From 2ad31a1f8ee73bf465a022c2c85f1b006defc6ac Mon Sep 17 00:00:00 2001 From: Yucheng Zhu Date: Thu, 30 Jul 2026 20:12:08 -0700 Subject: [PATCH] docs(otel/v2): restore pre-existing docstrings churned by the PR Restore the exact base wording of every comment and docstring the PR reworded without a code change, across logger.py, plumbing/routing.py, model/metadata.py, and plumbing/providers.py, so those lines drop out of the diff. Prose bound to a genuine code change or documenting new code is kept concise; no code logic changed. --- litellm/integrations/otel/logger.py | 237 ++++++++++++------ litellm/integrations/otel/model/metadata.py | 146 ++++++++--- .../integrations/otel/plumbing/providers.py | 83 +++--- litellm/integrations/otel/plumbing/routing.py | 78 +++--- 4 files changed, 347 insertions(+), 197 deletions(-) diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index 498a4f1daf0..114f0628b2a 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -77,9 +77,11 @@ def _span_error_from_exception( status_code: int | None = None, traceback_str: str | None = None, ) -> SpanError: - """A ``SpanError`` for a proxy-level failure that never produced a ``StandardLoggingPayload`` - (auth/validation/malformed-body rejections). ``status_code`` pins ``error.code`` to the real - response status.""" + """A ``SpanError`` for a proxy-level failure that never produced a + ``StandardLoggingPayload`` (auth / validation / malformed-body rejections), + mirroring ``_parse_error``'s field mapping so it stamps the same v2 keys a + failed LLM call does. ``status_code`` pins ``error.code`` to the real response + status, matching v1's SERVER-span behavior.""" from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup info = StandardLoggingPayloadSetup.get_error_information( @@ -103,8 +105,11 @@ _OTEL_MODULES = ( ) -# Cap on the open-call carrier map: a span opened at ``pre_call`` that never reaches a close -# callback (a stream that only fires stream events) would otherwise linger. Evicts the oldest. +# Cap on the open-call carrier map. A span opened at ``pre_call`` that never +# reaches a success/failure callback (e.g. a stream that only fires stream +# events) would otherwise linger; bounding the map evicts the oldest so memory +# stays flat on a long-running proxy while covering every concurrent in-flight +# call. _OPEN_CALLS_MAX = 10_000 @@ -164,8 +169,10 @@ class OpenTelemetryV2(CustomLogger): def _init_metrics(self, meter_provider: Any | None) -> "GenAIMetricRecorder | None": """Create the six GenAI histograms when metrics are enabled, else ``None``. - ``meter_provider`` is an explicit override (tests inject one); otherwise it is resolved - from the OTel global so the operator's readers/exporters receive the metrics. + ``meter_provider`` is an explicit override (tests inject one); otherwise the + provider is resolved from the OTel global so the operator's configured + readers/exporters receive the metrics, building and registering one only + when no global provider is set. """ if not self.config.enable_metrics: return None @@ -176,9 +183,11 @@ class OpenTelemetryV2(CustomLogger): def _init_events(self, logger_provider: LoggerProvider | None) -> "GenAIEventRecorder | None": """Create the GenAI event recorder when events are enabled, else ``None``. - ``logger_provider`` is an explicit override (tests inject one); otherwise it is resolved - from the OTel global. A ``None`` resolution means the operator opted out of logs, so no - recorder is built. + ``logger_provider`` is an explicit override (tests inject one); otherwise the + provider is resolved from the OTel global so an operator-configured logs + pipeline receives the events, building and registering one only when no + global provider is set. A ``None`` resolution means the operator opted out + of the logs signal, so no recorder is built. """ if not self.config.enable_events: return None @@ -226,11 +235,21 @@ class OpenTelemetryV2(CustomLogger): def log_pre_api_call(self, model, messages, kwargs): """Open the LLM-call span at the call boundary. - Runs synchronously in the request task, where the live server span is the ambient OTel - context, so the span parents to it natively; it is stashed and closed in the async - callback. When no recordable parent is visible (a thread-pool sync-only provider, where - the anchor doesn't follow), creation is deferred to the close callback. Synthetic - proxy-gate logs (auth/rate-limit rejections) made no upstream call and are skipped. + Runs synchronously inside the request task, before the upstream call — + the one place where the live server span is genuinely the ambient OTel + context — so the span parents to it natively, with no span threaded + through a metadata dict. The open span is stashed on the per-request + ``LiteLLMLoggingObj`` (a typed object) and closed in the async callback. + + When no recordable parent is visible (``pre_call`` was driven from a thread + pool for a sync-only provider, where contextvars — and so the anchor — + don't follow), creation is deferred: only the start time is recorded, and + the async callback — whose worker context was copied from the request task + and so still carries the anchor — creates the span then. + + Synthetic proxy-gate error logs (auth/rate-limit rejections) also fire this + hook but never made an upstream call; they are tagged and skipped so no + phantom LLM-call span is produced. """ call = LLMCallEvent.from_dict(kwargs) if call.is_no_upstream_call: @@ -249,8 +268,10 @@ class OpenTelemetryV2(CustomLogger): return start_time_ns = to_ns(datetime.now()) spans: tuple[Span, ...] = () - # Parent to the anchored root span (ambient on the SDK path); open live only when that - # resolves to a recordable parent, else defer to the close callback (the thread-pool case). + # Parent to the request's anchored root span (stable across the request), + # falling back to ambient on the SDK path. Open the span live only when + # that resolves to a recordable parent; otherwise defer to the close + # callback (the thread-pool case, where the anchor isn't visible here). parent_context = resolve_request_span_context() if is_recordable_span(get_current_span(parent_context)): spans = tuple( @@ -266,8 +287,9 @@ class OpenTelemetryV2(CustomLogger): ) ) self._open_llm_calls[call_id] = _LLMCallSpan(spans=spans, start_time_ns=start_time_ns) - # Evict the oldest open call if over budget; a call that opens but never closes would - # linger otherwise (the evicted span is dropped, never exported). + # Evict the oldest open call if the map is over budget. A call that opens + # but never closes (a stream that only fires stream events) would linger + # otherwise; the evicted span is simply dropped (never exported). if len(self._open_llm_calls) > _OPEN_CALLS_MAX: self._open_llm_calls.popitem(last=False) @@ -322,9 +344,10 @@ class OpenTelemetryV2(CustomLogger): self._record_failure_metrics(kwargs, start_time, end_time) def _seed_identity_baggage(self, identity: RequestIdentity, model: str | None, context: Context) -> Context: - """Seed authenticated request-identity Baggage onto ``context`` so the Baggage processor - stamps team/key/metadata onto the span. Read from the parsed payload, never the client's - ``params._meta`` carrier, so it can't be spoofed.""" + """Seed authenticated request-identity Baggage onto ``context`` so the Baggage + processor stamps team/key/metadata onto the span. Identity is read from the + parsed payload, never the client's ``params._meta`` carrier, so it can't be + spoofed.""" bag = promoted_baggage( identity, model, @@ -342,10 +365,14 @@ class OpenTelemetryV2(CustomLogger): ) -> bool: """Emit an MCP tool-call span when the closed request was a tool call. - MCP tool calls reach the callbacks with no ``pre_call`` carrier, so they get their own - CLIENT span here, emitted at once and deduped on the call id. Per the MCP semconv it - parents to the ``params._meta`` trace context (or a new root) and links the transport - span. Returns whether it handled the event, so the caller skips the LLM-call path. + MCP tool calls reach the success/failure callbacks like any other request + (with ``call_type`` ``call_mcp_tool``), but they are not LLM calls and have + no ``pre_call`` carrier — so they get their own CLIENT span here. Per the MCP + semconv it parents to the trace context the client propagated in + ``params._meta`` (or starts a new root) and links the transport span, rather + than nesting under the HTTP/session span. Returns whether it handled the + event, so the caller skips the LLM-call path. The whole span is emitted at + once (there is no boundary to open it at), deduped on the call id. """ raw_payload = kwargs.get("standard_logging_object") if not raw_payload or not is_mcp_tool_call(cast(Mapping[str, object], raw_payload)): @@ -354,15 +381,16 @@ class OpenTelemetryV2(CustomLogger): data = MCPToolCallSpanData.from_standard_logging_payload( payload, capture_content=self.config.capture_span_content ) - # Drop any stray LLM carrier for this id so it's neither leaked nor closed as a phantom span. + # A stray LLM carrier from a ``pre_call`` that mis-fired for this id would + # otherwise linger until evicted; drop it so it's neither leaked nor closed + # as a phantom LLM span. if data.identity.call_id: self._open_llm_calls.pop(data.identity.call_id, None) parent_context, links = resolve_mcp_span_context() parent_context = self._seed_identity_baggage(data.identity, None, parent_context) - # The tool-call span carries ``gen_ai.operation.name`` (execute_tool), so the - # fan-out processor treats it as a gen-AI span and skips it; route it to the - # request's admin destinations the same way the LLM-call span is, or it would - # reach the global exporter only and be missing from every tenant destination. + # The tool-call span carries ``gen_ai.operation.name``, so the fan-out processor + # treats it as a gen-AI span and skips it; route it to the request's admin + # destinations like the LLM-call span, or it reaches only the global exporter. call = LLMCallEvent.from_dict(kwargs) self._emitter.emit_fanout( SpanRole.MCP_TOOL_CALL, @@ -385,10 +413,12 @@ class OpenTelemetryV2(CustomLogger): ) -> bool: """Emit an MCP ``tools/list`` span when the closed request was a discovery call. - Like a tool call, listing has no ``pre_call`` carrier, so it gets its own CLIENT span. - Per the MCP semconv it parents to the ``params._meta`` trace context (or a new root) and - links the transport span. Returns whether it handled the event so the caller skips the - LLM-call path. + Like a tool call, listing reaches the success/failure callbacks (here with + ``call_type`` ``list_mcp_tools``) with no ``pre_call`` carrier, so it gets its + own CLIENT span. Per the MCP semconv it parents to the ``params._meta`` trace + context (or starts a new root) and links the transport span, rather than + nesting under the HTTP/session span. Returns whether it handled the event so + the caller skips the LLM-call path. """ raw_payload = kwargs.get("standard_logging_object") if not raw_payload or not is_mcp_list_tools(cast(Mapping[str, object], raw_payload)): @@ -559,13 +589,19 @@ class OpenTelemetryV2(CustomLogger): error_override: str | None, ) -> Span | None: data = ServiceSpanData.from_payload(payload, event_metadata=event_metadata) - # ``None`` role means metrics-only (framework instrumentation that duplicates a gen-AI - # span, or ``auth`` which gets a live phase span instead); it never enters the trace. + # Decide whether this service call is a span at all, and of what kind. + # ``None`` means metrics-only (framework instrumentation that duplicates a + # gen-AI span — ``self``/``router``/``proxy_pre_call`` — or ``auth``, which + # gets a live phase span instead). Those still feed Prometheus/Datadog via + # their own hooks; they just never enter the trace. role = span_role_for_service(data.service_name) if role is None: return None - # Skip a ping with neither timing nor a parent (in-memory queue gauges): a span would be a - # zero-duration root. Real background work passes start/end times; anything with a parent emits. + # A metrics-only ping with neither timing nor a parent (in-memory queue + # gauges) is not a traceable operation; a span for it would be a + # zero-duration root with no context, so skip it. Real background work + # (budget/reset jobs, spend flush) passes start/end times and still emits + # as a root; anything with a parent emits regardless. if error_override is None and start_time is None and end_time is None and parent_otel_span is None: return None if error_override is not None and data.error is None: @@ -575,9 +611,11 @@ class OpenTelemetryV2(CustomLogger): error=SpanError(message=error_override), event_metadata=data.event_metadata, ) - # Parent ambient-first (so the call nests under the active request phase, e.g. a DB lookup - # under ``auth``), falling back to the threaded ``parent_otel_span``; a background call has - # neither and starts its own root trace. + # Parent like every other span: ambient context first (so identity Baggage + # rides along and the call nests under whatever request phase is active — + # e.g. a DB lookup under the live ``auth`` span), falling back to the + # server span the proxy threaded as ``parent_otel_span``. A background + # service call has neither, so it starts its own root trace. parent_context = resolve_parent_context(threaded=parent_otel_span) return self._emitter.emit( role, @@ -595,9 +633,13 @@ class OpenTelemetryV2(CustomLogger): def seed_request_identity(self, user_api_key_dict: Any, model: Any = None) -> None: """Attach request-identity Baggage to the current context + server span. - Called once at the auth boundary so every span emitted afterwards inherits identity via - ``LiteLLMBaggageSpanProcessor``. Auth-internal DB lookups before the key resolves stay - unlabeled, which is correct. + Seeding identity into Baggage makes **every** span emitted afterwards for + this request — LLM call, guardrail, DB call — inherit it via + ``LiteLLMBaggageSpanProcessor``. Called once at the auth boundary (as soon + as the key resolves) so post-auth spans are labeled consistently; the + Baggage rides the request task's contextvar from there on. Auth-internal + DB lookups that run before the key is known stay unlabeled — identity + isn't determined yet, which is correct. """ try: identity = RequestIdentity.from_user_api_key_auth(user_api_key_dict) @@ -612,14 +654,19 @@ class OpenTelemetryV2(CustomLogger): # Attach (no detach): the contextvar is scoped to this request's # asyncio task and is reclaimed when the task ends. attach(set_request_baggage(bag, context=get_current())) - # The instrumentor started the server span before this ran, so the Baggage - # processor (fires only at span start) won't backfill it; stamp identity directly. - # Prefer the anchored root over ambient so identity lands on the server span even - # when seeding from inside the live ``auth`` phase span. + # The server span was started by the instrumentor before this ran, + # so the Baggage processor (which only fires at span start) won't + # backfill it — stamp identity on it directly. Prefer the anchored + # root span over the ambient one so identity still lands on the + # server span when seeding from inside the live ``auth`` phase span + # (the auth-failure path), where ``get_current_span`` is the phase + # span, not the request's root. server_span = request_root_span() or get_current_span() if is_recordable_span(server_span): - # Re-capture the anchor here too, covering entrypoints that bypass - # ``create_litellm_proxy_request_started_span`` (e.g. the SDK path). Idempotent. + # Re-capture the anchor here too: this runs post-auth with the + # server span active and covers entrypoints that bypass + # ``create_litellm_proxy_request_started_span`` (e.g. the SDK + # path's ``async_pre_call_hook``). Idempotent. set_request_root_span(server_span) for key, value in bag.items(): server_span.set_attribute(key, value) @@ -656,12 +703,14 @@ class OpenTelemetryV2(CustomLogger): exception: "Exception | None", status_code: int, ) -> None: - """Stamp v2 error.* attributes on the FastAPI-owned SERVER span for a failure that dies - before any LLM-call span exists (malformed body, auth/validation rejection). - - Only decorates the span (never sets status, ends it, or emits an exception event); the - instrumentor still owns the span's lifecycle, matching v1's SERVER-span behavior. - """ + """Stamp the v2 error.* attributes on the FastAPI-owned SERVER span for a + failure that dies before any LLM-call span exists (malformed body, auth / + validation rejection). Called from the proxy's global exception handler via + ``_close_dangling_otel_server_span``. The instrumentor still owns the span's + status and lifecycle, so this only decorates it — never sets status, never + ends it — and emits no exception event, matching v1's SERVER-span behavior + and avoiding a duplicate of the event ``async_post_call_failure_hook`` or + the ``auth`` phase span already records.""" if span is None or not is_recordable_span(span): return stamp_error( @@ -678,13 +727,18 @@ class OpenTelemetryV2(CustomLogger): user_api_key_dict: "UserAPIKeyAuth", traceback_str: "str | None" = None, ) -> None: - """Stamp error.* on the request's root SERVER span for a proxy-level failure that never - reached an LLM call (empty body rejected, auth failure), so it carries the same error keys - a failed LLM call does. + """Stamp error.* on the request's root SERVER span for a proxy-level + failure that never reached an LLM call (empty body rejected in the + endpoint, auth failure), so the failed request carries the same error keys + a failed LLM call does. v1's ``OpenTelemetry`` implemented this same hook; + v2 lost it when it stopped subclassing ``OpenTelemetry``, which is the + LIT-4179 regression for pre-call failures. - For an MCP message the session-task anchor is whatever request opened the session, so - prefer the transport the gateway published for this specific message. - """ + An MCP message is handled on the session's task, where the request-root + anchor is whatever request opened the session, so prefer the transport the + gateway published for this specific message. Without that, a failed tool + call aimed its error at the ``initialize`` request's finished span and the + SDK dropped it, leaving the POST that actually failed unmarked.""" span = mcp_message_transport_span() or request_root_span() or user_api_key_dict.parent_otel_span if span is None or not is_recordable_span(span): return None @@ -692,10 +746,18 @@ class OpenTelemetryV2(CustomLogger): return None def emit_guardrail_span(self, entry: "StandardLoggingGuardrailInformation") -> None: - # A guardrail is a sibling of the LLM call under the request's root span, so parent it to - # the explicit anchor, never the active span (which during a pre_call guardrail can be the - # live ``auth`` phase span). Emit with the guardrail's actual execution window so a - # pre_call guardrail is placed before the LLM call rather than at emission time. + # Emitted by the guardrail-recording code the moment a guardrail finishes, + # not from a post-call hook — that hook does not fire on every path (a + # pass-through request that passes its guardrails never reaches it), which + # left passing guardrails without a span. + # + # A guardrail is a sibling of the LLM call under the request's root span, + # so parent it to the explicit anchor — never the active span, which during + # a pre_call guardrail can be the live ``auth`` phase span. Emit with the + # guardrail's actual execution window so a pre_call guardrail is placed + # before the LLM call rather than at emission time. One entry in, one span + # out — the module-level entry point routes each entry to this single + # registered logger so a guardrail is never emitted more than once. data = GuardrailSpanData.from_logging_entry(entry) self._emitter.emit( SpanRole.GUARDRAIL, @@ -721,10 +783,21 @@ def select_global_otel_v2_logger( ) -> "OpenTelemetryV2": """The single ``OpenTelemetryV2`` whose provider should become the OTel global. - Prefer the canonical ``registered`` owner every other v2 entry point routes through, so the - server span and gen-ai spans share one provider and one trace. Fall back to a v2 logger in - ``in_memory_loggers`` (the SDK path), then build a generic one from ``OTEL_*`` only when none - was configured; each fallback avoids the second generic logger that orphaned the gen-ai spans. + The callback factory designates one logger as canonical the moment it builds + the first one (``_init_otel_logger_on_litellm_proxy`` sets + ``proxy_server.open_telemetry_logger``), and every other v2 entry point — + guardrail, identity seeding, phase spans — already routes through that same + ``registered`` owner. Reuse it here too so the global provider has one source + of truth instead of a second, independently-derived guess; this is the logger + a preset (arize, langfuse, …) folds the ``OTEL_*`` base exporter and its own + exporter into, so the FastAPI server span and the gen-ai spans share one + provider and one trace. + + Fall back to ``in_memory_loggers`` for the SDK path, where no proxy global is + set (selecting from there, not ``service_callback``, which a preset logger does + not always reach), and build a generic logger from ``OTEL_*`` only when none was + configured at all. Each fallback still avoids the second generic logger that + orphaned the gen-ai spans onto a different backend than the server span. """ if registered is not None: return registered @@ -739,9 +812,15 @@ def publish_global_otel_v2_provider( ) -> "OpenTelemetryV2": """Select the single v2 logger and publish its provider as the OTel global. - Called once at startup; ``registered`` (the canonical owner) makes the global reuse the - logger the rest of the v2 code emits through, and both it and ``set_global_provider`` are - injected so the publish step is unit-testable without touching real global OTel state. + The proxy calls this once at startup, after callbacks are initialized, so the + preset logger already exists; it passes ``registered`` (the canonical owner the + factory designated as ``proxy_server.open_telemetry_logger``) so the global + provider reuses the same logger the rest of the v2 code emits through (see + :func:`select_global_otel_v2_logger`). Both ``registered`` and + ``set_global_provider`` (the proxy passes + ``opentelemetry.trace.set_tracer_provider``) are injected so the publish step is + unit-testable without reading or mutating real global OTel state. Returns the + logger whose provider was published. """ logger = select_global_otel_v2_logger(in_memory_loggers, registered=registered) set_global_provider(logger._tracer_provider) @@ -760,9 +839,13 @@ def _registered_v2_logger() -> "OpenTelemetryV2 | None": def emit_guardrail_span(entry: "StandardLoggingGuardrailInformation") -> None: """Emit a guardrail span on the registered v2 OTel logger. - Called when a guardrail finishes, so a span is produced even when no post-call hook runs (the - pass-through allow path). Routes through the single canonical logger so a guardrail yields - exactly one span. Best-effort: emission must never break guardrail evaluation. + Called by the guardrail-recording code the moment a guardrail finishes, so a + span is produced regardless of whether a post-call hook later runs (it does + not on the pass-through allow path). Routes through the single canonical + logger — the same one every other v2 entry point uses — so a guardrail + recorded once yields exactly one span; fanning out across every reachable + ``OpenTelemetryV2`` instance double-emits the same entry. Best-effort: span + emission must never break guardrail evaluation. """ logger = _registered_v2_logger() if logger is None: diff --git a/litellm/integrations/otel/model/metadata.py b/litellm/integrations/otel/model/metadata.py index 6f7bde2c832..bda3c8e42a7 100644 --- a/litellm/integrations/otel/model/metadata.py +++ b/litellm/integrations/otel/model/metadata.py @@ -1,18 +1,37 @@ """The single translation layer between a request's metadata and the spans. -Every field is parsed **once**, here, out of the ``StandardLoggingPayload`` (or a -``UserAPIKeyAuth`` at the auth boundary), so span data, baggage, and the mappers -read typed fields instead of the raw ``metadata`` / ``hidden_params`` dicts. +Every relevant field litellm exposes about a request — the user-facing model, +the model actually dispatched to the provider, the deployment, and the caller's +identity (team, key, end-user) — is parsed **once**, here, out of the +``StandardLoggingPayload`` (or a ``UserAPIKeyAuth`` at the auth boundary). Span +data, baggage promotion, and the mappers then read these typed fields instead of +each digging into the raw ``metadata`` / ``hidden_params`` dicts. -:class:`RequestIdentity` holds caller identity (team/key/end-user), seeded into -Baggage at the auth boundary before routing has picked a deployment; its -``provider_model`` is absent from that early seed and filled only from the payload -at close. :class:`RequestContext` is the fully-resolved view at close, wrapping it. +Two models live here because a request's identity is known *before* its model +resolution is: -The request-vs-provider model split: a caller asks for a *model group* (``gpt-4o``) -that routes to a concrete deployment (``azure/my-deployment``). ``gen_ai.request.model`` -records the group and ``litellm.provider.model`` the dispatched model; they coincide -on the SDK path, which has no group. +* :class:`RequestIdentity` — team / key / end-user, seeded into Baggage at the + auth boundary (``from_user_api_key_auth``), before routing has picked a + deployment. ``provider_model`` is therefore absent from that early seed and is + only filled in from the payload once the call closes. +* :class:`RequestContext` — the full picture available at close: the resolved + request vs. provider model split, plus the response model, model group, model + id, and api base, wrapping the :class:`RequestIdentity`. + +The request-vs-provider model split is the subtle part. On the proxy a caller +asks for a *model group* (e.g. ``gpt-4o``) that routes to a concrete deployment +(e.g. ``azure/my-deployment``); the two are distinct and both worth recording. +``StandardLoggingPayload`` exposes them as: + +* ``model_group`` — the user-facing name the caller requested. +* ``model`` — already reconstructed (see ``reconstruct_model_name``) to the name + litellm dispatched to the provider (the deployment, provider-prefixed). +* ``hidden_params.litellm_model_name`` — a secondary source for the dispatched + model (populated only on some call paths, e.g. files). + +So ``gen_ai.request.model`` is the *group* (falling back to the call model on the +SDK path, which has no group), and ``litellm.provider.model`` is the *dispatched* +model. They coincide on the SDK path, which is correct. """ from __future__ import annotations @@ -36,22 +55,33 @@ class RequestIdentity: call_id: str | None = None team_id: str | None = None team_alias: str | None = None - # The team's free-form metadata, carried raw; filtered to an operator allowlist at Baggage-promotion time. + # The team's free-form metadata, carried raw (empty/missing -> None) and + # filtered to an operator allowlist only at Baggage-promotion time, so an + # unconfigured deployment never promotes any of it. team_metadata: Mapping[str, Any] | None = None key_hash: str | None = None end_user: str | None = None - # The model litellm dispatched to the provider; known only at close, so absent from the auth-time seed. + # The model litellm dispatched to the provider. Only known once the call + # completes (routing has picked a deployment), so it's absent from the + # auth-time seed and filled only from the payload. provider_model: str | None = None metadata: Mapping[str, str] = field(default_factory=dict) @classmethod def from_payload(cls, payload: "StandardLoggingPayload") -> "RequestIdentity": - """Parse caller identity (incl. resolved ``provider_model``) from a closed request's payload.""" + """Parse caller identity out of a closed request's payload metadata. + + ``provider_model`` is resolved here too (see :func:`resolve_provider_model`) + so the identity carried into Baggage labels every span with the dispatched + model, not just the user-facing one. + """ raw_meta = cast(Mapping[str, object], payload.get("metadata") or {}) metadata = {key: str(value) for key, value in raw_meta.items() if isinstance(value, (str, bool, int, float))} return cls( call_id=as_str(payload.get("litellm_call_id")) or as_str(payload.get("id")), - # Prefer the canonical ``user_api_key_team_id``; the bare ``team_id`` is a legacy alias. + # StandardLoggingMetadata's canonical key is ``user_api_key_team_id``; + # the bare ``team_id`` is a legacy alias and is often empty, so prefer + # the canonical key and fall back to the alias. team_id=as_str(raw_meta.get("user_api_key_team_id")) or as_str(raw_meta.get("team_id")), team_alias=as_str(raw_meta.get("user_api_key_team_alias")) or as_str(raw_meta.get("team_alias")), team_metadata=_team_metadata_dict(raw_meta.get("user_api_key_team_metadata")), @@ -63,10 +93,14 @@ class RequestIdentity: @classmethod def from_user_api_key_auth(cls, auth: object) -> "RequestIdentity": - """Identity from a ``UserAPIKeyAuth`` (duck-typed to avoid a proxy import). + """Identity from a ``UserAPIKeyAuth`` (duck-typed to keep this module + free of a proxy import). - Seeds Baggage at the pre-call hook so every span inherits identity. Metadata - sub-keys use the ``user_api_key_*`` names Baggage promotion expects. + Used in the pre-call hook to seed Baggage early — before any LLM, + guardrail, or service span is created — so the whole request's spans + inherit identity, not just the LLM-call span. Metadata sub-keys use the + ``user_api_key_*`` names that ``baggage.DEFAULT_BAGGAGE_METADATA_KEYS`` + promotes. """ get = lambda name: getattr(auth, name, None) # noqa: E731 metadata = { @@ -85,14 +119,20 @@ class RequestIdentity: team_metadata=_team_metadata_dict(get("team_metadata")), key_hash=as_str(get("api_key")), end_user=as_str(get("end_user_id")), - # ``provider_model`` is unknown at the auth boundary (routing hasn't picked a deployment yet). + # ``provider_model`` is unknown at the auth boundary — routing hasn't + # picked a deployment yet — so it's only populated from the payload. metadata=metadata, ) @dataclass(frozen=True) class RequestContext: - """The fully-resolved view of a closed request, parsed once from the payload.""" + """The fully-resolved view of a closed request, parsed once from the payload. + + ``request_model`` is the user-facing requested model and ``provider_model`` + (on :attr:`identity`) is the model litellm dispatched to the provider; the two + differ on the proxy (group vs. deployment) and coincide on the SDK path. + """ request_model: str response_model: str | None @@ -127,26 +167,43 @@ class RequestContext: # --- live-callback kwargs parsing ------------------------------------------- # -# Parse the live callback ``kwargs`` god object and raw pre/post-call ``data`` dicts — -# the untyped request state reaching a ``CustomLogger`` before a ``StandardLoggingPayload``. +# +# The model and helpers below parse the *live* callback ``kwargs`` god object (and +# the raw pre/post-call ``data`` dicts) — the untyped request state that reaches a +# ``CustomLogger`` before, or instead of, a ``StandardLoggingPayload``. They live +# here, with the payload/auth parsers, so every read out of a request's raw dicts +# is in one place rather than scattered across the ``CustomLogger``. @dataclass(frozen=True) class LLMCallEvent: - """The typed view of the live callback ``kwargs`` (``model_call_details``), parsed once.""" + """The typed view of the live callback ``kwargs`` (``model_call_details``). - # The ``litellm_call_id`` correlating ``pre_call`` with the close callback; the stable - # key for the open-call carrier. + litellm hands every callback an untyped ``kwargs`` god object. The fields the + OTel logger needs out of it are parsed **once**, here, so the ``CustomLogger`` + reads typed attributes instead of digging into the dict at each boundary. + """ + + # The ``litellm_call_id`` correlating ``pre_call`` with the close callback. + # Present in ``model_call_details`` at ``pre_call`` and in both the kwargs and + # the ``standard_logging_object`` at success/failure, so it's a stable key for + # the open-call carrier — no back-reference to the logging object required (the + # object isn't reachable from the callback kwargs at ``pre_call`` time). call_id: str | None - # The success/failure payload; ``None`` at ``pre_call`` or if the call closed with no payload. + # The ``StandardLoggingPayload`` carried on a success/failure callback; ``None`` + # at ``pre_call``, or when the call closed before any payload materialized (so + # there is nothing to stamp on the span). payload: "StandardLoggingPayload | None" otel_destinations: tuple[OtelDestination, ...] - # The request's ``standard_callback_dynamic_params`` (team/key OTLP credentials), or ``None`` - # when the call isn't scoped; routes the gen-AI span to a credential-scoped tracer. + # The ``standard_callback_dynamic_params`` routing the call to a per-tenant + # tracer (its own exporter/endpoint), or ``None`` when the call isn't scoped. dynamic_params: "StandardCallbackDynamicParams | None" - # True for synthetic proxy-gate logs (auth/rate-limit rejections): no upstream call, so no span. + # True for synthetic proxy-gate logs (auth / rate-limit rejections): they fire + # the ``pre_call`` hook but never made an upstream call, so they get no span. is_no_upstream_call: bool - # Best-effort ``"{operation} {model}"`` name at ``pre_call``; only matters for a leaked span (renamed at close). + # A best-effort ``"{operation} {model}"`` name known at ``pre_call`` time. The + # span is renamed from the typed payload at close (``finish_span``); this only + # needs to be reasonable for a span that never gets closed (a leak). provisional_span_name: str time_to_first_chunk_seconds: float | None @@ -168,8 +225,10 @@ class LLMCallEvent: def time_to_first_chunk_seconds(kwargs: Mapping[str, Any]) -> float | None: - """Seconds from upstream request (``api_call_start_time``) to first streamed chunk - (``completion_start_time``); ``None`` for non-streaming calls.""" + """Seconds from the upstream request being issued (``api_call_start_time``) + to the first streamed chunk (``completion_start_time``); ``None`` for + non-streaming calls, where ``completion_start_time`` is backfilled with the + end time and would not measure first-chunk latency.""" optional_params = cast(Mapping[str, Any], kwargs.get("optional_params") or {}) if not optional_params.get("stream"): return None @@ -190,7 +249,11 @@ def _call_id(payload: "StandardLoggingPayload | None", kwargs: Mapping[str, Any] def model_from_request_data(data: object) -> str | None: - """The user-facing ``model`` from a pre-call ``data`` dict (``None`` if absent).""" + """The user-facing ``model`` from a pre-call ``data`` dict (``None`` if absent). + + Read at the auth boundary to label early Baggage before routing has resolved + a deployment; ``data`` is duck-typed since it arrives untyped from the proxy. + """ if isinstance(data, Mapping): return as_str(data.get("model")) return None @@ -199,13 +262,16 @@ def model_from_request_data(data: object) -> str | None: def resolve_provider_model(payload: "StandardLoggingPayload") -> str | None: """The model litellm dispatched to the provider, from the payload. - Prefers ``metadata.deployment``, then ``hidden_params.litellm_model_name``, then the - top-level ``model`` (already resolved to the provider-prefixed deployment name). + Prefers the explicit ``hidden_params.litellm_model_name`` (set on call paths + that know it, e.g. files), then the top-level ``model`` — which + ``reconstruct_model_name`` has already resolved to the deployment's + provider-prefixed name. Returns ``None`` only when neither is present. """ raw_meta = cast(Mapping[str, object], payload.get("metadata") or {}) hidden = cast(Mapping[str, object], payload.get("hidden_params") or {}) return ( - # ``deployment`` (most precise) survives only on paths that don't strip it from metadata. + # ``deployment`` survives only on paths that don't strip it from metadata; + # harmless (and most precise) to prefer it when present. as_str(raw_meta.get("deployment")) or as_str(hidden.get("litellm_model_name")) or as_str(payload.get("model")) ) @@ -218,7 +284,13 @@ def _model_info_id(model_info: object) -> str | None: def _team_metadata_dict(value: object) -> Mapping[str, Any] | None: - """The team's free-form metadata as a raw mapping, or ``None`` when missing or empty.""" + """The team's free-form metadata as a raw mapping, or ``None`` when missing + or empty. + + Carried raw on the identity and filtered to an operator allowlist only at + Baggage-promotion time (see ``baggage.promoted_baggage``), so an empty case + is dropped rather than carrying a useless ``{}``. + """ if isinstance(value, Mapping) and value: return dict(value) return None diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index 6c276184a0d..99d2ebfae35 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -56,9 +56,12 @@ def to_otel_span_kind(kind: LiteLLMSpanKind) -> SpanKind: return _SPAN_KIND_BY_ROLE_KIND[kind] -# Custom exporter factories keyed by ``ExporterSpec.kind``. A preset registers one when its -# destination needs construction logic the built-in kinds can't express (e.g. an exporter that -# fetches an auth token lazily on first export). Keeps this module vendor-agnostic. +# Custom exporter factories keyed by ``ExporterSpec.kind``. A preset registers +# one here when its destination needs construction logic the built-in kinds +# can't express — e.g. an exporter that fetches an auth token lazily on its +# first export (off the event loop) instead of blocking at config-build time. +# Keeping the registry here lets this module stay vendor-agnostic: the factory +# lives with the integration that needs it. _EXPORTER_FACTORIES: dict[str, Callable[[ExporterSpec], SpanExporter]] = {} @@ -99,12 +102,16 @@ class LiteLLMBaggageSpanProcessor(SpanProcessor): def _otlp_traces_endpoint(endpoint: str | None) -> str | None: """Point an OTLP/HTTP base endpoint at the ``/v1/traces`` signal path. - An explicitly passed endpoint is used verbatim (unlike ``OTEL_EXPORTER_OTLP_ENDPOINT``), so a - base URL would POST to the root and 404; append the signal path here, leaving a correct path intact. + ``OTEL_EXPORTER_OTLP_ENDPOINT`` is a base URL (e.g. ``http://host:4318``). + The OTLP/HTTP exporter only appends the ``/v1/traces`` path when it reads + that env var itself; when an endpoint is passed explicitly it is used + verbatim, so a base URL would POST to the root and the collector returns + 404. Append the signal path here (leaving an already-correct path intact). """ if not endpoint: return endpoint endpoint = endpoint.rstrip("/") + # Splunk Observability uses ``/v2/trace/otlp``; never rewrite it. if endpoint.endswith("/v1/traces") or "/v2/trace/otlp" in endpoint or endpoint.endswith("/api/trace"): return endpoint for other_signal in ("/v1/logs", "/v1/metrics"): @@ -123,10 +130,8 @@ def default_otlp_kind_for_backend(callback_name: "str | None") -> str: def destination_resource_attrs(destination: "OtelDestination") -> Mapping[str, str]: """The destination's builder-declared Resource attributes (e.g. Arize's - ``model_id`` / ``arize.project.name``; empty for header-routed backends). - - Both export paths to a destination -- the fan-out processor and the per-tenant - clone provider -- read these so the gen-AI span and its parents share one Resource. + ``model_id`` / ``arize.project.name``; empty for header-routed backends), read + by both export paths so the gen-AI span and its parents share one Resource. """ return dict(destination.resource_attributes) @@ -198,8 +203,10 @@ def build_span_exporter(config: OpenTelemetryV2Config) -> SpanExporter: def _otlp_metrics_endpoint(endpoint: str | None) -> str | None: """Point an OTLP/HTTP base endpoint at the ``/v1/metrics`` signal path. - Mirrors ``_otlp_traces_endpoint`` for the metrics signal (an explicitly passed endpoint is - used verbatim, so a base URL would POST to the root). + The OTLP/HTTP exporter only appends ``/v1/metrics`` when it reads + ``OTEL_EXPORTER_OTLP_ENDPOINT`` itself; an explicitly passed endpoint is used + verbatim, so a base URL would POST to the root. Mirror ``_otlp_traces_endpoint`` + for the metrics signal (rewriting a sibling signal path when present). """ if not endpoint: return endpoint @@ -265,8 +272,10 @@ def build_metric_reader(config: OpenTelemetryV2Config) -> "MetricReader": def _otlp_logs_endpoint(endpoint: str | None) -> str | None: """Point an OTLP/HTTP base endpoint at the ``/v1/logs`` signal path. - Mirrors ``_otlp_traces_endpoint`` for the logs signal (an explicitly passed endpoint is used - verbatim, so a base URL would POST to the root). + The OTLP/HTTP exporter only appends ``/v1/logs`` when it reads + ``OTEL_EXPORTER_OTLP_ENDPOINT`` itself; an explicitly passed endpoint is used + verbatim, so a base URL would POST to the root. Mirror ``_otlp_traces_endpoint`` + for the logs signal (rewriting a sibling signal path when present). """ if not endpoint: return endpoint @@ -282,8 +291,10 @@ def _otlp_logs_endpoint(endpoint: str | None) -> str | None: def build_log_exporter(config: OpenTelemetryV2Config) -> LogExporter: """Build a log exporter mirroring the exporter selection of the other signals. - ``console`` (and any unrecognized kind) to the console; ``otlp_http``/``otlp_grpc`` over OTLP; - ``in_memory`` buffers for tests. Events ride the single-destination shorthand fields, not ``exporters``. + ``console`` (and any unrecognized kind) exports to the console; ``otlp_http`` + and ``otlp_grpc`` export over OTLP with the configured endpoint/headers; + ``in_memory`` buffers for tests. Like GenAI metrics, events ride the + single-destination shorthand fields, not the multi-exporter ``exporters`` list. """ kind = (config.exporter or "console").lower() if kind in ("in_memory", "inmemory", "memory"): @@ -318,8 +329,11 @@ def build_logger_provider( ) -> SDKLoggerProvider: """Build the :class:`LoggerProvider` GenAI events export through. - ``log_exporter`` is an explicit override (tests); otherwise selected from the config via - :func:`build_log_exporter`. Console/in-memory get a Simple processor, everything else Batch. + ``log_exporter`` is an explicit override (tests inject an + ``InMemoryLogExporter``); otherwise the exporter is selected from the config's + exporter kind via :func:`build_log_exporter`. Console and in-memory exporters + get a Simple processor (synchronous export, which tests rely on), everything + else a Batch processor — the same split as span processing. """ exporter = log_exporter if log_exporter is not None else build_log_exporter(config) provider = SDKLoggerProvider(resource=build_resource(config)) @@ -334,12 +348,14 @@ def resolve_logger_provider( config: OpenTelemetryV2Config, logger_provider: SDKLoggerProvider | None = None, ) -> SDKLoggerProvider | None: - """Resolve the :class:`LoggerProvider` GenAI events record through, or ``None`` when the - operator opted out of the logs signal. + """Resolve the :class:`LoggerProvider` GenAI events record through, or ``None`` + when the operator has opted out of the logs signal. - An injected provider wins (DI/tests); an operator-configured SDK global is reused; an explicit - ``NoOpLoggerProvider`` is an opt-out (``None``). Only the default placeholder global makes V2 - build and publish one from the config. + Same resolution order as :func:`resolve_meter_provider`: an injected provider + wins (DI/tests); an operator-configured SDK global is reused so events ride + their pipeline; an explicit ``NoOpLoggerProvider`` global is an opt-out and + yields ``None``, so no event is ever built. Only the default placeholder + global makes V2 build a provider from the config and publish it as the global. """ if logger_provider is not None: return logger_provider @@ -379,10 +395,14 @@ def resolve_meter_provider( ) -> MeterProvider: """Resolve the :class:`MeterProvider` GenAI metrics record through. - An injected provider wins (DI/tests); otherwise reuse the operator's configured global (a real - SDK provider or an explicit ``NoOpMeterProvider`` opt-out). Only the default proxy placeholder - makes V2 build and publish one from the config; the built provider is returned so its reader - thread stays live. + An injected provider wins (DI/tests). Otherwise reuse whatever the operator has + configured as the global, whether a real SDK provider or an explicit + ``NoOpMeterProvider``, so the GenAI histograms ride the operator's + readers/exporters and an explicit opt-out is honored. Only when the global is + still the default proxy placeholder does V2 build one from the config and + publish it as the global, mirroring how V2 owns trace export. The built + provider is the one returned, so its reader thread is always live, never + orphaned. """ if meter_provider is not None: return meter_provider @@ -416,13 +436,10 @@ def build_tracer_provider( tenant_fan_out_owner: str | None = None, attach_tenant_fan_out: bool = False, ) -> TracerProvider: - """Build the shared :class:`TracerProvider`: the Baggage processor first, then one - ``SpanProcessor`` per ``config.exporters`` entry (``exporter`` overrides with a test exporter). - - ``attach_tenant_fan_out``/``tenant_fan_out_owner`` add a ``TenantFanOutSpanProcessor`` that - forwards proxy-internal spans to the request's destinations. The main v2 provider always opts - in (so the server span reaches the destination and its gen-AI child isn't orphaned); per-tenant - clone providers pass neither. + """Build the shared :class:`TracerProvider`: Baggage processor first, then one + ``SpanProcessor`` per ``config.exporters`` entry (``exporter`` overrides for tests). + ``attach_tenant_fan_out``/``tenant_fan_out_owner`` add a ``TenantFanOutSpanProcessor`` + forwarding proxy-internal spans to the request's destinations. """ provider = TracerProvider(resource=build_resource(config)) if baggage_processor is None: diff --git a/litellm/integrations/otel/plumbing/routing.py b/litellm/integrations/otel/plumbing/routing.py index 54b72591b1a..87233526ddf 100644 --- a/litellm/integrations/otel/plumbing/routing.py +++ b/litellm/integrations/otel/plumbing/routing.py @@ -1,23 +1,7 @@ """Per-request multi-tenant tracer routing and span fan-out. -``TenantTracerCache`` routes the gen-AI LLM-call span, building per-tenant clone -``TracerProvider``s that export to the request's admin-owned destinations plus the -configured/global exporter. ``TenantFanOutSpanProcessor`` (at the bottom) forwards the -proxy-internal spans (server, auth, DB, cost) to every destination. The destinations both -read from a server-only contextvar, so a caller can neither redirect a trace nor spawn -providers through them. - -Separately, ``genai_tracers_for`` also routes the gen-AI span by the request's -``standard_callback_dynamic_params`` (team/key OTLP credentials), restoring the per-request -credential routing that predates the admin-destination refactor: it rewrites only the owned -exporter's headers, bounded by the same LRU. Those params are partly caller-influenced, so -this path can spawn a per-request provider; closing that for request-body-supplied credentials -(vs admin-configured team ``callback_vars``) is tracked separately. - -The gen-AI path (``tracers_for``) groups destinations by their backend-required Resource -attributes and builds one provider per group, because a span carries exactly one Resource and -a backend like Arize selects its project FROM it, so two Arize projects each get a correctly -tagged span instead of a last-wins merge. Empty destinations -> the logger's default (global only). +``TenantTracerCache`` routes the gen-AI span to per-tenant/destination providers; +``TenantFanOutSpanProcessor`` forwards proxy-internal spans to every admin-resolved destination. """ import threading @@ -42,18 +26,23 @@ from litellm.integrations.otel.presets import dynamic_otlp_headers if TYPE_CHECKING: from litellm.types.utils import StandardCallbackDynamicParams +# Exporter kinds that ignore headers — never rewritten with dynamic credentials. _NON_OTLP_KINDS = ("console", "in_memory", "inmemory", "memory") +# Cap on distinct credential-scoped providers held at once. ``dynamic_params`` +# can be populated from request metadata, so an unbounded cache lets a caller +# spawn one ``TracerProvider`` (plus its ``BatchSpanProcessor`` background +# thread) per unique credential set and exhaust the proxy. The LRU bound keeps +# the working set of active tenants resident while flushing and shutting down +# evicted providers so their threads are reclaimed. _MAX_CACHED_PROVIDERS = 256 def _shutdown_in_background(evicted: "TracerProvider | SpanProcessor") -> None: """Reclaim an evicted provider/processor's ``BatchSpanProcessor`` worker thread. - Dropping it from the cache without ``shutdown`` leaves that daemon thread running for - the life of the process (it does not "drain on its own"). ``shutdown`` force-flushes - and can do network I/O, so it runs fire-and-forget on a daemon thread rather than on - the request path; the evicted object is otherwise unreferenced. + Dropping it without ``shutdown`` leaks the daemon thread; ``shutdown`` force-flushes and can + do network I/O, so it runs fire-and-forget on a daemon thread rather than the request path. """ def _run() -> None: @@ -98,14 +87,9 @@ class TenantTracerCache: ) -> "tuple[Tracer, ...]": """The tracers for this request's gen-AI span, one per distinct Resource group. - A backend like Arize selects its project from the Resource, so destinations are grouped - by ``destination_resource_attrs`` and the caller emits the span once per tracer. The - configured/global exporters ride the FIRST group only, so the global receives the span - once. Empty ``destinations`` -> the logger's default tracer (deny). - - ``include_base_on_first`` is set to ``False`` by ``genai_tracers_for`` when a - credential-scoped tracer already carries the configured/global exporters, so the - destination groups don't also export the span to the global collector a second time. + Destinations are grouped by ``destination_resource_attrs`` (a backend like Arize selects its + project from the Resource); the configured/global exporters ride the first group only, and + ``include_base_on_first=False`` drops them when a credential-scoped tracer already carries them. """ if not destinations: return (default,) @@ -120,15 +104,9 @@ class TenantTracerCache: destinations: "tuple[OtelDestination, ...]", dynamic_params: "StandardCallbackDynamicParams | None", ) -> "tuple[Tracer, ...]": - """The gen-AI span's tracers, layering per-request credential routing over the - admin-destination fan-out. - - When the request carries this backend's team/key OTLP credentials - (``standard_callback_dynamic_params``), the global export rides a credential-scoped - provider (the configured exporters with this backend's own exporter rewritten to those - credentials), and the admin-destination groups omit the base exporters so the span - reaches the global collector exactly once. Without dynamic credentials this is the plain - destination fan-out (empty destinations -> the default tracer). + """The gen-AI span's tracers, layering per-request credential routing over the destination + fan-out: with this backend's team/key OTLP credentials the global export rides a + credential-scoped provider and the destination groups omit the base exporters; else plain fan-out. """ headers = dynamic_otlp_headers(self._callback_name, dynamic_params) if not headers: @@ -161,12 +139,14 @@ class TenantTracerCache: return get_tracer(provider, self._tracer_name) def _config_with_headers(self, headers: "dict[str, str]") -> OpenTelemetryV2Config: - """Clone the config, stamping ``headers`` onto this backend's own exporter only. + """Clone the config, stamping ``headers`` onto the credential's own exporter. - ``headers`` are the per-request credentials of ``self._callback_name``, so they apply - only to the exporter that integration contributed (``spec.owner``). A request carrying - one tenant's Arize key must never rewrite a co-configured Langfuse or self-hosted - collector exporter, which would leak that key to a different backend. + ``headers`` are the per-request credentials of ``self._callback_name`` (the + integration that built this cache), so they apply only to the exporter that + integration contributed (``spec.owner``). A request that carries one + tenant's Arize key must never rewrite the headers of a co-configured + Langfuse or self-hosted collector exporter, which would leak that key to a + different backend. """ header_str = ",".join(f"{key}={value}" for key, value in headers.items()) exporters = [ @@ -260,13 +240,11 @@ class TenantTracerCache: *, include_base_exporters: bool = True, ) -> OpenTelemetryV2Config: - """Clone the config, appending one exporter per resolved destination (its resolved host - and own auth headers) so one span exports to every destination. + """Clone the config, appending one exporter per destination so one span exports to every one. - ``include_base_exporters`` keeps the configured/global exporters; ``tracers_for`` sets it - only on the first Resource group so the global gets the span once, not once per group. The - clone's Resource folds in the destinations' ``destination_resource_attrs`` (Arize needs - ``model_id`` / ``arize.project.name``); callers group by those first so the merge is lossless.""" + ``include_base_exporters`` keeps the configured/global exporters (``tracers_for`` sets it only on + the first group); the clone's Resource folds in the destinations' ``destination_resource_attrs``. + """ from litellm.integrations.otel.plumbing.providers import ( destination_resource_attrs, )