scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-17 23:51:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 67

docs(otel/v2): restore pre-existing docstrings churned by the PR

Browse source 
Restore the exact base wording of every comment and docstring the PR reworded without a
code change, across logger.py, plumbing/routing.py, model/metadata.py, and
plumbing/providers.py, so those lines drop out of the diff. Prose bound to a genuine code
change or documenting new code is kept concise; no code logic changed.
This commit is contained in:
Yucheng Zhu 2026-07-30 20:12:08 -07:00
parent 10c42ffb73
commit 2ad31a1f8e
4 changed files with 347 additions and 197 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

237
litellm/integrations/otel/logger.py
View file

@ -77,9 +77,11 @@ def _span_error_from_exception(
status_code: int | None = None,
traceback_str: str | None = None,
) -> SpanError:
"""A ``SpanError`` for a proxy-level failure that never produced a ``StandardLoggingPayload``
(auth/validation/malformed-body rejections). ``status_code`` pins ``error.code`` to the real
response status."""
"""A ``SpanError`` for a proxy-level failure that never produced a
``StandardLoggingPayload`` (auth / validation / malformed-body rejections),
mirroring ``_parse_error``'s field mapping so it stamps the same v2 keys a
failed LLM call does. ``status_code`` pins ``error.code`` to the real response
status, matching v1's SERVER-span behavior."""
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
info = StandardLoggingPayloadSetup.get_error_information(
@ -103,8 +105,11 @@ _OTEL_MODULES = (
)
# Cap on the open-call carrier map: a span opened at ``pre_call`` that never reaches a close
# callback (a stream that only fires stream events) would otherwise linger. Evicts the oldest.
# Cap on the open-call carrier map. A span opened at ``pre_call`` that never
# reaches a success/failure callback (e.g. a stream that only fires stream
# events) would otherwise linger; bounding the map evicts the oldest so memory
# stays flat on a long-running proxy while covering every concurrent in-flight
# call.
_OPEN_CALLS_MAX = 10_000
@ -164,8 +169,10 @@ class OpenTelemetryV2(CustomLogger):
def _init_metrics(self, meter_provider: Any | None) -> "GenAIMetricRecorder | None":
"""Create the six GenAI histograms when metrics are enabled, else ``None``.
``meter_provider`` is an explicit override (tests inject one); otherwise it is resolved
from the OTel global so the operator's readers/exporters receive the metrics.
``meter_provider`` is an explicit override (tests inject one); otherwise the
provider is resolved from the OTel global so the operator's configured
readers/exporters receive the metrics, building and registering one only
when no global provider is set.
"""
if not self.config.enable_metrics:
return None
@ -176,9 +183,11 @@ class OpenTelemetryV2(CustomLogger):
def _init_events(self, logger_provider: LoggerProvider | None) -> "GenAIEventRecorder | None":
"""Create the GenAI event recorder when events are enabled, else ``None``.
``logger_provider`` is an explicit override (tests inject one); otherwise it is resolved
from the OTel global. A ``None`` resolution means the operator opted out of logs, so no
recorder is built.
``logger_provider`` is an explicit override (tests inject one); otherwise the
provider is resolved from the OTel global so an operator-configured logs
pipeline receives the events, building and registering one only when no
global provider is set. A ``None`` resolution means the operator opted out
of the logs signal, so no recorder is built.
"""
if not self.config.enable_events:
return None
@ -226,11 +235,21 @@ class OpenTelemetryV2(CustomLogger):
def log_pre_api_call(self, model, messages, kwargs):
"""Open the LLM-call span at the call boundary.
Runs synchronously in the request task, where the live server span is the ambient OTel
context, so the span parents to it natively; it is stashed and closed in the async
callback. When no recordable parent is visible (a thread-pool sync-only provider, where
the anchor doesn't follow), creation is deferred to the close callback. Synthetic
proxy-gate logs (auth/rate-limit rejections) made no upstream call and are skipped.
Runs synchronously inside the request task, before the upstream call —
the one place where the live server span is genuinely the ambient OTel
context — so the span parents to it natively, with no span threaded
through a metadata dict. The open span is stashed on the per-request
``LiteLLMLoggingObj`` (a typed object) and closed in the async callback.
When no recordable parent is visible (``pre_call`` was driven from a thread
pool for a sync-only provider, where contextvars — and so the anchor —
don't follow), creation is deferred: only the start time is recorded, and
the async callback — whose worker context was copied from the request task
and so still carries the anchor — creates the span then.
Synthetic proxy-gate error logs (auth/rate-limit rejections) also fire this
hook but never made an upstream call; they are tagged and skipped so no
phantom LLM-call span is produced.
"""
call = LLMCallEvent.from_dict(kwargs)
if call.is_no_upstream_call:
@ -249,8 +268,10 @@ class OpenTelemetryV2(CustomLogger):
return
start_time_ns = to_ns(datetime.now())
spans: tuple[Span, ...] = ()
# Parent to the anchored root span (ambient on the SDK path); open live only when that
# resolves to a recordable parent, else defer to the close callback (the thread-pool case).
# Parent to the request's anchored root span (stable across the request),
# falling back to ambient on the SDK path. Open the span live only when
# that resolves to a recordable parent; otherwise defer to the close
# callback (the thread-pool case, where the anchor isn't visible here).
parent_context = resolve_request_span_context()
if is_recordable_span(get_current_span(parent_context)):
spans = tuple(
@ -266,8 +287,9 @@ class OpenTelemetryV2(CustomLogger):
)
)
self._open_llm_calls[call_id] = _LLMCallSpan(spans=spans, start_time_ns=start_time_ns)
# Evict the oldest open call if over budget; a call that opens but never closes would
# linger otherwise (the evicted span is dropped, never exported).
# Evict the oldest open call if the map is over budget. A call that opens
# but never closes (a stream that only fires stream events) would linger
# otherwise; the evicted span is simply dropped (never exported).
if len(self._open_llm_calls) > _OPEN_CALLS_MAX:
self._open_llm_calls.popitem(last=False)
@ -322,9 +344,10 @@ class OpenTelemetryV2(CustomLogger):
self._record_failure_metrics(kwargs, start_time, end_time)
def _seed_identity_baggage(self, identity: RequestIdentity, model: str | None, context: Context) -> Context:
"""Seed authenticated request-identity Baggage onto ``context`` so the Baggage processor
stamps team/key/metadata onto the span. Read from the parsed payload, never the client's
``params._meta`` carrier, so it can't be spoofed."""
"""Seed authenticated request-identity Baggage onto ``context`` so the Baggage
processor stamps team/key/metadata onto the span. Identity is read from the
parsed payload, never the client's ``params._meta`` carrier, so it can't be
spoofed."""
bag = promoted_baggage(
identity,
model,
@ -342,10 +365,14 @@ class OpenTelemetryV2(CustomLogger):
) -> bool:
"""Emit an MCP tool-call span when the closed request was a tool call.
MCP tool calls reach the callbacks with no ``pre_call`` carrier, so they get their own
CLIENT span here, emitted at once and deduped on the call id. Per the MCP semconv it
parents to the ``params._meta`` trace context (or a new root) and links the transport
span. Returns whether it handled the event, so the caller skips the LLM-call path.
MCP tool calls reach the success/failure callbacks like any other request
(with ``call_type`` ``call_mcp_tool``), but they are not LLM calls and have
no ``pre_call`` carrier — so they get their own CLIENT span here. Per the MCP
semconv it parents to the trace context the client propagated in
``params._meta`` (or starts a new root) and links the transport span, rather
than nesting under the HTTP/session span. Returns whether it handled the
event, so the caller skips the LLM-call path. The whole span is emitted at
once (there is no boundary to open it at), deduped on the call id.
"""
raw_payload = kwargs.get("standard_logging_object")
if not raw_payload or not is_mcp_tool_call(cast(Mapping[str, object], raw_payload)):
@ -354,15 +381,16 @@ class OpenTelemetryV2(CustomLogger):
data = MCPToolCallSpanData.from_standard_logging_payload(
payload, capture_content=self.config.capture_span_content
)
# Drop any stray LLM carrier for this id so it's neither leaked nor closed as a phantom span.
# A stray LLM carrier from a ``pre_call`` that mis-fired for this id would
# otherwise linger until evicted; drop it so it's neither leaked nor closed
# as a phantom LLM span.
if data.identity.call_id:
self._open_llm_calls.pop(data.identity.call_id, None)
parent_context, links = resolve_mcp_span_context()
parent_context = self._seed_identity_baggage(data.identity, None, parent_context)
# The tool-call span carries ``gen_ai.operation.name`` (execute_tool), so the
# fan-out processor treats it as a gen-AI span and skips it; route it to the
# request's admin destinations the same way the LLM-call span is, or it would
# reach the global exporter only and be missing from every tenant destination.
# The tool-call span carries ``gen_ai.operation.name``, so the fan-out processor
# treats it as a gen-AI span and skips it; route it to the request's admin
# destinations like the LLM-call span, or it reaches only the global exporter.
call = LLMCallEvent.from_dict(kwargs)
self._emitter.emit_fanout(
SpanRole.MCP_TOOL_CALL,
@ -385,10 +413,12 @@ class OpenTelemetryV2(CustomLogger):
) -> bool:
"""Emit an MCP ``tools/list`` span when the closed request was a discovery call.
Like a tool call, listing has no ``pre_call`` carrier, so it gets its own CLIENT span.
Per the MCP semconv it parents to the ``params._meta`` trace context (or a new root) and
links the transport span. Returns whether it handled the event so the caller skips the
LLM-call path.
Like a tool call, listing reaches the success/failure callbacks (here with
``call_type`` ``list_mcp_tools``) with no ``pre_call`` carrier, so it gets its
own CLIENT span. Per the MCP semconv it parents to the ``params._meta`` trace
context (or starts a new root) and links the transport span, rather than
nesting under the HTTP/session span. Returns whether it handled the event so
the caller skips the LLM-call path.
"""
raw_payload = kwargs.get("standard_logging_object")
if not raw_payload or not is_mcp_list_tools(cast(Mapping[str, object], raw_payload)):
@ -559,13 +589,19 @@ class OpenTelemetryV2(CustomLogger):
error_override: str | None,
) -> Span | None:
data = ServiceSpanData.from_payload(payload, event_metadata=event_metadata)
# ``None`` role means metrics-only (framework instrumentation that duplicates a gen-AI
# span, or ``auth`` which gets a live phase span instead); it never enters the trace.
# Decide whether this service call is a span at all, and of what kind.
# ``None`` means metrics-only (framework instrumentation that duplicates a
# gen-AI span — ``self``/``router``/``proxy_pre_call`` — or ``auth``, which
# gets a live phase span instead). Those still feed Prometheus/Datadog via
# their own hooks; they just never enter the trace.
role = span_role_for_service(data.service_name)
if role is None:
return None
# Skip a ping with neither timing nor a parent (in-memory queue gauges): a span would be a
# zero-duration root. Real background work passes start/end times; anything with a parent emits.
# A metrics-only ping with neither timing nor a parent (in-memory queue
# gauges) is not a traceable operation; a span for it would be a
# zero-duration root with no context, so skip it. Real background work
# (budget/reset jobs, spend flush) passes start/end times and still emits
# as a root; anything with a parent emits regardless.
if error_override is None and start_time is None and end_time is None and parent_otel_span is None:
return None
if error_override is not None and data.error is None:
@ -575,9 +611,11 @@ class OpenTelemetryV2(CustomLogger):
error=SpanError(message=error_override),
event_metadata=data.event_metadata,
)
# Parent ambient-first (so the call nests under the active request phase, e.g. a DB lookup
# under ``auth``), falling back to the threaded ``parent_otel_span``; a background call has
# neither and starts its own root trace.
# Parent like every other span: ambient context first (so identity Baggage
# rides along and the call nests under whatever request phase is active —
# e.g. a DB lookup under the live ``auth`` span), falling back to the
# server span the proxy threaded as ``parent_otel_span``. A background
# service call has neither, so it starts its own root trace.
parent_context = resolve_parent_context(threaded=parent_otel_span)
return self._emitter.emit(
role,
@ -595,9 +633,13 @@ class OpenTelemetryV2(CustomLogger):
def seed_request_identity(self, user_api_key_dict: Any, model: Any = None) -> None:
"""Attach request-identity Baggage to the current context + server span.
Called once at the auth boundary so every span emitted afterwards inherits identity via
``LiteLLMBaggageSpanProcessor``. Auth-internal DB lookups before the key resolves stay
unlabeled, which is correct.
Seeding identity into Baggage makes **every** span emitted afterwards for
this request — LLM call, guardrail, DB call — inherit it via
``LiteLLMBaggageSpanProcessor``. Called once at the auth boundary (as soon
as the key resolves) so post-auth spans are labeled consistently; the
Baggage rides the request task's contextvar from there on. Auth-internal
DB lookups that run before the key is known stay unlabeled — identity
isn't determined yet, which is correct.
"""
try:
identity = RequestIdentity.from_user_api_key_auth(user_api_key_dict)
@ -612,14 +654,19 @@ class OpenTelemetryV2(CustomLogger):
# Attach (no detach): the contextvar is scoped to this request's
# asyncio task and is reclaimed when the task ends.
attach(set_request_baggage(bag, context=get_current()))
# The instrumentor started the server span before this ran, so the Baggage
# processor (fires only at span start) won't backfill it; stamp identity directly.
# Prefer the anchored root over ambient so identity lands on the server span even
# when seeding from inside the live ``auth`` phase span.
# The server span was started by the instrumentor before this ran,
# so the Baggage processor (which only fires at span start) won't
# backfill it — stamp identity on it directly. Prefer the anchored
# root span over the ambient one so identity still lands on the
# server span when seeding from inside the live ``auth`` phase span
# (the auth-failure path), where ``get_current_span`` is the phase
# span, not the request's root.
server_span = request_root_span() or get_current_span()
if is_recordable_span(server_span):
# Re-capture the anchor here too, covering entrypoints that bypass
# ``create_litellm_proxy_request_started_span`` (e.g. the SDK path). Idempotent.
# Re-capture the anchor here too: this runs post-auth with the
# server span active and covers entrypoints that bypass
# ``create_litellm_proxy_request_started_span`` (e.g. the SDK
# path's ``async_pre_call_hook``). Idempotent.
set_request_root_span(server_span)
for key, value in bag.items():
server_span.set_attribute(key, value)
@ -656,12 +703,14 @@ class OpenTelemetryV2(CustomLogger):
exception: "Exception | None",
status_code: int,
) -> None:
"""Stamp v2 error.* attributes on the FastAPI-owned SERVER span for a failure that dies
before any LLM-call span exists (malformed body, auth/validation rejection).
Only decorates the span (never sets status, ends it, or emits an exception event); the
instrumentor still owns the span's lifecycle, matching v1's SERVER-span behavior.
"""
"""Stamp the v2 error.* attributes on the FastAPI-owned SERVER span for a
failure that dies before any LLM-call span exists (malformed body, auth /
validation rejection). Called from the proxy's global exception handler via
``_close_dangling_otel_server_span``. The instrumentor still owns the span's
status and lifecycle, so this only decorates it — never sets status, never
ends it — and emits no exception event, matching v1's SERVER-span behavior
and avoiding a duplicate of the event ``async_post_call_failure_hook`` or
the ``auth`` phase span already records."""
if span is None or not is_recordable_span(span):
return
stamp_error(
@ -678,13 +727,18 @@ class OpenTelemetryV2(CustomLogger):
user_api_key_dict: "UserAPIKeyAuth",
traceback_str: "str | None" = None,
) -> None:
"""Stamp error.* on the request's root SERVER span for a proxy-level failure that never
reached an LLM call (empty body rejected, auth failure), so it carries the same error keys
a failed LLM call does.
"""Stamp error.* on the request's root SERVER span for a proxy-level
failure that never reached an LLM call (empty body rejected in the
endpoint, auth failure), so the failed request carries the same error keys
a failed LLM call does. v1's ``OpenTelemetry`` implemented this same hook;
v2 lost it when it stopped subclassing ``OpenTelemetry``, which is the
LIT-4179 regression for pre-call failures.
For an MCP message the session-task anchor is whatever request opened the session, so
prefer the transport the gateway published for this specific message.
"""
An MCP message is handled on the session's task, where the request-root
anchor is whatever request opened the session, so prefer the transport the
gateway published for this specific message. Without that, a failed tool
call aimed its error at the ``initialize`` request's finished span and the
SDK dropped it, leaving the POST that actually failed unmarked."""
span = mcp_message_transport_span() or request_root_span() or user_api_key_dict.parent_otel_span
if span is None or not is_recordable_span(span):
return None
@ -692,10 +746,18 @@ class OpenTelemetryV2(CustomLogger):
return None
def emit_guardrail_span(self, entry: "StandardLoggingGuardrailInformation") -> None:
# A guardrail is a sibling of the LLM call under the request's root span, so parent it to
# the explicit anchor, never the active span (which during a pre_call guardrail can be the
# live ``auth`` phase span). Emit with the guardrail's actual execution window so a
# pre_call guardrail is placed before the LLM call rather than at emission time.
# Emitted by the guardrail-recording code the moment a guardrail finishes,
# not from a post-call hook — that hook does not fire on every path (a
# pass-through request that passes its guardrails never reaches it), which
# left passing guardrails without a span.
#
# A guardrail is a sibling of the LLM call under the request's root span,
# so parent it to the explicit anchor — never the active span, which during
# a pre_call guardrail can be the live ``auth`` phase span. Emit with the
# guardrail's actual execution window so a pre_call guardrail is placed
# before the LLM call rather than at emission time. One entry in, one span
# out — the module-level entry point routes each entry to this single
# registered logger so a guardrail is never emitted more than once.
data = GuardrailSpanData.from_logging_entry(entry)
self._emitter.emit(
SpanRole.GUARDRAIL,
@ -721,10 +783,21 @@ def select_global_otel_v2_logger(
) -> "OpenTelemetryV2":
"""The single ``OpenTelemetryV2`` whose provider should become the OTel global.
Prefer the canonical ``registered`` owner every other v2 entry point routes through, so the
server span and gen-ai spans share one provider and one trace. Fall back to a v2 logger in
``in_memory_loggers`` (the SDK path), then build a generic one from ``OTEL_*`` only when none
was configured; each fallback avoids the second generic logger that orphaned the gen-ai spans.
The callback factory designates one logger as canonical the moment it builds
the first one (``_init_otel_logger_on_litellm_proxy`` sets
``proxy_server.open_telemetry_logger``), and every other v2 entry point —
guardrail, identity seeding, phase spans — already routes through that same
``registered`` owner. Reuse it here too so the global provider has one source
of truth instead of a second, independently-derived guess; this is the logger
a preset (arize, langfuse, …) folds the ``OTEL_*`` base exporter and its own
exporter into, so the FastAPI server span and the gen-ai spans share one
provider and one trace.
Fall back to ``in_memory_loggers`` for the SDK path, where no proxy global is
set (selecting from there, not ``service_callback``, which a preset logger does
not always reach), and build a generic logger from ``OTEL_*`` only when none was
configured at all. Each fallback still avoids the second generic logger that
orphaned the gen-ai spans onto a different backend than the server span.
"""
if registered is not None:
return registered
@ -739,9 +812,15 @@ def publish_global_otel_v2_provider(
) -> "OpenTelemetryV2":
"""Select the single v2 logger and publish its provider as the OTel global.
Called once at startup; ``registered`` (the canonical owner) makes the global reuse the
logger the rest of the v2 code emits through, and both it and ``set_global_provider`` are
injected so the publish step is unit-testable without touching real global OTel state.
The proxy calls this once at startup, after callbacks are initialized, so the
preset logger already exists; it passes ``registered`` (the canonical owner the
factory designated as ``proxy_server.open_telemetry_logger``) so the global
provider reuses the same logger the rest of the v2 code emits through (see
:func:`select_global_otel_v2_logger`). Both ``registered`` and
``set_global_provider`` (the proxy passes
``opentelemetry.trace.set_tracer_provider``) are injected so the publish step is
unit-testable without reading or mutating real global OTel state. Returns the
logger whose provider was published.
"""
logger = select_global_otel_v2_logger(in_memory_loggers, registered=registered)
set_global_provider(logger._tracer_provider)
@ -760,9 +839,13 @@ def _registered_v2_logger() -> "OpenTelemetryV2 | None":
def emit_guardrail_span(entry: "StandardLoggingGuardrailInformation") -> None:
"""Emit a guardrail span on the registered v2 OTel logger.
Called when a guardrail finishes, so a span is produced even when no post-call hook runs (the
pass-through allow path). Routes through the single canonical logger so a guardrail yields
exactly one span. Best-effort: emission must never break guardrail evaluation.
Called by the guardrail-recording code the moment a guardrail finishes, so a
span is produced regardless of whether a post-call hook later runs (it does
not on the pass-through allow path). Routes through the single canonical
logger — the same one every other v2 entry point uses — so a guardrail
recorded once yields exactly one span; fanning out across every reachable
``OpenTelemetryV2`` instance double-emits the same entry. Best-effort: span
emission must never break guardrail evaluation.
"""
logger = _registered_v2_logger()
if logger is None:

146
litellm/integrations/otel/model/metadata.py
View file

@ -1,18 +1,37 @@
"""The single translation layer between a request's metadata and the spans.
Every field is parsed **once**, here, out of the ``StandardLoggingPayload`` (or a
``UserAPIKeyAuth`` at the auth boundary), so span data, baggage, and the mappers
read typed fields instead of the raw ``metadata`` / ``hidden_params`` dicts.
Every relevant field litellm exposes about a request — the user-facing model,
the model actually dispatched to the provider, the deployment, and the caller's
identity (team, key, end-user) — is parsed **once**, here, out of the
``StandardLoggingPayload`` (or a ``UserAPIKeyAuth`` at the auth boundary). Span
data, baggage promotion, and the mappers then read these typed fields instead of
each digging into the raw ``metadata`` / ``hidden_params`` dicts.
:class:`RequestIdentity` holds caller identity (team/key/end-user), seeded into
Baggage at the auth boundary before routing has picked a deployment; its
``provider_model`` is absent from that early seed and filled only from the payload
at close. :class:`RequestContext` is the fully-resolved view at close, wrapping it.
Two models live here because a request's identity is known *before* its model
resolution is:
The request-vs-provider model split: a caller asks for a *model group* (``gpt-4o``)
that routes to a concrete deployment (``azure/my-deployment``). ``gen_ai.request.model``
records the group and ``litellm.provider.model`` the dispatched model; they coincide
on the SDK path, which has no group.
* :class:`RequestIdentity` — team / key / end-user, seeded into Baggage at the
auth boundary (``from_user_api_key_auth``), before routing has picked a
deployment. ``provider_model`` is therefore absent from that early seed and is
only filled in from the payload once the call closes.
* :class:`RequestContext` — the full picture available at close: the resolved
request vs. provider model split, plus the response model, model group, model
id, and api base, wrapping the :class:`RequestIdentity`.
The request-vs-provider model split is the subtle part. On the proxy a caller
asks for a *model group* (e.g. ``gpt-4o``) that routes to a concrete deployment
(e.g. ``azure/my-deployment``); the two are distinct and both worth recording.
``StandardLoggingPayload`` exposes them as:
* ``model_group`` — the user-facing name the caller requested.
* ``model`` — already reconstructed (see ``reconstruct_model_name``) to the name
litellm dispatched to the provider (the deployment, provider-prefixed).
* ``hidden_params.litellm_model_name`` — a secondary source for the dispatched
model (populated only on some call paths, e.g. files).
So ``gen_ai.request.model`` is the *group* (falling back to the call model on the
SDK path, which has no group), and ``litellm.provider.model`` is the *dispatched*
model. They coincide on the SDK path, which is correct.
"""
from __future__ import annotations
@ -36,22 +55,33 @@ class RequestIdentity:
call_id: str | None = None
team_id: str | None = None
team_alias: str | None = None
# The team's free-form metadata, carried raw; filtered to an operator allowlist at Baggage-promotion time.
# The team's free-form metadata, carried raw (empty/missing -> None) and
# filtered to an operator allowlist only at Baggage-promotion time, so an
# unconfigured deployment never promotes any of it.
team_metadata: Mapping[str, Any] | None = None
key_hash: str | None = None
end_user: str | None = None
# The model litellm dispatched to the provider; known only at close, so absent from the auth-time seed.
# The model litellm dispatched to the provider. Only known once the call
# completes (routing has picked a deployment), so it's absent from the
# auth-time seed and filled only from the payload.
provider_model: str | None = None
metadata: Mapping[str, str] = field(default_factory=dict)
@classmethod
def from_payload(cls, payload: "StandardLoggingPayload") -> "RequestIdentity":
"""Parse caller identity (incl. resolved ``provider_model``) from a closed request's payload."""
"""Parse caller identity out of a closed request's payload metadata.
``provider_model`` is resolved here too (see :func:`resolve_provider_model`)
so the identity carried into Baggage labels every span with the dispatched
model, not just the user-facing one.
"""
raw_meta = cast(Mapping[str, object], payload.get("metadata") or {})
metadata = {key: str(value) for key, value in raw_meta.items() if isinstance(value, (str, bool, int, float))}
return cls(
call_id=as_str(payload.get("litellm_call_id")) or as_str(payload.get("id")),
# Prefer the canonical ``user_api_key_team_id``; the bare ``team_id`` is a legacy alias.
# StandardLoggingMetadata's canonical key is ``user_api_key_team_id``;
# the bare ``team_id`` is a legacy alias and is often empty, so prefer
# the canonical key and fall back to the alias.
team_id=as_str(raw_meta.get("user_api_key_team_id")) or as_str(raw_meta.get("team_id")),
team_alias=as_str(raw_meta.get("user_api_key_team_alias")) or as_str(raw_meta.get("team_alias")),
team_metadata=_team_metadata_dict(raw_meta.get("user_api_key_team_metadata")),
@ -63,10 +93,14 @@ class RequestIdentity:
@classmethod
def from_user_api_key_auth(cls, auth: object) -> "RequestIdentity":
"""Identity from a ``UserAPIKeyAuth`` (duck-typed to avoid a proxy import).
"""Identity from a ``UserAPIKeyAuth`` (duck-typed to keep this module
free of a proxy import).
Seeds Baggage at the pre-call hook so every span inherits identity. Metadata
sub-keys use the ``user_api_key_*`` names Baggage promotion expects.
Used in the pre-call hook to seed Baggage early — before any LLM,
guardrail, or service span is created — so the whole request's spans
inherit identity, not just the LLM-call span. Metadata sub-keys use the
``user_api_key_*`` names that ``baggage.DEFAULT_BAGGAGE_METADATA_KEYS``
promotes.
"""
get = lambda name: getattr(auth, name, None) # noqa: E731
metadata = {
@ -85,14 +119,20 @@ class RequestIdentity:
team_metadata=_team_metadata_dict(get("team_metadata")),
key_hash=as_str(get("api_key")),
end_user=as_str(get("end_user_id")),
# ``provider_model`` is unknown at the auth boundary (routing hasn't picked a deployment yet).
# ``provider_model`` is unknown at the auth boundary — routing hasn't
# picked a deployment yet — so it's only populated from the payload.
metadata=metadata,
)
@dataclass(frozen=True)
class RequestContext:
"""The fully-resolved view of a closed request, parsed once from the payload."""
"""The fully-resolved view of a closed request, parsed once from the payload.
``request_model`` is the user-facing requested model and ``provider_model``
(on :attr:`identity`) is the model litellm dispatched to the provider; the two
differ on the proxy (group vs. deployment) and coincide on the SDK path.
"""
request_model: str
response_model: str | None
@ -127,26 +167,43 @@ class RequestContext:
# --- live-callback kwargs parsing ------------------------------------------- #
# Parse the live callback ``kwargs`` god object and raw pre/post-call ``data`` dicts —
# the untyped request state reaching a ``CustomLogger`` before a ``StandardLoggingPayload``.
#
# The model and helpers below parse the *live* callback ``kwargs`` god object (and
# the raw pre/post-call ``data`` dicts) — the untyped request state that reaches a
# ``CustomLogger`` before, or instead of, a ``StandardLoggingPayload``. They live
# here, with the payload/auth parsers, so every read out of a request's raw dicts
# is in one place rather than scattered across the ``CustomLogger``.
@dataclass(frozen=True)
class LLMCallEvent:
"""The typed view of the live callback ``kwargs`` (``model_call_details``), parsed once."""
"""The typed view of the live callback ``kwargs`` (``model_call_details``).
# The ``litellm_call_id`` correlating ``pre_call`` with the close callback; the stable
# key for the open-call carrier.
litellm hands every callback an untyped ``kwargs`` god object. The fields the
OTel logger needs out of it are parsed **once**, here, so the ``CustomLogger``
reads typed attributes instead of digging into the dict at each boundary.
"""
# The ``litellm_call_id`` correlating ``pre_call`` with the close callback.
# Present in ``model_call_details`` at ``pre_call`` and in both the kwargs and
# the ``standard_logging_object`` at success/failure, so it's a stable key for
# the open-call carrier — no back-reference to the logging object required (the
# object isn't reachable from the callback kwargs at ``pre_call`` time).
call_id: str | None
# The success/failure payload; ``None`` at ``pre_call`` or if the call closed with no payload.
# The ``StandardLoggingPayload`` carried on a success/failure callback; ``None``
# at ``pre_call``, or when the call closed before any payload materialized (so
# there is nothing to stamp on the span).
payload: "StandardLoggingPayload | None"
otel_destinations: tuple[OtelDestination, ...]
# The request's ``standard_callback_dynamic_params`` (team/key OTLP credentials), or ``None``
# when the call isn't scoped; routes the gen-AI span to a credential-scoped tracer.
# The ``standard_callback_dynamic_params`` routing the call to a per-tenant
# tracer (its own exporter/endpoint), or ``None`` when the call isn't scoped.
dynamic_params: "StandardCallbackDynamicParams | None"
# True for synthetic proxy-gate logs (auth/rate-limit rejections): no upstream call, so no span.
# True for synthetic proxy-gate logs (auth / rate-limit rejections): they fire
# the ``pre_call`` hook but never made an upstream call, so they get no span.
is_no_upstream_call: bool
# Best-effort ``"{operation} {model}"`` name at ``pre_call``; only matters for a leaked span (renamed at close).
# A best-effort ``"{operation} {model}"`` name known at ``pre_call`` time. The
# span is renamed from the typed payload at close (``finish_span``); this only
# needs to be reasonable for a span that never gets closed (a leak).
provisional_span_name: str
time_to_first_chunk_seconds: float | None
@ -168,8 +225,10 @@ class LLMCallEvent:
def time_to_first_chunk_seconds(kwargs: Mapping[str, Any]) -> float | None:
"""Seconds from upstream request (``api_call_start_time``) to first streamed chunk
(``completion_start_time``); ``None`` for non-streaming calls."""
"""Seconds from the upstream request being issued (``api_call_start_time``)
to the first streamed chunk (``completion_start_time``); ``None`` for
non-streaming calls, where ``completion_start_time`` is backfilled with the
end time and would not measure first-chunk latency."""
optional_params = cast(Mapping[str, Any], kwargs.get("optional_params") or {})
if not optional_params.get("stream"):
return None
@ -190,7 +249,11 @@ def _call_id(payload: "StandardLoggingPayload | None", kwargs: Mapping[str, Any]
def model_from_request_data(data: object) -> str | None:
"""The user-facing ``model`` from a pre-call ``data`` dict (``None`` if absent)."""
"""The user-facing ``model`` from a pre-call ``data`` dict (``None`` if absent).
Read at the auth boundary to label early Baggage before routing has resolved
a deployment; ``data`` is duck-typed since it arrives untyped from the proxy.
"""
if isinstance(data, Mapping):
return as_str(data.get("model"))
return None
@ -199,13 +262,16 @@ def model_from_request_data(data: object) -> str | None:
def resolve_provider_model(payload: "StandardLoggingPayload") -> str | None:
"""The model litellm dispatched to the provider, from the payload.
Prefers ``metadata.deployment``, then ``hidden_params.litellm_model_name``, then the
top-level ``model`` (already resolved to the provider-prefixed deployment name).
Prefers the explicit ``hidden_params.litellm_model_name`` (set on call paths
that know it, e.g. files), then the top-level ``model`` — which
``reconstruct_model_name`` has already resolved to the deployment's
provider-prefixed name. Returns ``None`` only when neither is present.
"""
raw_meta = cast(Mapping[str, object], payload.get("metadata") or {})
hidden = cast(Mapping[str, object], payload.get("hidden_params") or {})
return (
# ``deployment`` (most precise) survives only on paths that don't strip it from metadata.
# ``deployment`` survives only on paths that don't strip it from metadata;
# harmless (and most precise) to prefer it when present.
as_str(raw_meta.get("deployment")) or as_str(hidden.get("litellm_model_name")) or as_str(payload.get("model"))
)
@ -218,7 +284,13 @@ def _model_info_id(model_info: object) -> str | None:
def _team_metadata_dict(value: object) -> Mapping[str, Any] | None:
"""The team's free-form metadata as a raw mapping, or ``None`` when missing or empty."""
"""The team's free-form metadata as a raw mapping, or ``None`` when missing
or empty.
Carried raw on the identity and filtered to an operator allowlist only at
Baggage-promotion time (see ``baggage.promoted_baggage``), so an empty case
is dropped rather than carrying a useless ``{}``.
"""
if isinstance(value, Mapping) and value:
return dict(value)
return None

83
litellm/integrations/otel/plumbing/providers.py
View file

@ -56,9 +56,12 @@ def to_otel_span_kind(kind: LiteLLMSpanKind) -> SpanKind:
return _SPAN_KIND_BY_ROLE_KIND[kind]
# Custom exporter factories keyed by ``ExporterSpec.kind``. A preset registers one when its
# destination needs construction logic the built-in kinds can't express (e.g. an exporter that
# fetches an auth token lazily on first export). Keeps this module vendor-agnostic.
# Custom exporter factories keyed by ``ExporterSpec.kind``. A preset registers
# one here when its destination needs construction logic the built-in kinds
# can't express — e.g. an exporter that fetches an auth token lazily on its
# first export (off the event loop) instead of blocking at config-build time.
# Keeping the registry here lets this module stay vendor-agnostic: the factory
# lives with the integration that needs it.
_EXPORTER_FACTORIES: dict[str, Callable[[ExporterSpec], SpanExporter]] = {}
@ -99,12 +102,16 @@ class LiteLLMBaggageSpanProcessor(SpanProcessor):
def _otlp_traces_endpoint(endpoint: str | None) -> str | None:
"""Point an OTLP/HTTP base endpoint at the ``/v1/traces`` signal path.
An explicitly passed endpoint is used verbatim (unlike ``OTEL_EXPORTER_OTLP_ENDPOINT``), so a
base URL would POST to the root and 404; append the signal path here, leaving a correct path intact.
``OTEL_EXPORTER_OTLP_ENDPOINT`` is a base URL (e.g. ``http://host:4318``).
The OTLP/HTTP exporter only appends the ``/v1/traces`` path when it reads
that env var itself; when an endpoint is passed explicitly it is used
verbatim, so a base URL would POST to the root and the collector returns
404. Append the signal path here (leaving an already-correct path intact).
"""
if not endpoint:
return endpoint
endpoint = endpoint.rstrip("/")
# Splunk Observability uses ``/v2/trace/otlp``; never rewrite it.
if endpoint.endswith("/v1/traces") or "/v2/trace/otlp" in endpoint or endpoint.endswith("/api/trace"):
return endpoint
for other_signal in ("/v1/logs", "/v1/metrics"):
@ -123,10 +130,8 @@ def default_otlp_kind_for_backend(callback_name: "str | None") -> str:
def destination_resource_attrs(destination: "OtelDestination") -> Mapping[str, str]:
"""The destination's builder-declared Resource attributes (e.g. Arize's
``model_id`` / ``arize.project.name``; empty for header-routed backends).
Both export paths to a destination -- the fan-out processor and the per-tenant
clone provider -- read these so the gen-AI span and its parents share one Resource.
``model_id`` / ``arize.project.name``; empty for header-routed backends), read
by both export paths so the gen-AI span and its parents share one Resource.
"""
return dict(destination.resource_attributes)
@ -198,8 +203,10 @@ def build_span_exporter(config: OpenTelemetryV2Config) -> SpanExporter:
def _otlp_metrics_endpoint(endpoint: str | None) -> str | None:
"""Point an OTLP/HTTP base endpoint at the ``/v1/metrics`` signal path.
Mirrors ``_otlp_traces_endpoint`` for the metrics signal (an explicitly passed endpoint is
used verbatim, so a base URL would POST to the root).
The OTLP/HTTP exporter only appends ``/v1/metrics`` when it reads
``OTEL_EXPORTER_OTLP_ENDPOINT`` itself; an explicitly passed endpoint is used
verbatim, so a base URL would POST to the root. Mirror ``_otlp_traces_endpoint``
for the metrics signal (rewriting a sibling signal path when present).
"""
if not endpoint:
return endpoint
@ -265,8 +272,10 @@ def build_metric_reader(config: OpenTelemetryV2Config) -> "MetricReader":
def _otlp_logs_endpoint(endpoint: str | None) -> str | None:
"""Point an OTLP/HTTP base endpoint at the ``/v1/logs`` signal path.
Mirrors ``_otlp_traces_endpoint`` for the logs signal (an explicitly passed endpoint is used
verbatim, so a base URL would POST to the root).
The OTLP/HTTP exporter only appends ``/v1/logs`` when it reads
``OTEL_EXPORTER_OTLP_ENDPOINT`` itself; an explicitly passed endpoint is used
verbatim, so a base URL would POST to the root. Mirror ``_otlp_traces_endpoint``
for the logs signal (rewriting a sibling signal path when present).
"""
if not endpoint:
return endpoint
@ -282,8 +291,10 @@ def _otlp_logs_endpoint(endpoint: str | None) -> str | None:
def build_log_exporter(config: OpenTelemetryV2Config) -> LogExporter:
"""Build a log exporter mirroring the exporter selection of the other signals.
``console`` (and any unrecognized kind) to the console; ``otlp_http``/``otlp_grpc`` over OTLP;
``in_memory`` buffers for tests. Events ride the single-destination shorthand fields, not ``exporters``.
``console`` (and any unrecognized kind) exports to the console; ``otlp_http``
and ``otlp_grpc`` export over OTLP with the configured endpoint/headers;
``in_memory`` buffers for tests. Like GenAI metrics, events ride the
single-destination shorthand fields, not the multi-exporter ``exporters`` list.
"""
kind = (config.exporter or "console").lower()
if kind in ("in_memory", "inmemory", "memory"):
@ -318,8 +329,11 @@ def build_logger_provider(
) -> SDKLoggerProvider:
"""Build the :class:`LoggerProvider` GenAI events export through.
``log_exporter`` is an explicit override (tests); otherwise selected from the config via
:func:`build_log_exporter`. Console/in-memory get a Simple processor, everything else Batch.
``log_exporter`` is an explicit override (tests inject an
``InMemoryLogExporter``); otherwise the exporter is selected from the config's
exporter kind via :func:`build_log_exporter`. Console and in-memory exporters
get a Simple processor (synchronous export, which tests rely on), everything
else a Batch processor — the same split as span processing.
"""
exporter = log_exporter if log_exporter is not None else build_log_exporter(config)
provider = SDKLoggerProvider(resource=build_resource(config))
@ -334,12 +348,14 @@ def resolve_logger_provider(
config: OpenTelemetryV2Config,
logger_provider: SDKLoggerProvider | None = None,
) -> SDKLoggerProvider | None:
"""Resolve the :class:`LoggerProvider` GenAI events record through, or ``None`` when the
operator opted out of the logs signal.
"""Resolve the :class:`LoggerProvider` GenAI events record through, or ``None``
when the operator has opted out of the logs signal.
An injected provider wins (DI/tests); an operator-configured SDK global is reused; an explicit
``NoOpLoggerProvider`` is an opt-out (``None``). Only the default placeholder global makes V2
build and publish one from the config.
Same resolution order as :func:`resolve_meter_provider`: an injected provider
wins (DI/tests); an operator-configured SDK global is reused so events ride
their pipeline; an explicit ``NoOpLoggerProvider`` global is an opt-out and
yields ``None``, so no event is ever built. Only the default placeholder
global makes V2 build a provider from the config and publish it as the global.
"""
if logger_provider is not None:
return logger_provider
@ -379,10 +395,14 @@ def resolve_meter_provider(
) -> MeterProvider:
"""Resolve the :class:`MeterProvider` GenAI metrics record through.
An injected provider wins (DI/tests); otherwise reuse the operator's configured global (a real
SDK provider or an explicit ``NoOpMeterProvider`` opt-out). Only the default proxy placeholder
makes V2 build and publish one from the config; the built provider is returned so its reader
thread stays live.
An injected provider wins (DI/tests). Otherwise reuse whatever the operator has
configured as the global, whether a real SDK provider or an explicit
``NoOpMeterProvider``, so the GenAI histograms ride the operator's
readers/exporters and an explicit opt-out is honored. Only when the global is
still the default proxy placeholder does V2 build one from the config and
publish it as the global, mirroring how V2 owns trace export. The built
provider is the one returned, so its reader thread is always live, never
orphaned.
"""
if meter_provider is not None:
return meter_provider
@ -416,13 +436,10 @@ def build_tracer_provider(
tenant_fan_out_owner: str | None = None,
attach_tenant_fan_out: bool = False,
) -> TracerProvider:
"""Build the shared :class:`TracerProvider`: the Baggage processor first, then one
``SpanProcessor`` per ``config.exporters`` entry (``exporter`` overrides with a test exporter).
``attach_tenant_fan_out``/``tenant_fan_out_owner`` add a ``TenantFanOutSpanProcessor`` that
forwards proxy-internal spans to the request's destinations. The main v2 provider always opts
in (so the server span reaches the destination and its gen-AI child isn't orphaned); per-tenant
clone providers pass neither.
"""Build the shared :class:`TracerProvider`: Baggage processor first, then one
``SpanProcessor`` per ``config.exporters`` entry (``exporter`` overrides for tests).
``attach_tenant_fan_out``/``tenant_fan_out_owner`` add a ``TenantFanOutSpanProcessor``
forwarding proxy-internal spans to the request's destinations.
"""
provider = TracerProvider(resource=build_resource(config))
if baggage_processor is None:

78
litellm/integrations/otel/plumbing/routing.py
View file

@ -1,23 +1,7 @@
"""Per-request multi-tenant tracer routing and span fan-out.
``TenantTracerCache`` routes the gen-AI LLM-call span, building per-tenant clone
``TracerProvider``s that export to the request's admin-owned destinations plus the
configured/global exporter. ``TenantFanOutSpanProcessor`` (at the bottom) forwards the
proxy-internal spans (server, auth, DB, cost) to every destination. The destinations both
read from a server-only contextvar, so a caller can neither redirect a trace nor spawn
providers through them.
Separately, ``genai_tracers_for`` also routes the gen-AI span by the request's
``standard_callback_dynamic_params`` (team/key OTLP credentials), restoring the per-request
credential routing that predates the admin-destination refactor: it rewrites only the owned
exporter's headers, bounded by the same LRU. Those params are partly caller-influenced, so
this path can spawn a per-request provider; closing that for request-body-supplied credentials
(vs admin-configured team ``callback_vars``) is tracked separately.
The gen-AI path (``tracers_for``) groups destinations by their backend-required Resource
attributes and builds one provider per group, because a span carries exactly one Resource and
a backend like Arize selects its project FROM it, so two Arize projects each get a correctly
tagged span instead of a last-wins merge. Empty destinations -> the logger's default (global only).
``TenantTracerCache`` routes the gen-AI span to per-tenant/destination providers;
``TenantFanOutSpanProcessor`` forwards proxy-internal spans to every admin-resolved destination.
"""
import threading
@ -42,18 +26,23 @@ from litellm.integrations.otel.presets import dynamic_otlp_headers
if TYPE_CHECKING:
from litellm.types.utils import StandardCallbackDynamicParams
# Exporter kinds that ignore headers — never rewritten with dynamic credentials.
_NON_OTLP_KINDS = ("console", "in_memory", "inmemory", "memory")
# Cap on distinct credential-scoped providers held at once. ``dynamic_params``
# can be populated from request metadata, so an unbounded cache lets a caller
# spawn one ``TracerProvider`` (plus its ``BatchSpanProcessor`` background
# thread) per unique credential set and exhaust the proxy. The LRU bound keeps
# the working set of active tenants resident while flushing and shutting down
# evicted providers so their threads are reclaimed.
_MAX_CACHED_PROVIDERS = 256
def _shutdown_in_background(evicted: "TracerProvider | SpanProcessor") -> None:
"""Reclaim an evicted provider/processor's ``BatchSpanProcessor`` worker thread.
Dropping it from the cache without ``shutdown`` leaves that daemon thread running for
the life of the process (it does not "drain on its own"). ``shutdown`` force-flushes
and can do network I/O, so it runs fire-and-forget on a daemon thread rather than on
the request path; the evicted object is otherwise unreferenced.
Dropping it without ``shutdown`` leaks the daemon thread; ``shutdown`` force-flushes and can
do network I/O, so it runs fire-and-forget on a daemon thread rather than the request path.
"""
def _run() -> None:
@ -98,14 +87,9 @@ class TenantTracerCache:
) -> "tuple[Tracer, ...]":
"""The tracers for this request's gen-AI span, one per distinct Resource group.
A backend like Arize selects its project from the Resource, so destinations are grouped
by ``destination_resource_attrs`` and the caller emits the span once per tracer. The
configured/global exporters ride the FIRST group only, so the global receives the span
once. Empty ``destinations`` -> the logger's default tracer (deny).
``include_base_on_first`` is set to ``False`` by ``genai_tracers_for`` when a
credential-scoped tracer already carries the configured/global exporters, so the
destination groups don't also export the span to the global collector a second time.
Destinations are grouped by ``destination_resource_attrs`` (a backend like Arize selects its
project from the Resource); the configured/global exporters ride the first group only, and
``include_base_on_first=False`` drops them when a credential-scoped tracer already carries them.
"""
if not destinations:
return (default,)
@ -120,15 +104,9 @@ class TenantTracerCache:
destinations: "tuple[OtelDestination, ...]",
dynamic_params: "StandardCallbackDynamicParams | None",
) -> "tuple[Tracer, ...]":
"""The gen-AI span's tracers, layering per-request credential routing over the
admin-destination fan-out.
When the request carries this backend's team/key OTLP credentials
(``standard_callback_dynamic_params``), the global export rides a credential-scoped
provider (the configured exporters with this backend's own exporter rewritten to those
credentials), and the admin-destination groups omit the base exporters so the span
reaches the global collector exactly once. Without dynamic credentials this is the plain
destination fan-out (empty destinations -> the default tracer).
"""The gen-AI span's tracers, layering per-request credential routing over the destination
fan-out: with this backend's team/key OTLP credentials the global export rides a
credential-scoped provider and the destination groups omit the base exporters; else plain fan-out.
"""
headers = dynamic_otlp_headers(self._callback_name, dynamic_params)
if not headers:
@ -161,12 +139,14 @@ class TenantTracerCache:
return get_tracer(provider, self._tracer_name)
def _config_with_headers(self, headers: "dict[str, str]") -> OpenTelemetryV2Config:
"""Clone the config, stamping ``headers`` onto this backend's own exporter only.
"""Clone the config, stamping ``headers`` onto the credential's own exporter.
``headers`` are the per-request credentials of ``self._callback_name``, so they apply
only to the exporter that integration contributed (``spec.owner``). A request carrying
one tenant's Arize key must never rewrite a co-configured Langfuse or self-hosted
collector exporter, which would leak that key to a different backend.
``headers`` are the per-request credentials of ``self._callback_name`` (the
integration that built this cache), so they apply only to the exporter that
integration contributed (``spec.owner``). A request that carries one
tenant's Arize key must never rewrite the headers of a co-configured
Langfuse or self-hosted collector exporter, which would leak that key to a
different backend.
"""
header_str = ",".join(f"{key}={value}" for key, value in headers.items())
exporters = [
@ -260,13 +240,11 @@ class TenantTracerCache:
*,
include_base_exporters: bool = True,
) -> OpenTelemetryV2Config:
"""Clone the config, appending one exporter per resolved destination (its resolved host
and own auth headers) so one span exports to every destination.
"""Clone the config, appending one exporter per destination so one span exports to every one.
``include_base_exporters`` keeps the configured/global exporters; ``tracers_for`` sets it
only on the first Resource group so the global gets the span once, not once per group. The
clone's Resource folds in the destinations' ``destination_resource_attrs`` (Arize needs
``model_id`` / ``arize.project.name``); callers group by those first so the merge is lossless."""
``include_base_exporters`` keeps the configured/global exporters (``tracers_for`` sets it only on
the first group); the clone's Resource folds in the destinations' ``destination_resource_attrs``.
"""
from litellm.integrations.otel.plumbing.providers import (
destination_resource_attrs,
)