From 93061061fa22114ba0f03bb2379068ec3a2fa03f Mon Sep 17 00:00:00 2001 From: Yucheng Zhu Date: Fri, 7 Aug 2026 19:38:45 -0700 Subject: [PATCH] feat(otel): export the trace to the resolved destinations Fans each request's trace out to the destinations its identity resolved, applies per-request credentials only to the exporter their own backend contributed, and caches tracer providers per credential, endpoint and transport. Squashed onto the current staging tip; the previous history conflicted with staging's Final-annotation pass over the same OTEL v2 modules. --- litellm/__init__.py | 1 + litellm/integrations/otel/README.md | 18 +- .../integrations/otel/destination_logger.py | 171 ++++ litellm/integrations/otel/emitter.py | 48 ++ litellm/integrations/otel/logger.py | 239 ++++-- litellm/integrations/otel/model/config.py | 52 +- litellm/integrations/otel/model/metadata.py | 13 +- litellm/integrations/otel/mount.py | 22 +- .../integrations/otel/plumbing/providers.py | 42 +- litellm/integrations/otel/plumbing/routing.py | 429 +++++++++- litellm/integrations/otel/presets/__init__.py | 64 +- litellm/integrations/otel/presets/agentops.py | 23 +- litellm/integrations/otel/presets/arize.py | 58 +- litellm/integrations/otel/presets/base.py | 16 +- litellm/integrations/otel/presets/generic.py | 21 + litellm/integrations/otel/presets/langfuse.py | 53 +- .../integrations/otel/presets/langtrace.py | 1 + litellm/integrations/otel/presets/levo.py | 8 +- litellm/integrations/otel/presets/phoenix.py | 19 +- litellm/integrations/otel/presets/weave.py | 36 +- litellm/litellm_core_utils/litellm_logging.py | 76 +- .../proxy/_experimental/mcp_server/server.py | 20 + .../otel/test_otel_v2_components.py | 107 +++ .../integrations/otel/test_otel_v2_dynamic.py | 762 +++++++++++++++--- .../integrations/otel/test_otel_v2_fan_out.py | 390 +++++++++ .../integrations/otel/test_otel_v2_logger.py | 555 ++++++++++++- .../integrations/otel/test_otel_v2_mount.py | 8 + .../otel/test_otel_v2_multibackend.py | 30 + .../integrations/otel/test_otel_v2_presets.py | 231 +++++- .../test_litellm_logging.py | 446 ++++++++++ .../proxy/test_litellm_pre_call_utils.py | 23 + 31 files changed, 3619 insertions(+), 363 deletions(-) create mode 100644 litellm/integrations/otel/destination_logger.py create mode 100644 litellm/integrations/otel/presets/generic.py create mode 100644 tests/test_litellm/integrations/otel/test_otel_v2_fan_out.py diff --git a/litellm/__init__.py b/litellm/__init__.py index 056dd532f5f..1a7e29191d1 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -137,6 +137,7 @@ _custom_logger_compatible_callbacks_literal = Literal[ "langfuse", "langfuse_otel", "weave_otel", + "generic", "pagerduty", "humanloop", "azure_sentinel", diff --git a/litellm/integrations/otel/README.md b/litellm/integrations/otel/README.md index 338afe04e5e..1368befda5e 100644 --- a/litellm/integrations/otel/README.md +++ b/litellm/integrations/otel/README.md @@ -278,12 +278,13 @@ lives in [`plumbing/`](./plumbing): - [`presets/`](./presets) — each preset reads one integration's env vars and returns an `OpenTelemetryV2Config` (exporter destination + mapper vocabularies + resource attributes). `PRESET_BY_CALLBACK` maps a callback name (`"arize"`, - `"langfuse_otel"`, …) to its preset. Integrations that support team/key-scoped - credentials also provide a per-request OTLP header builder - (`DYNAMIC_HEADERS_BY_CALLBACK`). Presets do **no** network I/O at build time: - AgentOps, for example, mints its JWT lazily inside a custom exporter on the - first export (in the `BatchSpanProcessor` worker thread), never on the event - loop. + `"langfuse_otel"`, …) to its preset. Per-key/team routing is **not** here: a + destination is admin-owned infrastructure, resolved server-side from a named + credential into an `OtelDestination` (`destinations.py`) and applied by + `plumbing/routing.py`. Nothing in `presets/` reads vendor credentials or a host + off a request. Presets do **no** network I/O at build time: AgentOps, for + example, mints its JWT lazily inside a custom exporter on the first export (in + the `BatchSpanProcessor` worker thread), never on the event loop. ## Extending @@ -295,7 +296,8 @@ lives in [`plumbing/`](./plumbing): constructor, so the family stays bounded span-wide rather than per vocabulary. - **A new integration**: add a preset in `presets/` that returns an `OpenTelemetryV2Config`, and register it in `presets/__init__.PRESET_BY_CALLBACK`. - If it supports dynamic credentials, add a header builder to - `DYNAMIC_HEADERS_BY_CALLBACK`. + For admin-owned per-key/team destinations, add an adapter mapping the named + credential's values to an `OtelDestination` in `destinations._ADAPTERS` (or rely + on the generic `otel_endpoint`/`otel_headers` passthrough). - **A new span kind**: add a role to `spans.py` (registry entry + name builder), a payload dataclass in `payloads.py`, and a branch in the relevant mapper(s). diff --git a/litellm/integrations/otel/destination_logger.py b/litellm/integrations/otel/destination_logger.py new file mode 100644 index 00000000000..6c6150474d1 --- /dev/null +++ b/litellm/integrations/otel/destination_logger.py @@ -0,0 +1,171 @@ +"""Export to admin-owned destinations, independently of which logger owns a backend. + +A destination is a sink, never a reason to change ownership. Whether ``OpenTelemetryV2`` +or a legacy logger owns a backend is decided by the operator's own configuration; this +logger delivers the gen-AI span to whichever destinations the request resolved, so +registering a destination for one team cannot change any other tenant's pipeline. + +The span's vocabulary comes from the backend's preset called purely for its mappers and +semconv, with its exporters stripped: the preset's own exporter belongs to the backend's +owning logger, and including it here would export the same call twice. +""" + +from collections.abc import Mapping +from datetime import datetime +from functools import lru_cache +from typing import TYPE_CHECKING, Any + +from opentelemetry.trace import Span + +import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.integrations.otel.logger import OpenTelemetryV2 +from litellm.integrations.otel.model.config import OpenTelemetryV2Config +from litellm.integrations.otel.model.payloads import is_mcp_list_tools +from litellm.integrations.otel.plumbing.context import request_destinations +from litellm.integrations.otel.presets import PRESET_BY_CALLBACK +from litellm.litellm_core_utils.litellm_logging import otel_v2_owned_backends + +if TYPE_CHECKING: + from litellm.integrations.otel.model.destination import OtelDestination + from litellm.integrations.otel.model.event import LLMCallEvent + from litellm.types.utils import StandardCallbackDynamicParams, StandardLoggingPayload + + +def _vocabulary_config(backend: str) -> OpenTelemetryV2Config: + """The backend's span vocabulary with no exporter of its own.""" + preset_fn = PRESET_BY_CALLBACK.get(backend) + if preset_fn is None: + return OpenTelemetryV2Config() + try: + config = preset_fn(allow_missing_credentials=True) + except Exception: # noqa: BLE001 # an unbuildable preset still has a usable default vocabulary + return OpenTelemetryV2Config() + return config.model_copy(update={"exporters": ()}) + + +class _DestinationOnlyOtel(OpenTelemetryV2): + """An ``OpenTelemetryV2`` that reaches admin destinations and nothing else. + + It stays out of the proxy's global callback lists, and drops the request's own + ``callback_vars`` credentials: those name the tenant's account, which the backend's + owning logger already exports to. + """ + + def _init_otel_logger_on_litellm_proxy(self) -> None: + return None + + def _emit_deferred_llm_call( + self, + payload: "StandardLoggingPayload", + destinations: "tuple[OtelDestination, ...]", + start_time_ns: int | None, + end_time_ns: int | None, + time_to_first_chunk_seconds: float | None = None, + dynamic_params: "StandardCallbackDynamicParams | None" = None, + ) -> Span | None: + return super()._emit_deferred_llm_call( + payload, + destinations, + start_time_ns, + end_time_ns, + time_to_first_chunk_seconds, + None, + ) + + def _tracer_dynamic_params(self, call: "LLMCallEvent") -> "StandardCallbackDynamicParams | None": + return None + + def _emit_mcp_list_tools( + self, + kwargs: "Mapping[str, object]", + start_time: "datetime | float | None", + end_time: "datetime | float | None", + ) -> bool: + """Claim the event without emitting. + + A ``tools/list`` span carries no ``gen_ai.operation.name``, so the fan-out span + processor already routes the owning logger's span to the request's destinations. + Emitting a second one here would deliver the same discovery call twice; returning + ``True`` still stops the caller from closing it as an LLM call. + """ + raw_payload = kwargs.get("standard_logging_object") + return isinstance(raw_payload, Mapping) and is_mcp_list_tools(raw_payload) + + def export_to_destinations( + self, + kwargs: "Mapping[str, Any]", + start_time: "datetime | float | None", + end_time: "datetime | float | None", + ) -> None: + """Mirrors ``async_log_success_event``'s dispatch order. + + An MCP event is not an LLM call: closing it as one names the span from the LLM + vocabulary (``chat MCP: list_tools``, ``execute_tool MCP: -``), + drops every MCP semconv attribute, and stamps fabricated zero-token usage on it. + """ + if self._emit_mcp_tool_call(kwargs, start_time, end_time): + return + if self._emit_mcp_list_tools(kwargs, start_time, end_time): + return + self._close_llm_call(kwargs, start_time, end_time) + + +class AdminDestinationLogger(CustomLogger): + """Delivers each request's gen-AI span to the destinations its identity resolved.""" + + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + self._emitters: dict[str, _DestinationOnlyOtel] = {} # mutable-ok: bounded per-backend emitter cache + + def _emitter_for(self, backend: str) -> _DestinationOnlyOtel: + existing = self._emitters.get(backend) + if existing is not None: + return existing + emitter = _DestinationOnlyOtel(config=_vocabulary_config(backend), callback_name=backend) + self._emitters[backend] = emitter + return emitter + + def _export( + self, + kwargs: "Mapping[str, Any]", + start_time: "datetime | float | None", + end_time: "datetime | float | None", + ) -> None: + owned = otel_v2_owned_backends() + for backend in sorted({d.callback_name for d in request_destinations() if d.callback_name} - owned): + try: + self._emitter_for(backend).export_to_destinations(kwargs, start_time, end_time) + except Exception as exc: # noqa: BLE001 # one destination's failure must not break the request or the others + litellm.verbose_logger.debug("OTel V2 destination export for %s failed: %s", backend, exc) + + async def async_log_success_event( + self, + kwargs: "Mapping[str, Any]", + response_obj: Any, + start_time: "datetime | float | None", + end_time: "datetime | float | None", + ) -> None: + self._export(kwargs, start_time, end_time) + + async def async_log_failure_event( + self, + kwargs: "Mapping[str, Any]", + response_obj: Any, + start_time: "datetime | float | None", + end_time: "datetime | float | None", + ) -> None: + self._export(kwargs, start_time, end_time) + + +@lru_cache(maxsize=1) +def admin_destination_logger() -> AdminDestinationLogger: + return AdminDestinationLogger() + + +def register_admin_destination_logger() -> None: + """Put the destination sink on the proxy's async callback lists, once.""" + sink = admin_destination_logger() + for bucket in (litellm._async_success_callback, litellm._async_failure_callback): + if not any(callback is sink for callback in bucket): + bucket.append(sink) diff --git a/litellm/integrations/otel/emitter.py b/litellm/integrations/otel/emitter.py index 0850d867c7b..70c02b47cc9 100644 --- a/litellm/integrations/otel/emitter.py +++ b/litellm/integrations/otel/emitter.py @@ -214,6 +214,54 @@ class SpanEmitter: self.finish_span(role, span, data, end_time_ns=end_time_ns) return span + def emit_fanout( + self, + role: SpanRole, + data: SpanData, + parent_context: Context | None = None, + *, + start_time_ns: int | None = None, + end_time_ns: int | None = None, + tracers: Sequence[Tracer], + links: Sequence[Link] | None = None, + ) -> Span | None: + """Emit one logical span once per tracer, deduping the call ONCE. + + A backend that selects its target from the Resource (Arize's project) needs a + separately-tagged span per Resource group, so ``tracers`` is one tracer per group + (from ``TenantTracerCache.tracers_for``). Dedup runs once on the call id so a + sync+async double-firing still coalesces, then a span is started and finished on + each tracer (each provider stamps its own Resource). Returns the first span (or + ``None`` if deduped/empty). + """ + # A router retry reuses one ``litellm_call_id`` across every attempt, so the call id + # alone coalesced the successful attempt into the failed one it replaced and the + # retried call reached its destinations only as the failure. The response id splits + # the attempts apart while a sync+async double-firing of one attempt still shares it. + attempt_id = data.response_id if isinstance(data, LLMCallSpanData) else None + dedup_key = ( + (f"{data.identity.call_id}:{attempt_id}" if attempt_id else data.identity.call_id) + if isinstance(data, (LLMCallSpanData, MCPToolCallSpanData)) + else None + ) + if self._seen(dedup_key, role): + return None + name = _NAME_BUILDERS[role](data) + first: Span | None = None + for tracer in tracers: + span = self.start_span( + role, + name, + parent_context=parent_context, + start_time_ns=start_time_ns, + tracer=tracer, + links=links, + ) + self.finish_span(role, span, data, end_time_ns=end_time_ns) + if first is None: + first = span + return first + def finish_span( self, role: SpanRole, diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index 2c83406afed..2a98b82d7d0 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -62,8 +62,10 @@ from litellm.integrations.otel.plumbing.providers import ( from litellm.integrations.otel.plumbing.routing import TenantTracerCache if TYPE_CHECKING: + from litellm.integrations.otel.model.destination import OtelDestination from litellm.proxy._types import UserAPIKeyAuth from litellm.types.utils import ( + StandardCallbackDynamicParams, StandardLoggingGuardrailInformation, StandardLoggingPayload, ) @@ -116,18 +118,15 @@ _OPEN_CALLS_MAX: Final = 10_000 class _LLMCallSpan: """The state carried from the ``pre_call`` boundary to span close. - ``span`` is the live span when it could be opened at the boundary (the server - span was ambient), or ``None`` when creation was deferred because no ambient - parent was visible — in which case the async callback creates it against its - own (worker-copied) ambient context using ``start_time_ns``. The presence of - a carrier for a call at all is the proof that ``pre_call`` ran, i.e. that an - upstream call was actually attempted. + ``spans`` is one live span per destination Resource group, opened at the boundary; empty when + creation was deferred (no ambient parent visible), where the async callback creates them from + ``start_time_ns``. A carrier existing at all proves ``pre_call`` ran (an upstream call was attempted). """ - __slots__ = ("span", "start_time_ns") + __slots__ = ("spans", "start_time_ns") - def __init__(self, span: "Span | None", start_time_ns: int | None) -> None: - self.span = span + def __init__(self, spans: "tuple[Span, ...]", start_time_ns: int | None) -> None: + self.spans = spans self.start_time_ns = start_time_ns @@ -147,7 +146,13 @@ class OpenTelemetryV2(CustomLogger): self.config: OpenTelemetryV2Config = config or OpenTelemetryV2Config(**kwargs) self.callback_name = callback_name self._tracer_provider: TracerProvider = ( - tracer_provider if tracer_provider is not None else build_tracer_provider(self.config) + tracer_provider + if tracer_provider is not None + else build_tracer_provider( + self.config, + tenant_fan_out_owner=callback_name, + attach_tenant_fan_out=True, + ) ) self.tracer: Tracer = get_tracer(self._tracer_provider, LITELLM_TRACER_NAME) self._metrics_recorder = self._init_metrics(meter_provider) @@ -159,7 +164,8 @@ class OpenTelemetryV2(CustomLogger): event_recorder=self._init_events(logger_provider), ) self._tenant_tracers = TenantTracerCache(self.config, callback_name, LITELLM_TRACER_NAME) - self._open_llm_calls: OrderedDict[str, _LLMCallSpan] = OrderedDict() + self._open_llm_calls: OrderedDict[str, _LLMCallSpan] = OrderedDict() # mutable-ok: open-call span LRU + self._closed_call_ids: OrderedDict[str, None] = OrderedDict() # mutable-ok: bounded LRU of emitted call ids self._init_otel_logger_on_litellm_proxy() def _init_metrics(self, meter_provider: Any | None) -> "GenAIMetricRecorder | None": @@ -196,9 +202,21 @@ class OpenTelemetryV2(CustomLogger): # Proxy global registration # ====================================================================== # - def _register_in_callback_list(self, callbacks: list) -> None: - already_otel: Final = any( - cb.__class__.__module__.startswith(_OTEL_MODULES) for cb in callbacks if hasattr(cb, "__class__") + def _register_in_callback_list(self, callbacks: list, per_backend: bool = False) -> None: + """Add this logger to a proxy-global callback list, once. + + ``per_backend`` distinguishes the two kinds of list. ``service_callback`` wants a + single OTel owner, so any OTel-module callback already there wins. The event lists + are per-backend: v2 collapsed every backend onto this one class parameterised by + ``callback_name``, so a module-wide test let the first registrant lock out every + other backend, and each one's own exporter went dark. Matching the name as well + lets each backend register itself while still de-duplicating a repeat of itself. + """ + already_otel = any( + cb.__class__.__module__.startswith(_OTEL_MODULES) + and (not per_backend or getattr(cb, "callback_name", None) == self.callback_name) + for cb in callbacks + if hasattr(cb, "__class__") ) if not already_otel: callbacks.append(self) @@ -210,14 +228,28 @@ class OpenTelemetryV2(CustomLogger): return try: self._register_in_callback_list(litellm.service_callback) - self._register_in_callback_list(litellm.input_callback) - self._register_in_callback_list(litellm._async_success_callback) - self._register_in_callback_list(litellm._async_failure_callback) + self._register_in_callback_list(litellm.input_callback, per_backend=True) + self._register_in_callback_list(litellm._async_success_callback, per_backend=True) + self._register_in_callback_list(litellm._async_failure_callback, per_backend=True) except Exception: pass if getattr(proxy_server, "open_telemetry_logger", None) is None: setattr(proxy_server, "open_telemetry_logger", self) + def _destinations_for_backend(self, call: "LLMCallEvent") -> tuple: + """The call's admin-resolved destinations tagged with THIS logger's callback_name, + so each backend's span keeps its own attribute vocabulary.""" + return tuple(d for d in call.otel_destinations if d.callback_name == self.callback_name) + + def _tracer_dynamic_params(self, call: "LLMCallEvent") -> "StandardCallbackDynamicParams | None": + """The request's own backend credentials, used to pick a credential-scoped tracer. + + A subclass that exports only to admin destinations returns ``None``: the tenant's + own account is the owning logger's to reach, and layering a credential-scoped + tracer over the fan-out would export the call to it a second time. + """ + return call.dynamic_params + # ====================================================================== # # LLM-call callbacks — the span is opened at the ``pre_call`` boundary and # closed here. See ``log_pre_api_call``. @@ -252,22 +284,33 @@ class OpenTelemetryV2(CustomLogger): # call id; keep the first span so its start time is the true one. if call_id in self._open_llm_calls: return - start_time_ns: Final = to_ns(datetime.now()) - span: Span | None = None + # A router retry or fallback re-enters ``pre_call`` with the call id of an + # attempt that already closed, because the id is minted once per request. That + # is a genuinely new upstream attempt and gets its own span, so the marker left + # by the previous attempt is cleared here; leaving it made the close callback + # short-circuit and the successful attempt after a failure went untraced. + self._closed_call_ids.pop(call_id, None) + start_time_ns = to_ns(datetime.now()) + spans: tuple[Span, ...] = () # Parent to the request's anchored root span (stable across the request), # falling back to ambient on the SDK path. Open the span live only when # that resolves to a recordable parent; otherwise defer to the close # callback (the thread-pool case, where the anchor isn't visible here). parent_context: Final = resolve_request_span_context() if is_recordable_span(get_current_span(parent_context)): - span = self._emitter.start_span( - SpanRole.LLM_CALL, - call.provisional_span_name, - parent_context=parent_context, - start_time_ns=start_time_ns, - tracer=self._tenant_tracers.tracer_for(self.tracer, call.dynamic_params), + spans = tuple( + self._emitter.start_span( + SpanRole.LLM_CALL, + call.provisional_span_name, + parent_context=parent_context, + start_time_ns=start_time_ns, + tracer=tracer, + ) + for tracer in self._tenant_tracers.genai_tracers_for( + self.tracer, self._destinations_for_backend(call), call.dynamic_params + ) ) - self._open_llm_calls[call_id] = _LLMCallSpan(span=span, start_time_ns=start_time_ns) + self._open_llm_calls[call_id] = _LLMCallSpan(spans=spans, start_time_ns=start_time_ns) # Evict the oldest open call if the map is over budget. A call that opens # but never closes (a stream that only fires stream events) would linger # otherwise; the evicted span is simply dropped (never exported). @@ -369,12 +412,19 @@ class OpenTelemetryV2(CustomLogger): self._open_llm_calls.pop(data.identity.call_id, None) parent_context, links = resolve_mcp_span_context() parent_context = self._seed_identity_baggage(data.identity, None, parent_context) - self._emitter.emit( + # The tool-call span carries ``gen_ai.operation.name``, so the fan-out processor + # treats it as a gen-AI span and skips it; route it to the request's admin + # destinations like the LLM-call span, or it reaches only the global exporter. + call = LLMCallEvent.from_dict(kwargs) + self._emitter.emit_fanout( SpanRole.MCP_TOOL_CALL, data, parent_context=parent_context, start_time_ns=to_ns(start_time), end_time_ns=to_ns(end_time), + tracers=self._tenant_tracers.genai_tracers_for( + self.tracer, self._destinations_for_backend(call), self._tracer_dynamic_params(call) + ), links=links, ) return True @@ -423,49 +473,115 @@ class OpenTelemetryV2(CustomLogger): ) -> Span | None: """Finish the LLM-call span opened at ``pre_call`` (or create it deferred). - No carrier for this call id means ``pre_call`` never ran — the request was - rejected at the gate or blocked by a pre-call guardrail before any upstream - call — so there is nothing to record and no phantom span. + A missing carrier means either ``pre_call`` never ran (rejected at the gate or by a + pre-call guardrail, no payload, so dropping is correct) or this v2 instance was lazily + activated after ``pre_call``, where the payload IS set and names this backend, so a + deferred span is emitted with the success event's start time. + + Lazy activation has two causes, and both must emit. A resolved destination is one. + The other is a team carrying its own ``callback_vars`` credentials for this backend: + once v2 owns the backend the legacy logger is no longer built, so this instance is + the only thing left that can reach that team's account. Gating the deferred span on + destinations alone dropped those spans whenever the destination that made v2 take + the backend over belonged to a different team. """ - call: Final = LLMCallEvent.from_dict(kwargs) - call_id: Final = call.call_id - # ``pop`` is the dedup: this method runs from both the success and failure - # paths, and whichever fires first removes the carrier and closes the span. - carrier: Final = self._open_llm_calls.pop(call_id, None) if call_id else None + from litellm.integrations.otel.presets import dynamic_otlp_headers + + call = LLMCallEvent.from_dict(kwargs) + call_id = call.call_id + + carrier = self._open_llm_calls.pop(call_id, None) if call_id else None + payload = call.payload + + # The closed marker guards the carrier-less path only, where it stops a repeat + # success/failure callback re-emitting a span that already shipped. An open + # carrier is this attempt's own live span and must always be finished, or a + # retry's span would be left open and never exported. + # + # It is keyed on the payload's own id rather than the call id, because a router + # retry reuses one ``litellm_call_id`` for every attempt while each attempt gets + # its own payload id (the provider's response id, falling back to the call id). + # Keying on the call id made the successful attempt after a failure look like a + # duplicate, so a destination saw only the failure. + emit_key = (payload.get("id") if payload else None) or call_id + if carrier is None and emit_key and emit_key in self._closed_call_ids: + return None + if carrier is None: - return None - payload: Final = call.payload + destinations = self._destinations_for_backend(call) + own_credentials = bool(dynamic_otlp_headers(self.callback_name, call.dynamic_params)) + if call.is_no_upstream_call or payload is None or not (destinations or own_credentials): + return None + self._mark_closed(emit_key) + return self._emit_deferred_llm_call( + payload, + destinations, + to_ns(start_time), + to_ns(end_time), + call.time_to_first_chunk_seconds, + call.dynamic_params, + ) + + end_time_ns = to_ns(end_time) + self._mark_closed(emit_key) if payload is None: - if carrier.span is not None: - # Opened at the boundary but the payload never materialized — end - # it (named provisionally) so it isn't leaked as an open span. - carrier.span.end(end_time=to_ns(end_time)) + for span in carrier.spans: + span.end(end_time=end_time_ns) return None - data: Final = LLMCallSpanData.from_standard_logging_payload( + + data = LLMCallSpanData.from_standard_logging_payload( payload, capture_content=self.config.capture_span_content, time_to_first_chunk_seconds=call.time_to_first_chunk_seconds, ) - end_time_ns: Final = to_ns(end_time) - if carrier.span is not None: - # Born at the boundary: stamp attributes from the typed payload, set - # status, and end it. Its parent (the server span) was captured at - # creation from real ambient context. - self._emitter.finish_span(SpanRole.LLM_CALL, carrier.span, data, end_time_ns=end_time_ns) - return carrier.span - # Deferred: ``pre_call`` saw no recordable parent, so create the span now. - # The worker copied the request task's context, which carries the anchored - # root span — parent to it (ambient fallback on the SDK path). Seed identity - # Baggage so the span — and the SDK path, which has none — is labeled - # consistently. + if carrier.spans: + for span in carrier.spans: + self._emitter.finish_span(SpanRole.LLM_CALL, span, data, end_time_ns=end_time_ns) + return carrier.spans[0] + return self._emit_deferred_llm_call( + payload, + self._destinations_for_backend(call), + carrier.start_time_ns, + end_time_ns, + call.time_to_first_chunk_seconds, + call.dynamic_params, + ) + + def _mark_closed(self, call_id: str | None) -> None: + """Remember a call_id has been closed so a duplicate callback no-ops (bounded FIFO).""" + if not call_id: + return + self._closed_call_ids[call_id] = None + if len(self._closed_call_ids) > _OPEN_CALLS_MAX: + self._closed_call_ids.popitem(last=False) + + def _emit_deferred_llm_call( + self, + payload: "StandardLoggingPayload", + destinations: "tuple[OtelDestination, ...]", + start_time_ns: int | None, + end_time_ns: int | None, + time_to_first_chunk_seconds: float | None = None, + dynamic_params: "StandardCallbackDynamicParams | None" = None, + ) -> Span | None: + """Emit an LLM-call span outside the ``pre_call`` boundary. + + Two callers: the SDK thread-pool path and the destination-resolver path. Both anchor to + the request's root span via the worker-copied context and seed identity Baggage. + """ + data = LLMCallSpanData.from_standard_logging_payload( + payload, + capture_content=self.config.capture_span_content, + time_to_first_chunk_seconds=time_to_first_chunk_seconds, + ) parent_ctx = self._seed_identity_baggage(data.identity, data.request_model, resolve_request_span_context()) - return self._emitter.emit( + return self._emitter.emit_fanout( SpanRole.LLM_CALL, data, parent_context=parent_ctx, - start_time_ns=carrier.start_time_ns, + start_time_ns=start_time_ns, end_time_ns=end_time_ns, - tracer=self._tenant_tracers.tracer_for(self.tracer, call.dynamic_params), + tracers=self._tenant_tracers.genai_tracers_for(self.tracer, destinations, dynamic_params), ) # ====================================================================== # @@ -756,8 +872,15 @@ def publish_global_otel_v2_provider( unit-testable without reading or mutating real global OTel state. Returns the logger whose provider was published. """ - logger: Final = select_global_otel_v2_logger(in_memory_loggers, registered=registered) + # Local import: ``destination_logger`` imports this module for its emitter base, so a + # module-level import here would close the cycle. + from litellm.integrations.otel.destination_logger import ( + register_admin_destination_logger, + ) + + logger = select_global_otel_v2_logger(in_memory_loggers, registered=registered) set_global_provider(logger._tracer_provider) + register_admin_destination_logger() return logger diff --git a/litellm/integrations/otel/model/config.py b/litellm/integrations/otel/model/config.py index 53178b48991..0b51f8e97e8 100644 --- a/litellm/integrations/otel/model/config.py +++ b/litellm/integrations/otel/model/config.py @@ -4,9 +4,18 @@ from enum import Enum from functools import lru_cache from typing import Annotated, Any, Final -from pydantic import AliasChoices, BaseModel, Field, field_validator, model_validator +from pydantic import ( + AliasChoices, + BaseModel, + Field, + TypeAdapter, + ValidationError, + field_validator, + model_validator, +) from pydantic_settings import BaseSettings, NoDecode, SettingsConfigDict +from litellm._logging import verbose_logger from litellm.integrations.otel.model.baggage import ( BAGGAGE_PROMOTED_KEYS, DEFAULT_BAGGAGE_METADATA_KEYS, @@ -46,6 +55,35 @@ class _OTelV2Flag(BaseSettings): enabled: bool = Field(default=False, validation_alias=AliasChoices(OTEL_V2_ENV)) + @field_validator("enabled", mode="before") + @classmethod + def _unparseable_reads_as_off(cls, value: object) -> object: + """Never raise on this flag: an unusable value reads as off, loudly. + + The flag is read from ``instrument_fastapi_app`` while ``proxy_server`` is still + importing, so a parse error here takes the whole proxy down before it binds a + port. ``LITELLM_OTEL_V2=`` is routine in k8s ConfigMaps and ``.env`` files, and a + stray space or a word pydantic does not accept (``enabled``, ``2``) is an easy + typo; none of them is worth refusing to start over a feature that is off by + default. Surrounding whitespace is trimmed first so ``"true "`` still means true, + and anything left unrecognized warns and degrades rather than failing closed on + the whole process. + """ + if not isinstance(value, str): + return value + stripped = value.strip() + if not stripped: + return False + try: + return TypeAdapter(bool).validate_python(stripped) + except ValidationError: + verbose_logger.warning( + "%s=%r is not a recognized boolean; treating OpenTelemetry v2 as disabled. Use true or false.", + OTEL_V2_ENV, + value, + ) + return False + @lru_cache(maxsize=1) def is_otel_v2_enabled() -> bool: @@ -243,8 +281,16 @@ class OpenTelemetryV2Config(BaseSettings): if self.endpoint and self.exporter == "console": self.exporter = "otlp_http" # When no explicit destinations are given, fold the single-destination - # shorthand into one spec so the provider always has a destination. - if not self.exporters: + # shorthand into one spec so the provider has a destination. The exception is + # the "nothing configured" degrade case -- a preset returning a bare config + # because it found no credentials, where ``exporter`` is left at its default + # ``console`` and no endpoint is set: leave it exporter-less so the provider + # exports nothing, rather than degrading to a console exporter that prints every + # span (including prompt and completion content) to stdout synchronously on the + # request path. An explicitly chosen exporter (even ``console``), a non-console + # kind, or an endpoint all still fold. + console_by_default = self.exporter == "console" and "exporter" not in self.model_fields_set + if not self.exporters and (self.endpoint or not console_by_default): self.exporters = [ ExporterSpec( kind=self.exporter, diff --git a/litellm/integrations/otel/model/metadata.py b/litellm/integrations/otel/model/metadata.py index c7cdaae0417..d47e73898ad 100644 --- a/litellm/integrations/otel/model/metadata.py +++ b/litellm/integrations/otel/model/metadata.py @@ -41,11 +41,12 @@ from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, Final, cast from litellm.constants import LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL +from litellm.integrations.otel.model.destination import OtelDestination from litellm.integrations.otel.model.semconv import resolve_operation from litellm.integrations.otel.model.utils import as_str, to_seconds if TYPE_CHECKING: - from litellm.types.utils import StandardLoggingPayload + from litellm.types.utils import StandardCallbackDynamicParams, StandardLoggingPayload @dataclass(frozen=True) @@ -192,9 +193,10 @@ class LLMCallEvent: # at ``pre_call``, or when the call closed before any payload materialized (so # there is nothing to stamp on the span). payload: StandardLoggingPayload | None + otel_destinations: tuple[OtelDestination, ...] # The ``standard_callback_dynamic_params`` routing the call to a per-tenant # tracer (its own exporter/endpoint), or ``None`` when the call isn't scoped. - dynamic_params: Any + dynamic_params: StandardCallbackDynamicParams | None # True for synthetic proxy-gate logs (auth / rate-limit rejections): they fire # the ``pre_call`` hook but never made an upstream call, so they get no span. is_no_upstream_call: bool @@ -206,6 +208,12 @@ class LLMCallEvent: @classmethod def from_dict(cls, kwargs: Mapping[str, Any]) -> LLMCallEvent: + # Imported here rather than at module scope: this module is reachable from + # ``litellm/__init__`` and ``plumbing.context`` imports opentelemetry eagerly, so a + # top-level import makes the whole package a hard dependency of the proxy, which + # installs without the optional tracing extras. + from litellm.integrations.otel.plumbing.context import request_destinations + raw_payload: Final = kwargs.get("standard_logging_object") payload: Final = cast("StandardLoggingPayload", raw_payload) if raw_payload else None operation: Final = resolve_operation(as_str(kwargs.get("call_type"))) @@ -213,6 +221,7 @@ class LLMCallEvent: return cls( call_id=_call_id(payload, kwargs), payload=payload, + otel_destinations=request_destinations(), dynamic_params=kwargs.get("standard_callback_dynamic_params"), is_no_upstream_call=bool(kwargs.get(LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL)), provisional_span_name=f"{operation.value} {model}".strip(), diff --git a/litellm/integrations/otel/mount.py b/litellm/integrations/otel/mount.py index ac647c2c4f6..edd228cf83e 100644 --- a/litellm/integrations/otel/mount.py +++ b/litellm/integrations/otel/mount.py @@ -101,17 +101,21 @@ def instrument_fastapi_app(app: Any) -> None: after config load (see ``proxy_startup_event``), and the proxy delegates to it. That way server spans and gen-ai spans share one provider and the same trace. """ + if not is_otel_v2_enabled(): + return + try: - if not is_otel_v2_enabled(): - return - - # Lazy: only the V2-enabled path needs the optional - # ``opentelemetry-instrumentation-fastapi`` package, which is not part of the - # base ``litellm[proxy]`` install. Importing it at module top would make - # ``proxy_server``'s unconditional ``import`` of this module crash when the - # package is absent, even with the gate off. from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor + except ImportError: + verbose_logger.warning( + "LITELLM_OTEL_V2 is enabled but 'opentelemetry-instrumentation-fastapi' " + "is not installed. The FastAPI server span will not be created, so traces " + "exported to admin-owned destinations will be missing their root span " + "(orphaned children). Install 'opentelemetry-instrumentation-fastapi'." + ) + return + try: excluded_urls: Final = ( os.environ.get("OTEL_PYTHON_FASTAPI_EXCLUDED_URLS") if "OTEL_PYTHON_FASTAPI_EXCLUDED_URLS" in os.environ @@ -127,4 +131,4 @@ def instrument_fastapi_app(app: Any) -> None: exclude_spans=["receive", "send"], ) except Exception as e: - verbose_logger.debug("Skipping OTel V2 FastAPI instrumentation: %s", e) + verbose_logger.warning("OTel V2 FastAPI instrumentation failed: %s", e) diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index 80a8d01c061..48ddcf9518f 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -1,6 +1,6 @@ """Provider / exporter factory + the Baggage span processor.""" -from collections.abc import Callable, Iterable +from collections.abc import Callable, Iterable, Mapping from typing import TYPE_CHECKING, Any, Final from opentelemetry import _logs, baggage, metrics @@ -41,6 +41,8 @@ if TYPE_CHECKING: from opentelemetry.metrics import Meter from opentelemetry.sdk.metrics.export import MetricReader + from litellm.integrations.otel.model.destination import OtelDestination + _SPAN_KIND_BY_ROLE_KIND: Final[dict[LiteLLMSpanKind, SpanKind]] = { LiteLLMSpanKind.SERVER: SpanKind.SERVER, LiteLLMSpanKind.CLIENT: SpanKind.CLIENT, @@ -110,7 +112,7 @@ def _otlp_traces_endpoint(endpoint: str | None) -> str | None: return endpoint endpoint = endpoint.rstrip("/") # Splunk Observability uses ``/v2/trace/otlp``; never rewrite it. - if endpoint.endswith("/v1/traces") or "/v2/trace/otlp" in endpoint: + if endpoint.endswith("/v1/traces") or "/v2/trace/otlp" in endpoint or endpoint.endswith("/api/trace"): return endpoint for other_signal in ("/v1/logs", "/v1/metrics"): if endpoint.endswith(other_signal): @@ -118,6 +120,22 @@ def _otlp_traces_endpoint(endpoint: str | None) -> str | None: return endpoint + "/v1/traces" +_GRPC_BACKENDS = frozenset({"arize"}) + + +def default_otlp_kind_for_backend(callback_name: "str | None") -> str: + """The intrinsic OTLP transport for a backend's own OTLP endpoint.""" + return "otlp_grpc" if callback_name in _GRPC_BACKENDS else "otlp_http" + + +def destination_resource_attrs(destination: "OtelDestination") -> Mapping[str, str]: + """The destination's builder-declared Resource attributes (e.g. Arize's + ``model_id`` / ``arize.project.name``; empty for header-routed backends), read + by both export paths so the gen-AI span and its parents share one Resource. + """ + return dict(destination.resource_attributes) + + def parse_headers(raw: str | None) -> dict[str, str]: """Parse an OTLP ``"k=v,k=v"`` header string into a dict. @@ -415,20 +433,26 @@ def build_tracer_provider( exporter: SpanExporter | None = None, baggage_processor: SpanProcessor | None = None, use_simple_processor: bool | None = None, + tenant_fan_out_owner: str | None = None, + attach_tenant_fan_out: bool = False, ) -> TracerProvider: - """Build the shared :class:`TracerProvider`. - - Attach the Baggage processor first (so identity attributes land on each - span before any export decision), then add one ``SpanProcessor`` per - ``config.exporters`` entry — this is what fans spans out to multiple - backends. ``exporter`` and ``use_simple_processor`` are explicit overrides: - pass a single exporter to attach exactly that one (used by tests). + """Build the shared :class:`TracerProvider`: Baggage processor first, then one + ``SpanProcessor`` per ``config.exporters`` entry (``exporter`` overrides for tests). + ``attach_tenant_fan_out``/``tenant_fan_out_owner`` add a ``TenantFanOutSpanProcessor`` + forwarding proxy-internal spans to the request's destinations. """ provider: Final = TracerProvider(resource=build_resource(config)) if baggage_processor is None: baggage_processor = LiteLLMBaggageSpanProcessor(allowed_keys=config.baggage_promoted_keys) provider.add_span_processor(baggage_processor) + if attach_tenant_fan_out or tenant_fan_out_owner is not None: + from litellm.integrations.otel.plumbing.routing import ( + TenantFanOutSpanProcessor, + ) + + provider.add_span_processor(TenantFanOutSpanProcessor(owner_callback_name=tenant_fan_out_owner)) + if exporter is not None: provider.add_span_processor(_processor_for(exporter, use_simple_processor)) return provider diff --git a/litellm/integrations/otel/plumbing/routing.py b/litellm/integrations/otel/plumbing/routing.py index d9e9e84364a..2101e532de4 100644 --- a/litellm/integrations/otel/plumbing/routing.py +++ b/litellm/integrations/otel/plumbing/routing.py @@ -1,28 +1,31 @@ -"""Per-request multi-tenant tracer routing. +"""Per-request multi-tenant tracer routing and span fan-out. -When a request carries team/key vendor credentials in -``standard_callback_dynamic_params``, its spans must export through a -``TracerProvider`` whose OTLP headers carry those credentials. -``TenantTracerCache`` builds and caches one provider per distinct credential -set, and otherwise hands back the logger's default tracer. This lets a single -logger fan requests out to many tenants without needing a logger per tenant. +``TenantTracerCache`` routes the gen-AI span to per-tenant/destination providers; +``TenantFanOutSpanProcessor`` forwards proxy-internal spans to every admin-resolved destination. """ +import threading from collections import OrderedDict -from collections.abc import Mapping -from typing import Any, Final +from typing import TYPE_CHECKING, Final -from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.context import Context +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import ReadableSpan, Span, SpanProcessor, TracerProvider from opentelemetry.trace import Tracer from litellm._logging import verbose_logger -from litellm.integrations.otel.model.config import OpenTelemetryV2Config +from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config +from litellm.integrations.otel.model.destination import OtelDestination +from litellm.integrations.otel.plumbing.context import request_destinations from litellm.integrations.otel.plumbing.providers import ( build_tracer_provider, get_tracer, ) from litellm.integrations.otel.presets import dynamic_otlp_headers +if TYPE_CHECKING: + from litellm.types.utils import StandardCallbackDynamicParams + # Exporter kinds that ignore headers — never rewritten with dynamic credentials. _NON_OTLP_KINDS: Final = ("console", "in_memory", "inmemory", "memory") @@ -35,22 +38,24 @@ _NON_OTLP_KINDS: Final = ("console", "in_memory", "inmemory", "memory") _MAX_CACHED_PROVIDERS: Final = 256 -def _shutdown_provider(provider: TracerProvider) -> None: - """Flush + stop an evicted provider's processors (reclaims their threads). +def _shutdown_in_background(evicted: "TracerProvider | SpanProcessor") -> None: + """Reclaim an evicted provider/processor's ``BatchSpanProcessor`` worker thread. - ``TracerProvider.shutdown`` force-flushes each ``SpanProcessor`` before - stopping it, so any spans already handed to a ``BatchSpanProcessor`` are - exported rather than dropped. Best-effort: a shutdown failure must not break - the request that triggered the eviction. + Dropping it without ``shutdown`` leaks the daemon thread; ``shutdown`` force-flushes and can + do network I/O, so it runs fire-and-forget on a daemon thread rather than the request path. """ - try: - provider.shutdown() - except Exception as e: # pragma: no cover - defensive - verbose_logger.debug("OTel V2: error shutting down evicted provider: %s", e) + + def _run() -> None: + try: + evicted.shutdown() + except Exception as exc: # noqa: BLE001 # a failed shutdown must not surface on the hot path + verbose_logger.debug("OTel V2: background shutdown of evicted %s failed: %s", type(evicted).__name__, exc) + + threading.Thread(target=_run, name="litellm-otel-evict-shutdown", daemon=True).start() class TenantTracerCache: - """Credential-scoped ``TracerProvider`` cache keyed by the dynamic headers.""" + """Destination-scoped ``TracerProvider`` cache keyed by endpoint + headers.""" def __init__( self, @@ -61,32 +66,96 @@ class TenantTracerCache: self._config = config self._callback_name = callback_name self._tracer_name = tracer_name - self._providers: OrderedDict[tuple[tuple[str, str], ...], TracerProvider] = OrderedDict() + self._providers: OrderedDict[tuple[object, ...], TracerProvider] = ( + OrderedDict() + ) # mutable-ok: bounded LRU tracer-provider cache - def tracer_for(self, default: Tracer, dynamic_params: Any) -> Tracer: - """Return the tracer for this request. + def _evict_if_full(self) -> None: + """Drop the least-recently-used provider when over capacity, shutting it down off + the hot path so its ``BatchSpanProcessor`` worker thread is reclaimed rather than + leaked for the life of the process.""" + if len(self._providers) > _MAX_CACHED_PROVIDERS: + _, evicted = self._providers.popitem(last=False) + _shutdown_in_background(evicted) - Use ``default`` unless the request's dynamic credentials require a - credential-scoped tracer, in which case build (or reuse) one. The cache - is a bounded LRU: the least-recently-used provider is flushed and shut - down on overflow so its exporter threads don't accumulate. + def tracers_for( + self, + default: Tracer, + destinations: "tuple[OtelDestination, ...]", + *, + include_base_on_first: bool = True, + ) -> "tuple[Tracer, ...]": + """The tracers for this request's gen-AI span, one per distinct Resource group. + + Destinations are grouped by ``destination_resource_attrs`` (a backend like Arize selects its + project from the Resource). The configured/global exporters ride their own clean-Resource + provider (``default``), never folded into a destination group, so a Resource-routed + destination cannot stamp its own attributes (e.g. Arize's project) onto the global export. + ``include_base_on_first=False`` drops ``default`` when a credential-scoped tracer already + carries the configured exporters. + """ + if not destinations: + return (default,) + groups = tuple( + self._tracer_for_group(resource_key, group, include_base=False) + for resource_key, group in self._group_by_resource(destinations) + ) + return (default, *groups) if include_base_on_first else groups + + def genai_tracers_for( + self, + default: Tracer, + destinations: "tuple[OtelDestination, ...]", + dynamic_params: "StandardCallbackDynamicParams | None", + ) -> "tuple[Tracer, ...]": + """The gen-AI span's tracers, layering per-request credential routing over the destination + fan-out: with this backend's team/key OTLP credentials the global export rides a + credential-scoped provider and the destination groups omit the base exporters; else plain fan-out. """ headers: Final = dynamic_otlp_headers(self._callback_name, dynamic_params) if not headers: - return default - cache_key: Final = tuple(sorted(headers.items())) + return self.tracers_for(default, destinations) + dynamic = self._credential_scoped_tracer(headers, dynamic_params) + if not destinations: + return (dynamic,) + return (dynamic, *self.tracers_for(default, destinations, include_base_on_first=False)) + + def _credential_scoped_tracer( + self, + headers: "dict[str, str]", + dynamic_params: "StandardCallbackDynamicParams | None" = None, + ) -> Tracer: + """A cached provider that keeps the configured exporters and rewrites only this + backend's owned exporter's headers to ``headers`` (the per-request credentials). + + The endpoint and transport are part of the key: when the preset contributed no + exporter the synthesized one resolves both from the request's own credentials, so + two tenants sharing vendor credentials on different hosts would otherwise collide + and the second tenant's traces would ship to the first tenant's collector. + """ + from litellm.integrations.otel.presets import dynamic_otlp_destination + + destination = dynamic_otlp_destination(self._callback_name, dynamic_params) + cache_key: tuple[object, ...] = ( + "dynamic", + tuple(sorted(headers.items())), + destination.endpoint if destination is not None else None, + destination.protocol if destination is not None else None, + ) provider = self._providers.get(cache_key) if provider is not None: self._providers.move_to_end(cache_key) else: - provider = build_tracer_provider(self._config_with_headers(headers)) + provider = build_tracer_provider(self._config_with_headers(headers, dynamic_params)) self._providers[cache_key] = provider - if len(self._providers) > _MAX_CACHED_PROVIDERS: - _, evicted = self._providers.popitem(last=False) - _shutdown_provider(evicted) + self._evict_if_full() return get_tracer(provider, self._tracer_name) - def _config_with_headers(self, headers: Mapping[str, str]) -> OpenTelemetryV2Config: + def _config_with_headers( + self, + headers: "dict[str, str]", + dynamic_params: "StandardCallbackDynamicParams | None" = None, + ) -> OpenTelemetryV2Config: """Clone the config, stamping ``headers`` onto the credential's own exporter. ``headers`` are the per-request credentials of ``self._callback_name`` (the @@ -95,15 +164,295 @@ class TenantTracerCache: tenant's Arize key must never rewrite the headers of a co-configured Langfuse or self-hosted collector exporter, which would leak that key to a different backend. + + A preset built with ``allow_missing_credentials`` contributes no exporter of its + own, so there is nothing to stamp and the team's traces would go nowhere. One is + synthesized from the request's own credentials in that case, resolved through the + same builder an equivalent admin destination would use so the team reaches the + account its ``callback_vars`` name. """ - header_str: Final = ",".join(f"{key}={value}" for key, value in headers.items()) - header_update: Final[dict[str, str]] = {"headers": header_str} - exporters: Final = [ + header_str = ",".join(f"{key}={value}" for key, value in headers.items()) + owns_exporter = any( + spec.owner == self._callback_name and spec.kind.lower() not in _NON_OTLP_KINDS + for spec in self._config.exporters + ) + if not owns_exporter: + return self._config.model_copy( + update={"exporters": [*self._config.exporters, *self._synthesized_exporter(header_str, dynamic_params)]} + ) + exporters = [ ( - spec.model_copy(update=header_update) + spec.model_copy(update={"headers": header_str}) if spec.owner == self._callback_name and spec.kind.lower() not in _NON_OTLP_KINDS else spec ) for spec in self._config.exporters ] return self._config.model_copy(update={"exporters": exporters}) + + def _synthesized_exporter( + self, + header_str: str, + dynamic_params: "StandardCallbackDynamicParams | None", + ) -> "tuple[ExporterSpec, ...]": + """This backend's exporter built from the request's own credentials, or empty + when they don't resolve to an endpoint. + + The builder's ``protocol`` wins over the backend's intrinsic transport, matching + ``_config_with_destinations``: values that pin an HTTP collector for a gRPC-default + backend would otherwise be exported over gRPC and dropped. + """ + from litellm.integrations.otel.presets import dynamic_otlp_destination + + destination = dynamic_otlp_destination(self._callback_name, dynamic_params) + if destination is None or not destination.endpoint: + return () + return ( + ExporterSpec( + kind=destination.protocol or self._owned_otlp_kind(), + endpoint=destination.endpoint, + headers=header_str, + owner=self._callback_name, + ), + ) + + def _group_by_resource( + self, destinations: "tuple[OtelDestination, ...]" + ) -> "tuple[tuple[tuple[tuple[str, str], ...], tuple[OtelDestination, ...]], ...]": + """Destinations grouped by their backend-required Resource attributes. + + Groups sort deterministically by key, so the empty-Resource group (header-routed + backends) sorts first and the configured/global exporters attach to it. + """ + from litellm.integrations.otel.plumbing.providers import ( + destination_resource_attrs, + ) + + groups: OrderedDict[tuple[tuple[str, str], ...], list[OtelDestination]] = ( + OrderedDict() + ) # mutable-ok: insertion-order grouping accumulator, frozen before return + for destination in destinations: + key = tuple(sorted(destination_resource_attrs(destination).items())) + groups.setdefault(key, []).append(destination) + return tuple((key, tuple(group)) for key, group in sorted(groups.items())) + + def _tracer_for_group( + self, + resource_key: "tuple[tuple[str, str], ...]", + group: "tuple[OtelDestination, ...]", + *, + include_base: bool, + ) -> Tracer: + cache_key: tuple[object, ...] = ( + resource_key, + tuple(sorted((d.endpoint, tuple(sorted(d.headers.items())), d.protocol or "") for d in group)), + include_base, + ) + provider = self._providers.get(cache_key) + if provider is not None: + self._providers.move_to_end(cache_key) + else: + provider = build_tracer_provider( + self._config_with_destinations(tuple(group), include_base_exporters=include_base) + ) + self._providers[cache_key] = provider + self._evict_if_full() + return get_tracer(provider, self._tracer_name) + + def _owned_otlp_kind(self) -> str: + """The OTLP transport of this integration's own exporter (langfuse -> http, arize -> grpc). + + Prefers the admin's configured exporter kind; falls back to the backend's intrinsic + default so a lazily-activated backend with no owned spec still picks the right transport. + """ + from litellm.integrations.otel.plumbing.providers import ( + default_otlp_kind_for_backend, + ) + + for spec in self._config.exporters: + if spec.owner == self._callback_name and spec.kind.lower() not in _NON_OTLP_KINDS: + return spec.kind + return default_otlp_kind_for_backend(self._callback_name) + + def _config_with_destinations( + self, + destinations: "tuple[OtelDestination, ...]", + *, + include_base_exporters: bool = True, + ) -> OpenTelemetryV2Config: + """Clone the config, appending one exporter per destination so one span exports to every one. + + ``include_base_exporters`` keeps the configured/global exporters (``tracers_for`` sets it only on + the first group); the clone's Resource folds in the destinations' ``destination_resource_attrs``. + """ + from litellm.integrations.otel.plumbing.providers import ( + destination_resource_attrs, + ) + + kind = self._owned_otlp_kind() + appended = tuple( + ExporterSpec( + kind=d.protocol or kind, + endpoint=d.endpoint, + headers=d.header_string(), + owner=None, + ) + for d in destinations + ) + base_exporters = (*self._config.exporters,) if include_base_exporters else () + merged_resource_attrs = { + **self._config.resource_attributes, + **{key: value for d in destinations for key, value in destination_resource_attrs(d).items()}, + } + return self._config.model_copy( + update={ + "exporters": [*base_exporters, *appended], + "resource_attributes": merged_resource_attrs, + } + ) + + +_MAX_CACHED_PROCESSORS = 256 + +_GENAI_SPAN_ATTR = "gen_ai.operation.name" + + +def _processor_key(destination: OtelDestination) -> "tuple[str, tuple[tuple[str, str], ...], str | None]": + return (destination.endpoint, tuple(sorted(destination.headers.items())), destination.protocol) + + +def _is_genai_span(span: ReadableSpan) -> bool: + attributes = span.attributes or {} + return _GENAI_SPAN_ATTR in attributes + + +def _with_destination_resource(span: ReadableSpan, destination: OtelDestination) -> ReadableSpan: + """Return ``span`` with its Resource augmented by the destination's required attributes, + via a shallow wrapper that leaves the original span untouched.""" + from litellm.integrations.otel.plumbing.providers import ( + destination_resource_attrs, + ) + + extra = destination_resource_attrs(destination) + if not extra: + return span + merged = Resource.create({**dict(span.resource.attributes), **extra}) + return _ResourceWrappedReadableSpan(span, merged) + + +class _ResourceWrappedReadableSpan(ReadableSpan): + """A ``ReadableSpan`` view whose ``resource`` is overridden (for backend-specific attributes + like Arize's ``model_id``) without mutating the underlying span.""" + + def __init__(self, inner: ReadableSpan, resource: Resource) -> None: + super().__init__( + name=inner.name, + context=inner.context, + parent=inner.parent, + resource=resource, + attributes=inner.attributes, + events=inner.events, + links=inner.links, + kind=inner.kind, + status=inner.status, + start_time=inner.start_time, + end_time=inner.end_time, + instrumentation_scope=inner.instrumentation_scope, + ) + + +class TenantFanOutSpanProcessor(SpanProcessor): + """Forward each finished proxy-internal span to every admin-resolved destination. + + Destinations come from a request-scoped contextvar set during auth, so the processor is + stateless across requests and concurrent requests are isolated by contextvars. + """ + + def __init__(self, owner_callback_name: str | None) -> None: + self._owner = owner_callback_name + self._processors: OrderedDict[tuple, SpanProcessor] = ( + OrderedDict() + ) # mutable-ok: bounded LRU span-processor cache + + def on_start(self, span: Span, parent_context: Context | None = None) -> None: + return None + + def on_end(self, span: ReadableSpan) -> None: + destinations = request_destinations() + if not destinations: + return + if _is_genai_span(span): + return + for destination in destinations: + processor = self._processor_for(destination) + if processor is None: + continue + try: + processor.on_end(_with_destination_resource(span, destination)) + except Exception as exc: # noqa: BLE001 # best-effort fan-out; one destination's failure must not break the others or the request + verbose_logger.debug( + "OTel V2 fan-out: forwarding span to %s failed: %s", + destination.endpoint, + exc, + ) + + def shutdown(self) -> None: + # Snapshot before iterating: ``on_end`` mutates ``self._processors`` (insert / + # move_to_end / popitem) on the span-ending thread and can run concurrently with + # this SDK-driven shutdown, so iterating the live mapping risks a + # "mutated during iteration" RuntimeError that the per-item except can't catch. + for processor in tuple(self._processors.values()): + try: + processor.shutdown() + except Exception as exc: # noqa: BLE001 # a single processor's shutdown failure must not abort shutting down the rest + verbose_logger.debug("OTel V2 fan-out: processor shutdown failed: %s", exc) + self._processors.clear() + + def force_flush(self, timeout_millis: int = 30000) -> bool: + all_ok = True + # Snapshot before iterating (see ``shutdown``): a concurrent ``on_end`` mutating + # the processor cache must not abort the flush and drop the remaining destinations' + # buffered spans. + for processor in tuple(self._processors.values()): + try: + if not processor.force_flush(timeout_millis): + all_ok = False + except Exception: # noqa: BLE001 # a single processor's flush failure must not fail the whole force_flush + all_ok = False + return all_ok + + def _processor_for(self, destination: OtelDestination) -> SpanProcessor | None: + key = _processor_key(destination) + cached = self._processors.get(key) + if cached is not None: + self._processors.move_to_end(key) + return cached + from litellm.integrations.otel.plumbing.providers import ( + _exporter_from_spec, + default_otlp_kind_for_backend, + ) + from litellm.integrations.otel.plumbing.providers import ( + _processor_for as _build_processor, + ) + + try: + spec = ExporterSpec( + kind=destination.protocol or default_otlp_kind_for_backend(destination.callback_name), + endpoint=destination.endpoint, + headers=destination.header_string(), + owner=None, + ) + exporter = _exporter_from_spec(spec) + processor = _build_processor(exporter, use_simple=False) + except Exception as exc: # noqa: BLE001 # a malformed destination spec must not break fan-out; skip this destination + verbose_logger.debug( + "OTel V2 fan-out: failed to build processor for %s: %s", + destination.endpoint, + exc, + ) + return None + self._processors[key] = processor + if len(self._processors) > _MAX_CACHED_PROCESSORS: + _, evicted = self._processors.popitem(last=False) + _shutdown_in_background(evicted) + return processor diff --git a/litellm/integrations/otel/presets/__init__.py b/litellm/integrations/otel/presets/__init__.py index 95ac2783325..0296de458cc 100644 --- a/litellm/integrations/otel/presets/__init__.py +++ b/litellm/integrations/otel/presets/__init__.py @@ -6,14 +6,22 @@ vocabularies to apply, and any resource attributes. ``PRESET_BY_CALLBACK`` maps a callback name (``"arize"``, ``"langfuse_otel"``, ...) to its preset so the factory in ``litellm_logging`` can resolve a name and build a single ``OpenTelemetryV2`` instance from the result. + +Admin-owned trace destinations are resolved server-side from a named credential +into an ``OtelDestination`` (see ``litellm.integrations.otel.presets.destinations`` +and ``plumbing.routing``). Separately, per-request team/key OTLP credentials from +``standard_callback_dynamic_params`` route the gen-AI span to a credential-scoped +tracer for the integrations that support it; ``dynamic_otlp_headers`` below builds +those per-request headers. """ from collections.abc import Callable -from typing import Final +from typing import Final, TYPE_CHECKING from litellm.integrations.otel.presets.agentops import agentops_preset from litellm.integrations.otel.presets.arize import arize_dynamic_headers, arize_preset from litellm.integrations.otel.presets.base import Preset +from litellm.integrations.otel.presets.generic import generic_preset from litellm.integrations.otel.presets.langfuse import ( langfuse_dynamic_headers, langfuse_preset, @@ -24,23 +32,14 @@ from litellm.integrations.otel.presets.phoenix import phoenix_preset from litellm.integrations.otel.presets.weave import weave_dynamic_headers, weave_preset from litellm.types.utils import StandardCallbackDynamicParams -#: Callback name → preset. The ``Preset`` annotation makes mypy verify every -#: registered value matches the preset interface. -PRESET_BY_CALLBACK: Final[dict[str, Preset]] = { - "agentops": agentops_preset, - "arize": arize_preset, - "arize_phoenix": phoenix_preset, - "langfuse_otel": langfuse_preset, - "langtrace": langtrace_preset, - "levo": levo_preset, - "weave_otel": weave_preset, -} +if TYPE_CHECKING: + from litellm.integrations.otel.model.destination import OtelDestination #: Callback name → per-request OTLP header builder (team/key multi-tenant #: routing). Only integrations that support dynamic credentials appear here — -#: Arize-Phoenix/Langtrace/Levo/AgentOps don't, so they use the logger's +#: Arize-Phoenix/Langtrace/Levo/AgentOps/generic don't, so they use the logger's #: default tracer. -DYNAMIC_HEADERS_BY_CALLBACK: Final[dict[str, Callable[[StandardCallbackDynamicParams], dict[str, str]]]] = { +DYNAMIC_HEADERS_BY_CALLBACK: dict[str, Callable[[StandardCallbackDynamicParams], dict[str, str]]] = { "arize": arize_dynamic_headers, "langfuse_otel": langfuse_dynamic_headers, "weave_otel": weave_dynamic_headers, @@ -49,7 +48,7 @@ DYNAMIC_HEADERS_BY_CALLBACK: Final[dict[str, Callable[[StandardCallbackDynamicPa def dynamic_otlp_headers( callback_name: str | None, - dynamic_params: StandardCallbackDynamicParams | None, + dynamic_params: "StandardCallbackDynamicParams | None", ) -> dict[str, str] | None: """Per-request OTLP headers for ``callback_name``, or ``None`` if N/A. @@ -62,13 +61,48 @@ def dynamic_otlp_headers( return headers or None +def dynamic_otlp_destination( + callback_name: str | None, + dynamic_params: "StandardCallbackDynamicParams | None", +) -> "OtelDestination | None": + """The destination a request's own team/key credentials export to, or ``None``. + + Resolved through the admin-destination builders so a team's ``callback_vars`` reach + exactly the account an equivalent destination would, honouring per-tenant overrides + such as ``langfuse_host``. The builder's ``protocol`` comes with it: a transport the + values pin is the tenant's, not the backend's intrinsic default. + """ + from litellm.integrations.otel.presets.destinations import build_destination + + if callback_name not in DYNAMIC_HEADERS_BY_CALLBACK or not dynamic_params: + return None + values = {str(key): str(value) for key, value in dynamic_params.items() if isinstance(value, str)} + return build_destination(callback_name or "", values) + + +#: Callback name → preset. The ``Preset`` annotation makes mypy verify every +#: registered value matches the preset interface. +PRESET_BY_CALLBACK: dict[str, Preset] = { + "agentops": agentops_preset, + "arize": arize_preset, + "arize_phoenix": phoenix_preset, + "generic": generic_preset, + "langfuse_otel": langfuse_preset, + "langtrace": langtrace_preset, + "levo": levo_preset, + "weave_otel": weave_preset, +} + + __all__ = [ "DYNAMIC_HEADERS_BY_CALLBACK", "PRESET_BY_CALLBACK", "Preset", "agentops_preset", "arize_preset", + "dynamic_otlp_destination", "dynamic_otlp_headers", + "generic_preset", "langfuse_preset", "langtrace_preset", "levo_preset", diff --git a/litellm/integrations/otel/presets/agentops.py b/litellm/integrations/otel/presets/agentops.py index f45b1cd3cff..38f80aab36c 100644 --- a/litellm/integrations/otel/presets/agentops.py +++ b/litellm/integrations/otel/presets/agentops.py @@ -39,6 +39,7 @@ class _AgentOpsSettings(BaseSettings): def agentops_preset( *, config_overrides: OpenTelemetryV2Config | None = None, + allow_missing_credentials: bool = False, ) -> OpenTelemetryV2Config: """Build the AgentOps config without any network I/O. @@ -49,17 +50,21 @@ def agentops_preset( """ settings: Final = _AgentOpsSettings() base: Final = config_overrides or OpenTelemetryV2Config() + global_exporter = ( + () + if allow_missing_credentials and not settings.api_key + else ( + ExporterSpec( + kind=_AGENTOPS_EXPORTER_KIND, + endpoint=_AGENTOPS_ENDPOINT, + options=({"api_key": settings.api_key} if settings.api_key else None), + owner=ExporterOwner.AGENTOPS, + ), + ) + ) return base.model_copy( update={ - "exporters": [ - *base.exporters, - ExporterSpec( - kind=_AGENTOPS_EXPORTER_KIND, - endpoint=_AGENTOPS_ENDPOINT, - options=({"api_key": settings.api_key} if settings.api_key else None), - owner=ExporterOwner.AGENTOPS, - ), - ], + "exporters": [*base.exporters, *global_exporter], "resource_attributes": { **base.resource_attributes, "service.name": settings.service_name, diff --git a/litellm/integrations/otel/presets/arize.py b/litellm/integrations/otel/presets/arize.py index ee0de675657..fb6a49cb101 100644 --- a/litellm/integrations/otel/presets/arize.py +++ b/litellm/integrations/otel/presets/arize.py @@ -14,6 +14,8 @@ from litellm.integrations.otel.model.config import ( from litellm.integrations.otel.presets.utils import ensure_mappers from litellm.types.utils import StandardCallbackDynamicParams +ARIZE_PUBLIC_OTLP_ENDPOINT = "https://otlp.arize.com/v1" + class _ArizeSettings(BaseSettings): model_config = SettingsConfigDict(case_sensitive=False, extra="ignore") @@ -21,26 +23,35 @@ class _ArizeSettings(BaseSettings): # Standard OTLP headers env var, used as the fallback when no Arize # credentials are configured. otlp_traces_headers: str | None = Field(default=None, validation_alias="OTEL_EXPORTER_OTLP_TRACES_HEADERS") + grpc_endpoint: str | None = Field(default=None, validation_alias="ARIZE_ENDPOINT") + http_endpoint: str | None = Field(default=None, validation_alias="ARIZE_HTTP_ENDPOINT") def arize_preset( *, config_overrides: OpenTelemetryV2Config | None = None, + allow_missing_credentials: bool = False, ) -> OpenTelemetryV2Config: arize_cfg: Final = _V1ArizeLogger.get_arize_config() - headers: Final = _arize_headers(arize_cfg) + settings = _ArizeSettings() + has_own_endpoint = bool(settings.grpc_endpoint or settings.http_endpoint) + has_credentials = bool(arize_cfg.space_id or arize_cfg.space_key or arize_cfg.api_key) base: Final = config_overrides or OpenTelemetryV2Config() + global_exporter = ( + () + if allow_missing_credentials and not has_credentials and not has_own_endpoint + else ( + ExporterSpec( + kind=arize_cfg.protocol or "otlp_grpc", + endpoint=arize_cfg.endpoint or ARIZE_PUBLIC_OTLP_ENDPOINT, + headers=_arize_headers(arize_cfg, settings, has_own_endpoint), + owner=ExporterOwner.ARIZE_AX, + ), + ) + ) return base.model_copy( update={ - "exporters": [ - *base.exporters, - ExporterSpec( - kind=arize_cfg.protocol or "otlp_grpc", - endpoint=arize_cfg.endpoint or "https://otlp.arize.com/v1", - headers=headers, - owner=ExporterOwner.ARIZE_AX, - ), - ], + "exporters": [*base.exporters, *global_exporter], "mapper_names": ensure_mappers(base.mapper_names, "openinference"), "resource_attributes": { **base.resource_attributes, @@ -50,24 +61,25 @@ def arize_preset( ) -def _arize_headers(arize_cfg) -> str | None: - pieces: Final = [] - if arize_cfg.space_id or arize_cfg.space_key: - pieces.append(f"space_id={arize_cfg.space_id or arize_cfg.space_key}") - if arize_cfg.api_key: - pieces.append(f"api_key={arize_cfg.api_key}") - if not pieces: - # Fall back to the standard OTLP headers env var when no Arize - # credentials are configured. - return _ArizeSettings().otlp_traces_headers - return ",".join(pieces) +def _arize_headers(arize_cfg, settings: "_ArizeSettings", has_own_endpoint: bool) -> str | None: + space: Final = arize_cfg.space_id or arize_cfg.space_key + pieces: Final = ( + *((f"space_id={space}",) if space else ()), + *((f"api_key={arize_cfg.api_key}",) if arize_cfg.api_key else ()), + ) + if pieces: + return ",".join(pieces) + # Fall back to the standard OTLP headers env var only for an operator's own + # collector. Sending it to the public Arize endpoint would hand that collector's + # auth header to a vendor the operator has no account with. + return settings.otlp_traces_headers if has_own_endpoint else None def arize_dynamic_headers(params: StandardCallbackDynamicParams) -> dict[str, str]: """Per-request Arize OTLP headers from team/key dynamic params.""" - headers: Final[dict[str, str]] = {} + headers: dict[str, str] = {} # ``arize_space_key`` is the suggested param and wins over ``arize_space_id``. - space: Final = params.get("arize_space_key") or params.get("arize_space_id") + space = params.get("arize_space_key") or params.get("arize_space_id") if space: headers["arize-space-id"] = space api_key: Final = params.get("arize_api_key") diff --git a/litellm/integrations/otel/presets/base.py b/litellm/integrations/otel/presets/base.py index 3b9991f86a4..18628290299 100644 --- a/litellm/integrations/otel/presets/base.py +++ b/litellm/integrations/otel/presets/base.py @@ -18,6 +18,20 @@ class Preset(Protocol): ``config_overrides`` lets one preset layer onto another's config (or onto test-supplied defaults); the factory calls presets with no arguments. + + ``allow_missing_credentials`` distinguishes the two reasons a preset is built. + A credential-mandatory backend (weave/langfuse/levo) raises when its global env + credentials are absent so a misconfigured global callback fails loud at startup; + set this when the only reason for construction is an admin-owned destination, + which carries its own per-tenant credentials, so the preset degrades to an + exporter-less (mapper-only) config instead of raising. Credential-optional + backends (arize/phoenix/agentops/langtrace) already run without a global + exporter and ignore it. """ - def __call__(self, *, config_overrides: OpenTelemetryV2Config | None = None) -> OpenTelemetryV2Config: ... + def __call__( + self, + *, + config_overrides: OpenTelemetryV2Config | None = None, + allow_missing_credentials: bool = False, + ) -> OpenTelemetryV2Config: ... diff --git a/litellm/integrations/otel/presets/generic.py b/litellm/integrations/otel/presets/generic.py new file mode 100644 index 00000000000..501cccaf864 --- /dev/null +++ b/litellm/integrations/otel/presets/generic.py @@ -0,0 +1,21 @@ +"""Generic OTLP passthrough preset. + +A ``generic`` admin-owned destination carries only an ``otel_endpoint`` / +``otel_headers`` (no vendor adapter), so this preset is deliberately vendor-neutral: +it yields a base ``OpenTelemetryV2`` config with the standard ``gen_ai.*`` (+ ``legacy``) +span vocabulary, NO vendor mapper, and NO exporter of its own. The per-destination +exporter is appended by ``TenantTracerCache`` from the resolved ``OtelDestination``, so a +generic destination receives the same complete trace tree as Arize/Langfuse/Weave -- +root server span, auth/db internal spans, the ``chat `` gen-AI span, and +cost/error spans. It needs no global OTEL env vars and never raises. +""" + +from litellm.integrations.otel.model.config import OpenTelemetryV2Config + + +def generic_preset( + *, + config_overrides: OpenTelemetryV2Config | None = None, + allow_missing_credentials: bool = False, +) -> OpenTelemetryV2Config: + return config_overrides or OpenTelemetryV2Config() diff --git a/litellm/integrations/otel/presets/langfuse.py b/litellm/integrations/otel/presets/langfuse.py index 5104ee2ff55..3118aa12a53 100644 --- a/litellm/integrations/otel/presets/langfuse.py +++ b/litellm/integrations/otel/presets/langfuse.py @@ -14,29 +14,6 @@ from litellm.integrations.otel.presets.utils import ensure_mappers from litellm.types.utils import StandardCallbackDynamicParams -def langfuse_preset( - *, - config_overrides: OpenTelemetryV2Config | None = None, -) -> OpenTelemetryV2Config: - cfg: Final = _V1Langfuse.get_langfuse_otel_config() - kind: Final = cfg.exporter if isinstance(cfg.exporter, str) else "otlp_http" - base: Final = config_overrides or OpenTelemetryV2Config() - return base.model_copy( - update={ - "exporters": [ - *base.exporters, - ExporterSpec( - kind=kind, - endpoint=cfg.endpoint, - headers=cfg.headers, - owner=ExporterOwner.LANGFUSE_OTEL, - ), - ], - "mapper_names": ensure_mappers(base.mapper_names, "langfuse"), - } - ) - - def langfuse_dynamic_headers(params: StandardCallbackDynamicParams) -> dict[str, str]: """Per-request Langfuse OTLP headers from team/key dynamic params.""" public_key: Final = params.get("langfuse_public_key") @@ -48,3 +25,33 @@ def langfuse_dynamic_headers(params: StandardCallbackDynamicParams) -> dict[str, ) } return {} + + +def langfuse_preset( + *, + config_overrides: OpenTelemetryV2Config | None = None, + allow_missing_credentials: bool = False, +) -> OpenTelemetryV2Config: + base: Final = config_overrides or OpenTelemetryV2Config() + mappers = ensure_mappers(base.mapper_names, "langfuse") + try: + cfg = _V1Langfuse.get_langfuse_otel_config() + except Exception: + if not allow_missing_credentials: + raise + return base.model_copy(update={"mapper_names": mappers}) + kind: Final = cfg.exporter if isinstance(cfg.exporter, str) else "otlp_http" + return base.model_copy( + update={ + "exporters": [ + *base.exporters, + ExporterSpec( + kind=kind, + endpoint=cfg.endpoint, + headers=cfg.headers, + owner=ExporterOwner.LANGFUSE_OTEL, + ), + ], + "mapper_names": mappers, + } + ) diff --git a/litellm/integrations/otel/presets/langtrace.py b/litellm/integrations/otel/presets/langtrace.py index c88e4715ab0..2312575f04a 100644 --- a/litellm/integrations/otel/presets/langtrace.py +++ b/litellm/integrations/otel/presets/langtrace.py @@ -9,6 +9,7 @@ from litellm.integrations.otel.presets.utils import ensure_mappers def langtrace_preset( *, config_overrides: OpenTelemetryV2Config | None = None, + allow_missing_credentials: bool = False, ) -> OpenTelemetryV2Config: """Compose the Langtrace mapper on top of the customer's OTLP destination. diff --git a/litellm/integrations/otel/presets/levo.py b/litellm/integrations/otel/presets/levo.py index 41b3758cf3e..1d118e5223e 100644 --- a/litellm/integrations/otel/presets/levo.py +++ b/litellm/integrations/otel/presets/levo.py @@ -13,9 +13,15 @@ from litellm.integrations.otel.model.config import ( def levo_preset( *, config_overrides: OpenTelemetryV2Config | None = None, + allow_missing_credentials: bool = False, ) -> OpenTelemetryV2Config: - cfg: Final = _V1Levo.get_levo_config() base: Final = config_overrides or OpenTelemetryV2Config() + try: + cfg = _V1Levo.get_levo_config() + except Exception: + if not allow_missing_credentials: + raise + return base return base.model_copy( update={ "exporters": [ diff --git a/litellm/integrations/otel/presets/phoenix.py b/litellm/integrations/otel/presets/phoenix.py index fc1eb9f748f..18af6a8f62b 100644 --- a/litellm/integrations/otel/presets/phoenix.py +++ b/litellm/integrations/otel/presets/phoenix.py @@ -28,22 +28,23 @@ class _PhoenixSettings(BaseSettings): def phoenix_preset( *, config_overrides: OpenTelemetryV2Config | None = None, + allow_missing_credentials: bool = False, ) -> OpenTelemetryV2Config: cfg: Final = _V1Phoenix.get_arize_phoenix_config() headers: Final = cfg.otlp_auth_headers if hasattr(cfg, "otlp_auth_headers") else None project_name: Final = _PhoenixSettings().project_name base: Final = config_overrides or OpenTelemetryV2Config() + global_exporter = ( + ExporterSpec( + kind=cfg.protocol if hasattr(cfg, "protocol") else "otlp_http", + endpoint=cfg.endpoint, + headers=headers, + owner=ExporterOwner.ARIZE_PHOENIX, + ), + ) return base.model_copy( update={ - "exporters": [ - *base.exporters, - ExporterSpec( - kind=cfg.protocol if hasattr(cfg, "protocol") else "otlp_http", - endpoint=cfg.endpoint, - headers=headers, - owner=ExporterOwner.ARIZE_PHOENIX, - ), - ], + "exporters": [*base.exporters, *global_exporter], "mapper_names": ensure_mappers(base.mapper_names, "openinference"), "resource_attributes": { **base.resource_attributes, diff --git a/litellm/integrations/otel/presets/weave.py b/litellm/integrations/otel/presets/weave.py index 51d0ad01093..2a46ba50c24 100644 --- a/litellm/integrations/otel/presets/weave.py +++ b/litellm/integrations/otel/presets/weave.py @@ -15,12 +15,31 @@ from litellm.integrations.weave.weave_otel import ( from litellm.types.utils import StandardCallbackDynamicParams +def weave_dynamic_headers(params: StandardCallbackDynamicParams) -> dict[str, str]: + """Per-request Weave OTLP headers from team/key dynamic params.""" + headers: dict[str, str] = {} + api_key: Final = params.get("wandb_api_key") + if api_key: + headers["Authorization"] = _get_weave_authorization_header(api_key=api_key) + project_id: Final = params.get("weave_project_id") + if project_id: + headers["project_id"] = project_id + return headers + + def weave_preset( *, config_overrides: OpenTelemetryV2Config | None = None, + allow_missing_credentials: bool = False, ) -> OpenTelemetryV2Config: - weave_cfg: Final = get_weave_otel_config() base: Final = config_overrides or OpenTelemetryV2Config() + mappers = ensure_mappers(base.mapper_names, "openinference", "weave") + try: + weave_cfg = get_weave_otel_config() + except Exception: + if not allow_missing_credentials: + raise + return base.model_copy(update={"mapper_names": mappers}) return base.model_copy( update={ "exporters": [ @@ -32,19 +51,6 @@ def weave_preset( owner=ExporterOwner.WEAVE_OTEL, ), ], - # Weave consumes OpenInference + a small Weave-specific overlay. - "mapper_names": ensure_mappers(base.mapper_names, "openinference", "weave"), + "mapper_names": mappers, } ) - - -def weave_dynamic_headers(params: StandardCallbackDynamicParams) -> dict[str, str]: - """Per-request Weave OTLP headers from team/key dynamic params.""" - headers: Final[dict[str, str]] = {} - api_key: Final = params.get("wandb_api_key") - if api_key: - headers["Authorization"] = _get_weave_authorization_header(api_key=api_key) - project_id: Final = params.get("weave_project_id") - if project_id: - headers["project_id"] = project_id - return headers diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index b44d3e457d8..140c02025fe 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -60,6 +60,7 @@ from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.deepeval.deepeval import DeepEvalLogger from litellm.integrations.mlflow import MlflowLogger +from litellm.integrations.otel.model.config import OpenTelemetryV2Config from litellm.integrations.sqs import SQSLogger from litellm.litellm_core_utils.core_helpers import reconstruct_model_name from litellm.litellm_core_utils.get_litellm_params import get_litellm_params @@ -4024,9 +4025,11 @@ def _init_custom_logger_compatible_class( for callback in _in_memory_loggers: if type(callback) is OpenTelemetryV2: return callback - otel_logger_v2: Final = OpenTelemetryV2( - **_get_custom_logger_settings_from_proxy_server(callback_name=logging_integration) - ) + settings = _get_custom_logger_settings_from_proxy_server(callback_name=logging_integration) + config = OpenTelemetryV2Config(**settings) + if not config.exporters: + config = OpenTelemetryV2Config(**{**settings, "exporter": config.exporter}) + otel_logger_v2 = OpenTelemetryV2(config=config) _in_memory_loggers.append(otel_logger_v2) _maybe_auto_initialize_arize_phoenix(_in_memory_loggers) return otel_logger_v2 @@ -4237,6 +4240,8 @@ def _init_custom_logger_compatible_class( _otel_logger = WeaveOtelLogger(config=otel_config, callback_name="weave_otel") _in_memory_loggers.append(_otel_logger) return _otel_logger + elif logging_integration == "generic": + return _maybe_construct_otel_v2("generic", _in_memory_loggers) elif logging_integration == "pagerduty": for callback in _in_memory_loggers: if isinstance(callback, PagerDutyAlerting): @@ -4362,11 +4367,17 @@ def _init_custom_logger_compatible_class( def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[CustomLogger]) -> "OpenTelemetryV2 | None": - """If ``LITELLM_OTEL_V2`` is on, build (or reuse) a single ``OpenTelemetryV2`` - instance configured via the preset for ``callback_name``. + """Build (or reuse) a single ``OpenTelemetryV2`` instance configured via the + preset for ``callback_name`` when V2 owns this backend. - Returns ``None`` when V2 is off OR when there's no preset registered for - ``callback_name`` — callers should then fall through to the legacy path. + Ownership is the operator's configuration alone: the ``LITELLM_OTEL_V2`` flag plus + whatever credentials or ``OTEL_*`` settings the backend has. Returns ``None`` when the + flag is off, when no preset is registered for ``callback_name``, or when the preset + cannot build; callers then fall through to the legacy path. + + Admin-owned destinations deliberately play no part here. They are sinks, delivered to + by ``AdminDestinationLogger``, so registering one for a single team cannot move any + other tenant's backend onto a different logger. """ from litellm.integrations.otel.model.config import is_otel_v2_enabled @@ -4382,16 +4393,53 @@ def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[Custom if isinstance(callback, OpenTelemetryV2) and getattr(callback, "callback_name", None) == callback_name: return callback try: - config: Final = preset_fn() + config = preset_fn(allow_missing_credentials=False) except Exception: - # If env vars are missing or the preset raises, defer to the legacy path - # so customers get the same error story they had before V2 landed. return None - v2_logger: Final = OpenTelemetryV2(config=config, callback_name=callback_name) + if not config.exporters: + # An operator asked for this backend and nothing resolved to send spans to, so it + # would export nothing at all. v2 no longer degrades to a console exporter (that + # printed every span, prompt content included, synchronously on the request path), + # so without this the deployment is silently dark. + verbose_logger.warning( + "OTel v2: '%s' is enabled but no exporter is configured, so no spans will be " + "exported. Set OTEL_EXPORTER/OTEL_ENDPOINT, this backend's credentials, or " + "register a logging destination for it.", + callback_name, + ) + v2_logger = OpenTelemetryV2(config=config, callback_name=callback_name) _in_memory_loggers.append(v2_logger) return v2_logger +def otel_v2_owned_backends() -> frozenset[str]: + """Backend names an ``OpenTelemetryV2`` instance already owns. + + An owning logger fans its gen-AI span out to the request's destinations for its own + backend, so the destination sink must not emit for these or the destination receives + the same call twice as two sibling spans. + + Keyed on actual dispatch, not construction. A logger can be built and land in + ``_in_memory_loggers`` while never reaching the success-callback list, and treating + that as owned made the sink stand down for a backend nobody delivers, turning a + duplicate into a silent total loss of the tenant's traces. + + ``OpenTelemetryV2`` is imported here rather than at module scope because it pulls + ``opentelemetry``, which ships only in the proxy extras: importing it eagerly makes + ``import litellm`` fail for an SDK install that never asked for OTEL. + """ + from litellm.integrations.otel.logger import OpenTelemetryV2 + + dispatched = litellm._async_success_callback + return frozenset( + name + for logger in _in_memory_loggers + if isinstance(logger, OpenTelemetryV2) + and (name := getattr(logger, "callback_name", None)) + and any(cb is logger for cb in dispatched) + ) + + def _maybe_auto_initialize_arize_phoenix(_in_memory_loggers: list[CustomLogger]) -> None: """ Auto-initialize ArizePhoenixLogger when Phoenix env vars are detected. @@ -4556,6 +4604,12 @@ def get_custom_logger_compatible_class( for callback in _in_memory_loggers: if isinstance(callback, ArizeLogger) and callback.callback_name == "arize": return callback + elif logging_integration == "generic": + from litellm.integrations.otel.logger import OpenTelemetryV2 + + for callback in _in_memory_loggers: + if isinstance(callback, OpenTelemetryV2) and getattr(callback, "callback_name", None) == "generic": + return callback elif logging_integration == "logfire": if "LOGFIRE_TOKEN" not in os.environ: raise ValueError("LOGFIRE_TOKEN not found in environment variables") diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 49a1f1314f0..ee0ad5b614d 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -4790,6 +4790,25 @@ if MCP_AVAILABLE: return stored + async def _refresh_request_otel_destinations(user_api_key_auth: Any) -> None: + """Re-resolve the caller's admin destinations for THIS JSON-RPC message. + + A stateful MCP session dispatches every later message on descendants of the task + its ``initialize`` POST spawned, and a ContextVar is copied at task creation. The + destinations resolved for the first message therefore stay frozen on that task for + the session's whole life, so re-scoping or deleting a destination mid-session left + the revoked sink still receiving that team's spans. Resolving per message keeps the + access map authoritative for a session that outlives an admin's edit. + """ + if user_api_key_auth is None: + return + try: + from litellm.proxy.litellm_pre_call_utils import _apply_admin_logging_exporters + + await _apply_admin_logging_exporters(user_api_key_auth) + except Exception as exc: # noqa: BLE001 # a resolver failure must not break the MCP call + verbose_logger.debug("MCP: could not refresh otel destinations: %s", exc) + async def get_or_extract_auth_context() -> tuple[ UserAPIKeyAuth | None, str | None, @@ -4826,6 +4845,7 @@ if MCP_AVAILABLE: oauth2_headers = stored.oauth2_headers raw_headers = stored.raw_headers _client_ip = stored.client_ip + await _refresh_request_otel_destinations(user_api_key_auth) return ( user_api_key_auth, mcp_auth_header, diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_components.py b/tests/test_litellm/integrations/otel/test_otel_v2_components.py index d856d6871a3..138307b80d1 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_components.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_components.py @@ -489,6 +489,16 @@ def test_otlp_traces_endpoint_normalization(): norm("https://x.splunk.com/v2/trace/otlp") == "https://x.splunk.com/v2/trace/otlp" ) + # Langtrace ingests at /api/trace (a complete path, not an OTLP base) — appending + # /v1/traces 404s, so it must be left intact (regression for the double-append bug). + assert ( + norm("https://app.langtrace.ai/api/trace") + == "https://app.langtrace.ai/api/trace" + ) + assert ( + norm("https://app.langtrace.ai/api/trace/") + == "https://app.langtrace.ai/api/trace" + ) assert norm(None) is None @@ -1051,3 +1061,100 @@ def test_sanitize_event_metadata_caps_value_length_and_handles_none(): assert sanitize_event_metadata(None) == {} big = sanitize_event_metadata({"k": "v" * 5000}) assert len(big["k"]) == 1024 + + +def test_blank_otel_v2_flag_reads_as_off(monkeypatch): + """Regression: a declared-but-empty ``LITELLM_OTEL_V2=`` raised a pydantic + ValidationError. The flag is read while ``proxy_server`` is still importing, so the + error escaped module import and the proxy never bound its port; a blank var is routine + in k8s ConfigMaps and ``.env``. Lives here rather than beside the mount tests because + those need an optional instrumentation package CI does not install, which would leave + this uncovered. + """ + from litellm.integrations.otel.model.config import is_otel_v2_enabled + + for raw in ('', ' '): + monkeypatch.setenv('LITELLM_OTEL_V2', raw) + is_otel_v2_enabled.cache_clear() + assert is_otel_v2_enabled() is False + monkeypatch.setenv('LITELLM_OTEL_V2', '1') + is_otel_v2_enabled.cache_clear() + assert is_otel_v2_enabled() is True + is_otel_v2_enabled.cache_clear() + + +def test_no_otel_v2_flag_value_can_raise(monkeypatch): + """The flag is read while ``proxy_server`` is still importing, so any value that + raises takes the proxy down before it binds. A stray space or a word pydantic does not + accept is an easy typo and must degrade to off, while a padded boolean still means + what it says.""" + from litellm.integrations.otel.model.config import is_otel_v2_enabled + + def _read(raw): + monkeypatch.setenv('LITELLM_OTEL_V2', raw) + is_otel_v2_enabled.cache_clear() + return is_otel_v2_enabled() + + for raw in ('enabled', '2', 'maybe', 'TRUE!', '-1'): + assert _read(raw) is False, f'{raw!r} should degrade to off' + for raw in ('true ', ' TRUE', ' yes '): + assert _read(raw) is True, f'{raw!r} should read as on' + for raw in ('false ', ' 0 '): + assert _read(raw) is False, f'{raw!r} should read as off' + is_otel_v2_enabled.cache_clear() + + +def test_each_backend_registers_itself_on_the_per_backend_event_lists(): + """Regression: only the first OTel v2 backend ever dispatched. + + v2 collapsed every backend onto one class parameterised by ``callback_name``, but the + registration guard asked "is any OTel-module callback already here", so the first + registrant locked out every later backend. Each one was still constructed, so it looked + owned while never receiving an event, and its own configured exporter went dark. The + event lists are per-backend; ``service_callback`` stays single-owner, or the + proxy-internal service spans multiply once per configured backend. + """ + import litellm + from litellm.integrations.otel.logger import OpenTelemetryV2 + from litellm.integrations.otel.model.config import OpenTelemetryV2Config + + first = OpenTelemetryV2(config=OpenTelemetryV2Config(), callback_name="arize") + second = OpenTelemetryV2(config=OpenTelemetryV2Config(), callback_name="langfuse_otel") + + for lst in (litellm._async_success_callback, litellm._async_failure_callback, litellm.input_callback): + lst.clear() + first._register_in_callback_list(litellm._async_success_callback, per_backend=True) + second._register_in_callback_list(litellm._async_success_callback, per_backend=True) + first._register_in_callback_list(litellm._async_success_callback, per_backend=True) + + names = [getattr(cb, "callback_name", None) for cb in litellm._async_success_callback] + assert names == ["arize", "langfuse_otel"], names + + service: list = [] + first._register_in_callback_list(service) + second._register_in_callback_list(service) + assert len(service) == 1, "service_callback must keep a single OTel owner" + + +def test_owned_backends_counts_only_loggers_that_actually_dispatch(): + """Regression: ownership read construction, so the destination sink stood down for a + backend nobody delivers and the tenant's traces were lost outright rather than doubled. + """ + import litellm + from litellm.integrations.otel.logger import OpenTelemetryV2 + from litellm.integrations.otel.model.config import OpenTelemetryV2Config + from litellm.litellm_core_utils import litellm_logging as lm + + dispatched = OpenTelemetryV2(config=OpenTelemetryV2Config(), callback_name="arize") + constructed_only = OpenTelemetryV2(config=OpenTelemetryV2Config(), callback_name="langfuse_otel") + litellm._async_success_callback.clear() + litellm._async_success_callback.append(dispatched) + original = lm._in_memory_loggers[:] + try: + lm._in_memory_loggers.clear() + lm._in_memory_loggers.extend([dispatched, constructed_only]) + assert lm.otel_v2_owned_backends() == frozenset({"arize"}) + finally: + lm._in_memory_loggers.clear() + lm._in_memory_loggers.extend(original) + litellm._async_success_callback.clear() diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_dynamic.py b/tests/test_litellm/integrations/otel/test_otel_v2_dynamic.py index f7c0b5452fe..51c08e1ebef 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_dynamic.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_dynamic.py @@ -1,150 +1,185 @@ -"""Per-request multi-tenant credential routing (V1 parity).""" +"""Per-tenant tracer routing on admin-owned OTEL destinations, with fan-out, plus the +per-request credential routing layered over it. + +A request's identity chain is assigned a set of admin-owned exporters; the v2 logger fans +the trace out to all of them (plus the configured/global exporter). Separately, a request's +``standard_callback_dynamic_params`` (team/key OTLP credentials) route the gen-AI span to a +credential-scoped tracer for the integrations that support it (V1 parity). These tests lock +both: each destination's endpoint follows its resolved host (cross-host fix), the configured +exporters are kept (global also receives), a logger only exports the destinations tagged with +its own backend, and request credentials rewrite only their own backend's exporter headers. +""" import os import sys +import pytest + sys.path.insert(0, os.path.abspath("../../../..")) from opentelemetry.trace import NoOpTracer from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config -from litellm.integrations.otel.presets import dynamic_otlp_headers +from litellm.integrations.otel.model.destination import OtelDestination +from litellm.integrations.otel.model.metadata import LLMCallEvent +from litellm.integrations.otel.plumbing.context import ( + _request_destinations, + set_request_destinations, +) from litellm.integrations.otel.plumbing.routing import TenantTracerCache +@pytest.fixture(autouse=True) +def _reset_request_destinations(): + """The v2 router reads destinations from a server-only ContextVar. Reset it around + each test so a prior test's anchored destinations never leak into the next.""" + token = _request_destinations.set(()) + try: + yield + finally: + _request_destinations.reset(token) + + def _cache(callback_name, exporters=None): - cfg = OpenTelemetryV2Config(exporters=exporters or [ExporterSpec(kind="in_memory")]) + cfg = OpenTelemetryV2Config( + exporters=exporters or [ExporterSpec(kind="in_memory", owner=callback_name)] + ) return TenantTracerCache(cfg, callback_name, "litellm") -# --- header builders mirror the V1 construct_dynamic_otel_headers overrides --- # - - -def test_arize_dynamic_headers(): - headers = dynamic_otlp_headers( - "arize", {"arize_space_id": "S", "arize_api_key": "K"} +def _dest(endpoint, auth="Basic AAAA", backend="langfuse_otel"): + return OtelDestination( + endpoint=endpoint, headers={"Authorization": auth}, callback_name=backend ) - assert headers == {"arize-space-id": "S", "api_key": "K"} -def test_arize_space_key_overrides_space_id(): - headers = dynamic_otlp_headers( - "arize", {"arize_space_id": "S", "arize_space_key": "SK"} +def _event(destinations): + """Anchor destinations on the server-only ContextVar (the sole source the v2 router + reads) and build the call event from it, as the proxy does at request time.""" + set_request_destinations( + tuple(d if isinstance(d, OtelDestination) else OtelDestination.model_validate(d) for d in destinations) ) - assert headers == {"arize-space-id": "SK"} + return LLMCallEvent.from_dict({"call_type": "acompletion", "model": "gpt-4o"}) -def test_langfuse_dynamic_headers_need_both_keys(): - assert dynamic_otlp_headers("langfuse_otel", {"langfuse_public_key": "pk"}) is None - headers = dynamic_otlp_headers( - "langfuse_otel", {"langfuse_public_key": "pk", "langfuse_secret_key": "sk"} - ) - assert headers is not None and "Authorization" in headers +# --- routing only happens for admin destinations --------------------------- # -def test_weave_dynamic_headers(): - headers = dynamic_otlp_headers( - "weave_otel", {"wandb_api_key": "w", "weave_project_id": "p"} - ) - assert headers is not None - assert "Authorization" in headers and headers["project_id"] == "p" - - -def test_non_participating_callbacks_have_no_routing(): - # Phoenix subclasses the base in V1 (no override) → no dynamic routing. - assert dynamic_otlp_headers("arize_phoenix", {"arize_api_key": "K"}) is None - assert dynamic_otlp_headers("langtrace", {"arize_api_key": "K"}) is None - assert dynamic_otlp_headers(None, {"arize_api_key": "K"}) is None - - -def test_no_dynamic_params_is_no_routing(): - assert dynamic_otlp_headers("arize", None) is None - assert dynamic_otlp_headers("arize", {}) is None - - -# --- TenantTracerCache routes + caches a TracerProvider per credential set --- # - - -def test_provider_cached_per_credential_set(): - cache = _cache("arize") +def test_no_destinations_uses_default_tracer(): + cache = _cache("langfuse_otel") default = NoOpTracer() - creds_a = {"arize_space_id": "S", "arize_api_key": "K"} - creds_b = {"arize_space_id": "S2", "arize_api_key": "K2"} + assert cache.tracers_for(default, ()) == (default,) + assert cache._providers == {} - cache.tracer_for(default, creds_a) - cache.tracer_for(default, creds_a) # same set → reuse, no new provider + +def test_provider_cached_per_destination_set(): + cache = _cache("langfuse_otel") + default = NoOpTracer() + a = (_dest("https://eu.example/v1", "Basic A"),) + b = (_dest("https://eu.example/v1", "Basic B"),) + + cache.tracers_for(default, a) + cache.tracers_for(default, a) # same set -> reuse assert len(cache._providers) == 1 - cache.tracer_for(default, creds_b) # new set → new provider + cache.tracers_for(default, b) # different creds -> new provider assert len(cache._providers) == 2 -def test_provider_cache_is_bounded_and_evicts_lru(monkeypatch): - # The cache key derives from request-supplied dynamic credentials, so it - # must be bounded — an unbounded cache lets a caller spawn one provider (and - # its background exporter thread) per unique credential set. On overflow the - # least-recently-used provider is evicted and shut down. +def test_different_host_is_a_distinct_provider(): + """Two destinations with identical headers but different hosts must not collide; + the cache key includes each endpoint.""" + cache = _cache("langfuse_otel") + default = NoOpTracer() + eu = (_dest("https://cloud.langfuse.com/api/public/otel", "Basic X"),) + us = (_dest("https://us.cloud.langfuse.com/api/public/otel", "Basic X"),) + cache.tracers_for(default, eu) + cache.tracers_for(default, us) + assert len(cache._providers) == 2 + + +def test_destination_set_is_order_independent(): + cache = _cache("langfuse_otel") + default = NoOpTracer() + a = _dest("https://a/v1", "Basic A") + b = _dest("https://b/v1", "Basic B") + cache.tracers_for(default, (a, b)) + cache.tracers_for(default, (b, a)) # same set, different order -> one provider + assert len(cache._providers) == 1 + + +def test_provider_cache_evicts_lru_and_shuts_it_down_off_hot_path(monkeypatch): + """The provider cache stays bounded, and eviction shuts the LRU provider down so its + ``BatchSpanProcessor`` worker thread is reclaimed instead of leaked for the process + lifetime. ``shutdown`` force-flushes/can block, so it runs on a background daemon + thread rather than the request path: the evicted provider is shut down, the survivors + are not. Dropping the shutdown (the old leak) fails this.""" + import time + from unittest.mock import MagicMock + from litellm.integrations.otel.plumbing import routing as routing_mod monkeypatch.setattr(routing_mod, "_MAX_CACHED_PROVIDERS", 2) - shut_down = [] - monkeypatch.setattr( - routing_mod, "_shutdown_provider", lambda p: shut_down.append(p) - ) + real_build = routing_mod.build_tracer_provider + created = [] - cache = _cache("arize") + def spying_build(config): + provider = real_build(config) + provider.shutdown = MagicMock(wraps=provider.shutdown) + created.append(provider) + return provider + + monkeypatch.setattr(routing_mod, "build_tracer_provider", spying_build) + + cache = _cache("langfuse_otel") default = NoOpTracer() - - def creds(space): - return {"arize_space_id": space, "arize_api_key": "K"} - - cache.tracer_for(default, creds("1")) - cache.tracer_for(default, creds("2")) - cache.tracer_for(default, creds("1")) # touch "1" → "2" is now LRU - cache.tracer_for(default, creds("3")) # overflow → evict "2" + cache.tracers_for(default, (_dest("https://1/v1"),)) # created[0] + cache.tracers_for(default, (_dest("https://2/v1"),)) # created[1] + cache.tracers_for(default, (_dest("https://1/v1"),)) # touch "1" -> "2" is LRU + cache.tracers_for(default, (_dest("https://3/v1"),)) # created[2]: overflow -> evict "2" assert len(cache._providers) == 2 - assert len(shut_down) == 1 # exactly the evicted provider was shut down + # eviction shuts down off the hot path, so wait for the background daemon thread + deadline = time.time() + 5 + while not created[1].shutdown.called and time.time() < deadline: + time.sleep(0.02) + created[1].shutdown.assert_called() # the evicted LRU ("2") is reclaimed + created[0].shutdown.assert_not_called() # survivor + created[2].shutdown.assert_not_called() # survivor -def test_no_dynamic_params_uses_default_tracer(): - cache = _cache("arize") - default = NoOpTracer() - assert cache.tracer_for(default, {}) is default - assert cache._providers == {} +# --- fan-out: keep the configured exporters, append one per destination ----- # -def test_non_participating_callback_uses_default_tracer(): - cache = _cache("arize_phoenix") - default = NoOpTracer() - assert cache.tracer_for(default, {"arize_api_key": "K"}) is default - assert cache._providers == {} - - -def test_dynamic_headers_applied_to_otlp_exporter_only(): +@pytest.mark.parametrize("owner", ["langfuse_otel", "arize", "weave_otel"]) +def test_fan_out_appends_destination_with_resolved_endpoint(owner): + # The configured (global) exporter is kept; each destination is appended with its + # OWN resolved endpoint + headers (the cross-host fix, per owner). cache = _cache( - "arize", + owner, exporters=[ - ExporterSpec(kind="otlp_http", owner="arize"), - ExporterSpec(kind="in_memory", owner="arize"), + ExporterSpec( + kind="otlp_http", + endpoint="https://env-host.example/v1", + headers="Authorization=Basic ENV", + owner=owner, + ) ], ) - new_cfg = cache._config_with_headers({"arize-space-id": "S", "api_key": "K"}) - otlp, in_mem = new_cfg.exporters - assert otlp.headers == "arize-space-id=S,api_key=K" - assert in_mem.headers is None # console/in_memory left untouched + new = cache._config_with_destinations( + (_dest("https://resolved.example/v1", "Basic TEAM", backend=owner),) + ) + # global kept verbatim + assert new.exporters[0].endpoint == "https://env-host.example/v1" + assert new.exporters[0].headers == "Authorization=Basic ENV" + # destination appended at the resolved host with its own auth + assert new.exporters[-1].endpoint == "https://resolved.example/v1" + assert new.exporters[-1].headers == "Authorization=Basic TEAM" + assert len(new.exporters) == 2 -def test_dynamic_headers_do_not_leak_to_other_owners_exporter(): - """A tenant's Arize credentials must never be stamped onto a co-configured - exporter owned by a different backend (a self-hosted collector, Langfuse). - - Regression for the cross-backend credential leak: ``_config_with_headers`` - used to rewrite the headers of every OTLP exporter, so one request carrying - a team's Arize key clobbered the base collector's and Langfuse's headers - with that key. - """ +def test_fan_out_preserves_co_configured_exporters(): cache = _cache( - "arize", + "langfuse_otel", exporters=[ ExporterSpec( kind="otlp_http", @@ -154,22 +189,523 @@ def test_dynamic_headers_do_not_leak_to_other_owners_exporter(): ), ExporterSpec( kind="otlp_http", - endpoint="https://cloud.langfuse.com/api/public/otel", - headers="Authorization=Basic base-langfuse", + endpoint="https://us.cloud.langfuse.com/api/public/otel", + headers="Authorization=Basic ENV", owner="langfuse_otel", ), - ExporterSpec( - kind="otlp_grpc", - endpoint="https://otlp.arize.com/v1", - headers="space_id=base,api_key=base", - owner="arize", - ), ], ) - new_cfg = cache._config_with_headers( - {"arize-space-id": "TEAMX", "api_key": "TEAMX_KEY"} + new = cache._config_with_destinations( + (_dest("https://cloud.langfuse.com/api/public/otel", "Basic TEAM"),) ) - by_owner = {e.owner: e.headers for e in new_cfg.exporters} - assert by_owner["arize"] == "arize-space-id=TEAMX,api_key=TEAMX_KEY" - assert by_owner[None] == "x=base-collector" - assert by_owner["langfuse_otel"] == "Authorization=Basic base-langfuse" + # both originals preserved unchanged (no rewrite/leak) + assert new.exporters[0].endpoint == "http://self-hosted-collector:4318" + assert new.exporters[0].headers == "x=base-collector" + assert new.exporters[1].headers == "Authorization=Basic ENV" + # exactly one appended + assert new.exporters[-1].endpoint == "https://cloud.langfuse.com/api/public/otel" + assert len(new.exporters) == 3 + + +def test_fan_out_to_many_destinations_is_one_provider_with_all_exporters(): + cache = _cache( + "langfuse_otel", + exporters=[ + ExporterSpec( + kind="otlp_http", endpoint="https://env/v1", owner="langfuse_otel" + ) + ], + ) + new = cache._config_with_destinations( + (_dest("https://a/v1", "Basic A"), _dest("https://b/v1", "Basic B")) + ) + # global + 2 destinations -> 3 span processors, one provider, one span copied to all + assert [e.endpoint for e in new.exporters] == [ + "https://env/v1", + "https://a/v1", + "https://b/v1", + ] + cache.tracers_for(NoOpTracer(), (_dest("https://a/v1"), _dest("https://b/v1"))) + assert len(cache._providers) == 1 + + +# --- gen-AI span Resource must match its fanned-out parents ----------------- # +# +# The gen-AI LLM-call span is emitted through the TenantTracerCache clone +# provider, while the proxy-internal spans (server/auth/db) are forwarded by +# TenantFanOutSpanProcessor, which wraps each with the destination's backend- +# required Resource attrs (Arize needs model_id / arize.project.name). If the +# clone provider does NOT also carry those attrs, the gen-AI span reaches Arize +# with only service.name and Arize renders it as an orphaned subtree. These pin +# that the clone Resource carries the same attrs the fan-out wrap injects. + + +def _dest_with_resource(endpoint, backend, resource_attributes): + return OtelDestination( + endpoint=endpoint, + headers={"Authorization": "Basic AAAA"}, + callback_name=backend, + resource_attributes=resource_attributes, + ) + + +def test_clone_config_carries_destination_resource_attrs(): + cache = _cache("arize") + new = cache._config_with_destinations( + ( + _dest_with_resource( + "https://otlp.arize.com/v1", + "arize", + {"model_id": "team-b-proj", "arize.project.name": "team-b-proj"}, + ), + ) + ) + assert new.resource_attributes["model_id"] == "team-b-proj" + assert new.resource_attributes["arize.project.name"] == "team-b-proj" + + +def test_clone_config_carries_builder_declared_resource_attrs(monkeypatch): + """End-to-end: an arize credential that omits the project gets ARIZE_PROJECT_NAME + folded in at BUILD time (build_destination), and the clone config then carries + those Resource attrs generically -- the gen-AI span lands in the same project as + its fan-out'd parents. The clone path itself is backend-agnostic; it reads + whatever the builder declared.""" + monkeypatch.setenv("ARIZE_PROJECT_NAME", "env-proj") + from litellm.integrations.otel.presets.destinations import build_destination + + dest = build_destination("arize", {"arize_space_id": "s", "arize_api_key": "k"}) + assert dest is not None + cache = _cache("arize") + new = cache._config_with_destinations((dest,)) + assert new.resource_attributes["model_id"] == "env-proj" + assert new.resource_attributes["arize.project.name"] == "env-proj" + + +def test_clone_provider_emits_genai_span_with_destination_resource(): + """End-to-end regression: the span the clone provider actually exports must + carry the destination's Resource attrs. Pre-fix this Resource was + service.name only, orphaning the gen-AI span in Arize.""" + from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, + ) + + from litellm.integrations.otel.plumbing.providers import ( + build_tracer_provider, + get_tracer, + ) + + cfg = OpenTelemetryV2Config( + service_name="litellm-proxy", + exporters=[ExporterSpec(kind="in_memory", owner="arize")], + ) + cache = TenantTracerCache(cfg, "arize", "litellm") + dest = _dest_with_resource( + "https://otlp.arize.com/v1", + "arize", + {"model_id": "team-b-proj", "arize.project.name": "team-b-proj"}, + ) + tracers = cache.tracers_for(get_tracer(build_tracer_provider(cfg)), (dest,)) + tracer = tracers[-1] + with tracer.start_as_current_span("chat anthropic-haiku") as span: + span.set_attribute("gen_ai.operation.name", "chat") + + # The destination group carries the destination's Resource and only the + # destination's exporter; the configured in-memory exporter rides the base tracer + # (``tracers[0]``) on a clean Resource, so the tenant's project can never stamp it. + provider = next(iter(cache._providers.values())) + provider.force_flush() + resource_attrs = dict(provider.resource.attributes) + assert resource_attrs.get("model_id") == "team-b-proj" + assert not any( + isinstance(getattr(proc, "span_exporter", None), InMemorySpanExporter) + for proc in provider._active_span_processor._span_processors + ), "the configured exporter must not ride a destination group" + assert resource_attrs.get("arize.project.name") == "team-b-proj" + + +# --- request credentials route the gen-AI span (V1 parity) ------------------ # + + +_DYNAMIC_CREDS = [ + ("langfuse_otel", {"langfuse_public_key": "pk-teamA", "langfuse_secret_key": "sk-teamA"}, "Authorization="), + ("arize", {"arize_space_id": "space-teamA", "arize_api_key": "key-teamA"}, "arize-space-id=space-teamA"), + ("weave_otel", {"wandb_api_key": "wandb-teamA", "weave_project_id": "proj-teamA"}, "project_id=proj-teamA"), +] + + +@pytest.mark.parametrize("backend, creds, owned_fragment", _DYNAMIC_CREDS) +def test_request_credentials_rewrite_only_the_owned_exporter(backend, creds, owned_fragment): + """A request's team/key OTLP credentials rewrite the headers of THIS backend's own + exporter and nothing else, so the gen-AI span exports to that team's account while a + co-configured backend's exporter is untouched (no cross-backend key leak).""" + from litellm.integrations.otel.presets import dynamic_otlp_headers + + cache = _cache( + backend, + exporters=[ + ExporterSpec(kind="otlp_http", endpoint="http://collector/otel", headers="Authorization=Basic GLOBAL", owner=backend), + ExporterSpec(kind="otlp_http", endpoint="http://other/otel", headers="x=coconfigured", owner="levo"), + ], + ) + headers = dynamic_otlp_headers(backend, creds) + assert headers, f"{backend} must support dynamic credentials" + scoped = cache._config_with_headers(headers) + assert owned_fragment in (scoped.exporters[0].headers or "") + assert "GLOBAL" not in (scoped.exporters[0].headers or "") # the global header was rewritten away + assert scoped.exporters[1].headers == "x=coconfigured" # a different backend's exporter is untouched + + +@pytest.mark.parametrize("backend, creds, owned_fragment", _DYNAMIC_CREDS) +def test_genai_tracers_for_spawns_credential_scoped_provider(backend, creds, owned_fragment): + """genai_tracers_for with request creds and no admin destination returns exactly the + credential-scoped tracer (not the default) and caches its provider under a dynamic key.""" + cache = _cache( + backend, + exporters=[ExporterSpec(kind="otlp_http", endpoint="http://c/otel", headers="Authorization=Basic GLOBAL", owner=backend)], + ) + default = NoOpTracer() + tracers = cache.genai_tracers_for(default, (), creds) + assert len(tracers) == 1 + assert tracers[0] is not default + assert len(cache._providers) == 1 + assert next(iter(cache._providers))[0] == "dynamic" # namespaced key, never aliases a destination group + + +def test_genai_tracers_for_without_creds_is_plain_default(): + """No dynamic creds and no destinations -> the logger's default tracer, no provider spawned.""" + cache = _cache( + "langfuse_otel", + exporters=[ExporterSpec(kind="otlp_http", endpoint="http://c/otel", headers="", owner="langfuse_otel")], + ) + default = NoOpTracer() + assert cache.genai_tracers_for(default, (), None) == (default,) + assert cache._providers == {} + + +def test_genai_tracers_compose_credential_scoped_plus_destination(): + """Request creds AND an admin destination: the span exports via the credential-scoped + tracer (which carries the global exporter) plus one per-destination tracer that omits the + base exporters, so the global collector receives the span exactly once.""" + cache = _cache( + "langfuse_otel", + exporters=[ExporterSpec(kind="otlp_http", endpoint="http://global/otel", headers="Authorization=Basic GLOBAL", owner="langfuse_otel")], + ) + creds = {"langfuse_public_key": "pk-teamA", "langfuse_secret_key": "sk-teamA"} + destinations = (_dest("https://cloud.langfuse.com/api/public/otel", auth="Basic ADMIN"),) + tracers = cache.genai_tracers_for(NoOpTracer(), destinations, creds) + assert len(tracers) == 2 # credential-scoped (global) + one destination group + # the destination group provider carries no base exporter (base rides the dynamic tracer) + dest_only = cache._config_with_destinations(destinations, include_base_exporters=False) + assert all(spec.endpoint != "http://global/otel" for spec in dest_only.exporters) + + +def test_admin_destinations_route(): + event = _event( + [ + { + "callback_name": "langfuse_otel", + "endpoint": "https://cloud.langfuse.com/api/public/otel", + "headers": {"Authorization": "Basic ADMIN"}, + } + ] + ) + assert len(event.otel_destinations) == 1 + cache = _cache("langfuse_otel") + cache.tracers_for(NoOpTracer(), event.otel_destinations) + assert len(cache._providers) == 1 + + +# --- a logger only exports the destinations tagged with its own backend ----- # + + +def test_logger_filters_destinations_to_its_backend(): + from litellm.integrations.otel.logger import OpenTelemetryV2 + + event = _event( + [ + { + "callback_name": "langfuse_otel", + "endpoint": "https://lf/api/public/otel", + "headers": {"Authorization": "Basic A"}, + }, + { + "callback_name": "arize", + "endpoint": "https://otlp.arize.com/v1", + "headers": {"space_id": "S"}, + }, + ] + ) + + class _Shim: + callback_name = "langfuse_otel" + + got = OpenTelemetryV2._destinations_for_backend(_Shim(), event) + assert [d.endpoint for d in got] == ["https://lf/api/public/otel"] + + +# --- multi-destination, same-backend: group by Resource so each gets its span -- # +# +# A backend that selects its target FROM the Resource (Arize's project) needs a +# differently-tagged span per destination, because a span carries exactly one +# Resource (a TracerProvider property). Pre-fix the gen-AI clone folded every +# destination into ONE last-wins Resource, so two Arize projects collapsed to one +# and only that project received the gen-AI span. ``tracers_for`` groups by +# ``destination_resource_attrs`` and returns one tracer (one provider, one Resource) +# per distinct group. Header-routed backends declare no Resource attrs, so they stay +# in one group with multiple exporters and keep routing by per-exporter auth. + + +def _arize_dest(project, space="S", key="K"): + return OtelDestination( + endpoint="https://otlp.arize.com/v1", + headers={"space_id": space, "api_key": key}, + callback_name="arize", + resource_attributes={"model_id": project, "arize.project.name": project}, + ) + + +def _provider_project(provider): + return provider.resource.attributes.get("arize.project.name") + + +def test_tracers_for_empty_returns_default_only(): + cache = _cache("arize") + default = NoOpTracer() + assert cache.tracers_for(default, ()) == (default,) + assert cache._providers == {} + + +def test_tracers_for_single_destination_one_group(): + """A single Arize destination yields the clean base tracer (``default``) plus one + destination provider carrying its project; base is never folded into the project group.""" + default = NoOpTracer() + cache = _cache("arize") + tracers = cache.tracers_for(default, (_arize_dest("solo"),)) + assert tracers[0] is default # base rides its own clean tracer, returned first + assert len(tracers) == 2 # default + the one project group + assert {_provider_project(p) for p in cache._providers.values()} == {"solo"} + + +def test_tracers_for_two_arize_projects_split_into_separate_groups(): + """The fix: two Arize destinations with different Resource attrs must NOT + last-wins merge -- each project gets its own provider/Resource so each receives a + correctly-tagged gen-AI span.""" + cache = _cache("arize") + tracers = cache.tracers_for( + NoOpTracer(), (_arize_dest("projA"), _arize_dest("projB")) + ) + assert len(tracers) == 3 # default (clean base) + one tracer per project group + assert {_provider_project(p) for p in cache._providers.values()} == { + "projA", + "projB", + } + + +def test_two_arize_projects_each_provider_has_its_own_single_project(): + """Each group's Resource carries exactly its own project (not the other's, not a + merge).""" + cache = _cache("arize") + cache.tracers_for(NoOpTracer(), (_arize_dest("projA"), _arize_dest("projB"))) + by_project = { + _provider_project(p): p.resource.attributes for p in cache._providers.values() + } + assert by_project["projA"]["model_id"] == "projA" + assert by_project["projB"]["model_id"] == "projB" + + +def test_header_routed_destinations_stay_one_group_with_two_exporters(): + """Langfuse/Weave declare no Resource attrs, so two distinct destinations collapse + into ONE group (one provider) with one exporter each -- they route by per-exporter + auth, so no per-Resource split is needed or wanted.""" + cache = _cache( + "langfuse_otel", + exporters=[ + ExporterSpec(kind="otlp_http", endpoint="https://env/v1", owner=None) + ], + ) + tracers = cache.tracers_for( + NoOpTracer(), + (_dest("https://a/v1", "Basic A"), _dest("https://b/v1", "Basic B")), + ) + assert len(tracers) == 2 # default (clean base) + the single empty-Resource group + assert len(cache._providers) == 1 + (provider,) = cache._providers.values() + endpoints = " ".join( + str(getattr(getattr(sp, "span_exporter", None), "_endpoint", "")) + for sp in provider._active_span_processor._span_processors + ) + # both destinations live on the one group provider (the global exporter rides ``default``, + # not this group); OTLP normalizes the endpoint by appending /v1/traces, so match on prefix + assert "https://a/v1" in endpoints and "https://b/v1" in endpoints + + +def test_base_exporters_never_ride_a_destination_group(): + """The configured/global exporters must ride their own clean-Resource tracer + (``default``), never a destination group: folding them into a group made the global + export inherit that destination's Resource (e.g. Arize's project), so an operator's + own collector saw spans stamped with a tenant's project. The global still receives the + gen-AI span exactly once (via ``default``), not once per project.""" + cache = _cache( + "arize", + exporters=[ExporterSpec(kind="in_memory", endpoint=None, owner=None)], + ) + default = NoOpTracer() + tracers = cache.tracers_for(default, (_arize_dest("projA"), _arize_dest("projB"))) + assert tracers[0] is default # global rides its own clean tracer, exactly once + for provider in cache._providers.values(): + base_count = sum( + type(getattr(sp, "span_exporter", sp)).__name__ == "InMemorySpanExporter" + for sp in provider._active_span_processor._span_processors + ) + assert base_count == 0, "no destination group may carry the configured/global exporters" + # and each group's Resource stays its own project only + assert _provider_project(provider) in {"projA", "projB"} + + +def test_generic_backend_resolves_generic_destination(): + """A Generic OTLP destination (callback_name='generic') must be picked up by the + generic OpenTelemetryV2 logger, so its gen-AI span routes to the destination's + otel_endpoint. Regression: 'generic' had no preset, so no generic logger existed and + the gen-AI span was dropped (only proxy-internal spans fanned out).""" + from litellm.integrations.otel.logger import OpenTelemetryV2 + + event = _event( + [ + { + "callback_name": "generic", + "endpoint": "http://collector:4318", + "headers": {"x-tenant": "t1"}, + }, + { + "callback_name": "arize", + "endpoint": "https://otlp.arize.com/v1", + "headers": {"space_id": "S"}, + }, + ] + ) + + class _Shim: + callback_name = "generic" + + got = OpenTelemetryV2._destinations_for_backend(_Shim(), event) + assert [d.endpoint for d in got] == ["http://collector:4318"] + + +def test_otel_destination_is_frozen_and_renders_header_string(): + """OtelDestination is the immutable value the resolver hands to the runtime; + mutating one after resolution must fail (a request can never rewrite where its + traces go), and header_string renders the k=v,k2=v2 form the exporter expects.""" + import pytest + from pydantic import ValidationError + + dest = OtelDestination( + endpoint="https://c/v1", headers={"Authorization": "Bearer x", "x-k": "y"} + ) + with pytest.raises(ValidationError): + dest.endpoint = "https://evil/v1" + assert dest.header_string() == "Authorization=Bearer x,x-k=y" + + +@pytest.mark.parametrize( + "backend, params, expected_endpoint", + [ + ( + "langfuse_otel", + {"langfuse_public_key": "pk", "langfuse_secret_key": "sk", "langfuse_host": "https://lf.internal"}, + "https://lf.internal/api/public/otel", + ), + ("arize", {"arize_space_key": "S", "arize_api_key": "K"}, "https://otlp.arize.com/v1"), + ("weave_otel", {"wandb_api_key": "wk"}, "https://trace.wandb.ai/otel/v1/traces"), + ], +) +def test_team_credentials_still_export_when_the_preset_degraded(backend, params, expected_endpoint): + """Regression: a preset built with ``allow_missing_credentials`` contributes no owned + exporter, so a team's own ``callback_vars`` had nothing to be stamped onto and its + spans were dropped. The exporter is synthesized from the request's own credentials + instead, resolved through the same builder an equivalent admin destination uses. + """ + from litellm.integrations.otel.presets import dynamic_otlp_headers + + degraded = TenantTracerCache(OpenTelemetryV2Config(exporters=[]), backend, "litellm") + headers = dynamic_otlp_headers(backend, params) + assert headers, "the backend must recognise these per-team credentials" + + owned = [spec for spec in degraded._config_with_headers(headers, params).exporters if spec.owner == backend] + + assert len(owned) == 1, "a degraded preset must still export the team's own traces" + assert owned[0].endpoint == expected_endpoint + assert owned[0].headers == ",".join(f"{k}={v}" for k, v in headers.items()) + + +def test_degraded_preset_synthesizes_nothing_without_team_credentials(): + """No per-request credentials means no synthesized exporter, so a degraded backend + stays silent rather than inventing an uncredentialled vendor export.""" + degraded = TenantTracerCache(OpenTelemetryV2Config(exporters=[]), "arize", "litellm") + assert degraded._config_with_headers({}, None).exporters == [] + + +# --- transport is part of a destination's identity ------------------------- # +# Two destinations can agree on endpoint, headers and Resource attrs and still +# disagree on OTLP transport (an Arize credential naming ``arize_http_endpoint`` +# resolves to otlp_http where the backend's intrinsic default is gRPC). The +# provider cache and the synthesized exporter both have to carry that. + + +def _arize_dest_with_protocol(protocol): + return OtelDestination( + endpoint="https://collector.internal/v1", + headers={"space_id": "S", "api_key": "K"}, + callback_name="arize", + resource_attributes={"model_id": "proj", "arize.project.name": "proj"}, + protocol=protocol, + ) + + +def _exporter_transports(provider): + """The OTLP transport of each of a provider's exporters. Both exporters are named + ``OTLPSpanExporter``; only the defining module tells gRPC from HTTP apart.""" + return { + "http" if ".http." in type(sp.span_exporter).__module__ else "grpc" + for sp in provider._active_span_processor._span_processors + if getattr(sp, "span_exporter", None) is not None + and "otlp" in type(sp.span_exporter).__module__ + } + + +def test_same_endpoint_and_headers_but_different_protocol_are_distinct_providers(): + """Two destinations differing only in OTLP transport must not share a provider; + reusing the first one's exports the second's spans over the wrong transport.""" + cache = _cache("arize") + cache.tracers_for(NoOpTracer(), (_arize_dest_with_protocol(None),)) + cache.tracers_for(NoOpTracer(), (_arize_dest_with_protocol("otlp_http"),)) + + assert len(cache._providers) == 2 + transports = sorted(t for p in cache._providers.values() for t in _exporter_transports(p)) + assert transports == ["grpc", "http"] + + +def test_synthesized_exporter_uses_the_transport_the_credentials_pin(): + """A team whose Arize credentials name an HTTP collector gets an HTTP exporter, not + the backend's intrinsic gRPC default.""" + cache = _cache("arize", exporters=[ExporterSpec(kind="otlp_grpc", endpoint="https://env/v1", owner=None)]) + params = { + "arize_space_id": "S", + "arize_api_key": "K", + "arize_http_endpoint": "https://collector.internal/v1", + } + + config = cache._config_with_headers({"space_id": "S"}, params) + + (synthesized,) = [spec for spec in config.exporters if spec.owner == "arize"] + assert synthesized.endpoint == "https://collector.internal/v1" + assert synthesized.kind == "otlp_http" + + +def test_synthesized_exporter_keeps_the_backend_default_when_credentials_pin_nothing(): + """Credentials that name no transport still get the backend's intrinsic one.""" + cache = _cache("arize", exporters=[ExporterSpec(kind="otlp_grpc", endpoint="https://env/v1", owner=None)]) + + config = cache._config_with_headers({"space_id": "S"}, {"arize_space_id": "S", "arize_api_key": "K"}) + + (synthesized,) = [spec for spec in config.exporters if spec.owner == "arize"] + assert synthesized.kind == "otlp_grpc" diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_fan_out.py b/tests/test_litellm/integrations/otel/test_otel_v2_fan_out.py new file mode 100644 index 00000000000..607f1e82014 --- /dev/null +++ b/tests/test_litellm/integrations/otel/test_otel_v2_fan_out.py @@ -0,0 +1,390 @@ +"""Tests for the per-request tenant fan-out SpanProcessor (Hoist). + +The fan-out processor lives on the main v2 ``TracerProvider`` and forwards every +finished span to the admin-resolved per-tenant destinations carried on the +request's ``ContextVar``. The contract under test: + +- Spans on the main provider land at every per-tenant destination matching this + backend (the ``owner_callback_name``). +- When destinations is empty (an unassigned identity, the SDK path, a request + before the resolver ran), the processor is a no-op. +- Concurrent requests with different destinations stay isolated -- contextvars + scope per task, so one request's tenant doesn't receive another's spans. +- The processor caches one BatchSpanProcessor per ``(endpoint, headers)`` pair + and skips destinations whose ``callback_name`` doesn't match its owner. +""" + +import asyncio +from typing import Any + +import pytest + +pytest.importorskip("opentelemetry") + +from opentelemetry.sdk.trace import TracerProvider # noqa: E402 +from opentelemetry.sdk.trace.export import SimpleSpanProcessor # noqa: E402 +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( # noqa: E402 + InMemorySpanExporter, +) +from opentelemetry.sdk.trace.export import SpanExporter # noqa: E402 + +from litellm.integrations.otel.model.destination import OtelDestination # noqa: E402 +from litellm.integrations.otel.plumbing.context import ( # noqa: E402 + set_request_destinations, + _request_destinations, +) +from litellm.integrations.otel.plumbing.routing import ( # noqa: E402 + TenantFanOutSpanProcessor, + _processor_key, +) +from litellm.integrations.otel.plumbing.providers import ( # noqa: E402 + destination_resource_attrs, +) +from litellm.integrations.otel.plumbing import providers # noqa: E402 +from litellm.integrations.otel.model.config import OpenTelemetryV2Config # noqa: E402 + + +@pytest.fixture(autouse=True) +def _reset_request_destinations(): + _request_destinations.set(()) + yield + _request_destinations.set(()) + + +def _build_provider_with_fan_out( + owner: str, exporter: SpanExporter +) -> tuple[TracerProvider, TenantFanOutSpanProcessor]: + """Build a real TracerProvider with the in-memory exporter as the configured + backend, plus the fan-out processor wired in. The fan-out processor's + per-destination processors are real BatchSpanProcessors, replaced in the + test by an injected SimpleSpanProcessor against an in-memory exporter via + ``monkeypatch`` so the test can read what was forwarded.""" + cfg = OpenTelemetryV2Config(exporter="in_memory") + provider = providers.build_tracer_provider( + cfg, exporter=exporter, tenant_fan_out_owner=owner + ) + fan_out = next( + p + for p in provider._active_span_processor._span_processors + if isinstance(p, TenantFanOutSpanProcessor) + ) + return provider, fan_out + + +def test_fan_out_forwards_span_to_matching_destination(monkeypatch): + """A span on the main provider lands at the per-tenant destination whose + callback_name matches the fan-out owner. Removing the fan-out branch makes + this test fail (the tenant exporter never sees the span).""" + main_exporter = InMemorySpanExporter() + provider, fan_out = _build_provider_with_fan_out("langfuse_otel", main_exporter) + + tenant_exporter = InMemorySpanExporter() + + # Swap the lazy per-destination processor build for an in-memory one so we + # don't actually hit the network. + def _stub_processor_for(self, destination): + return SimpleSpanProcessor(tenant_exporter) + + monkeypatch.setattr( + TenantFanOutSpanProcessor, "_processor_for", _stub_processor_for + ) + + set_request_destinations( + ( + OtelDestination( + callback_name="langfuse_otel", + endpoint="https://cloud.langfuse.com/api/public/otel", + headers={"Authorization": "Bearer pk:sk"}, + ), + ) + ) + tracer = provider.get_tracer("test") + with tracer.start_as_current_span("auth /chat/completions"): + pass + + main_names = [s.name for s in main_exporter.get_finished_spans()] + tenant_names = [s.name for s in tenant_exporter.get_finished_spans()] + assert "auth /chat/completions" in main_names + assert "auth /chat/completions" in tenant_names + + +def test_fan_out_forwards_proxy_internal_spans_to_every_destination(monkeypatch): + """Proxy-internal spans (server, auth phase, postgres, post-call ledger) + are generic OTel semconv with no backend-specific vocabulary, so they ship + to every admin-resolved destination regardless of its ``callback_name``. + The owner discriminator is only used to skip the gen-AI span (the v2 logger + routes that itself through TenantTracerCache to avoid wrong-vocabulary + duplicates).""" + main_exporter = InMemorySpanExporter() + _provider, _ = _build_provider_with_fan_out("langfuse_otel", main_exporter) + + tenant_exporter = InMemorySpanExporter() + monkeypatch.setattr( + TenantFanOutSpanProcessor, + "_processor_for", + lambda self, destination: SimpleSpanProcessor(tenant_exporter), + ) + + # Destination is tagged for a DIFFERENT backend (``arize``) than the + # owner (``langfuse_otel``). Proxy-internal spans must still forward. + set_request_destinations( + ( + OtelDestination( + callback_name="arize", + endpoint="https://otlp.arize.com/v1", + headers={"api_key": "k", "space_id": "s"}, + ), + ) + ) + tracer = _provider.get_tracer("test") + with tracer.start_as_current_span("auth"): + pass + + names = [s.name for s in tenant_exporter.get_finished_spans()] + assert "auth" in names + + +def test_destination_resource_attrs_reads_declared_attrs(monkeypatch): + """``destination_resource_attrs`` is backend-agnostic: it returns exactly what + the destination's builder declared, never consulting env itself. The env-vs- + credential precedence is enforced at build time (see test_presets_destinations), + so setting ARIZE_PROJECT_NAME here must NOT override the destination's own attrs.""" + monkeypatch.setenv("ARIZE_PROJECT_NAME", "global-project") + destination = OtelDestination( + callback_name="arize", + endpoint="https://otlp.arize.com/v1", + headers={"api_key": "k", "space_id": "s"}, + resource_attributes={ + "model_id": "tenant-project", + "arize.project.name": "tenant-project", + }, + ) + + assert destination_resource_attrs(destination) == { + "model_id": "tenant-project", + "arize.project.name": "tenant-project", + } + + +def test_destination_resource_attrs_empty_for_header_routed_backend(): + """A langfuse/weave-style destination routes by header and declares no Resource + attrs; the helper returns {} and the fan-out leaves the span Resource untouched.""" + destination = OtelDestination( + callback_name="langfuse_otel", + endpoint="https://cloud.langfuse.com/api/public/otel", + headers={"Authorization": "Basic x"}, + ) + assert destination_resource_attrs(destination) == {} + + +def test_fan_out_skips_genai_span_to_avoid_double_export(monkeypatch): + """The gen-AI LLM-call span is routed by the v2 logger through + ``TenantTracerCache`` to per-tenant exporters with the right attribute + mapper. Forwarding it here too would deliver a duplicate with the wrong + vocabulary, surfacing in the destination as an orphaned second span.""" + main_exporter = InMemorySpanExporter() + _provider, _ = _build_provider_with_fan_out("langfuse_otel", main_exporter) + + tenant_exporter = InMemorySpanExporter() + monkeypatch.setattr( + TenantFanOutSpanProcessor, + "_processor_for", + lambda self, destination: SimpleSpanProcessor(tenant_exporter), + ) + + set_request_destinations( + ( + OtelDestination( + callback_name="arize", + endpoint="https://otlp.arize.com/v1", + headers={}, + ), + ) + ) + tracer = _provider.get_tracer("test") + # A gen-AI span sets ``gen_ai.operation.name`` (the v2 emitter does this + # via the SpanRole.LLM_CALL mapper). Fake it explicitly here. + with tracer.start_as_current_span("chat gpt-4o") as span: + span.set_attribute("gen_ai.operation.name", "chat") + # A proxy-internal span also fires. + with tracer.start_as_current_span("auth /v1/chat/completions"): + pass + + names = [s.name for s in tenant_exporter.get_finished_spans()] + assert "auth /v1/chat/completions" in names + assert "chat gpt-4o" not in names # gen-AI skipped + + +def test_fan_out_noop_when_no_destinations(monkeypatch): + """An unassigned identity / pre-auth / SDK path leaves the contextvar at + its empty-tuple default; the processor must short-circuit, NOT crash, NOT + forward.""" + main_exporter = InMemorySpanExporter() + provider, _ = _build_provider_with_fan_out("langfuse_otel", main_exporter) + forwarded: list[Any] = [] + monkeypatch.setattr( + TenantFanOutSpanProcessor, + "_processor_for", + lambda self, destination: forwarded.append(destination) or None, + ) + tracer = provider.get_tracer("test") + with tracer.start_as_current_span("auth"): + pass + assert forwarded == [] + + +def test_fan_out_caches_processor_per_destination_key(monkeypatch): + """Two requests with the same destination must share one cached + BatchSpanProcessor; two different destinations must build two. Otherwise + every request rebuilds the OTLP exporter (and its background thread).""" + main_exporter = InMemorySpanExporter() + _provider, fan_out = _build_provider_with_fan_out("langfuse_otel", main_exporter) + + built: list[OtelDestination] = [] + + real_processor_for = TenantFanOutSpanProcessor._processor_for + + def _spy(self, destination): + result = real_processor_for(self, destination) + if result is not None: + built.append(destination) + return result + + # Build always returns None to keep the test offline, but we tracked + # invocations via the spy above. Easier: stub the inner builder. + def _stub(self, destination): + built.append(destination) + key = _processor_key(destination) + if key in self._processors: + return self._processors[key] + proc = SimpleSpanProcessor(InMemorySpanExporter()) + self._processors[key] = proc + return proc + + monkeypatch.setattr(TenantFanOutSpanProcessor, "_processor_for", _stub) + + dest_a = OtelDestination( + callback_name="langfuse_otel", endpoint="https://a/", headers={"k": "1"} + ) + dest_b = OtelDestination( + callback_name="langfuse_otel", endpoint="https://b/", headers={"k": "1"} + ) + set_request_destinations((dest_a,)) + tracer = _provider.get_tracer("test") + with tracer.start_as_current_span("s1"): + pass + set_request_destinations((dest_a,)) + with tracer.start_as_current_span("s2"): + pass + set_request_destinations((dest_b,)) + with tracer.start_as_current_span("s3"): + pass + + # _stub appends every call; the cache size is what matters: two unique + # ``(endpoint, headers)`` pairs -> two cached processors. + assert len(fan_out._processors) == 2 + + +def test_fan_out_flush_and_shutdown_survive_concurrent_cache_mutation(): + """force_flush/shutdown snapshot the processor cache before iterating, so a + concurrent ``on_end`` inserting or evicting a processor cannot raise a + 'mutated during iteration' RuntimeError and drop the remaining destinations' + buffered spans.""" + _provider, fan_out = _build_provider_with_fan_out("langfuse_otel", InMemorySpanExporter()) + + touched: list[str] = [] + + class _MutatingProc: + def __init__(self, name: str, mutate: bool = False) -> None: + self.name = name + self.mutate = mutate + + def _maybe_mutate(self, key: str) -> None: + if self.mutate: # simulate a concurrent on_end caching a new processor + fan_out._processors[(key, "x")] = _MutatingProc(key) + + def force_flush(self, timeout_millis: int = 30000) -> bool: + touched.append(self.name) + self._maybe_mutate("late-flush") + return True + + def shutdown(self) -> None: + touched.append("sd-" + self.name) + self._maybe_mutate("late-shutdown") + + fan_out._processors.clear() + fan_out._processors[("a", "1")] = _MutatingProc("a", mutate=True) + fan_out._processors[("b", "2")] = _MutatingProc("b") + fan_out._processors[("c", "3")] = _MutatingProc("c") + + # No RuntimeError despite the mutation mid-iteration; every original processor ran. + assert fan_out.force_flush() is True + assert {"a", "b", "c"} <= set(touched) + + touched.clear() + fan_out._processors[("a", "1")] = _MutatingProc("a", mutate=True) + fan_out.shutdown() + assert "sd-a" in touched + + +def test_fan_out_per_request_isolation_with_concurrent_tasks(monkeypatch): + """Two requests running concurrently with different destinations must each + see only THEIR destination's spans. The contextvar scopes per asyncio task, + so this is a pin on the contextvar approach: switching to a global variable + would break this test.""" + main_exporter = InMemorySpanExporter() + provider, _ = _build_provider_with_fan_out("langfuse_otel", main_exporter) + + exporter_a = InMemorySpanExporter() + exporter_b = InMemorySpanExporter() + + def _stub(self, destination): + if destination.endpoint == "https://a/": + return SimpleSpanProcessor(exporter_a) + return SimpleSpanProcessor(exporter_b) + + monkeypatch.setattr(TenantFanOutSpanProcessor, "_processor_for", _stub) + tracer = provider.get_tracer("test") + + async def fire(label: str, endpoint: str): + set_request_destinations( + ( + OtelDestination( + callback_name="langfuse_otel", + endpoint=endpoint, + headers={}, + ), + ) + ) + with tracer.start_as_current_span(label): + await asyncio.sleep(0) + + async def driver(): + await asyncio.gather( + fire("req_a", "https://a/"), + fire("req_b", "https://b/"), + ) + + asyncio.run(driver()) + + names_a = {s.name for s in exporter_a.get_finished_spans()} + names_b = {s.name for s in exporter_b.get_finished_spans()} + assert "req_a" in names_a and "req_b" not in names_a + assert "req_b" in names_b and "req_a" not in names_b + + +def test_fan_out_owner_set_by_build_tracer_provider(): + """The v2 logger opts the MAIN provider into fan-out by passing its + callback_name; the TenantTracerCache clone providers do NOT, so the gen-AI + span emitted through them is not also forwarded by the fan-out processor. + Regressing this (adding the processor to clones) would double-export every + gen-AI span.""" + from litellm.integrations.otel.logger import OpenTelemetryV2 + + logger = OpenTelemetryV2(callback_name="arize") + main_procs = logger._tracer_provider._active_span_processor._span_processors + assert any(isinstance(p, TenantFanOutSpanProcessor) for p in main_procs) + # Build a tenant clone provider and confirm it has no fan-out processor. + clone = providers.build_tracer_provider(logger.config) + clone_procs = clone._active_span_processor._span_processors + assert not any(isinstance(p, TenantFanOutSpanProcessor) for p in clone_procs) diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py index 82b074220fa..fac5a7d9219 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py @@ -28,10 +28,12 @@ from litellm.integrations.otel import ( # noqa: E402 ) from litellm.integrations.otel.plumbing import providers # noqa: E402 from litellm.integrations.otel.plumbing.context import ( # noqa: E402 + _request_destinations, reset_mcp_message_trace_carrier, reset_mcp_message_transport_span, set_mcp_message_trace_carrier, set_mcp_message_transport_span, + set_request_destinations, set_request_root_span, ) from litellm.integrations.otel.logger import OpenTelemetryV2 # noqa: E402 @@ -41,6 +43,25 @@ from litellm.integrations.otel.model.spans import ( # noqa: E402 ) from litellm.integrations.otel.model.utils import to_ns, to_seconds # noqa: E402 + +def _anchor(dests): + """Anchor admin destinations on the server-only ContextVar the v2 router reads + (the proxy sets this at auth time; there is no request-carried carrier).""" + from litellm.integrations.otel.model.destination import OtelDestination + + set_request_destinations( + tuple(d if isinstance(d, OtelDestination) else OtelDestination.model_validate(d) for d in dests) + ) + + +@pytest.fixture(autouse=True) +def _reset_request_destinations(): + token = _request_destinations.set(()) + try: + yield + finally: + _request_destinations.reset(token) + # --------------------------------------------------------------------------- # # Fixtures # --------------------------------------------------------------------------- # @@ -429,6 +450,59 @@ def test_mcp_tool_call_is_not_logged_as_llm_call(): assert "gen_ai.request.model" not in span.attributes +def test_mcp_tool_call_routes_to_admin_destination(): + """The MCP tool-call span carries ``gen_ai.operation.name`` (execute_tool), so the + fan-out processor skips it; it must still be routed to the request's admin + destinations like the LLM-call span, not reach the global exporter only.""" + from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, + ) + + from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config + from litellm.integrations.otel.model.destination import OtelDestination + from litellm.integrations.otel.plumbing.context import _request_destinations + from litellm.integrations.otel.plumbing.providers import build_tracer_provider + + cfg = OpenTelemetryV2Config( + service_name="litellm-proxy", + exporters=[ExporterSpec(kind="in_memory")], + ) + base_provider = build_tracer_provider(cfg) + logger = OpenTelemetryV2(config=cfg, callback_name="generic", tracer_provider=base_provider) + endpoint = "http://127.0.0.1:1/v1/traces" + dest = OtelDestination(callback_name="generic", endpoint=endpoint, headers={"x": "1"}) + + token = _request_destinations.set((dest,)) + try: + kwargs = {"standard_logging_object": _mcp_payload()} + asyncio.run(logger.async_log_success_event(kwargs, None, None, None)) + finally: + _request_destinations.reset(token) + + # Routing: a destination provider was built and points at the destination's endpoint. If the + # tool-call span took the global-only path (no fan-out), no destination provider would exist. + assert logger._tenant_tracers._providers, "no destination provider was built for the tool-call" + destination_endpoints = " ".join( + str(getattr(getattr(proc, "span_exporter", None), "_endpoint", "")) + for provider in logger._tenant_tracers._providers.values() + for proc in provider._active_span_processor._span_processors + ) + assert endpoint in destination_endpoints, "no exporter pointed at the admin destination" + + # Emission: the tool-call span was really emitted (the configured/global exporter rides its own + # clean provider, so it is captured there rather than on a destination group). + base_provider.force_flush() + captured = [ + span + for proc in base_provider._active_span_processor._span_processors + if isinstance(getattr(proc, "span_exporter", None), InMemorySpanExporter) + for span in proc.span_exporter.get_finished_spans() + ] + assert any(s.attributes.get("mcp.method.name") == "tools/call" for s in captured), ( + "tool-call span was not emitted" + ) + + def test_mcp_tool_call_captures_io_when_enabled(): logger, exporter = _logger_capturing() kwargs = {"standard_logging_object": _mcp_payload()} @@ -863,6 +937,101 @@ def test_pre_call_idempotent_keeps_first_span(): assert first is second # not overwritten +def test_retry_reusing_a_closed_call_id_gets_its_own_span(): + """Regression: a router retry or fallback reuses the request's ``litellm_call_id``, + and the failed attempt's close marks that id closed before the retry runs. Treating + the marker as final meant the retry opened no span and emitted nothing, so a request + that failed once and then succeeded exported only the failure -- no tokens, no cost. + + The retry now reopens, and the reopened span is finished and exported, so it cannot + leak instead. + """ + logger, exporter = _logger() + kwargs = _kwargs() + server = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME) + + # attempt 1: opened, then closed by the failure callback + with trace.use_span(server, end_on_exit=False): + logger.log_pre_api_call(model="gpt-4o", messages=[], kwargs=kwargs) + assert "call_1" in logger._open_llm_calls + asyncio.run(logger.async_log_success_event(kwargs, None, None, None)) + assert "call_1" in logger._closed_call_ids + first_round = len([s for s in exporter.get_finished_spans() if s.name == "chat gpt-4o"]) + assert first_round == 1 + + # attempt 2 reuses the same call id and must get its own span + with trace.use_span(server, end_on_exit=False): + logger.log_pre_api_call(model="gpt-4o", messages=[], kwargs=kwargs) + assert "call_1" in logger._open_llm_calls, "the retry must reopen, not be skipped" + asyncio.run(logger.async_log_success_event(kwargs, None, None, None)) + server.end() + + assert "call_1" not in logger._open_llm_calls, "the reopened span must be finished, not leaked" + assert len([s for s in exporter.get_finished_spans() if s.name == "chat gpt-4o"]) == 2 + + +def test_retry_reaching_destinations_emits_a_span_per_attempt(monkeypatch): + """Regression, deferred/destination path: the destination logger is activated + lazily at success time, so both the failed attempt and the retry that replaced it + arrive as carrier-less closes sharing one ``litellm_call_id``. Deduping the fan-out + on the call id alone collapsed the two, and the destination saw only the failure -- + no tokens, no cost. Each attempt carries its own response id and must emit. + """ + logger, exporter = _logger() + monkeypatch.setattr(logger, "callback_name", "in_memory") + monkeypatch.setattr(logger._tenant_tracers, "genai_tracers_for", lambda default, dests, params: (default,)) + _anchor([{"callback_name": "in_memory", "endpoint": "https://otlp.example.com/v1", "headers": {"api_key": "k"}}]) + + failed = _payload( + id="attempt_1", + status="failure", + response={"id": "attempt_1", "model": "gpt-4o-2024", "choices": []}, + ) + succeeded = _payload( + id="attempt_2", + response={"id": "attempt_2", "model": "gpt-4o-2024", "choices": [{"finish_reason": "stop"}]}, + ) + assert failed["litellm_call_id"] == succeeded["litellm_call_id"], "a retry reuses one call id" + + asyncio.run(logger.async_log_success_event(_kwargs(failed), None, None, None)) + asyncio.run(logger.async_log_success_event(_kwargs(succeeded), None, None, None)) + + emitted = [s for s in exporter.get_finished_spans() if s.name == "chat gpt-4o"] + assert len(emitted) == 2, "the retry must reach the destination, not be deduped into the failure" + + +def test_repeat_deferred_callback_for_one_attempt_still_dedupes(monkeypatch): + """The counterpart the fan-out dedup still guards: a sync+async double-firing of a + single attempt shares its response id and must emit once.""" + logger, exporter = _logger() + monkeypatch.setattr(logger, "callback_name", "in_memory") + monkeypatch.setattr(logger._tenant_tracers, "genai_tracers_for", lambda default, dests, params: (default,)) + _anchor([{"callback_name": "in_memory", "endpoint": "https://otlp.example.com/v1", "headers": {"api_key": "k"}}]) + + payload = _payload(id="attempt_1", response={"id": "attempt_1", "model": "gpt-4o-2024", "choices": []}) + asyncio.run(logger.async_log_success_event(_kwargs(payload), None, None, None)) + asyncio.run(logger.async_log_success_event(_kwargs(payload), None, None, None)) + + assert len([s for s in exporter.get_finished_spans() if s.name == "chat gpt-4o"]) == 1 + + +def test_repeat_callback_without_an_open_span_emits_nothing(): + """The counterpart the closed marker still guards: a second success callback for a + call that has no open carrier must not re-emit through the deferred path.""" + logger, exporter = _logger() + kwargs = _kwargs() + server = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME) + with trace.use_span(server, end_on_exit=False): + logger.log_pre_api_call(model="gpt-4o", messages=[], kwargs=kwargs) + asyncio.run(logger.async_log_success_event(kwargs, None, None, None)) + emitted = len([s for s in exporter.get_finished_spans() if s.name == "chat gpt-4o"]) + + asyncio.run(logger.async_log_success_event(kwargs, None, None, None)) # duplicate + server.end() + + assert len([s for s in exporter.get_finished_spans() if s.name == "chat gpt-4o"]) == emitted + + # --------------------------------------------------------------------------- # # Parent resolution — ambient context at the boundary (no metadata threading) # --------------------------------------------------------------------------- # @@ -937,7 +1106,7 @@ def test_live_llm_span_anchors_to_root_with_no_active_span(): set_request_root_span(server) kwargs = _kwargs() logger.log_pre_api_call(model="gpt-4o", messages=[], kwargs=kwargs) - assert logger._open_llm_calls["call_1"].span is not None # live, via anchor + assert logger._open_llm_calls["call_1"].spans # live, via anchor asyncio.run(logger.async_log_success_event(kwargs, None, None, None)) server.end() by_name = {s.name: s for s in exporter.get_finished_spans()} @@ -957,7 +1126,7 @@ def test_deferred_llm_span_reads_anchor_at_close(): kwargs = _kwargs() # pre_call with NO anchor and no active span → deferred. logger.log_pre_api_call(model="gpt-4o", messages=[], kwargs=kwargs) - assert logger._open_llm_calls["call_1"].span is None # deferred + assert logger._open_llm_calls["call_1"].spans == () # deferred # Anchor becomes visible at close (worker copied the request task's context). set_request_root_span(server) asyncio.run(logger.async_log_success_event(kwargs, None, None, None)) @@ -997,6 +1166,199 @@ def test_synthetic_error_log_produces_no_llm_span(): assert "auth /chat/completions" in names # auth span itself still recorded +def test_lazy_activation_emits_llm_span_when_destination_resolves(monkeypatch): + """LIT-3850 lazy-activation seam: a v2 instance born inside the success path + (because the destination resolver appended its backend to ``success_callback`` + on this request) was not in the callback list when ``pre_call`` iterated, so + no carrier was opened. The close must still emit the gen-ai span when the + payload is present and the admin-resolved destinations name this backend, so + the per-tenant exporter ships it. Without the fallthrough, this test sees + zero spans.""" + logger, exporter = _logger() + monkeypatch.setattr(logger, "callback_name", "in_memory") + server = logger._emitter.start_span( + SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME + ) + set_request_root_span(server) + # Make ``tracers_for`` return the logger's default tracer regardless of the + # destinations passed: the test asserts the close-path emitted the span, not + # that the per-destination provider clone wired up an OTLP exporter (the + # routing cache's job, covered separately). The default tracer is bound to + # the in-memory exporter so the test can read the result. + tracer_for_calls: list[tuple] = [] + + def _fake_tracers_for(default, destinations): + tracer_for_calls.append(destinations) + return (default,) + + monkeypatch.setattr(logger._tenant_tracers, "tracers_for", _fake_tracers_for) + kwargs = _kwargs() + _anchor([ + { + "callback_name": "in_memory", + "endpoint": "https://otlp.example.com/v1", + "headers": {"api_key": "k"}, + } + ]) + assert "call_1" not in logger._open_llm_calls # no carrier opened + asyncio.run(logger.async_log_success_event(kwargs, None, None, None)) + server.end() + names = [s.name for s in exporter.get_finished_spans()] + assert "chat gpt-4o" in names + # ``tracers_for`` was invoked with exactly the resolved destination, proving + # the deferred path used per-tenant routing rather than the default tracer + # blindly. + assert len(tracer_for_calls) == 1 + (dests,) = tracer_for_calls + assert len(dests) == 1 and dests[0].endpoint == "https://otlp.example.com/v1" + + +def test_no_upstream_reject_emits_no_deferred_span_even_with_destinations(monkeypatch): + """A close for a no-upstream request (``LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL`` set) + with a payload and resolved destinations must emit no gen-AI span. Mirrors + ``test_lazy_activation_emits_llm_span_when_destination_resolves`` with the marker + set: that one emits, this one must not.""" + from litellm.constants import LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL + + logger, exporter = _logger() + monkeypatch.setattr(logger, "callback_name", "in_memory") + server = logger._emitter.start_span( + SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME + ) + set_request_root_span(server) + + tracer_for_calls: list[tuple] = [] + + def _fake_tracers_for(default, destinations): + tracer_for_calls.append(destinations) + return (default,) + + monkeypatch.setattr(logger._tenant_tracers, "tracers_for", _fake_tracers_for) + _anchor([ + { + "callback_name": "in_memory", + "endpoint": "https://otlp.example.com/v1", + "headers": {"api_key": "k"}, + } + ]) + payload = _payload( + status="failure", + error_information={"error_class": "ProxyException", "error_code": "429"}, + ) + kwargs = _kwargs(payload=payload) + kwargs[LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL] = True + + assert "call_1" not in logger._open_llm_calls + asyncio.run(logger.async_log_failure_event(kwargs, None, None, None)) + server.end() + + names = [s.name for s in exporter.get_finished_spans()] + assert "chat gpt-4o" not in names + assert tracer_for_calls == [] + + +def test_second_close_after_opened_call_does_not_emit_duplicate(monkeypatch): + logger, exporter = _logger() + monkeypatch.setattr(logger, "callback_name", "in_memory") + monkeypatch.setattr( + logger._tenant_tracers, + "tracers_for", + lambda default, destinations: (default,), + ) + server = logger._emitter.start_span( + SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME + ) + set_request_root_span(server) + kwargs = _kwargs() + _anchor([ + { + "callback_name": "in_memory", + "endpoint": "https://otlp.example.com/v1", + "headers": {"api_key": "k"}, + } + ]) + + logger.log_pre_api_call(model="gpt-4o", messages=[], kwargs=kwargs) + asyncio.run(logger.async_log_success_event(kwargs, None, None, None)) + asyncio.run(logger.async_log_failure_event(kwargs, None, None, None)) + server.end() + + names = [s.name for s in exporter.get_finished_spans()] + assert names.count("chat gpt-4o") == 1 + + +def test_close_without_carrier_and_without_destination_drops_silently(): + """The pre-existing early-return semantics (auth gate / pre-call guardrail + rejection with no destination resolving to this backend) must be preserved: + no phantom span. The fix only widens emit-on-close when the admin-resolved + destinations name this backend AND the payload exists.""" + logger, exporter = _logger() + server = logger._emitter.start_span( + SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME + ) + set_request_root_span(server) + asyncio.run(logger.async_log_success_event(_kwargs(), None, None, None)) + server.end() + names = [s.name for s in exporter.get_finished_spans()] + assert "chat gpt-4o" not in names + + +def test_close_without_carrier_drops_when_payload_missing(monkeypatch): + """No carrier + no payload = the auth-gate rejection case (no upstream call + happened). Must drop even when destinations resolve, so a phantom span is + never emitted for a request the gate refused.""" + logger, exporter = _logger() + monkeypatch.setattr(logger, "callback_name", "in_memory") + kwargs = {"litellm_params": {"metadata": {}}} + _anchor( + [ + { + "callback_name": "in_memory", + "endpoint": "https://otlp.example.com/v1", + "headers": {}, + } + ] + ) + asyncio.run(logger.async_log_success_event(kwargs, None, None, None)) + assert exporter.get_finished_spans() == () + + +def test_close_dedupes_duplicate_callbacks(monkeypatch): + """Cursor BugBot regression: a normal close pops the carrier and emits + the LLM span; a second callback for the same call_id must not emit a + duplicate. Before the dedup guard, the second close hit the + carrier-is-None branch and fired _emit_deferred_llm_call again whenever + payload + destinations remained on the kwargs, double-exporting the + span (e.g. success + failure callbacks both firing, or a custom callback + fanning out).""" + logger, exporter = _logger() + monkeypatch.setattr(logger, "callback_name", "in_memory") + server = logger._emitter.start_span( + SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME + ) + set_request_root_span(server) + monkeypatch.setattr( + logger._tenant_tracers, "tracers_for", lambda default, dests: (default,) + ) + kwargs = _kwargs() + _anchor([ + { + "callback_name": "in_memory", + "endpoint": "https://otlp.example.com/v1", + "headers": {}, + } + ]) + logger.log_pre_api_call(model="gpt-4o", messages=[], kwargs=kwargs) + asyncio.run(logger.async_log_success_event(kwargs, None, None, None)) + # Second callback for the same call_id: payload + destinations still on + # kwargs, but no second span must emit. + asyncio.run(logger.async_log_success_event(kwargs, None, None, None)) + asyncio.run(logger.async_log_failure_event(kwargs, None, None, None)) + server.end() + llm_spans = [s for s in exporter.get_finished_spans() if s.name == "chat gpt-4o"] + assert len(llm_spans) == 1 + + def test_create_request_started_span_captures_anchor(): """``create_litellm_proxy_request_started_span`` doubles as the anchor capture point: the active server span becomes the request root for later spans.""" @@ -2277,3 +2639,192 @@ def test_metrics_disabled_by_default_records_nothing(monkeypatch): ) ) assert _emitted_metric_names(reader) == set() + + +def _second_group_tracer(logger): + """A second independent in-memory provider standing in for a second Resource group + (e.g. a second Arize project); returns (tracer, exporter).""" + exporter = InMemorySpanExporter() + provider = providers.build_tracer_provider(logger.config, exporter=exporter) + return provider.get_tracer("litellm"), exporter + + +def test_genai_span_emitted_to_every_group_live(monkeypatch): + """Multi-destination fix (live path): when ``tracers_for`` returns two tracers (two + Resource groups, e.g. two Arize projects), the gen-AI span opened at ``pre_call`` + must be opened+finished on BOTH -- the bug was only one project receiving it.""" + logger, exporter_a = _logger() + monkeypatch.setattr(logger, "callback_name", "in_memory") + tracer_b, exporter_b = _second_group_tracer(logger) + monkeypatch.setattr( + logger._tenant_tracers, + "tracers_for", + lambda default, dests: (default, tracer_b), + ) + server = logger._emitter.start_span( + SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME + ) + set_request_root_span(server) + kwargs = _kwargs() + _anchor([ + {"callback_name": "in_memory", "endpoint": "https://x/v1", "headers": {}} + ]) + _emit_llm(logger, kwargs, ambient=server) + server.end() + assert [s.name for s in exporter_a.get_finished_spans()].count("chat gpt-4o") == 1 + assert [s.name for s in exporter_b.get_finished_spans()].count("chat gpt-4o") == 1 + + +def test_genai_span_emitted_to_every_group_deferred(monkeypatch): + """Same fix, deferred path (no carrier at ``pre_call``): ``emit_fanout`` dedups once + on the call id then emits the span on every group's tracer, so both projects get + exactly one.""" + logger, exporter_a = _logger() + monkeypatch.setattr(logger, "callback_name", "in_memory") + tracer_b, exporter_b = _second_group_tracer(logger) + monkeypatch.setattr( + logger._tenant_tracers, + "tracers_for", + lambda default, dests: (default, tracer_b), + ) + server = logger._emitter.start_span( + SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME + ) + set_request_root_span(server) + kwargs = _kwargs() + _anchor([ + {"callback_name": "in_memory", "endpoint": "https://x/v1", "headers": {}} + ]) + # no pre_call -> no carrier -> deferred close path + assert "call_1" not in logger._open_llm_calls + asyncio.run(logger.async_log_success_event(kwargs, None, None, None)) + server.end() + assert [s.name for s in exporter_a.get_finished_spans()].count("chat gpt-4o") == 1 + assert [s.name for s in exporter_b.get_finished_spans()].count("chat gpt-4o") == 1 + + +def _generic_dest(): + return { + "callback_name": "generic", + "endpoint": "http://collector:4318", + "headers": {}, + } + + +def test_generic_destination_emits_genai_span(monkeypatch): + """Acceptance #1: a request whose only admin destination is a Generic OTLP + destination emits the chat gen-AI span (regression: 'generic' had no preset, + so the gen-AI span was dropped and only proxy-internal spans reached the endpoint). + The destination's callback_name='generic' is matched by the generic logger, and the + span is routed through the per-destination tracer.""" + logger, exporter = _logger() + monkeypatch.setattr(logger, "callback_name", "generic") + routed: list = [] + monkeypatch.setattr( + logger._tenant_tracers, + "tracers_for", + lambda default, dests: (routed.append(dests) or (default,)), + ) + server = logger._emitter.start_span( + SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME + ) + set_request_root_span(server) + kwargs = _kwargs() + _anchor([_generic_dest()]) + _emit_llm(logger, kwargs, ambient=server) + server.end() + assert "chat gpt-4o" in [s.name for s in exporter.get_finished_spans()] + # the gen-AI span was routed for the generic destination (not an empty/global set) + assert any( + len(d) == 1 and d[0].endpoint == "http://collector:4318" for d in routed + ), "generic destination was not routed to the generic logger's tracer" + + +def test_generic_destination_emits_error_span_on_failure(monkeypatch): + """Acceptance #3: a FAILED call to a Generic OTLP destination still emits the + chat span, with OTEL status ERROR and the exception type.""" + logger, exporter = _logger() + monkeypatch.setattr(logger, "callback_name", "generic") + monkeypatch.setattr( + logger._tenant_tracers, "tracers_for", lambda default, dests: (default,) + ) + server = logger._emitter.start_span( + SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME + ) + set_request_root_span(server) + payload = _payload( + status="failure", + error_information={ + "error_class": "AuthenticationError", + "error_message": "Incorrect API key", + }, + ) + kwargs = _kwargs(payload=payload) + _anchor([_generic_dest()]) + _emit_llm(logger, kwargs, ambient=server, fail=True) + server.end() + spans = {s.name: s for s in exporter.get_finished_spans()} + assert "chat gpt-4o" in spans + assert spans["chat gpt-4o"].status.status_code is StatusCode.ERROR + assert spans["chat gpt-4o"].attributes["error.type"] == "AuthenticationError" + + +def test_lazy_activation_emits_llm_span_for_a_team_carrying_its_own_credentials(monkeypatch): + """Regression: a team's own ``callback_vars`` export must survive another team's grant. + + Once an admin registers a destination for a backend, v2 owns it and the legacy logger + is no longer built, so this instance is the only thing that can reach a team's own + account. That team has no destination of its own, so it is lazily activated at the + success event with no carrier. Gating the deferred span on destinations alone dropped + it: the team silently stopped exporting the moment a *different* team was granted a + destination for the same backend. + """ + logger, exporter = _logger() + monkeypatch.setattr(logger, "callback_name", "langfuse_otel") + server = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME) + set_request_root_span(server) + + genai_calls: list[tuple] = [] + + def _fake_genai_tracers_for(default, destinations, dynamic_params): + genai_calls.append((destinations, dynamic_params)) + return (default,) + + monkeypatch.setattr(logger._tenant_tracers, "genai_tracers_for", _fake_genai_tracers_for) + + kwargs = _kwargs() + kwargs["standard_callback_dynamic_params"] = { + "langfuse_public_key": "pk-team", + "langfuse_secret_key": "sk-team", + "langfuse_host": "https://team.langfuse.example", + } + _anchor([]) # no destination grants this identity + + assert "call_1" not in logger._open_llm_calls + asyncio.run(logger.async_log_success_event(kwargs, None, None, None)) + server.end() + + assert "chat gpt-4o" in [s.name for s in exporter.get_finished_spans()] + # routed by the request's own credentials, with no destinations in play + assert len(genai_calls) == 1 + destinations, dynamic_params = genai_calls[0] + assert destinations == () + assert dynamic_params["langfuse_public_key"] == "pk-team" + + +def test_lazy_activation_stays_silent_without_destination_or_credentials(monkeypatch): + """The counterpart: no destination and no per-request credentials for this backend + means nothing activated it, so the deferred path must emit nothing. Default-deny is + what keeps an ungranted identity off every destination.""" + logger, exporter = _logger() + monkeypatch.setattr(logger, "callback_name", "langfuse_otel") + server = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME) + set_request_root_span(server) + + kwargs = _kwargs() + _anchor([]) + + asyncio.run(logger.async_log_success_event(kwargs, None, None, None)) + server.end() + + assert "chat gpt-4o" not in [s.name for s in exporter.get_finished_spans()] diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_mount.py b/tests/test_litellm/integrations/otel/test_otel_v2_mount.py index 7240d49d022..a441f0489a7 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_mount.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_mount.py @@ -84,6 +84,14 @@ def test_gate_toggles_with_env(monkeypatch): assert is_otel_v2_enabled() is True +def test_blank_flag_does_not_escape_the_startup_mount(monkeypatch): + """The whole-proxy symptom: ``instrument_fastapi_app`` is called at + ``proxy_server`` import time, so it must not raise on a blank flag.""" + monkeypatch.setenv("LITELLM_OTEL_V2", "") + is_otel_v2_enabled.cache_clear() + instrument_fastapi_app(fastapi.FastAPI()) + + def test_instrumented_app_emits_server_span(): app, logger = _instrumented_app() exporter = InMemorySpanExporter() diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_multibackend.py b/tests/test_litellm/integrations/otel/test_otel_v2_multibackend.py index e879766c5c7..f843a8de8c7 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_multibackend.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_multibackend.py @@ -87,3 +87,33 @@ def test_config_folds_legacy_exporter_triple_into_exporters_list(): assert len(cfg.exporters) == 1 assert cfg.exporters[0].kind == "otlp_http" assert cfg.exporters[0].endpoint == "https://api.example.com" + + +def _processor_names(provider): + return [type(p).__name__ for p in provider._active_span_processor._span_processors] + + +def test_main_logger_provider_always_carries_fan_out(): + # The FastAPI server span (and auth/db/cost spans) bind to the main v2 logger's + # provider. That provider MUST carry the TenantFanOutSpanProcessor even for the + # generic global logger (no backend named) -- otherwise, for an admin-owned- + # destination-only deployment, the server span never reaches the destination and + # the gen-AI child is orphaned. Regression for the orphaned-span RCA. + from litellm.integrations.otel.logger import OpenTelemetryV2 + + generic = OpenTelemetryV2() + assert "TenantFanOutSpanProcessor" in _processor_names(generic._tracer_provider) + + named = OpenTelemetryV2(callback_name="arize") + assert "TenantFanOutSpanProcessor" in _processor_names(named._tracer_provider) + + +def test_build_tracer_provider_attach_fan_out_flag(): + cfg = OpenTelemetryV2Config(exporter="console") + # Per-tenant clone providers pass neither flag and must NOT fan out (else the + # gen-AI span they carry would be double-exported). + clone = build_tracer_provider(cfg) + assert "TenantFanOutSpanProcessor" not in _processor_names(clone) + # The main provider opts in explicitly. + main = build_tracer_provider(cfg, attach_tenant_fan_out=True) + assert "TenantFanOutSpanProcessor" in _processor_names(main) diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_presets.py b/tests/test_litellm/integrations/otel/test_otel_v2_presets.py index add2caf9a48..c21e59f36ba 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_presets.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_presets.py @@ -33,27 +33,130 @@ def test_agentops_preset_does_no_network_io(monkeypatch): assert spec.options == {"api_key": "ak-123"} # carried to the lazy exporter -def test_agentops_preset_without_key_omits_options(monkeypatch): +def test_agentops_preset_without_key_still_appends_exporter(monkeypatch): + # Additive parity with the pre-PR preset: a global agentops callback always + # contributes its exporter, with no api_key carried when none is configured. + # The JWT-minting exporter simply has nothing to mint until a key is set. monkeypatch.delenv("AGENTOPS_API_KEY", raising=False) cfg = agentops_preset() - spec = next(e for e in cfg.exporters if e.kind == _AGENTOPS_EXPORTER_KIND) - assert spec.options is None + specs = [e for e in cfg.exporters if e.kind == _AGENTOPS_EXPORTER_KIND] + assert len(specs) == 1 + assert specs[0].options is None + + +def test_arize_preset_without_credentials_still_appends_exporter(monkeypatch): + # Additive parity: a global arize callback always contributes its exporter at + # the configured (or default) endpoint, so a no-auth self-hosted collector + # reached via ARIZE_ENDPOINT keeps exporting exactly as before the PR. + from litellm.integrations.otel.model.config import ExporterOwner + from litellm.integrations.otel.presets.arize import arize_preset + + for var in ( + "ARIZE_SPACE_ID", + "ARIZE_SPACE_KEY", + "ARIZE_API_KEY", + "OTEL_EXPORTER_OTLP_TRACES_HEADERS", + ): + monkeypatch.delenv(var, raising=False) + cfg = arize_preset() + assert [e for e in cfg.exporters if e.owner == ExporterOwner.ARIZE_AX] != [] + + +@pytest.mark.parametrize( + "preset_name, owner_name, env", + [ + ("arize", "ARIZE_AX", ("ARIZE_SPACE_ID", "ARIZE_SPACE_KEY", "ARIZE_API_KEY", "ARIZE_ENDPOINT", "ARIZE_HTTP_ENDPOINT")), + ("agentops", "AGENTOPS", ("AGENTOPS_API_KEY",)), + ], +) +def test_credential_optional_presets_add_no_vendor_exporter_for_a_destination(monkeypatch, preset_name, owner_name, env): + # Regression: registering a team-scoped destination builds the backend proxy-wide with + # allow_missing_credentials=True. A credential-optional preset must then contribute NO + # global exporter: appending its public vendor endpoint anyway ships that tenant's spans + # to a vendor account nobody granted (and Arize additionally stamps the operator's own + # OTLP auth header onto that uninvited request). + from litellm.integrations.otel.model.config import ExporterOwner + from litellm.integrations.otel.presets import PRESET_BY_CALLBACK + + for var in env: + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv("OTEL_EXPORTER_OTLP_TRACES_HEADERS", "authorization=Bearer OPERATOR-COLLECTOR-SECRET") + + cfg = PRESET_BY_CALLBACK[preset_name](allow_missing_credentials=True) + owner = getattr(ExporterOwner, owner_name) + assert [e for e in cfg.exporters if e.owner == owner] == [] + assert "OPERATOR-COLLECTOR-SECRET" not in str([e.headers for e in cfg.exporters]) + assert cfg.mapper_names, "degrading must keep the preset's mappers" + + +def test_arize_keeps_its_exporter_for_an_operators_own_collector(monkeypatch): + # The carve-out: ARIZE_ENDPOINT with no Arize credentials is a legitimate self-hosted, + # no-auth collector authenticated via the standard OTLP headers env var. Degrading must + # not take that away, so the exporter stays and keeps carrying those headers. + from litellm.integrations.otel.model.config import ExporterOwner + from litellm.integrations.otel.presets.arize import arize_preset + + for var in ("ARIZE_SPACE_ID", "ARIZE_SPACE_KEY", "ARIZE_API_KEY", "ARIZE_HTTP_ENDPOINT"): + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv("ARIZE_ENDPOINT", "https://collector.internal/v1") + monkeypatch.setenv("OTEL_EXPORTER_OTLP_TRACES_HEADERS", "authorization=Bearer OPERATOR-COLLECTOR-SECRET") + + specs = [e for e in arize_preset(allow_missing_credentials=True).exporters if e.owner == ExporterOwner.ARIZE_AX] + assert len(specs) == 1 + assert specs[0].endpoint == "https://collector.internal/v1" + assert specs[0].headers == "authorization=Bearer OPERATOR-COLLECTOR-SECRET" + + +def test_arize_public_endpoint_spec_carries_no_operator_headers(monkeypatch): + # A global arize callback with no credentials still exports (additive parity), and the + # spec this preset contributes carries no headers of its own, so the operator's OTLP auth + # header is never written into an exporter aimed at Arize's public endpoint. + # + # This asserts the spec only. The OTLP SDK still reads OTEL_EXPORTER_OTLP_TRACES_HEADERS + # from the environment for any header-less exporter; that is standard OTLP behaviour that + # predates this PR and applies to every integration, so it is out of scope here. + from litellm.integrations.otel.model.config import ExporterOwner + from litellm.integrations.otel.presets.arize import ARIZE_PUBLIC_OTLP_ENDPOINT, arize_preset + + for var in ("ARIZE_SPACE_ID", "ARIZE_SPACE_KEY", "ARIZE_API_KEY", "ARIZE_ENDPOINT", "ARIZE_HTTP_ENDPOINT"): + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv("OTEL_EXPORTER_OTLP_TRACES_HEADERS", "authorization=Bearer OPERATOR-COLLECTOR-SECRET") + + specs = [e for e in arize_preset().exporters if e.owner == ExporterOwner.ARIZE_AX] + assert len(specs) == 1 + assert specs[0].endpoint == ARIZE_PUBLIC_OTLP_ENDPOINT + assert specs[0].headers is None + + +def test_phoenix_preset_without_config_appends_localhost_exporter(monkeypatch): + # Additive parity: unconfigured Phoenix defaults to http://localhost:6006 and + # the preset always contributes that exporter, so a local self-hosted Phoenix + # deployment with no env vars keeps receiving traces exactly as before the PR. + from litellm.integrations.otel.model.config import ExporterOwner + from litellm.integrations.otel.presets.phoenix import phoenix_preset + + for var in ("PHOENIX_API_KEY", "PHOENIX_COLLECTOR_ENDPOINT", "PHOENIX_COLLECTOR_HTTP_ENDPOINT"): + monkeypatch.delenv(var, raising=False) + cfg = phoenix_preset() + phoenix_exporters = [e for e in cfg.exporters if e.owner == ExporterOwner.ARIZE_PHOENIX] + assert len(phoenix_exporters) == 1 + assert phoenix_exporters[0].endpoint == "http://localhost:6006/v1/traces" def test_agentops_exporter_factory_is_registered(): assert _AGENTOPS_EXPORTER_KIND in providers._EXPORTER_FACTORIES -def test_dynamic_cred_presets_tag_exporter_with_matching_owner(monkeypatch): - """Each dynamic-credential preset must tag the exporter it contributes with - its own callback name, so per-request tenant routing - (``TenantTracerCache``) applies that integration's credentials only to its - own exporter and never bleeds them onto a co-configured backend. +def test_destination_routable_presets_tag_exporter_with_matching_owner(monkeypatch): + """Each destination-routable preset must tag the exporter it contributes with + its own callback name, so per-tenant routing (``TenantTracerCache``) points + that integration's admin destination at its own exporter only and never + rewrites a co-configured backend's exporter. """ - from litellm.integrations.otel.presets import ( - DYNAMIC_HEADERS_BY_CALLBACK, - PRESET_BY_CALLBACK, + from litellm.integrations.otel.presets.destinations import ( + OTEL_V2_DESTINATION_CALLBACKS, ) + from litellm.integrations.otel.presets import PRESET_BY_CALLBACK monkeypatch.setenv("ARIZE_SPACE_ID", "S") monkeypatch.setenv("ARIZE_API_KEY", "K") @@ -65,7 +168,7 @@ def test_dynamic_cred_presets_tag_exporter_with_matching_owner(monkeypatch): from litellm.integrations.otel.model.config import ExporterOwner - for callback_name in DYNAMIC_HEADERS_BY_CALLBACK: + for callback_name in OTEL_V2_DESTINATION_CALLBACKS: cfg = PRESET_BY_CALLBACK[callback_name]() owners = {e.owner for e in cfg.exporters} assert ExporterOwner(callback_name) in owners, ( @@ -160,3 +263,107 @@ def test_agentops_endpoint_points_at_live_host(): # so a typo or stale domain can never ship again. assert _AGENTOPS_ENDPOINT == "https://otlp.agentops.ai/v1/traces" assert "agentops.cloud" not in _AGENTOPS_ENDPOINT + + +def test_credential_mandatory_presets_raise_without_creds_by_default(monkeypatch): + # weave/langfuse/levo are credential-mandatory: with no global env creds and no + # opt-in to degrade, the preset must RAISE so a misconfigured global callback + # (e.g. ``callbacks: ["weave_otel"]`` with no keys) fails loud at startup, the + # same error story as before V2 landed. _maybe_construct_otel_v2 relies on this + # raise to defer to the legacy path. + from litellm.integrations.otel.presets.langfuse import langfuse_preset + from litellm.integrations.otel.presets.levo import levo_preset + from litellm.integrations.otel.presets.weave import weave_preset + + for var in ( + "WANDB_API_KEY", + "WANDB_PROJECT_ID", + "LANGFUSE_PUBLIC_KEY", + "LANGFUSE_SECRET_KEY", + "LEVOAI_API_KEY", + ): + monkeypatch.delenv(var, raising=False) + + for preset in (weave_preset, langfuse_preset, levo_preset): + with pytest.raises(Exception): + preset() + + +def test_credential_mandatory_presets_degrade_when_allowed(monkeypatch): + # When an admin-owned destination is the reason for construction it carries its + # own per-tenant credentials, so the preset is called with + # allow_missing_credentials=True and must degrade to an exporter-less config + # (keeping its mappers) instead of raising -- otherwise the destination never gets + # a v2 logger and its gen-AI span falls to the generic global logger. + from litellm.integrations.otel.model.config import ExporterOwner + from litellm.integrations.otel.presets.langfuse import langfuse_preset + from litellm.integrations.otel.presets.levo import levo_preset + from litellm.integrations.otel.presets.weave import weave_preset + + for var in ( + "WANDB_API_KEY", + "WANDB_PROJECT_ID", + "LANGFUSE_PUBLIC_KEY", + "LANGFUSE_SECRET_KEY", + "LEVOAI_API_KEY", + ): + monkeypatch.delenv(var, raising=False) + + for preset, owner in ( + (weave_preset, ExporterOwner.WEAVE_OTEL), + (langfuse_preset, ExporterOwner.LANGFUSE_OTEL), + (levo_preset, ExporterOwner.LEVO), + ): + cfg = preset(allow_missing_credentials=True) # must not raise + assert [e for e in cfg.exporters if e.owner == owner] == [] + + +def test_weave_preset_with_creds_contributes_exporter(monkeypatch): + from litellm.integrations.otel.model.config import ExporterOwner + from litellm.integrations.otel.presets.weave import weave_preset + + monkeypatch.setenv("WANDB_API_KEY", "w-key") + monkeypatch.setenv("WANDB_PROJECT_ID", "entity/project") + cfg = weave_preset() + assert [e for e in cfg.exporters if e.owner == ExporterOwner.WEAVE_OTEL] != [] + + +def test_generic_preset_needs_no_global_env_and_emits_genai(monkeypatch): + # Acceptance #2: the generic preset is vendor-neutral and admin-destination-only -- + # it must build with NO global OTEL env vars, never raise, carry the standard genai + # (+legacy) mappers, and contribute NO vendor exporter (the per-destination exporter + # is appended by the router). + from litellm.integrations.otel.presets.generic import generic_preset + + for var in ("OTEL_EXPORTER_OTLP_ENDPOINT", "OTEL_ENDPOINT", "OTEL_EXPORTER"): + monkeypatch.delenv(var, raising=False) + cfg = generic_preset() # must not raise, no env needed + assert "genai" in cfg.mapper_names + # A credential-less generic config must contribute NO exporter at all. In particular + # it must not degrade to the default console exporter, which prints every span + # (including prompt/completion content) to stdout synchronously on the request path. + assert cfg.exporters == [] + # the degrade flag is accepted (Preset protocol) and irrelevant -- still builds + assert generic_preset(allow_missing_credentials=True).exporters == [] + + +def test_bare_and_degraded_configs_do_not_console_flood(monkeypatch): + # The "nothing configured" degrade case (bare config, or a credential-mandatory + # preset degrading with allow_missing_credentials) must export nothing rather than + # folding the default console exporter in and dumping every span to stdout. + from litellm.integrations.otel.model.config import OpenTelemetryV2Config + from litellm.integrations.otel.presets.langfuse import langfuse_preset + from litellm.integrations.otel.presets.weave import weave_preset + + assert OpenTelemetryV2Config().exporters == [] + # An explicit endpoint (a real destination) still folds into one OTLP exporter. + assert len(OpenTelemetryV2Config(endpoint="http://collector/v1/traces").exporters) == 1 + # A deployment that *explicitly* selects console output still gets it -- only the + # default/degrade console (exporter left unset) is suppressed. + explicit_console = OpenTelemetryV2Config(exporter="console") + assert len(explicit_console.exporters) == 1 and explicit_console.exporters[0].kind == "console" + + for var in ("LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY", "WANDB_API_KEY", "WANDB_PROJECT_ID"): + monkeypatch.delenv(var, raising=False) + assert langfuse_preset(allow_missing_credentials=True).exporters == [] + assert weave_preset(allow_missing_credentials=True).exporters == [] diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 28a6c8dd18d..c6bc38bcd89 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -4156,6 +4156,175 @@ def test_failure_handler_zeroes_spend_without_recovered_usage(logging_obj): assert payload["total_tokens"] == 0 +def test_admin_owned_destination_does_not_activate_v2_without_flag(monkeypatch): + # LITELLM_OTEL_V2 is the sole activation gate: registering an admin-owned logging + # destination must NOT flip a v1 deployment onto v2. With the flag off, + # _maybe_construct_otel_v2 returns None whether or not a destination exists, so an + # existing v1 deployment is unaffected by merely registering a credential (and the + # flag-off + destination "orphaned tree" configuration can't arise). + from types import SimpleNamespace + + from litellm.integrations.otel.model.config import is_otel_v2_enabled + from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 + + monkeypatch.delenv("LITELLM_OTEL_V2", raising=False) + is_otel_v2_enabled.cache_clear() + assert is_otel_v2_enabled() is False + + # No logging credential for the backend -> legacy fallback (None). + monkeypatch.setattr(litellm, "credential_list", []) + assert _maybe_construct_otel_v2("arize", []) is None + + # A logging destination registered for the backend, flag still off -> still None. + monkeypatch.setattr( + litellm, + "credential_list", + [ + SimpleNamespace( + credential_name="arize-poc", + credential_info={ + "credential_type": "logging", + "description": "arize", + }, + ) + ], + ) + result = _maybe_construct_otel_v2("arize", []) + is_otel_v2_enabled.cache_clear() + assert result is None + + +def test_credential_mandatory_backend_global_misconfig_stays_loud(monkeypatch): + # Regression: weave/langfuse/levo are credential-mandatory; before V2 a global + # callback (e.g. ``callbacks: ["weave_otel"]``) with no credentials failed loud at + # startup. _maybe_construct_otel_v2 preserves that: with no creds the preset raises, + # _maybe_construct returns None, and the caller falls through to the legacy path. + # + # An admin-owned destination does not change that answer. Ownership is the operator's + # configuration alone, so registering a destination for one team cannot move any other + # tenant off the logger they already had; the destination is delivered to separately by + # AdminDestinationLogger. + from types import SimpleNamespace + + from litellm.integrations.otel.model.config import is_otel_v2_enabled + from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 + + for var in ("WANDB_API_KEY", "WANDB_PROJECT_ID"): + monkeypatch.delenv(var, raising=False) + + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + + monkeypatch.setattr(litellm, "credential_list", []) + assert _maybe_construct_otel_v2("weave_otel", []) is None + + monkeypatch.setattr( + litellm, + "credential_list", + [ + SimpleNamespace( + credential_name="wb-poc", + credential_values={"wandb_api_key": "wb-key"}, + credential_info={ + "credential_type": "logging", + "description": "weave_otel", + "access": {"teams": ["team-a"]}, + }, + ) + ], + ) + assert _maybe_construct_otel_v2("weave_otel", []) is None + is_otel_v2_enabled.cache_clear() + + +@pytest.mark.parametrize( + "credential_info, credential_values, why", + [ + ({"credential_type": "logging", "description": "weave_otel"}, {"wandb_api_key": "k"}, "no access at all"), + ( + {"credential_type": "logging", "description": "weave_otel", "access": {}}, + {"wandb_api_key": "k"}, + "empty access grants nobody", + ), + ( + { + "credential_type": "logging", + "description": "weave_otel", + "access": {"global": False, "teams": [], "orgs": []}, + }, + {"wandb_api_key": "k"}, + "explicitly revoked", + ), + ( + {"credential_type": "logging", "description": "weave_otel", "access": {"teams": ["team-a"]}}, + {}, + "granted but unbuildable", + ), + ], +) +def test_inert_destination_does_not_degrade_the_backend(monkeypatch, credential_info, credential_values, why): + # Regression: relaxing the preset's missing-credentials check changes how the backend + # is built for EVERY request on the proxy. A destination that routes to nobody -- no + # access, empty access, revoked access, or values that build no destination -- must + # not trigger it. Otherwise registering an inert row degrades the backend proxy-wide + # and silently drops the exports of teams carrying their own callback_vars for it. + from types import SimpleNamespace + + from litellm.integrations.otel.model.config import is_otel_v2_enabled + from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 + + for var in ("WANDB_API_KEY", "WANDB_PROJECT_ID"): + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + + monkeypatch.setattr( + litellm, + "credential_list", + [ + SimpleNamespace( + credential_name="inert", + credential_values=credential_values, + credential_info=credential_info, + ) + ], + ) + result = _maybe_construct_otel_v2("weave_otel", []) + is_otel_v2_enabled.cache_clear() + assert result is None, f"{why}: an inert destination must not relax the credential check" + + +def test_generic_admin_destination_needs_flag_to_build_otel_v2_logger(monkeypatch): + # The Generic OTLP passthrough ('generic') builds an OpenTelemetryV2 logger only + # when LITELLM_OTEL_V2 is on. Registering an admin-owned generic destination with + # the flag off must NOT construct a v2 logger (the flag is the sole activation + # gate); with the flag on it does. + from types import SimpleNamespace + + from litellm.integrations.otel.logger import OpenTelemetryV2 + from litellm.integrations.otel.model.config import is_otel_v2_enabled + from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 + + generic_dest = [ + SimpleNamespace( + credential_name="ui-generic", + credential_info={"credential_type": "logging", "description": "generic"}, + ) + ] + + # Flag off + destination registered -> still None (no v2, no flip onto v2). + monkeypatch.delenv("LITELLM_OTEL_V2", raising=False) + is_otel_v2_enabled.cache_clear() + monkeypatch.setattr(litellm, "credential_list", generic_dest) + assert _maybe_construct_otel_v2("generic", []) is None + + # Flag on + destination registered -> builds the v2 generic logger. + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + logger = _maybe_construct_otel_v2("generic", []) + is_otel_v2_enabled.cache_clear() + assert isinstance(logger, OpenTelemetryV2) + assert logger.callback_name == "generic" def test_set_cost_breakdown_stores_reasoning_cost(): """reasoning_cost is stored only when positive, mirroring the cache-cost fields.""" from datetime import datetime @@ -4539,3 +4708,280 @@ async def test_restore_correlation_context_works_across_asyncio_task_boundary(): finally: trace_id_var.set("") session_id_var.set("") + + +def test_zero_config_v2_warns_instead_of_going_silently_dark(monkeypatch, caplog): + """Regression: v2 stopped folding a console exporter into the nothing-configured case, + which is right (it printed every span, prompt content included, on the request path) + but left an operator with no exporter and no signal. Base printed to stdout; head must + at least say so, or the deployment is silently dark. + """ + import logging + + from litellm.integrations.otel.model.config import is_otel_v2_enabled + from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 + + # every alias the config reads, not just the short names -- the OTEL_EXPORTER_OTLP_* + # spellings are equally load-bearing and leak in from neighbouring suites + for var in ( + "OTEL_EXPORTER", + "OTEL_EXPORTER_OTLP_PROTOCOL", + "OTEL_ENDPOINT", + "OTEL_EXPORTER_OTLP_ENDPOINT", + "OTEL_HEADERS", + "OTEL_EXPORTER_OTLP_HEADERS", + ): + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + monkeypatch.setattr(litellm, "credential_list", []) + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + logger = _maybe_construct_otel_v2("generic", []) + is_otel_v2_enabled.cache_clear() + + assert logger is not None + assert logger.config.exporters == [] + assert any("no exporter is configured" in record.getMessage() for record in caplog.records) + + +def test_explicit_otel_callback_keeps_the_documented_console_default(monkeypatch): + """Regression: ``callbacks: ["otel"]`` with no endpoint must still export. + + The console fold is suppressed for a preset that degraded because it found no + credentials, which is the right call; the operator never asked for stdout there. An + operator who lists ``otel`` and sets no endpoint did ask for it, and the published docs + give ``console`` as the ``OTEL_EXPORTER`` default. Suppressing both left that + deployment silently dark with no warning. + """ + from litellm.integrations.otel.model.config import is_otel_v2_enabled + from litellm.integrations.otel.presets.generic import generic_preset + from litellm.litellm_core_utils.litellm_logging import _init_custom_logger_compatible_class + + for var in ("OTEL_EXPORTER", "OTEL_ENDPOINT", "OTEL_EXPORTER_OTLP_ENDPOINT", "OTEL_EXPORTER_OTLP_PROTOCOL"): + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + + logger = _init_custom_logger_compatible_class("otel", internal_usage_cache=None, llm_router=None) + is_otel_v2_enabled.cache_clear() + + assert logger is not None + kinds = [spec.kind for spec in logger.config.exporters] + assert kinds == ["console"], "an explicit otel callback with no endpoint must keep the documented console default" + assert generic_preset(allow_missing_credentials=True).exporters == [], "preset degrade must stay suppressed" + + +def test_destination_for_one_team_does_not_move_another_tenant_off_its_logger(monkeypatch): + """Regression: an admin destination must not change who owns a backend. + + ``_has_admin_owned_logging_destination`` answered "does a granting row exist anywhere" + and fed that into the preset's missing-credentials check, which decides whether v2 or + the legacy logger owns the backend for the whole process. Registering a destination + scoped to one team therefore moved every other tenant onto v2, and their traces lost + prompt content because v2 defaults to NO_CONTENT. + """ + from types import SimpleNamespace + + from litellm.integrations.langfuse.langfuse_otel import LangfuseOtelLogger + from litellm.integrations.otel.model.config import is_otel_v2_enabled + from litellm.litellm_core_utils.litellm_logging import _init_custom_logger_compatible_class + + for var in ("LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY", "LANGFUSE_HOST"): + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + + granted_to_someone_else = SimpleNamespace( + credential_name="lf-team-a", + credential_values={"langfuse_public_key": "pk", "langfuse_secret_key": "sk"}, + credential_info={ + "credential_type": "logging", + "description": "langfuse_otel", + "access": {"teams": ["team-a"]}, + }, + ) + + monkeypatch.setattr(litellm, "credential_list", []) + baseline = _init_custom_logger_compatible_class("langfuse_otel", internal_usage_cache=None, llm_router=None) + + monkeypatch.setattr(litellm, "credential_list", [granted_to_someone_else]) + with_destination = _init_custom_logger_compatible_class("langfuse_otel", internal_usage_cache=None, llm_router=None) + is_otel_v2_enabled.cache_clear() + + assert isinstance(baseline, LangfuseOtelLogger) + assert type(with_destination) is type(baseline), "another team's destination must not change this backend's owner" + + +def test_admin_destination_does_not_build_a_backend_logger(monkeypatch): + """A registered destination must not take a backend over. + + Ownership is the operator's configuration alone. A destination is delivered to by + ``AdminDestinationLogger``, whose per-backend emitter carries the backend's span + vocabulary and no exporter of its own: the preset's own exporter belongs to whichever + logger the operator configured, so including it here would export the call twice. + """ + from types import SimpleNamespace + + from litellm.integrations.otel.destination_logger import AdminDestinationLogger + from litellm.integrations.otel.model.config import is_otel_v2_enabled + from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 + + for var in ("WANDB_API_KEY", "WANDB_PROJECT_ID"): + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + monkeypatch.setattr( + litellm, + "credential_list", + [ + SimpleNamespace( + credential_name="wb", + credential_values={"wandb_api_key": "k"}, + credential_info={ + "credential_type": "logging", + "description": "weave_otel", + "access": {"teams": ["team-a"]}, + }, + ) + ], + ) + + assert _maybe_construct_otel_v2("weave_otel", []) is None + is_otel_v2_enabled.cache_clear() + + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-global") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-global") + emitter = AdminDestinationLogger()._emitter_for("langfuse_otel") + assert emitter.callback_name == "langfuse_otel" + assert list(emitter.config.exporters) == [], "the sink must not inherit the preset's own exporter" + + +def test_sink_skips_a_backend_an_otel_v2_logger_already_owns(monkeypatch): + """Regression: a destination whose backend is also a configured callback received the + gen-AI span twice, as two sibling spans under one parent. + + The owning ``OpenTelemetryV2`` already fans its span out to that backend's + destinations, so the sink emitting as well is a pure duplicate; ``_closed_call_ids`` + is per-instance and cannot dedupe across the two emitters. + """ + from litellm.integrations.otel import destination_logger as sink_module + from litellm.integrations.otel.destination_logger import AdminDestinationLogger + from litellm.integrations.otel.logger import OpenTelemetryV2 + from litellm.integrations.otel.model.config import OpenTelemetryV2Config + from litellm.litellm_core_utils import litellm_logging as logging_module + + owned = OpenTelemetryV2(config=OpenTelemetryV2Config(), callback_name="langfuse_otel") + monkeypatch.setattr(logging_module, "_in_memory_loggers", [owned]) + + class _Dest: + def __init__(self, name): + self.callback_name = name + + monkeypatch.setattr( + sink_module, + "request_destinations", + lambda: (_Dest("langfuse_otel"), _Dest("generic")), + ) + exported = [] + sink = AdminDestinationLogger() + monkeypatch.setattr( + sink, + "_emitter_for", + lambda backend: type( + "_E", (), {"export_to_destinations": lambda _self, *a: exported.append(backend)} + )(), + ) + + sink._export({}, None, None) + + assert exported == ["generic"], "the owned backend is the owning logger's to deliver" + + +def test_sink_does_not_close_an_mcp_event_as_an_llm_call(): + """Regression: the sink called ``_close_llm_call`` directly, bypassing the MCP + dispatch. A ``tools/call`` reached the destination named ``execute_tool MCP: + -`` with the tool smuggled into ``gen_ai.request.model``, fabricated + zero-token usage, and none of the MCP semconv attributes; the correct span never + arrived at all. + """ + from litellm.integrations.otel.destination_logger import _DestinationOnlyOtel + from litellm.integrations.otel.model.config import OpenTelemetryV2Config + + sink = _DestinationOnlyOtel(config=OpenTelemetryV2Config(), callback_name="generic") + calls = [] + sink._emit_mcp_tool_call = lambda *a: calls.append("tool_call") or True # type: ignore[method-assign] + sink._close_llm_call = lambda *a: calls.append("llm_call") # type: ignore[method-assign] + + sink.export_to_destinations({}, None, None) + + assert calls == ["tool_call"], "an MCP tool call must never be closed as an LLM call" + + +def test_sink_claims_list_tools_without_emitting_a_second_span(): + """``tools/list`` carries no ``gen_ai.operation.name``, so the fan-out span processor + already routes the owning logger's span to the destination. The sink must claim the + event (so it is not closed as an LLM call) without emitting its own duplicate.""" + from litellm.integrations.otel.destination_logger import _DestinationOnlyOtel + from litellm.integrations.otel.model.config import OpenTelemetryV2Config + + sink = _DestinationOnlyOtel(config=OpenTelemetryV2Config(), callback_name="generic") + emitted = [] + sink._emitter.emit = lambda *a, **k: emitted.append(a) # type: ignore[method-assign] + payload = {"call_type": "list_mcp_tools", "id": "abc"} + + handled = sink._emit_mcp_list_tools({"standard_logging_object": payload}, None, None) + + assert handled is True + assert emitted == [], "the fan-out processor already delivers tools/list" + + +def test_sink_never_layers_a_tenant_credential_tracer_over_the_fan_out(): + """The tenant's own backend account is the owning logger's to reach. If the sink + passed the request's ``dynamic_params`` through, ``genai_tracers_for`` would add a + credential-scoped tracer and export the call to that account a second time.""" + from litellm.integrations.otel.destination_logger import _DestinationOnlyOtel + from litellm.integrations.otel.logger import OpenTelemetryV2 + from litellm.integrations.otel.model.config import OpenTelemetryV2Config + from litellm.integrations.otel.model.metadata import LLMCallEvent + + call = LLMCallEvent.from_dict({"standard_callback_dynamic_params": {"langfuse_public_key": "pk"}}) + owning = OpenTelemetryV2(config=OpenTelemetryV2Config(), callback_name="langfuse_otel") + sink = _DestinationOnlyOtel(config=OpenTelemetryV2Config(), callback_name="langfuse_otel") + + assert owning._tracer_dynamic_params(call) is call.dynamic_params + assert sink._tracer_dynamic_params(call) is None + + +def test_importing_litellm_does_not_require_opentelemetry(): + """``opentelemetry`` ships only in the proxy extras, so a plain SDK install does not + have it. Importing the OTEL v2 logger at module scope made ``import litellm`` raise + ModuleNotFoundError for every one of those users; CI never sees it because the install + jobs sync all extras. Run in a child process with the package blocked at import time.""" + import subprocess + import textwrap + + program = textwrap.dedent( + """ + import sys + + class _Blocker: + def find_spec(self, name, path=None, target=None): + if name == "opentelemetry" or name.startswith("opentelemetry."): + raise ModuleNotFoundError(f"No module named '{name}'") + return None + + sys.meta_path.insert(0, _Blocker()) + import litellm # noqa: F401 + print("IMPORT_OK") + """ + ) + repo_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) + result = subprocess.run( + [sys.executable, "-c", program], + capture_output=True, + text=True, + timeout=300, + env={**os.environ, "PYTHONPATH": repo_root}, + ) + assert "IMPORT_OK" in result.stdout, f"import litellm failed without opentelemetry:\n{result.stderr[-3000:]}" diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index e0994f88d65..f8664f06853 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -1,3 +1,4 @@ +import inspect import asyncio import copy import json @@ -6820,6 +6821,28 @@ async def test_resolve_logging_exporters_noop_when_flag_off(monkeypatch): assert backends == () +@pytest.mark.asyncio +async def test_mcp_auth_chokepoint_refreshes_destinations_per_message(monkeypatch): + """Regression: a stateful MCP session outlived an admin's re-scope of a destination. + + Every JSON-RPC message after ``initialize`` is dispatched on a descendant of the task + that POST spawned, and a ContextVar is copied at task creation, so the destinations + resolved for the first message stayed frozen for the session's life. A destination + re-scoped to another team kept receiving the original team's spans until the client + disconnected. The shared auth chokepoint now re-resolves per message. + """ + import litellm.proxy._experimental.mcp_server.server as mcp_server + + src = inspect.getsource(mcp_server) + assert "_refresh_request_otel_destinations" in src + # It must hang off the one helper every JSON-RPC handler funnels through, not off a + # single handler; tools/list, get_prompt and read_resource all leaked precisely + # because only the tool-call path re-resolved. + chokepoint = src[src.index("async def get_or_extract_auth_context"):] + chokepoint = chokepoint[: chokepoint.index("def get_active_mcp_session")] + assert "await _refresh_request_otel_destinations(user_api_key_auth)" in chokepoint + + @pytest.mark.asyncio async def test_apply_admin_logging_exporters_needs_no_opentelemetry_when_flag_off(monkeypatch): """opentelemetry ships only in the proxy-runtime extra, so a litellm[proxy] install