mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-12 23:01:41 +00:00
feat(otel): export the trace to the resolved destinations
Fans each request's trace out to the destinations its identity resolved, applies per-request credentials only to the exporter their own backend contributed, and caches tracer providers per credential, endpoint and transport. Squashed onto the current staging tip; the previous history conflicted with staging's Final-annotation pass over the same OTEL v2 modules.
This commit is contained in:
Yucheng Zhu
parent
4588c3b3f0
commit
93061061fa
31 changed files with 3619 additions and 363 deletions
|
|
@ -137,6 +137,7 @@ _custom_logger_compatible_callbacks_literal = Literal[
|
|||
"langfuse",
|
||||
"langfuse_otel",
|
||||
"weave_otel",
|
||||
"generic",
|
||||
"pagerduty",
|
||||
"humanloop",
|
||||
"azure_sentinel",
|
||||
|
|
|
|||
|
|
@ -278,12 +278,13 @@ lives in [`plumbing/`](./plumbing):
|
|||
- [`presets/`](./presets) — each preset reads one integration's env vars and
|
||||
returns an `OpenTelemetryV2Config` (exporter destination + mapper vocabularies
|
||||
+ resource attributes). `PRESET_BY_CALLBACK` maps a callback name (`"arize"`,
|
||||
`"langfuse_otel"`, …) to its preset. Integrations that support team/key-scoped
|
||||
credentials also provide a per-request OTLP header builder
|
||||
(`DYNAMIC_HEADERS_BY_CALLBACK`). Presets do **no** network I/O at build time:
|
||||
AgentOps, for example, mints its JWT lazily inside a custom exporter on the
|
||||
first export (in the `BatchSpanProcessor` worker thread), never on the event
|
||||
loop.
|
||||
`"langfuse_otel"`, …) to its preset. Per-key/team routing is **not** here: a
|
||||
destination is admin-owned infrastructure, resolved server-side from a named
|
||||
credential into an `OtelDestination` (`destinations.py`) and applied by
|
||||
`plumbing/routing.py`. Nothing in `presets/` reads vendor credentials or a host
|
||||
off a request. Presets do **no** network I/O at build time: AgentOps, for
|
||||
example, mints its JWT lazily inside a custom exporter on the first export (in
|
||||
the `BatchSpanProcessor` worker thread), never on the event loop.
|
||||
|
||||
## Extending
|
||||
|
||||
|
|
@ -295,7 +296,8 @@ lives in [`plumbing/`](./plumbing):
|
|||
constructor, so the family stays bounded span-wide rather than per vocabulary.
|
||||
- **A new integration**: add a preset in `presets/` that returns an
|
||||
`OpenTelemetryV2Config`, and register it in `presets/__init__.PRESET_BY_CALLBACK`.
|
||||
If it supports dynamic credentials, add a header builder to
|
||||
`DYNAMIC_HEADERS_BY_CALLBACK`.
|
||||
For admin-owned per-key/team destinations, add an adapter mapping the named
|
||||
credential's values to an `OtelDestination` in `destinations._ADAPTERS` (or rely
|
||||
on the generic `otel_endpoint`/`otel_headers` passthrough).
|
||||
- **A new span kind**: add a role to `spans.py` (registry entry + name builder),
|
||||
a payload dataclass in `payloads.py`, and a branch in the relevant mapper(s).
|
||||
|
|
|
|||
171
litellm/integrations/otel/destination_logger.py
Normal file
171
litellm/integrations/otel/destination_logger.py
Normal file
|
|
@ -0,0 +1,171 @@
|
|||
"""Export to admin-owned destinations, independently of which logger owns a backend.
|
||||
|
||||
A destination is a sink, never a reason to change ownership. Whether ``OpenTelemetryV2``
|
||||
or a legacy logger owns a backend is decided by the operator's own configuration; this
|
||||
logger delivers the gen-AI span to whichever destinations the request resolved, so
|
||||
registering a destination for one team cannot change any other tenant's pipeline.
|
||||
|
||||
The span's vocabulary comes from the backend's preset called purely for its mappers and
|
||||
semconv, with its exporters stripped: the preset's own exporter belongs to the backend's
|
||||
owning logger, and including it here would export the same call twice.
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from datetime import datetime
|
||||
from functools import lru_cache
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from opentelemetry.trace import Span
|
||||
|
||||
import litellm
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.integrations.otel.logger import OpenTelemetryV2
|
||||
from litellm.integrations.otel.model.config import OpenTelemetryV2Config
|
||||
from litellm.integrations.otel.model.payloads import is_mcp_list_tools
|
||||
from litellm.integrations.otel.plumbing.context import request_destinations
|
||||
from litellm.integrations.otel.presets import PRESET_BY_CALLBACK
|
||||
from litellm.litellm_core_utils.litellm_logging import otel_v2_owned_backends
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.integrations.otel.model.destination import OtelDestination
|
||||
from litellm.integrations.otel.model.event import LLMCallEvent
|
||||
from litellm.types.utils import StandardCallbackDynamicParams, StandardLoggingPayload
|
||||
|
||||
|
||||
def _vocabulary_config(backend: str) -> OpenTelemetryV2Config:
|
||||
"""The backend's span vocabulary with no exporter of its own."""
|
||||
preset_fn = PRESET_BY_CALLBACK.get(backend)
|
||||
if preset_fn is None:
|
||||
return OpenTelemetryV2Config()
|
||||
try:
|
||||
config = preset_fn(allow_missing_credentials=True)
|
||||
except Exception: # noqa: BLE001 # an unbuildable preset still has a usable default vocabulary
|
||||
return OpenTelemetryV2Config()
|
||||
return config.model_copy(update={"exporters": ()})
|
||||
|
||||
|
||||
class _DestinationOnlyOtel(OpenTelemetryV2):
|
||||
"""An ``OpenTelemetryV2`` that reaches admin destinations and nothing else.
|
||||
|
||||
It stays out of the proxy's global callback lists, and drops the request's own
|
||||
``callback_vars`` credentials: those name the tenant's account, which the backend's
|
||||
owning logger already exports to.
|
||||
"""
|
||||
|
||||
def _init_otel_logger_on_litellm_proxy(self) -> None:
|
||||
return None
|
||||
|
||||
def _emit_deferred_llm_call(
|
||||
self,
|
||||
payload: "StandardLoggingPayload",
|
||||
destinations: "tuple[OtelDestination, ...]",
|
||||
start_time_ns: int | None,
|
||||
end_time_ns: int | None,
|
||||
time_to_first_chunk_seconds: float | None = None,
|
||||
dynamic_params: "StandardCallbackDynamicParams | None" = None,
|
||||
) -> Span | None:
|
||||
return super()._emit_deferred_llm_call(
|
||||
payload,
|
||||
destinations,
|
||||
start_time_ns,
|
||||
end_time_ns,
|
||||
time_to_first_chunk_seconds,
|
||||
None,
|
||||
)
|
||||
|
||||
def _tracer_dynamic_params(self, call: "LLMCallEvent") -> "StandardCallbackDynamicParams | None":
|
||||
return None
|
||||
|
||||
def _emit_mcp_list_tools(
|
||||
self,
|
||||
kwargs: "Mapping[str, object]",
|
||||
start_time: "datetime | float | None",
|
||||
end_time: "datetime | float | None",
|
||||
) -> bool:
|
||||
"""Claim the event without emitting.
|
||||
|
||||
A ``tools/list`` span carries no ``gen_ai.operation.name``, so the fan-out span
|
||||
processor already routes the owning logger's span to the request's destinations.
|
||||
Emitting a second one here would deliver the same discovery call twice; returning
|
||||
``True`` still stops the caller from closing it as an LLM call.
|
||||
"""
|
||||
raw_payload = kwargs.get("standard_logging_object")
|
||||
return isinstance(raw_payload, Mapping) and is_mcp_list_tools(raw_payload)
|
||||
|
||||
def export_to_destinations(
|
||||
self,
|
||||
kwargs: "Mapping[str, Any]",
|
||||
start_time: "datetime | float | None",
|
||||
end_time: "datetime | float | None",
|
||||
) -> None:
|
||||
"""Mirrors ``async_log_success_event``'s dispatch order.
|
||||
|
||||
An MCP event is not an LLM call: closing it as one names the span from the LLM
|
||||
vocabulary (``chat MCP: list_tools``, ``execute_tool MCP: <server>-<tool>``),
|
||||
drops every MCP semconv attribute, and stamps fabricated zero-token usage on it.
|
||||
"""
|
||||
if self._emit_mcp_tool_call(kwargs, start_time, end_time):
|
||||
return
|
||||
if self._emit_mcp_list_tools(kwargs, start_time, end_time):
|
||||
return
|
||||
self._close_llm_call(kwargs, start_time, end_time)
|
||||
|
||||
|
||||
class AdminDestinationLogger(CustomLogger):
|
||||
"""Delivers each request's gen-AI span to the destinations its identity resolved."""
|
||||
|
||||
def __init__(self, **kwargs: Any) -> None:
|
||||
super().__init__(**kwargs)
|
||||
self._emitters: dict[str, _DestinationOnlyOtel] = {} # mutable-ok: bounded per-backend emitter cache
|
||||
|
||||
def _emitter_for(self, backend: str) -> _DestinationOnlyOtel:
|
||||
existing = self._emitters.get(backend)
|
||||
if existing is not None:
|
||||
return existing
|
||||
emitter = _DestinationOnlyOtel(config=_vocabulary_config(backend), callback_name=backend)
|
||||
self._emitters[backend] = emitter
|
||||
return emitter
|
||||
|
||||
def _export(
|
||||
self,
|
||||
kwargs: "Mapping[str, Any]",
|
||||
start_time: "datetime | float | None",
|
||||
end_time: "datetime | float | None",
|
||||
) -> None:
|
||||
owned = otel_v2_owned_backends()
|
||||
for backend in sorted({d.callback_name for d in request_destinations() if d.callback_name} - owned):
|
||||
try:
|
||||
self._emitter_for(backend).export_to_destinations(kwargs, start_time, end_time)
|
||||
except Exception as exc: # noqa: BLE001 # one destination's failure must not break the request or the others
|
||||
litellm.verbose_logger.debug("OTel V2 destination export for %s failed: %s", backend, exc)
|
||||
|
||||
async def async_log_success_event(
|
||||
self,
|
||||
kwargs: "Mapping[str, Any]",
|
||||
response_obj: Any,
|
||||
start_time: "datetime | float | None",
|
||||
end_time: "datetime | float | None",
|
||||
) -> None:
|
||||
self._export(kwargs, start_time, end_time)
|
||||
|
||||
async def async_log_failure_event(
|
||||
self,
|
||||
kwargs: "Mapping[str, Any]",
|
||||
response_obj: Any,
|
||||
start_time: "datetime | float | None",
|
||||
end_time: "datetime | float | None",
|
||||
) -> None:
|
||||
self._export(kwargs, start_time, end_time)
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def admin_destination_logger() -> AdminDestinationLogger:
|
||||
return AdminDestinationLogger()
|
||||
|
||||
|
||||
def register_admin_destination_logger() -> None:
|
||||
"""Put the destination sink on the proxy's async callback lists, once."""
|
||||
sink = admin_destination_logger()
|
||||
for bucket in (litellm._async_success_callback, litellm._async_failure_callback):
|
||||
if not any(callback is sink for callback in bucket):
|
||||
bucket.append(sink)
|
||||
|
|
@ -214,6 +214,54 @@ class SpanEmitter:
|
|||
self.finish_span(role, span, data, end_time_ns=end_time_ns)
|
||||
return span
|
||||
|
||||
def emit_fanout(
|
||||
self,
|
||||
role: SpanRole,
|
||||
data: SpanData,
|
||||
parent_context: Context | None = None,
|
||||
*,
|
||||
start_time_ns: int | None = None,
|
||||
end_time_ns: int | None = None,
|
||||
tracers: Sequence[Tracer],
|
||||
links: Sequence[Link] | None = None,
|
||||
) -> Span | None:
|
||||
"""Emit one logical span once per tracer, deduping the call ONCE.
|
||||
|
||||
A backend that selects its target from the Resource (Arize's project) needs a
|
||||
separately-tagged span per Resource group, so ``tracers`` is one tracer per group
|
||||
(from ``TenantTracerCache.tracers_for``). Dedup runs once on the call id so a
|
||||
sync+async double-firing still coalesces, then a span is started and finished on
|
||||
each tracer (each provider stamps its own Resource). Returns the first span (or
|
||||
``None`` if deduped/empty).
|
||||
"""
|
||||
# A router retry reuses one ``litellm_call_id`` across every attempt, so the call id
|
||||
# alone coalesced the successful attempt into the failed one it replaced and the
|
||||
# retried call reached its destinations only as the failure. The response id splits
|
||||
# the attempts apart while a sync+async double-firing of one attempt still shares it.
|
||||
attempt_id = data.response_id if isinstance(data, LLMCallSpanData) else None
|
||||
dedup_key = (
|
||||
(f"{data.identity.call_id}:{attempt_id}" if attempt_id else data.identity.call_id)
|
||||
if isinstance(data, (LLMCallSpanData, MCPToolCallSpanData))
|
||||
else None
|
||||
)
|
||||
if self._seen(dedup_key, role):
|
||||
return None
|
||||
name = _NAME_BUILDERS[role](data)
|
||||
first: Span | None = None
|
||||
for tracer in tracers:
|
||||
span = self.start_span(
|
||||
role,
|
||||
name,
|
||||
parent_context=parent_context,
|
||||
start_time_ns=start_time_ns,
|
||||
tracer=tracer,
|
||||
links=links,
|
||||
)
|
||||
self.finish_span(role, span, data, end_time_ns=end_time_ns)
|
||||
if first is None:
|
||||
first = span
|
||||
return first
|
||||
|
||||
def finish_span(
|
||||
self,
|
||||
role: SpanRole,
|
||||
|
|
|
|||
|
|
@ -62,8 +62,10 @@ from litellm.integrations.otel.plumbing.providers import (
|
|||
from litellm.integrations.otel.plumbing.routing import TenantTracerCache
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.integrations.otel.model.destination import OtelDestination
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.types.utils import (
|
||||
StandardCallbackDynamicParams,
|
||||
StandardLoggingGuardrailInformation,
|
||||
StandardLoggingPayload,
|
||||
)
|
||||
|
|
@ -116,18 +118,15 @@ _OPEN_CALLS_MAX: Final = 10_000
|
|||
class _LLMCallSpan:
|
||||
"""The state carried from the ``pre_call`` boundary to span close.
|
||||
|
||||
``span`` is the live span when it could be opened at the boundary (the server
|
||||
span was ambient), or ``None`` when creation was deferred because no ambient
|
||||
parent was visible — in which case the async callback creates it against its
|
||||
own (worker-copied) ambient context using ``start_time_ns``. The presence of
|
||||
a carrier for a call at all is the proof that ``pre_call`` ran, i.e. that an
|
||||
upstream call was actually attempted.
|
||||
``spans`` is one live span per destination Resource group, opened at the boundary; empty when
|
||||
creation was deferred (no ambient parent visible), where the async callback creates them from
|
||||
``start_time_ns``. A carrier existing at all proves ``pre_call`` ran (an upstream call was attempted).
|
||||
"""
|
||||
|
||||
__slots__ = ("span", "start_time_ns")
|
||||
__slots__ = ("spans", "start_time_ns")
|
||||
|
||||
def __init__(self, span: "Span | None", start_time_ns: int | None) -> None:
|
||||
self.span = span
|
||||
def __init__(self, spans: "tuple[Span, ...]", start_time_ns: int | None) -> None:
|
||||
self.spans = spans
|
||||
self.start_time_ns = start_time_ns
|
||||
|
||||
|
||||
|
|
@ -147,7 +146,13 @@ class OpenTelemetryV2(CustomLogger):
|
|||
self.config: OpenTelemetryV2Config = config or OpenTelemetryV2Config(**kwargs)
|
||||
self.callback_name = callback_name
|
||||
self._tracer_provider: TracerProvider = (
|
||||
tracer_provider if tracer_provider is not None else build_tracer_provider(self.config)
|
||||
tracer_provider
|
||||
if tracer_provider is not None
|
||||
else build_tracer_provider(
|
||||
self.config,
|
||||
tenant_fan_out_owner=callback_name,
|
||||
attach_tenant_fan_out=True,
|
||||
)
|
||||
)
|
||||
self.tracer: Tracer = get_tracer(self._tracer_provider, LITELLM_TRACER_NAME)
|
||||
self._metrics_recorder = self._init_metrics(meter_provider)
|
||||
|
|
@ -159,7 +164,8 @@ class OpenTelemetryV2(CustomLogger):
|
|||
event_recorder=self._init_events(logger_provider),
|
||||
)
|
||||
self._tenant_tracers = TenantTracerCache(self.config, callback_name, LITELLM_TRACER_NAME)
|
||||
self._open_llm_calls: OrderedDict[str, _LLMCallSpan] = OrderedDict()
|
||||
self._open_llm_calls: OrderedDict[str, _LLMCallSpan] = OrderedDict() # mutable-ok: open-call span LRU
|
||||
self._closed_call_ids: OrderedDict[str, None] = OrderedDict() # mutable-ok: bounded LRU of emitted call ids
|
||||
self._init_otel_logger_on_litellm_proxy()
|
||||
|
||||
def _init_metrics(self, meter_provider: Any | None) -> "GenAIMetricRecorder | None":
|
||||
|
|
@ -196,9 +202,21 @@ class OpenTelemetryV2(CustomLogger):
|
|||
# Proxy global registration
|
||||
# ====================================================================== #
|
||||
|
||||
def _register_in_callback_list(self, callbacks: list) -> None:
|
||||
already_otel: Final = any(
|
||||
cb.__class__.__module__.startswith(_OTEL_MODULES) for cb in callbacks if hasattr(cb, "__class__")
|
||||
def _register_in_callback_list(self, callbacks: list, per_backend: bool = False) -> None:
|
||||
"""Add this logger to a proxy-global callback list, once.
|
||||
|
||||
``per_backend`` distinguishes the two kinds of list. ``service_callback`` wants a
|
||||
single OTel owner, so any OTel-module callback already there wins. The event lists
|
||||
are per-backend: v2 collapsed every backend onto this one class parameterised by
|
||||
``callback_name``, so a module-wide test let the first registrant lock out every
|
||||
other backend, and each one's own exporter went dark. Matching the name as well
|
||||
lets each backend register itself while still de-duplicating a repeat of itself.
|
||||
"""
|
||||
already_otel = any(
|
||||
cb.__class__.__module__.startswith(_OTEL_MODULES)
|
||||
and (not per_backend or getattr(cb, "callback_name", None) == self.callback_name)
|
||||
for cb in callbacks
|
||||
if hasattr(cb, "__class__")
|
||||
)
|
||||
if not already_otel:
|
||||
callbacks.append(self)
|
||||
|
|
@ -210,14 +228,28 @@ class OpenTelemetryV2(CustomLogger):
|
|||
return
|
||||
try:
|
||||
self._register_in_callback_list(litellm.service_callback)
|
||||
self._register_in_callback_list(litellm.input_callback)
|
||||
self._register_in_callback_list(litellm._async_success_callback)
|
||||
self._register_in_callback_list(litellm._async_failure_callback)
|
||||
self._register_in_callback_list(litellm.input_callback, per_backend=True)
|
||||
self._register_in_callback_list(litellm._async_success_callback, per_backend=True)
|
||||
self._register_in_callback_list(litellm._async_failure_callback, per_backend=True)
|
||||
except Exception:
|
||||
pass
|
||||
if getattr(proxy_server, "open_telemetry_logger", None) is None:
|
||||
setattr(proxy_server, "open_telemetry_logger", self)
|
||||
|
||||
def _destinations_for_backend(self, call: "LLMCallEvent") -> tuple:
|
||||
"""The call's admin-resolved destinations tagged with THIS logger's callback_name,
|
||||
so each backend's span keeps its own attribute vocabulary."""
|
||||
return tuple(d for d in call.otel_destinations if d.callback_name == self.callback_name)
|
||||
|
||||
def _tracer_dynamic_params(self, call: "LLMCallEvent") -> "StandardCallbackDynamicParams | None":
|
||||
"""The request's own backend credentials, used to pick a credential-scoped tracer.
|
||||
|
||||
A subclass that exports only to admin destinations returns ``None``: the tenant's
|
||||
own account is the owning logger's to reach, and layering a credential-scoped
|
||||
tracer over the fan-out would export the call to it a second time.
|
||||
"""
|
||||
return call.dynamic_params
|
||||
|
||||
# ====================================================================== #
|
||||
# LLM-call callbacks — the span is opened at the ``pre_call`` boundary and
|
||||
# closed here. See ``log_pre_api_call``.
|
||||
|
|
@ -252,22 +284,33 @@ class OpenTelemetryV2(CustomLogger):
|
|||
# call id; keep the first span so its start time is the true one.
|
||||
if call_id in self._open_llm_calls:
|
||||
return
|
||||
start_time_ns: Final = to_ns(datetime.now())
|
||||
span: Span | None = None
|
||||
# A router retry or fallback re-enters ``pre_call`` with the call id of an
|
||||
# attempt that already closed, because the id is minted once per request. That
|
||||
# is a genuinely new upstream attempt and gets its own span, so the marker left
|
||||
# by the previous attempt is cleared here; leaving it made the close callback
|
||||
# short-circuit and the successful attempt after a failure went untraced.
|
||||
self._closed_call_ids.pop(call_id, None)
|
||||
start_time_ns = to_ns(datetime.now())
|
||||
spans: tuple[Span, ...] = ()
|
||||
# Parent to the request's anchored root span (stable across the request),
|
||||
# falling back to ambient on the SDK path. Open the span live only when
|
||||
# that resolves to a recordable parent; otherwise defer to the close
|
||||
# callback (the thread-pool case, where the anchor isn't visible here).
|
||||
parent_context: Final = resolve_request_span_context()
|
||||
if is_recordable_span(get_current_span(parent_context)):
|
||||
span = self._emitter.start_span(
|
||||
SpanRole.LLM_CALL,
|
||||
call.provisional_span_name,
|
||||
parent_context=parent_context,
|
||||
start_time_ns=start_time_ns,
|
||||
tracer=self._tenant_tracers.tracer_for(self.tracer, call.dynamic_params),
|
||||
spans = tuple(
|
||||
self._emitter.start_span(
|
||||
SpanRole.LLM_CALL,
|
||||
call.provisional_span_name,
|
||||
parent_context=parent_context,
|
||||
start_time_ns=start_time_ns,
|
||||
tracer=tracer,
|
||||
)
|
||||
for tracer in self._tenant_tracers.genai_tracers_for(
|
||||
self.tracer, self._destinations_for_backend(call), call.dynamic_params
|
||||
)
|
||||
)
|
||||
self._open_llm_calls[call_id] = _LLMCallSpan(span=span, start_time_ns=start_time_ns)
|
||||
self._open_llm_calls[call_id] = _LLMCallSpan(spans=spans, start_time_ns=start_time_ns)
|
||||
# Evict the oldest open call if the map is over budget. A call that opens
|
||||
# but never closes (a stream that only fires stream events) would linger
|
||||
# otherwise; the evicted span is simply dropped (never exported).
|
||||
|
|
@ -369,12 +412,19 @@ class OpenTelemetryV2(CustomLogger):
|
|||
self._open_llm_calls.pop(data.identity.call_id, None)
|
||||
parent_context, links = resolve_mcp_span_context()
|
||||
parent_context = self._seed_identity_baggage(data.identity, None, parent_context)
|
||||
self._emitter.emit(
|
||||
# The tool-call span carries ``gen_ai.operation.name``, so the fan-out processor
|
||||
# treats it as a gen-AI span and skips it; route it to the request's admin
|
||||
# destinations like the LLM-call span, or it reaches only the global exporter.
|
||||
call = LLMCallEvent.from_dict(kwargs)
|
||||
self._emitter.emit_fanout(
|
||||
SpanRole.MCP_TOOL_CALL,
|
||||
data,
|
||||
parent_context=parent_context,
|
||||
start_time_ns=to_ns(start_time),
|
||||
end_time_ns=to_ns(end_time),
|
||||
tracers=self._tenant_tracers.genai_tracers_for(
|
||||
self.tracer, self._destinations_for_backend(call), self._tracer_dynamic_params(call)
|
||||
),
|
||||
links=links,
|
||||
)
|
||||
return True
|
||||
|
|
@ -423,49 +473,115 @@ class OpenTelemetryV2(CustomLogger):
|
|||
) -> Span | None:
|
||||
"""Finish the LLM-call span opened at ``pre_call`` (or create it deferred).
|
||||
|
||||
No carrier for this call id means ``pre_call`` never ran — the request was
|
||||
rejected at the gate or blocked by a pre-call guardrail before any upstream
|
||||
call — so there is nothing to record and no phantom span.
|
||||
A missing carrier means either ``pre_call`` never ran (rejected at the gate or by a
|
||||
pre-call guardrail, no payload, so dropping is correct) or this v2 instance was lazily
|
||||
activated after ``pre_call``, where the payload IS set and names this backend, so a
|
||||
deferred span is emitted with the success event's start time.
|
||||
|
||||
Lazy activation has two causes, and both must emit. A resolved destination is one.
|
||||
The other is a team carrying its own ``callback_vars`` credentials for this backend:
|
||||
once v2 owns the backend the legacy logger is no longer built, so this instance is
|
||||
the only thing left that can reach that team's account. Gating the deferred span on
|
||||
destinations alone dropped those spans whenever the destination that made v2 take
|
||||
the backend over belonged to a different team.
|
||||
"""
|
||||
call: Final = LLMCallEvent.from_dict(kwargs)
|
||||
call_id: Final = call.call_id
|
||||
# ``pop`` is the dedup: this method runs from both the success and failure
|
||||
# paths, and whichever fires first removes the carrier and closes the span.
|
||||
carrier: Final = self._open_llm_calls.pop(call_id, None) if call_id else None
|
||||
from litellm.integrations.otel.presets import dynamic_otlp_headers
|
||||
|
||||
call = LLMCallEvent.from_dict(kwargs)
|
||||
call_id = call.call_id
|
||||
|
||||
carrier = self._open_llm_calls.pop(call_id, None) if call_id else None
|
||||
payload = call.payload
|
||||
|
||||
# The closed marker guards the carrier-less path only, where it stops a repeat
|
||||
# success/failure callback re-emitting a span that already shipped. An open
|
||||
# carrier is this attempt's own live span and must always be finished, or a
|
||||
# retry's span would be left open and never exported.
|
||||
#
|
||||
# It is keyed on the payload's own id rather than the call id, because a router
|
||||
# retry reuses one ``litellm_call_id`` for every attempt while each attempt gets
|
||||
# its own payload id (the provider's response id, falling back to the call id).
|
||||
# Keying on the call id made the successful attempt after a failure look like a
|
||||
# duplicate, so a destination saw only the failure.
|
||||
emit_key = (payload.get("id") if payload else None) or call_id
|
||||
if carrier is None and emit_key and emit_key in self._closed_call_ids:
|
||||
return None
|
||||
|
||||
if carrier is None:
|
||||
return None
|
||||
payload: Final = call.payload
|
||||
destinations = self._destinations_for_backend(call)
|
||||
own_credentials = bool(dynamic_otlp_headers(self.callback_name, call.dynamic_params))
|
||||
if call.is_no_upstream_call or payload is None or not (destinations or own_credentials):
|
||||
return None
|
||||
self._mark_closed(emit_key)
|
||||
return self._emit_deferred_llm_call(
|
||||
payload,
|
||||
destinations,
|
||||
to_ns(start_time),
|
||||
to_ns(end_time),
|
||||
call.time_to_first_chunk_seconds,
|
||||
call.dynamic_params,
|
||||
)
|
||||
|
||||
end_time_ns = to_ns(end_time)
|
||||
self._mark_closed(emit_key)
|
||||
if payload is None:
|
||||
if carrier.span is not None:
|
||||
# Opened at the boundary but the payload never materialized — end
|
||||
# it (named provisionally) so it isn't leaked as an open span.
|
||||
carrier.span.end(end_time=to_ns(end_time))
|
||||
for span in carrier.spans:
|
||||
span.end(end_time=end_time_ns)
|
||||
return None
|
||||
data: Final = LLMCallSpanData.from_standard_logging_payload(
|
||||
|
||||
data = LLMCallSpanData.from_standard_logging_payload(
|
||||
payload,
|
||||
capture_content=self.config.capture_span_content,
|
||||
time_to_first_chunk_seconds=call.time_to_first_chunk_seconds,
|
||||
)
|
||||
end_time_ns: Final = to_ns(end_time)
|
||||
if carrier.span is not None:
|
||||
# Born at the boundary: stamp attributes from the typed payload, set
|
||||
# status, and end it. Its parent (the server span) was captured at
|
||||
# creation from real ambient context.
|
||||
self._emitter.finish_span(SpanRole.LLM_CALL, carrier.span, data, end_time_ns=end_time_ns)
|
||||
return carrier.span
|
||||
# Deferred: ``pre_call`` saw no recordable parent, so create the span now.
|
||||
# The worker copied the request task's context, which carries the anchored
|
||||
# root span — parent to it (ambient fallback on the SDK path). Seed identity
|
||||
# Baggage so the span — and the SDK path, which has none — is labeled
|
||||
# consistently.
|
||||
if carrier.spans:
|
||||
for span in carrier.spans:
|
||||
self._emitter.finish_span(SpanRole.LLM_CALL, span, data, end_time_ns=end_time_ns)
|
||||
return carrier.spans[0]
|
||||
return self._emit_deferred_llm_call(
|
||||
payload,
|
||||
self._destinations_for_backend(call),
|
||||
carrier.start_time_ns,
|
||||
end_time_ns,
|
||||
call.time_to_first_chunk_seconds,
|
||||
call.dynamic_params,
|
||||
)
|
||||
|
||||
def _mark_closed(self, call_id: str | None) -> None:
|
||||
"""Remember a call_id has been closed so a duplicate callback no-ops (bounded FIFO)."""
|
||||
if not call_id:
|
||||
return
|
||||
self._closed_call_ids[call_id] = None
|
||||
if len(self._closed_call_ids) > _OPEN_CALLS_MAX:
|
||||
self._closed_call_ids.popitem(last=False)
|
||||
|
||||
def _emit_deferred_llm_call(
|
||||
self,
|
||||
payload: "StandardLoggingPayload",
|
||||
destinations: "tuple[OtelDestination, ...]",
|
||||
start_time_ns: int | None,
|
||||
end_time_ns: int | None,
|
||||
time_to_first_chunk_seconds: float | None = None,
|
||||
dynamic_params: "StandardCallbackDynamicParams | None" = None,
|
||||
) -> Span | None:
|
||||
"""Emit an LLM-call span outside the ``pre_call`` boundary.
|
||||
|
||||
Two callers: the SDK thread-pool path and the destination-resolver path. Both anchor to
|
||||
the request's root span via the worker-copied context and seed identity Baggage.
|
||||
"""
|
||||
data = LLMCallSpanData.from_standard_logging_payload(
|
||||
payload,
|
||||
capture_content=self.config.capture_span_content,
|
||||
time_to_first_chunk_seconds=time_to_first_chunk_seconds,
|
||||
)
|
||||
parent_ctx = self._seed_identity_baggage(data.identity, data.request_model, resolve_request_span_context())
|
||||
return self._emitter.emit(
|
||||
return self._emitter.emit_fanout(
|
||||
SpanRole.LLM_CALL,
|
||||
data,
|
||||
parent_context=parent_ctx,
|
||||
start_time_ns=carrier.start_time_ns,
|
||||
start_time_ns=start_time_ns,
|
||||
end_time_ns=end_time_ns,
|
||||
tracer=self._tenant_tracers.tracer_for(self.tracer, call.dynamic_params),
|
||||
tracers=self._tenant_tracers.genai_tracers_for(self.tracer, destinations, dynamic_params),
|
||||
)
|
||||
|
||||
# ====================================================================== #
|
||||
|
|
@ -756,8 +872,15 @@ def publish_global_otel_v2_provider(
|
|||
unit-testable without reading or mutating real global OTel state. Returns the
|
||||
logger whose provider was published.
|
||||
"""
|
||||
logger: Final = select_global_otel_v2_logger(in_memory_loggers, registered=registered)
|
||||
# Local import: ``destination_logger`` imports this module for its emitter base, so a
|
||||
# module-level import here would close the cycle.
|
||||
from litellm.integrations.otel.destination_logger import (
|
||||
register_admin_destination_logger,
|
||||
)
|
||||
|
||||
logger = select_global_otel_v2_logger(in_memory_loggers, registered=registered)
|
||||
set_global_provider(logger._tracer_provider)
|
||||
register_admin_destination_logger()
|
||||
return logger
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -4,9 +4,18 @@ from enum import Enum
|
|||
from functools import lru_cache
|
||||
from typing import Annotated, Any, Final
|
||||
|
||||
from pydantic import AliasChoices, BaseModel, Field, field_validator, model_validator
|
||||
from pydantic import (
|
||||
AliasChoices,
|
||||
BaseModel,
|
||||
Field,
|
||||
TypeAdapter,
|
||||
ValidationError,
|
||||
field_validator,
|
||||
model_validator,
|
||||
)
|
||||
from pydantic_settings import BaseSettings, NoDecode, SettingsConfigDict
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.integrations.otel.model.baggage import (
|
||||
BAGGAGE_PROMOTED_KEYS,
|
||||
DEFAULT_BAGGAGE_METADATA_KEYS,
|
||||
|
|
@ -46,6 +55,35 @@ class _OTelV2Flag(BaseSettings):
|
|||
|
||||
enabled: bool = Field(default=False, validation_alias=AliasChoices(OTEL_V2_ENV))
|
||||
|
||||
@field_validator("enabled", mode="before")
|
||||
@classmethod
|
||||
def _unparseable_reads_as_off(cls, value: object) -> object:
|
||||
"""Never raise on this flag: an unusable value reads as off, loudly.
|
||||
|
||||
The flag is read from ``instrument_fastapi_app`` while ``proxy_server`` is still
|
||||
importing, so a parse error here takes the whole proxy down before it binds a
|
||||
port. ``LITELLM_OTEL_V2=`` is routine in k8s ConfigMaps and ``.env`` files, and a
|
||||
stray space or a word pydantic does not accept (``enabled``, ``2``) is an easy
|
||||
typo; none of them is worth refusing to start over a feature that is off by
|
||||
default. Surrounding whitespace is trimmed first so ``"true "`` still means true,
|
||||
and anything left unrecognized warns and degrades rather than failing closed on
|
||||
the whole process.
|
||||
"""
|
||||
if not isinstance(value, str):
|
||||
return value
|
||||
stripped = value.strip()
|
||||
if not stripped:
|
||||
return False
|
||||
try:
|
||||
return TypeAdapter(bool).validate_python(stripped)
|
||||
except ValidationError:
|
||||
verbose_logger.warning(
|
||||
"%s=%r is not a recognized boolean; treating OpenTelemetry v2 as disabled. Use true or false.",
|
||||
OTEL_V2_ENV,
|
||||
value,
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def is_otel_v2_enabled() -> bool:
|
||||
|
|
@ -243,8 +281,16 @@ class OpenTelemetryV2Config(BaseSettings):
|
|||
if self.endpoint and self.exporter == "console":
|
||||
self.exporter = "otlp_http"
|
||||
# When no explicit destinations are given, fold the single-destination
|
||||
# shorthand into one spec so the provider always has a destination.
|
||||
if not self.exporters:
|
||||
# shorthand into one spec so the provider has a destination. The exception is
|
||||
# the "nothing configured" degrade case -- a preset returning a bare config
|
||||
# because it found no credentials, where ``exporter`` is left at its default
|
||||
# ``console`` and no endpoint is set: leave it exporter-less so the provider
|
||||
# exports nothing, rather than degrading to a console exporter that prints every
|
||||
# span (including prompt and completion content) to stdout synchronously on the
|
||||
# request path. An explicitly chosen exporter (even ``console``), a non-console
|
||||
# kind, or an endpoint all still fold.
|
||||
console_by_default = self.exporter == "console" and "exporter" not in self.model_fields_set
|
||||
if not self.exporters and (self.endpoint or not console_by_default):
|
||||
self.exporters = [
|
||||
ExporterSpec(
|
||||
kind=self.exporter,
|
||||
|
|
|
|||
|
|
@ -41,11 +41,12 @@ from dataclasses import dataclass, field
|
|||
from typing import TYPE_CHECKING, Any, Final, cast
|
||||
|
||||
from litellm.constants import LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL
|
||||
from litellm.integrations.otel.model.destination import OtelDestination
|
||||
from litellm.integrations.otel.model.semconv import resolve_operation
|
||||
from litellm.integrations.otel.model.utils import as_str, to_seconds
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.types.utils import StandardLoggingPayload
|
||||
from litellm.types.utils import StandardCallbackDynamicParams, StandardLoggingPayload
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
|
|
@ -192,9 +193,10 @@ class LLMCallEvent:
|
|||
# at ``pre_call``, or when the call closed before any payload materialized (so
|
||||
# there is nothing to stamp on the span).
|
||||
payload: StandardLoggingPayload | None
|
||||
otel_destinations: tuple[OtelDestination, ...]
|
||||
# The ``standard_callback_dynamic_params`` routing the call to a per-tenant
|
||||
# tracer (its own exporter/endpoint), or ``None`` when the call isn't scoped.
|
||||
dynamic_params: Any
|
||||
dynamic_params: StandardCallbackDynamicParams | None
|
||||
# True for synthetic proxy-gate logs (auth / rate-limit rejections): they fire
|
||||
# the ``pre_call`` hook but never made an upstream call, so they get no span.
|
||||
is_no_upstream_call: bool
|
||||
|
|
@ -206,6 +208,12 @@ class LLMCallEvent:
|
|||
|
||||
@classmethod
|
||||
def from_dict(cls, kwargs: Mapping[str, Any]) -> LLMCallEvent:
|
||||
# Imported here rather than at module scope: this module is reachable from
|
||||
# ``litellm/__init__`` and ``plumbing.context`` imports opentelemetry eagerly, so a
|
||||
# top-level import makes the whole package a hard dependency of the proxy, which
|
||||
# installs without the optional tracing extras.
|
||||
from litellm.integrations.otel.plumbing.context import request_destinations
|
||||
|
||||
raw_payload: Final = kwargs.get("standard_logging_object")
|
||||
payload: Final = cast("StandardLoggingPayload", raw_payload) if raw_payload else None
|
||||
operation: Final = resolve_operation(as_str(kwargs.get("call_type")))
|
||||
|
|
@ -213,6 +221,7 @@ class LLMCallEvent:
|
|||
return cls(
|
||||
call_id=_call_id(payload, kwargs),
|
||||
payload=payload,
|
||||
otel_destinations=request_destinations(),
|
||||
dynamic_params=kwargs.get("standard_callback_dynamic_params"),
|
||||
is_no_upstream_call=bool(kwargs.get(LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL)),
|
||||
provisional_span_name=f"{operation.value} {model}".strip(),
|
||||
|
|
|
|||
|
|
@ -101,17 +101,21 @@ def instrument_fastapi_app(app: Any) -> None:
|
|||
after config load (see ``proxy_startup_event``), and the proxy delegates to it.
|
||||
That way server spans and gen-ai spans share one provider and the same trace.
|
||||
"""
|
||||
if not is_otel_v2_enabled():
|
||||
return
|
||||
|
||||
try:
|
||||
if not is_otel_v2_enabled():
|
||||
return
|
||||
|
||||
# Lazy: only the V2-enabled path needs the optional
|
||||
# ``opentelemetry-instrumentation-fastapi`` package, which is not part of the
|
||||
# base ``litellm[proxy]`` install. Importing it at module top would make
|
||||
# ``proxy_server``'s unconditional ``import`` of this module crash when the
|
||||
# package is absent, even with the gate off.
|
||||
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
|
||||
except ImportError:
|
||||
verbose_logger.warning(
|
||||
"LITELLM_OTEL_V2 is enabled but 'opentelemetry-instrumentation-fastapi' "
|
||||
"is not installed. The FastAPI server span will not be created, so traces "
|
||||
"exported to admin-owned destinations will be missing their root span "
|
||||
"(orphaned children). Install 'opentelemetry-instrumentation-fastapi'."
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
excluded_urls: Final = (
|
||||
os.environ.get("OTEL_PYTHON_FASTAPI_EXCLUDED_URLS")
|
||||
if "OTEL_PYTHON_FASTAPI_EXCLUDED_URLS" in os.environ
|
||||
|
|
@ -127,4 +131,4 @@ def instrument_fastapi_app(app: Any) -> None:
|
|||
exclude_spans=["receive", "send"],
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_logger.debug("Skipping OTel V2 FastAPI instrumentation: %s", e)
|
||||
verbose_logger.warning("OTel V2 FastAPI instrumentation failed: %s", e)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
"""Provider / exporter factory + the Baggage span processor."""
|
||||
|
||||
from collections.abc import Callable, Iterable
|
||||
from collections.abc import Callable, Iterable, Mapping
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
from opentelemetry import _logs, baggage, metrics
|
||||
|
|
@ -41,6 +41,8 @@ if TYPE_CHECKING:
|
|||
from opentelemetry.metrics import Meter
|
||||
from opentelemetry.sdk.metrics.export import MetricReader
|
||||
|
||||
from litellm.integrations.otel.model.destination import OtelDestination
|
||||
|
||||
_SPAN_KIND_BY_ROLE_KIND: Final[dict[LiteLLMSpanKind, SpanKind]] = {
|
||||
LiteLLMSpanKind.SERVER: SpanKind.SERVER,
|
||||
LiteLLMSpanKind.CLIENT: SpanKind.CLIENT,
|
||||
|
|
@ -110,7 +112,7 @@ def _otlp_traces_endpoint(endpoint: str | None) -> str | None:
|
|||
return endpoint
|
||||
endpoint = endpoint.rstrip("/")
|
||||
# Splunk Observability uses ``/v2/trace/otlp``; never rewrite it.
|
||||
if endpoint.endswith("/v1/traces") or "/v2/trace/otlp" in endpoint:
|
||||
if endpoint.endswith("/v1/traces") or "/v2/trace/otlp" in endpoint or endpoint.endswith("/api/trace"):
|
||||
return endpoint
|
||||
for other_signal in ("/v1/logs", "/v1/metrics"):
|
||||
if endpoint.endswith(other_signal):
|
||||
|
|
@ -118,6 +120,22 @@ def _otlp_traces_endpoint(endpoint: str | None) -> str | None:
|
|||
return endpoint + "/v1/traces"
|
||||
|
||||
|
||||
_GRPC_BACKENDS = frozenset({"arize"})
|
||||
|
||||
|
||||
def default_otlp_kind_for_backend(callback_name: "str | None") -> str:
|
||||
"""The intrinsic OTLP transport for a backend's own OTLP endpoint."""
|
||||
return "otlp_grpc" if callback_name in _GRPC_BACKENDS else "otlp_http"
|
||||
|
||||
|
||||
def destination_resource_attrs(destination: "OtelDestination") -> Mapping[str, str]:
|
||||
"""The destination's builder-declared Resource attributes (e.g. Arize's
|
||||
``model_id`` / ``arize.project.name``; empty for header-routed backends), read
|
||||
by both export paths so the gen-AI span and its parents share one Resource.
|
||||
"""
|
||||
return dict(destination.resource_attributes)
|
||||
|
||||
|
||||
def parse_headers(raw: str | None) -> dict[str, str]:
|
||||
"""Parse an OTLP ``"k=v,k=v"`` header string into a dict.
|
||||
|
||||
|
|
@ -415,20 +433,26 @@ def build_tracer_provider(
|
|||
exporter: SpanExporter | None = None,
|
||||
baggage_processor: SpanProcessor | None = None,
|
||||
use_simple_processor: bool | None = None,
|
||||
tenant_fan_out_owner: str | None = None,
|
||||
attach_tenant_fan_out: bool = False,
|
||||
) -> TracerProvider:
|
||||
"""Build the shared :class:`TracerProvider`.
|
||||
|
||||
Attach the Baggage processor first (so identity attributes land on each
|
||||
span before any export decision), then add one ``SpanProcessor`` per
|
||||
``config.exporters`` entry — this is what fans spans out to multiple
|
||||
backends. ``exporter`` and ``use_simple_processor`` are explicit overrides:
|
||||
pass a single exporter to attach exactly that one (used by tests).
|
||||
"""Build the shared :class:`TracerProvider`: Baggage processor first, then one
|
||||
``SpanProcessor`` per ``config.exporters`` entry (``exporter`` overrides for tests).
|
||||
``attach_tenant_fan_out``/``tenant_fan_out_owner`` add a ``TenantFanOutSpanProcessor``
|
||||
forwarding proxy-internal spans to the request's destinations.
|
||||
"""
|
||||
provider: Final = TracerProvider(resource=build_resource(config))
|
||||
if baggage_processor is None:
|
||||
baggage_processor = LiteLLMBaggageSpanProcessor(allowed_keys=config.baggage_promoted_keys)
|
||||
provider.add_span_processor(baggage_processor)
|
||||
|
||||
if attach_tenant_fan_out or tenant_fan_out_owner is not None:
|
||||
from litellm.integrations.otel.plumbing.routing import (
|
||||
TenantFanOutSpanProcessor,
|
||||
)
|
||||
|
||||
provider.add_span_processor(TenantFanOutSpanProcessor(owner_callback_name=tenant_fan_out_owner))
|
||||
|
||||
if exporter is not None:
|
||||
provider.add_span_processor(_processor_for(exporter, use_simple_processor))
|
||||
return provider
|
||||
|
|
|
|||
|
|
@ -1,28 +1,31 @@
|
|||
"""Per-request multi-tenant tracer routing.
|
||||
"""Per-request multi-tenant tracer routing and span fan-out.
|
||||
|
||||
When a request carries team/key vendor credentials in
|
||||
``standard_callback_dynamic_params``, its spans must export through a
|
||||
``TracerProvider`` whose OTLP headers carry those credentials.
|
||||
``TenantTracerCache`` builds and caches one provider per distinct credential
|
||||
set, and otherwise hands back the logger's default tracer. This lets a single
|
||||
logger fan requests out to many tenants without needing a logger per tenant.
|
||||
``TenantTracerCache`` routes the gen-AI span to per-tenant/destination providers;
|
||||
``TenantFanOutSpanProcessor`` forwards proxy-internal spans to every admin-resolved destination.
|
||||
"""
|
||||
|
||||
import threading
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Mapping
|
||||
from typing import Any, Final
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.context import Context
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.sdk.trace import ReadableSpan, Span, SpanProcessor, TracerProvider
|
||||
from opentelemetry.trace import Tracer
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.integrations.otel.model.config import OpenTelemetryV2Config
|
||||
from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config
|
||||
from litellm.integrations.otel.model.destination import OtelDestination
|
||||
from litellm.integrations.otel.plumbing.context import request_destinations
|
||||
from litellm.integrations.otel.plumbing.providers import (
|
||||
build_tracer_provider,
|
||||
get_tracer,
|
||||
)
|
||||
from litellm.integrations.otel.presets import dynamic_otlp_headers
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.types.utils import StandardCallbackDynamicParams
|
||||
|
||||
# Exporter kinds that ignore headers — never rewritten with dynamic credentials.
|
||||
_NON_OTLP_KINDS: Final = ("console", "in_memory", "inmemory", "memory")
|
||||
|
||||
|
|
@ -35,22 +38,24 @@ _NON_OTLP_KINDS: Final = ("console", "in_memory", "inmemory", "memory")
|
|||
_MAX_CACHED_PROVIDERS: Final = 256
|
||||
|
||||
|
||||
def _shutdown_provider(provider: TracerProvider) -> None:
|
||||
"""Flush + stop an evicted provider's processors (reclaims their threads).
|
||||
def _shutdown_in_background(evicted: "TracerProvider | SpanProcessor") -> None:
|
||||
"""Reclaim an evicted provider/processor's ``BatchSpanProcessor`` worker thread.
|
||||
|
||||
``TracerProvider.shutdown`` force-flushes each ``SpanProcessor`` before
|
||||
stopping it, so any spans already handed to a ``BatchSpanProcessor`` are
|
||||
exported rather than dropped. Best-effort: a shutdown failure must not break
|
||||
the request that triggered the eviction.
|
||||
Dropping it without ``shutdown`` leaks the daemon thread; ``shutdown`` force-flushes and can
|
||||
do network I/O, so it runs fire-and-forget on a daemon thread rather than the request path.
|
||||
"""
|
||||
try:
|
||||
provider.shutdown()
|
||||
except Exception as e: # pragma: no cover - defensive
|
||||
verbose_logger.debug("OTel V2: error shutting down evicted provider: %s", e)
|
||||
|
||||
def _run() -> None:
|
||||
try:
|
||||
evicted.shutdown()
|
||||
except Exception as exc: # noqa: BLE001 # a failed shutdown must not surface on the hot path
|
||||
verbose_logger.debug("OTel V2: background shutdown of evicted %s failed: %s", type(evicted).__name__, exc)
|
||||
|
||||
threading.Thread(target=_run, name="litellm-otel-evict-shutdown", daemon=True).start()
|
||||
|
||||
|
||||
class TenantTracerCache:
|
||||
"""Credential-scoped ``TracerProvider`` cache keyed by the dynamic headers."""
|
||||
"""Destination-scoped ``TracerProvider`` cache keyed by endpoint + headers."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
|
@ -61,32 +66,96 @@ class TenantTracerCache:
|
|||
self._config = config
|
||||
self._callback_name = callback_name
|
||||
self._tracer_name = tracer_name
|
||||
self._providers: OrderedDict[tuple[tuple[str, str], ...], TracerProvider] = OrderedDict()
|
||||
self._providers: OrderedDict[tuple[object, ...], TracerProvider] = (
|
||||
OrderedDict()
|
||||
) # mutable-ok: bounded LRU tracer-provider cache
|
||||
|
||||
def tracer_for(self, default: Tracer, dynamic_params: Any) -> Tracer:
|
||||
"""Return the tracer for this request.
|
||||
def _evict_if_full(self) -> None:
|
||||
"""Drop the least-recently-used provider when over capacity, shutting it down off
|
||||
the hot path so its ``BatchSpanProcessor`` worker thread is reclaimed rather than
|
||||
leaked for the life of the process."""
|
||||
if len(self._providers) > _MAX_CACHED_PROVIDERS:
|
||||
_, evicted = self._providers.popitem(last=False)
|
||||
_shutdown_in_background(evicted)
|
||||
|
||||
Use ``default`` unless the request's dynamic credentials require a
|
||||
credential-scoped tracer, in which case build (or reuse) one. The cache
|
||||
is a bounded LRU: the least-recently-used provider is flushed and shut
|
||||
down on overflow so its exporter threads don't accumulate.
|
||||
def tracers_for(
|
||||
self,
|
||||
default: Tracer,
|
||||
destinations: "tuple[OtelDestination, ...]",
|
||||
*,
|
||||
include_base_on_first: bool = True,
|
||||
) -> "tuple[Tracer, ...]":
|
||||
"""The tracers for this request's gen-AI span, one per distinct Resource group.
|
||||
|
||||
Destinations are grouped by ``destination_resource_attrs`` (a backend like Arize selects its
|
||||
project from the Resource). The configured/global exporters ride their own clean-Resource
|
||||
provider (``default``), never folded into a destination group, so a Resource-routed
|
||||
destination cannot stamp its own attributes (e.g. Arize's project) onto the global export.
|
||||
``include_base_on_first=False`` drops ``default`` when a credential-scoped tracer already
|
||||
carries the configured exporters.
|
||||
"""
|
||||
if not destinations:
|
||||
return (default,)
|
||||
groups = tuple(
|
||||
self._tracer_for_group(resource_key, group, include_base=False)
|
||||
for resource_key, group in self._group_by_resource(destinations)
|
||||
)
|
||||
return (default, *groups) if include_base_on_first else groups
|
||||
|
||||
def genai_tracers_for(
|
||||
self,
|
||||
default: Tracer,
|
||||
destinations: "tuple[OtelDestination, ...]",
|
||||
dynamic_params: "StandardCallbackDynamicParams | None",
|
||||
) -> "tuple[Tracer, ...]":
|
||||
"""The gen-AI span's tracers, layering per-request credential routing over the destination
|
||||
fan-out: with this backend's team/key OTLP credentials the global export rides a
|
||||
credential-scoped provider and the destination groups omit the base exporters; else plain fan-out.
|
||||
"""
|
||||
headers: Final = dynamic_otlp_headers(self._callback_name, dynamic_params)
|
||||
if not headers:
|
||||
return default
|
||||
cache_key: Final = tuple(sorted(headers.items()))
|
||||
return self.tracers_for(default, destinations)
|
||||
dynamic = self._credential_scoped_tracer(headers, dynamic_params)
|
||||
if not destinations:
|
||||
return (dynamic,)
|
||||
return (dynamic, *self.tracers_for(default, destinations, include_base_on_first=False))
|
||||
|
||||
def _credential_scoped_tracer(
|
||||
self,
|
||||
headers: "dict[str, str]",
|
||||
dynamic_params: "StandardCallbackDynamicParams | None" = None,
|
||||
) -> Tracer:
|
||||
"""A cached provider that keeps the configured exporters and rewrites only this
|
||||
backend's owned exporter's headers to ``headers`` (the per-request credentials).
|
||||
|
||||
The endpoint and transport are part of the key: when the preset contributed no
|
||||
exporter the synthesized one resolves both from the request's own credentials, so
|
||||
two tenants sharing vendor credentials on different hosts would otherwise collide
|
||||
and the second tenant's traces would ship to the first tenant's collector.
|
||||
"""
|
||||
from litellm.integrations.otel.presets import dynamic_otlp_destination
|
||||
|
||||
destination = dynamic_otlp_destination(self._callback_name, dynamic_params)
|
||||
cache_key: tuple[object, ...] = (
|
||||
"dynamic",
|
||||
tuple(sorted(headers.items())),
|
||||
destination.endpoint if destination is not None else None,
|
||||
destination.protocol if destination is not None else None,
|
||||
)
|
||||
provider = self._providers.get(cache_key)
|
||||
if provider is not None:
|
||||
self._providers.move_to_end(cache_key)
|
||||
else:
|
||||
provider = build_tracer_provider(self._config_with_headers(headers))
|
||||
provider = build_tracer_provider(self._config_with_headers(headers, dynamic_params))
|
||||
self._providers[cache_key] = provider
|
||||
if len(self._providers) > _MAX_CACHED_PROVIDERS:
|
||||
_, evicted = self._providers.popitem(last=False)
|
||||
_shutdown_provider(evicted)
|
||||
self._evict_if_full()
|
||||
return get_tracer(provider, self._tracer_name)
|
||||
|
||||
def _config_with_headers(self, headers: Mapping[str, str]) -> OpenTelemetryV2Config:
|
||||
def _config_with_headers(
|
||||
self,
|
||||
headers: "dict[str, str]",
|
||||
dynamic_params: "StandardCallbackDynamicParams | None" = None,
|
||||
) -> OpenTelemetryV2Config:
|
||||
"""Clone the config, stamping ``headers`` onto the credential's own exporter.
|
||||
|
||||
``headers`` are the per-request credentials of ``self._callback_name`` (the
|
||||
|
|
@ -95,15 +164,295 @@ class TenantTracerCache:
|
|||
tenant's Arize key must never rewrite the headers of a co-configured
|
||||
Langfuse or self-hosted collector exporter, which would leak that key to a
|
||||
different backend.
|
||||
|
||||
A preset built with ``allow_missing_credentials`` contributes no exporter of its
|
||||
own, so there is nothing to stamp and the team's traces would go nowhere. One is
|
||||
synthesized from the request's own credentials in that case, resolved through the
|
||||
same builder an equivalent admin destination would use so the team reaches the
|
||||
account its ``callback_vars`` name.
|
||||
"""
|
||||
header_str: Final = ",".join(f"{key}={value}" for key, value in headers.items())
|
||||
header_update: Final[dict[str, str]] = {"headers": header_str}
|
||||
exporters: Final = [
|
||||
header_str = ",".join(f"{key}={value}" for key, value in headers.items())
|
||||
owns_exporter = any(
|
||||
spec.owner == self._callback_name and spec.kind.lower() not in _NON_OTLP_KINDS
|
||||
for spec in self._config.exporters
|
||||
)
|
||||
if not owns_exporter:
|
||||
return self._config.model_copy(
|
||||
update={"exporters": [*self._config.exporters, *self._synthesized_exporter(header_str, dynamic_params)]}
|
||||
)
|
||||
exporters = [
|
||||
(
|
||||
spec.model_copy(update=header_update)
|
||||
spec.model_copy(update={"headers": header_str})
|
||||
if spec.owner == self._callback_name and spec.kind.lower() not in _NON_OTLP_KINDS
|
||||
else spec
|
||||
)
|
||||
for spec in self._config.exporters
|
||||
]
|
||||
return self._config.model_copy(update={"exporters": exporters})
|
||||
|
||||
def _synthesized_exporter(
|
||||
self,
|
||||
header_str: str,
|
||||
dynamic_params: "StandardCallbackDynamicParams | None",
|
||||
) -> "tuple[ExporterSpec, ...]":
|
||||
"""This backend's exporter built from the request's own credentials, or empty
|
||||
when they don't resolve to an endpoint.
|
||||
|
||||
The builder's ``protocol`` wins over the backend's intrinsic transport, matching
|
||||
``_config_with_destinations``: values that pin an HTTP collector for a gRPC-default
|
||||
backend would otherwise be exported over gRPC and dropped.
|
||||
"""
|
||||
from litellm.integrations.otel.presets import dynamic_otlp_destination
|
||||
|
||||
destination = dynamic_otlp_destination(self._callback_name, dynamic_params)
|
||||
if destination is None or not destination.endpoint:
|
||||
return ()
|
||||
return (
|
||||
ExporterSpec(
|
||||
kind=destination.protocol or self._owned_otlp_kind(),
|
||||
endpoint=destination.endpoint,
|
||||
headers=header_str,
|
||||
owner=self._callback_name,
|
||||
),
|
||||
)
|
||||
|
||||
def _group_by_resource(
|
||||
self, destinations: "tuple[OtelDestination, ...]"
|
||||
) -> "tuple[tuple[tuple[tuple[str, str], ...], tuple[OtelDestination, ...]], ...]":
|
||||
"""Destinations grouped by their backend-required Resource attributes.
|
||||
|
||||
Groups sort deterministically by key, so the empty-Resource group (header-routed
|
||||
backends) sorts first and the configured/global exporters attach to it.
|
||||
"""
|
||||
from litellm.integrations.otel.plumbing.providers import (
|
||||
destination_resource_attrs,
|
||||
)
|
||||
|
||||
groups: OrderedDict[tuple[tuple[str, str], ...], list[OtelDestination]] = (
|
||||
OrderedDict()
|
||||
) # mutable-ok: insertion-order grouping accumulator, frozen before return
|
||||
for destination in destinations:
|
||||
key = tuple(sorted(destination_resource_attrs(destination).items()))
|
||||
groups.setdefault(key, []).append(destination)
|
||||
return tuple((key, tuple(group)) for key, group in sorted(groups.items()))
|
||||
|
||||
def _tracer_for_group(
|
||||
self,
|
||||
resource_key: "tuple[tuple[str, str], ...]",
|
||||
group: "tuple[OtelDestination, ...]",
|
||||
*,
|
||||
include_base: bool,
|
||||
) -> Tracer:
|
||||
cache_key: tuple[object, ...] = (
|
||||
resource_key,
|
||||
tuple(sorted((d.endpoint, tuple(sorted(d.headers.items())), d.protocol or "") for d in group)),
|
||||
include_base,
|
||||
)
|
||||
provider = self._providers.get(cache_key)
|
||||
if provider is not None:
|
||||
self._providers.move_to_end(cache_key)
|
||||
else:
|
||||
provider = build_tracer_provider(
|
||||
self._config_with_destinations(tuple(group), include_base_exporters=include_base)
|
||||
)
|
||||
self._providers[cache_key] = provider
|
||||
self._evict_if_full()
|
||||
return get_tracer(provider, self._tracer_name)
|
||||
|
||||
def _owned_otlp_kind(self) -> str:
|
||||
"""The OTLP transport of this integration's own exporter (langfuse -> http, arize -> grpc).
|
||||
|
||||
Prefers the admin's configured exporter kind; falls back to the backend's intrinsic
|
||||
default so a lazily-activated backend with no owned spec still picks the right transport.
|
||||
"""
|
||||
from litellm.integrations.otel.plumbing.providers import (
|
||||
default_otlp_kind_for_backend,
|
||||
)
|
||||
|
||||
for spec in self._config.exporters:
|
||||
if spec.owner == self._callback_name and spec.kind.lower() not in _NON_OTLP_KINDS:
|
||||
return spec.kind
|
||||
return default_otlp_kind_for_backend(self._callback_name)
|
||||
|
||||
def _config_with_destinations(
|
||||
self,
|
||||
destinations: "tuple[OtelDestination, ...]",
|
||||
*,
|
||||
include_base_exporters: bool = True,
|
||||
) -> OpenTelemetryV2Config:
|
||||
"""Clone the config, appending one exporter per destination so one span exports to every one.
|
||||
|
||||
``include_base_exporters`` keeps the configured/global exporters (``tracers_for`` sets it only on
|
||||
the first group); the clone's Resource folds in the destinations' ``destination_resource_attrs``.
|
||||
"""
|
||||
from litellm.integrations.otel.plumbing.providers import (
|
||||
destination_resource_attrs,
|
||||
)
|
||||
|
||||
kind = self._owned_otlp_kind()
|
||||
appended = tuple(
|
||||
ExporterSpec(
|
||||
kind=d.protocol or kind,
|
||||
endpoint=d.endpoint,
|
||||
headers=d.header_string(),
|
||||
owner=None,
|
||||
)
|
||||
for d in destinations
|
||||
)
|
||||
base_exporters = (*self._config.exporters,) if include_base_exporters else ()
|
||||
merged_resource_attrs = {
|
||||
**self._config.resource_attributes,
|
||||
**{key: value for d in destinations for key, value in destination_resource_attrs(d).items()},
|
||||
}
|
||||
return self._config.model_copy(
|
||||
update={
|
||||
"exporters": [*base_exporters, *appended],
|
||||
"resource_attributes": merged_resource_attrs,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
_MAX_CACHED_PROCESSORS = 256
|
||||
|
||||
_GENAI_SPAN_ATTR = "gen_ai.operation.name"
|
||||
|
||||
|
||||
def _processor_key(destination: OtelDestination) -> "tuple[str, tuple[tuple[str, str], ...], str | None]":
|
||||
return (destination.endpoint, tuple(sorted(destination.headers.items())), destination.protocol)
|
||||
|
||||
|
||||
def _is_genai_span(span: ReadableSpan) -> bool:
|
||||
attributes = span.attributes or {}
|
||||
return _GENAI_SPAN_ATTR in attributes
|
||||
|
||||
|
||||
def _with_destination_resource(span: ReadableSpan, destination: OtelDestination) -> ReadableSpan:
|
||||
"""Return ``span`` with its Resource augmented by the destination's required attributes,
|
||||
via a shallow wrapper that leaves the original span untouched."""
|
||||
from litellm.integrations.otel.plumbing.providers import (
|
||||
destination_resource_attrs,
|
||||
)
|
||||
|
||||
extra = destination_resource_attrs(destination)
|
||||
if not extra:
|
||||
return span
|
||||
merged = Resource.create({**dict(span.resource.attributes), **extra})
|
||||
return _ResourceWrappedReadableSpan(span, merged)
|
||||
|
||||
|
||||
class _ResourceWrappedReadableSpan(ReadableSpan):
|
||||
"""A ``ReadableSpan`` view whose ``resource`` is overridden (for backend-specific attributes
|
||||
like Arize's ``model_id``) without mutating the underlying span."""
|
||||
|
||||
def __init__(self, inner: ReadableSpan, resource: Resource) -> None:
|
||||
super().__init__(
|
||||
name=inner.name,
|
||||
context=inner.context,
|
||||
parent=inner.parent,
|
||||
resource=resource,
|
||||
attributes=inner.attributes,
|
||||
events=inner.events,
|
||||
links=inner.links,
|
||||
kind=inner.kind,
|
||||
status=inner.status,
|
||||
start_time=inner.start_time,
|
||||
end_time=inner.end_time,
|
||||
instrumentation_scope=inner.instrumentation_scope,
|
||||
)
|
||||
|
||||
|
||||
class TenantFanOutSpanProcessor(SpanProcessor):
|
||||
"""Forward each finished proxy-internal span to every admin-resolved destination.
|
||||
|
||||
Destinations come from a request-scoped contextvar set during auth, so the processor is
|
||||
stateless across requests and concurrent requests are isolated by contextvars.
|
||||
"""
|
||||
|
||||
def __init__(self, owner_callback_name: str | None) -> None:
|
||||
self._owner = owner_callback_name
|
||||
self._processors: OrderedDict[tuple, SpanProcessor] = (
|
||||
OrderedDict()
|
||||
) # mutable-ok: bounded LRU span-processor cache
|
||||
|
||||
def on_start(self, span: Span, parent_context: Context | None = None) -> None:
|
||||
return None
|
||||
|
||||
def on_end(self, span: ReadableSpan) -> None:
|
||||
destinations = request_destinations()
|
||||
if not destinations:
|
||||
return
|
||||
if _is_genai_span(span):
|
||||
return
|
||||
for destination in destinations:
|
||||
processor = self._processor_for(destination)
|
||||
if processor is None:
|
||||
continue
|
||||
try:
|
||||
processor.on_end(_with_destination_resource(span, destination))
|
||||
except Exception as exc: # noqa: BLE001 # best-effort fan-out; one destination's failure must not break the others or the request
|
||||
verbose_logger.debug(
|
||||
"OTel V2 fan-out: forwarding span to %s failed: %s",
|
||||
destination.endpoint,
|
||||
exc,
|
||||
)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
# Snapshot before iterating: ``on_end`` mutates ``self._processors`` (insert /
|
||||
# move_to_end / popitem) on the span-ending thread and can run concurrently with
|
||||
# this SDK-driven shutdown, so iterating the live mapping risks a
|
||||
# "mutated during iteration" RuntimeError that the per-item except can't catch.
|
||||
for processor in tuple(self._processors.values()):
|
||||
try:
|
||||
processor.shutdown()
|
||||
except Exception as exc: # noqa: BLE001 # a single processor's shutdown failure must not abort shutting down the rest
|
||||
verbose_logger.debug("OTel V2 fan-out: processor shutdown failed: %s", exc)
|
||||
self._processors.clear()
|
||||
|
||||
def force_flush(self, timeout_millis: int = 30000) -> bool:
|
||||
all_ok = True
|
||||
# Snapshot before iterating (see ``shutdown``): a concurrent ``on_end`` mutating
|
||||
# the processor cache must not abort the flush and drop the remaining destinations'
|
||||
# buffered spans.
|
||||
for processor in tuple(self._processors.values()):
|
||||
try:
|
||||
if not processor.force_flush(timeout_millis):
|
||||
all_ok = False
|
||||
except Exception: # noqa: BLE001 # a single processor's flush failure must not fail the whole force_flush
|
||||
all_ok = False
|
||||
return all_ok
|
||||
|
||||
def _processor_for(self, destination: OtelDestination) -> SpanProcessor | None:
|
||||
key = _processor_key(destination)
|
||||
cached = self._processors.get(key)
|
||||
if cached is not None:
|
||||
self._processors.move_to_end(key)
|
||||
return cached
|
||||
from litellm.integrations.otel.plumbing.providers import (
|
||||
_exporter_from_spec,
|
||||
default_otlp_kind_for_backend,
|
||||
)
|
||||
from litellm.integrations.otel.plumbing.providers import (
|
||||
_processor_for as _build_processor,
|
||||
)
|
||||
|
||||
try:
|
||||
spec = ExporterSpec(
|
||||
kind=destination.protocol or default_otlp_kind_for_backend(destination.callback_name),
|
||||
endpoint=destination.endpoint,
|
||||
headers=destination.header_string(),
|
||||
owner=None,
|
||||
)
|
||||
exporter = _exporter_from_spec(spec)
|
||||
processor = _build_processor(exporter, use_simple=False)
|
||||
except Exception as exc: # noqa: BLE001 # a malformed destination spec must not break fan-out; skip this destination
|
||||
verbose_logger.debug(
|
||||
"OTel V2 fan-out: failed to build processor for %s: %s",
|
||||
destination.endpoint,
|
||||
exc,
|
||||
)
|
||||
return None
|
||||
self._processors[key] = processor
|
||||
if len(self._processors) > _MAX_CACHED_PROCESSORS:
|
||||
_, evicted = self._processors.popitem(last=False)
|
||||
_shutdown_in_background(evicted)
|
||||
return processor
|
||||
|
|
|
|||
|
|
@ -6,14 +6,22 @@ vocabularies to apply, and any resource attributes. ``PRESET_BY_CALLBACK``
|
|||
maps a callback name (``"arize"``, ``"langfuse_otel"``, ...) to its preset so
|
||||
the factory in ``litellm_logging`` can resolve a name and build a single
|
||||
``OpenTelemetryV2`` instance from the result.
|
||||
|
||||
Admin-owned trace destinations are resolved server-side from a named credential
|
||||
into an ``OtelDestination`` (see ``litellm.integrations.otel.presets.destinations``
|
||||
and ``plumbing.routing``). Separately, per-request team/key OTLP credentials from
|
||||
``standard_callback_dynamic_params`` route the gen-AI span to a credential-scoped
|
||||
tracer for the integrations that support it; ``dynamic_otlp_headers`` below builds
|
||||
those per-request headers.
|
||||
"""
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Final
|
||||
from typing import Final, TYPE_CHECKING
|
||||
|
||||
from litellm.integrations.otel.presets.agentops import agentops_preset
|
||||
from litellm.integrations.otel.presets.arize import arize_dynamic_headers, arize_preset
|
||||
from litellm.integrations.otel.presets.base import Preset
|
||||
from litellm.integrations.otel.presets.generic import generic_preset
|
||||
from litellm.integrations.otel.presets.langfuse import (
|
||||
langfuse_dynamic_headers,
|
||||
langfuse_preset,
|
||||
|
|
@ -24,23 +32,14 @@ from litellm.integrations.otel.presets.phoenix import phoenix_preset
|
|||
from litellm.integrations.otel.presets.weave import weave_dynamic_headers, weave_preset
|
||||
from litellm.types.utils import StandardCallbackDynamicParams
|
||||
|
||||
#: Callback name → preset. The ``Preset`` annotation makes mypy verify every
|
||||
#: registered value matches the preset interface.
|
||||
PRESET_BY_CALLBACK: Final[dict[str, Preset]] = {
|
||||
"agentops": agentops_preset,
|
||||
"arize": arize_preset,
|
||||
"arize_phoenix": phoenix_preset,
|
||||
"langfuse_otel": langfuse_preset,
|
||||
"langtrace": langtrace_preset,
|
||||
"levo": levo_preset,
|
||||
"weave_otel": weave_preset,
|
||||
}
|
||||
if TYPE_CHECKING:
|
||||
from litellm.integrations.otel.model.destination import OtelDestination
|
||||
|
||||
#: Callback name → per-request OTLP header builder (team/key multi-tenant
|
||||
#: routing). Only integrations that support dynamic credentials appear here —
|
||||
#: Arize-Phoenix/Langtrace/Levo/AgentOps don't, so they use the logger's
|
||||
#: Arize-Phoenix/Langtrace/Levo/AgentOps/generic don't, so they use the logger's
|
||||
#: default tracer.
|
||||
DYNAMIC_HEADERS_BY_CALLBACK: Final[dict[str, Callable[[StandardCallbackDynamicParams], dict[str, str]]]] = {
|
||||
DYNAMIC_HEADERS_BY_CALLBACK: dict[str, Callable[[StandardCallbackDynamicParams], dict[str, str]]] = {
|
||||
"arize": arize_dynamic_headers,
|
||||
"langfuse_otel": langfuse_dynamic_headers,
|
||||
"weave_otel": weave_dynamic_headers,
|
||||
|
|
@ -49,7 +48,7 @@ DYNAMIC_HEADERS_BY_CALLBACK: Final[dict[str, Callable[[StandardCallbackDynamicPa
|
|||
|
||||
def dynamic_otlp_headers(
|
||||
callback_name: str | None,
|
||||
dynamic_params: StandardCallbackDynamicParams | None,
|
||||
dynamic_params: "StandardCallbackDynamicParams | None",
|
||||
) -> dict[str, str] | None:
|
||||
"""Per-request OTLP headers for ``callback_name``, or ``None`` if N/A.
|
||||
|
||||
|
|
@ -62,13 +61,48 @@ def dynamic_otlp_headers(
|
|||
return headers or None
|
||||
|
||||
|
||||
def dynamic_otlp_destination(
|
||||
callback_name: str | None,
|
||||
dynamic_params: "StandardCallbackDynamicParams | None",
|
||||
) -> "OtelDestination | None":
|
||||
"""The destination a request's own team/key credentials export to, or ``None``.
|
||||
|
||||
Resolved through the admin-destination builders so a team's ``callback_vars`` reach
|
||||
exactly the account an equivalent destination would, honouring per-tenant overrides
|
||||
such as ``langfuse_host``. The builder's ``protocol`` comes with it: a transport the
|
||||
values pin is the tenant's, not the backend's intrinsic default.
|
||||
"""
|
||||
from litellm.integrations.otel.presets.destinations import build_destination
|
||||
|
||||
if callback_name not in DYNAMIC_HEADERS_BY_CALLBACK or not dynamic_params:
|
||||
return None
|
||||
values = {str(key): str(value) for key, value in dynamic_params.items() if isinstance(value, str)}
|
||||
return build_destination(callback_name or "", values)
|
||||
|
||||
|
||||
#: Callback name → preset. The ``Preset`` annotation makes mypy verify every
|
||||
#: registered value matches the preset interface.
|
||||
PRESET_BY_CALLBACK: dict[str, Preset] = {
|
||||
"agentops": agentops_preset,
|
||||
"arize": arize_preset,
|
||||
"arize_phoenix": phoenix_preset,
|
||||
"generic": generic_preset,
|
||||
"langfuse_otel": langfuse_preset,
|
||||
"langtrace": langtrace_preset,
|
||||
"levo": levo_preset,
|
||||
"weave_otel": weave_preset,
|
||||
}
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DYNAMIC_HEADERS_BY_CALLBACK",
|
||||
"PRESET_BY_CALLBACK",
|
||||
"Preset",
|
||||
"agentops_preset",
|
||||
"arize_preset",
|
||||
"dynamic_otlp_destination",
|
||||
"dynamic_otlp_headers",
|
||||
"generic_preset",
|
||||
"langfuse_preset",
|
||||
"langtrace_preset",
|
||||
"levo_preset",
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ class _AgentOpsSettings(BaseSettings):
|
|||
def agentops_preset(
|
||||
*,
|
||||
config_overrides: OpenTelemetryV2Config | None = None,
|
||||
allow_missing_credentials: bool = False,
|
||||
) -> OpenTelemetryV2Config:
|
||||
"""Build the AgentOps config without any network I/O.
|
||||
|
||||
|
|
@ -49,17 +50,21 @@ def agentops_preset(
|
|||
"""
|
||||
settings: Final = _AgentOpsSettings()
|
||||
base: Final = config_overrides or OpenTelemetryV2Config()
|
||||
global_exporter = (
|
||||
()
|
||||
if allow_missing_credentials and not settings.api_key
|
||||
else (
|
||||
ExporterSpec(
|
||||
kind=_AGENTOPS_EXPORTER_KIND,
|
||||
endpoint=_AGENTOPS_ENDPOINT,
|
||||
options=({"api_key": settings.api_key} if settings.api_key else None),
|
||||
owner=ExporterOwner.AGENTOPS,
|
||||
),
|
||||
)
|
||||
)
|
||||
return base.model_copy(
|
||||
update={
|
||||
"exporters": [
|
||||
*base.exporters,
|
||||
ExporterSpec(
|
||||
kind=_AGENTOPS_EXPORTER_KIND,
|
||||
endpoint=_AGENTOPS_ENDPOINT,
|
||||
options=({"api_key": settings.api_key} if settings.api_key else None),
|
||||
owner=ExporterOwner.AGENTOPS,
|
||||
),
|
||||
],
|
||||
"exporters": [*base.exporters, *global_exporter],
|
||||
"resource_attributes": {
|
||||
**base.resource_attributes,
|
||||
"service.name": settings.service_name,
|
||||
|
|
|
|||
|
|
@ -14,6 +14,8 @@ from litellm.integrations.otel.model.config import (
|
|||
from litellm.integrations.otel.presets.utils import ensure_mappers
|
||||
from litellm.types.utils import StandardCallbackDynamicParams
|
||||
|
||||
ARIZE_PUBLIC_OTLP_ENDPOINT = "https://otlp.arize.com/v1"
|
||||
|
||||
|
||||
class _ArizeSettings(BaseSettings):
|
||||
model_config = SettingsConfigDict(case_sensitive=False, extra="ignore")
|
||||
|
|
@ -21,26 +23,35 @@ class _ArizeSettings(BaseSettings):
|
|||
# Standard OTLP headers env var, used as the fallback when no Arize
|
||||
# credentials are configured.
|
||||
otlp_traces_headers: str | None = Field(default=None, validation_alias="OTEL_EXPORTER_OTLP_TRACES_HEADERS")
|
||||
grpc_endpoint: str | None = Field(default=None, validation_alias="ARIZE_ENDPOINT")
|
||||
http_endpoint: str | None = Field(default=None, validation_alias="ARIZE_HTTP_ENDPOINT")
|
||||
|
||||
|
||||
def arize_preset(
|
||||
*,
|
||||
config_overrides: OpenTelemetryV2Config | None = None,
|
||||
allow_missing_credentials: bool = False,
|
||||
) -> OpenTelemetryV2Config:
|
||||
arize_cfg: Final = _V1ArizeLogger.get_arize_config()
|
||||
headers: Final = _arize_headers(arize_cfg)
|
||||
settings = _ArizeSettings()
|
||||
has_own_endpoint = bool(settings.grpc_endpoint or settings.http_endpoint)
|
||||
has_credentials = bool(arize_cfg.space_id or arize_cfg.space_key or arize_cfg.api_key)
|
||||
base: Final = config_overrides or OpenTelemetryV2Config()
|
||||
global_exporter = (
|
||||
()
|
||||
if allow_missing_credentials and not has_credentials and not has_own_endpoint
|
||||
else (
|
||||
ExporterSpec(
|
||||
kind=arize_cfg.protocol or "otlp_grpc",
|
||||
endpoint=arize_cfg.endpoint or ARIZE_PUBLIC_OTLP_ENDPOINT,
|
||||
headers=_arize_headers(arize_cfg, settings, has_own_endpoint),
|
||||
owner=ExporterOwner.ARIZE_AX,
|
||||
),
|
||||
)
|
||||
)
|
||||
return base.model_copy(
|
||||
update={
|
||||
"exporters": [
|
||||
*base.exporters,
|
||||
ExporterSpec(
|
||||
kind=arize_cfg.protocol or "otlp_grpc",
|
||||
endpoint=arize_cfg.endpoint or "https://otlp.arize.com/v1",
|
||||
headers=headers,
|
||||
owner=ExporterOwner.ARIZE_AX,
|
||||
),
|
||||
],
|
||||
"exporters": [*base.exporters, *global_exporter],
|
||||
"mapper_names": ensure_mappers(base.mapper_names, "openinference"),
|
||||
"resource_attributes": {
|
||||
**base.resource_attributes,
|
||||
|
|
@ -50,24 +61,25 @@ def arize_preset(
|
|||
)
|
||||
|
||||
|
||||
def _arize_headers(arize_cfg) -> str | None:
|
||||
pieces: Final = []
|
||||
if arize_cfg.space_id or arize_cfg.space_key:
|
||||
pieces.append(f"space_id={arize_cfg.space_id or arize_cfg.space_key}")
|
||||
if arize_cfg.api_key:
|
||||
pieces.append(f"api_key={arize_cfg.api_key}")
|
||||
if not pieces:
|
||||
# Fall back to the standard OTLP headers env var when no Arize
|
||||
# credentials are configured.
|
||||
return _ArizeSettings().otlp_traces_headers
|
||||
return ",".join(pieces)
|
||||
def _arize_headers(arize_cfg, settings: "_ArizeSettings", has_own_endpoint: bool) -> str | None:
|
||||
space: Final = arize_cfg.space_id or arize_cfg.space_key
|
||||
pieces: Final = (
|
||||
*((f"space_id={space}",) if space else ()),
|
||||
*((f"api_key={arize_cfg.api_key}",) if arize_cfg.api_key else ()),
|
||||
)
|
||||
if pieces:
|
||||
return ",".join(pieces)
|
||||
# Fall back to the standard OTLP headers env var only for an operator's own
|
||||
# collector. Sending it to the public Arize endpoint would hand that collector's
|
||||
# auth header to a vendor the operator has no account with.
|
||||
return settings.otlp_traces_headers if has_own_endpoint else None
|
||||
|
||||
|
||||
def arize_dynamic_headers(params: StandardCallbackDynamicParams) -> dict[str, str]:
|
||||
"""Per-request Arize OTLP headers from team/key dynamic params."""
|
||||
headers: Final[dict[str, str]] = {}
|
||||
headers: dict[str, str] = {}
|
||||
# ``arize_space_key`` is the suggested param and wins over ``arize_space_id``.
|
||||
space: Final = params.get("arize_space_key") or params.get("arize_space_id")
|
||||
space = params.get("arize_space_key") or params.get("arize_space_id")
|
||||
if space:
|
||||
headers["arize-space-id"] = space
|
||||
api_key: Final = params.get("arize_api_key")
|
||||
|
|
|
|||
|
|
@ -18,6 +18,20 @@ class Preset(Protocol):
|
|||
|
||||
``config_overrides`` lets one preset layer onto another's config (or onto
|
||||
test-supplied defaults); the factory calls presets with no arguments.
|
||||
|
||||
``allow_missing_credentials`` distinguishes the two reasons a preset is built.
|
||||
A credential-mandatory backend (weave/langfuse/levo) raises when its global env
|
||||
credentials are absent so a misconfigured global callback fails loud at startup;
|
||||
set this when the only reason for construction is an admin-owned destination,
|
||||
which carries its own per-tenant credentials, so the preset degrades to an
|
||||
exporter-less (mapper-only) config instead of raising. Credential-optional
|
||||
backends (arize/phoenix/agentops/langtrace) already run without a global
|
||||
exporter and ignore it.
|
||||
"""
|
||||
|
||||
def __call__(self, *, config_overrides: OpenTelemetryV2Config | None = None) -> OpenTelemetryV2Config: ...
|
||||
def __call__(
|
||||
self,
|
||||
*,
|
||||
config_overrides: OpenTelemetryV2Config | None = None,
|
||||
allow_missing_credentials: bool = False,
|
||||
) -> OpenTelemetryV2Config: ...
|
||||
|
|
|
|||
21
litellm/integrations/otel/presets/generic.py
Normal file
21
litellm/integrations/otel/presets/generic.py
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
"""Generic OTLP passthrough preset.
|
||||
|
||||
A ``generic`` admin-owned destination carries only an ``otel_endpoint`` /
|
||||
``otel_headers`` (no vendor adapter), so this preset is deliberately vendor-neutral:
|
||||
it yields a base ``OpenTelemetryV2`` config with the standard ``gen_ai.*`` (+ ``legacy``)
|
||||
span vocabulary, NO vendor mapper, and NO exporter of its own. The per-destination
|
||||
exporter is appended by ``TenantTracerCache`` from the resolved ``OtelDestination``, so a
|
||||
generic destination receives the same complete trace tree as Arize/Langfuse/Weave --
|
||||
root server span, auth/db internal spans, the ``chat <model>`` gen-AI span, and
|
||||
cost/error spans. It needs no global OTEL env vars and never raises.
|
||||
"""
|
||||
|
||||
from litellm.integrations.otel.model.config import OpenTelemetryV2Config
|
||||
|
||||
|
||||
def generic_preset(
|
||||
*,
|
||||
config_overrides: OpenTelemetryV2Config | None = None,
|
||||
allow_missing_credentials: bool = False,
|
||||
) -> OpenTelemetryV2Config:
|
||||
return config_overrides or OpenTelemetryV2Config()
|
||||
|
|
@ -14,29 +14,6 @@ from litellm.integrations.otel.presets.utils import ensure_mappers
|
|||
from litellm.types.utils import StandardCallbackDynamicParams
|
||||
|
||||
|
||||
def langfuse_preset(
|
||||
*,
|
||||
config_overrides: OpenTelemetryV2Config | None = None,
|
||||
) -> OpenTelemetryV2Config:
|
||||
cfg: Final = _V1Langfuse.get_langfuse_otel_config()
|
||||
kind: Final = cfg.exporter if isinstance(cfg.exporter, str) else "otlp_http"
|
||||
base: Final = config_overrides or OpenTelemetryV2Config()
|
||||
return base.model_copy(
|
||||
update={
|
||||
"exporters": [
|
||||
*base.exporters,
|
||||
ExporterSpec(
|
||||
kind=kind,
|
||||
endpoint=cfg.endpoint,
|
||||
headers=cfg.headers,
|
||||
owner=ExporterOwner.LANGFUSE_OTEL,
|
||||
),
|
||||
],
|
||||
"mapper_names": ensure_mappers(base.mapper_names, "langfuse"),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def langfuse_dynamic_headers(params: StandardCallbackDynamicParams) -> dict[str, str]:
|
||||
"""Per-request Langfuse OTLP headers from team/key dynamic params."""
|
||||
public_key: Final = params.get("langfuse_public_key")
|
||||
|
|
@ -48,3 +25,33 @@ def langfuse_dynamic_headers(params: StandardCallbackDynamicParams) -> dict[str,
|
|||
)
|
||||
}
|
||||
return {}
|
||||
|
||||
|
||||
def langfuse_preset(
|
||||
*,
|
||||
config_overrides: OpenTelemetryV2Config | None = None,
|
||||
allow_missing_credentials: bool = False,
|
||||
) -> OpenTelemetryV2Config:
|
||||
base: Final = config_overrides or OpenTelemetryV2Config()
|
||||
mappers = ensure_mappers(base.mapper_names, "langfuse")
|
||||
try:
|
||||
cfg = _V1Langfuse.get_langfuse_otel_config()
|
||||
except Exception:
|
||||
if not allow_missing_credentials:
|
||||
raise
|
||||
return base.model_copy(update={"mapper_names": mappers})
|
||||
kind: Final = cfg.exporter if isinstance(cfg.exporter, str) else "otlp_http"
|
||||
return base.model_copy(
|
||||
update={
|
||||
"exporters": [
|
||||
*base.exporters,
|
||||
ExporterSpec(
|
||||
kind=kind,
|
||||
endpoint=cfg.endpoint,
|
||||
headers=cfg.headers,
|
||||
owner=ExporterOwner.LANGFUSE_OTEL,
|
||||
),
|
||||
],
|
||||
"mapper_names": mappers,
|
||||
}
|
||||
)
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ from litellm.integrations.otel.presets.utils import ensure_mappers
|
|||
def langtrace_preset(
|
||||
*,
|
||||
config_overrides: OpenTelemetryV2Config | None = None,
|
||||
allow_missing_credentials: bool = False,
|
||||
) -> OpenTelemetryV2Config:
|
||||
"""Compose the Langtrace mapper on top of the customer's OTLP destination.
|
||||
|
||||
|
|
|
|||
|
|
@ -13,9 +13,15 @@ from litellm.integrations.otel.model.config import (
|
|||
def levo_preset(
|
||||
*,
|
||||
config_overrides: OpenTelemetryV2Config | None = None,
|
||||
allow_missing_credentials: bool = False,
|
||||
) -> OpenTelemetryV2Config:
|
||||
cfg: Final = _V1Levo.get_levo_config()
|
||||
base: Final = config_overrides or OpenTelemetryV2Config()
|
||||
try:
|
||||
cfg = _V1Levo.get_levo_config()
|
||||
except Exception:
|
||||
if not allow_missing_credentials:
|
||||
raise
|
||||
return base
|
||||
return base.model_copy(
|
||||
update={
|
||||
"exporters": [
|
||||
|
|
|
|||
|
|
@ -28,22 +28,23 @@ class _PhoenixSettings(BaseSettings):
|
|||
def phoenix_preset(
|
||||
*,
|
||||
config_overrides: OpenTelemetryV2Config | None = None,
|
||||
allow_missing_credentials: bool = False,
|
||||
) -> OpenTelemetryV2Config:
|
||||
cfg: Final = _V1Phoenix.get_arize_phoenix_config()
|
||||
headers: Final = cfg.otlp_auth_headers if hasattr(cfg, "otlp_auth_headers") else None
|
||||
project_name: Final = _PhoenixSettings().project_name
|
||||
base: Final = config_overrides or OpenTelemetryV2Config()
|
||||
global_exporter = (
|
||||
ExporterSpec(
|
||||
kind=cfg.protocol if hasattr(cfg, "protocol") else "otlp_http",
|
||||
endpoint=cfg.endpoint,
|
||||
headers=headers,
|
||||
owner=ExporterOwner.ARIZE_PHOENIX,
|
||||
),
|
||||
)
|
||||
return base.model_copy(
|
||||
update={
|
||||
"exporters": [
|
||||
*base.exporters,
|
||||
ExporterSpec(
|
||||
kind=cfg.protocol if hasattr(cfg, "protocol") else "otlp_http",
|
||||
endpoint=cfg.endpoint,
|
||||
headers=headers,
|
||||
owner=ExporterOwner.ARIZE_PHOENIX,
|
||||
),
|
||||
],
|
||||
"exporters": [*base.exporters, *global_exporter],
|
||||
"mapper_names": ensure_mappers(base.mapper_names, "openinference"),
|
||||
"resource_attributes": {
|
||||
**base.resource_attributes,
|
||||
|
|
|
|||
|
|
@ -15,12 +15,31 @@ from litellm.integrations.weave.weave_otel import (
|
|||
from litellm.types.utils import StandardCallbackDynamicParams
|
||||
|
||||
|
||||
def weave_dynamic_headers(params: StandardCallbackDynamicParams) -> dict[str, str]:
|
||||
"""Per-request Weave OTLP headers from team/key dynamic params."""
|
||||
headers: dict[str, str] = {}
|
||||
api_key: Final = params.get("wandb_api_key")
|
||||
if api_key:
|
||||
headers["Authorization"] = _get_weave_authorization_header(api_key=api_key)
|
||||
project_id: Final = params.get("weave_project_id")
|
||||
if project_id:
|
||||
headers["project_id"] = project_id
|
||||
return headers
|
||||
|
||||
|
||||
def weave_preset(
|
||||
*,
|
||||
config_overrides: OpenTelemetryV2Config | None = None,
|
||||
allow_missing_credentials: bool = False,
|
||||
) -> OpenTelemetryV2Config:
|
||||
weave_cfg: Final = get_weave_otel_config()
|
||||
base: Final = config_overrides or OpenTelemetryV2Config()
|
||||
mappers = ensure_mappers(base.mapper_names, "openinference", "weave")
|
||||
try:
|
||||
weave_cfg = get_weave_otel_config()
|
||||
except Exception:
|
||||
if not allow_missing_credentials:
|
||||
raise
|
||||
return base.model_copy(update={"mapper_names": mappers})
|
||||
return base.model_copy(
|
||||
update={
|
||||
"exporters": [
|
||||
|
|
@ -32,19 +51,6 @@ def weave_preset(
|
|||
owner=ExporterOwner.WEAVE_OTEL,
|
||||
),
|
||||
],
|
||||
# Weave consumes OpenInference + a small Weave-specific overlay.
|
||||
"mapper_names": ensure_mappers(base.mapper_names, "openinference", "weave"),
|
||||
"mapper_names": mappers,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def weave_dynamic_headers(params: StandardCallbackDynamicParams) -> dict[str, str]:
|
||||
"""Per-request Weave OTLP headers from team/key dynamic params."""
|
||||
headers: Final[dict[str, str]] = {}
|
||||
api_key: Final = params.get("wandb_api_key")
|
||||
if api_key:
|
||||
headers["Authorization"] = _get_weave_authorization_header(api_key=api_key)
|
||||
project_id: Final = params.get("weave_project_id")
|
||||
if project_id:
|
||||
headers["project_id"] = project_id
|
||||
return headers
|
||||
|
|
|
|||
|
|
@ -60,6 +60,7 @@ from litellm.integrations.custom_guardrail import CustomGuardrail
|
|||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.integrations.deepeval.deepeval import DeepEvalLogger
|
||||
from litellm.integrations.mlflow import MlflowLogger
|
||||
from litellm.integrations.otel.model.config import OpenTelemetryV2Config
|
||||
from litellm.integrations.sqs import SQSLogger
|
||||
from litellm.litellm_core_utils.core_helpers import reconstruct_model_name
|
||||
from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
|
||||
|
|
@ -4024,9 +4025,11 @@ def _init_custom_logger_compatible_class(
|
|||
for callback in _in_memory_loggers:
|
||||
if type(callback) is OpenTelemetryV2:
|
||||
return callback
|
||||
otel_logger_v2: Final = OpenTelemetryV2(
|
||||
**_get_custom_logger_settings_from_proxy_server(callback_name=logging_integration)
|
||||
)
|
||||
settings = _get_custom_logger_settings_from_proxy_server(callback_name=logging_integration)
|
||||
config = OpenTelemetryV2Config(**settings)
|
||||
if not config.exporters:
|
||||
config = OpenTelemetryV2Config(**{**settings, "exporter": config.exporter})
|
||||
otel_logger_v2 = OpenTelemetryV2(config=config)
|
||||
_in_memory_loggers.append(otel_logger_v2)
|
||||
_maybe_auto_initialize_arize_phoenix(_in_memory_loggers)
|
||||
return otel_logger_v2
|
||||
|
|
@ -4237,6 +4240,8 @@ def _init_custom_logger_compatible_class(
|
|||
_otel_logger = WeaveOtelLogger(config=otel_config, callback_name="weave_otel")
|
||||
_in_memory_loggers.append(_otel_logger)
|
||||
return _otel_logger
|
||||
elif logging_integration == "generic":
|
||||
return _maybe_construct_otel_v2("generic", _in_memory_loggers)
|
||||
elif logging_integration == "pagerduty":
|
||||
for callback in _in_memory_loggers:
|
||||
if isinstance(callback, PagerDutyAlerting):
|
||||
|
|
@ -4362,11 +4367,17 @@ def _init_custom_logger_compatible_class(
|
|||
|
||||
|
||||
def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[CustomLogger]) -> "OpenTelemetryV2 | None":
|
||||
"""If ``LITELLM_OTEL_V2`` is on, build (or reuse) a single ``OpenTelemetryV2``
|
||||
instance configured via the preset for ``callback_name``.
|
||||
"""Build (or reuse) a single ``OpenTelemetryV2`` instance configured via the
|
||||
preset for ``callback_name`` when V2 owns this backend.
|
||||
|
||||
Returns ``None`` when V2 is off OR when there's no preset registered for
|
||||
``callback_name`` — callers should then fall through to the legacy path.
|
||||
Ownership is the operator's configuration alone: the ``LITELLM_OTEL_V2`` flag plus
|
||||
whatever credentials or ``OTEL_*`` settings the backend has. Returns ``None`` when the
|
||||
flag is off, when no preset is registered for ``callback_name``, or when the preset
|
||||
cannot build; callers then fall through to the legacy path.
|
||||
|
||||
Admin-owned destinations deliberately play no part here. They are sinks, delivered to
|
||||
by ``AdminDestinationLogger``, so registering one for a single team cannot move any
|
||||
other tenant's backend onto a different logger.
|
||||
"""
|
||||
from litellm.integrations.otel.model.config import is_otel_v2_enabled
|
||||
|
||||
|
|
@ -4382,16 +4393,53 @@ def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[Custom
|
|||
if isinstance(callback, OpenTelemetryV2) and getattr(callback, "callback_name", None) == callback_name:
|
||||
return callback
|
||||
try:
|
||||
config: Final = preset_fn()
|
||||
config = preset_fn(allow_missing_credentials=False)
|
||||
except Exception:
|
||||
# If env vars are missing or the preset raises, defer to the legacy path
|
||||
# so customers get the same error story they had before V2 landed.
|
||||
return None
|
||||
v2_logger: Final = OpenTelemetryV2(config=config, callback_name=callback_name)
|
||||
if not config.exporters:
|
||||
# An operator asked for this backend and nothing resolved to send spans to, so it
|
||||
# would export nothing at all. v2 no longer degrades to a console exporter (that
|
||||
# printed every span, prompt content included, synchronously on the request path),
|
||||
# so without this the deployment is silently dark.
|
||||
verbose_logger.warning(
|
||||
"OTel v2: '%s' is enabled but no exporter is configured, so no spans will be "
|
||||
"exported. Set OTEL_EXPORTER/OTEL_ENDPOINT, this backend's credentials, or "
|
||||
"register a logging destination for it.",
|
||||
callback_name,
|
||||
)
|
||||
v2_logger = OpenTelemetryV2(config=config, callback_name=callback_name)
|
||||
_in_memory_loggers.append(v2_logger)
|
||||
return v2_logger
|
||||
|
||||
|
||||
def otel_v2_owned_backends() -> frozenset[str]:
|
||||
"""Backend names an ``OpenTelemetryV2`` instance already owns.
|
||||
|
||||
An owning logger fans its gen-AI span out to the request's destinations for its own
|
||||
backend, so the destination sink must not emit for these or the destination receives
|
||||
the same call twice as two sibling spans.
|
||||
|
||||
Keyed on actual dispatch, not construction. A logger can be built and land in
|
||||
``_in_memory_loggers`` while never reaching the success-callback list, and treating
|
||||
that as owned made the sink stand down for a backend nobody delivers, turning a
|
||||
duplicate into a silent total loss of the tenant's traces.
|
||||
|
||||
``OpenTelemetryV2`` is imported here rather than at module scope because it pulls
|
||||
``opentelemetry``, which ships only in the proxy extras: importing it eagerly makes
|
||||
``import litellm`` fail for an SDK install that never asked for OTEL.
|
||||
"""
|
||||
from litellm.integrations.otel.logger import OpenTelemetryV2
|
||||
|
||||
dispatched = litellm._async_success_callback
|
||||
return frozenset(
|
||||
name
|
||||
for logger in _in_memory_loggers
|
||||
if isinstance(logger, OpenTelemetryV2)
|
||||
and (name := getattr(logger, "callback_name", None))
|
||||
and any(cb is logger for cb in dispatched)
|
||||
)
|
||||
|
||||
|
||||
def _maybe_auto_initialize_arize_phoenix(_in_memory_loggers: list[CustomLogger]) -> None:
|
||||
"""
|
||||
Auto-initialize ArizePhoenixLogger when Phoenix env vars are detected.
|
||||
|
|
@ -4556,6 +4604,12 @@ def get_custom_logger_compatible_class(
|
|||
for callback in _in_memory_loggers:
|
||||
if isinstance(callback, ArizeLogger) and callback.callback_name == "arize":
|
||||
return callback
|
||||
elif logging_integration == "generic":
|
||||
from litellm.integrations.otel.logger import OpenTelemetryV2
|
||||
|
||||
for callback in _in_memory_loggers:
|
||||
if isinstance(callback, OpenTelemetryV2) and getattr(callback, "callback_name", None) == "generic":
|
||||
return callback
|
||||
elif logging_integration == "logfire":
|
||||
if "LOGFIRE_TOKEN" not in os.environ:
|
||||
raise ValueError("LOGFIRE_TOKEN not found in environment variables")
|
||||
|
|
|
|||
|
|
@ -4790,6 +4790,25 @@ if MCP_AVAILABLE:
|
|||
|
||||
return stored
|
||||
|
||||
async def _refresh_request_otel_destinations(user_api_key_auth: Any) -> None:
|
||||
"""Re-resolve the caller's admin destinations for THIS JSON-RPC message.
|
||||
|
||||
A stateful MCP session dispatches every later message on descendants of the task
|
||||
its ``initialize`` POST spawned, and a ContextVar is copied at task creation. The
|
||||
destinations resolved for the first message therefore stay frozen on that task for
|
||||
the session's whole life, so re-scoping or deleting a destination mid-session left
|
||||
the revoked sink still receiving that team's spans. Resolving per message keeps the
|
||||
access map authoritative for a session that outlives an admin's edit.
|
||||
"""
|
||||
if user_api_key_auth is None:
|
||||
return
|
||||
try:
|
||||
from litellm.proxy.litellm_pre_call_utils import _apply_admin_logging_exporters
|
||||
|
||||
await _apply_admin_logging_exporters(user_api_key_auth)
|
||||
except Exception as exc: # noqa: BLE001 # a resolver failure must not break the MCP call
|
||||
verbose_logger.debug("MCP: could not refresh otel destinations: %s", exc)
|
||||
|
||||
async def get_or_extract_auth_context() -> tuple[
|
||||
UserAPIKeyAuth | None,
|
||||
str | None,
|
||||
|
|
@ -4826,6 +4845,7 @@ if MCP_AVAILABLE:
|
|||
oauth2_headers = stored.oauth2_headers
|
||||
raw_headers = stored.raw_headers
|
||||
_client_ip = stored.client_ip
|
||||
await _refresh_request_otel_destinations(user_api_key_auth)
|
||||
return (
|
||||
user_api_key_auth,
|
||||
mcp_auth_header,
|
||||
|
|
|
|||
|
|
@ -489,6 +489,16 @@ def test_otlp_traces_endpoint_normalization():
|
|||
norm("https://x.splunk.com/v2/trace/otlp")
|
||||
== "https://x.splunk.com/v2/trace/otlp"
|
||||
)
|
||||
# Langtrace ingests at /api/trace (a complete path, not an OTLP base) — appending
|
||||
# /v1/traces 404s, so it must be left intact (regression for the double-append bug).
|
||||
assert (
|
||||
norm("https://app.langtrace.ai/api/trace")
|
||||
== "https://app.langtrace.ai/api/trace"
|
||||
)
|
||||
assert (
|
||||
norm("https://app.langtrace.ai/api/trace/")
|
||||
== "https://app.langtrace.ai/api/trace"
|
||||
)
|
||||
assert norm(None) is None
|
||||
|
||||
|
||||
|
|
@ -1051,3 +1061,100 @@ def test_sanitize_event_metadata_caps_value_length_and_handles_none():
|
|||
assert sanitize_event_metadata(None) == {}
|
||||
big = sanitize_event_metadata({"k": "v" * 5000})
|
||||
assert len(big["k"]) == 1024
|
||||
|
||||
|
||||
def test_blank_otel_v2_flag_reads_as_off(monkeypatch):
|
||||
"""Regression: a declared-but-empty ``LITELLM_OTEL_V2=`` raised a pydantic
|
||||
ValidationError. The flag is read while ``proxy_server`` is still importing, so the
|
||||
error escaped module import and the proxy never bound its port; a blank var is routine
|
||||
in k8s ConfigMaps and ``.env``. Lives here rather than beside the mount tests because
|
||||
those need an optional instrumentation package CI does not install, which would leave
|
||||
this uncovered.
|
||||
"""
|
||||
from litellm.integrations.otel.model.config import is_otel_v2_enabled
|
||||
|
||||
for raw in ('', ' '):
|
||||
monkeypatch.setenv('LITELLM_OTEL_V2', raw)
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
assert is_otel_v2_enabled() is False
|
||||
monkeypatch.setenv('LITELLM_OTEL_V2', '1')
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
assert is_otel_v2_enabled() is True
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
|
||||
|
||||
def test_no_otel_v2_flag_value_can_raise(monkeypatch):
|
||||
"""The flag is read while ``proxy_server`` is still importing, so any value that
|
||||
raises takes the proxy down before it binds. A stray space or a word pydantic does not
|
||||
accept is an easy typo and must degrade to off, while a padded boolean still means
|
||||
what it says."""
|
||||
from litellm.integrations.otel.model.config import is_otel_v2_enabled
|
||||
|
||||
def _read(raw):
|
||||
monkeypatch.setenv('LITELLM_OTEL_V2', raw)
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
return is_otel_v2_enabled()
|
||||
|
||||
for raw in ('enabled', '2', 'maybe', 'TRUE!', '-1'):
|
||||
assert _read(raw) is False, f'{raw!r} should degrade to off'
|
||||
for raw in ('true ', ' TRUE', ' yes '):
|
||||
assert _read(raw) is True, f'{raw!r} should read as on'
|
||||
for raw in ('false ', ' 0 '):
|
||||
assert _read(raw) is False, f'{raw!r} should read as off'
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
|
||||
|
||||
def test_each_backend_registers_itself_on_the_per_backend_event_lists():
|
||||
"""Regression: only the first OTel v2 backend ever dispatched.
|
||||
|
||||
v2 collapsed every backend onto one class parameterised by ``callback_name``, but the
|
||||
registration guard asked "is any OTel-module callback already here", so the first
|
||||
registrant locked out every later backend. Each one was still constructed, so it looked
|
||||
owned while never receiving an event, and its own configured exporter went dark. The
|
||||
event lists are per-backend; ``service_callback`` stays single-owner, or the
|
||||
proxy-internal service spans multiply once per configured backend.
|
||||
"""
|
||||
import litellm
|
||||
from litellm.integrations.otel.logger import OpenTelemetryV2
|
||||
from litellm.integrations.otel.model.config import OpenTelemetryV2Config
|
||||
|
||||
first = OpenTelemetryV2(config=OpenTelemetryV2Config(), callback_name="arize")
|
||||
second = OpenTelemetryV2(config=OpenTelemetryV2Config(), callback_name="langfuse_otel")
|
||||
|
||||
for lst in (litellm._async_success_callback, litellm._async_failure_callback, litellm.input_callback):
|
||||
lst.clear()
|
||||
first._register_in_callback_list(litellm._async_success_callback, per_backend=True)
|
||||
second._register_in_callback_list(litellm._async_success_callback, per_backend=True)
|
||||
first._register_in_callback_list(litellm._async_success_callback, per_backend=True)
|
||||
|
||||
names = [getattr(cb, "callback_name", None) for cb in litellm._async_success_callback]
|
||||
assert names == ["arize", "langfuse_otel"], names
|
||||
|
||||
service: list = []
|
||||
first._register_in_callback_list(service)
|
||||
second._register_in_callback_list(service)
|
||||
assert len(service) == 1, "service_callback must keep a single OTel owner"
|
||||
|
||||
|
||||
def test_owned_backends_counts_only_loggers_that_actually_dispatch():
|
||||
"""Regression: ownership read construction, so the destination sink stood down for a
|
||||
backend nobody delivers and the tenant's traces were lost outright rather than doubled.
|
||||
"""
|
||||
import litellm
|
||||
from litellm.integrations.otel.logger import OpenTelemetryV2
|
||||
from litellm.integrations.otel.model.config import OpenTelemetryV2Config
|
||||
from litellm.litellm_core_utils import litellm_logging as lm
|
||||
|
||||
dispatched = OpenTelemetryV2(config=OpenTelemetryV2Config(), callback_name="arize")
|
||||
constructed_only = OpenTelemetryV2(config=OpenTelemetryV2Config(), callback_name="langfuse_otel")
|
||||
litellm._async_success_callback.clear()
|
||||
litellm._async_success_callback.append(dispatched)
|
||||
original = lm._in_memory_loggers[:]
|
||||
try:
|
||||
lm._in_memory_loggers.clear()
|
||||
lm._in_memory_loggers.extend([dispatched, constructed_only])
|
||||
assert lm.otel_v2_owned_backends() == frozenset({"arize"})
|
||||
finally:
|
||||
lm._in_memory_loggers.clear()
|
||||
lm._in_memory_loggers.extend(original)
|
||||
litellm._async_success_callback.clear()
|
||||
|
|
|
|||
|
|
@ -1,150 +1,185 @@
|
|||
"""Per-request multi-tenant credential routing (V1 parity)."""
|
||||
"""Per-tenant tracer routing on admin-owned OTEL destinations, with fan-out, plus the
|
||||
per-request credential routing layered over it.
|
||||
|
||||
A request's identity chain is assigned a set of admin-owned exporters; the v2 logger fans
|
||||
the trace out to all of them (plus the configured/global exporter). Separately, a request's
|
||||
``standard_callback_dynamic_params`` (team/key OTLP credentials) route the gen-AI span to a
|
||||
credential-scoped tracer for the integrations that support it (V1 parity). These tests lock
|
||||
both: each destination's endpoint follows its resolved host (cross-host fix), the configured
|
||||
exporters are kept (global also receives), a logger only exports the destinations tagged with
|
||||
its own backend, and request credentials rewrite only their own backend's exporter headers.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, os.path.abspath("../../../.."))
|
||||
|
||||
from opentelemetry.trace import NoOpTracer
|
||||
|
||||
from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config
|
||||
from litellm.integrations.otel.presets import dynamic_otlp_headers
|
||||
from litellm.integrations.otel.model.destination import OtelDestination
|
||||
from litellm.integrations.otel.model.metadata import LLMCallEvent
|
||||
from litellm.integrations.otel.plumbing.context import (
|
||||
_request_destinations,
|
||||
set_request_destinations,
|
||||
)
|
||||
from litellm.integrations.otel.plumbing.routing import TenantTracerCache
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_request_destinations():
|
||||
"""The v2 router reads destinations from a server-only ContextVar. Reset it around
|
||||
each test so a prior test's anchored destinations never leak into the next."""
|
||||
token = _request_destinations.set(())
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_request_destinations.reset(token)
|
||||
|
||||
|
||||
def _cache(callback_name, exporters=None):
|
||||
cfg = OpenTelemetryV2Config(exporters=exporters or [ExporterSpec(kind="in_memory")])
|
||||
cfg = OpenTelemetryV2Config(
|
||||
exporters=exporters or [ExporterSpec(kind="in_memory", owner=callback_name)]
|
||||
)
|
||||
return TenantTracerCache(cfg, callback_name, "litellm")
|
||||
|
||||
|
||||
# --- header builders mirror the V1 construct_dynamic_otel_headers overrides --- #
|
||||
|
||||
|
||||
def test_arize_dynamic_headers():
|
||||
headers = dynamic_otlp_headers(
|
||||
"arize", {"arize_space_id": "S", "arize_api_key": "K"}
|
||||
def _dest(endpoint, auth="Basic AAAA", backend="langfuse_otel"):
|
||||
return OtelDestination(
|
||||
endpoint=endpoint, headers={"Authorization": auth}, callback_name=backend
|
||||
)
|
||||
assert headers == {"arize-space-id": "S", "api_key": "K"}
|
||||
|
||||
|
||||
def test_arize_space_key_overrides_space_id():
|
||||
headers = dynamic_otlp_headers(
|
||||
"arize", {"arize_space_id": "S", "arize_space_key": "SK"}
|
||||
def _event(destinations):
|
||||
"""Anchor destinations on the server-only ContextVar (the sole source the v2 router
|
||||
reads) and build the call event from it, as the proxy does at request time."""
|
||||
set_request_destinations(
|
||||
tuple(d if isinstance(d, OtelDestination) else OtelDestination.model_validate(d) for d in destinations)
|
||||
)
|
||||
assert headers == {"arize-space-id": "SK"}
|
||||
return LLMCallEvent.from_dict({"call_type": "acompletion", "model": "gpt-4o"})
|
||||
|
||||
|
||||
def test_langfuse_dynamic_headers_need_both_keys():
|
||||
assert dynamic_otlp_headers("langfuse_otel", {"langfuse_public_key": "pk"}) is None
|
||||
headers = dynamic_otlp_headers(
|
||||
"langfuse_otel", {"langfuse_public_key": "pk", "langfuse_secret_key": "sk"}
|
||||
)
|
||||
assert headers is not None and "Authorization" in headers
|
||||
# --- routing only happens for admin destinations --------------------------- #
|
||||
|
||||
|
||||
def test_weave_dynamic_headers():
|
||||
headers = dynamic_otlp_headers(
|
||||
"weave_otel", {"wandb_api_key": "w", "weave_project_id": "p"}
|
||||
)
|
||||
assert headers is not None
|
||||
assert "Authorization" in headers and headers["project_id"] == "p"
|
||||
|
||||
|
||||
def test_non_participating_callbacks_have_no_routing():
|
||||
# Phoenix subclasses the base in V1 (no override) → no dynamic routing.
|
||||
assert dynamic_otlp_headers("arize_phoenix", {"arize_api_key": "K"}) is None
|
||||
assert dynamic_otlp_headers("langtrace", {"arize_api_key": "K"}) is None
|
||||
assert dynamic_otlp_headers(None, {"arize_api_key": "K"}) is None
|
||||
|
||||
|
||||
def test_no_dynamic_params_is_no_routing():
|
||||
assert dynamic_otlp_headers("arize", None) is None
|
||||
assert dynamic_otlp_headers("arize", {}) is None
|
||||
|
||||
|
||||
# --- TenantTracerCache routes + caches a TracerProvider per credential set --- #
|
||||
|
||||
|
||||
def test_provider_cached_per_credential_set():
|
||||
cache = _cache("arize")
|
||||
def test_no_destinations_uses_default_tracer():
|
||||
cache = _cache("langfuse_otel")
|
||||
default = NoOpTracer()
|
||||
creds_a = {"arize_space_id": "S", "arize_api_key": "K"}
|
||||
creds_b = {"arize_space_id": "S2", "arize_api_key": "K2"}
|
||||
assert cache.tracers_for(default, ()) == (default,)
|
||||
assert cache._providers == {}
|
||||
|
||||
cache.tracer_for(default, creds_a)
|
||||
cache.tracer_for(default, creds_a) # same set → reuse, no new provider
|
||||
|
||||
def test_provider_cached_per_destination_set():
|
||||
cache = _cache("langfuse_otel")
|
||||
default = NoOpTracer()
|
||||
a = (_dest("https://eu.example/v1", "Basic A"),)
|
||||
b = (_dest("https://eu.example/v1", "Basic B"),)
|
||||
|
||||
cache.tracers_for(default, a)
|
||||
cache.tracers_for(default, a) # same set -> reuse
|
||||
assert len(cache._providers) == 1
|
||||
cache.tracer_for(default, creds_b) # new set → new provider
|
||||
cache.tracers_for(default, b) # different creds -> new provider
|
||||
assert len(cache._providers) == 2
|
||||
|
||||
|
||||
def test_provider_cache_is_bounded_and_evicts_lru(monkeypatch):
|
||||
# The cache key derives from request-supplied dynamic credentials, so it
|
||||
# must be bounded — an unbounded cache lets a caller spawn one provider (and
|
||||
# its background exporter thread) per unique credential set. On overflow the
|
||||
# least-recently-used provider is evicted and shut down.
|
||||
def test_different_host_is_a_distinct_provider():
|
||||
"""Two destinations with identical headers but different hosts must not collide;
|
||||
the cache key includes each endpoint."""
|
||||
cache = _cache("langfuse_otel")
|
||||
default = NoOpTracer()
|
||||
eu = (_dest("https://cloud.langfuse.com/api/public/otel", "Basic X"),)
|
||||
us = (_dest("https://us.cloud.langfuse.com/api/public/otel", "Basic X"),)
|
||||
cache.tracers_for(default, eu)
|
||||
cache.tracers_for(default, us)
|
||||
assert len(cache._providers) == 2
|
||||
|
||||
|
||||
def test_destination_set_is_order_independent():
|
||||
cache = _cache("langfuse_otel")
|
||||
default = NoOpTracer()
|
||||
a = _dest("https://a/v1", "Basic A")
|
||||
b = _dest("https://b/v1", "Basic B")
|
||||
cache.tracers_for(default, (a, b))
|
||||
cache.tracers_for(default, (b, a)) # same set, different order -> one provider
|
||||
assert len(cache._providers) == 1
|
||||
|
||||
|
||||
def test_provider_cache_evicts_lru_and_shuts_it_down_off_hot_path(monkeypatch):
|
||||
"""The provider cache stays bounded, and eviction shuts the LRU provider down so its
|
||||
``BatchSpanProcessor`` worker thread is reclaimed instead of leaked for the process
|
||||
lifetime. ``shutdown`` force-flushes/can block, so it runs on a background daemon
|
||||
thread rather than the request path: the evicted provider is shut down, the survivors
|
||||
are not. Dropping the shutdown (the old leak) fails this."""
|
||||
import time
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from litellm.integrations.otel.plumbing import routing as routing_mod
|
||||
|
||||
monkeypatch.setattr(routing_mod, "_MAX_CACHED_PROVIDERS", 2)
|
||||
shut_down = []
|
||||
monkeypatch.setattr(
|
||||
routing_mod, "_shutdown_provider", lambda p: shut_down.append(p)
|
||||
)
|
||||
real_build = routing_mod.build_tracer_provider
|
||||
created = []
|
||||
|
||||
cache = _cache("arize")
|
||||
def spying_build(config):
|
||||
provider = real_build(config)
|
||||
provider.shutdown = MagicMock(wraps=provider.shutdown)
|
||||
created.append(provider)
|
||||
return provider
|
||||
|
||||
monkeypatch.setattr(routing_mod, "build_tracer_provider", spying_build)
|
||||
|
||||
cache = _cache("langfuse_otel")
|
||||
default = NoOpTracer()
|
||||
|
||||
def creds(space):
|
||||
return {"arize_space_id": space, "arize_api_key": "K"}
|
||||
|
||||
cache.tracer_for(default, creds("1"))
|
||||
cache.tracer_for(default, creds("2"))
|
||||
cache.tracer_for(default, creds("1")) # touch "1" → "2" is now LRU
|
||||
cache.tracer_for(default, creds("3")) # overflow → evict "2"
|
||||
cache.tracers_for(default, (_dest("https://1/v1"),)) # created[0]
|
||||
cache.tracers_for(default, (_dest("https://2/v1"),)) # created[1]
|
||||
cache.tracers_for(default, (_dest("https://1/v1"),)) # touch "1" -> "2" is LRU
|
||||
cache.tracers_for(default, (_dest("https://3/v1"),)) # created[2]: overflow -> evict "2"
|
||||
|
||||
assert len(cache._providers) == 2
|
||||
assert len(shut_down) == 1 # exactly the evicted provider was shut down
|
||||
# eviction shuts down off the hot path, so wait for the background daemon thread
|
||||
deadline = time.time() + 5
|
||||
while not created[1].shutdown.called and time.time() < deadline:
|
||||
time.sleep(0.02)
|
||||
created[1].shutdown.assert_called() # the evicted LRU ("2") is reclaimed
|
||||
created[0].shutdown.assert_not_called() # survivor
|
||||
created[2].shutdown.assert_not_called() # survivor
|
||||
|
||||
|
||||
def test_no_dynamic_params_uses_default_tracer():
|
||||
cache = _cache("arize")
|
||||
default = NoOpTracer()
|
||||
assert cache.tracer_for(default, {}) is default
|
||||
assert cache._providers == {}
|
||||
# --- fan-out: keep the configured exporters, append one per destination ----- #
|
||||
|
||||
|
||||
def test_non_participating_callback_uses_default_tracer():
|
||||
cache = _cache("arize_phoenix")
|
||||
default = NoOpTracer()
|
||||
assert cache.tracer_for(default, {"arize_api_key": "K"}) is default
|
||||
assert cache._providers == {}
|
||||
|
||||
|
||||
def test_dynamic_headers_applied_to_otlp_exporter_only():
|
||||
@pytest.mark.parametrize("owner", ["langfuse_otel", "arize", "weave_otel"])
|
||||
def test_fan_out_appends_destination_with_resolved_endpoint(owner):
|
||||
# The configured (global) exporter is kept; each destination is appended with its
|
||||
# OWN resolved endpoint + headers (the cross-host fix, per owner).
|
||||
cache = _cache(
|
||||
"arize",
|
||||
owner,
|
||||
exporters=[
|
||||
ExporterSpec(kind="otlp_http", owner="arize"),
|
||||
ExporterSpec(kind="in_memory", owner="arize"),
|
||||
ExporterSpec(
|
||||
kind="otlp_http",
|
||||
endpoint="https://env-host.example/v1",
|
||||
headers="Authorization=Basic ENV",
|
||||
owner=owner,
|
||||
)
|
||||
],
|
||||
)
|
||||
new_cfg = cache._config_with_headers({"arize-space-id": "S", "api_key": "K"})
|
||||
otlp, in_mem = new_cfg.exporters
|
||||
assert otlp.headers == "arize-space-id=S,api_key=K"
|
||||
assert in_mem.headers is None # console/in_memory left untouched
|
||||
new = cache._config_with_destinations(
|
||||
(_dest("https://resolved.example/v1", "Basic TEAM", backend=owner),)
|
||||
)
|
||||
# global kept verbatim
|
||||
assert new.exporters[0].endpoint == "https://env-host.example/v1"
|
||||
assert new.exporters[0].headers == "Authorization=Basic ENV"
|
||||
# destination appended at the resolved host with its own auth
|
||||
assert new.exporters[-1].endpoint == "https://resolved.example/v1"
|
||||
assert new.exporters[-1].headers == "Authorization=Basic TEAM"
|
||||
assert len(new.exporters) == 2
|
||||
|
||||
|
||||
def test_dynamic_headers_do_not_leak_to_other_owners_exporter():
|
||||
"""A tenant's Arize credentials must never be stamped onto a co-configured
|
||||
exporter owned by a different backend (a self-hosted collector, Langfuse).
|
||||
|
||||
Regression for the cross-backend credential leak: ``_config_with_headers``
|
||||
used to rewrite the headers of every OTLP exporter, so one request carrying
|
||||
a team's Arize key clobbered the base collector's and Langfuse's headers
|
||||
with that key.
|
||||
"""
|
||||
def test_fan_out_preserves_co_configured_exporters():
|
||||
cache = _cache(
|
||||
"arize",
|
||||
"langfuse_otel",
|
||||
exporters=[
|
||||
ExporterSpec(
|
||||
kind="otlp_http",
|
||||
|
|
@ -154,22 +189,523 @@ def test_dynamic_headers_do_not_leak_to_other_owners_exporter():
|
|||
),
|
||||
ExporterSpec(
|
||||
kind="otlp_http",
|
||||
endpoint="https://cloud.langfuse.com/api/public/otel",
|
||||
headers="Authorization=Basic base-langfuse",
|
||||
endpoint="https://us.cloud.langfuse.com/api/public/otel",
|
||||
headers="Authorization=Basic ENV",
|
||||
owner="langfuse_otel",
|
||||
),
|
||||
ExporterSpec(
|
||||
kind="otlp_grpc",
|
||||
endpoint="https://otlp.arize.com/v1",
|
||||
headers="space_id=base,api_key=base",
|
||||
owner="arize",
|
||||
),
|
||||
],
|
||||
)
|
||||
new_cfg = cache._config_with_headers(
|
||||
{"arize-space-id": "TEAMX", "api_key": "TEAMX_KEY"}
|
||||
new = cache._config_with_destinations(
|
||||
(_dest("https://cloud.langfuse.com/api/public/otel", "Basic TEAM"),)
|
||||
)
|
||||
by_owner = {e.owner: e.headers for e in new_cfg.exporters}
|
||||
assert by_owner["arize"] == "arize-space-id=TEAMX,api_key=TEAMX_KEY"
|
||||
assert by_owner[None] == "x=base-collector"
|
||||
assert by_owner["langfuse_otel"] == "Authorization=Basic base-langfuse"
|
||||
# both originals preserved unchanged (no rewrite/leak)
|
||||
assert new.exporters[0].endpoint == "http://self-hosted-collector:4318"
|
||||
assert new.exporters[0].headers == "x=base-collector"
|
||||
assert new.exporters[1].headers == "Authorization=Basic ENV"
|
||||
# exactly one appended
|
||||
assert new.exporters[-1].endpoint == "https://cloud.langfuse.com/api/public/otel"
|
||||
assert len(new.exporters) == 3
|
||||
|
||||
|
||||
def test_fan_out_to_many_destinations_is_one_provider_with_all_exporters():
|
||||
cache = _cache(
|
||||
"langfuse_otel",
|
||||
exporters=[
|
||||
ExporterSpec(
|
||||
kind="otlp_http", endpoint="https://env/v1", owner="langfuse_otel"
|
||||
)
|
||||
],
|
||||
)
|
||||
new = cache._config_with_destinations(
|
||||
(_dest("https://a/v1", "Basic A"), _dest("https://b/v1", "Basic B"))
|
||||
)
|
||||
# global + 2 destinations -> 3 span processors, one provider, one span copied to all
|
||||
assert [e.endpoint for e in new.exporters] == [
|
||||
"https://env/v1",
|
||||
"https://a/v1",
|
||||
"https://b/v1",
|
||||
]
|
||||
cache.tracers_for(NoOpTracer(), (_dest("https://a/v1"), _dest("https://b/v1")))
|
||||
assert len(cache._providers) == 1
|
||||
|
||||
|
||||
# --- gen-AI span Resource must match its fanned-out parents ----------------- #
|
||||
#
|
||||
# The gen-AI LLM-call span is emitted through the TenantTracerCache clone
|
||||
# provider, while the proxy-internal spans (server/auth/db) are forwarded by
|
||||
# TenantFanOutSpanProcessor, which wraps each with the destination's backend-
|
||||
# required Resource attrs (Arize needs model_id / arize.project.name). If the
|
||||
# clone provider does NOT also carry those attrs, the gen-AI span reaches Arize
|
||||
# with only service.name and Arize renders it as an orphaned subtree. These pin
|
||||
# that the clone Resource carries the same attrs the fan-out wrap injects.
|
||||
|
||||
|
||||
def _dest_with_resource(endpoint, backend, resource_attributes):
|
||||
return OtelDestination(
|
||||
endpoint=endpoint,
|
||||
headers={"Authorization": "Basic AAAA"},
|
||||
callback_name=backend,
|
||||
resource_attributes=resource_attributes,
|
||||
)
|
||||
|
||||
|
||||
def test_clone_config_carries_destination_resource_attrs():
|
||||
cache = _cache("arize")
|
||||
new = cache._config_with_destinations(
|
||||
(
|
||||
_dest_with_resource(
|
||||
"https://otlp.arize.com/v1",
|
||||
"arize",
|
||||
{"model_id": "team-b-proj", "arize.project.name": "team-b-proj"},
|
||||
),
|
||||
)
|
||||
)
|
||||
assert new.resource_attributes["model_id"] == "team-b-proj"
|
||||
assert new.resource_attributes["arize.project.name"] == "team-b-proj"
|
||||
|
||||
|
||||
def test_clone_config_carries_builder_declared_resource_attrs(monkeypatch):
|
||||
"""End-to-end: an arize credential that omits the project gets ARIZE_PROJECT_NAME
|
||||
folded in at BUILD time (build_destination), and the clone config then carries
|
||||
those Resource attrs generically -- the gen-AI span lands in the same project as
|
||||
its fan-out'd parents. The clone path itself is backend-agnostic; it reads
|
||||
whatever the builder declared."""
|
||||
monkeypatch.setenv("ARIZE_PROJECT_NAME", "env-proj")
|
||||
from litellm.integrations.otel.presets.destinations import build_destination
|
||||
|
||||
dest = build_destination("arize", {"arize_space_id": "s", "arize_api_key": "k"})
|
||||
assert dest is not None
|
||||
cache = _cache("arize")
|
||||
new = cache._config_with_destinations((dest,))
|
||||
assert new.resource_attributes["model_id"] == "env-proj"
|
||||
assert new.resource_attributes["arize.project.name"] == "env-proj"
|
||||
|
||||
|
||||
def test_clone_provider_emits_genai_span_with_destination_resource():
|
||||
"""End-to-end regression: the span the clone provider actually exports must
|
||||
carry the destination's Resource attrs. Pre-fix this Resource was
|
||||
service.name only, orphaning the gen-AI span in Arize."""
|
||||
from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
|
||||
InMemorySpanExporter,
|
||||
)
|
||||
|
||||
from litellm.integrations.otel.plumbing.providers import (
|
||||
build_tracer_provider,
|
||||
get_tracer,
|
||||
)
|
||||
|
||||
cfg = OpenTelemetryV2Config(
|
||||
service_name="litellm-proxy",
|
||||
exporters=[ExporterSpec(kind="in_memory", owner="arize")],
|
||||
)
|
||||
cache = TenantTracerCache(cfg, "arize", "litellm")
|
||||
dest = _dest_with_resource(
|
||||
"https://otlp.arize.com/v1",
|
||||
"arize",
|
||||
{"model_id": "team-b-proj", "arize.project.name": "team-b-proj"},
|
||||
)
|
||||
tracers = cache.tracers_for(get_tracer(build_tracer_provider(cfg)), (dest,))
|
||||
tracer = tracers[-1]
|
||||
with tracer.start_as_current_span("chat anthropic-haiku") as span:
|
||||
span.set_attribute("gen_ai.operation.name", "chat")
|
||||
|
||||
# The destination group carries the destination's Resource and only the
|
||||
# destination's exporter; the configured in-memory exporter rides the base tracer
|
||||
# (``tracers[0]``) on a clean Resource, so the tenant's project can never stamp it.
|
||||
provider = next(iter(cache._providers.values()))
|
||||
provider.force_flush()
|
||||
resource_attrs = dict(provider.resource.attributes)
|
||||
assert resource_attrs.get("model_id") == "team-b-proj"
|
||||
assert not any(
|
||||
isinstance(getattr(proc, "span_exporter", None), InMemorySpanExporter)
|
||||
for proc in provider._active_span_processor._span_processors
|
||||
), "the configured exporter must not ride a destination group"
|
||||
assert resource_attrs.get("arize.project.name") == "team-b-proj"
|
||||
|
||||
|
||||
# --- request credentials route the gen-AI span (V1 parity) ------------------ #
|
||||
|
||||
|
||||
_DYNAMIC_CREDS = [
|
||||
("langfuse_otel", {"langfuse_public_key": "pk-teamA", "langfuse_secret_key": "sk-teamA"}, "Authorization="),
|
||||
("arize", {"arize_space_id": "space-teamA", "arize_api_key": "key-teamA"}, "arize-space-id=space-teamA"),
|
||||
("weave_otel", {"wandb_api_key": "wandb-teamA", "weave_project_id": "proj-teamA"}, "project_id=proj-teamA"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend, creds, owned_fragment", _DYNAMIC_CREDS)
|
||||
def test_request_credentials_rewrite_only_the_owned_exporter(backend, creds, owned_fragment):
|
||||
"""A request's team/key OTLP credentials rewrite the headers of THIS backend's own
|
||||
exporter and nothing else, so the gen-AI span exports to that team's account while a
|
||||
co-configured backend's exporter is untouched (no cross-backend key leak)."""
|
||||
from litellm.integrations.otel.presets import dynamic_otlp_headers
|
||||
|
||||
cache = _cache(
|
||||
backend,
|
||||
exporters=[
|
||||
ExporterSpec(kind="otlp_http", endpoint="http://collector/otel", headers="Authorization=Basic GLOBAL", owner=backend),
|
||||
ExporterSpec(kind="otlp_http", endpoint="http://other/otel", headers="x=coconfigured", owner="levo"),
|
||||
],
|
||||
)
|
||||
headers = dynamic_otlp_headers(backend, creds)
|
||||
assert headers, f"{backend} must support dynamic credentials"
|
||||
scoped = cache._config_with_headers(headers)
|
||||
assert owned_fragment in (scoped.exporters[0].headers or "")
|
||||
assert "GLOBAL" not in (scoped.exporters[0].headers or "") # the global header was rewritten away
|
||||
assert scoped.exporters[1].headers == "x=coconfigured" # a different backend's exporter is untouched
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend, creds, owned_fragment", _DYNAMIC_CREDS)
|
||||
def test_genai_tracers_for_spawns_credential_scoped_provider(backend, creds, owned_fragment):
|
||||
"""genai_tracers_for with request creds and no admin destination returns exactly the
|
||||
credential-scoped tracer (not the default) and caches its provider under a dynamic key."""
|
||||
cache = _cache(
|
||||
backend,
|
||||
exporters=[ExporterSpec(kind="otlp_http", endpoint="http://c/otel", headers="Authorization=Basic GLOBAL", owner=backend)],
|
||||
)
|
||||
default = NoOpTracer()
|
||||
tracers = cache.genai_tracers_for(default, (), creds)
|
||||
assert len(tracers) == 1
|
||||
assert tracers[0] is not default
|
||||
assert len(cache._providers) == 1
|
||||
assert next(iter(cache._providers))[0] == "dynamic" # namespaced key, never aliases a destination group
|
||||
|
||||
|
||||
def test_genai_tracers_for_without_creds_is_plain_default():
|
||||
"""No dynamic creds and no destinations -> the logger's default tracer, no provider spawned."""
|
||||
cache = _cache(
|
||||
"langfuse_otel",
|
||||
exporters=[ExporterSpec(kind="otlp_http", endpoint="http://c/otel", headers="", owner="langfuse_otel")],
|
||||
)
|
||||
default = NoOpTracer()
|
||||
assert cache.genai_tracers_for(default, (), None) == (default,)
|
||||
assert cache._providers == {}
|
||||
|
||||
|
||||
def test_genai_tracers_compose_credential_scoped_plus_destination():
|
||||
"""Request creds AND an admin destination: the span exports via the credential-scoped
|
||||
tracer (which carries the global exporter) plus one per-destination tracer that omits the
|
||||
base exporters, so the global collector receives the span exactly once."""
|
||||
cache = _cache(
|
||||
"langfuse_otel",
|
||||
exporters=[ExporterSpec(kind="otlp_http", endpoint="http://global/otel", headers="Authorization=Basic GLOBAL", owner="langfuse_otel")],
|
||||
)
|
||||
creds = {"langfuse_public_key": "pk-teamA", "langfuse_secret_key": "sk-teamA"}
|
||||
destinations = (_dest("https://cloud.langfuse.com/api/public/otel", auth="Basic ADMIN"),)
|
||||
tracers = cache.genai_tracers_for(NoOpTracer(), destinations, creds)
|
||||
assert len(tracers) == 2 # credential-scoped (global) + one destination group
|
||||
# the destination group provider carries no base exporter (base rides the dynamic tracer)
|
||||
dest_only = cache._config_with_destinations(destinations, include_base_exporters=False)
|
||||
assert all(spec.endpoint != "http://global/otel" for spec in dest_only.exporters)
|
||||
|
||||
|
||||
def test_admin_destinations_route():
|
||||
event = _event(
|
||||
[
|
||||
{
|
||||
"callback_name": "langfuse_otel",
|
||||
"endpoint": "https://cloud.langfuse.com/api/public/otel",
|
||||
"headers": {"Authorization": "Basic ADMIN"},
|
||||
}
|
||||
]
|
||||
)
|
||||
assert len(event.otel_destinations) == 1
|
||||
cache = _cache("langfuse_otel")
|
||||
cache.tracers_for(NoOpTracer(), event.otel_destinations)
|
||||
assert len(cache._providers) == 1
|
||||
|
||||
|
||||
# --- a logger only exports the destinations tagged with its own backend ----- #
|
||||
|
||||
|
||||
def test_logger_filters_destinations_to_its_backend():
|
||||
from litellm.integrations.otel.logger import OpenTelemetryV2
|
||||
|
||||
event = _event(
|
||||
[
|
||||
{
|
||||
"callback_name": "langfuse_otel",
|
||||
"endpoint": "https://lf/api/public/otel",
|
||||
"headers": {"Authorization": "Basic A"},
|
||||
},
|
||||
{
|
||||
"callback_name": "arize",
|
||||
"endpoint": "https://otlp.arize.com/v1",
|
||||
"headers": {"space_id": "S"},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
class _Shim:
|
||||
callback_name = "langfuse_otel"
|
||||
|
||||
got = OpenTelemetryV2._destinations_for_backend(_Shim(), event)
|
||||
assert [d.endpoint for d in got] == ["https://lf/api/public/otel"]
|
||||
|
||||
|
||||
# --- multi-destination, same-backend: group by Resource so each gets its span -- #
|
||||
#
|
||||
# A backend that selects its target FROM the Resource (Arize's project) needs a
|
||||
# differently-tagged span per destination, because a span carries exactly one
|
||||
# Resource (a TracerProvider property). Pre-fix the gen-AI clone folded every
|
||||
# destination into ONE last-wins Resource, so two Arize projects collapsed to one
|
||||
# and only that project received the gen-AI span. ``tracers_for`` groups by
|
||||
# ``destination_resource_attrs`` and returns one tracer (one provider, one Resource)
|
||||
# per distinct group. Header-routed backends declare no Resource attrs, so they stay
|
||||
# in one group with multiple exporters and keep routing by per-exporter auth.
|
||||
|
||||
|
||||
def _arize_dest(project, space="S", key="K"):
|
||||
return OtelDestination(
|
||||
endpoint="https://otlp.arize.com/v1",
|
||||
headers={"space_id": space, "api_key": key},
|
||||
callback_name="arize",
|
||||
resource_attributes={"model_id": project, "arize.project.name": project},
|
||||
)
|
||||
|
||||
|
||||
def _provider_project(provider):
|
||||
return provider.resource.attributes.get("arize.project.name")
|
||||
|
||||
|
||||
def test_tracers_for_empty_returns_default_only():
|
||||
cache = _cache("arize")
|
||||
default = NoOpTracer()
|
||||
assert cache.tracers_for(default, ()) == (default,)
|
||||
assert cache._providers == {}
|
||||
|
||||
|
||||
def test_tracers_for_single_destination_one_group():
|
||||
"""A single Arize destination yields the clean base tracer (``default``) plus one
|
||||
destination provider carrying its project; base is never folded into the project group."""
|
||||
default = NoOpTracer()
|
||||
cache = _cache("arize")
|
||||
tracers = cache.tracers_for(default, (_arize_dest("solo"),))
|
||||
assert tracers[0] is default # base rides its own clean tracer, returned first
|
||||
assert len(tracers) == 2 # default + the one project group
|
||||
assert {_provider_project(p) for p in cache._providers.values()} == {"solo"}
|
||||
|
||||
|
||||
def test_tracers_for_two_arize_projects_split_into_separate_groups():
|
||||
"""The fix: two Arize destinations with different Resource attrs must NOT
|
||||
last-wins merge -- each project gets its own provider/Resource so each receives a
|
||||
correctly-tagged gen-AI span."""
|
||||
cache = _cache("arize")
|
||||
tracers = cache.tracers_for(
|
||||
NoOpTracer(), (_arize_dest("projA"), _arize_dest("projB"))
|
||||
)
|
||||
assert len(tracers) == 3 # default (clean base) + one tracer per project group
|
||||
assert {_provider_project(p) for p in cache._providers.values()} == {
|
||||
"projA",
|
||||
"projB",
|
||||
}
|
||||
|
||||
|
||||
def test_two_arize_projects_each_provider_has_its_own_single_project():
|
||||
"""Each group's Resource carries exactly its own project (not the other's, not a
|
||||
merge)."""
|
||||
cache = _cache("arize")
|
||||
cache.tracers_for(NoOpTracer(), (_arize_dest("projA"), _arize_dest("projB")))
|
||||
by_project = {
|
||||
_provider_project(p): p.resource.attributes for p in cache._providers.values()
|
||||
}
|
||||
assert by_project["projA"]["model_id"] == "projA"
|
||||
assert by_project["projB"]["model_id"] == "projB"
|
||||
|
||||
|
||||
def test_header_routed_destinations_stay_one_group_with_two_exporters():
|
||||
"""Langfuse/Weave declare no Resource attrs, so two distinct destinations collapse
|
||||
into ONE group (one provider) with one exporter each -- they route by per-exporter
|
||||
auth, so no per-Resource split is needed or wanted."""
|
||||
cache = _cache(
|
||||
"langfuse_otel",
|
||||
exporters=[
|
||||
ExporterSpec(kind="otlp_http", endpoint="https://env/v1", owner=None)
|
||||
],
|
||||
)
|
||||
tracers = cache.tracers_for(
|
||||
NoOpTracer(),
|
||||
(_dest("https://a/v1", "Basic A"), _dest("https://b/v1", "Basic B")),
|
||||
)
|
||||
assert len(tracers) == 2 # default (clean base) + the single empty-Resource group
|
||||
assert len(cache._providers) == 1
|
||||
(provider,) = cache._providers.values()
|
||||
endpoints = " ".join(
|
||||
str(getattr(getattr(sp, "span_exporter", None), "_endpoint", ""))
|
||||
for sp in provider._active_span_processor._span_processors
|
||||
)
|
||||
# both destinations live on the one group provider (the global exporter rides ``default``,
|
||||
# not this group); OTLP normalizes the endpoint by appending /v1/traces, so match on prefix
|
||||
assert "https://a/v1" in endpoints and "https://b/v1" in endpoints
|
||||
|
||||
|
||||
def test_base_exporters_never_ride_a_destination_group():
|
||||
"""The configured/global exporters must ride their own clean-Resource tracer
|
||||
(``default``), never a destination group: folding them into a group made the global
|
||||
export inherit that destination's Resource (e.g. Arize's project), so an operator's
|
||||
own collector saw spans stamped with a tenant's project. The global still receives the
|
||||
gen-AI span exactly once (via ``default``), not once per project."""
|
||||
cache = _cache(
|
||||
"arize",
|
||||
exporters=[ExporterSpec(kind="in_memory", endpoint=None, owner=None)],
|
||||
)
|
||||
default = NoOpTracer()
|
||||
tracers = cache.tracers_for(default, (_arize_dest("projA"), _arize_dest("projB")))
|
||||
assert tracers[0] is default # global rides its own clean tracer, exactly once
|
||||
for provider in cache._providers.values():
|
||||
base_count = sum(
|
||||
type(getattr(sp, "span_exporter", sp)).__name__ == "InMemorySpanExporter"
|
||||
for sp in provider._active_span_processor._span_processors
|
||||
)
|
||||
assert base_count == 0, "no destination group may carry the configured/global exporters"
|
||||
# and each group's Resource stays its own project only
|
||||
assert _provider_project(provider) in {"projA", "projB"}
|
||||
|
||||
|
||||
def test_generic_backend_resolves_generic_destination():
|
||||
"""A Generic OTLP destination (callback_name='generic') must be picked up by the
|
||||
generic OpenTelemetryV2 logger, so its gen-AI span routes to the destination's
|
||||
otel_endpoint. Regression: 'generic' had no preset, so no generic logger existed and
|
||||
the gen-AI span was dropped (only proxy-internal spans fanned out)."""
|
||||
from litellm.integrations.otel.logger import OpenTelemetryV2
|
||||
|
||||
event = _event(
|
||||
[
|
||||
{
|
||||
"callback_name": "generic",
|
||||
"endpoint": "http://collector:4318",
|
||||
"headers": {"x-tenant": "t1"},
|
||||
},
|
||||
{
|
||||
"callback_name": "arize",
|
||||
"endpoint": "https://otlp.arize.com/v1",
|
||||
"headers": {"space_id": "S"},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
class _Shim:
|
||||
callback_name = "generic"
|
||||
|
||||
got = OpenTelemetryV2._destinations_for_backend(_Shim(), event)
|
||||
assert [d.endpoint for d in got] == ["http://collector:4318"]
|
||||
|
||||
|
||||
def test_otel_destination_is_frozen_and_renders_header_string():
|
||||
"""OtelDestination is the immutable value the resolver hands to the runtime;
|
||||
mutating one after resolution must fail (a request can never rewrite where its
|
||||
traces go), and header_string renders the k=v,k2=v2 form the exporter expects."""
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
dest = OtelDestination(
|
||||
endpoint="https://c/v1", headers={"Authorization": "Bearer x", "x-k": "y"}
|
||||
)
|
||||
with pytest.raises(ValidationError):
|
||||
dest.endpoint = "https://evil/v1"
|
||||
assert dest.header_string() == "Authorization=Bearer x,x-k=y"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"backend, params, expected_endpoint",
|
||||
[
|
||||
(
|
||||
"langfuse_otel",
|
||||
{"langfuse_public_key": "pk", "langfuse_secret_key": "sk", "langfuse_host": "https://lf.internal"},
|
||||
"https://lf.internal/api/public/otel",
|
||||
),
|
||||
("arize", {"arize_space_key": "S", "arize_api_key": "K"}, "https://otlp.arize.com/v1"),
|
||||
("weave_otel", {"wandb_api_key": "wk"}, "https://trace.wandb.ai/otel/v1/traces"),
|
||||
],
|
||||
)
|
||||
def test_team_credentials_still_export_when_the_preset_degraded(backend, params, expected_endpoint):
|
||||
"""Regression: a preset built with ``allow_missing_credentials`` contributes no owned
|
||||
exporter, so a team's own ``callback_vars`` had nothing to be stamped onto and its
|
||||
spans were dropped. The exporter is synthesized from the request's own credentials
|
||||
instead, resolved through the same builder an equivalent admin destination uses.
|
||||
"""
|
||||
from litellm.integrations.otel.presets import dynamic_otlp_headers
|
||||
|
||||
degraded = TenantTracerCache(OpenTelemetryV2Config(exporters=[]), backend, "litellm")
|
||||
headers = dynamic_otlp_headers(backend, params)
|
||||
assert headers, "the backend must recognise these per-team credentials"
|
||||
|
||||
owned = [spec for spec in degraded._config_with_headers(headers, params).exporters if spec.owner == backend]
|
||||
|
||||
assert len(owned) == 1, "a degraded preset must still export the team's own traces"
|
||||
assert owned[0].endpoint == expected_endpoint
|
||||
assert owned[0].headers == ",".join(f"{k}={v}" for k, v in headers.items())
|
||||
|
||||
|
||||
def test_degraded_preset_synthesizes_nothing_without_team_credentials():
|
||||
"""No per-request credentials means no synthesized exporter, so a degraded backend
|
||||
stays silent rather than inventing an uncredentialled vendor export."""
|
||||
degraded = TenantTracerCache(OpenTelemetryV2Config(exporters=[]), "arize", "litellm")
|
||||
assert degraded._config_with_headers({}, None).exporters == []
|
||||
|
||||
|
||||
# --- transport is part of a destination's identity ------------------------- #
|
||||
# Two destinations can agree on endpoint, headers and Resource attrs and still
|
||||
# disagree on OTLP transport (an Arize credential naming ``arize_http_endpoint``
|
||||
# resolves to otlp_http where the backend's intrinsic default is gRPC). The
|
||||
# provider cache and the synthesized exporter both have to carry that.
|
||||
|
||||
|
||||
def _arize_dest_with_protocol(protocol):
|
||||
return OtelDestination(
|
||||
endpoint="https://collector.internal/v1",
|
||||
headers={"space_id": "S", "api_key": "K"},
|
||||
callback_name="arize",
|
||||
resource_attributes={"model_id": "proj", "arize.project.name": "proj"},
|
||||
protocol=protocol,
|
||||
)
|
||||
|
||||
|
||||
def _exporter_transports(provider):
|
||||
"""The OTLP transport of each of a provider's exporters. Both exporters are named
|
||||
``OTLPSpanExporter``; only the defining module tells gRPC from HTTP apart."""
|
||||
return {
|
||||
"http" if ".http." in type(sp.span_exporter).__module__ else "grpc"
|
||||
for sp in provider._active_span_processor._span_processors
|
||||
if getattr(sp, "span_exporter", None) is not None
|
||||
and "otlp" in type(sp.span_exporter).__module__
|
||||
}
|
||||
|
||||
|
||||
def test_same_endpoint_and_headers_but_different_protocol_are_distinct_providers():
|
||||
"""Two destinations differing only in OTLP transport must not share a provider;
|
||||
reusing the first one's exports the second's spans over the wrong transport."""
|
||||
cache = _cache("arize")
|
||||
cache.tracers_for(NoOpTracer(), (_arize_dest_with_protocol(None),))
|
||||
cache.tracers_for(NoOpTracer(), (_arize_dest_with_protocol("otlp_http"),))
|
||||
|
||||
assert len(cache._providers) == 2
|
||||
transports = sorted(t for p in cache._providers.values() for t in _exporter_transports(p))
|
||||
assert transports == ["grpc", "http"]
|
||||
|
||||
|
||||
def test_synthesized_exporter_uses_the_transport_the_credentials_pin():
|
||||
"""A team whose Arize credentials name an HTTP collector gets an HTTP exporter, not
|
||||
the backend's intrinsic gRPC default."""
|
||||
cache = _cache("arize", exporters=[ExporterSpec(kind="otlp_grpc", endpoint="https://env/v1", owner=None)])
|
||||
params = {
|
||||
"arize_space_id": "S",
|
||||
"arize_api_key": "K",
|
||||
"arize_http_endpoint": "https://collector.internal/v1",
|
||||
}
|
||||
|
||||
config = cache._config_with_headers({"space_id": "S"}, params)
|
||||
|
||||
(synthesized,) = [spec for spec in config.exporters if spec.owner == "arize"]
|
||||
assert synthesized.endpoint == "https://collector.internal/v1"
|
||||
assert synthesized.kind == "otlp_http"
|
||||
|
||||
|
||||
def test_synthesized_exporter_keeps_the_backend_default_when_credentials_pin_nothing():
|
||||
"""Credentials that name no transport still get the backend's intrinsic one."""
|
||||
cache = _cache("arize", exporters=[ExporterSpec(kind="otlp_grpc", endpoint="https://env/v1", owner=None)])
|
||||
|
||||
config = cache._config_with_headers({"space_id": "S"}, {"arize_space_id": "S", "arize_api_key": "K"})
|
||||
|
||||
(synthesized,) = [spec for spec in config.exporters if spec.owner == "arize"]
|
||||
assert synthesized.kind == "otlp_grpc"
|
||||
|
|
|
|||
390
tests/test_litellm/integrations/otel/test_otel_v2_fan_out.py
Normal file
390
tests/test_litellm/integrations/otel/test_otel_v2_fan_out.py
Normal file
|
|
@ -0,0 +1,390 @@
|
|||
"""Tests for the per-request tenant fan-out SpanProcessor (Hoist).
|
||||
|
||||
The fan-out processor lives on the main v2 ``TracerProvider`` and forwards every
|
||||
finished span to the admin-resolved per-tenant destinations carried on the
|
||||
request's ``ContextVar``. The contract under test:
|
||||
|
||||
- Spans on the main provider land at every per-tenant destination matching this
|
||||
backend (the ``owner_callback_name``).
|
||||
- When destinations is empty (an unassigned identity, the SDK path, a request
|
||||
before the resolver ran), the processor is a no-op.
|
||||
- Concurrent requests with different destinations stay isolated -- contextvars
|
||||
scope per task, so one request's tenant doesn't receive another's spans.
|
||||
- The processor caches one BatchSpanProcessor per ``(endpoint, headers)`` pair
|
||||
and skips destinations whose ``callback_name`` doesn't match its owner.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
pytest.importorskip("opentelemetry")
|
||||
|
||||
from opentelemetry.sdk.trace import TracerProvider # noqa: E402
|
||||
from opentelemetry.sdk.trace.export import SimpleSpanProcessor # noqa: E402
|
||||
from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( # noqa: E402
|
||||
InMemorySpanExporter,
|
||||
)
|
||||
from opentelemetry.sdk.trace.export import SpanExporter # noqa: E402
|
||||
|
||||
from litellm.integrations.otel.model.destination import OtelDestination # noqa: E402
|
||||
from litellm.integrations.otel.plumbing.context import ( # noqa: E402
|
||||
set_request_destinations,
|
||||
_request_destinations,
|
||||
)
|
||||
from litellm.integrations.otel.plumbing.routing import ( # noqa: E402
|
||||
TenantFanOutSpanProcessor,
|
||||
_processor_key,
|
||||
)
|
||||
from litellm.integrations.otel.plumbing.providers import ( # noqa: E402
|
||||
destination_resource_attrs,
|
||||
)
|
||||
from litellm.integrations.otel.plumbing import providers # noqa: E402
|
||||
from litellm.integrations.otel.model.config import OpenTelemetryV2Config # noqa: E402
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_request_destinations():
|
||||
_request_destinations.set(())
|
||||
yield
|
||||
_request_destinations.set(())
|
||||
|
||||
|
||||
def _build_provider_with_fan_out(
|
||||
owner: str, exporter: SpanExporter
|
||||
) -> tuple[TracerProvider, TenantFanOutSpanProcessor]:
|
||||
"""Build a real TracerProvider with the in-memory exporter as the configured
|
||||
backend, plus the fan-out processor wired in. The fan-out processor's
|
||||
per-destination processors are real BatchSpanProcessors, replaced in the
|
||||
test by an injected SimpleSpanProcessor against an in-memory exporter via
|
||||
``monkeypatch`` so the test can read what was forwarded."""
|
||||
cfg = OpenTelemetryV2Config(exporter="in_memory")
|
||||
provider = providers.build_tracer_provider(
|
||||
cfg, exporter=exporter, tenant_fan_out_owner=owner
|
||||
)
|
||||
fan_out = next(
|
||||
p
|
||||
for p in provider._active_span_processor._span_processors
|
||||
if isinstance(p, TenantFanOutSpanProcessor)
|
||||
)
|
||||
return provider, fan_out
|
||||
|
||||
|
||||
def test_fan_out_forwards_span_to_matching_destination(monkeypatch):
|
||||
"""A span on the main provider lands at the per-tenant destination whose
|
||||
callback_name matches the fan-out owner. Removing the fan-out branch makes
|
||||
this test fail (the tenant exporter never sees the span)."""
|
||||
main_exporter = InMemorySpanExporter()
|
||||
provider, fan_out = _build_provider_with_fan_out("langfuse_otel", main_exporter)
|
||||
|
||||
tenant_exporter = InMemorySpanExporter()
|
||||
|
||||
# Swap the lazy per-destination processor build for an in-memory one so we
|
||||
# don't actually hit the network.
|
||||
def _stub_processor_for(self, destination):
|
||||
return SimpleSpanProcessor(tenant_exporter)
|
||||
|
||||
monkeypatch.setattr(
|
||||
TenantFanOutSpanProcessor, "_processor_for", _stub_processor_for
|
||||
)
|
||||
|
||||
set_request_destinations(
|
||||
(
|
||||
OtelDestination(
|
||||
callback_name="langfuse_otel",
|
||||
endpoint="https://cloud.langfuse.com/api/public/otel",
|
||||
headers={"Authorization": "Bearer pk:sk"},
|
||||
),
|
||||
)
|
||||
)
|
||||
tracer = provider.get_tracer("test")
|
||||
with tracer.start_as_current_span("auth /chat/completions"):
|
||||
pass
|
||||
|
||||
main_names = [s.name for s in main_exporter.get_finished_spans()]
|
||||
tenant_names = [s.name for s in tenant_exporter.get_finished_spans()]
|
||||
assert "auth /chat/completions" in main_names
|
||||
assert "auth /chat/completions" in tenant_names
|
||||
|
||||
|
||||
def test_fan_out_forwards_proxy_internal_spans_to_every_destination(monkeypatch):
|
||||
"""Proxy-internal spans (server, auth phase, postgres, post-call ledger)
|
||||
are generic OTel semconv with no backend-specific vocabulary, so they ship
|
||||
to every admin-resolved destination regardless of its ``callback_name``.
|
||||
The owner discriminator is only used to skip the gen-AI span (the v2 logger
|
||||
routes that itself through TenantTracerCache to avoid wrong-vocabulary
|
||||
duplicates)."""
|
||||
main_exporter = InMemorySpanExporter()
|
||||
_provider, _ = _build_provider_with_fan_out("langfuse_otel", main_exporter)
|
||||
|
||||
tenant_exporter = InMemorySpanExporter()
|
||||
monkeypatch.setattr(
|
||||
TenantFanOutSpanProcessor,
|
||||
"_processor_for",
|
||||
lambda self, destination: SimpleSpanProcessor(tenant_exporter),
|
||||
)
|
||||
|
||||
# Destination is tagged for a DIFFERENT backend (``arize``) than the
|
||||
# owner (``langfuse_otel``). Proxy-internal spans must still forward.
|
||||
set_request_destinations(
|
||||
(
|
||||
OtelDestination(
|
||||
callback_name="arize",
|
||||
endpoint="https://otlp.arize.com/v1",
|
||||
headers={"api_key": "k", "space_id": "s"},
|
||||
),
|
||||
)
|
||||
)
|
||||
tracer = _provider.get_tracer("test")
|
||||
with tracer.start_as_current_span("auth"):
|
||||
pass
|
||||
|
||||
names = [s.name for s in tenant_exporter.get_finished_spans()]
|
||||
assert "auth" in names
|
||||
|
||||
|
||||
def test_destination_resource_attrs_reads_declared_attrs(monkeypatch):
|
||||
"""``destination_resource_attrs`` is backend-agnostic: it returns exactly what
|
||||
the destination's builder declared, never consulting env itself. The env-vs-
|
||||
credential precedence is enforced at build time (see test_presets_destinations),
|
||||
so setting ARIZE_PROJECT_NAME here must NOT override the destination's own attrs."""
|
||||
monkeypatch.setenv("ARIZE_PROJECT_NAME", "global-project")
|
||||
destination = OtelDestination(
|
||||
callback_name="arize",
|
||||
endpoint="https://otlp.arize.com/v1",
|
||||
headers={"api_key": "k", "space_id": "s"},
|
||||
resource_attributes={
|
||||
"model_id": "tenant-project",
|
||||
"arize.project.name": "tenant-project",
|
||||
},
|
||||
)
|
||||
|
||||
assert destination_resource_attrs(destination) == {
|
||||
"model_id": "tenant-project",
|
||||
"arize.project.name": "tenant-project",
|
||||
}
|
||||
|
||||
|
||||
def test_destination_resource_attrs_empty_for_header_routed_backend():
|
||||
"""A langfuse/weave-style destination routes by header and declares no Resource
|
||||
attrs; the helper returns {} and the fan-out leaves the span Resource untouched."""
|
||||
destination = OtelDestination(
|
||||
callback_name="langfuse_otel",
|
||||
endpoint="https://cloud.langfuse.com/api/public/otel",
|
||||
headers={"Authorization": "Basic x"},
|
||||
)
|
||||
assert destination_resource_attrs(destination) == {}
|
||||
|
||||
|
||||
def test_fan_out_skips_genai_span_to_avoid_double_export(monkeypatch):
|
||||
"""The gen-AI LLM-call span is routed by the v2 logger through
|
||||
``TenantTracerCache`` to per-tenant exporters with the right attribute
|
||||
mapper. Forwarding it here too would deliver a duplicate with the wrong
|
||||
vocabulary, surfacing in the destination as an orphaned second span."""
|
||||
main_exporter = InMemorySpanExporter()
|
||||
_provider, _ = _build_provider_with_fan_out("langfuse_otel", main_exporter)
|
||||
|
||||
tenant_exporter = InMemorySpanExporter()
|
||||
monkeypatch.setattr(
|
||||
TenantFanOutSpanProcessor,
|
||||
"_processor_for",
|
||||
lambda self, destination: SimpleSpanProcessor(tenant_exporter),
|
||||
)
|
||||
|
||||
set_request_destinations(
|
||||
(
|
||||
OtelDestination(
|
||||
callback_name="arize",
|
||||
endpoint="https://otlp.arize.com/v1",
|
||||
headers={},
|
||||
),
|
||||
)
|
||||
)
|
||||
tracer = _provider.get_tracer("test")
|
||||
# A gen-AI span sets ``gen_ai.operation.name`` (the v2 emitter does this
|
||||
# via the SpanRole.LLM_CALL mapper). Fake it explicitly here.
|
||||
with tracer.start_as_current_span("chat gpt-4o") as span:
|
||||
span.set_attribute("gen_ai.operation.name", "chat")
|
||||
# A proxy-internal span also fires.
|
||||
with tracer.start_as_current_span("auth /v1/chat/completions"):
|
||||
pass
|
||||
|
||||
names = [s.name for s in tenant_exporter.get_finished_spans()]
|
||||
assert "auth /v1/chat/completions" in names
|
||||
assert "chat gpt-4o" not in names # gen-AI skipped
|
||||
|
||||
|
||||
def test_fan_out_noop_when_no_destinations(monkeypatch):
|
||||
"""An unassigned identity / pre-auth / SDK path leaves the contextvar at
|
||||
its empty-tuple default; the processor must short-circuit, NOT crash, NOT
|
||||
forward."""
|
||||
main_exporter = InMemorySpanExporter()
|
||||
provider, _ = _build_provider_with_fan_out("langfuse_otel", main_exporter)
|
||||
forwarded: list[Any] = []
|
||||
monkeypatch.setattr(
|
||||
TenantFanOutSpanProcessor,
|
||||
"_processor_for",
|
||||
lambda self, destination: forwarded.append(destination) or None,
|
||||
)
|
||||
tracer = provider.get_tracer("test")
|
||||
with tracer.start_as_current_span("auth"):
|
||||
pass
|
||||
assert forwarded == []
|
||||
|
||||
|
||||
def test_fan_out_caches_processor_per_destination_key(monkeypatch):
|
||||
"""Two requests with the same destination must share one cached
|
||||
BatchSpanProcessor; two different destinations must build two. Otherwise
|
||||
every request rebuilds the OTLP exporter (and its background thread)."""
|
||||
main_exporter = InMemorySpanExporter()
|
||||
_provider, fan_out = _build_provider_with_fan_out("langfuse_otel", main_exporter)
|
||||
|
||||
built: list[OtelDestination] = []
|
||||
|
||||
real_processor_for = TenantFanOutSpanProcessor._processor_for
|
||||
|
||||
def _spy(self, destination):
|
||||
result = real_processor_for(self, destination)
|
||||
if result is not None:
|
||||
built.append(destination)
|
||||
return result
|
||||
|
||||
# Build always returns None to keep the test offline, but we tracked
|
||||
# invocations via the spy above. Easier: stub the inner builder.
|
||||
def _stub(self, destination):
|
||||
built.append(destination)
|
||||
key = _processor_key(destination)
|
||||
if key in self._processors:
|
||||
return self._processors[key]
|
||||
proc = SimpleSpanProcessor(InMemorySpanExporter())
|
||||
self._processors[key] = proc
|
||||
return proc
|
||||
|
||||
monkeypatch.setattr(TenantFanOutSpanProcessor, "_processor_for", _stub)
|
||||
|
||||
dest_a = OtelDestination(
|
||||
callback_name="langfuse_otel", endpoint="https://a/", headers={"k": "1"}
|
||||
)
|
||||
dest_b = OtelDestination(
|
||||
callback_name="langfuse_otel", endpoint="https://b/", headers={"k": "1"}
|
||||
)
|
||||
set_request_destinations((dest_a,))
|
||||
tracer = _provider.get_tracer("test")
|
||||
with tracer.start_as_current_span("s1"):
|
||||
pass
|
||||
set_request_destinations((dest_a,))
|
||||
with tracer.start_as_current_span("s2"):
|
||||
pass
|
||||
set_request_destinations((dest_b,))
|
||||
with tracer.start_as_current_span("s3"):
|
||||
pass
|
||||
|
||||
# _stub appends every call; the cache size is what matters: two unique
|
||||
# ``(endpoint, headers)`` pairs -> two cached processors.
|
||||
assert len(fan_out._processors) == 2
|
||||
|
||||
|
||||
def test_fan_out_flush_and_shutdown_survive_concurrent_cache_mutation():
|
||||
"""force_flush/shutdown snapshot the processor cache before iterating, so a
|
||||
concurrent ``on_end`` inserting or evicting a processor cannot raise a
|
||||
'mutated during iteration' RuntimeError and drop the remaining destinations'
|
||||
buffered spans."""
|
||||
_provider, fan_out = _build_provider_with_fan_out("langfuse_otel", InMemorySpanExporter())
|
||||
|
||||
touched: list[str] = []
|
||||
|
||||
class _MutatingProc:
|
||||
def __init__(self, name: str, mutate: bool = False) -> None:
|
||||
self.name = name
|
||||
self.mutate = mutate
|
||||
|
||||
def _maybe_mutate(self, key: str) -> None:
|
||||
if self.mutate: # simulate a concurrent on_end caching a new processor
|
||||
fan_out._processors[(key, "x")] = _MutatingProc(key)
|
||||
|
||||
def force_flush(self, timeout_millis: int = 30000) -> bool:
|
||||
touched.append(self.name)
|
||||
self._maybe_mutate("late-flush")
|
||||
return True
|
||||
|
||||
def shutdown(self) -> None:
|
||||
touched.append("sd-" + self.name)
|
||||
self._maybe_mutate("late-shutdown")
|
||||
|
||||
fan_out._processors.clear()
|
||||
fan_out._processors[("a", "1")] = _MutatingProc("a", mutate=True)
|
||||
fan_out._processors[("b", "2")] = _MutatingProc("b")
|
||||
fan_out._processors[("c", "3")] = _MutatingProc("c")
|
||||
|
||||
# No RuntimeError despite the mutation mid-iteration; every original processor ran.
|
||||
assert fan_out.force_flush() is True
|
||||
assert {"a", "b", "c"} <= set(touched)
|
||||
|
||||
touched.clear()
|
||||
fan_out._processors[("a", "1")] = _MutatingProc("a", mutate=True)
|
||||
fan_out.shutdown()
|
||||
assert "sd-a" in touched
|
||||
|
||||
|
||||
def test_fan_out_per_request_isolation_with_concurrent_tasks(monkeypatch):
|
||||
"""Two requests running concurrently with different destinations must each
|
||||
see only THEIR destination's spans. The contextvar scopes per asyncio task,
|
||||
so this is a pin on the contextvar approach: switching to a global variable
|
||||
would break this test."""
|
||||
main_exporter = InMemorySpanExporter()
|
||||
provider, _ = _build_provider_with_fan_out("langfuse_otel", main_exporter)
|
||||
|
||||
exporter_a = InMemorySpanExporter()
|
||||
exporter_b = InMemorySpanExporter()
|
||||
|
||||
def _stub(self, destination):
|
||||
if destination.endpoint == "https://a/":
|
||||
return SimpleSpanProcessor(exporter_a)
|
||||
return SimpleSpanProcessor(exporter_b)
|
||||
|
||||
monkeypatch.setattr(TenantFanOutSpanProcessor, "_processor_for", _stub)
|
||||
tracer = provider.get_tracer("test")
|
||||
|
||||
async def fire(label: str, endpoint: str):
|
||||
set_request_destinations(
|
||||
(
|
||||
OtelDestination(
|
||||
callback_name="langfuse_otel",
|
||||
endpoint=endpoint,
|
||||
headers={},
|
||||
),
|
||||
)
|
||||
)
|
||||
with tracer.start_as_current_span(label):
|
||||
await asyncio.sleep(0)
|
||||
|
||||
async def driver():
|
||||
await asyncio.gather(
|
||||
fire("req_a", "https://a/"),
|
||||
fire("req_b", "https://b/"),
|
||||
)
|
||||
|
||||
asyncio.run(driver())
|
||||
|
||||
names_a = {s.name for s in exporter_a.get_finished_spans()}
|
||||
names_b = {s.name for s in exporter_b.get_finished_spans()}
|
||||
assert "req_a" in names_a and "req_b" not in names_a
|
||||
assert "req_b" in names_b and "req_a" not in names_b
|
||||
|
||||
|
||||
def test_fan_out_owner_set_by_build_tracer_provider():
|
||||
"""The v2 logger opts the MAIN provider into fan-out by passing its
|
||||
callback_name; the TenantTracerCache clone providers do NOT, so the gen-AI
|
||||
span emitted through them is not also forwarded by the fan-out processor.
|
||||
Regressing this (adding the processor to clones) would double-export every
|
||||
gen-AI span."""
|
||||
from litellm.integrations.otel.logger import OpenTelemetryV2
|
||||
|
||||
logger = OpenTelemetryV2(callback_name="arize")
|
||||
main_procs = logger._tracer_provider._active_span_processor._span_processors
|
||||
assert any(isinstance(p, TenantFanOutSpanProcessor) for p in main_procs)
|
||||
# Build a tenant clone provider and confirm it has no fan-out processor.
|
||||
clone = providers.build_tracer_provider(logger.config)
|
||||
clone_procs = clone._active_span_processor._span_processors
|
||||
assert not any(isinstance(p, TenantFanOutSpanProcessor) for p in clone_procs)
|
||||
|
|
@ -28,10 +28,12 @@ from litellm.integrations.otel import ( # noqa: E402
|
|||
)
|
||||
from litellm.integrations.otel.plumbing import providers # noqa: E402
|
||||
from litellm.integrations.otel.plumbing.context import ( # noqa: E402
|
||||
_request_destinations,
|
||||
reset_mcp_message_trace_carrier,
|
||||
reset_mcp_message_transport_span,
|
||||
set_mcp_message_trace_carrier,
|
||||
set_mcp_message_transport_span,
|
||||
set_request_destinations,
|
||||
set_request_root_span,
|
||||
)
|
||||
from litellm.integrations.otel.logger import OpenTelemetryV2 # noqa: E402
|
||||
|
|
@ -41,6 +43,25 @@ from litellm.integrations.otel.model.spans import ( # noqa: E402
|
|||
)
|
||||
from litellm.integrations.otel.model.utils import to_ns, to_seconds # noqa: E402
|
||||
|
||||
|
||||
def _anchor(dests):
|
||||
"""Anchor admin destinations on the server-only ContextVar the v2 router reads
|
||||
(the proxy sets this at auth time; there is no request-carried carrier)."""
|
||||
from litellm.integrations.otel.model.destination import OtelDestination
|
||||
|
||||
set_request_destinations(
|
||||
tuple(d if isinstance(d, OtelDestination) else OtelDestination.model_validate(d) for d in dests)
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_request_destinations():
|
||||
token = _request_destinations.set(())
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_request_destinations.reset(token)
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Fixtures
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
|
@ -429,6 +450,59 @@ def test_mcp_tool_call_is_not_logged_as_llm_call():
|
|||
assert "gen_ai.request.model" not in span.attributes
|
||||
|
||||
|
||||
def test_mcp_tool_call_routes_to_admin_destination():
|
||||
"""The MCP tool-call span carries ``gen_ai.operation.name`` (execute_tool), so the
|
||||
fan-out processor skips it; it must still be routed to the request's admin
|
||||
destinations like the LLM-call span, not reach the global exporter only."""
|
||||
from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
|
||||
InMemorySpanExporter,
|
||||
)
|
||||
|
||||
from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config
|
||||
from litellm.integrations.otel.model.destination import OtelDestination
|
||||
from litellm.integrations.otel.plumbing.context import _request_destinations
|
||||
from litellm.integrations.otel.plumbing.providers import build_tracer_provider
|
||||
|
||||
cfg = OpenTelemetryV2Config(
|
||||
service_name="litellm-proxy",
|
||||
exporters=[ExporterSpec(kind="in_memory")],
|
||||
)
|
||||
base_provider = build_tracer_provider(cfg)
|
||||
logger = OpenTelemetryV2(config=cfg, callback_name="generic", tracer_provider=base_provider)
|
||||
endpoint = "http://127.0.0.1:1/v1/traces"
|
||||
dest = OtelDestination(callback_name="generic", endpoint=endpoint, headers={"x": "1"})
|
||||
|
||||
token = _request_destinations.set((dest,))
|
||||
try:
|
||||
kwargs = {"standard_logging_object": _mcp_payload()}
|
||||
asyncio.run(logger.async_log_success_event(kwargs, None, None, None))
|
||||
finally:
|
||||
_request_destinations.reset(token)
|
||||
|
||||
# Routing: a destination provider was built and points at the destination's endpoint. If the
|
||||
# tool-call span took the global-only path (no fan-out), no destination provider would exist.
|
||||
assert logger._tenant_tracers._providers, "no destination provider was built for the tool-call"
|
||||
destination_endpoints = " ".join(
|
||||
str(getattr(getattr(proc, "span_exporter", None), "_endpoint", ""))
|
||||
for provider in logger._tenant_tracers._providers.values()
|
||||
for proc in provider._active_span_processor._span_processors
|
||||
)
|
||||
assert endpoint in destination_endpoints, "no exporter pointed at the admin destination"
|
||||
|
||||
# Emission: the tool-call span was really emitted (the configured/global exporter rides its own
|
||||
# clean provider, so it is captured there rather than on a destination group).
|
||||
base_provider.force_flush()
|
||||
captured = [
|
||||
span
|
||||
for proc in base_provider._active_span_processor._span_processors
|
||||
if isinstance(getattr(proc, "span_exporter", None), InMemorySpanExporter)
|
||||
for span in proc.span_exporter.get_finished_spans()
|
||||
]
|
||||
assert any(s.attributes.get("mcp.method.name") == "tools/call" for s in captured), (
|
||||
"tool-call span was not emitted"
|
||||
)
|
||||
|
||||
|
||||
def test_mcp_tool_call_captures_io_when_enabled():
|
||||
logger, exporter = _logger_capturing()
|
||||
kwargs = {"standard_logging_object": _mcp_payload()}
|
||||
|
|
@ -863,6 +937,101 @@ def test_pre_call_idempotent_keeps_first_span():
|
|||
assert first is second # not overwritten
|
||||
|
||||
|
||||
def test_retry_reusing_a_closed_call_id_gets_its_own_span():
|
||||
"""Regression: a router retry or fallback reuses the request's ``litellm_call_id``,
|
||||
and the failed attempt's close marks that id closed before the retry runs. Treating
|
||||
the marker as final meant the retry opened no span and emitted nothing, so a request
|
||||
that failed once and then succeeded exported only the failure -- no tokens, no cost.
|
||||
|
||||
The retry now reopens, and the reopened span is finished and exported, so it cannot
|
||||
leak instead.
|
||||
"""
|
||||
logger, exporter = _logger()
|
||||
kwargs = _kwargs()
|
||||
server = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME)
|
||||
|
||||
# attempt 1: opened, then closed by the failure callback
|
||||
with trace.use_span(server, end_on_exit=False):
|
||||
logger.log_pre_api_call(model="gpt-4o", messages=[], kwargs=kwargs)
|
||||
assert "call_1" in logger._open_llm_calls
|
||||
asyncio.run(logger.async_log_success_event(kwargs, None, None, None))
|
||||
assert "call_1" in logger._closed_call_ids
|
||||
first_round = len([s for s in exporter.get_finished_spans() if s.name == "chat gpt-4o"])
|
||||
assert first_round == 1
|
||||
|
||||
# attempt 2 reuses the same call id and must get its own span
|
||||
with trace.use_span(server, end_on_exit=False):
|
||||
logger.log_pre_api_call(model="gpt-4o", messages=[], kwargs=kwargs)
|
||||
assert "call_1" in logger._open_llm_calls, "the retry must reopen, not be skipped"
|
||||
asyncio.run(logger.async_log_success_event(kwargs, None, None, None))
|
||||
server.end()
|
||||
|
||||
assert "call_1" not in logger._open_llm_calls, "the reopened span must be finished, not leaked"
|
||||
assert len([s for s in exporter.get_finished_spans() if s.name == "chat gpt-4o"]) == 2
|
||||
|
||||
|
||||
def test_retry_reaching_destinations_emits_a_span_per_attempt(monkeypatch):
|
||||
"""Regression, deferred/destination path: the destination logger is activated
|
||||
lazily at success time, so both the failed attempt and the retry that replaced it
|
||||
arrive as carrier-less closes sharing one ``litellm_call_id``. Deduping the fan-out
|
||||
on the call id alone collapsed the two, and the destination saw only the failure --
|
||||
no tokens, no cost. Each attempt carries its own response id and must emit.
|
||||
"""
|
||||
logger, exporter = _logger()
|
||||
monkeypatch.setattr(logger, "callback_name", "in_memory")
|
||||
monkeypatch.setattr(logger._tenant_tracers, "genai_tracers_for", lambda default, dests, params: (default,))
|
||||
_anchor([{"callback_name": "in_memory", "endpoint": "https://otlp.example.com/v1", "headers": {"api_key": "k"}}])
|
||||
|
||||
failed = _payload(
|
||||
id="attempt_1",
|
||||
status="failure",
|
||||
response={"id": "attempt_1", "model": "gpt-4o-2024", "choices": []},
|
||||
)
|
||||
succeeded = _payload(
|
||||
id="attempt_2",
|
||||
response={"id": "attempt_2", "model": "gpt-4o-2024", "choices": [{"finish_reason": "stop"}]},
|
||||
)
|
||||
assert failed["litellm_call_id"] == succeeded["litellm_call_id"], "a retry reuses one call id"
|
||||
|
||||
asyncio.run(logger.async_log_success_event(_kwargs(failed), None, None, None))
|
||||
asyncio.run(logger.async_log_success_event(_kwargs(succeeded), None, None, None))
|
||||
|
||||
emitted = [s for s in exporter.get_finished_spans() if s.name == "chat gpt-4o"]
|
||||
assert len(emitted) == 2, "the retry must reach the destination, not be deduped into the failure"
|
||||
|
||||
|
||||
def test_repeat_deferred_callback_for_one_attempt_still_dedupes(monkeypatch):
|
||||
"""The counterpart the fan-out dedup still guards: a sync+async double-firing of a
|
||||
single attempt shares its response id and must emit once."""
|
||||
logger, exporter = _logger()
|
||||
monkeypatch.setattr(logger, "callback_name", "in_memory")
|
||||
monkeypatch.setattr(logger._tenant_tracers, "genai_tracers_for", lambda default, dests, params: (default,))
|
||||
_anchor([{"callback_name": "in_memory", "endpoint": "https://otlp.example.com/v1", "headers": {"api_key": "k"}}])
|
||||
|
||||
payload = _payload(id="attempt_1", response={"id": "attempt_1", "model": "gpt-4o-2024", "choices": []})
|
||||
asyncio.run(logger.async_log_success_event(_kwargs(payload), None, None, None))
|
||||
asyncio.run(logger.async_log_success_event(_kwargs(payload), None, None, None))
|
||||
|
||||
assert len([s for s in exporter.get_finished_spans() if s.name == "chat gpt-4o"]) == 1
|
||||
|
||||
|
||||
def test_repeat_callback_without_an_open_span_emits_nothing():
|
||||
"""The counterpart the closed marker still guards: a second success callback for a
|
||||
call that has no open carrier must not re-emit through the deferred path."""
|
||||
logger, exporter = _logger()
|
||||
kwargs = _kwargs()
|
||||
server = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME)
|
||||
with trace.use_span(server, end_on_exit=False):
|
||||
logger.log_pre_api_call(model="gpt-4o", messages=[], kwargs=kwargs)
|
||||
asyncio.run(logger.async_log_success_event(kwargs, None, None, None))
|
||||
emitted = len([s for s in exporter.get_finished_spans() if s.name == "chat gpt-4o"])
|
||||
|
||||
asyncio.run(logger.async_log_success_event(kwargs, None, None, None)) # duplicate
|
||||
server.end()
|
||||
|
||||
assert len([s for s in exporter.get_finished_spans() if s.name == "chat gpt-4o"]) == emitted
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Parent resolution — ambient context at the boundary (no metadata threading)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
|
@ -937,7 +1106,7 @@ def test_live_llm_span_anchors_to_root_with_no_active_span():
|
|||
set_request_root_span(server)
|
||||
kwargs = _kwargs()
|
||||
logger.log_pre_api_call(model="gpt-4o", messages=[], kwargs=kwargs)
|
||||
assert logger._open_llm_calls["call_1"].span is not None # live, via anchor
|
||||
assert logger._open_llm_calls["call_1"].spans # live, via anchor
|
||||
asyncio.run(logger.async_log_success_event(kwargs, None, None, None))
|
||||
server.end()
|
||||
by_name = {s.name: s for s in exporter.get_finished_spans()}
|
||||
|
|
@ -957,7 +1126,7 @@ def test_deferred_llm_span_reads_anchor_at_close():
|
|||
kwargs = _kwargs()
|
||||
# pre_call with NO anchor and no active span → deferred.
|
||||
logger.log_pre_api_call(model="gpt-4o", messages=[], kwargs=kwargs)
|
||||
assert logger._open_llm_calls["call_1"].span is None # deferred
|
||||
assert logger._open_llm_calls["call_1"].spans == () # deferred
|
||||
# Anchor becomes visible at close (worker copied the request task's context).
|
||||
set_request_root_span(server)
|
||||
asyncio.run(logger.async_log_success_event(kwargs, None, None, None))
|
||||
|
|
@ -997,6 +1166,199 @@ def test_synthetic_error_log_produces_no_llm_span():
|
|||
assert "auth /chat/completions" in names # auth span itself still recorded
|
||||
|
||||
|
||||
def test_lazy_activation_emits_llm_span_when_destination_resolves(monkeypatch):
|
||||
"""LIT-3850 lazy-activation seam: a v2 instance born inside the success path
|
||||
(because the destination resolver appended its backend to ``success_callback``
|
||||
on this request) was not in the callback list when ``pre_call`` iterated, so
|
||||
no carrier was opened. The close must still emit the gen-ai span when the
|
||||
payload is present and the admin-resolved destinations name this backend, so
|
||||
the per-tenant exporter ships it. Without the fallthrough, this test sees
|
||||
zero spans."""
|
||||
logger, exporter = _logger()
|
||||
monkeypatch.setattr(logger, "callback_name", "in_memory")
|
||||
server = logger._emitter.start_span(
|
||||
SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME
|
||||
)
|
||||
set_request_root_span(server)
|
||||
# Make ``tracers_for`` return the logger's default tracer regardless of the
|
||||
# destinations passed: the test asserts the close-path emitted the span, not
|
||||
# that the per-destination provider clone wired up an OTLP exporter (the
|
||||
# routing cache's job, covered separately). The default tracer is bound to
|
||||
# the in-memory exporter so the test can read the result.
|
||||
tracer_for_calls: list[tuple] = []
|
||||
|
||||
def _fake_tracers_for(default, destinations):
|
||||
tracer_for_calls.append(destinations)
|
||||
return (default,)
|
||||
|
||||
monkeypatch.setattr(logger._tenant_tracers, "tracers_for", _fake_tracers_for)
|
||||
kwargs = _kwargs()
|
||||
_anchor([
|
||||
{
|
||||
"callback_name": "in_memory",
|
||||
"endpoint": "https://otlp.example.com/v1",
|
||||
"headers": {"api_key": "k"},
|
||||
}
|
||||
])
|
||||
assert "call_1" not in logger._open_llm_calls # no carrier opened
|
||||
asyncio.run(logger.async_log_success_event(kwargs, None, None, None))
|
||||
server.end()
|
||||
names = [s.name for s in exporter.get_finished_spans()]
|
||||
assert "chat gpt-4o" in names
|
||||
# ``tracers_for`` was invoked with exactly the resolved destination, proving
|
||||
# the deferred path used per-tenant routing rather than the default tracer
|
||||
# blindly.
|
||||
assert len(tracer_for_calls) == 1
|
||||
(dests,) = tracer_for_calls
|
||||
assert len(dests) == 1 and dests[0].endpoint == "https://otlp.example.com/v1"
|
||||
|
||||
|
||||
def test_no_upstream_reject_emits_no_deferred_span_even_with_destinations(monkeypatch):
|
||||
"""A close for a no-upstream request (``LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL`` set)
|
||||
with a payload and resolved destinations must emit no gen-AI span. Mirrors
|
||||
``test_lazy_activation_emits_llm_span_when_destination_resolves`` with the marker
|
||||
set: that one emits, this one must not."""
|
||||
from litellm.constants import LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL
|
||||
|
||||
logger, exporter = _logger()
|
||||
monkeypatch.setattr(logger, "callback_name", "in_memory")
|
||||
server = logger._emitter.start_span(
|
||||
SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME
|
||||
)
|
||||
set_request_root_span(server)
|
||||
|
||||
tracer_for_calls: list[tuple] = []
|
||||
|
||||
def _fake_tracers_for(default, destinations):
|
||||
tracer_for_calls.append(destinations)
|
||||
return (default,)
|
||||
|
||||
monkeypatch.setattr(logger._tenant_tracers, "tracers_for", _fake_tracers_for)
|
||||
_anchor([
|
||||
{
|
||||
"callback_name": "in_memory",
|
||||
"endpoint": "https://otlp.example.com/v1",
|
||||
"headers": {"api_key": "k"},
|
||||
}
|
||||
])
|
||||
payload = _payload(
|
||||
status="failure",
|
||||
error_information={"error_class": "ProxyException", "error_code": "429"},
|
||||
)
|
||||
kwargs = _kwargs(payload=payload)
|
||||
kwargs[LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL] = True
|
||||
|
||||
assert "call_1" not in logger._open_llm_calls
|
||||
asyncio.run(logger.async_log_failure_event(kwargs, None, None, None))
|
||||
server.end()
|
||||
|
||||
names = [s.name for s in exporter.get_finished_spans()]
|
||||
assert "chat gpt-4o" not in names
|
||||
assert tracer_for_calls == []
|
||||
|
||||
|
||||
def test_second_close_after_opened_call_does_not_emit_duplicate(monkeypatch):
|
||||
logger, exporter = _logger()
|
||||
monkeypatch.setattr(logger, "callback_name", "in_memory")
|
||||
monkeypatch.setattr(
|
||||
logger._tenant_tracers,
|
||||
"tracers_for",
|
||||
lambda default, destinations: (default,),
|
||||
)
|
||||
server = logger._emitter.start_span(
|
||||
SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME
|
||||
)
|
||||
set_request_root_span(server)
|
||||
kwargs = _kwargs()
|
||||
_anchor([
|
||||
{
|
||||
"callback_name": "in_memory",
|
||||
"endpoint": "https://otlp.example.com/v1",
|
||||
"headers": {"api_key": "k"},
|
||||
}
|
||||
])
|
||||
|
||||
logger.log_pre_api_call(model="gpt-4o", messages=[], kwargs=kwargs)
|
||||
asyncio.run(logger.async_log_success_event(kwargs, None, None, None))
|
||||
asyncio.run(logger.async_log_failure_event(kwargs, None, None, None))
|
||||
server.end()
|
||||
|
||||
names = [s.name for s in exporter.get_finished_spans()]
|
||||
assert names.count("chat gpt-4o") == 1
|
||||
|
||||
|
||||
def test_close_without_carrier_and_without_destination_drops_silently():
|
||||
"""The pre-existing early-return semantics (auth gate / pre-call guardrail
|
||||
rejection with no destination resolving to this backend) must be preserved:
|
||||
no phantom span. The fix only widens emit-on-close when the admin-resolved
|
||||
destinations name this backend AND the payload exists."""
|
||||
logger, exporter = _logger()
|
||||
server = logger._emitter.start_span(
|
||||
SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME
|
||||
)
|
||||
set_request_root_span(server)
|
||||
asyncio.run(logger.async_log_success_event(_kwargs(), None, None, None))
|
||||
server.end()
|
||||
names = [s.name for s in exporter.get_finished_spans()]
|
||||
assert "chat gpt-4o" not in names
|
||||
|
||||
|
||||
def test_close_without_carrier_drops_when_payload_missing(monkeypatch):
|
||||
"""No carrier + no payload = the auth-gate rejection case (no upstream call
|
||||
happened). Must drop even when destinations resolve, so a phantom span is
|
||||
never emitted for a request the gate refused."""
|
||||
logger, exporter = _logger()
|
||||
monkeypatch.setattr(logger, "callback_name", "in_memory")
|
||||
kwargs = {"litellm_params": {"metadata": {}}}
|
||||
_anchor(
|
||||
[
|
||||
{
|
||||
"callback_name": "in_memory",
|
||||
"endpoint": "https://otlp.example.com/v1",
|
||||
"headers": {},
|
||||
}
|
||||
]
|
||||
)
|
||||
asyncio.run(logger.async_log_success_event(kwargs, None, None, None))
|
||||
assert exporter.get_finished_spans() == ()
|
||||
|
||||
|
||||
def test_close_dedupes_duplicate_callbacks(monkeypatch):
|
||||
"""Cursor BugBot regression: a normal close pops the carrier and emits
|
||||
the LLM span; a second callback for the same call_id must not emit a
|
||||
duplicate. Before the dedup guard, the second close hit the
|
||||
carrier-is-None branch and fired _emit_deferred_llm_call again whenever
|
||||
payload + destinations remained on the kwargs, double-exporting the
|
||||
span (e.g. success + failure callbacks both firing, or a custom callback
|
||||
fanning out)."""
|
||||
logger, exporter = _logger()
|
||||
monkeypatch.setattr(logger, "callback_name", "in_memory")
|
||||
server = logger._emitter.start_span(
|
||||
SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME
|
||||
)
|
||||
set_request_root_span(server)
|
||||
monkeypatch.setattr(
|
||||
logger._tenant_tracers, "tracers_for", lambda default, dests: (default,)
|
||||
)
|
||||
kwargs = _kwargs()
|
||||
_anchor([
|
||||
{
|
||||
"callback_name": "in_memory",
|
||||
"endpoint": "https://otlp.example.com/v1",
|
||||
"headers": {},
|
||||
}
|
||||
])
|
||||
logger.log_pre_api_call(model="gpt-4o", messages=[], kwargs=kwargs)
|
||||
asyncio.run(logger.async_log_success_event(kwargs, None, None, None))
|
||||
# Second callback for the same call_id: payload + destinations still on
|
||||
# kwargs, but no second span must emit.
|
||||
asyncio.run(logger.async_log_success_event(kwargs, None, None, None))
|
||||
asyncio.run(logger.async_log_failure_event(kwargs, None, None, None))
|
||||
server.end()
|
||||
llm_spans = [s for s in exporter.get_finished_spans() if s.name == "chat gpt-4o"]
|
||||
assert len(llm_spans) == 1
|
||||
|
||||
|
||||
def test_create_request_started_span_captures_anchor():
|
||||
"""``create_litellm_proxy_request_started_span`` doubles as the anchor capture
|
||||
point: the active server span becomes the request root for later spans."""
|
||||
|
|
@ -2277,3 +2639,192 @@ def test_metrics_disabled_by_default_records_nothing(monkeypatch):
|
|||
)
|
||||
)
|
||||
assert _emitted_metric_names(reader) == set()
|
||||
|
||||
|
||||
def _second_group_tracer(logger):
|
||||
"""A second independent in-memory provider standing in for a second Resource group
|
||||
(e.g. a second Arize project); returns (tracer, exporter)."""
|
||||
exporter = InMemorySpanExporter()
|
||||
provider = providers.build_tracer_provider(logger.config, exporter=exporter)
|
||||
return provider.get_tracer("litellm"), exporter
|
||||
|
||||
|
||||
def test_genai_span_emitted_to_every_group_live(monkeypatch):
|
||||
"""Multi-destination fix (live path): when ``tracers_for`` returns two tracers (two
|
||||
Resource groups, e.g. two Arize projects), the gen-AI span opened at ``pre_call``
|
||||
must be opened+finished on BOTH -- the bug was only one project receiving it."""
|
||||
logger, exporter_a = _logger()
|
||||
monkeypatch.setattr(logger, "callback_name", "in_memory")
|
||||
tracer_b, exporter_b = _second_group_tracer(logger)
|
||||
monkeypatch.setattr(
|
||||
logger._tenant_tracers,
|
||||
"tracers_for",
|
||||
lambda default, dests: (default, tracer_b),
|
||||
)
|
||||
server = logger._emitter.start_span(
|
||||
SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME
|
||||
)
|
||||
set_request_root_span(server)
|
||||
kwargs = _kwargs()
|
||||
_anchor([
|
||||
{"callback_name": "in_memory", "endpoint": "https://x/v1", "headers": {}}
|
||||
])
|
||||
_emit_llm(logger, kwargs, ambient=server)
|
||||
server.end()
|
||||
assert [s.name for s in exporter_a.get_finished_spans()].count("chat gpt-4o") == 1
|
||||
assert [s.name for s in exporter_b.get_finished_spans()].count("chat gpt-4o") == 1
|
||||
|
||||
|
||||
def test_genai_span_emitted_to_every_group_deferred(monkeypatch):
|
||||
"""Same fix, deferred path (no carrier at ``pre_call``): ``emit_fanout`` dedups once
|
||||
on the call id then emits the span on every group's tracer, so both projects get
|
||||
exactly one."""
|
||||
logger, exporter_a = _logger()
|
||||
monkeypatch.setattr(logger, "callback_name", "in_memory")
|
||||
tracer_b, exporter_b = _second_group_tracer(logger)
|
||||
monkeypatch.setattr(
|
||||
logger._tenant_tracers,
|
||||
"tracers_for",
|
||||
lambda default, dests: (default, tracer_b),
|
||||
)
|
||||
server = logger._emitter.start_span(
|
||||
SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME
|
||||
)
|
||||
set_request_root_span(server)
|
||||
kwargs = _kwargs()
|
||||
_anchor([
|
||||
{"callback_name": "in_memory", "endpoint": "https://x/v1", "headers": {}}
|
||||
])
|
||||
# no pre_call -> no carrier -> deferred close path
|
||||
assert "call_1" not in logger._open_llm_calls
|
||||
asyncio.run(logger.async_log_success_event(kwargs, None, None, None))
|
||||
server.end()
|
||||
assert [s.name for s in exporter_a.get_finished_spans()].count("chat gpt-4o") == 1
|
||||
assert [s.name for s in exporter_b.get_finished_spans()].count("chat gpt-4o") == 1
|
||||
|
||||
|
||||
def _generic_dest():
|
||||
return {
|
||||
"callback_name": "generic",
|
||||
"endpoint": "http://collector:4318",
|
||||
"headers": {},
|
||||
}
|
||||
|
||||
|
||||
def test_generic_destination_emits_genai_span(monkeypatch):
|
||||
"""Acceptance #1: a request whose only admin destination is a Generic OTLP
|
||||
destination emits the chat <model> gen-AI span (regression: 'generic' had no preset,
|
||||
so the gen-AI span was dropped and only proxy-internal spans reached the endpoint).
|
||||
The destination's callback_name='generic' is matched by the generic logger, and the
|
||||
span is routed through the per-destination tracer."""
|
||||
logger, exporter = _logger()
|
||||
monkeypatch.setattr(logger, "callback_name", "generic")
|
||||
routed: list = []
|
||||
monkeypatch.setattr(
|
||||
logger._tenant_tracers,
|
||||
"tracers_for",
|
||||
lambda default, dests: (routed.append(dests) or (default,)),
|
||||
)
|
||||
server = logger._emitter.start_span(
|
||||
SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME
|
||||
)
|
||||
set_request_root_span(server)
|
||||
kwargs = _kwargs()
|
||||
_anchor([_generic_dest()])
|
||||
_emit_llm(logger, kwargs, ambient=server)
|
||||
server.end()
|
||||
assert "chat gpt-4o" in [s.name for s in exporter.get_finished_spans()]
|
||||
# the gen-AI span was routed for the generic destination (not an empty/global set)
|
||||
assert any(
|
||||
len(d) == 1 and d[0].endpoint == "http://collector:4318" for d in routed
|
||||
), "generic destination was not routed to the generic logger's tracer"
|
||||
|
||||
|
||||
def test_generic_destination_emits_error_span_on_failure(monkeypatch):
|
||||
"""Acceptance #3: a FAILED call to a Generic OTLP destination still emits the
|
||||
chat <model> span, with OTEL status ERROR and the exception type."""
|
||||
logger, exporter = _logger()
|
||||
monkeypatch.setattr(logger, "callback_name", "generic")
|
||||
monkeypatch.setattr(
|
||||
logger._tenant_tracers, "tracers_for", lambda default, dests: (default,)
|
||||
)
|
||||
server = logger._emitter.start_span(
|
||||
SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME
|
||||
)
|
||||
set_request_root_span(server)
|
||||
payload = _payload(
|
||||
status="failure",
|
||||
error_information={
|
||||
"error_class": "AuthenticationError",
|
||||
"error_message": "Incorrect API key",
|
||||
},
|
||||
)
|
||||
kwargs = _kwargs(payload=payload)
|
||||
_anchor([_generic_dest()])
|
||||
_emit_llm(logger, kwargs, ambient=server, fail=True)
|
||||
server.end()
|
||||
spans = {s.name: s for s in exporter.get_finished_spans()}
|
||||
assert "chat gpt-4o" in spans
|
||||
assert spans["chat gpt-4o"].status.status_code is StatusCode.ERROR
|
||||
assert spans["chat gpt-4o"].attributes["error.type"] == "AuthenticationError"
|
||||
|
||||
|
||||
def test_lazy_activation_emits_llm_span_for_a_team_carrying_its_own_credentials(monkeypatch):
|
||||
"""Regression: a team's own ``callback_vars`` export must survive another team's grant.
|
||||
|
||||
Once an admin registers a destination for a backend, v2 owns it and the legacy logger
|
||||
is no longer built, so this instance is the only thing that can reach a team's own
|
||||
account. That team has no destination of its own, so it is lazily activated at the
|
||||
success event with no carrier. Gating the deferred span on destinations alone dropped
|
||||
it: the team silently stopped exporting the moment a *different* team was granted a
|
||||
destination for the same backend.
|
||||
"""
|
||||
logger, exporter = _logger()
|
||||
monkeypatch.setattr(logger, "callback_name", "langfuse_otel")
|
||||
server = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME)
|
||||
set_request_root_span(server)
|
||||
|
||||
genai_calls: list[tuple] = []
|
||||
|
||||
def _fake_genai_tracers_for(default, destinations, dynamic_params):
|
||||
genai_calls.append((destinations, dynamic_params))
|
||||
return (default,)
|
||||
|
||||
monkeypatch.setattr(logger._tenant_tracers, "genai_tracers_for", _fake_genai_tracers_for)
|
||||
|
||||
kwargs = _kwargs()
|
||||
kwargs["standard_callback_dynamic_params"] = {
|
||||
"langfuse_public_key": "pk-team",
|
||||
"langfuse_secret_key": "sk-team",
|
||||
"langfuse_host": "https://team.langfuse.example",
|
||||
}
|
||||
_anchor([]) # no destination grants this identity
|
||||
|
||||
assert "call_1" not in logger._open_llm_calls
|
||||
asyncio.run(logger.async_log_success_event(kwargs, None, None, None))
|
||||
server.end()
|
||||
|
||||
assert "chat gpt-4o" in [s.name for s in exporter.get_finished_spans()]
|
||||
# routed by the request's own credentials, with no destinations in play
|
||||
assert len(genai_calls) == 1
|
||||
destinations, dynamic_params = genai_calls[0]
|
||||
assert destinations == ()
|
||||
assert dynamic_params["langfuse_public_key"] == "pk-team"
|
||||
|
||||
|
||||
def test_lazy_activation_stays_silent_without_destination_or_credentials(monkeypatch):
|
||||
"""The counterpart: no destination and no per-request credentials for this backend
|
||||
means nothing activated it, so the deferred path must emit nothing. Default-deny is
|
||||
what keeps an ungranted identity off every destination."""
|
||||
logger, exporter = _logger()
|
||||
monkeypatch.setattr(logger, "callback_name", "langfuse_otel")
|
||||
server = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME)
|
||||
set_request_root_span(server)
|
||||
|
||||
kwargs = _kwargs()
|
||||
_anchor([])
|
||||
|
||||
asyncio.run(logger.async_log_success_event(kwargs, None, None, None))
|
||||
server.end()
|
||||
|
||||
assert "chat gpt-4o" not in [s.name for s in exporter.get_finished_spans()]
|
||||
|
|
|
|||
|
|
@ -84,6 +84,14 @@ def test_gate_toggles_with_env(monkeypatch):
|
|||
assert is_otel_v2_enabled() is True
|
||||
|
||||
|
||||
def test_blank_flag_does_not_escape_the_startup_mount(monkeypatch):
|
||||
"""The whole-proxy symptom: ``instrument_fastapi_app`` is called at
|
||||
``proxy_server`` import time, so it must not raise on a blank flag."""
|
||||
monkeypatch.setenv("LITELLM_OTEL_V2", "")
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
instrument_fastapi_app(fastapi.FastAPI())
|
||||
|
||||
|
||||
def test_instrumented_app_emits_server_span():
|
||||
app, logger = _instrumented_app()
|
||||
exporter = InMemorySpanExporter()
|
||||
|
|
|
|||
|
|
@ -87,3 +87,33 @@ def test_config_folds_legacy_exporter_triple_into_exporters_list():
|
|||
assert len(cfg.exporters) == 1
|
||||
assert cfg.exporters[0].kind == "otlp_http"
|
||||
assert cfg.exporters[0].endpoint == "https://api.example.com"
|
||||
|
||||
|
||||
def _processor_names(provider):
|
||||
return [type(p).__name__ for p in provider._active_span_processor._span_processors]
|
||||
|
||||
|
||||
def test_main_logger_provider_always_carries_fan_out():
|
||||
# The FastAPI server span (and auth/db/cost spans) bind to the main v2 logger's
|
||||
# provider. That provider MUST carry the TenantFanOutSpanProcessor even for the
|
||||
# generic global logger (no backend named) -- otherwise, for an admin-owned-
|
||||
# destination-only deployment, the server span never reaches the destination and
|
||||
# the gen-AI child is orphaned. Regression for the orphaned-span RCA.
|
||||
from litellm.integrations.otel.logger import OpenTelemetryV2
|
||||
|
||||
generic = OpenTelemetryV2()
|
||||
assert "TenantFanOutSpanProcessor" in _processor_names(generic._tracer_provider)
|
||||
|
||||
named = OpenTelemetryV2(callback_name="arize")
|
||||
assert "TenantFanOutSpanProcessor" in _processor_names(named._tracer_provider)
|
||||
|
||||
|
||||
def test_build_tracer_provider_attach_fan_out_flag():
|
||||
cfg = OpenTelemetryV2Config(exporter="console")
|
||||
# Per-tenant clone providers pass neither flag and must NOT fan out (else the
|
||||
# gen-AI span they carry would be double-exported).
|
||||
clone = build_tracer_provider(cfg)
|
||||
assert "TenantFanOutSpanProcessor" not in _processor_names(clone)
|
||||
# The main provider opts in explicitly.
|
||||
main = build_tracer_provider(cfg, attach_tenant_fan_out=True)
|
||||
assert "TenantFanOutSpanProcessor" in _processor_names(main)
|
||||
|
|
|
|||
|
|
@ -33,27 +33,130 @@ def test_agentops_preset_does_no_network_io(monkeypatch):
|
|||
assert spec.options == {"api_key": "ak-123"} # carried to the lazy exporter
|
||||
|
||||
|
||||
def test_agentops_preset_without_key_omits_options(monkeypatch):
|
||||
def test_agentops_preset_without_key_still_appends_exporter(monkeypatch):
|
||||
# Additive parity with the pre-PR preset: a global agentops callback always
|
||||
# contributes its exporter, with no api_key carried when none is configured.
|
||||
# The JWT-minting exporter simply has nothing to mint until a key is set.
|
||||
monkeypatch.delenv("AGENTOPS_API_KEY", raising=False)
|
||||
cfg = agentops_preset()
|
||||
spec = next(e for e in cfg.exporters if e.kind == _AGENTOPS_EXPORTER_KIND)
|
||||
assert spec.options is None
|
||||
specs = [e for e in cfg.exporters if e.kind == _AGENTOPS_EXPORTER_KIND]
|
||||
assert len(specs) == 1
|
||||
assert specs[0].options is None
|
||||
|
||||
|
||||
def test_arize_preset_without_credentials_still_appends_exporter(monkeypatch):
|
||||
# Additive parity: a global arize callback always contributes its exporter at
|
||||
# the configured (or default) endpoint, so a no-auth self-hosted collector
|
||||
# reached via ARIZE_ENDPOINT keeps exporting exactly as before the PR.
|
||||
from litellm.integrations.otel.model.config import ExporterOwner
|
||||
from litellm.integrations.otel.presets.arize import arize_preset
|
||||
|
||||
for var in (
|
||||
"ARIZE_SPACE_ID",
|
||||
"ARIZE_SPACE_KEY",
|
||||
"ARIZE_API_KEY",
|
||||
"OTEL_EXPORTER_OTLP_TRACES_HEADERS",
|
||||
):
|
||||
monkeypatch.delenv(var, raising=False)
|
||||
cfg = arize_preset()
|
||||
assert [e for e in cfg.exporters if e.owner == ExporterOwner.ARIZE_AX] != []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"preset_name, owner_name, env",
|
||||
[
|
||||
("arize", "ARIZE_AX", ("ARIZE_SPACE_ID", "ARIZE_SPACE_KEY", "ARIZE_API_KEY", "ARIZE_ENDPOINT", "ARIZE_HTTP_ENDPOINT")),
|
||||
("agentops", "AGENTOPS", ("AGENTOPS_API_KEY",)),
|
||||
],
|
||||
)
|
||||
def test_credential_optional_presets_add_no_vendor_exporter_for_a_destination(monkeypatch, preset_name, owner_name, env):
|
||||
# Regression: registering a team-scoped destination builds the backend proxy-wide with
|
||||
# allow_missing_credentials=True. A credential-optional preset must then contribute NO
|
||||
# global exporter: appending its public vendor endpoint anyway ships that tenant's spans
|
||||
# to a vendor account nobody granted (and Arize additionally stamps the operator's own
|
||||
# OTLP auth header onto that uninvited request).
|
||||
from litellm.integrations.otel.model.config import ExporterOwner
|
||||
from litellm.integrations.otel.presets import PRESET_BY_CALLBACK
|
||||
|
||||
for var in env:
|
||||
monkeypatch.delenv(var, raising=False)
|
||||
monkeypatch.setenv("OTEL_EXPORTER_OTLP_TRACES_HEADERS", "authorization=Bearer OPERATOR-COLLECTOR-SECRET")
|
||||
|
||||
cfg = PRESET_BY_CALLBACK[preset_name](allow_missing_credentials=True)
|
||||
owner = getattr(ExporterOwner, owner_name)
|
||||
assert [e for e in cfg.exporters if e.owner == owner] == []
|
||||
assert "OPERATOR-COLLECTOR-SECRET" not in str([e.headers for e in cfg.exporters])
|
||||
assert cfg.mapper_names, "degrading must keep the preset's mappers"
|
||||
|
||||
|
||||
def test_arize_keeps_its_exporter_for_an_operators_own_collector(monkeypatch):
|
||||
# The carve-out: ARIZE_ENDPOINT with no Arize credentials is a legitimate self-hosted,
|
||||
# no-auth collector authenticated via the standard OTLP headers env var. Degrading must
|
||||
# not take that away, so the exporter stays and keeps carrying those headers.
|
||||
from litellm.integrations.otel.model.config import ExporterOwner
|
||||
from litellm.integrations.otel.presets.arize import arize_preset
|
||||
|
||||
for var in ("ARIZE_SPACE_ID", "ARIZE_SPACE_KEY", "ARIZE_API_KEY", "ARIZE_HTTP_ENDPOINT"):
|
||||
monkeypatch.delenv(var, raising=False)
|
||||
monkeypatch.setenv("ARIZE_ENDPOINT", "https://collector.internal/v1")
|
||||
monkeypatch.setenv("OTEL_EXPORTER_OTLP_TRACES_HEADERS", "authorization=Bearer OPERATOR-COLLECTOR-SECRET")
|
||||
|
||||
specs = [e for e in arize_preset(allow_missing_credentials=True).exporters if e.owner == ExporterOwner.ARIZE_AX]
|
||||
assert len(specs) == 1
|
||||
assert specs[0].endpoint == "https://collector.internal/v1"
|
||||
assert specs[0].headers == "authorization=Bearer OPERATOR-COLLECTOR-SECRET"
|
||||
|
||||
|
||||
def test_arize_public_endpoint_spec_carries_no_operator_headers(monkeypatch):
|
||||
# A global arize callback with no credentials still exports (additive parity), and the
|
||||
# spec this preset contributes carries no headers of its own, so the operator's OTLP auth
|
||||
# header is never written into an exporter aimed at Arize's public endpoint.
|
||||
#
|
||||
# This asserts the spec only. The OTLP SDK still reads OTEL_EXPORTER_OTLP_TRACES_HEADERS
|
||||
# from the environment for any header-less exporter; that is standard OTLP behaviour that
|
||||
# predates this PR and applies to every integration, so it is out of scope here.
|
||||
from litellm.integrations.otel.model.config import ExporterOwner
|
||||
from litellm.integrations.otel.presets.arize import ARIZE_PUBLIC_OTLP_ENDPOINT, arize_preset
|
||||
|
||||
for var in ("ARIZE_SPACE_ID", "ARIZE_SPACE_KEY", "ARIZE_API_KEY", "ARIZE_ENDPOINT", "ARIZE_HTTP_ENDPOINT"):
|
||||
monkeypatch.delenv(var, raising=False)
|
||||
monkeypatch.setenv("OTEL_EXPORTER_OTLP_TRACES_HEADERS", "authorization=Bearer OPERATOR-COLLECTOR-SECRET")
|
||||
|
||||
specs = [e for e in arize_preset().exporters if e.owner == ExporterOwner.ARIZE_AX]
|
||||
assert len(specs) == 1
|
||||
assert specs[0].endpoint == ARIZE_PUBLIC_OTLP_ENDPOINT
|
||||
assert specs[0].headers is None
|
||||
|
||||
|
||||
def test_phoenix_preset_without_config_appends_localhost_exporter(monkeypatch):
|
||||
# Additive parity: unconfigured Phoenix defaults to http://localhost:6006 and
|
||||
# the preset always contributes that exporter, so a local self-hosted Phoenix
|
||||
# deployment with no env vars keeps receiving traces exactly as before the PR.
|
||||
from litellm.integrations.otel.model.config import ExporterOwner
|
||||
from litellm.integrations.otel.presets.phoenix import phoenix_preset
|
||||
|
||||
for var in ("PHOENIX_API_KEY", "PHOENIX_COLLECTOR_ENDPOINT", "PHOENIX_COLLECTOR_HTTP_ENDPOINT"):
|
||||
monkeypatch.delenv(var, raising=False)
|
||||
cfg = phoenix_preset()
|
||||
phoenix_exporters = [e for e in cfg.exporters if e.owner == ExporterOwner.ARIZE_PHOENIX]
|
||||
assert len(phoenix_exporters) == 1
|
||||
assert phoenix_exporters[0].endpoint == "http://localhost:6006/v1/traces"
|
||||
|
||||
|
||||
def test_agentops_exporter_factory_is_registered():
|
||||
assert _AGENTOPS_EXPORTER_KIND in providers._EXPORTER_FACTORIES
|
||||
|
||||
|
||||
def test_dynamic_cred_presets_tag_exporter_with_matching_owner(monkeypatch):
|
||||
"""Each dynamic-credential preset must tag the exporter it contributes with
|
||||
its own callback name, so per-request tenant routing
|
||||
(``TenantTracerCache``) applies that integration's credentials only to its
|
||||
own exporter and never bleeds them onto a co-configured backend.
|
||||
def test_destination_routable_presets_tag_exporter_with_matching_owner(monkeypatch):
|
||||
"""Each destination-routable preset must tag the exporter it contributes with
|
||||
its own callback name, so per-tenant routing (``TenantTracerCache``) points
|
||||
that integration's admin destination at its own exporter only and never
|
||||
rewrites a co-configured backend's exporter.
|
||||
"""
|
||||
from litellm.integrations.otel.presets import (
|
||||
DYNAMIC_HEADERS_BY_CALLBACK,
|
||||
PRESET_BY_CALLBACK,
|
||||
from litellm.integrations.otel.presets.destinations import (
|
||||
OTEL_V2_DESTINATION_CALLBACKS,
|
||||
)
|
||||
from litellm.integrations.otel.presets import PRESET_BY_CALLBACK
|
||||
|
||||
monkeypatch.setenv("ARIZE_SPACE_ID", "S")
|
||||
monkeypatch.setenv("ARIZE_API_KEY", "K")
|
||||
|
|
@ -65,7 +168,7 @@ def test_dynamic_cred_presets_tag_exporter_with_matching_owner(monkeypatch):
|
|||
|
||||
from litellm.integrations.otel.model.config import ExporterOwner
|
||||
|
||||
for callback_name in DYNAMIC_HEADERS_BY_CALLBACK:
|
||||
for callback_name in OTEL_V2_DESTINATION_CALLBACKS:
|
||||
cfg = PRESET_BY_CALLBACK[callback_name]()
|
||||
owners = {e.owner for e in cfg.exporters}
|
||||
assert ExporterOwner(callback_name) in owners, (
|
||||
|
|
@ -160,3 +263,107 @@ def test_agentops_endpoint_points_at_live_host():
|
|||
# so a typo or stale domain can never ship again.
|
||||
assert _AGENTOPS_ENDPOINT == "https://otlp.agentops.ai/v1/traces"
|
||||
assert "agentops.cloud" not in _AGENTOPS_ENDPOINT
|
||||
|
||||
|
||||
def test_credential_mandatory_presets_raise_without_creds_by_default(monkeypatch):
|
||||
# weave/langfuse/levo are credential-mandatory: with no global env creds and no
|
||||
# opt-in to degrade, the preset must RAISE so a misconfigured global callback
|
||||
# (e.g. ``callbacks: ["weave_otel"]`` with no keys) fails loud at startup, the
|
||||
# same error story as before V2 landed. _maybe_construct_otel_v2 relies on this
|
||||
# raise to defer to the legacy path.
|
||||
from litellm.integrations.otel.presets.langfuse import langfuse_preset
|
||||
from litellm.integrations.otel.presets.levo import levo_preset
|
||||
from litellm.integrations.otel.presets.weave import weave_preset
|
||||
|
||||
for var in (
|
||||
"WANDB_API_KEY",
|
||||
"WANDB_PROJECT_ID",
|
||||
"LANGFUSE_PUBLIC_KEY",
|
||||
"LANGFUSE_SECRET_KEY",
|
||||
"LEVOAI_API_KEY",
|
||||
):
|
||||
monkeypatch.delenv(var, raising=False)
|
||||
|
||||
for preset in (weave_preset, langfuse_preset, levo_preset):
|
||||
with pytest.raises(Exception):
|
||||
preset()
|
||||
|
||||
|
||||
def test_credential_mandatory_presets_degrade_when_allowed(monkeypatch):
|
||||
# When an admin-owned destination is the reason for construction it carries its
|
||||
# own per-tenant credentials, so the preset is called with
|
||||
# allow_missing_credentials=True and must degrade to an exporter-less config
|
||||
# (keeping its mappers) instead of raising -- otherwise the destination never gets
|
||||
# a v2 logger and its gen-AI span falls to the generic global logger.
|
||||
from litellm.integrations.otel.model.config import ExporterOwner
|
||||
from litellm.integrations.otel.presets.langfuse import langfuse_preset
|
||||
from litellm.integrations.otel.presets.levo import levo_preset
|
||||
from litellm.integrations.otel.presets.weave import weave_preset
|
||||
|
||||
for var in (
|
||||
"WANDB_API_KEY",
|
||||
"WANDB_PROJECT_ID",
|
||||
"LANGFUSE_PUBLIC_KEY",
|
||||
"LANGFUSE_SECRET_KEY",
|
||||
"LEVOAI_API_KEY",
|
||||
):
|
||||
monkeypatch.delenv(var, raising=False)
|
||||
|
||||
for preset, owner in (
|
||||
(weave_preset, ExporterOwner.WEAVE_OTEL),
|
||||
(langfuse_preset, ExporterOwner.LANGFUSE_OTEL),
|
||||
(levo_preset, ExporterOwner.LEVO),
|
||||
):
|
||||
cfg = preset(allow_missing_credentials=True) # must not raise
|
||||
assert [e for e in cfg.exporters if e.owner == owner] == []
|
||||
|
||||
|
||||
def test_weave_preset_with_creds_contributes_exporter(monkeypatch):
|
||||
from litellm.integrations.otel.model.config import ExporterOwner
|
||||
from litellm.integrations.otel.presets.weave import weave_preset
|
||||
|
||||
monkeypatch.setenv("WANDB_API_KEY", "w-key")
|
||||
monkeypatch.setenv("WANDB_PROJECT_ID", "entity/project")
|
||||
cfg = weave_preset()
|
||||
assert [e for e in cfg.exporters if e.owner == ExporterOwner.WEAVE_OTEL] != []
|
||||
|
||||
|
||||
def test_generic_preset_needs_no_global_env_and_emits_genai(monkeypatch):
|
||||
# Acceptance #2: the generic preset is vendor-neutral and admin-destination-only --
|
||||
# it must build with NO global OTEL env vars, never raise, carry the standard genai
|
||||
# (+legacy) mappers, and contribute NO vendor exporter (the per-destination exporter
|
||||
# is appended by the router).
|
||||
from litellm.integrations.otel.presets.generic import generic_preset
|
||||
|
||||
for var in ("OTEL_EXPORTER_OTLP_ENDPOINT", "OTEL_ENDPOINT", "OTEL_EXPORTER"):
|
||||
monkeypatch.delenv(var, raising=False)
|
||||
cfg = generic_preset() # must not raise, no env needed
|
||||
assert "genai" in cfg.mapper_names
|
||||
# A credential-less generic config must contribute NO exporter at all. In particular
|
||||
# it must not degrade to the default console exporter, which prints every span
|
||||
# (including prompt/completion content) to stdout synchronously on the request path.
|
||||
assert cfg.exporters == []
|
||||
# the degrade flag is accepted (Preset protocol) and irrelevant -- still builds
|
||||
assert generic_preset(allow_missing_credentials=True).exporters == []
|
||||
|
||||
|
||||
def test_bare_and_degraded_configs_do_not_console_flood(monkeypatch):
|
||||
# The "nothing configured" degrade case (bare config, or a credential-mandatory
|
||||
# preset degrading with allow_missing_credentials) must export nothing rather than
|
||||
# folding the default console exporter in and dumping every span to stdout.
|
||||
from litellm.integrations.otel.model.config import OpenTelemetryV2Config
|
||||
from litellm.integrations.otel.presets.langfuse import langfuse_preset
|
||||
from litellm.integrations.otel.presets.weave import weave_preset
|
||||
|
||||
assert OpenTelemetryV2Config().exporters == []
|
||||
# An explicit endpoint (a real destination) still folds into one OTLP exporter.
|
||||
assert len(OpenTelemetryV2Config(endpoint="http://collector/v1/traces").exporters) == 1
|
||||
# A deployment that *explicitly* selects console output still gets it -- only the
|
||||
# default/degrade console (exporter left unset) is suppressed.
|
||||
explicit_console = OpenTelemetryV2Config(exporter="console")
|
||||
assert len(explicit_console.exporters) == 1 and explicit_console.exporters[0].kind == "console"
|
||||
|
||||
for var in ("LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY", "WANDB_API_KEY", "WANDB_PROJECT_ID"):
|
||||
monkeypatch.delenv(var, raising=False)
|
||||
assert langfuse_preset(allow_missing_credentials=True).exporters == []
|
||||
assert weave_preset(allow_missing_credentials=True).exporters == []
|
||||
|
|
|
|||
|
|
@ -4156,6 +4156,175 @@ def test_failure_handler_zeroes_spend_without_recovered_usage(logging_obj):
|
|||
assert payload["total_tokens"] == 0
|
||||
|
||||
|
||||
def test_admin_owned_destination_does_not_activate_v2_without_flag(monkeypatch):
|
||||
# LITELLM_OTEL_V2 is the sole activation gate: registering an admin-owned logging
|
||||
# destination must NOT flip a v1 deployment onto v2. With the flag off,
|
||||
# _maybe_construct_otel_v2 returns None whether or not a destination exists, so an
|
||||
# existing v1 deployment is unaffected by merely registering a credential (and the
|
||||
# flag-off + destination "orphaned tree" configuration can't arise).
|
||||
from types import SimpleNamespace
|
||||
|
||||
from litellm.integrations.otel.model.config import is_otel_v2_enabled
|
||||
from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2
|
||||
|
||||
monkeypatch.delenv("LITELLM_OTEL_V2", raising=False)
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
assert is_otel_v2_enabled() is False
|
||||
|
||||
# No logging credential for the backend -> legacy fallback (None).
|
||||
monkeypatch.setattr(litellm, "credential_list", [])
|
||||
assert _maybe_construct_otel_v2("arize", []) is None
|
||||
|
||||
# A logging destination registered for the backend, flag still off -> still None.
|
||||
monkeypatch.setattr(
|
||||
litellm,
|
||||
"credential_list",
|
||||
[
|
||||
SimpleNamespace(
|
||||
credential_name="arize-poc",
|
||||
credential_info={
|
||||
"credential_type": "logging",
|
||||
"description": "arize",
|
||||
},
|
||||
)
|
||||
],
|
||||
)
|
||||
result = _maybe_construct_otel_v2("arize", [])
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_credential_mandatory_backend_global_misconfig_stays_loud(monkeypatch):
|
||||
# Regression: weave/langfuse/levo are credential-mandatory; before V2 a global
|
||||
# callback (e.g. ``callbacks: ["weave_otel"]``) with no credentials failed loud at
|
||||
# startup. _maybe_construct_otel_v2 preserves that: with no creds the preset raises,
|
||||
# _maybe_construct returns None, and the caller falls through to the legacy path.
|
||||
#
|
||||
# An admin-owned destination does not change that answer. Ownership is the operator's
|
||||
# configuration alone, so registering a destination for one team cannot move any other
|
||||
# tenant off the logger they already had; the destination is delivered to separately by
|
||||
# AdminDestinationLogger.
|
||||
from types import SimpleNamespace
|
||||
|
||||
from litellm.integrations.otel.model.config import is_otel_v2_enabled
|
||||
from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2
|
||||
|
||||
for var in ("WANDB_API_KEY", "WANDB_PROJECT_ID"):
|
||||
monkeypatch.delenv(var, raising=False)
|
||||
|
||||
monkeypatch.setenv("LITELLM_OTEL_V2", "true")
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
|
||||
monkeypatch.setattr(litellm, "credential_list", [])
|
||||
assert _maybe_construct_otel_v2("weave_otel", []) is None
|
||||
|
||||
monkeypatch.setattr(
|
||||
litellm,
|
||||
"credential_list",
|
||||
[
|
||||
SimpleNamespace(
|
||||
credential_name="wb-poc",
|
||||
credential_values={"wandb_api_key": "wb-key"},
|
||||
credential_info={
|
||||
"credential_type": "logging",
|
||||
"description": "weave_otel",
|
||||
"access": {"teams": ["team-a"]},
|
||||
},
|
||||
)
|
||||
],
|
||||
)
|
||||
assert _maybe_construct_otel_v2("weave_otel", []) is None
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"credential_info, credential_values, why",
|
||||
[
|
||||
({"credential_type": "logging", "description": "weave_otel"}, {"wandb_api_key": "k"}, "no access at all"),
|
||||
(
|
||||
{"credential_type": "logging", "description": "weave_otel", "access": {}},
|
||||
{"wandb_api_key": "k"},
|
||||
"empty access grants nobody",
|
||||
),
|
||||
(
|
||||
{
|
||||
"credential_type": "logging",
|
||||
"description": "weave_otel",
|
||||
"access": {"global": False, "teams": [], "orgs": []},
|
||||
},
|
||||
{"wandb_api_key": "k"},
|
||||
"explicitly revoked",
|
||||
),
|
||||
(
|
||||
{"credential_type": "logging", "description": "weave_otel", "access": {"teams": ["team-a"]}},
|
||||
{},
|
||||
"granted but unbuildable",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_inert_destination_does_not_degrade_the_backend(monkeypatch, credential_info, credential_values, why):
|
||||
# Regression: relaxing the preset's missing-credentials check changes how the backend
|
||||
# is built for EVERY request on the proxy. A destination that routes to nobody -- no
|
||||
# access, empty access, revoked access, or values that build no destination -- must
|
||||
# not trigger it. Otherwise registering an inert row degrades the backend proxy-wide
|
||||
# and silently drops the exports of teams carrying their own callback_vars for it.
|
||||
from types import SimpleNamespace
|
||||
|
||||
from litellm.integrations.otel.model.config import is_otel_v2_enabled
|
||||
from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2
|
||||
|
||||
for var in ("WANDB_API_KEY", "WANDB_PROJECT_ID"):
|
||||
monkeypatch.delenv(var, raising=False)
|
||||
monkeypatch.setenv("LITELLM_OTEL_V2", "true")
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
|
||||
monkeypatch.setattr(
|
||||
litellm,
|
||||
"credential_list",
|
||||
[
|
||||
SimpleNamespace(
|
||||
credential_name="inert",
|
||||
credential_values=credential_values,
|
||||
credential_info=credential_info,
|
||||
)
|
||||
],
|
||||
)
|
||||
result = _maybe_construct_otel_v2("weave_otel", [])
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
assert result is None, f"{why}: an inert destination must not relax the credential check"
|
||||
|
||||
|
||||
def test_generic_admin_destination_needs_flag_to_build_otel_v2_logger(monkeypatch):
|
||||
# The Generic OTLP passthrough ('generic') builds an OpenTelemetryV2 logger only
|
||||
# when LITELLM_OTEL_V2 is on. Registering an admin-owned generic destination with
|
||||
# the flag off must NOT construct a v2 logger (the flag is the sole activation
|
||||
# gate); with the flag on it does.
|
||||
from types import SimpleNamespace
|
||||
|
||||
from litellm.integrations.otel.logger import OpenTelemetryV2
|
||||
from litellm.integrations.otel.model.config import is_otel_v2_enabled
|
||||
from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2
|
||||
|
||||
generic_dest = [
|
||||
SimpleNamespace(
|
||||
credential_name="ui-generic",
|
||||
credential_info={"credential_type": "logging", "description": "generic"},
|
||||
)
|
||||
]
|
||||
|
||||
# Flag off + destination registered -> still None (no v2, no flip onto v2).
|
||||
monkeypatch.delenv("LITELLM_OTEL_V2", raising=False)
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
monkeypatch.setattr(litellm, "credential_list", generic_dest)
|
||||
assert _maybe_construct_otel_v2("generic", []) is None
|
||||
|
||||
# Flag on + destination registered -> builds the v2 generic logger.
|
||||
monkeypatch.setenv("LITELLM_OTEL_V2", "true")
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
logger = _maybe_construct_otel_v2("generic", [])
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
assert isinstance(logger, OpenTelemetryV2)
|
||||
assert logger.callback_name == "generic"
|
||||
def test_set_cost_breakdown_stores_reasoning_cost():
|
||||
"""reasoning_cost is stored only when positive, mirroring the cache-cost fields."""
|
||||
from datetime import datetime
|
||||
|
|
@ -4539,3 +4708,280 @@ async def test_restore_correlation_context_works_across_asyncio_task_boundary():
|
|||
finally:
|
||||
trace_id_var.set("")
|
||||
session_id_var.set("")
|
||||
|
||||
|
||||
def test_zero_config_v2_warns_instead_of_going_silently_dark(monkeypatch, caplog):
|
||||
"""Regression: v2 stopped folding a console exporter into the nothing-configured case,
|
||||
which is right (it printed every span, prompt content included, on the request path)
|
||||
but left an operator with no exporter and no signal. Base printed to stdout; head must
|
||||
at least say so, or the deployment is silently dark.
|
||||
"""
|
||||
import logging
|
||||
|
||||
from litellm.integrations.otel.model.config import is_otel_v2_enabled
|
||||
from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2
|
||||
|
||||
# every alias the config reads, not just the short names -- the OTEL_EXPORTER_OTLP_*
|
||||
# spellings are equally load-bearing and leak in from neighbouring suites
|
||||
for var in (
|
||||
"OTEL_EXPORTER",
|
||||
"OTEL_EXPORTER_OTLP_PROTOCOL",
|
||||
"OTEL_ENDPOINT",
|
||||
"OTEL_EXPORTER_OTLP_ENDPOINT",
|
||||
"OTEL_HEADERS",
|
||||
"OTEL_EXPORTER_OTLP_HEADERS",
|
||||
):
|
||||
monkeypatch.delenv(var, raising=False)
|
||||
monkeypatch.setenv("LITELLM_OTEL_V2", "true")
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
monkeypatch.setattr(litellm, "credential_list", [])
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger="LiteLLM"):
|
||||
logger = _maybe_construct_otel_v2("generic", [])
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
|
||||
assert logger is not None
|
||||
assert logger.config.exporters == []
|
||||
assert any("no exporter is configured" in record.getMessage() for record in caplog.records)
|
||||
|
||||
|
||||
def test_explicit_otel_callback_keeps_the_documented_console_default(monkeypatch):
|
||||
"""Regression: ``callbacks: ["otel"]`` with no endpoint must still export.
|
||||
|
||||
The console fold is suppressed for a preset that degraded because it found no
|
||||
credentials, which is the right call; the operator never asked for stdout there. An
|
||||
operator who lists ``otel`` and sets no endpoint did ask for it, and the published docs
|
||||
give ``console`` as the ``OTEL_EXPORTER`` default. Suppressing both left that
|
||||
deployment silently dark with no warning.
|
||||
"""
|
||||
from litellm.integrations.otel.model.config import is_otel_v2_enabled
|
||||
from litellm.integrations.otel.presets.generic import generic_preset
|
||||
from litellm.litellm_core_utils.litellm_logging import _init_custom_logger_compatible_class
|
||||
|
||||
for var in ("OTEL_EXPORTER", "OTEL_ENDPOINT", "OTEL_EXPORTER_OTLP_ENDPOINT", "OTEL_EXPORTER_OTLP_PROTOCOL"):
|
||||
monkeypatch.delenv(var, raising=False)
|
||||
monkeypatch.setenv("LITELLM_OTEL_V2", "true")
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
|
||||
logger = _init_custom_logger_compatible_class("otel", internal_usage_cache=None, llm_router=None)
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
|
||||
assert logger is not None
|
||||
kinds = [spec.kind for spec in logger.config.exporters]
|
||||
assert kinds == ["console"], "an explicit otel callback with no endpoint must keep the documented console default"
|
||||
assert generic_preset(allow_missing_credentials=True).exporters == [], "preset degrade must stay suppressed"
|
||||
|
||||
|
||||
def test_destination_for_one_team_does_not_move_another_tenant_off_its_logger(monkeypatch):
|
||||
"""Regression: an admin destination must not change who owns a backend.
|
||||
|
||||
``_has_admin_owned_logging_destination`` answered "does a granting row exist anywhere"
|
||||
and fed that into the preset's missing-credentials check, which decides whether v2 or
|
||||
the legacy logger owns the backend for the whole process. Registering a destination
|
||||
scoped to one team therefore moved every other tenant onto v2, and their traces lost
|
||||
prompt content because v2 defaults to NO_CONTENT.
|
||||
"""
|
||||
from types import SimpleNamespace
|
||||
|
||||
from litellm.integrations.langfuse.langfuse_otel import LangfuseOtelLogger
|
||||
from litellm.integrations.otel.model.config import is_otel_v2_enabled
|
||||
from litellm.litellm_core_utils.litellm_logging import _init_custom_logger_compatible_class
|
||||
|
||||
for var in ("LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY", "LANGFUSE_HOST"):
|
||||
monkeypatch.delenv(var, raising=False)
|
||||
monkeypatch.setenv("LITELLM_OTEL_V2", "true")
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
|
||||
granted_to_someone_else = SimpleNamespace(
|
||||
credential_name="lf-team-a",
|
||||
credential_values={"langfuse_public_key": "pk", "langfuse_secret_key": "sk"},
|
||||
credential_info={
|
||||
"credential_type": "logging",
|
||||
"description": "langfuse_otel",
|
||||
"access": {"teams": ["team-a"]},
|
||||
},
|
||||
)
|
||||
|
||||
monkeypatch.setattr(litellm, "credential_list", [])
|
||||
baseline = _init_custom_logger_compatible_class("langfuse_otel", internal_usage_cache=None, llm_router=None)
|
||||
|
||||
monkeypatch.setattr(litellm, "credential_list", [granted_to_someone_else])
|
||||
with_destination = _init_custom_logger_compatible_class("langfuse_otel", internal_usage_cache=None, llm_router=None)
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
|
||||
assert isinstance(baseline, LangfuseOtelLogger)
|
||||
assert type(with_destination) is type(baseline), "another team's destination must not change this backend's owner"
|
||||
|
||||
|
||||
def test_admin_destination_does_not_build_a_backend_logger(monkeypatch):
|
||||
"""A registered destination must not take a backend over.
|
||||
|
||||
Ownership is the operator's configuration alone. A destination is delivered to by
|
||||
``AdminDestinationLogger``, whose per-backend emitter carries the backend's span
|
||||
vocabulary and no exporter of its own: the preset's own exporter belongs to whichever
|
||||
logger the operator configured, so including it here would export the call twice.
|
||||
"""
|
||||
from types import SimpleNamespace
|
||||
|
||||
from litellm.integrations.otel.destination_logger import AdminDestinationLogger
|
||||
from litellm.integrations.otel.model.config import is_otel_v2_enabled
|
||||
from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2
|
||||
|
||||
for var in ("WANDB_API_KEY", "WANDB_PROJECT_ID"):
|
||||
monkeypatch.delenv(var, raising=False)
|
||||
monkeypatch.setenv("LITELLM_OTEL_V2", "true")
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
monkeypatch.setattr(
|
||||
litellm,
|
||||
"credential_list",
|
||||
[
|
||||
SimpleNamespace(
|
||||
credential_name="wb",
|
||||
credential_values={"wandb_api_key": "k"},
|
||||
credential_info={
|
||||
"credential_type": "logging",
|
||||
"description": "weave_otel",
|
||||
"access": {"teams": ["team-a"]},
|
||||
},
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
assert _maybe_construct_otel_v2("weave_otel", []) is None
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
|
||||
monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-global")
|
||||
monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-global")
|
||||
emitter = AdminDestinationLogger()._emitter_for("langfuse_otel")
|
||||
assert emitter.callback_name == "langfuse_otel"
|
||||
assert list(emitter.config.exporters) == [], "the sink must not inherit the preset's own exporter"
|
||||
|
||||
|
||||
def test_sink_skips_a_backend_an_otel_v2_logger_already_owns(monkeypatch):
|
||||
"""Regression: a destination whose backend is also a configured callback received the
|
||||
gen-AI span twice, as two sibling spans under one parent.
|
||||
|
||||
The owning ``OpenTelemetryV2`` already fans its span out to that backend's
|
||||
destinations, so the sink emitting as well is a pure duplicate; ``_closed_call_ids``
|
||||
is per-instance and cannot dedupe across the two emitters.
|
||||
"""
|
||||
from litellm.integrations.otel import destination_logger as sink_module
|
||||
from litellm.integrations.otel.destination_logger import AdminDestinationLogger
|
||||
from litellm.integrations.otel.logger import OpenTelemetryV2
|
||||
from litellm.integrations.otel.model.config import OpenTelemetryV2Config
|
||||
from litellm.litellm_core_utils import litellm_logging as logging_module
|
||||
|
||||
owned = OpenTelemetryV2(config=OpenTelemetryV2Config(), callback_name="langfuse_otel")
|
||||
monkeypatch.setattr(logging_module, "_in_memory_loggers", [owned])
|
||||
|
||||
class _Dest:
|
||||
def __init__(self, name):
|
||||
self.callback_name = name
|
||||
|
||||
monkeypatch.setattr(
|
||||
sink_module,
|
||||
"request_destinations",
|
||||
lambda: (_Dest("langfuse_otel"), _Dest("generic")),
|
||||
)
|
||||
exported = []
|
||||
sink = AdminDestinationLogger()
|
||||
monkeypatch.setattr(
|
||||
sink,
|
||||
"_emitter_for",
|
||||
lambda backend: type(
|
||||
"_E", (), {"export_to_destinations": lambda _self, *a: exported.append(backend)}
|
||||
)(),
|
||||
)
|
||||
|
||||
sink._export({}, None, None)
|
||||
|
||||
assert exported == ["generic"], "the owned backend is the owning logger's to deliver"
|
||||
|
||||
|
||||
def test_sink_does_not_close_an_mcp_event_as_an_llm_call():
|
||||
"""Regression: the sink called ``_close_llm_call`` directly, bypassing the MCP
|
||||
dispatch. A ``tools/call`` reached the destination named ``execute_tool MCP:
|
||||
<server>-<tool>`` with the tool smuggled into ``gen_ai.request.model``, fabricated
|
||||
zero-token usage, and none of the MCP semconv attributes; the correct span never
|
||||
arrived at all.
|
||||
"""
|
||||
from litellm.integrations.otel.destination_logger import _DestinationOnlyOtel
|
||||
from litellm.integrations.otel.model.config import OpenTelemetryV2Config
|
||||
|
||||
sink = _DestinationOnlyOtel(config=OpenTelemetryV2Config(), callback_name="generic")
|
||||
calls = []
|
||||
sink._emit_mcp_tool_call = lambda *a: calls.append("tool_call") or True # type: ignore[method-assign]
|
||||
sink._close_llm_call = lambda *a: calls.append("llm_call") # type: ignore[method-assign]
|
||||
|
||||
sink.export_to_destinations({}, None, None)
|
||||
|
||||
assert calls == ["tool_call"], "an MCP tool call must never be closed as an LLM call"
|
||||
|
||||
|
||||
def test_sink_claims_list_tools_without_emitting_a_second_span():
|
||||
"""``tools/list`` carries no ``gen_ai.operation.name``, so the fan-out span processor
|
||||
already routes the owning logger's span to the destination. The sink must claim the
|
||||
event (so it is not closed as an LLM call) without emitting its own duplicate."""
|
||||
from litellm.integrations.otel.destination_logger import _DestinationOnlyOtel
|
||||
from litellm.integrations.otel.model.config import OpenTelemetryV2Config
|
||||
|
||||
sink = _DestinationOnlyOtel(config=OpenTelemetryV2Config(), callback_name="generic")
|
||||
emitted = []
|
||||
sink._emitter.emit = lambda *a, **k: emitted.append(a) # type: ignore[method-assign]
|
||||
payload = {"call_type": "list_mcp_tools", "id": "abc"}
|
||||
|
||||
handled = sink._emit_mcp_list_tools({"standard_logging_object": payload}, None, None)
|
||||
|
||||
assert handled is True
|
||||
assert emitted == [], "the fan-out processor already delivers tools/list"
|
||||
|
||||
|
||||
def test_sink_never_layers_a_tenant_credential_tracer_over_the_fan_out():
|
||||
"""The tenant's own backend account is the owning logger's to reach. If the sink
|
||||
passed the request's ``dynamic_params`` through, ``genai_tracers_for`` would add a
|
||||
credential-scoped tracer and export the call to that account a second time."""
|
||||
from litellm.integrations.otel.destination_logger import _DestinationOnlyOtel
|
||||
from litellm.integrations.otel.logger import OpenTelemetryV2
|
||||
from litellm.integrations.otel.model.config import OpenTelemetryV2Config
|
||||
from litellm.integrations.otel.model.metadata import LLMCallEvent
|
||||
|
||||
call = LLMCallEvent.from_dict({"standard_callback_dynamic_params": {"langfuse_public_key": "pk"}})
|
||||
owning = OpenTelemetryV2(config=OpenTelemetryV2Config(), callback_name="langfuse_otel")
|
||||
sink = _DestinationOnlyOtel(config=OpenTelemetryV2Config(), callback_name="langfuse_otel")
|
||||
|
||||
assert owning._tracer_dynamic_params(call) is call.dynamic_params
|
||||
assert sink._tracer_dynamic_params(call) is None
|
||||
|
||||
|
||||
def test_importing_litellm_does_not_require_opentelemetry():
|
||||
"""``opentelemetry`` ships only in the proxy extras, so a plain SDK install does not
|
||||
have it. Importing the OTEL v2 logger at module scope made ``import litellm`` raise
|
||||
ModuleNotFoundError for every one of those users; CI never sees it because the install
|
||||
jobs sync all extras. Run in a child process with the package blocked at import time."""
|
||||
import subprocess
|
||||
import textwrap
|
||||
|
||||
program = textwrap.dedent(
|
||||
"""
|
||||
import sys
|
||||
|
||||
class _Blocker:
|
||||
def find_spec(self, name, path=None, target=None):
|
||||
if name == "opentelemetry" or name.startswith("opentelemetry."):
|
||||
raise ModuleNotFoundError(f"No module named '{name}'")
|
||||
return None
|
||||
|
||||
sys.meta_path.insert(0, _Blocker())
|
||||
import litellm # noqa: F401
|
||||
print("IMPORT_OK")
|
||||
"""
|
||||
)
|
||||
repo_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", ".."))
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-c", program],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=300,
|
||||
env={**os.environ, "PYTHONPATH": repo_root},
|
||||
)
|
||||
assert "IMPORT_OK" in result.stdout, f"import litellm failed without opentelemetry:\n{result.stderr[-3000:]}"
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import inspect
|
||||
import asyncio
|
||||
import copy
|
||||
import json
|
||||
|
|
@ -6820,6 +6821,28 @@ async def test_resolve_logging_exporters_noop_when_flag_off(monkeypatch):
|
|||
assert backends == ()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mcp_auth_chokepoint_refreshes_destinations_per_message(monkeypatch):
|
||||
"""Regression: a stateful MCP session outlived an admin's re-scope of a destination.
|
||||
|
||||
Every JSON-RPC message after ``initialize`` is dispatched on a descendant of the task
|
||||
that POST spawned, and a ContextVar is copied at task creation, so the destinations
|
||||
resolved for the first message stayed frozen for the session's life. A destination
|
||||
re-scoped to another team kept receiving the original team's spans until the client
|
||||
disconnected. The shared auth chokepoint now re-resolves per message.
|
||||
"""
|
||||
import litellm.proxy._experimental.mcp_server.server as mcp_server
|
||||
|
||||
src = inspect.getsource(mcp_server)
|
||||
assert "_refresh_request_otel_destinations" in src
|
||||
# It must hang off the one helper every JSON-RPC handler funnels through, not off a
|
||||
# single handler; tools/list, get_prompt and read_resource all leaked precisely
|
||||
# because only the tool-call path re-resolved.
|
||||
chokepoint = src[src.index("async def get_or_extract_auth_context"):]
|
||||
chokepoint = chokepoint[: chokepoint.index("def get_active_mcp_session")]
|
||||
assert "await _refresh_request_otel_destinations(user_api_key_auth)" in chokepoint
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_apply_admin_logging_exporters_needs_no_opentelemetry_when_flag_off(monkeypatch):
|
||||
"""opentelemetry ships only in the proxy-runtime extra, so a litellm[proxy] install
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue