From 09976043ac990aa4e3f74569b568cb461b648035 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Tue, 8 Sep 2026 16:29:16 -0700 Subject: [PATCH] feat(otel): backport tenant trace destinations to rc/1.101.0 Backport #39654 without conflict resolution or implementation changes (cherry picked from commit 3165951e6d0b795290ddb18453984a14bff9bf5c) --- litellm/__init__.py | 3 + litellm/integrations/otel/logger.py | 55 +- litellm/integrations/otel/mappers/legacy.py | 3 +- litellm/integrations/otel/model/config.py | 6 +- .../integrations/otel/model/destination.py | 49 + litellm/integrations/otel/model/semconv.py | 3 + litellm/integrations/otel/plumbing/context.py | 68 +- .../integrations/otel/plumbing/providers.py | 690 ++++- litellm/integrations/otel/plumbing/routing.py | 16 +- litellm/integrations/otel/presets/agentops.py | 1 + litellm/integrations/otel/presets/arize.py | 6 +- litellm/integrations/otel/presets/base.py | 14 +- .../integrations/otel/presets/destinations.py | 152 + litellm/integrations/otel/presets/langfuse.py | 23 +- .../integrations/otel/presets/langtrace.py | 1 + litellm/integrations/otel/presets/levo.py | 1 + litellm/integrations/otel/presets/newrelic.py | 1 + litellm/integrations/otel/presets/phoenix.py | 1 + litellm/integrations/otel/presets/utils.py | 31 + litellm/integrations/otel/presets/weave.py | 21 +- litellm/integrations/weave/weave_otel.py | 20 +- litellm/litellm_core_utils/litellm_logging.py | 57 +- .../proxy/_experimental/mcp_server/server.py | 66 +- litellm/proxy/auth/user_api_key_auth.py | 38 + litellm/proxy/litellm_pre_call_utils.py | 140 + .../otel/test_otel_v2_destinations.py | 2719 +++++++++++++++++ .../integrations/otel/test_otel_v2_logger.py | 4 +- .../test_litellm_logging.py | 9 +- .../mcp_server/test_mcp_server.py | 69 + 29 files changed, 4214 insertions(+), 53 deletions(-) create mode 100644 litellm/integrations/otel/model/destination.py create mode 100644 litellm/integrations/otel/presets/destinations.py create mode 100644 tests/test_litellm/integrations/otel/test_otel_v2_destinations.py diff --git a/litellm/__init__.py b/litellm/__init__.py index 42c0ea881fd..14327d54897 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -325,6 +325,9 @@ ssl_certificate: Optional[str] = None user_url_validation: bool = True user_url_allowed_hosts: List[str] = [] provider_url_destination_allowed_hosts: List[str] = [] +#: "override" (default) or "additive": whether a key or team destination replaces +#: the operator's exporter for that backend or exports alongside it. +otel_tenant_destination_mode: str | None = None ssl_ecdh_curve: Optional[str] = None # Set to 'X25519' to disable PQC and improve performance disable_streaming_logging: bool = False disable_token_counter: bool = False diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index 5519896a961..630aa313dc9 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -16,9 +16,11 @@ from opentelemetry.trace import ( Span, Tracer, get_current_span, + get_tracer_provider, set_span_in_context, use_span, ) +from opentelemetry.trace import TracerProvider as ApiTracerProvider import litellm from litellm._logging import verbose_logger @@ -63,6 +65,7 @@ from litellm.integrations.otel.plumbing.metrics import ( create_genai_metrics, ) from litellm.integrations.otel.plumbing.providers import ( + attach_tenant_fan_out, build_tracer_provider, get_event_logger, get_meter, @@ -85,6 +88,7 @@ if TYPE_CHECKING: ) LITELLM_TRACER_NAME: Final = "litellm" +_published_v2_provider: ApiTracerProvider | None = None def _span_error_from_exception( @@ -180,7 +184,9 @@ class OpenTelemetryV2(CustomLogger): self.config: OpenTelemetryV2Config = config or OpenTelemetryV2Config(**kwargs) self.callback_name = callback_name self._tracer_provider: TracerProvider = ( - tracer_provider if tracer_provider is not None else build_tracer_provider(self.config) + tracer_provider + if tracer_provider is not None + else build_tracer_provider(self.config, tenant_overrides=True) ) self.tracer: Tracer = get_tracer(self._tracer_provider, LITELLM_TRACER_NAME) self._metrics_recorder = self._init_metrics(meter_provider) @@ -195,6 +201,11 @@ class OpenTelemetryV2(CustomLogger): self._open_llm_calls: OrderedDict[str, _LLMCallSpan] = OrderedDict() self._init_otel_logger_on_litellm_proxy() + @property + def tracer_provider(self) -> TracerProvider: + """The provider this logger emits through, read-only to its callers.""" + return self._tracer_provider + def _init_metrics(self, meter_provider: "MeterProvider | None") -> "GenAIMetricRecorder | None": """Create the six GenAI histograms when metrics are enabled, else ``None``. @@ -863,12 +874,33 @@ def publish_global_otel_v2_provider( ``opentelemetry.trace.set_tracer_provider``) are injected so the publish step is unit-testable without reading or mutating real global OTel state. Returns the logger whose provider was published. + + The published provider is also the one that fans spans out to key/team + destinations, because it is the only provider the whole request tree passes + through; see :func:`attach_tenant_fan_out`. It is remembered for + :func:`fan_out_provider` because neither the OTel global (``set_tracer_provider`` + keeps the first provider it was ever handed) nor + ``proxy_server.open_telemetry_logger`` (a legacy v1 logger can hold that slot) + reliably leads back to it. """ + global _published_v2_provider logger: Final = select_global_otel_v2_logger(in_memory_loggers, registered=registered) - set_global_provider(logger._tracer_provider) + attach_tenant_fan_out(logger.tracer_provider, *_v2_configs(in_memory_loggers, logger)) + set_global_provider(logger.tracer_provider) + _published_v2_provider = logger.tracer_provider # rebind-ok: startup records the one provider carrying the fan-out return logger +def _v2_configs(in_memory_loggers: Sequence[object], logger: "OpenTelemetryV2") -> tuple[OpenTelemetryV2Config, ...]: + """Every v2 logger's config, the published logger's first. + + Each preset keeps its own provider and exporters, so the accounts the operator + writes to are spread over all of them, not held by the published logger alone. + """ + others: Final = tuple(cb.config for cb in in_memory_loggers if isinstance(cb, OpenTelemetryV2) and cb is not logger) + return (logger.config, *others) + + def _registered_v2_logger() -> "OpenTelemetryV2 | None": try: from litellm.proxy import proxy_server @@ -904,6 +936,25 @@ def seed_request_identity(user_api_key_dict: object, model: str | None = None) - logger.seed_request_identity(user_api_key_dict, model=model) +def fan_out_provider() -> ApiTracerProvider: + """The provider :func:`publish_global_otel_v2_provider` gave the tenant fan-out. + + Read off the publish itself, not the OTel global and not the registered logger: + the global keeps whichever provider claimed it first (auto-instrumentation, a + legacy logger), and the registered slot can hold a v1 logger while the publish + picked a v2 one from ``_in_memory_loggers``. Either detour lands on a provider + with no fan-out and drops every destination at auth. + """ + published: Final = _published_v2_provider + if published is not None: + return published + logger: Final = _registered_v2_logger() + if logger is not None: + attach_tenant_fan_out(logger.tracer_provider, logger.config) + return logger.tracer_provider + return get_tracer_provider() + + @contextmanager def phase_span(name: str) -> "Iterator[Span | None]": logger: Final = _registered_v2_logger() diff --git a/litellm/integrations/otel/mappers/legacy.py b/litellm/integrations/otel/mappers/legacy.py index 37475acb8f7..d25c25cd127 100644 --- a/litellm/integrations/otel/mappers/legacy.py +++ b/litellm/integrations/otel/mappers/legacy.py @@ -23,6 +23,7 @@ from litellm.integrations.otel.model.payloads import ( ServiceSpanData, ToolDefinition, ) +from litellm.integrations.otel.model.semconv import Error # Attribute keys in the semconv-ai / Traceloop vocabulary. _LEGACY_SYSTEM: Final = "gen_ai.system" @@ -36,7 +37,7 @@ _LEGACY_PRESENCE_PENALTY: Final = "llm.presence_penalty" _LEGACY_STOP_SEQUENCES: Final = "llm.chat.stop_sequences" _LEGACY_SERVICE: Final = "service" _LEGACY_CALL_TYPE: Final = "call_type" -_LEGACY_ERROR: Final = "error" +_LEGACY_ERROR: Final = Error.MESSAGE_LEGACY class LegacyMapper: diff --git a/litellm/integrations/otel/model/config.py b/litellm/integrations/otel/model/config.py index 9e3064c2bff..f0f5befe430 100644 --- a/litellm/integrations/otel/model/config.py +++ b/litellm/integrations/otel/model/config.py @@ -253,7 +253,9 @@ class OpenTelemetryV2Config(BaseSettings): if self.endpoint and self.exporter == "console": self.exporter = "otlp_http" # When no explicit destinations are given, fold the single-destination - # shorthand into one spec so the provider always has a destination. + # shorthand into one spec so the provider always has a destination. A spec + # with no fields set is how the presets tell "nothing configured" from an + # operator who asked for the console by name. if not self.exporters: self.exporters = [ ExporterSpec( @@ -261,6 +263,8 @@ class OpenTelemetryV2Config(BaseSettings): endpoint=self.endpoint, headers=self.headers, ) + if not self.model_fields_set.isdisjoint(("exporter", "endpoint", "headers")) + else ExporterSpec() ] # Ensure ``genai`` is always present and first. names = list(self.mapper_names) diff --git a/litellm/integrations/otel/model/destination.py b/litellm/integrations/otel/model/destination.py new file mode 100644 index 00000000000..299253cac77 --- /dev/null +++ b/litellm/integrations/otel/model/destination.py @@ -0,0 +1,49 @@ +"""The resolved OTLP destination a request's traces export to. + +Backend-agnostic on purpose: every OTEL backend reduces to an endpoint plus auth +headers. The per-backend field mapping lives in ``presets.destinations``. +""" + +from collections.abc import Mapping +from typing import Final +from urllib.parse import quote + +from pydantic import BaseModel, ConfigDict, Field + + +class OtelDestination(BaseModel): + model_config = ConfigDict(frozen=True) + + endpoint: str + headers: Mapping[str, str] = Field(default_factory=dict) + resource_attributes: Mapping[str, str] = Field(default_factory=dict) + callback_name: str | None = None + protocol: str | None = Field( + default=None, + description=( + "OTLP transport, defaulting to the backend's own. Not derivable from the " + "scheme: Arize's ``https://otlp.arize.com/v1`` is gRPC." + ), + ) + + def header_string(self) -> str: + """Render headers as the ``k=v,k2=v2`` form an ``ExporterSpec`` expects. + + Values are percent-encoded because ``providers.parse_headers`` decodes them + with the SDK's W3C-Baggage parser: a value carrying a ``,`` or ``=`` (a + Langfuse project name, a base64 Authorization payload ending in ``==``) + would otherwise be split into bogus pairs on the way back out. + """ + return ",".join(f"{key}={quote(value, safe='')}" for key, value in self.headers.items()) + + def cache_key(self) -> tuple[str, tuple[tuple[str, str], ...], tuple[tuple[str, str], ...], str | None]: + """Identity for processor reuse, so one destination means one exporter.""" + return ( + self.endpoint, + tuple(sorted(self.headers.items())), + tuple(sorted(self.resource_attributes.items())), + self.protocol, + ) + + +NO_DESTINATIONS: Final[tuple[OtelDestination, ...]] = () diff --git a/litellm/integrations/otel/model/semconv.py b/litellm/integrations/otel/model/semconv.py index af5327cbd41..d3628005bac 100644 --- a/litellm/integrations/otel/model/semconv.py +++ b/litellm/integrations/otel/model/semconv.py @@ -204,6 +204,9 @@ class Error: TYPE: Final = "error.type" MESSAGE: Final = "error.message" + # The same text under the bare key the semconv-ai / Traceloop vocabulary uses + # (see ``LegacyMapper``), so anything reading or redacting error text covers both. + MESSAGE_LEGACY: Final = "error" class LiteLLMError: diff --git a/litellm/integrations/otel/plumbing/context.py b/litellm/integrations/otel/plumbing/context.py index aa7cc8e2afd..21e61c71fb7 100644 --- a/litellm/integrations/otel/plumbing/context.py +++ b/litellm/integrations/otel/plumbing/context.py @@ -1,8 +1,9 @@ """Trace-context + Baggage helpers.""" +import os from collections.abc import Mapping from contextvars import ContextVar, Token -from typing import Final +from typing import TYPE_CHECKING, Final from opentelemetry import baggage from opentelemetry.context import Context, get_current @@ -21,6 +22,9 @@ from opentelemetry.trace.propagation.tracecontext import ( from litellm.integrations.otel.model.semconv import HTTP +if TYPE_CHECKING: + from litellm.integrations.otel.model.destination import OtelDestination + _PROPAGATOR: Final = TraceContextTextMapPropagator() # The request's root span — the FastAPI-owned SERVER span — captured ONCE when the @@ -304,3 +308,65 @@ def extract_traceparent(headers: Mapping[str, str]) -> Context | None: return None carrier: Final = {str(key).lower(): value for key, value in headers.items()} return _PROPAGATOR.extract(carrier) + + +# The OTLP destinations this request's key or team pointed its traces at, resolved +# once during auth. A ``ContextVar`` for the same reason the root span above is one: +# it rides the request task's context into the ``asyncio.create_task`` children that +# close the LLM span, and it is visible to every ``SpanProcessor.on_end`` that fires +# on the request task. Stateful MCP handlers set and reset it per message; the +# request-task value otherwise dies with that task. +_request_destinations: Final['ContextVar[tuple["OtelDestination", ...]]'] = ContextVar( + "litellm_otel_request_destinations", default=() +) + + +def set_request_destinations(destinations: 'tuple["OtelDestination", ...]') -> "Token[tuple[OtelDestination, ...]]": + """Anchor the destinations this request exports to and return a reset token.""" + return _request_destinations.set(destinations) + + +def reset_request_destinations(token: "Token[tuple[OtelDestination, ...]]") -> None: + _request_destinations.reset(token) + + +def request_destinations() -> 'tuple["OtelDestination", ...]': + """The destinations resolved for this request, empty outside a proxy request.""" + return _request_destinations.get() + + +#: ``litellm_settings: otel_tenant_destination_mode`` and its env equivalent. +ADDITIVE_DESTINATION_MODE: Final = "additive" +OTEL_TENANT_DESTINATION_MODE_ENV: Final = "LITELLM_OTEL_TENANT_DESTINATION_MODE" + + +def tenant_destinations_are_additive() -> bool: + """Whether a tenant destination exports alongside the operator's own exporter. + + Override is the default: the tenant's traffic reaches the tenant's account and + nowhere else. Operators running one org-wide backend across every team set this + to ``additive`` so the same trace lands in both places. + """ + import litellm + + configured: Final = litellm.otel_tenant_destination_mode or os.environ.get(OTEL_TENANT_DESTINATION_MODE_ENV) + return isinstance(configured, str) and configured.strip().lower() == ADDITIVE_DESTINATION_MODE + + +def destination_backends() -> frozenset[str]: + """Backends this request resolved a tenant destination for. + + The fan-out already carries the whole trace to those destinations, so the + per-request tracer route must never send a second copy, in either mode. + """ + return frozenset(d.callback_name for d in _request_destinations.get() if d.callback_name) + + +def suppressed_backends() -> frozenset[str]: + """Backends whose operator-level exporters this request must NOT reach. + + Empty under ``additive``, where the operator keeps its copy of every span. + """ + if tenant_destinations_are_additive(): + return frozenset() + return destination_backends() diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index fb74ff85e5b..afd502962cf 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -1,9 +1,14 @@ """Provider / exporter factory + the Baggage span processor.""" -from collections.abc import Callable, Iterable +import queue +import threading +import time +from collections import OrderedDict +from collections.abc import Callable, Iterable, Mapping, Sequence +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal -from opentelemetry import _logs, baggage, metrics +from opentelemetry import _logs, baggage, metrics, trace from opentelemetry._events import EventLogger from opentelemetry._logs import LoggerProvider, NoOpLoggerProvider from opentelemetry.context import Context @@ -19,7 +24,8 @@ from opentelemetry.sdk._logs.export import ( ) from opentelemetry.sdk.metrics import MeterProvider as SDKMeterProvider from opentelemetry.sdk.resources import Resource -from opentelemetry.sdk.trace import ReadableSpan, SpanProcessor, TracerProvider +from opentelemetry.sdk.trace import Event, ReadableSpan, SpanProcessor, TracerProvider +from opentelemetry.sdk.trace import Span as SDKSpan from opentelemetry.sdk.trace.export import ( BatchSpanProcessor, ConsoleSpanExporter, @@ -29,18 +35,35 @@ from opentelemetry.sdk.trace.export import ( from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( InMemorySpanExporter, ) -from opentelemetry.trace import Span, SpanKind, Tracer +from opentelemetry.trace import Span, SpanKind, Status, Tracer from opentelemetry.util.re import parse_env_headers +from opentelemetry.util.types import Attributes, AttributeValue +from litellm._logging import verbose_logger from litellm._version import version as litellm_version from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config -from litellm.integrations.otel.model.semconv import LiteLLM +from litellm.integrations.otel.model.semconv import ( + DB, + MCP, + Error, + ExceptionEvent, + GenAI, + LiteLLM, + LiteLLMError, + Server, +) from litellm.integrations.otel.model.spans import LiteLLMSpanKind +from litellm.integrations.otel.plumbing.context import ( + request_destinations, + suppressed_backends, +) if TYPE_CHECKING: from opentelemetry.metrics import Meter from opentelemetry.sdk.metrics.export import MetricReader + from litellm.integrations.otel.model.destination import OtelDestination + _SPAN_KIND_BY_ROLE_KIND: Final[dict[LiteLLMSpanKind, SpanKind]] = { LiteLLMSpanKind.SERVER: SpanKind.SERVER, LiteLLMSpanKind.CLIENT: SpanKind.CLIENT, @@ -194,6 +217,555 @@ def _processor_for(exporter: SpanExporter, use_simple: bool | None) -> SpanProce return SimpleSpanProcessor(exporter) if use_simple else BatchSpanProcessor(exporter) +#: Distinct tenant destinations whose exporters stay alive. Each holds a connection +#: pool and a batch thread, so the cache is bounded and evicts least-recently-used. +_MAX_CACHED_DESTINATION_PROCESSORS: Final = 32 + +#: Workers closing shed destination processors, bounding the threads a tenant can +#: create by cycling its destination config. +_DRAIN_WORKERS: Final = 2 + +#: Shed processors waiting to be closed before the fan-out stops building new ones. +#: Each still owns a batch thread until its close returns, and a collector that never +#: answers makes every close take the exporter's full timeout, so past this many the +#: operator's exporter keeps the span instead (see ``deliverable``). +_MAX_PENDING_DRAINS: Final = 64 + +#: How long ``shutdown`` waits for spans already being forwarded, so teardown closes +#: no processor under one. Bounded: an exporter that never returns must not hold the +#: proxy open. +_SHUTDOWN_DRAIN_SECONDS: Final = 5.0 + +#: An exporter's account: its normalized endpoint and the credentials it presents. +_SinkKey = tuple[str, tuple[tuple[str, str], ...]] + +#: Header names that spell one credential two ways. Arize's operator exporter sends +#: ``space_id`` where a tenant destination sends ``arize-space-id``. +_CREDENTIAL_ALIASES: Final = MappingProxyType({"arize_space_id": "space_id"}) + + +class _DrainPool: + """Closes shed destination processors off the span-export path. + + ``shutdown`` flushes over the network and is reached from ``on_end``, so closing + one inline would let a single unreachable tenant collector stall every other + tenant's spans behind it. A fixed set of workers rather than a thread per + processor means a tenant cycling its destination config cannot spawn threads as + fast as it can send requests; slow shutdowns queue behind each other. + + The workers are daemons and belong to the fan-out that sheds the processors, so + neither an unreachable collector nor a lazily built process-wide singleton can + hold the proxy open on the way down. + """ + + def __init__( + self, + workers: int = _DRAIN_WORKERS, + pending: "queue.Queue[SpanProcessor | None] | None" = None, + capacity: int = _MAX_PENDING_DRAINS, + ) -> None: + self._workers: Final = workers + self._capacity: Final = capacity + self._lock: Final = threading.Lock() + self._closed = False + self._backlog = 0 # guarded by ``_lock``: submitted processors whose close has not returned + self._pending: Final[queue.Queue[SpanProcessor | None]] = pending if pending is not None else queue.Queue() + self._threads: Final = tuple( + threading.Thread(target=self._drain_until_closed, daemon=True, name="litellm-otel-destination-drain") + for _ in range(workers) + ) + for worker in self._threads: + worker.start() + + def submit(self, processor: SpanProcessor) -> None: + """Queue ``processor`` for closing, or hand it off once the pool is retired. + + The check and the put share one lock. Reading a closed flag on its own leaves + room for :meth:`close` to run in between, and the processor would land behind + the sentinels every worker has already exited on. + + Past close there is no worker left to take it, and the caller is whichever + thread just ended a span, so closing it inline would park that thread on a + network flush the shutdown deadline has already stopped waiting for. The extra + thread is bounded by the same close: the fan-out stops handing processors out + at that point, so only the ones already exporting when it happened arrive here. + """ + with self._lock: + if not self._closed: + self._backlog += 1 + self._pending.put(processor) + return + threading.Thread( + target=_shutdown_quietly, + args=(processor,), + daemon=True, + name="litellm-otel-destination-drain-straggler", + ).start() + + def saturated(self) -> bool: + """Whether enough closes are outstanding that building another processor must wait. + + The workers close in order and each close blocks for as long as its exporter + does, so a collector that stopped answering would otherwise turn every new + destination into one more batch thread parked behind them, for as long as the + tenants keep rotating. Holding the count here rather than reading the queue + keeps the two processors a worker is mid-close on in the total. + """ + with self._lock: + return self._backlog >= self._capacity + + def close(self, timeout: float | None = None) -> None: + """Retire the workers once they have closed everything already queued. + + A proxy that rebuilds its telemetry builds another fan-out, so workers that + outlive the one that started them are two more threads per reload, forever. + + ``timeout`` bounds how long the caller waits for that draining to finish. The + workers are daemons, so whatever is still flushing when it expires is dropped + by the interpreter rather than holding it open. + """ + with self._lock: + if self._closed: + return + self._closed = True + for _ in range(self._workers): + self._pending.put(None) + if timeout is None: + return + deadline: Final = time.monotonic() + timeout + for worker in self._threads: + worker.join(timeout=max(0.0, deadline - time.monotonic())) + + def _drain_until_closed(self) -> None: + while True: + processor: SpanProcessor | None = self._pending.get() # rebind-ok: loop variable + if processor is None: + return + _shutdown_quietly(processor) + with self._lock: + self._backlog -= 1 + + +_NO_ATTRIBUTES: Final[Mapping[str, AttributeValue]] = MappingProxyType({}) +_DB_SYSTEM_KEYS: Final = frozenset({DB.SYSTEM_NAME, DB.SYSTEM_LEGACY}) +# Keys on a database span that describe the proxy's own datastore: its host, its +# port, and its schema. +_DATASTORE_ENDPOINT_KEYS: Final = frozenset({Server.ADDRESS, Server.PORT, DB.NAMESPACE}) +# A span carrying one of these describes the tenant's own call (the model call, the +# MCP call, the guardrail), so its error text is theirs to see. Every other span is +# the proxy's own work, whose error text names the operator's infrastructure. +_TENANT_OWNED_KEYS: Final = frozenset({GenAI.OPERATION_NAME, MCP.METHOD_NAME, LiteLLM.GUARDRAIL_NAME}) +_PROXY_ERROR_TEXT_KEYS: Final = frozenset({Error.MESSAGE, Error.MESSAGE_LEGACY}) +# A guardrail that never answered carries the exception it raised as its response, +# which names the operator's guardrail endpoint. The second spelling is the legacy +# status the request-level logger still maps. +_GUARDRAIL_UNREACHABLE_STATUSES: Final = frozenset({"guardrail_failed_to_respond", "failure"}) +# Attribute prefixes the FastAPI instrumentor uses for headers the operator opted to +# capture (``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_*``). The request +# side carries the caller's bearer token verbatim. +_CAPTURED_HEADER_PREFIXES: Final = ("http.request.header.", "http.response.header.") +# The instrumentor stamps the request URL on the server span with its query string, +# under the old convention and the new one, and litellm accepts a virtual key as a +# ``?key=`` query parameter. +_URL_KEYS: Final = frozenset({"http.url", "http.target", "url.full"}) +_URL_QUERY_KEY: Final = "url.query" + + +class _TenantSpanView(ReadableSpan): + """A ``ReadableSpan`` view for one destination, leaving the operator's own span alone.""" + + def __init__( + self, + inner: ReadableSpan, + resource: Resource, + attributes: Attributes, + events: Sequence[Event], + status: Status, + ) -> None: + super().__init__( + name=inner.name, + context=inner.context, + parent=inner.parent, + resource=resource, + attributes=attributes, + events=events, + links=inner.links, + kind=inner.kind, + status=status, + start_time=inner.start_time, + end_time=inner.end_time, + instrumentation_scope=inner.instrumentation_scope, + ) + + +def _is_database_span(attributes: Mapping[str, AttributeValue]) -> bool: + return any(key in attributes for key in _DB_SYSTEM_KEYS) + + +def _is_tenant_owned_span(attributes: Mapping[str, AttributeValue]) -> bool: + return any(key in attributes for key in _TENANT_OWNED_KEYS) + + +def _guardrail_unreachable(attributes: Mapping[str, AttributeValue]) -> bool: + return attributes.get(LiteLLM.GUARDRAIL_STATUS) in _GUARDRAIL_UNREACHABLE_STATUSES + + +def _tenant_visible(key: str, database: bool, owned: bool, unreachable_guardrail: bool) -> bool: + if key.startswith(_CAPTURED_HEADER_PREFIXES) or key in (LiteLLMError.STACK_TRACE, _URL_QUERY_KEY): + return False + if database and key in _DATASTORE_ENDPOINT_KEYS: + return False + if unreachable_guardrail and key == LiteLLM.GUARDRAIL_RESPONSE: + return False + return owned or key not in _PROXY_ERROR_TEXT_KEYS + + +def _without_query(key: str, value: AttributeValue) -> AttributeValue: + if key not in _URL_KEYS or not isinstance(value, str): + return value + return value.partition("?")[0] + + +def _same_attributes(kept: Mapping[str, AttributeValue], attributes: Mapping[str, AttributeValue]) -> bool: + return len(kept) == len(attributes) and all(kept[key] is value for key, value in attributes.items()) + + +def _without_stack_trace(event: Event) -> Event: + attributes: Final = event.attributes or _NO_ATTRIBUTES + if ExceptionEvent.STACKTRACE not in attributes: + return event + return Event( + name=event.name, + attributes=MappingProxyType( + {key: value for key, value in attributes.items() if key != ExceptionEvent.STACKTRACE} + ), + timestamp=event.timestamp, + ) + + +def _for_destination(span: ReadableSpan, destination: "OtelDestination") -> ReadableSpan: + """The view of ``span`` a tenant destination receives. + + A span the tenant's own call produced keeps its error text. Every other span is + the proxy's own work (the request root, auth, the database), and its error text, + its events and its status description come off, since a Prisma failure there + spells out the operator's Postgres endpoint. A database span loses that endpoint + too, and a guardrail that failed to respond loses its response text, which is the + exception it raised and names the operator's guardrail endpoint. Stack traces walk + the operator's install and come off every span, as do the headers the operator + captures on the server span, whose request side holds the caller's bearer token, + and the query string of the request URL, which can hold the same key. The span + itself stays, so the tenant still gets the whole trace tree. + """ + extra: Final = destination.resource_attributes + attributes: Final = span.attributes or _NO_ATTRIBUTES + database: Final = _is_database_span(attributes) + owned: Final = _is_tenant_owned_span(attributes) + unreachable: Final = _guardrail_unreachable(attributes) + kept: Final = MappingProxyType( + { + key: _without_query(key, value) + for key, value in attributes.items() + if _tenant_visible(key, database, owned, unreachable) + } + ) + recorded: Final = span.events + events: Final = tuple(_without_stack_trace(event) for event in recorded) if owned else () + unchanged: Final = owned and _same_attributes(kept, attributes) and all(a is b for a, b in zip(events, recorded)) + if not extra and unchanged: + return span + resource: Final = span.resource.merge(Resource(extra)) if extra else span.resource + status: Final = span.status if owned else Status(span.status.status_code) + return _TenantSpanView(span, resource, kept, events, status) + + +class TenantFanOutSpanProcessor(SpanProcessor): + """Export every finished span to each destination this request resolved. + + Destinations ride a request-scoped ``ContextVar`` set during auth, so concurrent + requests stay isolated. The forwarded view keeps the original trace and parent + ids, so the tenant gets the same tree the operator would have received. + + Exactly one provider carries this processor, the one published as the OTel global + (see :func:`attach_tenant_fan_out`). That provider is the only one every span + passes through: the FastAPI server span, the auth span and the post-call database + spans are emitted on the global, while a second v2 logger's provider sees only + that logger's own gen-AI span. Attaching the fan-out per logger would hand a + tenant a one-span trace whenever its backend is not the global one, and two + copies of the model call whenever it is. + """ + + def __init__( + self, + processor_factory: 'Callable[["OtelDestination"], SpanProcessor | None] | None' = None, + shutdown_drain_seconds: float = _SHUTDOWN_DRAIN_SECONDS, + operator_sinks: frozenset[_SinkKey] = frozenset(), + pending_drains: int = _MAX_PENDING_DRAINS, + drain_pool: _DrainPool | None = None, + ) -> None: + self._operator_sinks: Final = operator_sinks + self._drain_seconds: Final = shutdown_drain_seconds + self._lock: Final = threading.Condition() + self._closed = False # guarded by ``_lock``: an unlocked read races the teardown it gates + self._build: Final = processor_factory if processor_factory is not None else _destination_processor + self._processors: OrderedDict[object, SpanProcessor] = OrderedDict() # mutable-ok: bounded LRU + self._retired: OrderedDict[int, SpanProcessor] = OrderedDict() # mutable-ok: drains as exports finish + self._exporting: dict[int, int] = {} # mutable-ok: per-processor in-flight export count + self._drain: Final = drain_pool if drain_pool is not None else _DrainPool(capacity=pending_drains) + + def on_start(self, span: SDKSpan, parent_context: Context | None = None) -> None: + return None + + def on_end(self, span: ReadableSpan) -> None: + suppressed: Final = suppressed_backends() + for destination in request_destinations(): + if self._operator_already_writes(destination, suppressed): + continue + processor = self._acquire(destination) # rebind-ok: loop variable; pyright forbids Final in a loop + if processor is None: + continue + try: + processor.on_end(_for_destination(span, destination)) + except Exception as exc: # noqa: BLE001 # one destination's failure must not cost the others their span + verbose_logger.debug("OTel V2 fan-out: forwarding to %s failed: %s", destination.endpoint, exc) + finally: + self._release(processor) + + def _operator_already_writes(self, destination: "OtelDestination", suppressed: frozenset[str]) -> bool: + """Whether the operator's own exporter is sending this span to the same account. + + Only reachable under ``additive``, where nothing is suppressed: a team that + names the operator's own project would otherwise have every span written + there twice, once by the operator's exporter and once by the fan-out. + """ + return ( + destination.callback_name not in suppressed + and _sink_key(destination.endpoint, destination.headers) in self._operator_sinks + ) + + def shutdown(self) -> None: + """Close every destination processor, once the spans in flight have landed. + + ``on_end`` runs on whichever thread ends a span and can reach this fan-out + while the SDK is tearing the provider down, so closing blind would drop a + trace mid-forward and would hand the next caller a fresh exporter nothing + will ever close. Refusing new work and then waiting out the in-flight ones + keeps both from happening. A straggler past the bound is retired instead of + closed: the thread still exporting it closes it through the drain as soon as + its export returns, so no span is dropped mid-forward. + + Every close then goes to the drain rather than running here. Closing a + destination processor flushes it over the network and the SDK joins its own + worker with no timeout of its own, so one tenant collector that answers but + never finishes a response would otherwise hold process teardown open for as + long as it likes. The drain's workers are daemons, and the whole teardown + shares one deadline. + """ + deadline: Final = time.monotonic() + self._drain_seconds + with self._lock: + self._closed = True + self._lock.wait_for(lambda: not self._exporting, timeout=self._drain_seconds) + live: Final = tuple((id(p), p) for p in (*self._processors.values(), *self._retired.values())) + closing: Final = tuple(p for ident, p in live if ident not in self._exporting) + self._processors.clear() + self._retired = OrderedDict( # mutable-ok: the same bounded map, keeping only what is still exporting + (ident, p) for ident, p in live if ident in self._exporting + ) + for processor in closing: + self._drain.submit(processor) + self._drain.close(timeout=max(0.0, deadline - time.monotonic())) + + def force_flush(self, timeout_millis: int = 30000) -> bool: + results: Final = tuple(self._flush_one(processor, timeout_millis) for processor in self._snapshot()) + return all(results) + + def _snapshot(self) -> tuple[SpanProcessor, ...]: + with self._lock: + return (*self._processors.values(), *self._retired.values()) + + @staticmethod + def _flush_one(processor: SpanProcessor, timeout_millis: int) -> bool: + try: + return processor.force_flush(timeout_millis) + except Exception: # noqa: BLE001 # one exporter's flush failure must not fail the whole flush + return False + + def deliverable(self, destinations: Iterable["OtelDestination"]) -> tuple["OtelDestination", ...]: + """The subset of ``destinations`` this fan-out can actually export to. + + A destination whose exporter will not build (a protocol whose package is not + installed, a malformed endpoint) has to be dropped before the request anchors + it, not when its first span ends. By then the operator's own exporter has been + told to hold that backend's spans back for this request, so dropping there + loses the span outright instead of leaving it where it would have gone with no + override at all. + """ + return tuple(destination for destination in destinations if self._buildable(destination)) + + def _buildable(self, destination: "OtelDestination") -> bool: + """Whether a processor for ``destination`` exists or can be built right now.""" + with self._lock: + if self._closed: + return False + built: Final = self._cached_or_built_locked(destination, anchored=False) + drained: Final = self._drainable_locked() + for shed in drained: + self._drain.submit(shed) + return built is not None + + def _acquire(self, destination: "OtelDestination") -> SpanProcessor | None: + """The processor for ``destination``, marked busy until ``_release``. + + The build happens under the same lock that reads the cache, so a cold cache + met by a burst of concurrent requests yields one exporter rather than one per + thread with all but the winner shed. Building an exporter opens no connection, + so the cost of holding the lock is a constructor, once per destination. + """ + with self._lock: + if self._closed: + return None + processor: Final = self._cached_or_built_locked(destination, anchored=True) + if processor is None: + return None + self._exporting[id(processor)] = self._exporting.get(id(processor), 0) + 1 + drained: Final = self._drainable_locked() + for shed in drained: + self._drain.submit(shed) + return processor + + def _cached_or_built_locked(self, destination: "OtelDestination", *, anchored: bool) -> SpanProcessor | None: + """The cached processor for ``destination``, or a new one if the drain can take it. + + Every build past the cache cap sheds one processor into the drain, so while the + shed ones are stuck closing against a collector that stopped answering, a + destination that is not yet anchored is refused rather than parked behind them: + ``deliverable`` then leaves its spans with the operator's exporter until the + drain catches up. One the request already anchored is rebuilt regardless. The + operator's exporter has stood down for it, so refusing here would drop the span, + and other tenants' auths can evict it in the meantime, with that eviction being + what tips the drain over. Eviction holds while the drain is saturated, so such a + rebuild costs the cache one entry rather than shedding another processor, and + the total stays at one per destination in flight. + """ + key: Final = destination.cache_key() + if (cached := self._processors.get(key)) is not None: + self._processors.move_to_end(key) + self._retire_overflow_locked() + return cached + if not anchored and self._drain.saturated(): + verbose_logger.debug("OTel V2 fan-out: drain saturated, not building for %s", destination.endpoint) + return None + return self._build_locked(destination, key) + + def _build_locked(self, destination: "OtelDestination", key: object) -> SpanProcessor | None: + built: Final = self._build(destination) + if built is None: + return None + self._processors[key] = built + self._retire_overflow_locked() + return built + + def _release(self, processor: SpanProcessor) -> None: + with self._lock: + remaining: Final = self._exporting.get(id(processor), 1) - 1 + if remaining > 0: + self._exporting[id(processor)] = remaining + else: + self._exporting.pop(id(processor), None) + if not self._exporting: + self._lock.notify_all() + drained: Final = self._drainable_locked() + for retired in drained: + self._drain.submit(retired) + + def _retire_overflow_locked(self) -> None: + """Move the LRU processor out of the cache once it is past the cap, drain permitting. + + Eviction is what feeds the drain, and a destination a request already anchored + is rebuilt on its next span, which would shed another one. While the shed ones + are stuck closing against a collector that stopped answering, evicting would + churn the cache at one more processor, and one more batch thread, per span. + Holding above the cap instead keeps the total at one processor per destination + in flight, since ``deliverable`` anchors no new destination while the drain is + saturated. Once it has room again, every hit and build trims one entry. + """ + if len(self._processors) <= _MAX_CACHED_DESTINATION_PROCESSORS or self._drain.saturated(): + return + _, evicted = self._processors.popitem(last=False) + self._retired[id(evicted)] = evicted + + def _drainable_locked(self) -> tuple[SpanProcessor, ...]: + """Retired processors no thread is exporting through, removed from the list. + + ``on_end`` holds a processor across an export, so closing an evicted one there + drops the span it is holding. A retiree is out of the cache and can never be + handed out again, so once its export count reaches zero it stays there. + """ + idle: Final = tuple(key for key in self._retired if self._exporting.get(key, 0) == 0) + return tuple(self._retired.pop(key) for key in idle) + + +def _destination_processor(destination: "OtelDestination") -> SpanProcessor | None: + """A batching OTLP processor aimed at ``destination``, or ``None`` if unbuildable. + + A protocol that resolves to a headerless exporter is unbuildable too: the + console fallback would swallow the tenant's credentials and print its spans to + the proxy's stdout while the operator's exporter stands down for them. + """ + kind: Final = destination.protocol or "otlp_http" + if exporter_transport(kind) == "headerless": + verbose_logger.debug("OTel V2 fan-out: no OTLP transport for protocol %r at %s", kind, destination.endpoint) + return None + try: + spec: Final = ExporterSpec( + kind=kind, + endpoint=destination.endpoint, + headers=destination.header_string(), + owner=None, + ) + return _processor_for(_exporter_from_spec(spec), use_simple=False) + except Exception as exc: # noqa: BLE001 # a malformed destination must not break the request or the other destinations + verbose_logger.debug("OTel V2 fan-out: no processor for %s: %s", destination.endpoint, exc) + return None + + +def _shutdown_quietly(processor: SpanProcessor) -> None: + try: + processor.shutdown() + except Exception as exc: # noqa: BLE001 # defensive: shedding a spare processor must not raise + verbose_logger.debug("OTel V2 fan-out: discarding processor failed: %s", exc) + + +class _OverriddenBackendFilter(SpanProcessor): + """Hold a span back from ``owner``'s operator-level exporter when the request + pointed ``owner`` at a tenant's own account. + + Wrapping is the only place this works: ``SynchronousMultiSpanProcessor.on_end`` + ignores return values, so a sibling processor can never veto the export. + + Under ``additive`` mode nothing is suppressed, so the wrapper passes every span + straight through and the operator keeps its copy. + """ + + def __init__(self, inner: SpanProcessor, owner: str) -> None: + self._inner: Final = inner + self._owner: Final = owner + + def on_start(self, span: SDKSpan, parent_context: Context | None = None) -> None: + self._inner.on_start(span, parent_context) + + def on_end(self, span: ReadableSpan) -> None: + if self._owner in suppressed_backends(): + return + self._inner.on_end(span) + + def shutdown(self) -> None: + self._inner.shutdown() + + def force_flush(self, timeout_millis: int = 30000) -> bool: + return self._inner.force_flush(timeout_millis) + + def build_span_exporter(config: OpenTelemetryV2Config) -> SpanExporter: """Build a single exporter from the top-level config fields. @@ -437,6 +1009,7 @@ def build_tracer_provider( exporter: SpanExporter | None = None, baggage_processor: SpanProcessor | None = None, use_simple_processor: bool | None = None, + tenant_overrides: bool = False, ) -> TracerProvider: """Build the shared :class:`TracerProvider`. @@ -445,6 +1018,13 @@ def build_tracer_provider( ``config.exporters`` entry — this is what fans spans out to multiple backends. ``exporter`` and ``use_simple_processor`` are explicit overrides: pass a single exporter to attach exactly that one (used by tests). + + ``tenant_overrides`` wraps each owned exporter so a request that pointed that + backend at a key's or team's own account skips it. Every v2 logger's provider + wants it, since any of them may own the overridden backend; delivering to the + tenant is a separate job, done once by :func:`attach_tenant_fan_out`. The + per-tenant providers this same function builds must leave it off, or they would + filter out the very spans they exist to carry. """ provider: Final = TracerProvider(resource=build_resource(config)) if baggage_processor is None: @@ -461,15 +1041,107 @@ def build_tracer_provider( if spec.requires_headers and not spec.headers: continue exp = _exporter_from_spec(spec) + processor = _processor_for( + exp, + (spec.use_simple_processor if spec.use_simple_processor is not None else use_simple_processor), + ) + owner = spec.owner.value if spec.owner is not None else None provider.add_span_processor( - _processor_for( - exp, - (spec.use_simple_processor if spec.use_simple_processor is not None else use_simple_processor), - ) + _OverriddenBackendFilter(processor, owner) if tenant_overrides and owner is not None else processor ) return provider +_FAN_OUT_ATTACH_LOCK: Final = threading.Lock() + + +def attach_tenant_fan_out(provider: TracerProvider, *configs: OpenTelemetryV2Config) -> None: + """Give ``provider`` the fan-out that delivers spans to key/team destinations. + + Called on the one provider published as the OTel global, and idempotent so a + second publish (a test, a re-initialized proxy) cannot double-export. Concurrent + first calls (requests racing to anchor before any publish) serialize on one lock + so exactly one fan-out lands. ``configs`` name the operator's own exporters, one + config per v2 logger since each keeps its own provider and still writes its + account, so an additive destination pointing at any of them is delivered once + rather than twice. + """ + with _FAN_OUT_ATTACH_LOCK: + if any(isinstance(processor, TenantFanOutSpanProcessor) for processor in _attached_processors(provider)): + return + provider.add_span_processor(TenantFanOutSpanProcessor(operator_sinks=operator_sink_keys(*configs))) + + +def deliverable_destinations( + destinations: Iterable["OtelDestination"], + provider: trace.TracerProvider | None = None, +) -> tuple["OtelDestination", ...]: + """The destinations a request can anchor, given what is published to carry them. + + Anchoring a destination is what tells the operator's own exporter to stand down + for that backend, so one nothing can deliver has to be dropped here: with no + fan-out attached, or with an exporter that will not build, the request keeps + exactly the routing it would have had without any override. + """ + fan_out: Final = next( + ( + processor + for processor in _attached_processors(provider if provider is not None else trace.get_tracer_provider()) + if isinstance(processor, TenantFanOutSpanProcessor) + ), + None, + ) + return fan_out.deliverable(destinations) if fan_out is not None else () + + +def operator_sink_keys(*configs: OpenTelemetryV2Config) -> frozenset[_SinkKey]: + """The accounts the operator's own exporters write to, in destination terms. + + Every v2 logger's config counts, since each logger exports through its own + provider. An exporter with no endpoint of its own resolves one from the + environment at export time, so it has no comparable identity and is left out, + and so is one that never reaches the wire: a console kind ignores the endpoint, + and a header-gated spec with no credentials is skipped when the provider is built. + """ + return frozenset( + key + for config in configs + for spec in config.exporters + if _exports_to_the_wire(spec) and (key := _sink_key(spec.endpoint, parse_headers(spec.headers))) is not None + ) + + +def _exports_to_the_wire(spec: ExporterSpec) -> bool: + """Whether ``build_tracer_provider`` gives ``spec`` an exporter that sends OTLP.""" + return exporter_transport(spec.kind) != "headerless" and not (spec.requires_headers and not spec.headers) + + +def _sink_key(endpoint: str | None, headers: Mapping[str, str]) -> "_SinkKey | None": + """The account an exporter writes to, or ``None`` when it has no fixed one. + + Normalized on the three counts that make one account look like two: the operator's + spec carries the signal path a tenant destination leaves for the exporter to + append, header names survive one round trip lowercased and the other not, and one + credential answers to more than one name (see :data:`_CREDENTIAL_ALIASES`). + """ + normalized: Final = _otlp_traces_endpoint(endpoint) + if normalized is None: + return None + return (normalized, tuple(sorted((_credential_name(name), value) for name, value in headers.items()))) + + +def _credential_name(header: str) -> str: + """The credential a header carries, under whichever name the backend spells it.""" + normalized: Final = header.strip().lower().replace("-", "_") + return _CREDENTIAL_ALIASES.get(normalized, normalized) + + +def _attached_processors(provider: trace.TracerProvider) -> "tuple[SpanProcessor, ...]": + """The processors already on ``provider``, or empty when the SDK hides them.""" + multi: Final = getattr(provider, "_active_span_processor", None) + return tuple(getattr(multi, "_span_processors", ())) + + def get_tracer(provider: TracerProvider, name: str = "litellm") -> Tracer: # Stamp the instrumentation scope with the LiteLLM package version so every # emitted span carries a deterministic ``scope.version`` (the standard OTel diff --git a/litellm/integrations/otel/plumbing/routing.py b/litellm/integrations/otel/plumbing/routing.py index 227e18f3663..f78d18d943c 100644 --- a/litellm/integrations/otel/plumbing/routing.py +++ b/litellm/integrations/otel/plumbing/routing.py @@ -25,6 +25,7 @@ from opentelemetry.trace import Tracer from litellm._logging import verbose_logger from litellm.constants import OTEL_SERVICE_NAME_METADATA_KEYS from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config +from litellm.integrations.otel.plumbing.context import destination_backends from litellm.integrations.otel.plumbing.providers import ( build_tracer_provider, exporter_transport, @@ -231,10 +232,21 @@ class TenantTracerCache: concurrent overflow eviction can't shut it down between selection and the caller's span start. The caller must ``release`` it exactly once. """ + # A backend with a destination is delivered by the fan-out processor, which + # carries the whole trace and already carries this tenant's credentials and + # service name. Routing here too would detach this span onto a second provider, + # so the tenant would get the request tree plus a stray one-span trace. + if self._callback_name is not None and self._callback_name in destination_backends(): + return TenantRoute(tracer=default, detached=False) credential_headers: Final = self._credential_headers(dynamic_params) project_headers: Final = self._project_headers(auth_metadata) service_name: Final = tenant_service_name(auth_metadata) - if not credential_headers and not project_headers and service_name is None: + tenant_account: Final = bool(credential_headers) or bool(project_headers) + # A service name on its own only relabels the operator's own backend, so moving + # the span to a second provider for it while some other backend has a + # destination would drop the model call out of the trace the fan-out delivers. + # The destination stamps the same service name itself. + if not tenant_account and (service_name is None or destination_backends()): return TenantRoute(tracer=default, detached=False) # A fixed per-integration region endpoint (New Relic us/eu), never a # caller-supplied host; ``None`` keeps the preset's own endpoint. @@ -255,7 +267,7 @@ class TenantTracerCache: _shutdown_provider(evicted) return TenantRoute( tracer=get_tracer(provider, self._tracer_name), - detached=bool(project_headers) or bool(credential_headers), + detached=tenant_account, provider=provider, ) diff --git a/litellm/integrations/otel/presets/agentops.py b/litellm/integrations/otel/presets/agentops.py index f45b1cd3cff..965213f2ee4 100644 --- a/litellm/integrations/otel/presets/agentops.py +++ b/litellm/integrations/otel/presets/agentops.py @@ -39,6 +39,7 @@ class _AgentOpsSettings(BaseSettings): def agentops_preset( *, config_overrides: OpenTelemetryV2Config | None = None, + allow_missing_credentials: bool = False, ) -> OpenTelemetryV2Config: """Build the AgentOps config without any network I/O. diff --git a/litellm/integrations/otel/presets/arize.py b/litellm/integrations/otel/presets/arize.py index ee0de675657..d7ce87f5552 100644 --- a/litellm/integrations/otel/presets/arize.py +++ b/litellm/integrations/otel/presets/arize.py @@ -26,10 +26,12 @@ class _ArizeSettings(BaseSettings): def arize_preset( *, config_overrides: OpenTelemetryV2Config | None = None, + allow_missing_credentials: bool = False, ) -> OpenTelemetryV2Config: + base: Final = config_overrides or OpenTelemetryV2Config() + mappers: Final = ensure_mappers(base.mapper_names, "openinference") arize_cfg: Final = _V1ArizeLogger.get_arize_config() headers: Final = _arize_headers(arize_cfg) - base: Final = config_overrides or OpenTelemetryV2Config() return base.model_copy( update={ "exporters": [ @@ -41,7 +43,7 @@ def arize_preset( owner=ExporterOwner.ARIZE_AX, ), ], - "mapper_names": ensure_mappers(base.mapper_names, "openinference"), + "mapper_names": mappers, "resource_attributes": { **base.resource_attributes, **({"model_id": arize_cfg.project_name} if arize_cfg.project_name else {}), diff --git a/litellm/integrations/otel/presets/base.py b/litellm/integrations/otel/presets/base.py index 3b9991f86a4..3a768a08a4f 100644 --- a/litellm/integrations/otel/presets/base.py +++ b/litellm/integrations/otel/presets/base.py @@ -18,6 +18,18 @@ class Preset(Protocol): ``config_overrides`` lets one preset layer onto another's config (or onto test-supplied defaults); the factory calls presets with no arguments. + + ``allow_missing_credentials`` lets a credential-mandatory backend (langfuse and + weave) degrade to an exporter-less, mapper-only config instead of raising when the + operator set no env credentials of their own. That is a real + deployment: every team brings its own account and the operator keeps none, and + without it the whole V2 path silently falls back to the legacy integration, so + no team destination is ever reached. Credential-optional backends ignore it. """ - def __call__(self, *, config_overrides: OpenTelemetryV2Config | None = None) -> OpenTelemetryV2Config: ... + def __call__( + self, + *, + config_overrides: OpenTelemetryV2Config | None = None, + allow_missing_credentials: bool = False, + ) -> OpenTelemetryV2Config: ... diff --git a/litellm/integrations/otel/presets/destinations.py b/litellm/integrations/otel/presets/destinations.py new file mode 100644 index 00000000000..2bf9bfa5261 --- /dev/null +++ b/litellm/integrations/otel/presets/destinations.py @@ -0,0 +1,152 @@ +"""Map a key's or team's callback vars to the OTLP destination its traces export to. + +Header building is delegated to each preset's existing ``*_dynamic_headers`` builder, +so a destination authenticates exactly the way the per-request tracer route already +did; only the endpoint and transport need a per-backend rule. +""" + +import os +from collections.abc import Callable, Mapping +from functools import lru_cache +from types import MappingProxyType +from typing import Final + +import litellm +from litellm._logging import verbose_logger +from litellm.integrations.otel.model.destination import OtelDestination +from litellm.litellm_core_utils.url_utils import is_url_destination_allowed_by_host +from litellm.types.utils import StandardCallbackDynamicParams + +#: An endpoint plus the OTLP transport to reach it with, or ``None`` when the backend +#: names no destination. The transport is ``None`` where the backend has only one. +_Destination = tuple[str, str | None] + + +@lru_cache(maxsize=128) +def _warn_host_not_allowlisted(host: str) -> None: + """Cached so one misconfigured team logs once rather than once per request.""" + verbose_logger.warning( + "OTel V2: not exporting to key/team Langfuse host '%s'. Add it to " + "litellm_settings.provider_url_destination_allowed_hosts to permit it", + host, + ) + + +def _langfuse_destination(params: StandardCallbackDynamicParams) -> "_Destination | None": + """The tenant's own Langfuse host, else the operator's, else Langfuse US cloud. + + A host the tenant named has to be allowlisted by the operator, the same way a + URL-valued ``model`` is: anyone who can mint a key can write it, and it becomes an + endpoint the proxy posts the request's whole trace to, carrying the tenant's own + credentials. The operator's own ``LANGFUSE_HOST`` is not checked, since an internal + collector there is a deployment choice. + """ + from litellm.integrations.langfuse.langfuse_otel import ( + LANGFUSE_CLOUD_US_ENDPOINT, + LangfuseOtelLogger, + ) + + tenant_host: Final = params.get("langfuse_host") or None + host: Final = tenant_host or LangfuseOtelLogger._get_langfuse_otel_host() # pyright: ignore[reportPrivateUsage] # reuse the backend's own env host resolver rather than duplicating it + if not host: + return (LANGFUSE_CLOUD_US_ENDPOINT, None) + normalized: Final = host if host.startswith("http") else f"https://{host}" + endpoint: Final = f"{normalized.rstrip('/')}/api/public/otel" + if tenant_host is None: + return (endpoint, None) + if not is_url_destination_allowed_by_host(endpoint, litellm.provider_url_destination_allowed_hosts): + _warn_host_not_allowlisted(host) + return None + return (endpoint, None) + + +def _arize_destination(params: StandardCallbackDynamicParams) -> "_Destination | None": + from litellm.integrations.arize.arize import ArizeLogger + + config: Final = ArizeLogger.get_arize_config() + return (config.endpoint, config.protocol) + + +def _weave_destination(params: StandardCallbackDynamicParams) -> "_Destination | None": + from litellm.integrations.weave.weave_otel import weave_otel_endpoint + + return (weave_otel_endpoint(os.environ.get("WANDB_HOST")), None) + + +def _newrelic_destination(params: StandardCallbackDynamicParams) -> "_Destination | None": + from litellm.integrations.otel.presets.newrelic import newrelic_dynamic_endpoint + + endpoint: Final = newrelic_dynamic_endpoint(params) + return (endpoint, None) if endpoint else None + + +#: Callback name -> destination resolver. A backend is destination-capable exactly +#: when it appears here AND in ``DYNAMIC_HEADERS_BY_CALLBACK``: without a header +#: builder the destination would carry no tenant credentials, and the exporter +#: would post the tenant's traffic to the operator's account. +_DESTINATION_BY_CALLBACK: Final[Mapping[str, Callable[[StandardCallbackDynamicParams], "_Destination | None"]]] = ( + MappingProxyType( + { + "langfuse_otel": _langfuse_destination, + "arize": _arize_destination, + "weave_otel": _weave_destination, + "newrelic": _newrelic_destination, + } + ) +) + +#: Headers a destination must carry to authenticate. Several dynamic-header builders +#: gate each credential independently, so a half-configured backend yields a non-empty +#: but unusable header set; accepting it would suppress the operator's own exporter and +#: send the request's whole trace where it cannot be stored. +_REQUIRED_HEADERS_BY_CALLBACK: Final[Mapping[str, frozenset[str]]] = MappingProxyType( + { + "langfuse_otel": frozenset({"Authorization"}), + "arize": frozenset({"arize-space-id", "api_key"}), + "weave_otel": frozenset({"Authorization", "project_id"}), + "newrelic": frozenset({"api-key"}), + } +) + +_NO_ATTRS: Final[Mapping[str, str]] = MappingProxyType({}) + + +def destination_capable_backends() -> frozenset[str]: + """Backends a key or team can point at its own account.""" + from litellm.integrations.otel.presets import DYNAMIC_HEADERS_BY_CALLBACK + + return frozenset(_DESTINATION_BY_CALLBACK) & frozenset(DYNAMIC_HEADERS_BY_CALLBACK) + + +def destination_for( + callback_name: str, + params: StandardCallbackDynamicParams, + service_name: str | None = None, +) -> OtelDestination | None: + """The destination ``params`` names for ``callback_name``, or ``None``. + + ``None`` means the caller configured nothing usable for this backend, so the + request keeps the operator's global exporters. ``service_name`` is the key's or + team's ``otel_service_name``, which the per-request tracer route applies when the + backend is not overridden and the destination has to apply once it is. + """ + from litellm.integrations.otel.presets import DYNAMIC_HEADERS_BY_CALLBACK + + header_builder: Final = DYNAMIC_HEADERS_BY_CALLBACK.get(callback_name) + destination_builder: Final = _DESTINATION_BY_CALLBACK.get(callback_name) + if header_builder is None or destination_builder is None: + return None + headers: Final = header_builder(params) + if not headers or not _REQUIRED_HEADERS_BY_CALLBACK[callback_name] <= frozenset(headers): + return None + resolved: Final = destination_builder(params) + if resolved is None: + return None + endpoint, protocol = resolved + return OtelDestination( + endpoint=endpoint, + headers=MappingProxyType(dict(headers)), # mutable-ok: MappingProxyType needs a concrete mapping to wrap + resource_attributes=MappingProxyType({"service.name": service_name}) if service_name else _NO_ATTRS, + callback_name=callback_name, + protocol=protocol, + ) diff --git a/litellm/integrations/otel/presets/langfuse.py b/litellm/integrations/otel/presets/langfuse.py index c2f64422eff..9149e0c0d94 100644 --- a/litellm/integrations/otel/presets/langfuse.py +++ b/litellm/integrations/otel/presets/langfuse.py @@ -10,17 +10,32 @@ from litellm.integrations.otel.model.config import ( ExporterSpec, OpenTelemetryV2Config, ) -from litellm.integrations.otel.presets.utils import ensure_mappers +from litellm.integrations.otel.presets.utils import ( + credential_gated_exporters, + ensure_mappers, +) from litellm.types.utils import StandardCallbackDynamicParams def langfuse_preset( *, config_overrides: OpenTelemetryV2Config | None = None, + allow_missing_credentials: bool = False, ) -> OpenTelemetryV2Config: - cfg: Final = _V1Langfuse.get_langfuse_otel_config() - kind: Final = cfg.exporter if isinstance(cfg.exporter, str) else "otlp_http" base: Final = config_overrides or OpenTelemetryV2Config() + mappers: Final = ensure_mappers(base.mapper_names, "langfuse") + try: + cfg: Final = _V1Langfuse.get_langfuse_otel_config() + except Exception: + if not allow_missing_credentials: + raise + return base.model_copy( + update={ # mutable-ok: pydantic model_copy takes a plain update mapping + "exporters": credential_gated_exporters(base.exporters, ExporterOwner.LANGFUSE_OTEL), + "mapper_names": mappers, + } + ) + kind: Final = cfg.exporter if isinstance(cfg.exporter, str) else "otlp_http" return base.model_copy( update={ "exporters": [ @@ -32,7 +47,7 @@ def langfuse_preset( owner=ExporterOwner.LANGFUSE_OTEL, ), ], - "mapper_names": ensure_mappers(base.mapper_names, "langfuse"), + "mapper_names": mappers, } ) diff --git a/litellm/integrations/otel/presets/langtrace.py b/litellm/integrations/otel/presets/langtrace.py index c88e4715ab0..2312575f04a 100644 --- a/litellm/integrations/otel/presets/langtrace.py +++ b/litellm/integrations/otel/presets/langtrace.py @@ -9,6 +9,7 @@ from litellm.integrations.otel.presets.utils import ensure_mappers def langtrace_preset( *, config_overrides: OpenTelemetryV2Config | None = None, + allow_missing_credentials: bool = False, ) -> OpenTelemetryV2Config: """Compose the Langtrace mapper on top of the customer's OTLP destination. diff --git a/litellm/integrations/otel/presets/levo.py b/litellm/integrations/otel/presets/levo.py index 41b3758cf3e..c1580cf7a5b 100644 --- a/litellm/integrations/otel/presets/levo.py +++ b/litellm/integrations/otel/presets/levo.py @@ -13,6 +13,7 @@ from litellm.integrations.otel.model.config import ( def levo_preset( *, config_overrides: OpenTelemetryV2Config | None = None, + allow_missing_credentials: bool = False, ) -> OpenTelemetryV2Config: cfg: Final = _V1Levo.get_levo_config() base: Final = config_overrides or OpenTelemetryV2Config() diff --git a/litellm/integrations/otel/presets/newrelic.py b/litellm/integrations/otel/presets/newrelic.py index 4660a707355..771b3f643c8 100644 --- a/litellm/integrations/otel/presets/newrelic.py +++ b/litellm/integrations/otel/presets/newrelic.py @@ -44,6 +44,7 @@ class _NewRelicSettings(BaseSettings): def newrelic_preset( *, config_overrides: OpenTelemetryV2Config | None = None, + allow_missing_credentials: bool = False, ) -> OpenTelemetryV2Config: settings: Final = _NewRelicSettings() base: Final = config_overrides or OpenTelemetryV2Config() diff --git a/litellm/integrations/otel/presets/phoenix.py b/litellm/integrations/otel/presets/phoenix.py index eef407b6c1b..f4f34ee7525 100644 --- a/litellm/integrations/otel/presets/phoenix.py +++ b/litellm/integrations/otel/presets/phoenix.py @@ -60,6 +60,7 @@ def phoenix_project_headers(auth_metadata: Mapping[str, str] | None) -> Mapping[ def phoenix_preset( *, config_overrides: OpenTelemetryV2Config | None = None, + allow_missing_credentials: bool = False, ) -> OpenTelemetryV2Config: cfg: Final = _V1Phoenix.get_arize_phoenix_config() headers: Final = cfg.otlp_auth_headers if hasattr(cfg, "otlp_auth_headers") else None diff --git a/litellm/integrations/otel/presets/utils.py b/litellm/integrations/otel/presets/utils.py index 328569d3daf..1270c41e77b 100644 --- a/litellm/integrations/otel/presets/utils.py +++ b/litellm/integrations/otel/presets/utils.py @@ -3,6 +3,8 @@ from collections.abc import Iterable from typing import Final +from litellm.integrations.otel.model.config import ExporterOwner, ExporterSpec + def ensure_mappers(mapper_names: Iterable[str], *names: str) -> list[str]: """Return ``mapper_names`` with each of ``names`` appended if not already present. @@ -15,3 +17,32 @@ def ensure_mappers(mapper_names: Iterable[str], *names: str) -> list[str]: if name not in result: result.append(name) return result + + +def credential_gated_exporters( + exporters: "Iterable[ExporterSpec]", owner: "ExporterOwner" +) -> "tuple[ExporterSpec, ...]": + """``exporters`` with the operator's destination replaced by a header-gated one. + + Used when a credential-mandatory backend is asked to build without the operator's + own credentials, so only key/team destinations receive spans. Two things have to + happen for that to mean "export nowhere": the placeholder console spec that + ``OpenTelemetryV2Config`` folds in for an empty exporter list is dropped, or every + span would be printed to stdout, and the gated spec keeps the owner so the + override filter still recognises which backend this provider speaks for. + """ + return ( + *(spec for spec in exporters if not is_unconfigured_placeholder(spec)), + ExporterSpec(owner=owner, requires_headers=True), + ) + + +def is_unconfigured_placeholder(spec: "ExporterSpec") -> bool: + """Whether ``spec`` is the one ``_normalize`` folds in when nothing was configured. + + No field set is what says the operator asked for nothing: an exporter they did + configure survives, even ``OTEL_EXPORTER=console`` whose value matches the default, + and so does the gated spec this module appends, which would otherwise eat itself + when one preset layers onto another. + """ + return not spec.model_fields_set diff --git a/litellm/integrations/otel/presets/weave.py b/litellm/integrations/otel/presets/weave.py index 51d0ad01093..644cd39ad36 100644 --- a/litellm/integrations/otel/presets/weave.py +++ b/litellm/integrations/otel/presets/weave.py @@ -7,7 +7,10 @@ from litellm.integrations.otel.model.config import ( ExporterSpec, OpenTelemetryV2Config, ) -from litellm.integrations.otel.presets.utils import ensure_mappers +from litellm.integrations.otel.presets.utils import ( + credential_gated_exporters, + ensure_mappers, +) from litellm.integrations.weave.weave_otel import ( _get_weave_authorization_header, get_weave_otel_config, @@ -18,9 +21,21 @@ from litellm.types.utils import StandardCallbackDynamicParams def weave_preset( *, config_overrides: OpenTelemetryV2Config | None = None, + allow_missing_credentials: bool = False, ) -> OpenTelemetryV2Config: - weave_cfg: Final = get_weave_otel_config() base: Final = config_overrides or OpenTelemetryV2Config() + mappers: Final = ensure_mappers(base.mapper_names, "openinference", "weave") + try: + weave_cfg: Final = get_weave_otel_config() + except Exception: + if not allow_missing_credentials: + raise + return base.model_copy( + update={ # mutable-ok: pydantic model_copy takes a plain update mapping + "exporters": credential_gated_exporters(base.exporters, ExporterOwner.WEAVE_OTEL), + "mapper_names": mappers, + } + ) return base.model_copy( update={ "exporters": [ @@ -33,7 +48,7 @@ def weave_preset( ), ], # Weave consumes OpenInference + a small Weave-specific overlay. - "mapper_names": ensure_mappers(base.mapper_names, "openinference", "weave"), + "mapper_names": mappers, } ) diff --git a/litellm/integrations/weave/weave_otel.py b/litellm/integrations/weave/weave_otel.py index f2cc64a9ba2..50289263f38 100644 --- a/litellm/integrations/weave/weave_otel.py +++ b/litellm/integrations/weave/weave_otel.py @@ -117,6 +117,14 @@ def _get_weave_authorization_header(api_key: str) -> str: return f"Basic {auth_header}" +def weave_otel_endpoint(host: str | None) -> str: + """The OTLP traces endpoint for a self-managed ``host``, else Weave cloud.""" + if not host: + return WEAVE_BASE_URL + WEAVE_OTEL_ENDPOINT + normalized: Final = host if host.startswith("http") else f"https://{host}" + return normalized.rstrip("/") + WEAVE_OTEL_ENDPOINT + + def get_weave_otel_config() -> WeaveOtelConfig: """ Retrieves the Weave OpenTelemetry configuration based on environment variables. @@ -134,7 +142,6 @@ def get_weave_otel_config() -> WeaveOtelConfig: """ api_key: Final = os.getenv("WANDB_API_KEY") project_id: Final = os.getenv("WANDB_PROJECT_ID") - host = os.getenv("WANDB_HOST") if not api_key: raise ValueError("WANDB_API_KEY must be set for Weave OpenTelemetry integration.") @@ -144,15 +151,8 @@ def get_weave_otel_config() -> WeaveOtelConfig: "WANDB_PROJECT_ID must be set for Weave OpenTelemetry integration. Format: /" ) - if host: - if not host.startswith("http"): - host = "https://" + host - # Self-managed instances use a different path - endpoint = host.rstrip("/") + WEAVE_OTEL_ENDPOINT - verbose_logger.debug("Using Weave OTEL endpoint from host: %s", endpoint) - else: - endpoint = WEAVE_BASE_URL + WEAVE_OTEL_ENDPOINT - verbose_logger.debug("Using Weave cloud endpoint: %s", endpoint) + endpoint: Final = weave_otel_endpoint(os.getenv("WANDB_HOST")) + verbose_logger.debug("Using Weave OTEL endpoint: %s", endpoint) # Weave uses Basic auth with format: api: auth_header: Final = _get_weave_authorization_header(api_key=api_key) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index c31c4323157..f326d604ae2 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -200,6 +200,7 @@ if TYPE_CHECKING: from mcp.types import EmbeddedResource, ImageContent, TextContent from litellm.integrations.otel.logger import OpenTelemetryV2 + from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig try: from litellm_enterprise.enterprise_callbacks.callback_controls import ( @@ -4800,31 +4801,83 @@ def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[Custom Returns ``None`` when V2 is off OR when there's no preset registered for ``callback_name`` — callers should then fall through to the legacy path. + + A preset that needs operator credentials it cannot find is allowed to build + only when this request has a key/team destination for that backend and another + V2 logger is already registered to carry the fan-out. The resulting logger keeps + only its credential-gated exporter, while the registered logger owns operator + delivery. Without that carrier, a preset that raises or that ends up with nothing + but its gated exporter and the default console placeholder returns ``None``, so the + caller falls through to the legacy path exactly as before V2 landed. """ from litellm.integrations.otel.model.config import is_otel_v2_enabled if not is_otel_v2_enabled(): return None from litellm.integrations.otel.logger import OpenTelemetryV2, build_otel_v2_logger + from litellm.integrations.otel.plumbing.context import destination_backends from litellm.integrations.otel.presets import PRESET_BY_CALLBACK preset_fn: Final = PRESET_BY_CALLBACK.get(callback_name) if preset_fn is None: return None + serves_a_destination: Final = callback_name in destination_backends() + has_v2_logger: Final = any(isinstance(callback, OpenTelemetryV2) for callback in _in_memory_loggers) + carried: Final = serves_a_destination and has_v2_logger for callback in _in_memory_loggers: - if isinstance(callback, OpenTelemetryV2) and getattr(callback, "callback_name", None) == callback_name: + if ( + isinstance(callback, OpenTelemetryV2) + and getattr(callback, "callback_name", None) == callback_name + and (serves_a_destination or not _exports_nowhere(callback.config)) + ): return callback try: - config: Final = preset_fn() + built: Final = preset_fn(allow_missing_credentials=carried) except Exception: # If env vars are missing or the preset raises, defer to the legacy path # so customers get the same error story they had before V2 landed. return None + gated: Final = _is_credential_gated(built) + if gated and not carried and not _has_operator_exporter(built): + return None + config: Final = _only_the_gated_exporter(built) if gated and carried else built + if _exports_nowhere(config): + verbose_logger.warning( + "OTel V2: no operator credentials for '%s'; only key/team destinations will receive its traces", + callback_name, + ) v2_logger: Final = build_otel_v2_logger(config=config, callback_name=callback_name) _in_memory_loggers.append(v2_logger) return v2_logger +def _exports_nowhere(config: "OpenTelemetryV2Config") -> bool: + """Whether every exporter in ``config`` is waiting on credentials it never got.""" + return all(_is_gated(spec) for spec in config.exporters) + + +def _is_credential_gated(config: "OpenTelemetryV2Config") -> bool: + """Whether the preset built without the operator's own credentials for its backend.""" + return any(_is_gated(spec) for spec in config.exporters) + + +def _has_operator_exporter(config: "OpenTelemetryV2Config") -> bool: + """Whether the operator configured somewhere real to export, beyond the default console placeholder.""" + from litellm.integrations.otel.presets.utils import is_unconfigured_placeholder + + return any(not _is_gated(spec) and not is_unconfigured_placeholder(spec) for spec in config.exporters) + + +def _only_the_gated_exporter(config: "OpenTelemetryV2Config") -> "OpenTelemetryV2Config": + return config.model_copy( + update={"exporters": [spec for spec in config.exporters if _is_gated(spec)]} # mutable-ok: model_copy update + ) + + +def _is_gated(spec: "ExporterSpec") -> bool: + return spec.requires_headers and not spec.headers + + def _maybe_auto_initialize_arize_phoenix(_in_memory_loggers: list[CustomLogger]) -> None: """ Auto-initialize ArizePhoenixLogger when Phoenix env vars are detected. diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 26f5d6e7c8c..d90b0cd5d9f 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -19,7 +19,7 @@ from typing import TYPE_CHECKING, Any, Final, Protocol import httpx from fastapi import FastAPI, HTTPException -from pydantic import AnyUrl, ConfigDict +from pydantic import AnyUrl, ConfigDict, TypeAdapter, ValidationError from starlette.requests import Request as StarletteRequest from starlette.responses import JSONResponse from starlette.types import Message, Receive, Scope, Send @@ -107,9 +107,9 @@ _MAX_STATEFUL_SESSIONS_PER_OWNER: Final = 100 # prevents an authenticated client from forcing the proxy to buffer an # arbitrarily large body just to make a routing decision. _MCP_ROUTING_PEEK_MAX_BYTES: Final = 4096 -# ASGI scope key holding the tracing span of the request carrying an MCP -# message, written on the request task and read back by the message handler. +# ASGI scope keys carrying OTel request state into a stateful MCP message handler. _MCP_TRANSPORT_SPAN_SCOPE_KEY: Final = "litellm_otel_transport_span" +_MCP_DESTINATIONS_SCOPE_KEY: Final = "litellm_otel_request_destinations" def _invalidate_byok_cred_cache(user_id: str, server_id: str) -> None: @@ -327,18 +327,17 @@ def _otel_publish_transport_span_on_scope(scope: Scope) -> None: scope[_MCP_TRANSPORT_SPAN_SCOPE_KEY] = span -def _otel_transport_span_from_message(req_ctx: object) -> object: - """The tracing span of the HTTP request that carried this MCP message. - - Read off that request's ASGI scope, reached through the ``Request`` the - streamable-HTTP transport attaches to each message, so it is this message's - transport and not whichever request happens to have touched the session last. - Returns whatever the scope holds; the otel plumbing validates it.""" +def _otel_value_from_message_scope(req_ctx: object, key: str) -> object: request: Final = getattr(req_ctx, "request", None) scope: Final = getattr(request, "scope", None) if not isinstance(scope, Mapping): return None - return scope.get(_MCP_TRANSPORT_SPAN_SCOPE_KEY) + return scope.get(key) + + +def _otel_transport_span_from_message(req_ctx: object) -> object: + """The tracing span of the HTTP request that carried this MCP message.""" + return _otel_value_from_message_scope(req_ctx, _MCP_TRANSPORT_SPAN_SCOPE_KEY) def _otel_set_mcp_transport_span(span: object) -> object: @@ -371,6 +370,44 @@ def _otel_reset_mcp_transport_span(token: object) -> None: return +def _otel_publish_request_destinations_on_scope(scope: Scope) -> None: + try: + from litellm.integrations.otel.plumbing.context import request_destinations + + scope[_MCP_DESTINATIONS_SCOPE_KEY] = request_destinations() + except ImportError: + return + + +def _otel_set_mcp_request_destinations(req_ctx: object) -> object: + destinations: Final = _otel_value_from_message_scope(req_ctx, _MCP_DESTINATIONS_SCOPE_KEY) + if not isinstance(destinations, tuple): + return None + try: + from litellm.integrations.otel.model.destination import OtelDestination + from litellm.integrations.otel.plumbing.context import set_request_destinations + + destination_adapter: Final[TypeAdapter[tuple[OtelDestination, ...]]] = TypeAdapter( + tuple[OtelDestination, ...], + config=ConfigDict(revalidate_instances="always"), + ) + validated_destinations: Final = destination_adapter.validate_python(destinations, strict=True) + return set_request_destinations(validated_destinations) + except (ImportError, ValidationError): + return None + + +def _otel_reset_mcp_request_destinations(token: object) -> None: + if token is None: + return + try: + from litellm.integrations.otel.plumbing.context import reset_request_destinations + + reset_request_destinations(token) + except ImportError: + return + + def _proxy_exception_to_http_exception(exc: ProxyException) -> HTTPException: """Map a ``ProxyException`` to an ``HTTPException`` that preserves its real status code and headers. @@ -762,10 +799,12 @@ if MCP_AVAILABLE: _session_reset_token = active_mcp_session_var.set(req_ctx.session) _trace_token = None _transport_token = None + _destinations_token = None try: _trace_token = _otel_set_mcp_trace_carrier(_mcp_meta_trace_carrier(req_ctx)) _transport_token = _otel_set_mcp_transport_span(_otel_transport_span_from_message(req_ctx)) + _destinations_token = _otel_set_mcp_request_destinations(req_ctx) # Get user authentication from context variable ( user_api_key_auth, @@ -822,6 +861,7 @@ if MCP_AVAILABLE: # This prevents the HTTP stream from failing and allows the client to get a response return [] finally: + _otel_reset_mcp_request_destinations(_destinations_token) _otel_reset_mcp_transport_span(_transport_token) _otel_reset_mcp_trace_carrier(_trace_token) if _session_reset_token is not None: @@ -1006,10 +1046,12 @@ if MCP_AVAILABLE: _session_reset_token = active_mcp_session_var.set(req_ctx.session) _trace_token = None _transport_token = None + _destinations_token = None try: _trace_token = _otel_set_mcp_trace_carrier(_mcp_meta_trace_carrier(req_ctx)) _transport_token = _otel_set_mcp_transport_span(_otel_transport_span_from_message(req_ctx)) + _destinations_token = _otel_set_mcp_request_destinations(req_ctx) # Validate arguments ( user_api_key_auth, @@ -1147,6 +1189,7 @@ if MCP_AVAILABLE: return response finally: + _otel_reset_mcp_request_destinations(_destinations_token) _otel_reset_mcp_transport_span(_transport_token) _otel_reset_mcp_trace_carrier(_trace_token) if _session_reset_token is not None: @@ -4397,6 +4440,7 @@ if MCP_AVAILABLE: async def _dispatch() -> None: _otel_publish_transport_span_on_scope(scope) + _otel_publish_request_destinations_on_scope(scope) auth_user: Final = _set_or_update_auth_context( user_api_key_auth=user_api_key_auth, mcp_auth_header=mcp_auth_header, diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 93293db24c6..0ec28b62a3a 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -2845,6 +2845,43 @@ async def _authorize_authenticated_request( @tracer.wrap() +def _seed_request_destinations(user_api_key_dict: UserAPIKeyAuth, request: Request | None = None) -> None: + """Anchor the OTLP destinations this key or team overrides its traces to. + + Called inside the ``auth`` phase span so that span reaches the tenant's account + as well, and on the request task so the ``ContextVar`` is inherited by the logging + tasks that close the LLM span. Best-effort: trace routing must never fail auth. + + ``request`` carries the headers, so a backend this request disabled with + ``x-litellm-disable-callbacks`` resolves to no destination. + + Only destinations the published fan-out can build are anchored. Anchoring one is + what tells the operator's exporter to hold that backend's spans back under + ``override``, so an unbuildable one would leave the span with nowhere to go. + + The ``postgres`` spans under ``auth`` close before this runs, because they are the + reads that resolve the identity being read here. They never reach the tenant's + account, and they are never withheld from the operator's backend, whichever mode + is set. + """ + try: + from litellm.integrations.otel.logger import fan_out_provider + from litellm.integrations.otel.plumbing.context import set_request_destinations + from litellm.integrations.otel.plumbing.providers import deliverable_destinations + from litellm.proxy.litellm_pre_call_utils import ( + resolve_tenant_otel_destinations, + ) + + set_request_destinations( + deliverable_destinations( + resolve_tenant_otel_destinations(user_api_key_dict, _safe_get_request_headers(request)), + fan_out_provider(), + ) + ) + except Exception as exc: # noqa: BLE001 # telemetry routing is best-effort and must never break authentication + verbose_proxy_logger.debug("OTel V2: tenant destination resolution failed: %s", exc) + + async def user_api_key_auth( request: Request, api_key: str = fastapi.Security(api_key_header), @@ -2891,6 +2928,7 @@ async def user_api_key_auth( raise body_parse_exception raise user_api_key_auth_obj.budget_reservation = None + _seed_request_destinations(user_api_key_auth_obj, request) # A body that never parsed is authenticated (so the trace carries identity # and this ``auth`` span) but not authorized: there is no model to check it diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index d026c5510e6..0ced2294248 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -10,6 +10,7 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, cast from fastapi import HTTPException, Request +from pydantic import TypeAdapter from pydantic import ValidationError as PydanticValidationError from starlette.datastructures import Headers @@ -26,6 +27,7 @@ from litellm.constants import ( SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, SESSION_ID_GENERATED_METADATA_KEY, SESSION_ID_OMITTED_METADATA_KEY, + X_LITELLM_DISABLE_CALLBACKS, ) from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( @@ -157,6 +159,7 @@ from litellm.types.utils import ( CustomPricingLiteLLMParams, LlmProviders, ProviderSpecificHeader, + StandardCallbackDynamicParams, StandardLoggingUserAPIKeyMetadata, SupportedCacheControls, ) @@ -170,6 +173,7 @@ _ENABLE_TEAM_STALE_ALIAS_BYPASS: bool | None = None if TYPE_CHECKING: + from litellm.integrations.otel.model.destination import OtelDestination from litellm.proxy.proxy_server import ProxyConfig as _ProxyConfig from litellm.types.proxy.policy_engine import Policy, PolicyMatchContext @@ -974,6 +978,142 @@ def _get_dynamic_logging_metadata( return callback_settings_obj +_TENANT_OTEL_PARAMS: Final = TypeAdapter(StandardCallbackDynamicParams) + + +def _tenant_otel_params(callback_vars: Mapping[str, str]) -> StandardCallbackDynamicParams: + try: + return _TENANT_OTEL_PARAMS.validate_python(callback_vars) + except PydanticValidationError: + return StandardCallbackDynamicParams() + + +_NO_REQUEST_HEADERS: Final[Mapping[str, str]] = MappingProxyType({}) + + +def _dynamically_disabled_backends( + user_api_key_dict: UserAPIKeyAuth, + request_headers: Mapping[str, str] | None, +) -> frozenset[str]: + """The callbacks this request turned off, read the way dispatch reads them. + + Same sources, precedence, and premium gate ``EnterpriseCallbackControls`` applies + before it skips a callback: the ``x-litellm-disable-callbacks`` header wins over the + key's stored list, team settings are not a source, and a non-premium proxy honours + neither. A destination has to agree with that decision, or a backend the key turned + off would still be exported to, now through the fan-out instead of the callback. + """ + from litellm.proxy.proxy_server import premium_user + + if litellm.allow_dynamic_callback_disabling is not True or not premium_user: + return frozenset() + header: Final = (request_headers if request_headers is not None else _NO_REQUEST_HEADERS).get( + X_LITELLM_DISABLE_CALLBACKS + ) + if header is not None: + return frozenset(name.strip().lower() for name in header.split(",")) + metadata: Final = user_api_key_dict.metadata + disabled: Final = metadata.get("litellm_disabled_callbacks") if metadata else None + if not isinstance(disabled, list): + return frozenset() + return frozenset(name.lower() for name in disabled if isinstance(name, str)) + + +def resolve_tenant_otel_destinations( + user_api_key_dict: UserAPIKeyAuth, + request_headers: Mapping[str, str] | None = None, +) -> "tuple[OtelDestination, ...]": + """The OTLP destinations this request's key or team config overrides its traces to. + + Key settings win over team settings outright, the same precedence + ``_get_dynamic_logging_metadata`` applies, so one caller never exports the same + backend to two accounts. An empty key-level list counts as configured, since that + is what disabling a key's callbacks writes. Returns empty when OTEL V2 is off, when + neither level named a destination-capable backend, or when the config is + incomplete, and the request then keeps the operator's own exporters. + + Two entries naming the same backend merge their ``callback_vars`` last-wins, the + way ``convert_key_logging_metadata_to_callback`` merges them, so the destination + and the per-request tracer routing cannot read one config two ways. + + A ``failure``-only entry is skipped: a destination is resolved during auth, before + the request has an outcome, so honouring the filter would mean holding every span + back until the call finishes. Those entries keep today's behaviour instead, where + the tenant's credentials reach the backend through per-request tracer routing and + the operator's exporter is left alone. + + A backend the request disabled dynamically, through the key's + ``litellm_disabled_callbacks`` or the ``x-litellm-disable-callbacks`` header in + ``request_headers``, resolves to no destination, so the fan-out never carries the + request tree to that account and the operator's exporter is never suppressed for + it. That leaves the request exactly where it stood before destinations existed: + the OTel V2 logger itself is not on the disable list's class registry, so its own + span still routes to the tenant's credentials the way it did then. + """ + from litellm.integrations.otel.model.config import is_otel_v2_enabled + from litellm.integrations.otel.presets.destinations import destination_for + + if not is_otel_v2_enabled(): + return () + key_entries: Final = KeyAndTeamLoggingSettings.get_key_dynamic_logging_settings(user_api_key_dict) + entries: Final = ( + key_entries + if key_entries is not None + else KeyAndTeamLoggingSettings.get_team_dynamic_logging_settings(user_api_key_dict) + ) + if not entries: + return () + disabled: Final = _dynamically_disabled_backends(user_api_key_dict, request_headers) + callbacks: Final = tuple( + callback + for item in entries + if (callback := _get_validated_callback_metadata(item=item, source="otel-destination")) is not None + if callback.callback_type != "failure" + if callback.callback_name.lower() not in disabled + ) + return tuple( + destination + for name in dict.fromkeys(callback.callback_name for callback in callbacks) + if ( + destination := destination_for( + name, + _tenant_otel_params( + MappingProxyType( + { + var: value + for callback in callbacks + if callback.callback_name == name + for var, value in callback.callback_vars.items() + } + ) + ), + _tenant_service_name(user_api_key_dict), + ) + ) + is not None + ) + + +def _tenant_service_name(user_api_key_dict: UserAPIKeyAuth) -> str | None: + """The ``service.name`` this key or team configured, the key winning over its team. + + Same fields and same precedence the request-metadata build applies, read straight + off the auth object because destinations resolve during auth, before that metadata + is assembled. + """ + sources: Final = (user_api_key_dict.metadata, user_api_key_dict.team_metadata) + return next( + ( + stripped + for source in sources + if source + for field in OTEL_SERVICE_NAME_METADATA_KEYS + if isinstance(value := source.get(field), str) and (stripped := value.strip()) + ), + None, + ) + + def clean_headers( headers: Headers, litellm_key_header_name: str | None = None, diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py new file mode 100644 index 00000000000..1799381bada --- /dev/null +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -0,0 +1,2719 @@ +"""Key/team OTLP destinations override the operator's exporters for that backend.""" + +import contextvars +import time +from collections.abc import Mapping +from types import MappingProxyType + +import pytest +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import Status, StatusCode + +import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.integrations.otel import logger as otel_logger +from litellm.integrations.otel.logger import ( + OpenTelemetryV2, + build_otel_v2_logger, + fan_out_provider, + publish_global_otel_v2_provider, +) +from litellm.integrations.otel.model.config import ( + ExporterOwner, + ExporterSpec, + OpenTelemetryV2Config, + is_otel_v2_enabled, +) +from litellm.integrations.otel.model.destination import OtelDestination +from litellm.integrations.otel.plumbing import providers as otel_providers +from litellm.integrations.otel.plumbing.context import ( + destination_backends, + request_destinations, + set_request_destinations, +) +from litellm.integrations.otel.plumbing.providers import ( + TenantFanOutSpanProcessor, + _OverriddenBackendFilter, + _sink_key, + build_tracer_provider, + deliverable_destinations, + operator_sink_keys, +) +from litellm.integrations.otel.plumbing.routing import TenantTracerCache, get_tracer +from litellm.integrations.otel.presets.arize import arize_preset +from litellm.integrations.otel.presets.destinations import ( + destination_capable_backends, + destination_for, +) +from litellm.integrations.otel.presets.langfuse import langfuse_preset +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.litellm_pre_call_utils import resolve_tenant_otel_destinations +from litellm.types.utils import StandardCallbackDynamicParams + +LANGFUSE_DEST = OtelDestination( + endpoint="http://tenant.local/api/public/otel", + headers={"Authorization": "Basic dGVuYW50"}, + callback_name="langfuse_otel", +) + + +@pytest.fixture +def allow_test_hosts(monkeypatch): + """A tenant-supplied host must be allowlisted by the operator. Allowlist the ones + these fixtures name so the resolution tests stay about resolution; + ``TestTenantHostSsrfGuard`` covers the guard itself.""" + monkeypatch.setattr( + litellm, "provider_url_destination_allowed_hosts", ["team.local", "key.local", "x"], raising=False + ) + + +@pytest.fixture(autouse=True) +def isolate_published_provider(monkeypatch): + """Publishing records the fan-out carrier in module state; one test's publish must + not become the next test's provider.""" + monkeypatch.setattr(otel_logger, "_published_v2_provider", None) + + +def in_fresh_context(fn, *args): + """Run ``fn`` in its own context so one test's destinations never leak.""" + return contextvars.copy_context().run(fn, *args) + + +def emit(provider: TracerProvider, name: str = "chat gpt-4") -> None: + with get_tracer(provider, "litellm").start_as_current_span(name): + pass + + +def wired_provider(dest_exporter: InMemorySpanExporter, global_exporter: InMemorySpanExporter) -> TracerProvider: + """The operator's provider: one owned exporter plus the tenant fan-out.""" + provider = TracerProvider() + provider.add_span_processor(_OverriddenBackendFilter(SimpleSpanProcessor(global_exporter), "langfuse_otel")) + provider.add_span_processor( + TenantFanOutSpanProcessor(processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter)) + ) + return provider + + +class TestOverrideSuppression: + def test_operator_exporter_keeps_the_span_when_no_destination_is_resolved(self): + global_exporter, dest_exporter = InMemorySpanExporter(), InMemorySpanExporter() + provider = wired_provider(dest_exporter, global_exporter) + + in_fresh_context(emit, provider) + + assert [s.name for s in global_exporter.get_finished_spans()] == ["chat gpt-4"] + assert dest_exporter.get_finished_spans() == () + + def test_operator_exporter_is_skipped_once_the_backend_is_overridden(self): + global_exporter, dest_exporter = InMemorySpanExporter(), InMemorySpanExporter() + provider = wired_provider(dest_exporter, global_exporter) + + def run(): + set_request_destinations((LANGFUSE_DEST,)) + emit(provider) + + in_fresh_context(run) + + assert global_exporter.get_finished_spans() == () + assert [s.name for s in dest_exporter.get_finished_spans()] == ["chat gpt-4"] + + def test_a_backend_the_request_did_not_override_still_exports(self): + arize_exporter, dest_exporter = InMemorySpanExporter(), InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(_OverriddenBackendFilter(SimpleSpanProcessor(arize_exporter), "arize")) + provider.add_span_processor( + TenantFanOutSpanProcessor(processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter)) + ) + + def run(): + set_request_destinations((LANGFUSE_DEST,)) + emit(provider) + + in_fresh_context(run) + + assert [s.name for s in arize_exporter.get_finished_spans()] == ["chat gpt-4"] + + +class TestRoutingMode: + """The operator's choice between replacing its own exporter and exporting alongside it. + + One org-wide backend across every team is a real deployment, and losing it the + moment a team configures its own is what ``additive`` exists to prevent. + """ + + OPERATOR_SINK = ("https://cloud.langfuse.com/api/public/otel/v1/traces", (("authorization", "Basic op"),)) + #: What a tenant destination for that same project looks like before normalizing: + #: no signal path yet, and the header name cased the way the backend writes it. + SAME_ACCOUNT_ENDPOINT = "https://cloud.langfuse.com/api/public/otel" + + @staticmethod + def _additive(monkeypatch): + monkeypatch.setattr(litellm, "otel_tenant_destination_mode", "additive", raising=False) + + @staticmethod + def _tree(provider): + tracer = get_tracer(provider, "litellm") + with tracer.start_as_current_span("POST /v1/chat/completions"): + with tracer.start_as_current_span("auth /v1/chat/completions"): + pass + with tracer.start_as_current_span("chat gpt-4"): + pass + + def _run(self, provider, destinations=(LANGFUSE_DEST,)): + def run(): + set_request_destinations(destinations) + self._tree(provider) + + in_fresh_context(run) + + def test_global_only_keeps_every_span_and_delivers_to_nobody(self): + """No team destination resolved, so the operator's backbone is untouched.""" + global_exporter, dest_exporter = InMemorySpanExporter(), InMemorySpanExporter() + provider = wired_provider(dest_exporter, global_exporter) + + self._run(provider, destinations=()) + + assert len(global_exporter.get_finished_spans()) == 3 + assert dest_exporter.get_finished_spans() == () + + def test_team_only_gets_the_whole_tree_with_no_operator_exporter(self): + """A deployment with no operator credentials still gives the team its trace.""" + dest_exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor( + TenantFanOutSpanProcessor(processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter)) + ) + + self._run(provider) + + assert {s.name for s in dest_exporter.get_finished_spans()} == { + "POST /v1/chat/completions", + "auth /v1/chat/completions", + "chat gpt-4", + } + + def test_additive_gives_the_operator_and_the_team_the_same_tree(self, monkeypatch): + self._additive(monkeypatch) + global_exporter, dest_exporter = InMemorySpanExporter(), InMemorySpanExporter() + provider = wired_provider(dest_exporter, global_exporter) + + self._run(provider) + + names = {"POST /v1/chat/completions", "auth /v1/chat/completions", "chat gpt-4"} + assert {s.name for s in global_exporter.get_finished_spans()} == names + assert {s.name for s in dest_exporter.get_finished_spans()} == names + assert len(global_exporter.get_finished_spans()) == 3, "the operator must not get a span twice" + + def test_override_moves_the_tree_off_the_operator(self): + """The default, unchanged: the tenant's traffic reaches the tenant and nowhere else.""" + global_exporter, dest_exporter = InMemorySpanExporter(), InMemorySpanExporter() + provider = wired_provider(dest_exporter, global_exporter) + + self._run(provider) + + assert global_exporter.get_finished_spans() == () + assert len(dest_exporter.get_finished_spans()) == 3 + + def test_a_team_naming_the_operators_own_project_is_written_once(self, monkeypatch): + """Fanning out to two accounts is the point. Writing the same account twice + is a duplicate the operator would see in their own project.""" + self._additive(monkeypatch) + shared = InMemorySpanExporter() + same = OtelDestination( + endpoint=self.SAME_ACCOUNT_ENDPOINT, + headers=MappingProxyType({"Authorization": "Basic op"}), + callback_name="langfuse_otel", + ) + provider = TracerProvider() + provider.add_span_processor(_OverriddenBackendFilter(SimpleSpanProcessor(shared), "langfuse_otel")) + provider.add_span_processor( + TenantFanOutSpanProcessor( + processor_factory=lambda _d: SimpleSpanProcessor(shared), + operator_sinks=frozenset({self.OPERATOR_SINK}), + ) + ) + + self._run(provider, destinations=(same,)) + + assert len(shared.get_finished_spans()) == 3, "the same account received the trace twice" + + def test_in_override_a_team_naming_the_operators_project_still_gets_the_trace(self): + """Override suppresses the operator's own exporter, so the fan-out is the only + thing left delivering. Skipping it on a matching account leaves the team with + nothing at all.""" + shared = InMemorySpanExporter() + same = OtelDestination( + endpoint=self.SAME_ACCOUNT_ENDPOINT, + headers=MappingProxyType({"Authorization": "Basic op"}), + callback_name="langfuse_otel", + ) + provider = TracerProvider() + provider.add_span_processor(_OverriddenBackendFilter(SimpleSpanProcessor(shared), "langfuse_otel")) + provider.add_span_processor( + TenantFanOutSpanProcessor( + processor_factory=lambda _d: SimpleSpanProcessor(shared), + operator_sinks=frozenset({self.OPERATOR_SINK}), + ) + ) + + self._run(provider, destinations=(same,)) + + assert len(shared.get_finished_spans()) == 3, "the team's own destination received nothing" + + def test_a_team_naming_a_different_project_still_gets_its_copy(self, monkeypatch): + """The dedup keys on the account, so a second project is still a second copy.""" + self._additive(monkeypatch) + global_exporter, dest_exporter = InMemorySpanExporter(), InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(_OverriddenBackendFilter(SimpleSpanProcessor(global_exporter), "langfuse_otel")) + provider.add_span_processor( + TenantFanOutSpanProcessor( + processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter), + operator_sinks=frozenset({self.OPERATOR_SINK}), + ) + ) + + self._run(provider) + + assert len(global_exporter.get_finished_spans()) == 3 + assert len(dest_exporter.get_finished_spans()) == 3 + + @pytest.mark.parametrize("additive", [True, False]) + def test_a_failing_team_destination_leaves_the_operator_alone(self, monkeypatch, additive): + """A tenant collector that raises on every span must not cost the operator + its own telemetry, nor take the request down with it.""" + if additive: + self._additive(monkeypatch) + global_exporter, arize_exporter = InMemorySpanExporter(), InMemorySpanExporter() + + class Exploding(SimpleSpanProcessor): + def on_end(self, span): + raise RuntimeError("tenant collector is down") + + provider = TracerProvider() + provider.add_span_processor(_OverriddenBackendFilter(SimpleSpanProcessor(global_exporter), "langfuse_otel")) + provider.add_span_processor(_OverriddenBackendFilter(SimpleSpanProcessor(arize_exporter), "arize")) + provider.add_span_processor( + TenantFanOutSpanProcessor(processor_factory=lambda _d: Exploding(InMemorySpanExporter())) + ) + + self._run(provider) + + assert len(arize_exporter.get_finished_spans()) == 3, "an unrelated backend lost spans" + assert len(global_exporter.get_finished_spans()) == (3 if additive else 0) + + def test_the_env_var_turns_additive_on_without_a_config_file(self, monkeypatch): + monkeypatch.setattr(litellm, "otel_tenant_destination_mode", None, raising=False) + monkeypatch.setenv("LITELLM_OTEL_TENANT_DESTINATION_MODE", "Additive") + global_exporter, dest_exporter = InMemorySpanExporter(), InMemorySpanExporter() + provider = wired_provider(dest_exporter, global_exporter) + + self._run(provider) + + assert len(global_exporter.get_finished_spans()) == 3 + assert len(dest_exporter.get_finished_spans()) == 3 + + def test_an_unrecognized_mode_stays_on_override(self, monkeypatch): + monkeypatch.setattr(litellm, "otel_tenant_destination_mode", "both", raising=False) + global_exporter, dest_exporter = InMemorySpanExporter(), InMemorySpanExporter() + provider = wired_provider(dest_exporter, global_exporter) + + self._run(provider) + + assert global_exporter.get_finished_spans() == () + + def test_operator_sink_keys_skips_an_exporter_with_no_endpoint_of_its_own(self): + """Such an exporter resolves its endpoint from the environment at export + time, so it has no identity to compare a destination against.""" + config = OpenTelemetryV2Config( + exporters=( + ExporterSpec(kind="otlp_http", endpoint=self.OPERATOR_SINK[0], headers="authorization=Basic op"), + ExporterSpec(kind="otlp_http", endpoint=None, headers="authorization=Basic other"), + ) + ) + + assert operator_sink_keys(config) == frozenset({self.OPERATOR_SINK}) + + def test_operator_sink_keys_skips_exporters_that_never_reach_the_wire(self): + """A console kind ignores the endpoint and a header-gated spec with no + credentials is dropped when the provider is built, so treating either as an + account the operator writes to would silently withhold a team's own spans + under additive.""" + config = OpenTelemetryV2Config( + exporters=( + ExporterSpec(kind="otlp_http", endpoint=self.OPERATOR_SINK[0], headers="authorization=Basic op"), + ExporterSpec(kind="console", endpoint="http://team.local/v1/traces"), + ExporterSpec(kind="otlp_http", endpoint="http://gated.local/v1/traces", requires_headers=True), + ) + ) + + assert operator_sink_keys(config) == frozenset({self.OPERATOR_SINK}) + + def test_operator_sink_keys_spans_every_config_it_is_handed(self): + first = OpenTelemetryV2Config( + exporters=( + ExporterSpec( + kind="otlp_http", + endpoint=self.OPERATOR_SINK[0], + headers="authorization=Basic op", + ), + ) + ) + second = OpenTelemetryV2Config( + exporters=( + ExporterSpec( + kind="otlp_http", + endpoint="https://otlp.arize.com/v1/traces", + headers="space_id=s,api_key=k", + ), + ) + ) + + assert operator_sink_keys(first, second) == { + self.OPERATOR_SINK, + _sink_key("https://otlp.arize.com/v1/traces", {"space_id": "s", "api_key": "k"}), + } + + def test_a_team_pointing_at_a_credential_less_operator_exporter_still_gets_its_spans(self, monkeypatch): + """Under additive the fan-out skips a destination the operator already writes + to. An exporter the provider never built writes nothing, so skipping it would + cost the team every span.""" + monkeypatch.setenv("LITELLM_OTEL_TENANT_DESTINATION_MODE", "additive") + gated_endpoint = "http://gated.local/v1/traces" + destination = OtelDestination(endpoint=gated_endpoint, callback_name="newrelic") + config = OpenTelemetryV2Config( + exporters=(ExporterSpec(kind="otlp_http", endpoint=gated_endpoint, requires_headers=True),) + ) + dest_exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor( + TenantFanOutSpanProcessor( + processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter), + operator_sinks=operator_sink_keys(config), + ) + ) + + def run(): + set_request_destinations((destination,)) + emit(provider) + + in_fresh_context(run) + + assert [s.name for s in dest_exporter.get_finished_spans()] == ["chat gpt-4"] + + def test_the_operators_own_langfuse_and_a_team_naming_it_are_one_account(self, monkeypatch): + """The two sides are built by different code that writes the endpoint and the + header names differently, so comparing them raw silently never matches.""" + monkeypatch.setenv("LANGFUSE_HOST", "https://lf.internal") + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-op") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-op") + monkeypatch.setattr(litellm, "provider_url_destination_allowed_hosts", ["lf.internal"], raising=False) + operator = operator_sink_keys(langfuse_preset()) + + def sink(public_key, secret_key): + destination = destination_for( + "langfuse_otel", + StandardCallbackDynamicParams( + langfuse_public_key=public_key, + langfuse_secret_key=secret_key, + langfuse_host="https://lf.internal", + ), + ) + assert destination is not None + return _sink_key(destination.endpoint, destination.headers) + + assert sink("pk-op", "sk-op") in operator, "a team naming the operator's own project" + assert sink("pk-team", "sk-team") not in operator, "a different project on the same server" + + def test_two_accounts_holding_the_same_strings_in_different_roles_are_not_one(self): + """The values alone are not the identity. Two accounts can hold the same pair + of strings with the space id and the api key the other way round, and folding + them together would leave the second one's team with no trace at all.""" + endpoint = "https://otlp.arize.com/v1" + + assert _sink_key(endpoint, {"space_id": "a", "api_key": "b"}) != _sink_key( + endpoint, {"space_id": "b", "api_key": "a"} + ) + + def test_the_operators_own_arize_space_and_a_team_naming_it_are_one_account(self, monkeypatch): + """One account answers to two header names here: the operator's exporter sends + ``space_id`` and a team destination sends ``arize-space-id``. Keyed on the names, + additive would write the operator's own space twice for every request.""" + monkeypatch.setenv("ARIZE_SPACE_ID", "space-op") + monkeypatch.setenv("ARIZE_API_KEY", "key-op") + monkeypatch.delenv("ARIZE_SPACE_KEY", raising=False) + operator = operator_sink_keys(arize_preset()) + + def sink(space, api_key): + destination = destination_for( + "arize", + StandardCallbackDynamicParams(arize_space_key=space, arize_api_key=api_key), + ) + assert destination is not None + return _sink_key(destination.endpoint, destination.headers) + + assert sink("space-op", "key-op") in operator, "a team naming the operator's own space" + assert sink("space-team", "key-team") not in operator, "a different Arize space" + + +class TestFanOut: + def test_every_span_of_the_request_reaches_the_destination_in_one_trace(self): + """The whole tree, gen-AI span included, parented as the operator would see it.""" + dest_exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor( + TenantFanOutSpanProcessor(processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter)) + ) + tracer = get_tracer(provider, "litellm") + + def run(): + set_request_destinations((LANGFUSE_DEST,)) + with tracer.start_as_current_span("POST /v1/chat/completions"): + with tracer.start_as_current_span("auth /v1/chat/completions"): + pass + with tracer.start_as_current_span("chat gpt-4"): + pass + + in_fresh_context(run) + + spans = dest_exporter.get_finished_spans() + by_name = {s.name: s for s in spans} + assert set(by_name) == {"POST /v1/chat/completions", "auth /v1/chat/completions", "chat gpt-4"} + root = by_name["POST /v1/chat/completions"] + assert len({s.context.trace_id for s in spans}) == 1, "the tenant must receive one connected trace" + for child in ("auth /v1/chat/completions", "chat gpt-4"): + assert by_name[child].parent.span_id == root.context.span_id + + def test_a_team_naming_two_backends_gets_the_trace_at_both(self): + """The fan-out rides one provider, so it cannot skip a destination on the + grounds that some other backend owns it: nothing else would deliver it.""" + langfuse, arize = InMemorySpanExporter(), InMemorySpanExporter() + by_endpoint = {"http://a.local": langfuse, "http://b.local": arize} + provider = TracerProvider() + provider.add_span_processor( + TenantFanOutSpanProcessor(processor_factory=lambda d: SimpleSpanProcessor(by_endpoint[d.endpoint])) + ) + + def run(): + set_request_destinations( + ( + OtelDestination(endpoint="http://a.local", callback_name="langfuse_otel"), + OtelDestination(endpoint="http://b.local", callback_name="arize"), + ) + ) + emit(provider) + + in_fresh_context(run) + + assert [s.name for s in langfuse.get_finished_spans()] == ["chat gpt-4"] + assert [s.name for s in arize.get_finished_spans()] == ["chat gpt-4"] + + def test_a_destination_carries_the_tenants_service_name(self): + """An overridden backend skips per-request tracer routing, so the service name + that route used to apply has to travel on the destination instead.""" + dest = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(TenantFanOutSpanProcessor(processor_factory=lambda _d: SimpleSpanProcessor(dest))) + + def run(): + set_request_destinations( + ( + OtelDestination( + endpoint="http://a.local", + callback_name="langfuse_otel", + resource_attributes={"service.name": "team-checkout"}, + ), + ) + ) + emit(provider) + + in_fresh_context(run) + + assert {s.resource.attributes["service.name"] for s in dest.get_finished_spans()} == {"team-checkout"} + + def test_the_operators_database_endpoint_does_not_ride_along_to_the_tenant(self): + """A database span describes the proxy's own Postgres, so the tenant gets the + span and its timing without the host, the port, the schema or the error text + that names them. The operator's own copy keeps everything.""" + dest_exporter, operator_exporter = InMemorySpanExporter(), InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(operator_exporter)) + provider.add_span_processor( + TenantFanOutSpanProcessor(processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter)) + ) + tracer = get_tracer(provider, "litellm") + unreachable = "Can't reach database server at db.internal.example:15400" + + def run(): + set_request_destinations((LANGFUSE_DEST,)) + with tracer.start_as_current_span("postgres get_data") as db_span: + db_span.set_attributes( + { + "db.system.name": "postgresql", + "db.system": "postgresql", + "db.operation.name": "get_data", + "server.address": "db.internal.example", + "server.port": 15400, + "db.namespace": "litellm", + "error.type": "PrismaError", + "error.message": unreachable, + "error": unreachable, + "litellm.provider.error.stack_trace": f"Traceback: {unreachable}", + } + ) + db_span.add_event("exception", {"exception.message": unreachable}) + db_span.set_status(Status(StatusCode.ERROR, unreachable)) + with tracer.start_as_current_span("chat claude-haiku") as llm_span: + llm_span.set_attribute("server.address", "api.anthropic.com") + + in_fresh_context(run) + + tenant = {s.name: s for s in dest_exporter.get_finished_spans()} + operator = {s.name: s for s in operator_exporter.get_finished_spans()} + assert set(tenant) == {"postgres get_data", "chat claude-haiku"}, "the tenant keeps the whole tree" + tenant_db = tenant["postgres get_data"] + assert dict(tenant_db.attributes) == { + "db.system.name": "postgresql", + "db.system": "postgresql", + "db.operation.name": "get_data", + "error.type": "PrismaError", + } + assert list(tenant_db.events) == [] + assert tenant_db.status.status_code is StatusCode.ERROR, "the tenant still sees that the call failed" + assert tenant_db.status.description is None + assert "db.internal.example" not in tenant_db.to_json() + assert tenant["chat claude-haiku"].attributes["server.address"] == "api.anthropic.com", ( + "only the operator's datastore is redacted, never the model endpoint" + ) + operator_db = operator["postgres get_data"] + assert operator_db.attributes["server.address"] == "db.internal.example" + assert operator_db.attributes["server.port"] == 15400 + assert operator_db.attributes["db.namespace"] == "litellm" + assert operator_db.attributes["error.message"] == unreachable + assert operator_db.attributes["error"] == unreachable + assert operator_db.status.description == unreachable + assert [event.name for event in operator_db.events] == ["exception"] + + @pytest.mark.parametrize("failure_status", ["guardrail_failed_to_respond", "failure"]) + def test_a_guardrails_failure_text_does_not_ride_along_to_the_tenant(self, failure_status): + dest_exporter, operator_exporter = InMemorySpanExporter(), InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(operator_exporter)) + provider.add_span_processor( + TenantFanOutSpanProcessor(processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter)) + ) + tracer = get_tracer(provider, "litellm") + unreachable = "Cannot connect to host guardrail.internal.example:9000" + verdict = '{"action": "block", "categories": ["pii"]}' + + def run(): + set_request_destinations((LANGFUSE_DEST,)) + with tracer.start_as_current_span("POST /v1/chat/completions"): + with tracer.start_as_current_span("execute_guardrail pii") as down: + down.set_attributes( + { + "litellm.guardrail.name": "pii", + "litellm.guardrail.status": failure_status, + "litellm.guardrail.response": unreachable, + } + ) + with tracer.start_as_current_span("execute_guardrail toxicity") as up: + up.set_attributes( + { + "litellm.guardrail.name": "toxicity", + "litellm.guardrail.status": "guardrail_intervened", + "litellm.guardrail.response": verdict, + } + ) + + in_fresh_context(run) + + tenant = {s.name: s for s in dest_exporter.get_finished_spans()} + operator = {s.name: s for s in operator_exporter.get_finished_spans()} + assert dict(tenant["execute_guardrail pii"].attributes) == { + "litellm.guardrail.name": "pii", + "litellm.guardrail.status": failure_status, + } + assert "guardrail.internal.example" not in tenant["execute_guardrail pii"].to_json() + assert tenant["execute_guardrail toxicity"].attributes["litellm.guardrail.response"] == verdict + assert operator["execute_guardrail pii"].attributes["litellm.guardrail.response"] == unreachable + + def test_the_callers_key_in_the_query_string_does_not_ride_along_to_the_tenant(self): + """A Google AI Studio style request authenticates with ``?key=``, + and the instrumentor stamps the full request URL on the server span. The + tenant keeps the URL up to the query string, and the operator's copy keeps it + whole.""" + dest_exporter, operator_exporter = InMemorySpanExporter(), InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(operator_exporter)) + provider.add_span_processor( + TenantFanOutSpanProcessor(processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter)) + ) + tracer = get_tracer(provider, "litellm") + path = "/v1beta/models/gemini-2.5-flash:generateContent" + query = "key=sk-another-members-virtual-key&alt=sse" + + def run(): + set_request_destinations((LANGFUSE_DEST,)) + with tracer.start_as_current_span(f"POST {path}") as server_span: + server_span.set_attributes( + { + "http.method": "POST", + "http.route": path, + "http.target": f"{path}?{query}", + "http.url": f"http://proxy.example:4000{path}?{query}", + "url.path": path, + "url.query": query, + "http.status_code": 200, + } + ) + with tracer.start_as_current_span("generate_content gemini-2.5-flash") as llm_span: + llm_span.set_attributes( + { + "gen_ai.operation.name": "generate_content", + "url.full": f"https://generativelanguage.googleapis.com{path}?key=AIza-operator-provider-key", + } + ) + + in_fresh_context(run) + + tenant = {s.name: s for s in dest_exporter.get_finished_spans()} + assert dict(tenant[f"POST {path}"].attributes) == { + "http.method": "POST", + "http.route": path, + "http.target": path, + "http.url": f"http://proxy.example:4000{path}", + "url.path": path, + "http.status_code": 200, + } + assert "sk-another-members-virtual-key" not in tenant[f"POST {path}"].to_json() + assert tenant["generate_content gemini-2.5-flash"].attributes["url.full"] == ( + f"https://generativelanguage.googleapis.com{path}" + ), "the tenant's own span keeps its error text, and still loses a query string" + operator = {s.name: s for s in operator_exporter.get_finished_spans()} + assert operator[f"POST {path}"].attributes["http.url"] == f"http://proxy.example:4000{path}?{query}" + assert operator[f"POST {path}"].attributes["url.query"] == query + assert "AIza-operator-provider-key" in operator["generate_content gemini-2.5-flash"].to_json() + + def test_captured_request_headers_do_not_ride_along_to_the_tenant(self): + """With ``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST`` set, the + server span carries the caller's bearer token. A team admin's collector must + not receive it, while the operator's own copy keeps it and the tenant keeps the + rest of the span.""" + dest_exporter, operator_exporter = InMemorySpanExporter(), InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(operator_exporter)) + provider.add_span_processor( + TenantFanOutSpanProcessor(processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter)) + ) + tracer = get_tracer(provider, "litellm") + bearer = "Bearer sk-another-members-virtual-key" + + def run(): + set_request_destinations((LANGFUSE_DEST,)) + with tracer.start_as_current_span("POST /v1/chat/completions") as server_span: + server_span.set_attributes( + { + "http.request.method": "POST", + "http.route": "/v1/chat/completions", + "http.request.header.authorization": (bearer,), + "http.request.header.x_litellm_api_key": (bearer,), + "http.response.header.set_cookie": ("session=abc",), + } + ) + server_span.set_status(Status(StatusCode.ERROR)) + + in_fresh_context(run) + + tenant = dest_exporter.get_finished_spans()[0] + assert dict(tenant.attributes) == {"http.request.method": "POST", "http.route": "/v1/chat/completions"} + assert bearer not in tenant.to_json() + assert tenant.status.status_code is StatusCode.ERROR + operator = operator_exporter.get_finished_spans()[0] + assert operator.attributes["http.request.header.authorization"] == (bearer,) + assert operator.attributes["http.response.header.set_cookie"] == ("session=abc",) + + def test_the_proxys_own_error_text_does_not_ride_along_to_the_tenant(self): + """Postgres failing during auth surfaces as a ``ProxyException`` whose message + quotes the Prisma error, so the auth span and the request root carry the + operator's database endpoint in ``error.message``, in the exception event and + in the status description. None of it is the tenant's, so it all comes off, + while the failure itself (its type, its code, its status) stays. The tenant's + own model call keeps its error text, less the stack trace that walks the + operator's install. The operator's copy keeps everything.""" + dest_exporter, operator_exporter = InMemorySpanExporter(), InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(operator_exporter)) + provider.add_span_processor( + TenantFanOutSpanProcessor(processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter)) + ) + tracer = get_tracer(provider, "litellm") + unreachable = "Authentication Error, Can't reach database server at db.internal.example:15400" + install = "/srv/litellm/.venv/lib/python3.13/site-packages/opentelemetry/trace/__init__.py" + provider_error = "AnthropicException - invalid x-api-key" + + def fail(span, message: str) -> None: + span.set_attributes( + { + "error.type": "ProxyException", + "error.message": message, + "litellm.provider.error.code": "500", + "litellm.provider.error.stack_trace": f"Traceback\n File {install}\n{message}", + } + ) + span.add_event( + "exception", + {"exception.type": "ProxyException", "exception.message": message, "exception.stacktrace": install}, + ) + span.set_status(Status(StatusCode.ERROR, message)) + + def run(): + set_request_destinations((LANGFUSE_DEST,)) + with tracer.start_as_current_span("POST /v1/chat/completions") as root: + with tracer.start_as_current_span("auth /v1/chat/completions") as auth: + fail(auth, unreachable) + with tracer.start_as_current_span("chat claude-haiku") as llm: + llm.set_attribute("gen_ai.operation.name", "chat") + fail(llm, provider_error) + fail(root, unreachable) + + in_fresh_context(run) + + tenant = {s.name: s for s in dest_exporter.get_finished_spans()} + operator = {s.name: s for s in operator_exporter.get_finished_spans()} + assert set(tenant) == {"POST /v1/chat/completions", "auth /v1/chat/completions", "chat claude-haiku"} + for name in ("POST /v1/chat/completions", "auth /v1/chat/completions"): + proxy_span = tenant[name] + assert dict(proxy_span.attributes) == {"error.type": "ProxyException", "litellm.provider.error.code": "500"} + assert list(proxy_span.events) == [] + assert proxy_span.status.status_code is StatusCode.ERROR + assert proxy_span.status.description is None + assert "db.internal.example" not in proxy_span.to_json() + assert install not in proxy_span.to_json() + llm_span = tenant["chat claude-haiku"] + assert llm_span.attributes["error.message"] == provider_error, "the tenant's own call keeps its error text" + assert "litellm.provider.error.stack_trace" not in llm_span.attributes + assert llm_span.status.description == provider_error + assert [dict(event.attributes) for event in llm_span.events] == [ + {"exception.type": "ProxyException", "exception.message": provider_error} + ] + assert install not in llm_span.to_json() + for name, message in (("auth /v1/chat/completions", unreachable), ("chat claude-haiku", provider_error)): + assert operator[name].attributes["error.message"] == message + assert install in operator[name].attributes["litellm.provider.error.stack_trace"] + assert operator[name].events[0].attributes["exception.stacktrace"] == install + assert operator[name].status.description == message + + def test_a_tenants_service_name_is_layered_onto_the_operators_resource(self): + """The destination's ``service.name`` replaces the operator's on the tenant's + copy and every other resource attribute travels unchanged. Nothing is detected + afresh per span, so no attribute the operator did not configure appears.""" + dest = InMemorySpanExporter() + provider = TracerProvider( + resource=Resource({"service.name": "litellm-proxy", "deployment.environment.name": "prod"}) + ) + provider.add_span_processor(TenantFanOutSpanProcessor(processor_factory=lambda _d: SimpleSpanProcessor(dest))) + + def run(): + set_request_destinations( + ( + OtelDestination( + endpoint="http://a.local", + callback_name="langfuse_otel", + resource_attributes={"service.name": "team-checkout"}, + ), + ) + ) + emit(provider) + + in_fresh_context(run) + + (span,) = dest.get_finished_spans() + assert dict(span.resource.attributes) == { + "service.name": "team-checkout", + "deployment.environment.name": "prod", + } + + def test_a_destination_that_cannot_build_a_processor_is_skipped_quietly(self): + """An unbuildable destination must not cost the caller its request.""" + attempts = [] + reached_the_end = [] + + def factory(destination): + attempts.append(destination.endpoint) + + provider = TracerProvider() + provider.add_span_processor(TenantFanOutSpanProcessor(processor_factory=factory)) + + def run(): + set_request_destinations((LANGFUSE_DEST,)) + emit(provider) + reached_the_end.append(True) + + in_fresh_context(run) + + assert attempts == [LANGFUSE_DEST.endpoint] + assert reached_the_end == [True] + + def test_an_unbuildable_destination_leaves_the_span_with_the_operator(self): + """Anchoring the destination is what makes the operator's exporter stand down + for the backend, so a destination nothing can deliver to must never be anchored, + or the span reaches neither account.""" + global_exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(_OverriddenBackendFilter(SimpleSpanProcessor(global_exporter), "langfuse_otel")) + provider.add_span_processor(TenantFanOutSpanProcessor(processor_factory=lambda _d: None)) + + def run(): + set_request_destinations(deliverable_destinations((LANGFUSE_DEST,), provider)) + emit(provider) + return request_destinations() + + anchored = in_fresh_context(run) + + assert anchored == () + assert [s.name for s in global_exporter.get_finished_spans()] == ["chat gpt-4"] + + def test_a_buildable_destination_is_still_anchored_and_still_overrides(self): + global_exporter, dest_exporter = InMemorySpanExporter(), InMemorySpanExporter() + provider = wired_provider(dest_exporter, global_exporter) + + def run(): + set_request_destinations(deliverable_destinations((LANGFUSE_DEST,), provider)) + emit(provider) + return request_destinations() + + anchored = in_fresh_context(run) + + assert anchored == (LANGFUSE_DEST,) + assert global_exporter.get_finished_spans() == () + assert [s.name for s in dest_exporter.get_finished_spans()] == ["chat gpt-4"] + + def test_only_the_unbuildable_destination_is_dropped_from_a_mixed_set(self): + dest_exporter = InMemorySpanExporter() + other = LANGFUSE_DEST.model_copy(update={"endpoint": "http://broken.local/otel"}) + fan_out = TenantFanOutSpanProcessor( + processor_factory=lambda d: None if d.endpoint == other.endpoint else SimpleSpanProcessor(dest_exporter) + ) + + assert fan_out.deliverable((other, LANGFUSE_DEST)) == (LANGFUSE_DEST,) + + def test_no_fan_out_means_nothing_is_anchored(self): + """With nothing to carry the spans to the tenant, anchoring would only stop the + operator's exporter from writing them.""" + provider = TracerProvider() + + assert deliverable_destinations((LANGFUSE_DEST,), provider) == () + + def test_a_protocol_with_no_otlp_transport_is_not_deliverable(self): + """An unknown exporter kind falls back to the console exporter, which ignores the + tenant's credentials and prints its spans to the proxy's stdout. Treating that as + deliverable would stand the operator's exporter down for spans nobody stores.""" + typo = LANGFUSE_DEST.model_copy(update={"protocol": "consle"}) + fan_out = TenantFanOutSpanProcessor() + try: + assert fan_out.deliverable((typo, LANGFUSE_DEST)) == (LANGFUSE_DEST,) + finally: + fan_out.shutdown() + + def test_a_closed_fan_out_anchors_nothing(self): + fan_out = TenantFanOutSpanProcessor(processor_factory=lambda _d: SimpleSpanProcessor(InMemorySpanExporter())) + provider = TracerProvider() + provider.add_span_processor(fan_out) + fan_out.shutdown() + + assert deliverable_destinations((LANGFUSE_DEST,), provider) == () + + def test_the_processor_built_to_check_deliverability_is_the_one_that_exports(self): + built = [] + + def factory(_destination): + built.append(SimpleSpanProcessor(InMemorySpanExporter())) + return built[-1] + + provider = TracerProvider() + provider.add_span_processor(TenantFanOutSpanProcessor(processor_factory=factory)) + + def run(): + set_request_destinations(deliverable_destinations((LANGFUSE_DEST,), provider)) + emit(provider) + + in_fresh_context(run) + + assert len(built) == 1 + + def test_one_processor_is_reused_across_spans_of_the_same_destination(self): + built = [] + + def factory(_destination): + processor = SimpleSpanProcessor(InMemorySpanExporter()) + built.append(processor) + return processor + + provider = TracerProvider() + provider.add_span_processor(TenantFanOutSpanProcessor(processor_factory=factory)) + + def run(): + set_request_destinations((LANGFUSE_DEST,)) + emit(provider, "one") + emit(provider, "two") + + in_fresh_context(run) + + assert len(built) == 1 + + +class TestProviderWiring: + def test_build_tracer_provider_only_filters_when_asked(self): + config = OpenTelemetryV2Config(exporters=[ExporterSpec(kind="in_memory", owner=ExporterOwner.LANGFUSE_OTEL)]) + operator = build_tracer_provider(config, tenant_overrides=True) + tenant = build_tracer_provider(config) + + def kinds(provider): + return [type(p).__name__ for p in provider._active_span_processor._span_processors] + + assert "_OverriddenBackendFilter" in kinds(operator) + assert "_OverriddenBackendFilter" not in kinds(tenant), "a per-tenant provider must not filter itself out" + assert "TenantFanOutSpanProcessor" not in kinds(operator), "delivery belongs to the published global alone" + assert "TenantFanOutSpanProcessor" not in kinds(tenant) + + def test_only_the_published_global_provider_delivers_to_tenants(self): + """A second v2 logger's provider never sees the server, auth or database spans, + so fanning out from it would hand the tenant a one-span trace. Publishing is + what picks the one provider the whole request tree passes through.""" + config = OpenTelemetryV2Config(exporters=[ExporterSpec(kind="in_memory", owner=ExporterOwner.ARIZE_AX)]) + published, other = OpenTelemetryV2(config=config, callback_name="arize"), OpenTelemetryV2(config=config) + + publish_global_otel_v2_provider([other], lambda _p: None, registered=published) + + def kinds(logger): + return [type(p).__name__ for p in logger._tracer_provider._active_span_processor._span_processors] + + assert kinds(published).count("TenantFanOutSpanProcessor") == 1 + assert "TenantFanOutSpanProcessor" not in kinds(other) + + @pytest.mark.parametrize("canonical", ["langfuse_otel", "arize"]) + def test_publishing_tells_the_fan_out_about_every_v2_loggers_account(self, monkeypatch, canonical): + monkeypatch.setenv("LITELLM_OTEL_TENANT_DESTINATION_MODE", "additive") + shared = InMemorySpanExporter() + monkeypatch.setattr(otel_providers, "_destination_processor", lambda _d: SimpleSpanProcessor(shared)) + accounts = { + "langfuse_otel": ( + "https://cloud.langfuse.com/api/public/otel/v1/traces", + "authorization=Basic op", + ), + "arize": ( + "https://otlp.arize.com/v1/traces", + "space_id=space-op,api_key=key-op", + ), + } + loggers = { + name: OpenTelemetryV2( + config=OpenTelemetryV2Config( + exporters=(ExporterSpec(kind="otlp_http", endpoint=endpoint, headers=headers),) + ), + callback_name=name, + tracer_provider=TracerProvider(), + ) + for name, (endpoint, headers) in accounts.items() + } + other = "arize" if canonical == "langfuse_otel" else "langfuse_otel" + published = publish_global_otel_v2_provider( + [loggers[other]], + lambda _p: None, + registered=loggers[canonical], + ) + + def destination(name, headers): + return OtelDestination(endpoint=accounts[name][0], headers=headers, callback_name=name) + + def run(destinations): + set_request_destinations(destinations) + emit(published.tracer_provider) + + in_fresh_context(run, (destination(canonical, dict(pair.split("=") for pair in accounts[canonical][1].split(","))),)) + in_fresh_context(run, (destination(other, dict(pair.split("=") for pair in accounts[other][1].split(","))),)) + assert shared.get_finished_spans() == (), "an account the operator already writes to was written twice" + + in_fresh_context(run, (destination(other, {"authorization": "Basic team"}),)) + assert [s.name for s in shared.get_finished_spans()] == ["chat gpt-4"] + + def test_publishing_twice_does_not_double_export(self): + config = OpenTelemetryV2Config(exporters=[ExporterSpec(kind="in_memory", owner=ExporterOwner.ARIZE_AX)]) + logger = OpenTelemetryV2(config=config, callback_name="arize") + + publish_global_otel_v2_provider([], lambda _p: None, registered=logger) + publish_global_otel_v2_provider([], lambda _p: None, registered=logger) + + kinds = [type(p).__name__ for p in logger._tracer_provider._active_span_processor._span_processors] + assert kinds.count("TenantFanOutSpanProcessor") == 1 + + def test_anchoring_reads_the_fan_out_off_the_published_provider_not_the_otel_global(self, monkeypatch): + """``set_tracer_provider`` keeps the first provider it was handed. When + auto-instrumentation or a legacy logger claimed it before the proxy published, + the OTel global carries no fan-out, so reading it there would refuse every + destination the published provider delivers.""" + from litellm.proxy import proxy_server + + config = OpenTelemetryV2Config(exporters=[ExporterSpec(kind="in_memory", owner=ExporterOwner.LANGFUSE_OTEL)]) + logger = OpenTelemetryV2(config=config, callback_name="langfuse_otel") + publish_global_otel_v2_provider([], lambda _p: None, registered=logger) + monkeypatch.setattr(proxy_server, "open_telemetry_logger", logger) + claimed_first = TracerProvider() + + assert fan_out_provider() is logger.tracer_provider + assert deliverable_destinations((LANGFUSE_DEST,), claimed_first) == () + assert deliverable_destinations((LANGFUSE_DEST,), fan_out_provider()) == (LANGFUSE_DEST,) + + def test_a_legacy_v1_logger_holding_the_registered_slot_does_not_hide_the_fan_out(self, monkeypatch): + """The proxy publishes with ``registered=None`` when ``open_telemetry_logger`` + holds a v1 logger, so the fan-out lands on a v2 logger taken from + ``_in_memory_loggers``. Reading the registered slot finds no v2 logger there and + the OTel global belongs to v1, so both detours refuse every destination the + published provider delivers.""" + from litellm.integrations.opentelemetry import OpenTelemetry + from litellm.proxy import proxy_server + + config = OpenTelemetryV2Config(exporters=[ExporterSpec(kind="in_memory", owner=ExporterOwner.LANGFUSE_OTEL)]) + v2 = OpenTelemetryV2(config=config, callback_name="langfuse_otel") + publish_global_otel_v2_provider([v2], lambda _p: None, registered=None) + monkeypatch.setattr(proxy_server, "open_telemetry_logger", OpenTelemetry()) + + assert fan_out_provider() is v2.tracer_provider + assert deliverable_destinations((LANGFUSE_DEST,), fan_out_provider()) == (LANGFUSE_DEST,) + + def test_without_a_publish_anchoring_attaches_fan_out_to_registered_v2_logger(self, monkeypatch): + from litellm.proxy import proxy_server + + config = OpenTelemetryV2Config(exporters=[ExporterSpec(kind="in_memory", owner=ExporterOwner.LANGFUSE_OTEL)]) + logger = OpenTelemetryV2(config=config, callback_name="langfuse_otel") + monkeypatch.setattr(proxy_server, "open_telemetry_logger", logger) + + assert fan_out_provider() is logger.tracer_provider + assert deliverable_destinations((LANGFUSE_DEST,), fan_out_provider()) == (LANGFUSE_DEST,) + + def test_concurrent_anchoring_attaches_exactly_one_fan_out(self): + """Requests race to anchor when the startup publish never ran, and a fan-out + attached twice delivers every tenant span twice.""" + import threading + + from litellm.integrations.otel.plumbing.providers import attach_tenant_fan_out + + class SlowAttachProvider(TracerProvider): + def add_span_processor(self, span_processor): + time.sleep(0.05) + super().add_span_processor(span_processor) + + provider = SlowAttachProvider() + config = OpenTelemetryV2Config(exporters=[ExporterSpec(kind="in_memory", owner=ExporterOwner.LANGFUSE_OTEL)]) + barrier = threading.Barrier(8) + + def anchor(): + barrier.wait(timeout=10) + attach_tenant_fan_out(provider, config) + + threads = [threading.Thread(target=anchor) for _ in range(8)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=10) + + kinds = [type(p).__name__ for p in provider._active_span_processor._span_processors] + assert kinds.count("TenantFanOutSpanProcessor") == 1, f"one fan-out per provider, got {kinds}" + + def test_without_a_publish_anchoring_falls_back_to_the_otel_global(self, monkeypatch): + from opentelemetry import trace + + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "open_telemetry_logger", None) + + assert fan_out_provider() is trace.get_tracer_provider() + + def test_auth_seeds_the_request_with_destinations_the_registered_logger_can_deliver( + self, monkeypatch, allow_test_hosts + ): + from litellm.proxy import proxy_server + from litellm.proxy.auth.user_api_key_auth import _seed_request_destinations + + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + config = OpenTelemetryV2Config(exporters=[ExporterSpec(kind="in_memory", owner=ExporterOwner.LANGFUSE_OTEL)]) + logger = OpenTelemetryV2(config=config, callback_name="langfuse_otel") + publish_global_otel_v2_provider([], lambda _p: None, registered=logger) + monkeypatch.setattr(proxy_server, "open_telemetry_logger", logger) + auth = UserAPIKeyAuth( + team_metadata={ + "logging": [ + { + "callback_name": "langfuse_otel", + "callback_type": "success", + "callback_vars": { + "langfuse_public_key": "pk-team", + "langfuse_secret_key": "sk-team", + "langfuse_host": "http://team.local", + }, + } + ] + } + ) + expected = resolve_tenant_otel_destinations(auth) + assert expected, "the fixture must resolve to a destination for the test to mean anything" + + def run(): + _seed_request_destinations(auth) + return request_destinations() + + assert deliverable_destinations(expected, TracerProvider()) == () + assert in_fresh_context(run) == expected + + +class TestRouting: + def test_an_overridden_backend_is_not_detached_onto_a_second_provider(self): + config = OpenTelemetryV2Config( + exporters=[ExporterSpec(kind="otlp_http", endpoint="http://op.local", owner=ExporterOwner.LANGFUSE_OTEL)] + ) + cache = TenantTracerCache(config, "langfuse_otel", "litellm") + default = get_tracer(TracerProvider(), "litellm") + params = {"langfuse_public_key": "pk", "langfuse_secret_key": "sk"} + + assert cache.route_for(default, params).detached is True + + def run(): + set_request_destinations((LANGFUSE_DEST,)) + return cache.route_for(default, params) + + route = in_fresh_context(run) + assert route.detached is False + assert route.tracer is default + assert route.provider is None + + def test_an_overridden_backend_does_not_detach_on_a_service_name_either(self): + """A key or team service name is its own reason to build a second provider, so + clearing only the credentials would still take the model call out of the tree.""" + config = OpenTelemetryV2Config( + exporters=[ExporterSpec(kind="otlp_http", endpoint="http://op.local", owner=ExporterOwner.LANGFUSE_OTEL)] + ) + cache = TenantTracerCache(config, "langfuse_otel", "litellm") + default = get_tracer(TracerProvider(), "litellm") + auth_metadata = {"otel_service_name": "team-checkout"} + + assert cache.route_for(default, None, auth_metadata).detached is False + assert cache.route_for(default, None, auth_metadata).tracer is not default + + def run(): + set_request_destinations((LANGFUSE_DEST,)) + return cache.route_for(default, None, auth_metadata) + + route = in_fresh_context(run) + assert route.tracer is default, "the fan-out carries the service name on the destination instead" + assert route.provider is None + + @pytest.mark.parametrize("callback_name", ["arize", None]) + def test_a_service_name_does_not_detach_a_backend_the_destination_does_not_name(self, callback_name): + """The fan-out only sees spans on the published provider, so relabelling this + logger's span onto a second provider would drop the model call out of the + trace another backend's destination receives.""" + config = OpenTelemetryV2Config( + exporters=[ExporterSpec(kind="otlp_http", endpoint="http://op.local", owner=ExporterOwner.ARIZE_AX)] + ) + cache = TenantTracerCache(config, callback_name, "litellm") + default = get_tracer(TracerProvider(), "litellm") + auth_metadata = {"otel_service_name": "team-checkout"} + + relabelled = cache.route_for(default, None, auth_metadata) + assert relabelled.tracer is not default + cache.release(relabelled.provider) + + def run(): + set_request_destinations((LANGFUSE_DEST,)) + return cache.route_for(default, None, auth_metadata) + + route = in_fresh_context(run) + assert route.tracer is default + assert route.detached is False + assert route.provider is None + + @pytest.mark.parametrize( + ("owner", "params", "auth_metadata"), + [ + (ExporterOwner.ARIZE_AX, {"arize_space_key": "space", "arize_api_key": "key"}, {}), + (ExporterOwner.ARIZE_PHOENIX, None, {"phoenix_project_name": "team-project"}), + ], + ) + def test_a_backend_pointed_at_its_own_account_still_routes_next_to_another_backend_destination( + self, owner, params, auth_metadata + ): + """Credentials or a project name the tenant's own account for this backend, which + the other backend's destination cannot stand in for.""" + config = OpenTelemetryV2Config( + exporters=[ExporterSpec(kind="otlp_http", endpoint="http://op.local", owner=owner)] + ) + cache = TenantTracerCache(config, owner.value, "litellm") + default = get_tracer(TracerProvider(), "litellm") + + def run(): + set_request_destinations((LANGFUSE_DEST,)) + return cache.route_for(default, params, {"otel_service_name": "team-checkout", **auth_metadata}) + + route = in_fresh_context(run) + assert route.tracer is not default + assert route.detached is True + cache.release(route.provider) + + +@pytest.mark.usefixtures("allow_test_hosts") +class TestDestinationResolution: + def test_a_langfuse_key_pair_and_host_become_a_destination(self, monkeypatch): + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + auth = UserAPIKeyAuth( + team_metadata={ + "logging": [ + { + "callback_name": "langfuse_otel", + "callback_type": "success", + "callback_vars": { + "langfuse_public_key": "pk-team", + "langfuse_secret_key": "sk-team", + "langfuse_host": "http://team.local", + }, + } + ] + } + ) + + destinations = resolve_tenant_otel_destinations(auth) + + assert [d.endpoint for d in destinations] == ["http://team.local/api/public/otel"] + assert destinations[0].callback_name == "langfuse_otel" + + def test_a_keys_service_name_outranks_its_teams_on_the_destination(self, monkeypatch): + """The key/team ``otel_service_name`` used to reach the backend through + per-request tracer routing, which an overridden backend skips.""" + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + auth = UserAPIKeyAuth( + metadata={"otel_service_name": "key-svc"}, + team_metadata={ + "otel_service_name": "team-svc", + "logging": [ + { + "callback_name": "langfuse_otel", + "callback_type": "success", + "callback_vars": { + "langfuse_public_key": "pk-team", + "langfuse_secret_key": "sk-team", + "langfuse_host": "http://team.local", + }, + } + ], + }, + ) + + destinations = resolve_tenant_otel_destinations(auth) + + assert dict(destinations[0].resource_attributes) == {"service.name": "key-svc"} + + def test_a_team_that_named_no_service_name_gets_no_resource_override(self, monkeypatch): + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + auth = UserAPIKeyAuth( + team_metadata={ + "otel_service_name": " ", + "logging": [ + { + "callback_name": "langfuse_otel", + "callback_type": "success", + "callback_vars": { + "langfuse_public_key": "pk-team", + "langfuse_secret_key": "sk-team", + "langfuse_host": "http://team.local", + }, + } + ], + } + ) + + destinations = resolve_tenant_otel_destinations(auth) + + assert dict(destinations[0].resource_attributes) == {} + + def test_the_key_wins_over_the_team_for_the_same_backend(self, monkeypatch): + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + + def entry(host: str) -> Mapping[str, object]: + return { + "callback_name": "langfuse_otel", + "callback_type": "success", + "callback_vars": { + "langfuse_public_key": "pk", + "langfuse_secret_key": "sk", + "langfuse_host": host, + }, + } + + auth = UserAPIKeyAuth( + metadata={"logging": [entry("http://key.local")]}, + team_metadata={"logging": [entry("http://team.local")]}, + ) + + assert [d.endpoint for d in resolve_tenant_otel_destinations(auth)] == ["http://key.local/api/public/otel"] + + def test_nothing_resolves_while_otel_v2_is_off(self, monkeypatch): + monkeypatch.delenv("LITELLM_OTEL_V2", raising=False) + is_otel_v2_enabled.cache_clear() + auth = UserAPIKeyAuth( + team_metadata={ + "logging": [ + { + "callback_name": "langfuse_otel", + "callback_type": "success", + "callback_vars": {"langfuse_public_key": "pk", "langfuse_secret_key": "sk"}, + } + ] + } + ) + + assert resolve_tenant_otel_destinations(auth) == () + + def test_a_host_without_its_key_pair_resolves_to_nothing(self): + assert destination_for("langfuse_otel", {"langfuse_host": "http://team.local"}) is None + + def test_a_backend_with_no_dynamic_credentials_has_no_destination(self): + assert "arize_phoenix" not in destination_capable_backends() + assert destination_for("arize_phoenix", {"arize_api_key": "k"}) is None + + def test_the_destination_header_string_survives_the_exporter_round_trip(self): + from litellm.integrations.otel.plumbing.providers import parse_headers + + destination = destination_for( + "langfuse_otel", + {"langfuse_public_key": "pk", "langfuse_secret_key": "sk", "langfuse_host": "http://x"}, + ) + assert parse_headers(destination.header_string())["authorization"] == destination.headers["Authorization"] + + +#: Anything that makes ``OpenTelemetryV2Config`` synthesize a real operator destination. +_OTEL_SHORTHAND_ENV = ( + "OTEL_ENDPOINT", + "OTEL_HEADERS", + "OTEL_EXPORTER", + "OTEL_EXPORTER_OTLP_ENDPOINT", + "OTEL_EXPORTER_OTLP_HEADERS", + "OTEL_EXPORTER_OTLP_PROTOCOL", +) + + +def credential_less_proxy(monkeypatch) -> None: + """An operator with no Langfuse account and no generic OTLP collector.""" + for name in ("LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY", *_OTEL_SHORTHAND_ENV): + monkeypatch.delenv(name, raising=False) + with pytest.raises(ValueError, match="LANGFUSE_PUBLIC_KEY"): + langfuse_preset() + + +class TestPresetDegradation: + def test_a_credential_less_langfuse_exports_nowhere_instead_of_to_the_console(self, monkeypatch, capfd): + """``_normalize`` folds a console exporter in for an empty list, which would + print every span on a proxy whose teams bring their own credentials.""" + credential_less_proxy(monkeypatch) + + config = langfuse_preset(allow_missing_credentials=True) + provider = build_tracer_provider(config, tenant_overrides=True) + capfd.readouterr() + in_fresh_context(emit, provider) + provider.force_flush() + + assert '"name": "chat gpt-4"' not in capfd.readouterr().out + assert "langfuse" in config.mapper_names + + def test_langfuse_still_raises_for_a_global_callback_with_no_credentials(self, monkeypatch): + monkeypatch.delenv("LANGFUSE_PUBLIC_KEY", raising=False) + monkeypatch.delenv("LANGFUSE_SECRET_KEY", raising=False) + + with pytest.raises(ValueError, match="LANGFUSE_PUBLIC_KEY"): + langfuse_preset() + + def test_a_credential_less_proxy_builds_the_gated_logger_beside_a_v2_carrier(self, monkeypatch): + from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 + + credential_less_proxy(monkeypatch) + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + carrier = build_otel_v2_logger(OpenTelemetryV2Config(exporter="in_memory")) + + def run(): + set_request_destinations((LANGFUSE_DEST,)) + return _maybe_construct_otel_v2("langfuse_otel", [carrier]) + + is_otel_v2_enabled.cache_clear() + logger = in_fresh_context(run) + is_otel_v2_enabled.cache_clear() + + assert logger is not None + assert all(spec.requires_headers and not spec.headers for spec in logger.config.exporters) + + def test_a_credential_less_proxy_with_no_destinations_falls_back_to_the_legacy_path(self, monkeypatch): + """Nothing can use a credential-less langfuse here, so the operator has to get + the same story as before v2: the legacy integration, not a global provider + that exports nowhere.""" + from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 + + credential_less_proxy(monkeypatch) + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + + is_otel_v2_enabled.cache_clear() + logger = in_fresh_context(_maybe_construct_otel_v2, "langfuse_otel", []) + is_otel_v2_enabled.cache_clear() + + assert logger is None + + def test_a_valid_newrelic_base_exporter_survives_without_a_license_key(self, monkeypatch): + from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 + + monkeypatch.delenv("NEW_RELIC_LICENSE_KEY", raising=False) + monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.local:4318") + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + + is_otel_v2_enabled.cache_clear() + logger = in_fresh_context(_maybe_construct_otel_v2, "newrelic", []) + is_otel_v2_enabled.cache_clear() + + assert logger is not None + assert [spec.endpoint for spec in logger.config.exporters] == [ + "http://collector.local:4318", + "https://otlp.nr-data.net", + ] + + def test_a_credentialless_newrelic_without_a_base_exporter_falls_back(self, monkeypatch): + from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 + + monkeypatch.delenv("NEW_RELIC_LICENSE_KEY", raising=False) + for name in _OTEL_SHORTHAND_ENV: + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + + is_otel_v2_enabled.cache_clear() + logger = in_fresh_context(_maybe_construct_otel_v2, "newrelic", []) + is_otel_v2_enabled.cache_clear() + + assert logger is None + + def test_an_explicit_console_exporter_keeps_a_credentialless_preset_on_v2(self, monkeypatch, capfd): + """``OTEL_EXPORTER=console`` reads exactly like the placeholder ``_normalize`` + folds in, but the operator asked for it, so a credential-less New Relic keeps + the V2 logger and its spans reach stdout instead of the legacy path.""" + from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 + + monkeypatch.delenv("NEW_RELIC_LICENSE_KEY", raising=False) + for name in _OTEL_SHORTHAND_ENV: + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("OTEL_EXPORTER", "console") + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + + is_otel_v2_enabled.cache_clear() + logger = in_fresh_context(_maybe_construct_otel_v2, "newrelic", []) + is_otel_v2_enabled.cache_clear() + + assert logger is not None + assert logger.config.exporters[0].kind == "console" + assert not logger.config.exporters[0].requires_headers + + def test_a_destination_for_one_backend_does_not_degrade_another(self, monkeypatch): + from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 + + credential_less_proxy(monkeypatch) + monkeypatch.delenv("WANDB_API_KEY", raising=False) + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + + def run(): + set_request_destinations((LANGFUSE_DEST,)) + return _maybe_construct_otel_v2("weave_otel", []) + + is_otel_v2_enabled.cache_clear() + logger = in_fresh_context(run) + is_otel_v2_enabled.cache_clear() + + assert logger is None + + def test_the_exporter_less_logger_is_not_reused_by_a_request_without_destinations(self, monkeypatch): + """Reusing it would let one team's destination decide how every later request + without one is logged, long after the degrade was justified.""" + from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 + + credential_less_proxy(monkeypatch) + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + loggers = [build_otel_v2_logger(OpenTelemetryV2Config(exporter="in_memory"))] + + def with_destination(): + set_request_destinations((LANGFUSE_DEST,)) + return _maybe_construct_otel_v2("langfuse_otel", loggers) + + is_otel_v2_enabled.cache_clear() + degraded = in_fresh_context(with_destination) + plain = in_fresh_context(_maybe_construct_otel_v2, "langfuse_otel", loggers) + is_otel_v2_enabled.cache_clear() + + assert degraded is not None + assert plain is None + + def test_a_credentialed_logger_is_still_reused_across_requests(self, monkeypatch): + from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 + + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-lf-1") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-lf-1") + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + loggers = [] + + is_otel_v2_enabled.cache_clear() + first = in_fresh_context(_maybe_construct_otel_v2, "langfuse_otel", loggers) + second = in_fresh_context(_maybe_construct_otel_v2, "langfuse_otel", loggers) + is_otel_v2_enabled.cache_clear() + + assert first is not None + assert second is first + + @staticmethod + def _degraded_langfuse_beside(loggers, monkeypatch): + from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 + + credential_less_proxy(monkeypatch) + monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.local:4318") + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + + def run(): + set_request_destinations((LANGFUSE_DEST,)) + return _maybe_construct_otel_v2("langfuse_otel", loggers) + + is_otel_v2_enabled.cache_clear() + logger = in_fresh_context(run) + is_otel_v2_enabled.cache_clear() + assert logger is not None + return logger + + def test_a_degraded_logger_beside_another_v2_logger_leaves_the_collector_to_it(self, monkeypatch): + """The other logger's provider already exports every span to the operator's + collector, so a second model span from this one would land there twice.""" + collector_logger = build_otel_v2_logger(OpenTelemetryV2Config(exporter="in_memory")) + + logger = self._degraded_langfuse_beside([collector_logger], monkeypatch) + + assert [spec.endpoint for spec in logger.config.exporters] == [None] + assert all(spec.requires_headers and not spec.headers for spec in logger.config.exporters) + + @pytest.mark.parametrize("registered", [(), (CustomLogger(),)]) + def test_a_credential_less_proxy_with_a_destination_but_no_v2_carrier_falls_back(self, monkeypatch, registered): + """Only a V2 logger publishes the provider the fan-out rides on, so a legacy + callback beside this one leaves the destination just as unreachable as no + callback at all, and the operator keeps the pre-V2 story.""" + from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 + + credential_less_proxy(monkeypatch) + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + + def run(): + set_request_destinations((LANGFUSE_DEST,)) + return _maybe_construct_otel_v2("langfuse_otel", list(registered)) + + is_otel_v2_enabled.cache_clear() + logger = in_fresh_context(run) + is_otel_v2_enabled.cache_clear() + + assert logger is None + + def test_a_credentialed_logger_beside_another_v2_logger_keeps_every_exporter(self, monkeypatch): + """Only a degraded preset gives the collector up; an operator who configured + both the backend and the collector still exports to both, as on base.""" + from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 + + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-lf-1") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-lf-1") + monkeypatch.setenv("LANGFUSE_HOST", "https://cloud.langfuse.com") + monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.local:4318") + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + collector_logger = build_otel_v2_logger(OpenTelemetryV2Config(exporter="in_memory")) + + is_otel_v2_enabled.cache_clear() + logger = in_fresh_context(_maybe_construct_otel_v2, "langfuse_otel", [collector_logger]) + is_otel_v2_enabled.cache_clear() + + assert logger is not None + assert [spec.endpoint for spec in logger.config.exporters] == [ + "http://collector.local:4318", + "https://cloud.langfuse.com/api/public/otel", + ] + assert all(spec.headers for spec in logger.config.exporters if spec.requires_headers) + + +class TestContextIsolation: + def test_destinations_do_not_leak_between_requests(self): + def first(): + set_request_destinations((LANGFUSE_DEST,)) + return destination_backends() + + assert in_fresh_context(first) == frozenset({"langfuse_otel"}) + assert in_fresh_context(request_destinations) == () + + +class TestOperatorShorthandSurvivesDegradation: + def test_a_generic_otlp_collector_keeps_receiving_when_langfuse_has_no_credentials(self, monkeypatch): + """Only the stdout placeholder is dropped. An operator who set the standard + OTLP env vars configured a real destination and must keep it.""" + monkeypatch.delenv("LANGFUSE_PUBLIC_KEY", raising=False) + monkeypatch.delenv("LANGFUSE_SECRET_KEY", raising=False) + monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.local:4318") + + config = langfuse_preset(allow_missing_credentials=True) + + assert [spec.endpoint for spec in config.exporters] == ["http://collector.local:4318", None] + assert [spec.kind for spec in config.exporters] == ["otlp_http", "console"] + + def test_the_stdout_placeholder_is_still_dropped_when_it_is_the_only_exporter(self, monkeypatch): + credential_less_proxy(monkeypatch) + + config = langfuse_preset(allow_missing_credentials=True) + + assert all(spec.requires_headers and not spec.headers for spec in config.exporters) + + +class TestBackendEndpointParity: + def test_arize_follows_its_own_http_endpoint_instead_of_the_grpc_default(self, monkeypatch): + monkeypatch.delenv("ARIZE_ENDPOINT", raising=False) + monkeypatch.setenv("ARIZE_HTTP_ENDPOINT", "https://otlp.arize.com/v1/traces") + + destination = destination_for("arize", {"arize_space_id": "s", "arize_api_key": "k"}) + + assert destination.endpoint == "https://otlp.arize.com/v1/traces" + assert destination.protocol == "otlp_http" + + def test_arize_uses_grpc_when_nothing_is_configured(self, monkeypatch): + monkeypatch.delenv("ARIZE_ENDPOINT", raising=False) + monkeypatch.delenv("ARIZE_HTTP_ENDPOINT", raising=False) + + destination = destination_for("arize", {"arize_space_id": "s", "arize_api_key": "k"}) + + assert destination.endpoint == "https://otlp.arize.com/v1" + assert destination.protocol == "otlp_grpc" + + def test_weave_follows_a_self_hosted_wandb_host(self, monkeypatch): + monkeypatch.setenv("WANDB_HOST", "weave.internal.example") + + destination = destination_for("weave_otel", {"wandb_api_key": "k", "weave_project_id": "e/p"}) + + assert destination.endpoint == "https://weave.internal.example/otel/v1/traces" + + def test_weave_uses_the_cloud_endpoint_without_a_host(self, monkeypatch): + monkeypatch.delenv("WANDB_HOST", raising=False) + + destination = destination_for("weave_otel", {"wandb_api_key": "k", "weave_project_id": "e/p"}) + + assert destination.endpoint == "https://trace.wandb.ai/otel/v1/traces" + + +class TestIncompleteCredentials: + """Half a credential set builds a non-empty but unusable header dict. Accepting it + would suppress the operator's exporter and send the trace where it cannot land.""" + + @pytest.mark.parametrize( + "callback_name,callback_vars", + [ + ("arize", {"arize_api_key": "k"}), + ("arize", {"arize_space_id": "s"}), + ("weave_otel", {"wandb_api_key": "k"}), + ("weave_otel", {"weave_project_id": "e/p"}), + ("langfuse_otel", {"langfuse_public_key": "pk"}), + ], + ) + def test_a_partial_credential_set_resolves_to_nothing(self, callback_name, callback_vars): + assert destination_for(callback_name, callback_vars) is None + + @pytest.mark.parametrize( + "callback_name,callback_vars", + [ + ("arize", {"arize_space_id": "s", "arize_api_key": "k"}), + ("weave_otel", {"wandb_api_key": "k", "weave_project_id": "e/p"}), + ("newrelic", {"newrelic_api_key": "k"}), + ], + ) + def test_a_complete_credential_set_resolves(self, callback_name, callback_vars): + assert destination_for(callback_name, callback_vars) is not None + + +@pytest.mark.usefixtures("allow_test_hosts") +class TestCallbackTypeFilter: + @staticmethod + def _auth(callback_type: str | None) -> UserAPIKeyAuth: + return UserAPIKeyAuth( + team_metadata={ + "logging": [ + { + "callback_name": "langfuse_otel", + "callback_type": callback_type, + "callback_vars": { + "langfuse_public_key": "pk", + "langfuse_secret_key": "sk", + "langfuse_host": "http://team.local", + }, + } + ] + } + ) + + @pytest.mark.parametrize("callback_type", ["success", "success_and_failure", None]) + def test_an_entry_that_wants_success_traces_gets_the_whole_trace(self, monkeypatch, callback_type): + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + + assert resolve_tenant_otel_destinations(self._auth(callback_type)) != () + + def test_a_failure_only_entry_does_not_take_over_the_trace(self, monkeypatch): + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + + assert resolve_tenant_otel_destinations(self._auth("failure")) == () + + +class TestTenantConfigAgreement: + """The destination resolver and ``convert_key_logging_metadata_to_callback`` read + the same stored config, so they must not read it two different ways.""" + + @pytest.fixture(autouse=True) + def _v2_on(self, monkeypatch): + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + monkeypatch.setattr( + litellm, "provider_url_destination_allowed_hosts", ["team.local", "key.local"], raising=False + ) + is_otel_v2_enabled.cache_clear() + yield + is_otel_v2_enabled.cache_clear() + + @staticmethod + def _entry(host, **extra): + return { + "callback_name": "langfuse_otel", + "callback_vars": {"langfuse_public_key": "pk", "langfuse_secret_key": "sk", "langfuse_host": host, **extra}, + } + + def test_a_key_that_disabled_its_callbacks_does_not_fall_back_to_the_team(self): + """Disabling a key's callbacks stores an empty list, which the sibling parser + reads as 'the key configured none'.""" + auth = UserAPIKeyAuth( + metadata={"logging": []}, + team_metadata={"logging": [self._entry("http://team.local")]}, + ) + + assert resolve_tenant_otel_destinations(auth) == () + + def test_two_entries_for_one_backend_merge_their_vars_last_wins(self): + auth = UserAPIKeyAuth( + team_metadata={ + "logging": [ + self._entry("http://team.local"), + {"callback_name": "langfuse_otel", "callback_vars": {"langfuse_host": "http://key.local"}}, + ] + } + ) + + destinations = resolve_tenant_otel_destinations(auth) + + assert [d.endpoint for d in destinations] == ["http://key.local/api/public/otel"] + + @pytest.fixture + def premium(self, monkeypatch): + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "premium_user", True) + monkeypatch.setattr(litellm, "allow_dynamic_callback_disabling", True) + + @pytest.mark.usefixtures("premium") + def test_a_backend_the_key_disabled_resolves_to_no_destination(self): + """Dispatch skips a callback named in the key's ``litellm_disabled_callbacks``, + so the fan-out must not deliver to it either.""" + auth = UserAPIKeyAuth( + metadata={"litellm_disabled_callbacks": ["Langfuse_OTEL"]}, + team_metadata={"logging": [self._entry("http://team.local")]}, + ) + + assert resolve_tenant_otel_destinations(auth) == () + + @pytest.mark.usefixtures("premium") + @pytest.mark.parametrize( + ("header", "resolved"), + [ + ("langfuse_otel", False), + (" LANGFUSE_OTEL ,arize", False), + ("arize", True), + ], + ) + def test_the_disable_header_wins_over_the_key_list(self, header, resolved): + """Same precedence as dispatch: a header that names other backends re-enables + the one the key stored.""" + auth = UserAPIKeyAuth( + metadata={"litellm_disabled_callbacks": ["langfuse_otel"]}, + team_metadata={"logging": [self._entry("http://team.local")]}, + ) + + destinations = resolve_tenant_otel_destinations(auth, {"x-litellm-disable-callbacks": header}) + + assert bool(destinations) is resolved + + def test_a_non_premium_proxy_ignores_the_disabled_list_like_dispatch_does(self, monkeypatch): + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "premium_user", False) + auth = UserAPIKeyAuth( + metadata={"litellm_disabled_callbacks": ["langfuse_otel"]}, + team_metadata={"logging": [self._entry("http://team.local")]}, + ) + + assert resolve_tenant_otel_destinations(auth, {"x-litellm-disable-callbacks": "langfuse_otel"}) != () + + +class TestEvictionSafety: + class Recording(SimpleSpanProcessor): + def __init__(self): + super().__init__(InMemorySpanExporter()) + self.shutdown_calls = 0 + + def shutdown(self): + self.shutdown_calls += 1 + + def _fan_out(self): + built = [] + + def factory(_destination): + built.append(self.Recording()) + return built[-1] + + return TenantFanOutSpanProcessor(processor_factory=factory), built + + @staticmethod + def _dest(index): + return LANGFUSE_DEST.model_copy(update={"endpoint": f"http://d{index}/otel"}) + + @staticmethod + def _settle(fan_out, processor=None): + """Wait for retirement to clear and, when given, for the drain to run. + + The drain pool is shared and bounded, so a shed processor is closed once a + worker picks it up rather than the moment it is handed over. + """ + for _ in range(500): + if not fan_out._retired and (processor is None or processor.shutdown_calls): + return + time.sleep(0.02) + + def test_a_processor_still_exporting_a_span_is_not_closed_under_it(self): + """``on_end`` holds a processor across the export, so closing an evicted one + there drops the span it is holding.""" + from litellm.integrations.otel.plumbing.providers import _MAX_CACHED_DESTINATION_PROCESSORS + + fan_out, built = self._fan_out() + held = fan_out._acquire(self._dest(0)) + for index in range(1, _MAX_CACHED_DESTINATION_PROCESSORS + 1): + fan_out._acquire(self._dest(index)) + fan_out._release(built[-1]) + + assert held.shutdown_calls == 0 + assert id(held) in fan_out._retired + + fan_out._release(held) + self._settle(fan_out, held) + + assert held.shutdown_calls == 1 + + def test_a_recently_used_destination_is_not_the_one_evicted(self): + """Without the refresh the cache sheds by insertion order, so the busiest + destination is the one whose exporter is rebuilt on every overflow.""" + from litellm.integrations.otel.plumbing.providers import _MAX_CACHED_DESTINATION_PROCESSORS + + fan_out, built = self._fan_out() + for index in range(_MAX_CACHED_DESTINATION_PROCESSORS): + fan_out._release(fan_out._acquire(self._dest(index))) + fan_out._release(fan_out._acquire(self._dest(0))) + fan_out._release(fan_out._acquire(self._dest(_MAX_CACHED_DESTINATION_PROCESSORS))) + self._settle(fan_out, built[1]) + + assert built[1].shutdown_calls == 1 + assert built[0].shutdown_calls == 0, "the destination used most recently was the one shed" + + def test_an_idle_evicted_processor_is_closed_off_the_export_path(self): + from litellm.integrations.otel.plumbing.providers import _MAX_CACHED_DESTINATION_PROCESSORS + + fan_out, built = self._fan_out() + for index in range(_MAX_CACHED_DESTINATION_PROCESSORS + 1): + fan_out._acquire(self._dest(index)) + fan_out._release(built[-1]) + self._settle(fan_out, built[0]) + + assert built[0].shutdown_calls == 1 + assert len(fan_out._processors) == _MAX_CACHED_DESTINATION_PROCESSORS + + def test_a_slow_collector_does_not_hold_up_the_export_path(self): + """``shutdown`` flushes over the network and is reached from ``on_end``, so + closing a shed processor inline lets one unreachable tenant collector stall + every other tenant's spans.""" + from litellm.integrations.otel.plumbing.providers import _MAX_CACHED_DESTINATION_PROCESSORS + + class Slow(self.Recording): + def shutdown(self): + time.sleep(3) + super().shutdown() + + built = [] + + def factory(_destination): + built.append(Slow()) + return built[-1] + + fan_out = TenantFanOutSpanProcessor(processor_factory=factory) + started = time.monotonic() + for index in range(_MAX_CACHED_DESTINATION_PROCESSORS + 1): + fan_out._acquire(self._dest(index)) + fan_out._release(built[-1]) + + assert time.monotonic() - started < 2 + + def test_shedding_many_processors_does_not_spawn_a_thread_each(self): + """A tenant that cycles its destination config sheds a processor per request, + so a thread per shed processor is a thread per request against a slow + collector.""" + import threading + + from litellm.integrations.otel.plumbing.providers import _MAX_CACHED_DESTINATION_PROCESSORS + + release = threading.Event() + + class Blocking(self.Recording): + def shutdown(self): + release.wait(timeout=10) + super().shutdown() + + built = [] + + def factory(_destination): + built.append(Blocking()) + return built[-1] + + fan_out = TenantFanOutSpanProcessor(processor_factory=factory) + before = self._drain_workers() + try: + for index in range(_MAX_CACHED_DESTINATION_PROCESSORS + 30): + fan_out._acquire(self._dest(index)) + fan_out._release(built[-1]) + grew = self._drain_workers() - before + assert grew == 0, f"one drain thread per shed processor: {grew} new threads" + finally: + release.set() + self._settle(fan_out, built[0]) + + def test_a_saturated_drain_leaves_new_destinations_with_the_operator(self): + """A shed processor keeps its batch thread until its close returns, and against + a collector that never answers every close waits out the exporter's timeout. + Tenants rotating past the cache cap would otherwise queue one more processor, + and one more thread, per request for as long as the outage lasts.""" + import threading + + from litellm.integrations.otel.plumbing.providers import _MAX_CACHED_DESTINATION_PROCESSORS + + release = threading.Event() + + class Blocking(self.Recording): + def shutdown(self): + release.wait(timeout=10) + super().shutdown() + + built = [] + + def factory(_destination): + built.append(Blocking()) + return built[-1] + + fan_out = TenantFanOutSpanProcessor(processor_factory=factory, pending_drains=3) + try: + anchored = tuple( + fan_out.deliverable((self._dest(index),)) for index in range(_MAX_CACHED_DESTINATION_PROCESSORS + 40) + ) + + assert len(built) == _MAX_CACHED_DESTINATION_PROCESSORS + 3, "a processor per request during the outage" + assert sum(1 for accepted in anchored if accepted) == len(built), "anchored what it could not build" + assert fan_out.deliverable((self._dest(999),)) == (), "the span would vanish instead of staying with the operator" + finally: + release.set() + for _ in range(500): + if not fan_out._drain.saturated(): + break + time.sleep(0.02) + + assert fan_out.deliverable((self._dest(999),)) == (self._dest(999),), "the fan-out never recovered" + + def test_an_anchored_destination_evicted_under_a_saturated_drain_still_gets_the_span(self): + """``deliverable`` accepted the destination, so the operator's exporter has stood + down for it. Other tenants' auths can then evict it, and the eviction is what + tips the drain into saturation, so refusing the rebuild at ``on_end`` would drop + the span outright.""" + import threading + + from litellm.integrations.otel.plumbing.providers import _MAX_CACHED_DESTINATION_PROCESSORS + + release = threading.Event() + + class Blocking(self.Recording): + def shutdown(self): + release.wait(timeout=10) + super().shutdown() + + built = [] + + def factory(_destination): + built.append(Blocking()) + return built[-1] + + fan_out = TenantFanOutSpanProcessor( + processor_factory=factory, pending_drains=_MAX_CACHED_DESTINATION_PROCESSORS + 1 + ) + provider = TracerProvider() + provider.add_span_processor(fan_out) + tracer = get_tracer(provider, "litellm") + anchored = self._dest(0) + try: + for index in range(1, _MAX_CACHED_DESTINATION_PROCESSORS + 1): + assert fan_out.deliverable((self._dest(index),)) + assert fan_out.deliverable((anchored,)) == (anchored,) + first = built[-1] + for index in range(_MAX_CACHED_DESTINATION_PROCESSORS + 1, 2 * _MAX_CACHED_DESTINATION_PROCESSORS + 1): + assert fan_out.deliverable((self._dest(index),)) + assert fan_out._drain.saturated(), "the anchored destination's own eviction saturates the drain" + assert first not in fan_out._processors.values(), "the anchored destination was not evicted" + + def run(): + set_request_destinations((anchored,)) + with tracer.start_as_current_span("chat anthropic"): + pass + + before = len(built) + in_fresh_context(run) + assert len(built) == before + 1, "the anchored destination was not rebuilt, so its span went nowhere" + assert [span.name for span in built[-1].span_exporter.get_finished_spans()] == ["chat anthropic"] + assert first.span_exporter.get_finished_spans() == (), "the shed processor was handed out again" + finally: + release.set() + + def _saturated_by_anchoring(self, pending_drains, extra): + """A fan-out whose drain ``extra`` anchorings past the cache cap have saturated. + + Returns it with the processors built, the destinations that anchored, and the + event that lets the blocked closes finish. + """ + import threading + + from litellm.integrations.otel.plumbing.providers import _MAX_CACHED_DESTINATION_PROCESSORS + + release = threading.Event() + + class Blocking(self.Recording): + def shutdown(self): + release.wait(timeout=10) + super().shutdown() + + built = [] + + def factory(_destination): + built.append(Blocking()) + return built[-1] + + fan_out = TenantFanOutSpanProcessor(processor_factory=factory, pending_drains=pending_drains) + destinations = tuple(self._dest(index) for index in range(_MAX_CACHED_DESTINATION_PROCESSORS + extra)) + anchored = tuple(destination for destination in destinations if fan_out.deliverable((destination,))) + assert fan_out._drain.saturated(), "anchoring past the cap did not saturate the drain" + assert len(anchored) > _MAX_CACHED_DESTINATION_PROCESSORS, "not enough destinations in flight to churn" + return fan_out, built, anchored, release + + def test_anchored_rebuilds_under_a_saturated_drain_do_not_grow_with_the_spans(self): + """Every anchored rebuild past the cap evicts another anchored destination, whose + next span rebuilds it in turn. With more destinations in flight than the cache + holds, each span would then cost one more processor, one more batch thread and + one more close queued behind a collector that never answers.""" + from litellm.integrations.otel.plumbing.providers import _MAX_CACHED_DESTINATION_PROCESSORS + + fan_out, built, anchored, release = self._saturated_by_anchoring(pending_drains=4, extra=8) + try: + after_anchoring = len(built) + for _ in range(5): + for destination in anchored: + fan_out._release(fan_out._acquire(destination)) + + rebuilt = len(built) - after_anchoring + assert rebuilt == len(anchored) - _MAX_CACHED_DESTINATION_PROCESSORS, ( + f"{rebuilt} rebuilds over 5 rounds of {len(anchored)} anchored destinations: one per evicted one expected" + ) + assert len(fan_out._processors) == len(anchored), "an anchored destination was shed under a saturated drain" + assert all(destination in fan_out.deliverable((destination,)) for destination in anchored) + finally: + release.set() + + def test_the_cache_returns_to_its_cap_once_the_drain_has_room(self): + """Holding above the cap is for the outage only: with the drain caught up, the + entries kept for the destinations in flight are the ones to shed.""" + from litellm.integrations.otel.plumbing.providers import _MAX_CACHED_DESTINATION_PROCESSORS + + fan_out, built, anchored, release = self._saturated_by_anchoring(pending_drains=4, extra=8) + for destination in anchored: + fan_out._release(fan_out._acquire(destination)) + assert len(fan_out._processors) > _MAX_CACHED_DESTINATION_PROCESSORS + + release.set() + for _ in range(500): + for destination in anchored[-4:]: + fan_out._release(fan_out._acquire(destination)) + if len(fan_out._processors) <= _MAX_CACHED_DESTINATION_PROCESSORS: + break + time.sleep(0.02) + + assert len(fan_out._processors) == _MAX_CACHED_DESTINATION_PROCESSORS, "the cache never came back to its cap" + shed = len(built) - _MAX_CACHED_DESTINATION_PROCESSORS + for _ in range(500): + if sum(processor.shutdown_calls for processor in built) == shed: + break + time.sleep(0.02) + + assert sum(processor.shutdown_calls for processor in built) == shed, "a shed processor was never closed" + + def test_concurrent_eviction_cannot_build_between_retirement_and_drain_submission(self): + """A second request cannot build while the first eviction is being handed to + the drain, or concurrent churn can outrun the pending-drain limit.""" + import threading + + from litellm.integrations.otel.plumbing.providers import ( + _DrainPool, + _MAX_CACHED_DESTINATION_PROCESSORS, + ) + + class GatedDrain(_DrainPool): + def __init__(self): + super().__init__(workers=0) + self.started = threading.Event() + self.release = threading.Event() + + def saturated(self): + return False + + def submit(self, processor): + if not self.started.is_set(): + self.started.set() + self.release.wait(timeout=5) + + built = [] + + def factory(_destination): + built.append(self.Recording()) + return built[-1] + + drain = GatedDrain() + fan_out = TenantFanOutSpanProcessor(processor_factory=factory, drain_pool=drain) + for index in range(_MAX_CACHED_DESTINATION_PROCESSORS): + fan_out._release(fan_out._acquire(self._dest(index))) + + first = threading.Thread(target=lambda: fan_out._release(fan_out._acquire(self._dest(32)))) + first.start() + assert drain.started.wait(timeout=5) + second = threading.Thread(target=lambda: fan_out._release(fan_out._acquire(self._dest(33)))) + second.start() + time.sleep(0.1) + + assert len(built) == _MAX_CACHED_DESTINATION_PROCESSORS + 1 + + drain.release.set() + first.join(timeout=5) + second.join(timeout=5) + assert not first.is_alive() and not second.is_alive() + assert len(built) == _MAX_CACHED_DESTINATION_PROCESSORS + 2 + + def test_drain_workers_are_daemons(self): + """Python joins a ThreadPoolExecutor's workers at interpreter exit, so one + unreachable tenant collector would hold the proxy open for its export + timeout on the way down.""" + import threading + + self._fan_out() + workers = [t for t in threading.enumerate() if t.name.startswith("litellm-otel-destination-drain")] + + assert workers, "no drain worker was started" + assert all(t.daemon for t in workers), "a non-daemon drain worker blocks interpreter exit" + + def test_a_burst_of_first_evictions_starts_one_set_of_drain_workers(self): + """A drain pool built lazily on first use is not built once: several threads + can each finish the build, and every pool but the winner is left with its + workers blocked on a queue nothing will ever feed again.""" + import threading + + from litellm.integrations.otel.plumbing.providers import ( + _DRAIN_WORKERS, + _MAX_CACHED_DESTINATION_PROCESSORS, + ) + + for _ in range(3): + before = self._drain_workers() + fan_out, built = self._fan_out() + for index in range(_MAX_CACHED_DESTINATION_PROCESSORS): + fan_out._release(fan_out._acquire(self._dest(index))) + barrier = threading.Barrier(16) + + def shed(index, fan_out=fan_out, barrier=barrier): + barrier.wait(timeout=10) + fan_out._release(fan_out._acquire(self._dest(index))) + + threads = [ + threading.Thread(target=shed, args=(_MAX_CACHED_DESTINATION_PROCESSORS + index,)) for index in range(16) + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=10) + self._settle(fan_out) + + assert self._drain_workers() - before == _DRAIN_WORKERS + + @staticmethod + def _drain_workers(): + import threading + + return len([t for t in threading.enumerate() if t.name.startswith("litellm-otel-destination-drain")]) + + def test_shutdown_does_not_close_a_processor_under_an_in_flight_export(self): + """``on_end`` runs on whichever thread ends a span, so it reaches the fan-out + while the SDK tears the provider down.""" + import threading + + fan_out, _ = self._fan_out() + held = fan_out._acquire(self._dest(0)) + closed = threading.Thread(target=fan_out.shutdown) + closed.start() + try: + time.sleep(0.3) + + assert held.shutdown_calls == 0, "closed a processor with a span still being forwarded" + finally: + fan_out._release(held) + closed.join(timeout=10) + + assert held.shutdown_calls == 1 + + def test_a_closed_fan_out_builds_no_new_processor(self): + """A processor built after shutdown is one nothing will ever close, and it + exports to a tenant on a provider the SDK has already torn down.""" + fan_out, built = self._fan_out() + fan_out.shutdown() + + assert fan_out._acquire(self._dest(0)) is None + assert built == [] + + def test_shutdown_gives_up_on_an_export_that_never_finishes(self): + """The wait is bounded: an exporter stuck on a dead collector must not hold + the proxy open on the way down.""" + import threading + + fan_out = TenantFanOutSpanProcessor(processor_factory=lambda _d: self.Recording(), shutdown_drain_seconds=0.2) + fan_out._acquire(self._dest(0)) + closed = threading.Thread(target=fan_out.shutdown) + closed.start() + closed.join(timeout=5) + + assert not closed.is_alive(), "shutdown blocked on an export that never finished" + + def test_shutdown_retires_the_drain_workers(self): + """A proxy that rebuilds its telemetry builds another fan-out, so workers that + outlive the one that started them are two more threads per reload.""" + from litellm.integrations.otel.plumbing.providers import _DRAIN_WORKERS + + before = self._drain_workers() + fan_out, _ = self._fan_out() + assert self._drain_workers() - before == _DRAIN_WORKERS + + fan_out.shutdown() + for _ in range(500): + if self._drain_workers() == before: + break + time.sleep(0.02) + + assert self._drain_workers() == before, "the drain workers outlived their fan-out" + + def test_a_processor_shed_after_shutdown_is_still_closed(self): + """``close`` retires the workers, so anything handed to the pool afterwards + would sit in a queue nobody reads.""" + fan_out, _ = self._fan_out() + stray = self.Recording() + fan_out.shutdown() + fan_out._drain.submit(stray) + + for _ in range(500): + if stray.shutdown_calls: + break + time.sleep(0.02) + + assert stray.shutdown_calls == 1 + + def test_releasing_a_straggler_after_shutdown_does_not_block_the_span_thread(self): + """The teardown deadline has already expired by then, so closing the straggler + inline would park whichever thread just ended a span on the very flush the + deadline gave up waiting for.""" + import threading + + never = threading.Event() + + class Stuck(self.Recording): + def shutdown(self): + never.wait() + + def factory(_destination): + return Stuck() + + fan_out = TenantFanOutSpanProcessor(processor_factory=factory, shutdown_drain_seconds=0.05) + held = fan_out._acquire(self._dest(0)) + fan_out.shutdown() + + released = threading.Event() + caller = threading.Thread(target=lambda: (fan_out._release(held), released.set()), daemon=True) + caller.start() + came_back = released.wait(timeout=5) + never.set() + + assert came_back, "the thread that ended the span was left holding a stuck teardown" + + def test_shutdown_waits_out_an_export_that_lands_inside_the_bound(self): + """Without the wait the closing is left to a daemon thread, which the + interpreter can retire before it runs, so the last spans never reach the + tenant.""" + import threading + + fan_out, built = self._fan_out() + held = fan_out._acquire(self._dest(0)) + threading.Timer(0.2, lambda: fan_out._release(held)).start() + + fan_out.shutdown() + + assert held.shutdown_calls == 1, "shutdown returned before the export it should have waited out" + + def test_a_straggler_past_the_drain_bound_is_closed_by_its_own_thread(self): + """The wait is bounded so one dead collector cannot hold the proxy open, which + means a processor still exporting when it expires has to be left to the thread + holding it rather than closed under the span it is carrying.""" + built = [] + + def factory(_destination): + built.append(self.Recording()) + return built[-1] + + fan_out = TenantFanOutSpanProcessor(processor_factory=factory, shutdown_drain_seconds=0.05) + held = fan_out._acquire(self._dest(0)) + + fan_out.shutdown() + + assert held.shutdown_calls == 0 + + fan_out._release(held) + self._settle(fan_out, held) + + assert held.shutdown_calls == 1 + + def test_a_processor_built_while_shutdown_waits_is_still_closed(self): + """Shutdown cannot slip between the build and the insert, which would leave a + live exporter, with its batch thread and its connection pool, in a map nothing + will read again.""" + import threading + + built = [] + + def slow(_destination): + time.sleep(0.4) + built.append(self.Recording()) + return built[-1] + + fan_out = TenantFanOutSpanProcessor(processor_factory=slow, shutdown_drain_seconds=0.05) + acquired = [] + caller = threading.Thread(target=lambda: acquired.append(fan_out._acquire(self._dest(0)))) + caller.start() + time.sleep(0.1) + fan_out.shutdown() + caller.join(timeout=10) + + assert acquired == built, "the build shutdown waited out was thrown away" + + fan_out._release(built[0]) + self._settle(fan_out, built[0]) + + assert built[0].shutdown_calls == 1, "the exporter outlived the fan-out" + assert fan_out._processors == {}, "an exporter was left in a cleared cache" + + def test_shutdown_returns_when_a_destination_never_finishes_closing(self): + """Closing an exporter flushes over the network and the SDK joins its own + worker with no timeout, so a tenant collector that answers but never finishes + a response would hold process teardown open for as long as it likes.""" + import threading + + never = threading.Event() + + class Stuck(self.Recording): + def shutdown(self): + never.wait() + + built = [] + + def factory(_destination): + built.append(Stuck()) + return built[-1] + + fan_out = TenantFanOutSpanProcessor(processor_factory=factory, shutdown_drain_seconds=0.3) + fan_out._release(fan_out._acquire(self._dest(0))) + returned = threading.Event() + threading.Thread(target=lambda: (fan_out.shutdown(), returned.set()), daemon=True).start() + + came_back = returned.wait(timeout=8) + never.set() + + assert came_back, "shutdown never returned while a collector held its exporter open" + + def test_a_cold_cache_met_by_a_burst_builds_one_processor_per_destination(self): + """Building outside the cache lock let every thread of the burst construct its + own exporter, each with a batch thread and a connection pool, and shed all but + one into the drain.""" + import threading + + built = [] + + def factory(_destination): + time.sleep(0.01) + built.append(self.Recording()) + return built[-1] + + fan_out = TenantFanOutSpanProcessor(processor_factory=factory) + ready = threading.Barrier(8) + + def acquire(): + ready.wait() + fan_out._release(fan_out._acquire(self._dest(0))) + + callers = [threading.Thread(target=acquire) for _ in range(8)] + for caller in callers: + caller.start() + for caller in callers: + caller.join(timeout=10) + + assert len(built) == 1, f"one destination, {len(built)} exporters built" + + def test_a_submit_racing_close_is_never_stranded_behind_the_sentinels(self): + """A submit that read the closed state and then let ``close`` run queues its + processor after every sentinel, where the workers have already exited.""" + import queue + import threading + + from litellm.integrations.otel.plumbing.providers import _DrainPool + + at_the_put, close_returned = threading.Event(), threading.Event() + + class Gated(queue.Queue): + def put(self, item, *args, **kwargs): + if item is not None: + at_the_put.set() + close_returned.wait(timeout=1) + super().put(item, *args, **kwargs) + + pool = _DrainPool(pending=Gated()) + submitted = self.Recording() + submitter = threading.Thread(target=pool.submit, args=(submitted,)) + submitter.start() + assert at_the_put.wait(timeout=5) + closer = threading.Thread(target=pool.close) + closer.start() + closer.join(timeout=1.5) + close_returned.set() + submitter.join(timeout=5) + closer.join(timeout=5) + for _ in range(250): + if submitted.shutdown_calls: + break + time.sleep(0.02) + + assert submitted.shutdown_calls == 1, "a processor was queued behind the sentinels and never closed" + + def test_a_retired_processor_is_still_closed_after_shutdown(self): + """Eviction and shutdown can both land while a span is being forwarded, and the + evicted processor still has to be closed once that export returns.""" + from litellm.integrations.otel.plumbing.providers import _MAX_CACHED_DESTINATION_PROCESSORS + + fan_out, built = self._fan_out() + held = fan_out._acquire(self._dest(0)) + for index in range(1, _MAX_CACHED_DESTINATION_PROCESSORS + 1): + fan_out._acquire(self._dest(index)) + fan_out._release(built[-1]) + fan_out.shutdown() + + assert held.shutdown_calls == 0 + + fan_out._release(held) + self._settle(fan_out, held) + + assert held.shutdown_calls == 1 + + +class TestCredentialGatedExporters: + def test_layering_a_second_preset_does_not_eat_the_first_gated_exporter(self, monkeypatch): + """``base.Preset`` advertises ``config_overrides`` layering, and the gated spec + is itself a console exporter with no endpoint.""" + credential_less_proxy(monkeypatch) + from litellm.integrations.otel.presets.utils import credential_gated_exporters + + once = credential_gated_exporters((), ExporterOwner.LANGFUSE_OTEL) + twice = credential_gated_exporters(once, ExporterOwner.WEAVE_OTEL) + + assert [spec.owner for spec in twice] == [ExporterOwner.LANGFUSE_OTEL, ExporterOwner.WEAVE_OTEL] + + def test_an_exporter_the_operator_configured_survives(self): + from litellm.integrations.otel.presets.utils import credential_gated_exporters + + operator_console = ExporterSpec(kind="console", use_simple_processor=True) + + kept = credential_gated_exporters((operator_console,), ExporterOwner.LANGFUSE_OTEL) + + assert kept[0] == operator_console + + def test_an_otlp_exporter_on_its_default_endpoint_survives(self): + """``OTEL_EXPORTER=otlp_http`` with no endpoint is a real collector on the SDK's + default port, not the placeholder, so the transport is what tells them apart.""" + from litellm.integrations.otel.presets.utils import credential_gated_exporters + + operator_otlp = ExporterSpec(kind="otlp_http", endpoint=None, headers=None) + + kept = credential_gated_exporters((operator_otlp,), ExporterOwner.LANGFUSE_OTEL) + + assert kept[0] == operator_otlp + + def test_an_in_memory_exporter_the_operator_asked_for_survives(self): + """``OTEL_EXPORTER=in_memory`` stores spans, so it is a destination the operator + chose, not the placeholder that stands in for choosing nothing.""" + from litellm.integrations.otel.presets.utils import credential_gated_exporters + + operator_memory = ExporterSpec(kind="in_memory", endpoint=None, headers=None) + + kept = credential_gated_exporters((operator_memory,), ExporterOwner.LANGFUSE_OTEL) + + assert kept[0] == operator_memory + + def test_the_synthesized_stdout_placeholder_is_dropped(self, monkeypatch): + from litellm.integrations.otel.presets.utils import credential_gated_exporters + + for name in _OTEL_SHORTHAND_ENV: + monkeypatch.delenv(name, raising=False) + placeholder = OpenTelemetryV2Config().exporters[0] + + kept = credential_gated_exporters((placeholder,), ExporterOwner.LANGFUSE_OTEL) + + assert [spec.owner for spec in kept] == [ExporterOwner.LANGFUSE_OTEL] + + def test_a_console_exporter_the_operator_named_survives(self, monkeypatch): + """Same kind, endpoint and headers as the placeholder; only the fact that the + operator set ``OTEL_EXPORTER`` tells them apart.""" + from litellm.integrations.otel.presets.utils import credential_gated_exporters + + for name in _OTEL_SHORTHAND_ENV: + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("OTEL_EXPORTER", "console") + operator_console = OpenTelemetryV2Config().exporters[0] + + kept = credential_gated_exporters((operator_console,), ExporterOwner.LANGFUSE_OTEL) + + assert kept[0] is operator_console + + +class TestTenantHostSsrfGuard: + """Anyone who can mint a key can write ``langfuse_host``, so the host it names has + to be one the operator approved.""" + + @pytest.fixture(autouse=True) + def _guard_on(self, monkeypatch): + from litellm.integrations.otel.presets.destinations import _warn_host_not_allowlisted + + monkeypatch.setattr(litellm, "provider_url_destination_allowed_hosts", [], raising=False) + _warn_host_not_allowlisted.cache_clear() + yield + _warn_host_not_allowlisted.cache_clear() + + @staticmethod + def _langfuse(host: str) -> Mapping[str, str]: + return {"langfuse_public_key": "pk", "langfuse_secret_key": "sk", "langfuse_host": host} + + @pytest.mark.parametrize( + "host", + [ + "http://127.0.0.1:9111", + "http://169.254.169.254", + "http://10.0.0.5:3000", + "https://collector.example.com", + "https://langfuse.corp:99999", + "ftp://collector.example.com", + ], + ) + def test_a_host_the_operator_never_approved_resolves_to_nothing(self, host): + assert destination_for("langfuse_otel", self._langfuse(host)) is None + + def test_userinfo_naming_an_allowlisted_host_does_not_smuggle_a_second_one(self, monkeypatch): + """``https://allowed@10.0.0.5`` reads as the allowlisted host to the eye and + posts to 10.0.0.5 on the wire.""" + monkeypatch.setattr(litellm, "provider_url_destination_allowed_hosts", ["collector.example.com"], raising=False) + + assert destination_for("langfuse_otel", self._langfuse("https://collector.example.com@10.0.0.5")) is None + + def test_a_malformed_host_does_not_take_the_other_backends_with_it(self, monkeypatch): + """``urlparse(...).port`` raises a bare ValueError, which would escape + ``destination_for`` and kill the whole resolution.""" + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + monkeypatch.setenv("NEW_RELIC_OTEL_ENDPOINT", "https://otlp.nr-data.net") + monkeypatch.setattr(litellm, "provider_url_destination_allowed_hosts", ["collector.example.com"], raising=False) + is_otel_v2_enabled.cache_clear() + auth = UserAPIKeyAuth( + token="hashed", + team_metadata={ + "logging": [ + {"callback_name": "langfuse_otel", "callback_vars": self._langfuse("https://lf.corp:99999")}, + {"callback_name": "newrelic", "callback_vars": {"newrelic_api_key": "nr"}}, + ] + }, + ) + + assert [d.callback_name for d in resolve_tenant_otel_destinations(auth)] == ["newrelic"] + + def test_the_operator_can_allowlist_its_teams_internal_langfuse(self, monkeypatch): + monkeypatch.setattr(litellm, "provider_url_destination_allowed_hosts", ["127.0.0.1:9111"], raising=False) + + destination = destination_for("langfuse_otel", self._langfuse("http://127.0.0.1:9111")) + + assert destination.endpoint == "http://127.0.0.1:9111/api/public/otel" + + def test_the_operators_own_internal_host_is_never_blocked(self, monkeypatch): + """The operator configures ``LANGFUSE_HOST`` themselves, so an internal + collector there is a deployment choice rather than caller-supplied input.""" + monkeypatch.setenv("LANGFUSE_HOST", "http://127.0.0.1:9111") + + destination = destination_for("langfuse_otel", {"langfuse_public_key": "pk", "langfuse_secret_key": "sk"}) + + assert destination.endpoint == "http://127.0.0.1:9111/api/public/otel" + + def test_an_allowlisted_host_is_taken_without_resolving_it(self, monkeypatch): + """The check runs on the asyncio auth path, so it must not block on a name the + caller chose. ``.invalid`` never resolves, and it is still accepted.""" + monkeypatch.setattr(litellm, "provider_url_destination_allowed_hosts", ["lf.invalid"], raising=False) + + destination = destination_for("langfuse_otel", self._langfuse("https://lf.invalid")) + + assert destination.endpoint == "https://lf.invalid/api/public/otel" + + def test_a_rejected_host_is_warned_about_once(self, caplog): + with caplog.at_level("WARNING", logger="LiteLLM"): + for _ in range(3): + destination_for("langfuse_otel", self._langfuse("http://10.0.0.5:3000")) + + assert sum("provider_url_destination_allowed_hosts" in record.message for record in caplog.records) == 1 diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py index b735abaf7bf..2869c804c07 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py @@ -2041,7 +2041,7 @@ def test_select_global_otel_v2_logger_builds_one_when_none_registered(): assert isinstance(chosen, OpenTelemetryV2) -def test_publish_global_otel_v2_provider_sets_selected_logger_provider(): +def test_publish_global_otel_v2_provider_sets_selected_logger_provider(monkeypatch): """The startup publish must set the OTel global provider to the *selected* logger's provider (the preset logger that owns every exporter), so the FastAPI server span and the gen-ai spans share one provider and one trace. @@ -2051,8 +2051,10 @@ def test_publish_global_otel_v2_provider_sets_selected_logger_provider(): test would otherwise miss: that the published provider is the selected logger's, not some other. """ + from litellm.integrations.otel import logger as otel_logger from litellm.integrations.otel.logger import publish_global_otel_v2_provider + monkeypatch.setattr(otel_logger, "_published_v2_provider", None) cfg = OpenTelemetryV2Config(exporter="in_memory") tp = providers.build_tracer_provider(cfg) preset_logger = OpenTelemetryV2( diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 16a99713a06..2817c079e4b 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -5953,15 +5953,17 @@ def test_prompt_hooks_skip_prompt_managers_when_no_prompt_id(logging_obj, tmp_pa for hook in [cb for cb in litellm.callbacks if isinstance(cb, VectorStorePreCallHook)]: litellm.logging_callback_manager.remove_callback_from_list_by_object(litellm.callbacks, hook) def test_newrelic_dispatch_prefers_otel_v2_when_flag_on(monkeypatch): - """With LITELLM_OTEL_V2 on, the "newrelic" callback builds the OTel v2 - logger (per-team credential routing); with the flag off (default) it keeps - the legacy agent-based logger, so existing deployments are untouched.""" + """With LITELLM_OTEL_V2 on and operator credentials present, the "newrelic" + callback builds the OTel v2 logger (per-team credential routing); with the + flag off (default) it keeps the legacy agent-based logger, so existing + deployments are untouched.""" from litellm.integrations.otel.logger import OpenTelemetryV2 from litellm.integrations.otel.model.config import is_otel_v2_enabled from litellm.litellm_core_utils import litellm_logging as logging_module logging_module._in_memory_loggers.clear() monkeypatch.setenv("LITELLM_OTEL_V2", "true") + monkeypatch.setenv("NEW_RELIC_LICENSE_KEY", "test-license-key") is_otel_v2_enabled.cache_clear() try: v2_logger = logging_module._init_custom_logger_compatible_class( @@ -6016,6 +6018,7 @@ def test_get_custom_logger_compatible_class_finds_v2_newrelic(monkeypatch): logging_module._in_memory_loggers.clear() monkeypatch.setenv("LITELLM_OTEL_V2", "true") + monkeypatch.setenv("NEW_RELIC_LICENSE_KEY", "test-license-key") is_otel_v2_enabled.cache_clear() try: created = logging_module._init_custom_logger_compatible_class( diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py index 086ab854e36..bacd0d6db42 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py @@ -7319,6 +7319,75 @@ class TestMCPMetaTraceCarrier: assert _mcp_meta_trace_carrier(SimpleNamespace(meta=only_progress)) is None +@pytest.mark.asyncio +async def test_stateful_mcp_tool_call_uses_current_requests_otel_destinations() -> None: + from types import SimpleNamespace + + from mcp.server.lowlevel.server import request_ctx + from mcp.shared.context import RequestContext + + from litellm.integrations.otel.model.destination import OtelDestination + from litellm.integrations.otel.plumbing.context import ( + request_destinations, + reset_request_destinations, + set_request_destinations, + ) + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + from litellm.proxy._experimental.mcp_server.server import ( + _MCP_DESTINATIONS_SCOPE_KEY, + mcp_server_tool_call, + set_auth_context, + ) + from litellm.proxy._experimental.mcp_server.tool_registry import ( + global_mcp_tool_registry, + ) + + initialized_destination = OtelDestination(endpoint="https://initialize.example", callback_name="langfuse_otel") + current_destination = OtelDestination(endpoint="https://current.example", callback_name="arize") + server = MCPServer( + server_id="otel-context-test", + name="otelcontext", + transport=MCPTransport.http, + allow_all_keys=True, + ) + + async def observe_destinations() -> str: + assert request_destinations() == (current_destination,) + return "ok" + + global_mcp_server_manager.registry[server.server_id] = server + global_mcp_server_manager.tool_name_to_mcp_server_name_mapping["otelcontext-observe"] = server.name + global_mcp_tool_registry.register_tool( + name="otelcontext-observe", + description="Observe request destinations", + input_schema={"type": "object"}, + handler=observe_destinations, + ) + set_auth_context(None, raw_headers={}) + destinations_token = set_request_destinations((initialized_destination,)) + scope = {_MCP_DESTINATIONS_SCOPE_KEY: (current_destination,)} + current_request_context = RequestContext( + request_id=1, + meta=None, + session=SimpleNamespace(), + lifespan_context=None, + request=SimpleNamespace(scope=scope), + ) + request_token = request_ctx.set(current_request_context) + try: + result = await mcp_server_tool_call("otelcontext-observe", {}) + assert result.isError is False + assert request_destinations() == (initialized_destination,) + finally: + request_ctx.reset(request_token) + reset_request_destinations(destinations_token) + global_mcp_tool_registry.tools.pop("otelcontext-observe", None) + global_mcp_server_manager.registry.pop(server.server_id, None) + global_mcp_server_manager.tool_name_to_mcp_server_name_mapping.pop("otelcontext-observe", None) + + @pytest.mark.asyncio async def test_get_allowed_mcp_servers_includes_active_servers_submitted_by_user(): """BYOM submitters can see approved servers they submitted without allow_all_keys."""