From bcba86e426fedea716c703d5c1fa4c7a65e479b7 Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Thu, 3 Sep 2026 14:42:07 -0700 Subject: [PATCH 01/47] feat(otel v2): send a key's or team's whole trace to its own destination A key or team that configures its own Langfuse, Arize, Weave or New Relic credentials used to get a single detached span in its account while the rest of the request trace stayed on the operator's backend, so neither side held a complete trace. Resolve the destination during auth, forward every span of the request to it, and hold the same request back from the operator's exporter for that backend, so the tenant gets the tree the operator would have seen and the operator gets nothing for that request. Also let a credential-mandatory preset build without the operator's own env credentials. Without that, a proxy whose teams each bring their own account fell back to the legacy integration and never ran a line of the v2 path. --- litellm/integrations/otel/logger.py | 4 +- .../integrations/otel/model/destination.py | 55 +++ litellm/integrations/otel/plumbing/context.py | 36 +- .../integrations/otel/plumbing/providers.py | 203 +++++++++- litellm/integrations/otel/plumbing/routing.py | 10 +- litellm/integrations/otel/presets/agentops.py | 1 + litellm/integrations/otel/presets/arize.py | 23 +- litellm/integrations/otel/presets/base.py | 14 +- .../integrations/otel/presets/destinations.py | 118 ++++++ litellm/integrations/otel/presets/langfuse.py | 23 +- .../integrations/otel/presets/langtrace.py | 1 + litellm/integrations/otel/presets/levo.py | 1 + litellm/integrations/otel/presets/newrelic.py | 1 + litellm/integrations/otel/presets/phoenix.py | 1 + litellm/integrations/otel/presets/utils.py | 23 ++ litellm/integrations/otel/presets/weave.py | 21 +- litellm/litellm_core_utils/litellm_logging.py | 7 +- litellm/proxy/auth/user_api_key_auth.py | 23 ++ litellm/proxy/litellm_pre_call_utils.py | 48 +++ .../otel/test_otel_v2_destinations.py | 360 ++++++++++++++++++ 20 files changed, 952 insertions(+), 21 deletions(-) create mode 100644 litellm/integrations/otel/model/destination.py create mode 100644 litellm/integrations/otel/presets/destinations.py create mode 100644 tests/test_litellm/integrations/otel/test_otel_v2_destinations.py diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index 5519896a961..ec3c3eaa004 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -180,7 +180,9 @@ class OpenTelemetryV2(CustomLogger): self.config: OpenTelemetryV2Config = config or OpenTelemetryV2Config(**kwargs) self.callback_name = callback_name self._tracer_provider: TracerProvider = ( - tracer_provider if tracer_provider is not None else build_tracer_provider(self.config) + tracer_provider + if tracer_provider is not None + else build_tracer_provider(self.config, tenant_overrides=True) ) self.tracer: Tracer = get_tracer(self._tracer_provider, LITELLM_TRACER_NAME) self._metrics_recorder = self._init_metrics(meter_provider) diff --git a/litellm/integrations/otel/model/destination.py b/litellm/integrations/otel/model/destination.py new file mode 100644 index 00000000000..b88a240c966 --- /dev/null +++ b/litellm/integrations/otel/model/destination.py @@ -0,0 +1,55 @@ +"""The resolved OTLP destination a request's traces export to. + +A destination is a backend-agnostic target: an endpoint plus the auth headers the +exporter sends. The proxy builds one per backend from the key or team logging +config resolved at auth, and the fan-out span processor exports the request's +spans through it. Every OTEL backend reduces to this shape; the per-backend field +mapping lives in ``litellm.integrations.otel.presets.destinations``. +""" + +from collections.abc import Mapping +from typing import Final +from urllib.parse import quote + +from pydantic import BaseModel, ConfigDict, Field + + +class OtelDestination(BaseModel): + model_config = ConfigDict(frozen=True) + + endpoint: str + headers: Mapping[str, str] = Field(default_factory=dict) + resource_attributes: Mapping[str, str] = Field(default_factory=dict) + callback_name: str | None = None + protocol: str | None = Field( + default=None, + description=( + "OTLP transport for this endpoint (``otlp_http`` / ``otlp_grpc``). The " + "backend's intrinsic default is used when unset. A backend whose own cloud " + "endpoint is gRPC can still be pointed at an HTTP collector, which the " + "scheme alone cannot express: Arize's own ``https://otlp.arize.com/v1`` is gRPC." + ), + ) + + def header_string(self) -> str: + """Render headers as the ``k=v,k2=v2`` form an ``ExporterSpec`` expects. + + Values are percent-encoded because ``providers.parse_headers`` decodes them + with the SDK's W3C-Baggage parser: a value carrying a ``,`` or ``=`` (a + Langfuse project name, a base64 Authorization payload ending in ``==``) + would otherwise be split into bogus pairs on the way back out. + """ + return ",".join(f"{key}={quote(value, safe='')}" for key, value in self.headers.items()) + + def cache_key(self) -> tuple[str, tuple[tuple[str, str], ...], tuple[tuple[str, str], ...], str | None]: + """Identity for processor reuse: two requests naming the same destination + must share one exporter rather than minting a connection pool each.""" + return ( + self.endpoint, + tuple(sorted(self.headers.items())), + tuple(sorted(self.resource_attributes.items())), + self.protocol, + ) + + +NO_DESTINATIONS: Final[tuple[OtelDestination, ...]] = () diff --git a/litellm/integrations/otel/plumbing/context.py b/litellm/integrations/otel/plumbing/context.py index aa7cc8e2afd..3d8d8b53388 100644 --- a/litellm/integrations/otel/plumbing/context.py +++ b/litellm/integrations/otel/plumbing/context.py @@ -2,7 +2,7 @@ from collections.abc import Mapping from contextvars import ContextVar, Token -from typing import Final +from typing import TYPE_CHECKING, Final from opentelemetry import baggage from opentelemetry.context import Context, get_current @@ -21,6 +21,9 @@ from opentelemetry.trace.propagation.tracecontext import ( from litellm.integrations.otel.model.semconv import HTTP +if TYPE_CHECKING: + from litellm.integrations.otel.model.destination import OtelDestination + _PROPAGATOR: Final = TraceContextTextMapPropagator() # The request's root span — the FastAPI-owned SERVER span — captured ONCE when the @@ -304,3 +307,34 @@ def extract_traceparent(headers: Mapping[str, str]) -> Context | None: return None carrier: Final = {str(key).lower(): value for key, value in headers.items()} return _PROPAGATOR.extract(carrier) + + +# The OTLP destinations this request's key or team pointed its traces at, resolved +# once during auth. A ``ContextVar`` for the same reason the root span above is one: +# it rides the request task's context into the ``asyncio.create_task`` children that +# close the LLM span, and it is visible to every ``SpanProcessor.on_end`` that fires +# on the request task. Never reset -- it dies with the task. +_request_destinations: Final['ContextVar[tuple["OtelDestination", ...]]'] = ContextVar( + "litellm_otel_request_destinations", default=() +) + + +def set_request_destinations(destinations: 'tuple["OtelDestination", ...]') -> None: + """Anchor the destinations this request exports to.""" + _request_destinations.set(destinations) + + +def request_destinations() -> 'tuple["OtelDestination", ...]': + """The destinations resolved for this request, empty outside a proxy request.""" + return _request_destinations.get() + + +def overridden_backends() -> frozenset[str]: + """Backends whose global exporters this request must NOT reach. + + A team destination is an override, not an addition: once the request resolved a + destination for a backend, that backend's operator-level exporters are suppressed + for every span of the request, so the tenant's traffic reaches the tenant's + account and nowhere else. + """ + return frozenset(d.callback_name for d in _request_destinations.get() if d.callback_name) diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index fb74ff85e5b..00198234f54 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -1,5 +1,7 @@ """Provider / exporter factory + the Baggage span processor.""" +import threading +from collections import OrderedDict from collections.abc import Callable, Iterable from typing import TYPE_CHECKING, Any, Final, Literal @@ -20,6 +22,7 @@ from opentelemetry.sdk._logs.export import ( from opentelemetry.sdk.metrics import MeterProvider as SDKMeterProvider from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import ReadableSpan, SpanProcessor, TracerProvider +from opentelemetry.sdk.trace import Span as SDKSpan from opentelemetry.sdk.trace.export import ( BatchSpanProcessor, ConsoleSpanExporter, @@ -32,15 +35,22 @@ from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( from opentelemetry.trace import Span, SpanKind, Tracer from opentelemetry.util.re import parse_env_headers +from litellm._logging import verbose_logger from litellm._version import version as litellm_version from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config from litellm.integrations.otel.model.semconv import LiteLLM from litellm.integrations.otel.model.spans import LiteLLMSpanKind +from litellm.integrations.otel.plumbing.context import ( + overridden_backends, + request_destinations, +) if TYPE_CHECKING: from opentelemetry.metrics import Meter from opentelemetry.sdk.metrics.export import MetricReader + from litellm.integrations.otel.model.destination import OtelDestination + _SPAN_KIND_BY_ROLE_KIND: Final[dict[LiteLLMSpanKind, SpanKind]] = { LiteLLMSpanKind.SERVER: SpanKind.SERVER, LiteLLMSpanKind.CLIENT: SpanKind.CLIENT, @@ -194,6 +204,178 @@ def _processor_for(exporter: SpanExporter, use_simple: bool | None) -> SpanProce return SimpleSpanProcessor(exporter) if use_simple else BatchSpanProcessor(exporter) +#: Distinct tenant destinations whose exporters stay alive. Each holds a connection +#: pool and a batch thread, so the cache is bounded and evicts least-recently-used. +_MAX_CACHED_DESTINATION_PROCESSORS: Final = 32 + + +class _ResourceWrappedReadableSpan(ReadableSpan): + """A ``ReadableSpan`` view with an overridden Resource, leaving the original alone.""" + + def __init__(self, inner: ReadableSpan, resource: Resource) -> None: + super().__init__( + name=inner.name, + context=inner.context, + parent=inner.parent, + resource=resource, + attributes=inner.attributes, + events=inner.events, + links=inner.links, + kind=inner.kind, + status=inner.status, + start_time=inner.start_time, + end_time=inner.end_time, + instrumentation_scope=inner.instrumentation_scope, + ) + + +def _with_destination_resource(span: ReadableSpan, destination: "OtelDestination") -> ReadableSpan: + extra: Final = destination.resource_attributes + if not extra: + return span + merged: Final = Resource.create( + {**dict(span.resource.attributes), **dict(extra)} # mutable-ok: the OTel SDK takes a concrete attribute mapping + ) + return _ResourceWrappedReadableSpan(span, merged) + + +class TenantFanOutSpanProcessor(SpanProcessor): + """Export every finished span to each destination this request resolved. + + Destinations ride a request-scoped ``ContextVar`` set during auth, so the + processor keeps no per-request state and concurrent requests stay isolated. + Every span is forwarded, the gen-AI span included: the tenant's account gets the + tree the operator's would have received, still parented, because the forwarded + view keeps the original span's trace and parent ids. + """ + + def __init__( + self, + processor_factory: 'Callable[["OtelDestination"], SpanProcessor | None] | None' = None, + ) -> None: + self._lock: Final = threading.Lock() + self._build: Final = processor_factory if processor_factory is not None else _destination_processor + self._processors: OrderedDict[object, SpanProcessor] = OrderedDict() # mutable-ok: bounded LRU + + def on_start(self, span: SDKSpan, parent_context: Context | None = None) -> None: + return None + + def on_end(self, span: ReadableSpan) -> None: + for destination in request_destinations(): + processor = self._processor_for(destination) # rebind-ok: loop variable; pyright forbids Final in a loop + if processor is None: + continue + try: + processor.on_end(_with_destination_resource(span, destination)) + except Exception as exc: # noqa: BLE001 # one destination's failure must not cost the others their span + verbose_logger.debug("OTel V2 fan-out: forwarding to %s failed: %s", destination.endpoint, exc) + + def shutdown(self) -> None: + # Snapshot first: ``on_end`` mutates the cache on whichever thread ends a span + # and can run concurrently with this SDK-driven shutdown, so iterating the live + # mapping risks a "mutated during iteration" the per-item except cannot catch. + for processor in self._snapshot(): + try: + processor.shutdown() + except Exception as exc: # noqa: BLE001 # one processor's shutdown must not abort the rest + verbose_logger.debug("OTel V2 fan-out: processor shutdown failed: %s", exc) + with self._lock: + self._processors.clear() + + def force_flush(self, timeout_millis: int = 30000) -> bool: + results: Final = tuple(self._flush_one(processor, timeout_millis) for processor in self._snapshot()) + return all(results) + + def _snapshot(self) -> tuple[SpanProcessor, ...]: + with self._lock: + return tuple(self._processors.values()) + + @staticmethod + def _flush_one(processor: SpanProcessor, timeout_millis: int) -> bool: + try: + return processor.force_flush(timeout_millis) + except Exception: # noqa: BLE001 # one exporter's flush failure must not fail the whole flush + return False + + def _processor_for(self, destination: "OtelDestination") -> SpanProcessor | None: + key: Final = destination.cache_key() + with self._lock: + cached: Final = self._processors.get(key) + if cached is not None: + self._processors.move_to_end(key) + return cached + built: Final = self._build(destination) + if built is None: + return None + with self._lock: + existing: Final = self._processors.get(key) + if existing is not None: + # Another thread won the race; drop ours rather than leak its thread. + _shutdown_quietly(built) + return existing + self._processors[key] = built + evicted: Final = ( + self._processors.popitem(last=False)[1] + if len(self._processors) > _MAX_CACHED_DESTINATION_PROCESSORS + else None + ) + if evicted is not None: + _shutdown_quietly(evicted) + return built + + +def _destination_processor(destination: "OtelDestination") -> SpanProcessor | None: + """A batching OTLP processor aimed at ``destination``, or ``None`` if unbuildable.""" + try: + spec: Final = ExporterSpec( + kind=destination.protocol or "otlp_http", + endpoint=destination.endpoint, + headers=destination.header_string(), + owner=None, + ) + return _processor_for(_exporter_from_spec(spec), use_simple=False) + except Exception as exc: # noqa: BLE001 # a malformed destination must not break the request or the other destinations + verbose_logger.debug("OTel V2 fan-out: no processor for %s: %s", destination.endpoint, exc) + return None + + +def _shutdown_quietly(processor: SpanProcessor) -> None: + try: + processor.shutdown() + except Exception as exc: # noqa: BLE001 # defensive: shedding a spare processor must not raise + verbose_logger.debug("OTel V2 fan-out: discarding processor failed: %s", exc) + + +class _OverriddenBackendFilter(SpanProcessor): + """Hold a span back from ``owner``'s operator-level exporter when the request + pointed ``owner`` at a tenant's own account. + + A team destination is an override rather than an addition, and a ``SpanProcessor`` + cannot veto its siblings (``SynchronousMultiSpanProcessor.on_end`` ignores return + values), so suppression has to wrap the exporter's own processor. Dropping here + also keeps the span out of ``BatchSpanProcessor``'s bounded queue instead of + filling it with spans that will never ship. + """ + + def __init__(self, inner: SpanProcessor, owner: str) -> None: + self._inner: Final = inner + self._owner: Final = owner + + def on_start(self, span: SDKSpan, parent_context: Context | None = None) -> None: + self._inner.on_start(span, parent_context) + + def on_end(self, span: ReadableSpan) -> None: + if self._owner in overridden_backends(): + return + self._inner.on_end(span) + + def shutdown(self) -> None: + self._inner.shutdown() + + def force_flush(self, timeout_millis: int = 30000) -> bool: + return self._inner.force_flush(timeout_millis) + + def build_span_exporter(config: OpenTelemetryV2Config) -> SpanExporter: """Build a single exporter from the top-level config fields. @@ -437,6 +619,7 @@ def build_tracer_provider( exporter: SpanExporter | None = None, baggage_processor: SpanProcessor | None = None, use_simple_processor: bool | None = None, + tenant_overrides: bool = False, ) -> TracerProvider: """Build the shared :class:`TracerProvider`. @@ -445,6 +628,12 @@ def build_tracer_provider( ``config.exporters`` entry — this is what fans spans out to multiple backends. ``exporter`` and ``use_simple_processor`` are explicit overrides: pass a single exporter to attach exactly that one (used by tests). + + ``tenant_overrides`` belongs to the operator-level provider alone: it wraps each + owned exporter so a request that pointed that backend at a key's or team's own + account skips it, and adds the fan-out processor that delivers to that account + instead. The per-tenant providers this same function builds must leave it off, + or they would filter out the very spans they exist to carry. """ provider: Final = TracerProvider(resource=build_resource(config)) if baggage_processor is None: @@ -461,12 +650,16 @@ def build_tracer_provider( if spec.requires_headers and not spec.headers: continue exp = _exporter_from_spec(spec) - provider.add_span_processor( - _processor_for( - exp, - (spec.use_simple_processor if spec.use_simple_processor is not None else use_simple_processor), - ) + processor = _processor_for( + exp, + (spec.use_simple_processor if spec.use_simple_processor is not None else use_simple_processor), ) + owner = spec.owner.value if spec.owner is not None else None + provider.add_span_processor( + _OverriddenBackendFilter(processor, owner) if tenant_overrides and owner is not None else processor + ) + if tenant_overrides: + provider.add_span_processor(TenantFanOutSpanProcessor()) return provider diff --git a/litellm/integrations/otel/plumbing/routing.py b/litellm/integrations/otel/plumbing/routing.py index 227e18f3663..710c51d4942 100644 --- a/litellm/integrations/otel/plumbing/routing.py +++ b/litellm/integrations/otel/plumbing/routing.py @@ -25,6 +25,7 @@ from opentelemetry.trace import Tracer from litellm._logging import verbose_logger from litellm.constants import OTEL_SERVICE_NAME_METADATA_KEYS from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config +from litellm.integrations.otel.plumbing.context import overridden_backends from litellm.integrations.otel.plumbing.providers import ( build_tracer_provider, exporter_transport, @@ -231,7 +232,14 @@ class TenantTracerCache: concurrent overflow eviction can't shut it down between selection and the caller's span start. The caller must ``release`` it exactly once. """ - credential_headers: Final = self._credential_headers(dynamic_params) + # An overridden backend is delivered by the fan-out processor, which carries the + # whole trace. Routing here too would detach this span onto a second provider, + # so the tenant would get the request tree plus a stray one-span trace. + credential_headers: Final = ( + _NO_HEADERS + if self._callback_name is not None and self._callback_name in overridden_backends() + else self._credential_headers(dynamic_params) + ) project_headers: Final = self._project_headers(auth_metadata) service_name: Final = tenant_service_name(auth_metadata) if not credential_headers and not project_headers and service_name is None: diff --git a/litellm/integrations/otel/presets/agentops.py b/litellm/integrations/otel/presets/agentops.py index f45b1cd3cff..965213f2ee4 100644 --- a/litellm/integrations/otel/presets/agentops.py +++ b/litellm/integrations/otel/presets/agentops.py @@ -39,6 +39,7 @@ class _AgentOpsSettings(BaseSettings): def agentops_preset( *, config_overrides: OpenTelemetryV2Config | None = None, + allow_missing_credentials: bool = False, ) -> OpenTelemetryV2Config: """Build the AgentOps config without any network I/O. diff --git a/litellm/integrations/otel/presets/arize.py b/litellm/integrations/otel/presets/arize.py index ee0de675657..63222856b2a 100644 --- a/litellm/integrations/otel/presets/arize.py +++ b/litellm/integrations/otel/presets/arize.py @@ -11,7 +11,10 @@ from litellm.integrations.otel.model.config import ( ExporterSpec, OpenTelemetryV2Config, ) -from litellm.integrations.otel.presets.utils import ensure_mappers +from litellm.integrations.otel.presets.utils import ( + credential_gated_exporters, + ensure_mappers, +) from litellm.types.utils import StandardCallbackDynamicParams @@ -26,10 +29,22 @@ class _ArizeSettings(BaseSettings): def arize_preset( *, config_overrides: OpenTelemetryV2Config | None = None, + allow_missing_credentials: bool = False, ) -> OpenTelemetryV2Config: - arize_cfg: Final = _V1ArizeLogger.get_arize_config() - headers: Final = _arize_headers(arize_cfg) base: Final = config_overrides or OpenTelemetryV2Config() + mappers: Final = ensure_mappers(base.mapper_names, "openinference") + try: + arize_cfg: Final = _V1ArizeLogger.get_arize_config() + except Exception: + if not allow_missing_credentials: + raise + return base.model_copy( + update={ # mutable-ok: pydantic model_copy takes a plain update mapping + "exporters": credential_gated_exporters(base.exporters, ExporterOwner.ARIZE_AX), + "mapper_names": mappers, + } + ) + headers: Final = _arize_headers(arize_cfg) return base.model_copy( update={ "exporters": [ @@ -41,7 +56,7 @@ def arize_preset( owner=ExporterOwner.ARIZE_AX, ), ], - "mapper_names": ensure_mappers(base.mapper_names, "openinference"), + "mapper_names": mappers, "resource_attributes": { **base.resource_attributes, **({"model_id": arize_cfg.project_name} if arize_cfg.project_name else {}), diff --git a/litellm/integrations/otel/presets/base.py b/litellm/integrations/otel/presets/base.py index 3b9991f86a4..3b7264abd88 100644 --- a/litellm/integrations/otel/presets/base.py +++ b/litellm/integrations/otel/presets/base.py @@ -18,6 +18,18 @@ class Preset(Protocol): ``config_overrides`` lets one preset layer onto another's config (or onto test-supplied defaults); the factory calls presets with no arguments. + + ``allow_missing_credentials`` lets a credential-mandatory backend (langfuse / + arize / weave) degrade to an exporter-less, mapper-only config instead of + raising when the operator set no env credentials of their own. That is a real + deployment: every team brings its own account and the operator keeps none, and + without it the whole V2 path silently falls back to the legacy integration, so + no team destination is ever reached. Credential-optional backends ignore it. """ - def __call__(self, *, config_overrides: OpenTelemetryV2Config | None = None) -> OpenTelemetryV2Config: ... + def __call__( + self, + *, + config_overrides: OpenTelemetryV2Config | None = None, + allow_missing_credentials: bool = False, + ) -> OpenTelemetryV2Config: ... diff --git a/litellm/integrations/otel/presets/destinations.py b/litellm/integrations/otel/presets/destinations.py new file mode 100644 index 00000000000..966d8f4e1f4 --- /dev/null +++ b/litellm/integrations/otel/presets/destinations.py @@ -0,0 +1,118 @@ +"""Map a key's or team's callback vars to the OTLP destination its traces export to. + +The auth path resolves one destination per backend the caller configured, and the +fan-out span processor exports the whole request through it. Header building is +delegated to each preset's existing ``*_dynamic_headers`` builder, so a destination +authenticates exactly the way the per-request tracer route already did; only the +endpoint needs a per-backend rule, because a backend's host is either fixed, taken +from a region table, or named by the tenant alongside its own key pair. +""" + +import os +from collections.abc import Callable, Mapping +from types import MappingProxyType +from typing import Final + +from litellm.integrations.otel.model.destination import OtelDestination +from litellm.types.utils import StandardCallbackDynamicParams + +#: gRPC is Arize's own transport; an explicitly named HTTP collector overrides it. +_ARIZE_GRPC_ENDPOINT: Final = "https://otlp.arize.com/v1" + + +def _langfuse_endpoint(params: StandardCallbackDynamicParams) -> str | None: + """The tenant's own Langfuse host, else the operator's, else Langfuse US cloud. + + Falling back to the operator's host is safe and is what V1 does: the tenant's + own key pair still selects its own project, and a self-hosted deployment where + every team lives on one Langfuse server is the common shape. + """ + from litellm.integrations.langfuse.langfuse_otel import ( + LANGFUSE_CLOUD_US_ENDPOINT, + LangfuseOtelLogger, + ) + + host: Final = params.get("langfuse_host") or LangfuseOtelLogger._get_langfuse_otel_host() # pyright: ignore[reportPrivateUsage] # reuse the backend's own env host resolver rather than duplicating it + if not host: + return LANGFUSE_CLOUD_US_ENDPOINT + normalized: Final = host if host.startswith("http") else f"https://{host}" + return f"{normalized.rstrip('/')}/api/public/otel" + + +def _arize_endpoint(params: StandardCallbackDynamicParams) -> str | None: + return os.environ.get("ARIZE_ENDPOINT") or _ARIZE_GRPC_ENDPOINT + + +def _arize_protocol(params: StandardCallbackDynamicParams) -> str | None: + return "otlp_http" if os.environ.get("ARIZE_HTTP_ENDPOINT") and not os.environ.get("ARIZE_ENDPOINT") else None + + +def _weave_endpoint(params: StandardCallbackDynamicParams) -> str | None: + from litellm.integrations.weave.weave_otel import WEAVE_BASE_URL, WEAVE_OTEL_ENDPOINT + + return WEAVE_BASE_URL + WEAVE_OTEL_ENDPOINT + + +def _newrelic_endpoint(params: StandardCallbackDynamicParams) -> str | None: + from litellm.integrations.otel.presets.newrelic import newrelic_dynamic_endpoint + + return newrelic_dynamic_endpoint(params) + + +#: Callback name -> endpoint resolver. A backend is destination-capable exactly +#: when it appears here AND in ``DYNAMIC_HEADERS_BY_CALLBACK``: without a header +#: builder the destination would carry no tenant credentials, and the exporter +#: would post the tenant's traffic to the operator's account. +_ENDPOINT_BY_CALLBACK: Final[Mapping[str, Callable[[StandardCallbackDynamicParams], str | None]]] = MappingProxyType( + { + "langfuse_otel": _langfuse_endpoint, + "arize": _arize_endpoint, + "weave_otel": _weave_endpoint, + "newrelic": _newrelic_endpoint, + } +) + +_PROTOCOL_BY_CALLBACK: Final[Mapping[str, Callable[[StandardCallbackDynamicParams], str | None]]] = MappingProxyType( + { + "arize": _arize_protocol, + } +) + +_NO_ATTRS: Final[Mapping[str, str]] = MappingProxyType({}) + + +def destination_capable_backends() -> frozenset[str]: + """Backends a key or team can point at its own account.""" + from litellm.integrations.otel.presets import DYNAMIC_HEADERS_BY_CALLBACK + + return frozenset(_ENDPOINT_BY_CALLBACK) & frozenset(DYNAMIC_HEADERS_BY_CALLBACK) + + +def destination_for(callback_name: str, params: StandardCallbackDynamicParams) -> OtelDestination | None: + """The destination ``params`` names for ``callback_name``, or ``None``. + + ``None`` means the caller configured nothing usable for this backend, so the + request keeps the operator's global exporters. A partial config (a host with + no key pair) resolves to ``None`` rather than to the operator's endpoint with + the tenant's host, which would post the operator's credentials elsewhere. + """ + from litellm.integrations.otel.presets import DYNAMIC_HEADERS_BY_CALLBACK + + header_builder: Final = DYNAMIC_HEADERS_BY_CALLBACK.get(callback_name) + endpoint_builder: Final = _ENDPOINT_BY_CALLBACK.get(callback_name) + if header_builder is None or endpoint_builder is None: + return None + headers: Final = header_builder(params) + if not headers: + return None + endpoint: Final = endpoint_builder(params) + if not endpoint: + return None + protocol_builder: Final = _PROTOCOL_BY_CALLBACK.get(callback_name) + return OtelDestination( + endpoint=endpoint, + headers=MappingProxyType(dict(headers)), + resource_attributes=_NO_ATTRS, + callback_name=callback_name, + protocol=protocol_builder(params) if protocol_builder is not None else None, + ) diff --git a/litellm/integrations/otel/presets/langfuse.py b/litellm/integrations/otel/presets/langfuse.py index c2f64422eff..9149e0c0d94 100644 --- a/litellm/integrations/otel/presets/langfuse.py +++ b/litellm/integrations/otel/presets/langfuse.py @@ -10,17 +10,32 @@ from litellm.integrations.otel.model.config import ( ExporterSpec, OpenTelemetryV2Config, ) -from litellm.integrations.otel.presets.utils import ensure_mappers +from litellm.integrations.otel.presets.utils import ( + credential_gated_exporters, + ensure_mappers, +) from litellm.types.utils import StandardCallbackDynamicParams def langfuse_preset( *, config_overrides: OpenTelemetryV2Config | None = None, + allow_missing_credentials: bool = False, ) -> OpenTelemetryV2Config: - cfg: Final = _V1Langfuse.get_langfuse_otel_config() - kind: Final = cfg.exporter if isinstance(cfg.exporter, str) else "otlp_http" base: Final = config_overrides or OpenTelemetryV2Config() + mappers: Final = ensure_mappers(base.mapper_names, "langfuse") + try: + cfg: Final = _V1Langfuse.get_langfuse_otel_config() + except Exception: + if not allow_missing_credentials: + raise + return base.model_copy( + update={ # mutable-ok: pydantic model_copy takes a plain update mapping + "exporters": credential_gated_exporters(base.exporters, ExporterOwner.LANGFUSE_OTEL), + "mapper_names": mappers, + } + ) + kind: Final = cfg.exporter if isinstance(cfg.exporter, str) else "otlp_http" return base.model_copy( update={ "exporters": [ @@ -32,7 +47,7 @@ def langfuse_preset( owner=ExporterOwner.LANGFUSE_OTEL, ), ], - "mapper_names": ensure_mappers(base.mapper_names, "langfuse"), + "mapper_names": mappers, } ) diff --git a/litellm/integrations/otel/presets/langtrace.py b/litellm/integrations/otel/presets/langtrace.py index c88e4715ab0..2312575f04a 100644 --- a/litellm/integrations/otel/presets/langtrace.py +++ b/litellm/integrations/otel/presets/langtrace.py @@ -9,6 +9,7 @@ from litellm.integrations.otel.presets.utils import ensure_mappers def langtrace_preset( *, config_overrides: OpenTelemetryV2Config | None = None, + allow_missing_credentials: bool = False, ) -> OpenTelemetryV2Config: """Compose the Langtrace mapper on top of the customer's OTLP destination. diff --git a/litellm/integrations/otel/presets/levo.py b/litellm/integrations/otel/presets/levo.py index 41b3758cf3e..c1580cf7a5b 100644 --- a/litellm/integrations/otel/presets/levo.py +++ b/litellm/integrations/otel/presets/levo.py @@ -13,6 +13,7 @@ from litellm.integrations.otel.model.config import ( def levo_preset( *, config_overrides: OpenTelemetryV2Config | None = None, + allow_missing_credentials: bool = False, ) -> OpenTelemetryV2Config: cfg: Final = _V1Levo.get_levo_config() base: Final = config_overrides or OpenTelemetryV2Config() diff --git a/litellm/integrations/otel/presets/newrelic.py b/litellm/integrations/otel/presets/newrelic.py index 4660a707355..771b3f643c8 100644 --- a/litellm/integrations/otel/presets/newrelic.py +++ b/litellm/integrations/otel/presets/newrelic.py @@ -44,6 +44,7 @@ class _NewRelicSettings(BaseSettings): def newrelic_preset( *, config_overrides: OpenTelemetryV2Config | None = None, + allow_missing_credentials: bool = False, ) -> OpenTelemetryV2Config: settings: Final = _NewRelicSettings() base: Final = config_overrides or OpenTelemetryV2Config() diff --git a/litellm/integrations/otel/presets/phoenix.py b/litellm/integrations/otel/presets/phoenix.py index eef407b6c1b..f4f34ee7525 100644 --- a/litellm/integrations/otel/presets/phoenix.py +++ b/litellm/integrations/otel/presets/phoenix.py @@ -60,6 +60,7 @@ def phoenix_project_headers(auth_metadata: Mapping[str, str] | None) -> Mapping[ def phoenix_preset( *, config_overrides: OpenTelemetryV2Config | None = None, + allow_missing_credentials: bool = False, ) -> OpenTelemetryV2Config: cfg: Final = _V1Phoenix.get_arize_phoenix_config() headers: Final = cfg.otlp_auth_headers if hasattr(cfg, "otlp_auth_headers") else None diff --git a/litellm/integrations/otel/presets/utils.py b/litellm/integrations/otel/presets/utils.py index 328569d3daf..ef6e80ff33e 100644 --- a/litellm/integrations/otel/presets/utils.py +++ b/litellm/integrations/otel/presets/utils.py @@ -3,6 +3,11 @@ from collections.abc import Iterable from typing import Final +from litellm.integrations.otel.model.config import ExporterOwner, ExporterSpec + +#: What ``OpenTelemetryV2Config._normalize`` folds in when no destination is configured. +_DEFAULT_SHORTHAND_EXPORTER: Final = ExporterSpec() + def ensure_mappers(mapper_names: Iterable[str], *names: str) -> list[str]: """Return ``mapper_names`` with each of ``names`` appended if not already present. @@ -15,3 +20,21 @@ def ensure_mappers(mapper_names: Iterable[str], *names: str) -> list[str]: if name not in result: result.append(name) return result + + +def credential_gated_exporters( + exporters: "Iterable[ExporterSpec]", owner: "ExporterOwner" +) -> "tuple[ExporterSpec, ...]": + """``exporters`` with the operator's destination replaced by a header-gated one. + + Used when a credential-mandatory backend is asked to build without the operator's + own credentials, so only key/team destinations receive spans. Two things have to + happen for that to mean "export nowhere": the placeholder console spec that + ``OpenTelemetryV2Config`` folds in for an empty exporter list is dropped, or every + span would be printed to stdout, and the gated spec keeps the owner so the + override filter still recognises which backend this provider speaks for. + """ + return ( + *(spec for spec in exporters if spec != _DEFAULT_SHORTHAND_EXPORTER), + ExporterSpec(owner=owner, requires_headers=True), + ) diff --git a/litellm/integrations/otel/presets/weave.py b/litellm/integrations/otel/presets/weave.py index 51d0ad01093..644cd39ad36 100644 --- a/litellm/integrations/otel/presets/weave.py +++ b/litellm/integrations/otel/presets/weave.py @@ -7,7 +7,10 @@ from litellm.integrations.otel.model.config import ( ExporterSpec, OpenTelemetryV2Config, ) -from litellm.integrations.otel.presets.utils import ensure_mappers +from litellm.integrations.otel.presets.utils import ( + credential_gated_exporters, + ensure_mappers, +) from litellm.integrations.weave.weave_otel import ( _get_weave_authorization_header, get_weave_otel_config, @@ -18,9 +21,21 @@ from litellm.types.utils import StandardCallbackDynamicParams def weave_preset( *, config_overrides: OpenTelemetryV2Config | None = None, + allow_missing_credentials: bool = False, ) -> OpenTelemetryV2Config: - weave_cfg: Final = get_weave_otel_config() base: Final = config_overrides or OpenTelemetryV2Config() + mappers: Final = ensure_mappers(base.mapper_names, "openinference", "weave") + try: + weave_cfg: Final = get_weave_otel_config() + except Exception: + if not allow_missing_credentials: + raise + return base.model_copy( + update={ # mutable-ok: pydantic model_copy takes a plain update mapping + "exporters": credential_gated_exporters(base.exporters, ExporterOwner.WEAVE_OTEL), + "mapper_names": mappers, + } + ) return base.model_copy( update={ "exporters": [ @@ -33,7 +48,7 @@ def weave_preset( ), ], # Weave consumes OpenInference + a small Weave-specific overlay. - "mapper_names": ensure_mappers(base.mapper_names, "openinference", "weave"), + "mapper_names": mappers, } ) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 989777e9412..c7e8678be68 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -4815,11 +4815,16 @@ def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[Custom if isinstance(callback, OpenTelemetryV2) and getattr(callback, "callback_name", None) == callback_name: return callback try: - config: Final = preset_fn() + config: Final = preset_fn(allow_missing_credentials=True) except Exception: # If env vars are missing or the preset raises, defer to the legacy path # so customers get the same error story they had before V2 landed. return None + if all(spec.requires_headers and not spec.headers for spec in config.exporters): + verbose_logger.warning( + "OTel V2: no operator credentials for '%s'; only key/team destinations will receive its traces", + callback_name, + ) v2_logger: Final = build_otel_v2_logger(config=config, callback_name=callback_name) _in_memory_loggers.append(v2_logger) return v2_logger diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 93293db24c6..4a0257311d1 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -2845,6 +2845,28 @@ async def _authorize_authenticated_request( @tracer.wrap() +def _seed_request_destinations(user_api_key_dict: UserAPIKeyAuth) -> None: + """Anchor the OTLP destinations this key or team overrides its traces to. + + Called inside the ``auth`` phase span so that span reaches the tenant's account + as well, and on the request task so the ``ContextVar`` is inherited by the logging + tasks that close the LLM span. Best-effort: trace routing must never fail auth. + + The two ``postgres`` spans under ``auth`` close before this runs, because they are + the reads that resolve the identity being read here, so they keep going to the + operator's backend alone. + """ + try: + from litellm.integrations.otel.plumbing.context import set_request_destinations + from litellm.proxy.litellm_pre_call_utils import ( + resolve_tenant_otel_destinations, + ) + + set_request_destinations(resolve_tenant_otel_destinations(user_api_key_dict)) + except Exception as exc: # noqa: BLE001 # telemetry routing is best-effort and must never break authentication + verbose_proxy_logger.debug("OTel V2: tenant destination resolution failed: %s", exc) + + async def user_api_key_auth( request: Request, api_key: str = fastapi.Security(api_key_header), @@ -2891,6 +2913,7 @@ async def user_api_key_auth( raise body_parse_exception raise user_api_key_auth_obj.budget_reservation = None + _seed_request_destinations(user_api_key_auth_obj) # A body that never parsed is authenticated (so the trace carries identity # and this ``auth`` span) but not authorized: there is no model to check it diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index d026c5510e6..6069ed9a8c6 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -10,6 +10,7 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, cast from fastapi import HTTPException, Request +from pydantic import TypeAdapter from pydantic import ValidationError as PydanticValidationError from starlette.datastructures import Headers @@ -157,6 +158,7 @@ from litellm.types.utils import ( CustomPricingLiteLLMParams, LlmProviders, ProviderSpecificHeader, + StandardCallbackDynamicParams, StandardLoggingUserAPIKeyMetadata, SupportedCacheControls, ) @@ -170,6 +172,7 @@ _ENABLE_TEAM_STALE_ALIAS_BYPASS: bool | None = None if TYPE_CHECKING: + from litellm.integrations.otel.model.destination import OtelDestination from litellm.proxy.proxy_server import ProxyConfig as _ProxyConfig from litellm.types.proxy.policy_engine import Policy, PolicyMatchContext @@ -974,6 +977,51 @@ def _get_dynamic_logging_metadata( return callback_settings_obj +_TENANT_OTEL_PARAMS: Final = TypeAdapter(StandardCallbackDynamicParams) + + +def _tenant_otel_params(callback_vars: Mapping[str, str]) -> StandardCallbackDynamicParams: + try: + return _TENANT_OTEL_PARAMS.validate_python(callback_vars) + except PydanticValidationError: + return StandardCallbackDynamicParams() + + +def resolve_tenant_otel_destinations( + user_api_key_dict: UserAPIKeyAuth, +) -> "tuple[OtelDestination, ...]": + """The OTLP destinations this request's key or team config overrides its traces to. + + Key settings win over team settings outright, the same precedence + ``_get_dynamic_logging_metadata`` applies, so one caller never exports the same + backend to two accounts. Returns empty when OTEL V2 is off, when neither level + named a destination-capable backend, or when the config is incomplete, and the + request then keeps the operator's own exporters. + """ + from litellm.integrations.otel.model.config import is_otel_v2_enabled + from litellm.integrations.otel.presets.destinations import destination_for + + if not is_otel_v2_enabled(): + return () + entries: Final = KeyAndTeamLoggingSettings.get_key_dynamic_logging_settings( + user_api_key_dict + ) or KeyAndTeamLoggingSettings.get_team_dynamic_logging_settings(user_api_key_dict) + if not entries: + return () + resolved: Final = tuple( + destination + for item in entries + if (callback := _get_validated_callback_metadata(item=item, source="otel-destination")) is not None + if (destination := destination_for(callback.callback_name, _tenant_otel_params(callback.callback_vars))) + is not None + ) + return tuple( + destination + for index, destination in enumerate(resolved) + if destination.callback_name not in tuple(earlier.callback_name for earlier in resolved[:index]) + ) + + def clean_headers( headers: Headers, litellm_key_header_name: str | None = None, diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py new file mode 100644 index 00000000000..ee67160533f --- /dev/null +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -0,0 +1,360 @@ +"""Key/team OTLP destinations override the operator's exporters for that backend.""" + +import contextvars +from collections.abc import Mapping + +import pytest +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from litellm.integrations.otel.model.config import ( + ExporterOwner, + ExporterSpec, + OpenTelemetryV2Config, + is_otel_v2_enabled, +) +from litellm.integrations.otel.model.destination import OtelDestination +from litellm.integrations.otel.plumbing.context import ( + overridden_backends, + request_destinations, + set_request_destinations, +) +from litellm.integrations.otel.plumbing.providers import ( + TenantFanOutSpanProcessor, + _OverriddenBackendFilter, + build_tracer_provider, +) +from litellm.integrations.otel.plumbing.routing import TenantTracerCache, get_tracer +from litellm.integrations.otel.presets.destinations import ( + destination_capable_backends, + destination_for, +) +from litellm.integrations.otel.presets.langfuse import langfuse_preset +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.litellm_pre_call_utils import resolve_tenant_otel_destinations + +LANGFUSE_DEST = OtelDestination( + endpoint="http://tenant.local/api/public/otel", + headers={"Authorization": "Basic dGVuYW50"}, + callback_name="langfuse_otel", +) + + +def in_fresh_context(fn, *args): + """Run ``fn`` in its own context so one test's destinations never leak.""" + return contextvars.copy_context().run(fn, *args) + + +def emit(provider: TracerProvider, name: str = "chat gpt-4") -> None: + with get_tracer(provider, "litellm").start_as_current_span(name): + pass + + +def wired_provider(dest_exporter: InMemorySpanExporter, global_exporter: InMemorySpanExporter) -> TracerProvider: + """The operator's provider: one owned exporter plus the tenant fan-out.""" + provider = TracerProvider() + provider.add_span_processor(_OverriddenBackendFilter(SimpleSpanProcessor(global_exporter), "langfuse_otel")) + provider.add_span_processor( + TenantFanOutSpanProcessor(processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter)) + ) + return provider + + +class TestOverrideSuppression: + def test_operator_exporter_keeps_the_span_when_no_destination_is_resolved(self): + global_exporter, dest_exporter = InMemorySpanExporter(), InMemorySpanExporter() + provider = wired_provider(dest_exporter, global_exporter) + + in_fresh_context(emit, provider) + + assert [s.name for s in global_exporter.get_finished_spans()] == ["chat gpt-4"] + assert dest_exporter.get_finished_spans() == () + + def test_operator_exporter_is_skipped_once_the_backend_is_overridden(self): + global_exporter, dest_exporter = InMemorySpanExporter(), InMemorySpanExporter() + provider = wired_provider(dest_exporter, global_exporter) + + def run(): + set_request_destinations((LANGFUSE_DEST,)) + emit(provider) + + in_fresh_context(run) + + assert global_exporter.get_finished_spans() == () + assert [s.name for s in dest_exporter.get_finished_spans()] == ["chat gpt-4"] + + def test_a_backend_the_request_did_not_override_still_exports(self): + arize_exporter, dest_exporter = InMemorySpanExporter(), InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(_OverriddenBackendFilter(SimpleSpanProcessor(arize_exporter), "arize")) + provider.add_span_processor( + TenantFanOutSpanProcessor(processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter)) + ) + + def run(): + set_request_destinations((LANGFUSE_DEST,)) + emit(provider) + + in_fresh_context(run) + + assert [s.name for s in arize_exporter.get_finished_spans()] == ["chat gpt-4"] + + +class TestFanOut: + def test_every_span_of_the_request_reaches_the_destination_in_one_trace(self): + """The whole tree, gen-AI span included, parented as the operator would see it.""" + dest_exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor( + TenantFanOutSpanProcessor(processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter)) + ) + tracer = get_tracer(provider, "litellm") + + def run(): + set_request_destinations((LANGFUSE_DEST,)) + with tracer.start_as_current_span("POST /v1/chat/completions"): + with tracer.start_as_current_span("auth /v1/chat/completions"): + pass + with tracer.start_as_current_span("chat gpt-4"): + pass + + in_fresh_context(run) + + spans = dest_exporter.get_finished_spans() + by_name = {s.name: s for s in spans} + assert set(by_name) == {"POST /v1/chat/completions", "auth /v1/chat/completions", "chat gpt-4"} + root = by_name["POST /v1/chat/completions"] + assert len({s.context.trace_id for s in spans}) == 1, "the tenant must receive one connected trace" + for child in ("auth /v1/chat/completions", "chat gpt-4"): + assert by_name[child].parent.span_id == root.context.span_id + + def test_two_destinations_each_receive_their_own_copy(self): + first, second = InMemorySpanExporter(), InMemorySpanExporter() + by_endpoint = {"http://a.local": first, "http://b.local": second} + provider = TracerProvider() + provider.add_span_processor( + TenantFanOutSpanProcessor( + processor_factory=lambda d: SimpleSpanProcessor(by_endpoint[d.endpoint]), + ) + ) + + def run(): + set_request_destinations( + ( + OtelDestination(endpoint="http://a.local", callback_name="langfuse_otel"), + OtelDestination(endpoint="http://b.local", callback_name="arize"), + ) + ) + emit(provider) + + in_fresh_context(run) + + assert [s.name for s in first.get_finished_spans()] == ["chat gpt-4"] + assert [s.name for s in second.get_finished_spans()] == ["chat gpt-4"] + + def test_a_destination_that_cannot_build_a_processor_is_skipped_quietly(self): + """An unbuildable destination must not cost the caller its request.""" + attempts = [] + reached_the_end = [] + + def factory(destination): + attempts.append(destination.endpoint) + + provider = TracerProvider() + provider.add_span_processor(TenantFanOutSpanProcessor(processor_factory=factory)) + + def run(): + set_request_destinations((LANGFUSE_DEST,)) + emit(provider) + reached_the_end.append(True) + + in_fresh_context(run) + + assert attempts == [LANGFUSE_DEST.endpoint] + assert reached_the_end == [True] + + def test_one_processor_is_reused_across_spans_of_the_same_destination(self): + built = [] + + def factory(_destination): + processor = SimpleSpanProcessor(InMemorySpanExporter()) + built.append(processor) + return processor + + provider = TracerProvider() + provider.add_span_processor(TenantFanOutSpanProcessor(processor_factory=factory)) + + def run(): + set_request_destinations((LANGFUSE_DEST,)) + emit(provider, "one") + emit(provider, "two") + + in_fresh_context(run) + + assert len(built) == 1 + + +class TestProviderWiring: + def test_build_tracer_provider_only_filters_when_asked(self): + config = OpenTelemetryV2Config(exporters=[ExporterSpec(kind="in_memory", owner=ExporterOwner.LANGFUSE_OTEL)]) + operator = build_tracer_provider(config, tenant_overrides=True) + tenant = build_tracer_provider(config) + + def kinds(provider): + return [type(p).__name__ for p in provider._active_span_processor._span_processors] + + assert "_OverriddenBackendFilter" in kinds(operator) + assert "TenantFanOutSpanProcessor" in kinds(operator) + assert "_OverriddenBackendFilter" not in kinds(tenant), "a per-tenant provider must not filter itself out" + assert "TenantFanOutSpanProcessor" not in kinds(tenant) + + +class TestRouting: + def test_an_overridden_backend_is_not_detached_onto_a_second_provider(self): + config = OpenTelemetryV2Config( + exporters=[ExporterSpec(kind="otlp_http", endpoint="http://op.local", owner=ExporterOwner.LANGFUSE_OTEL)] + ) + cache = TenantTracerCache(config, "langfuse_otel", "litellm") + default = get_tracer(TracerProvider(), "litellm") + params = {"langfuse_public_key": "pk", "langfuse_secret_key": "sk"} + + assert cache.route_for(default, params).detached is True + + def run(): + set_request_destinations((LANGFUSE_DEST,)) + return cache.route_for(default, params) + + route = in_fresh_context(run) + assert route.detached is False + assert route.tracer is default + assert route.provider is None + + +class TestDestinationResolution: + def test_a_langfuse_key_pair_and_host_become_a_destination(self, monkeypatch): + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + auth = UserAPIKeyAuth( + team_metadata={ + "logging": [ + { + "callback_name": "langfuse_otel", + "callback_type": "success", + "callback_vars": { + "langfuse_public_key": "pk-team", + "langfuse_secret_key": "sk-team", + "langfuse_host": "http://team.local", + }, + } + ] + } + ) + + destinations = resolve_tenant_otel_destinations(auth) + + assert [d.endpoint for d in destinations] == ["http://team.local/api/public/otel"] + assert destinations[0].callback_name == "langfuse_otel" + + def test_the_key_wins_over_the_team_for_the_same_backend(self, monkeypatch): + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + + def entry(host: str) -> Mapping[str, object]: + return { + "callback_name": "langfuse_otel", + "callback_type": "success", + "callback_vars": { + "langfuse_public_key": "pk", + "langfuse_secret_key": "sk", + "langfuse_host": host, + }, + } + + auth = UserAPIKeyAuth( + metadata={"logging": [entry("http://key.local")]}, + team_metadata={"logging": [entry("http://team.local")]}, + ) + + assert [d.endpoint for d in resolve_tenant_otel_destinations(auth)] == ["http://key.local/api/public/otel"] + + def test_nothing_resolves_while_otel_v2_is_off(self, monkeypatch): + monkeypatch.delenv("LITELLM_OTEL_V2", raising=False) + is_otel_v2_enabled.cache_clear() + auth = UserAPIKeyAuth( + team_metadata={ + "logging": [ + { + "callback_name": "langfuse_otel", + "callback_type": "success", + "callback_vars": {"langfuse_public_key": "pk", "langfuse_secret_key": "sk"}, + } + ] + } + ) + + assert resolve_tenant_otel_destinations(auth) == () + + def test_a_host_without_its_key_pair_resolves_to_nothing(self): + assert destination_for("langfuse_otel", {"langfuse_host": "http://team.local"}) is None + + def test_a_backend_with_no_dynamic_credentials_has_no_destination(self): + assert "arize_phoenix" not in destination_capable_backends() + assert destination_for("arize_phoenix", {"arize_api_key": "k"}) is None + + def test_the_destination_header_string_survives_the_exporter_round_trip(self): + from litellm.integrations.otel.plumbing.providers import parse_headers + + destination = destination_for( + "langfuse_otel", + {"langfuse_public_key": "pk", "langfuse_secret_key": "sk", "langfuse_host": "http://x"}, + ) + assert parse_headers(destination.header_string())["authorization"] == destination.headers["Authorization"] + + +class TestPresetDegradation: + def test_a_credential_less_langfuse_exports_nowhere_instead_of_to_the_console(self, monkeypatch, capsys): + """``_normalize`` folds a console exporter in for an empty list, which would + print every span on a proxy whose teams bring their own credentials.""" + monkeypatch.delenv("LANGFUSE_PUBLIC_KEY", raising=False) + monkeypatch.delenv("LANGFUSE_SECRET_KEY", raising=False) + + config = langfuse_preset(allow_missing_credentials=True) + provider = build_tracer_provider(config, tenant_overrides=True) + capsys.readouterr() + in_fresh_context(emit, provider) + provider.force_flush() + + assert capsys.readouterr().out == "" + assert "langfuse" in config.mapper_names + + def test_langfuse_still_raises_for_a_global_callback_with_no_credentials(self, monkeypatch): + monkeypatch.delenv("LANGFUSE_PUBLIC_KEY", raising=False) + monkeypatch.delenv("LANGFUSE_SECRET_KEY", raising=False) + + with pytest.raises(ValueError, match="LANGFUSE_PUBLIC_KEY"): + langfuse_preset() + + def test_a_credential_less_proxy_still_builds_the_v2_logger(self, monkeypatch): + from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 + + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + monkeypatch.delenv("LANGFUSE_PUBLIC_KEY", raising=False) + monkeypatch.delenv("LANGFUSE_SECRET_KEY", raising=False) + + is_otel_v2_enabled.cache_clear() + logger = _maybe_construct_otel_v2("langfuse_otel", []) + is_otel_v2_enabled.cache_clear() + + assert logger is not None, "team-only deployments must not fall back to the legacy integration" + assert all(spec.requires_headers and not spec.headers for spec in logger.config.exporters) + + +class TestContextIsolation: + def test_destinations_do_not_leak_between_requests(self): + def first(): + set_request_destinations((LANGFUSE_DEST,)) + return overridden_backends() + + assert in_fresh_context(first) == frozenset({"langfuse_otel"}) + assert in_fresh_context(request_destinations) == () From 1f6b80e659623cc636b7bcc23356c72bf396ce65 Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Thu, 3 Sep 2026 15:50:06 -0700 Subject: [PATCH 02/47] fix(otel v2): validate tenant destinations and match each backend's own endpoint Review round on the tenant destination routing. - A key/team Langfuse host is user-supplied input, so it goes through the proxy's SSRF guard. A private address is refused, the operator keeps the trace, and the warning names user_url_allowed_hosts. The operator's own LANGFUSE_HOST is not checked. - Arize and Weave destinations now resolve their endpoint and transport through the backend's own config, so an ARIZE_HTTP_ENDPOINT collector and a self-hosted WANDB_HOST are honoured instead of the cloud default. - A half-configured backend no longer resolves: several dynamic header builders gate each credential separately, so an api key with no space id produced a non-empty but unusable header set that suppressed the operator's exporter. - A callback_type of "failure" no longer takes over the trace. The destination is resolved during auth, before the outcome is known. - The fan-out cache evicts without shutting the processor down, matching ArizePhoenixLogger: a concurrent on_end may still hold it. - The stdout placeholder is identified by what it does rather than by equality with an import-time default, so an operator's OTEL_EXPORTER_OTLP_* collector survives the credential-less path. --- .../integrations/otel/model/destination.py | 16 +- .../integrations/otel/plumbing/providers.py | 27 +- .../integrations/otel/presets/destinations.py | 73 +++-- litellm/integrations/otel/presets/utils.py | 16 +- litellm/integrations/weave/weave_otel.py | 20 +- litellm/litellm_core_utils/url_utils.py | 49 ++++ litellm/proxy/litellm_pre_call_utils.py | 7 + .../otel/test_otel_v2_destinations.py | 251 +++++++++++++++++- 8 files changed, 390 insertions(+), 69 deletions(-) diff --git a/litellm/integrations/otel/model/destination.py b/litellm/integrations/otel/model/destination.py index b88a240c966..299253cac77 100644 --- a/litellm/integrations/otel/model/destination.py +++ b/litellm/integrations/otel/model/destination.py @@ -1,10 +1,7 @@ """The resolved OTLP destination a request's traces export to. -A destination is a backend-agnostic target: an endpoint plus the auth headers the -exporter sends. The proxy builds one per backend from the key or team logging -config resolved at auth, and the fan-out span processor exports the request's -spans through it. Every OTEL backend reduces to this shape; the per-backend field -mapping lives in ``litellm.integrations.otel.presets.destinations``. +Backend-agnostic on purpose: every OTEL backend reduces to an endpoint plus auth +headers. The per-backend field mapping lives in ``presets.destinations``. """ from collections.abc import Mapping @@ -24,10 +21,8 @@ class OtelDestination(BaseModel): protocol: str | None = Field( default=None, description=( - "OTLP transport for this endpoint (``otlp_http`` / ``otlp_grpc``). The " - "backend's intrinsic default is used when unset. A backend whose own cloud " - "endpoint is gRPC can still be pointed at an HTTP collector, which the " - "scheme alone cannot express: Arize's own ``https://otlp.arize.com/v1`` is gRPC." + "OTLP transport, defaulting to the backend's own. Not derivable from the " + "scheme: Arize's ``https://otlp.arize.com/v1`` is gRPC." ), ) @@ -42,8 +37,7 @@ class OtelDestination(BaseModel): return ",".join(f"{key}={quote(value, safe='')}" for key, value in self.headers.items()) def cache_key(self) -> tuple[str, tuple[tuple[str, str], ...], tuple[tuple[str, str], ...], str | None]: - """Identity for processor reuse: two requests naming the same destination - must share one exporter rather than minting a connection pool each.""" + """Identity for processor reuse, so one destination means one exporter.""" return ( self.endpoint, tuple(sorted(self.headers.items())), diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index 00198234f54..232827e3d47 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -242,11 +242,9 @@ def _with_destination_resource(span: ReadableSpan, destination: "OtelDestination class TenantFanOutSpanProcessor(SpanProcessor): """Export every finished span to each destination this request resolved. - Destinations ride a request-scoped ``ContextVar`` set during auth, so the - processor keeps no per-request state and concurrent requests stay isolated. - Every span is forwarded, the gen-AI span included: the tenant's account gets the - tree the operator's would have received, still parented, because the forwarded - view keeps the original span's trace and parent ids. + Destinations ride a request-scoped ``ContextVar`` set during auth, so concurrent + requests stay isolated. The forwarded view keeps the original trace and parent + ids, so the tenant gets the same tree the operator would have received. """ def __init__( @@ -314,13 +312,11 @@ class TenantFanOutSpanProcessor(SpanProcessor): _shutdown_quietly(built) return existing self._processors[key] = built - evicted: Final = ( - self._processors.popitem(last=False)[1] - if len(self._processors) > _MAX_CACHED_DESTINATION_PROCESSORS - else None - ) - if evicted is not None: - _shutdown_quietly(evicted) + if len(self._processors) > _MAX_CACHED_DESTINATION_PROCESSORS: + # Evict without shutting down: another thread may be inside ``on_end`` + # holding the victim, and a shut-down BatchSpanProcessor drops spans + # silently. Same rule as ArizePhoenixLogger's per-project cache. + self._processors.popitem(last=False) return built @@ -350,11 +346,8 @@ class _OverriddenBackendFilter(SpanProcessor): """Hold a span back from ``owner``'s operator-level exporter when the request pointed ``owner`` at a tenant's own account. - A team destination is an override rather than an addition, and a ``SpanProcessor`` - cannot veto its siblings (``SynchronousMultiSpanProcessor.on_end`` ignores return - values), so suppression has to wrap the exporter's own processor. Dropping here - also keeps the span out of ``BatchSpanProcessor``'s bounded queue instead of - filling it with spans that will never ship. + Wrapping is the only place this works: ``SynchronousMultiSpanProcessor.on_end`` + ignores return values, so a sibling processor can never veto the export. """ def __init__(self, inner: SpanProcessor, owner: str) -> None: diff --git a/litellm/integrations/otel/presets/destinations.py b/litellm/integrations/otel/presets/destinations.py index 966d8f4e1f4..2589f5dbf3e 100644 --- a/litellm/integrations/otel/presets/destinations.py +++ b/litellm/integrations/otel/presets/destinations.py @@ -1,11 +1,8 @@ """Map a key's or team's callback vars to the OTLP destination its traces export to. -The auth path resolves one destination per backend the caller configured, and the -fan-out span processor exports the whole request through it. Header building is -delegated to each preset's existing ``*_dynamic_headers`` builder, so a destination -authenticates exactly the way the per-request tracer route already did; only the -endpoint needs a per-backend rule, because a backend's host is either fixed, taken -from a region table, or named by the tenant alongside its own key pair. +Header building is delegated to each preset's existing ``*_dynamic_headers`` builder, +so a destination authenticates exactly the way the per-request tracer route already +did; only the endpoint and transport need a per-backend rule. """ import os @@ -13,44 +10,63 @@ from collections.abc import Callable, Mapping from types import MappingProxyType from typing import Final +from litellm._logging import verbose_logger from litellm.integrations.otel.model.destination import OtelDestination +from litellm.litellm_core_utils.url_utils import SSRFError, assert_public_url from litellm.types.utils import StandardCallbackDynamicParams -#: gRPC is Arize's own transport; an explicitly named HTTP collector overrides it. -_ARIZE_GRPC_ENDPOINT: Final = "https://otlp.arize.com/v1" - def _langfuse_endpoint(params: StandardCallbackDynamicParams) -> str | None: """The tenant's own Langfuse host, else the operator's, else Langfuse US cloud. - Falling back to the operator's host is safe and is what V1 does: the tenant's - own key pair still selects its own project, and a self-hosted deployment where - every team lives on one Langfuse server is the common shape. + A host the tenant named goes through the proxy's SSRF guard first. Anyone who can + mint a key can write it, so without the check it points the exporter, and the + tenant credentials it carries, at any address the proxy can reach. The operator's + own ``LANGFUSE_HOST`` is not checked: an internal collector is a normal + deployment and the operator is the one configuring it. """ from litellm.integrations.langfuse.langfuse_otel import ( LANGFUSE_CLOUD_US_ENDPOINT, LangfuseOtelLogger, ) - host: Final = params.get("langfuse_host") or LangfuseOtelLogger._get_langfuse_otel_host() # pyright: ignore[reportPrivateUsage] # reuse the backend's own env host resolver rather than duplicating it + tenant_host: Final = params.get("langfuse_host") or None + host: Final = tenant_host or LangfuseOtelLogger._get_langfuse_otel_host() # pyright: ignore[reportPrivateUsage] # reuse the backend's own env host resolver rather than duplicating it if not host: return LANGFUSE_CLOUD_US_ENDPOINT normalized: Final = host if host.startswith("http") else f"https://{host}" - return f"{normalized.rstrip('/')}/api/public/otel" + endpoint: Final = f"{normalized.rstrip('/')}/api/public/otel" + if tenant_host is None: + return endpoint + try: + assert_public_url(endpoint) + except SSRFError as exc: + verbose_logger.warning( + "OTel V2: not exporting to key/team Langfuse host '%s' (%s). " + "Add it to general_settings.user_url_allowed_hosts to permit it", + host, + exc, + ) + return None + return endpoint def _arize_endpoint(params: StandardCallbackDynamicParams) -> str | None: - return os.environ.get("ARIZE_ENDPOINT") or _ARIZE_GRPC_ENDPOINT + from litellm.integrations.arize.arize import ArizeLogger + + return ArizeLogger.get_arize_config().endpoint def _arize_protocol(params: StandardCallbackDynamicParams) -> str | None: - return "otlp_http" if os.environ.get("ARIZE_HTTP_ENDPOINT") and not os.environ.get("ARIZE_ENDPOINT") else None + from litellm.integrations.arize.arize import ArizeLogger + + return ArizeLogger.get_arize_config().protocol def _weave_endpoint(params: StandardCallbackDynamicParams) -> str | None: - from litellm.integrations.weave.weave_otel import WEAVE_BASE_URL, WEAVE_OTEL_ENDPOINT + from litellm.integrations.weave.weave_otel import weave_otel_endpoint - return WEAVE_BASE_URL + WEAVE_OTEL_ENDPOINT + return weave_otel_endpoint(os.environ.get("WANDB_HOST")) def _newrelic_endpoint(params: StandardCallbackDynamicParams) -> str | None: @@ -78,6 +94,19 @@ _PROTOCOL_BY_CALLBACK: Final[Mapping[str, Callable[[StandardCallbackDynamicParam } ) +#: Headers a destination must carry to authenticate. Several dynamic-header builders +#: gate each credential independently, so a half-configured backend yields a non-empty +#: but unusable header set; accepting it would suppress the operator's own exporter and +#: send the request's whole trace where it cannot be stored. +_REQUIRED_HEADERS_BY_CALLBACK: Final[Mapping[str, frozenset[str]]] = MappingProxyType( + { + "langfuse_otel": frozenset({"Authorization"}), + "arize": frozenset({"arize-space-id", "api_key"}), + "weave_otel": frozenset({"Authorization", "project_id"}), + "newrelic": frozenset({"api-key"}), + } +) + _NO_ATTRS: Final[Mapping[str, str]] = MappingProxyType({}) @@ -92,9 +121,7 @@ def destination_for(callback_name: str, params: StandardCallbackDynamicParams) - """The destination ``params`` names for ``callback_name``, or ``None``. ``None`` means the caller configured nothing usable for this backend, so the - request keeps the operator's global exporters. A partial config (a host with - no key pair) resolves to ``None`` rather than to the operator's endpoint with - the tenant's host, which would post the operator's credentials elsewhere. + request keeps the operator's global exporters. """ from litellm.integrations.otel.presets import DYNAMIC_HEADERS_BY_CALLBACK @@ -103,7 +130,7 @@ def destination_for(callback_name: str, params: StandardCallbackDynamicParams) - if header_builder is None or endpoint_builder is None: return None headers: Final = header_builder(params) - if not headers: + if not _REQUIRED_HEADERS_BY_CALLBACK.get(callback_name, frozenset()) <= frozenset(headers): return None endpoint: Final = endpoint_builder(params) if not endpoint: @@ -111,7 +138,7 @@ def destination_for(callback_name: str, params: StandardCallbackDynamicParams) - protocol_builder: Final = _PROTOCOL_BY_CALLBACK.get(callback_name) return OtelDestination( endpoint=endpoint, - headers=MappingProxyType(dict(headers)), + headers=MappingProxyType(dict(headers)), # mutable-ok: MappingProxyType needs a concrete mapping to wrap resource_attributes=_NO_ATTRS, callback_name=callback_name, protocol=protocol_builder(params) if protocol_builder is not None else None, diff --git a/litellm/integrations/otel/presets/utils.py b/litellm/integrations/otel/presets/utils.py index ef6e80ff33e..002d5894e64 100644 --- a/litellm/integrations/otel/presets/utils.py +++ b/litellm/integrations/otel/presets/utils.py @@ -5,9 +5,6 @@ from typing import Final from litellm.integrations.otel.model.config import ExporterOwner, ExporterSpec -#: What ``OpenTelemetryV2Config._normalize`` folds in when no destination is configured. -_DEFAULT_SHORTHAND_EXPORTER: Final = ExporterSpec() - def ensure_mappers(mapper_names: Iterable[str], *names: str) -> list[str]: """Return ``mapper_names`` with each of ``names`` appended if not already present. @@ -35,6 +32,17 @@ def credential_gated_exporters( override filter still recognises which backend this provider speaks for. """ return ( - *(spec for spec in exporters if spec != _DEFAULT_SHORTHAND_EXPORTER), + *(spec for spec in exporters if not _prints_to_stdout(spec)), ExporterSpec(owner=owner, requires_headers=True), ) + + +def _prints_to_stdout(spec: "ExporterSpec") -> bool: + """Whether ``spec`` is the placeholder ``_normalize`` folds in for an empty list. + + Identified by what it does rather than by equality with a default instance: + ``OpenTelemetryV2Config`` reads the standard ``OTEL_EXPORTER_OTLP_*`` env vars, so + the shorthand it synthesizes is a real operator destination whenever any of them + is set, and only a console exporter with no endpoint prints every span. + """ + return spec.kind == "console" and spec.endpoint is None diff --git a/litellm/integrations/weave/weave_otel.py b/litellm/integrations/weave/weave_otel.py index f2cc64a9ba2..50289263f38 100644 --- a/litellm/integrations/weave/weave_otel.py +++ b/litellm/integrations/weave/weave_otel.py @@ -117,6 +117,14 @@ def _get_weave_authorization_header(api_key: str) -> str: return f"Basic {auth_header}" +def weave_otel_endpoint(host: str | None) -> str: + """The OTLP traces endpoint for a self-managed ``host``, else Weave cloud.""" + if not host: + return WEAVE_BASE_URL + WEAVE_OTEL_ENDPOINT + normalized: Final = host if host.startswith("http") else f"https://{host}" + return normalized.rstrip("/") + WEAVE_OTEL_ENDPOINT + + def get_weave_otel_config() -> WeaveOtelConfig: """ Retrieves the Weave OpenTelemetry configuration based on environment variables. @@ -134,7 +142,6 @@ def get_weave_otel_config() -> WeaveOtelConfig: """ api_key: Final = os.getenv("WANDB_API_KEY") project_id: Final = os.getenv("WANDB_PROJECT_ID") - host = os.getenv("WANDB_HOST") if not api_key: raise ValueError("WANDB_API_KEY must be set for Weave OpenTelemetry integration.") @@ -144,15 +151,8 @@ def get_weave_otel_config() -> WeaveOtelConfig: "WANDB_PROJECT_ID must be set for Weave OpenTelemetry integration. Format: /" ) - if host: - if not host.startswith("http"): - host = "https://" + host - # Self-managed instances use a different path - endpoint = host.rstrip("/") + WEAVE_OTEL_ENDPOINT - verbose_logger.debug("Using Weave OTEL endpoint from host: %s", endpoint) - else: - endpoint = WEAVE_BASE_URL + WEAVE_OTEL_ENDPOINT - verbose_logger.debug("Using Weave cloud endpoint: %s", endpoint) + endpoint: Final = weave_otel_endpoint(os.getenv("WANDB_HOST")) + verbose_logger.debug("Using Weave OTEL endpoint: %s", endpoint) # Weave uses Basic auth with format: api: auth_header: Final = _get_weave_authorization_header(api_key=api_key) diff --git a/litellm/litellm_core_utils/url_utils.py b/litellm/litellm_core_utils/url_utils.py index 1e43117933d..cee248b8384 100644 --- a/litellm/litellm_core_utils/url_utils.py +++ b/litellm/litellm_core_utils/url_utils.py @@ -20,6 +20,7 @@ Admins can opt out via two ``litellm`` globals (wired from proxy config): """ import socket +from functools import lru_cache from ipaddress import ip_address, ip_network from typing import Any, Final, Protocol from urllib.parse import quote, urlparse, urlunparse @@ -363,6 +364,54 @@ def validate_url(url: str) -> tuple[str, str]: return rewritten, host_header +def assert_public_url(url: str) -> None: + """Raise ``SSRFError`` unless ``url``'s host resolves only to public addresses. + + The validation half of :func:`validate_url`, for callers that must keep the + original hostname on the wire (TLS SNI, vendor-side Host routing) and so cannot + use its IP-rewriting form. It honours the same ``litellm.user_url_validation`` + master switch and ``litellm.user_url_allowed_hosts`` allowlist. Because the + caller still connects by name, this rejects a host that resolves somewhere + private; it does not close a DNS rebind between the check and the connection. + """ + if not getattr(litellm, "user_url_validation", True): + return + rejection: Final = _public_host_rejection(url, tuple(getattr(litellm, "user_url_allowed_hosts", None) or ())) + if rejection is not None: + raise SSRFError(rejection) + + +@lru_cache(maxsize=512) +def _public_host_rejection(url: str, allowed_hosts: tuple[str, ...]) -> str | None: + """Why ``url`` is not safe to reach, or ``None``. + + A verdict rather than an exception so both outcomes are cached: callers check + the same handful of destinations on every request and ``getaddrinfo`` blocks. + ``allowed_hosts`` is part of the key so a config reload takes effect. + """ + parsed: Final = urlparse(url) + if parsed.scheme not in _ALLOWED_SCHEMES: + return f"URL scheme '{parsed.scheme}' is not allowed" + + hostname: Final = parsed.hostname + if not hostname: + return "URL has no hostname" + + effective_port: Final = parsed.port if parsed.port is not None else _default_port_for_scheme(parsed.scheme) + if _is_host_allowlisted(hostname, effective_port): + return None + + try: + addrinfo: Final = socket.getaddrinfo(hostname, effective_port, proto=socket.IPPROTO_TCP) + except socket.gaierror as e: + return f"DNS resolution failed for '{hostname}': {e}" + + blocked: Final = tuple( + address for address in (_sockaddr_host(info[4]) for info in addrinfo) if _is_blocked_ip(address) + ) + return f"'{hostname}' resolves to a non-public address ({blocked[0]})" if blocked else None + + def assert_same_origin(candidate_url: str, expected_url: str) -> None: """Verify ``candidate_url`` shares scheme, host, and port with ``expected_url``. diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 6069ed9a8c6..dc78e636e37 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -997,6 +997,12 @@ def resolve_tenant_otel_destinations( backend to two accounts. Returns empty when OTEL V2 is off, when neither level named a destination-capable backend, or when the config is incomplete, and the request then keeps the operator's own exporters. + + A ``failure``-only entry is skipped: a destination is resolved during auth, before + the request has an outcome, so honouring the filter would mean holding every span + back until the call finishes. Those entries keep today's behaviour instead, where + the tenant's credentials reach the backend through per-request tracer routing and + the operator's exporter is left alone. """ from litellm.integrations.otel.model.config import is_otel_v2_enabled from litellm.integrations.otel.presets.destinations import destination_for @@ -1012,6 +1018,7 @@ def resolve_tenant_otel_destinations( destination for item in entries if (callback := _get_validated_callback_metadata(item=item, source="otel-destination")) is not None + if callback.callback_type != "failure" if (destination := destination_for(callback.callback_name, _tenant_otel_params(callback.callback_vars))) is not None ) diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index ee67160533f..c64ef05ce77 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -3,6 +3,7 @@ import contextvars from collections.abc import Mapping +import litellm import pytest from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import SimpleSpanProcessor @@ -41,6 +42,20 @@ LANGFUSE_DEST = OtelDestination( ) +@pytest.fixture +def allow_test_hosts(monkeypatch): + """Hosts named by these fixtures do not resolve, and a tenant-supplied host now + goes through the SSRF guard. Allowlist them so the resolution tests stay about + resolution; ``TestTenantHostSsrfGuard`` covers the guard itself.""" + from litellm.litellm_core_utils.url_utils import _public_host_rejection + + monkeypatch.setattr(litellm, "user_url_validation", True, raising=False) + monkeypatch.setattr(litellm, "user_url_allowed_hosts", ["team.local", "key.local", "x"], raising=False) + _public_host_rejection.cache_clear() + yield + _public_host_rejection.cache_clear() + + def in_fresh_context(fn, *args): """Run ``fn`` in its own context so one test's destinations never leak.""" return contextvars.copy_context().run(fn, *args) @@ -231,6 +246,7 @@ class TestRouting: assert route.provider is None +@pytest.mark.usefixtures("allow_test_hosts") class TestDestinationResolution: def test_a_langfuse_key_pair_and_host_become_a_destination(self, monkeypatch): monkeypatch.setenv("LITELLM_OTEL_V2", "true") @@ -312,12 +328,30 @@ class TestDestinationResolution: assert parse_headers(destination.header_string())["authorization"] == destination.headers["Authorization"] +#: Anything that makes ``OpenTelemetryV2Config`` synthesize a real operator destination. +_OTEL_SHORTHAND_ENV = ( + "OTEL_ENDPOINT", + "OTEL_HEADERS", + "OTEL_EXPORTER", + "OTEL_EXPORTER_OTLP_ENDPOINT", + "OTEL_EXPORTER_OTLP_HEADERS", + "OTEL_EXPORTER_OTLP_PROTOCOL", +) + + +def credential_less_proxy(monkeypatch) -> None: + """An operator with no Langfuse account and no generic OTLP collector.""" + for name in ("LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY", *_OTEL_SHORTHAND_ENV): + monkeypatch.delenv(name, raising=False) + with pytest.raises(ValueError, match="LANGFUSE_PUBLIC_KEY"): + langfuse_preset() + + class TestPresetDegradation: def test_a_credential_less_langfuse_exports_nowhere_instead_of_to_the_console(self, monkeypatch, capsys): """``_normalize`` folds a console exporter in for an empty list, which would print every span on a proxy whose teams bring their own credentials.""" - monkeypatch.delenv("LANGFUSE_PUBLIC_KEY", raising=False) - monkeypatch.delenv("LANGFUSE_SECRET_KEY", raising=False) + credential_less_proxy(monkeypatch) config = langfuse_preset(allow_missing_credentials=True) provider = build_tracer_provider(config, tenant_overrides=True) @@ -338,9 +372,8 @@ class TestPresetDegradation: def test_a_credential_less_proxy_still_builds_the_v2_logger(self, monkeypatch): from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 + credential_less_proxy(monkeypatch) monkeypatch.setenv("LITELLM_OTEL_V2", "true") - monkeypatch.delenv("LANGFUSE_PUBLIC_KEY", raising=False) - monkeypatch.delenv("LANGFUSE_SECRET_KEY", raising=False) is_otel_v2_enabled.cache_clear() logger = _maybe_construct_otel_v2("langfuse_otel", []) @@ -358,3 +391,213 @@ class TestContextIsolation: assert in_fresh_context(first) == frozenset({"langfuse_otel"}) assert in_fresh_context(request_destinations) == () + + +class TestOperatorShorthandSurvivesDegradation: + def test_a_generic_otlp_collector_keeps_receiving_when_langfuse_has_no_credentials(self, monkeypatch): + """Only the stdout placeholder is dropped. An operator who set the standard + OTLP env vars configured a real destination and must keep it.""" + monkeypatch.delenv("LANGFUSE_PUBLIC_KEY", raising=False) + monkeypatch.delenv("LANGFUSE_SECRET_KEY", raising=False) + monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.local:4318") + + config = langfuse_preset(allow_missing_credentials=True) + + assert [spec.endpoint for spec in config.exporters] == ["http://collector.local:4318", None] + assert [spec.kind for spec in config.exporters] == ["otlp_http", "console"] + + def test_the_stdout_placeholder_is_still_dropped_when_it_is_the_only_exporter(self, monkeypatch): + credential_less_proxy(monkeypatch) + + config = langfuse_preset(allow_missing_credentials=True) + + assert all(spec.requires_headers and not spec.headers for spec in config.exporters) + + +class TestBackendEndpointParity: + def test_arize_follows_its_own_http_endpoint_instead_of_the_grpc_default(self, monkeypatch): + monkeypatch.delenv("ARIZE_ENDPOINT", raising=False) + monkeypatch.setenv("ARIZE_HTTP_ENDPOINT", "https://otlp.arize.com/v1/traces") + + destination = destination_for("arize", {"arize_space_id": "s", "arize_api_key": "k"}) + + assert destination.endpoint == "https://otlp.arize.com/v1/traces" + assert destination.protocol == "otlp_http" + + def test_arize_uses_grpc_when_nothing_is_configured(self, monkeypatch): + monkeypatch.delenv("ARIZE_ENDPOINT", raising=False) + monkeypatch.delenv("ARIZE_HTTP_ENDPOINT", raising=False) + + destination = destination_for("arize", {"arize_space_id": "s", "arize_api_key": "k"}) + + assert destination.endpoint == "https://otlp.arize.com/v1" + assert destination.protocol == "otlp_grpc" + + def test_weave_follows_a_self_hosted_wandb_host(self, monkeypatch): + monkeypatch.setenv("WANDB_HOST", "weave.internal.example") + + destination = destination_for("weave_otel", {"wandb_api_key": "k", "weave_project_id": "e/p"}) + + assert destination.endpoint == "https://weave.internal.example/otel/v1/traces" + + def test_weave_uses_the_cloud_endpoint_without_a_host(self, monkeypatch): + monkeypatch.delenv("WANDB_HOST", raising=False) + + destination = destination_for("weave_otel", {"wandb_api_key": "k", "weave_project_id": "e/p"}) + + assert destination.endpoint == "https://trace.wandb.ai/otel/v1/traces" + + +class TestIncompleteCredentials: + """Half a credential set builds a non-empty but unusable header dict. Accepting it + would suppress the operator's exporter and send the trace where it cannot land.""" + + @pytest.mark.parametrize( + "callback_name,callback_vars", + [ + ("arize", {"arize_api_key": "k"}), + ("arize", {"arize_space_id": "s"}), + ("weave_otel", {"wandb_api_key": "k"}), + ("weave_otel", {"weave_project_id": "e/p"}), + ("langfuse_otel", {"langfuse_public_key": "pk"}), + ], + ) + def test_a_partial_credential_set_resolves_to_nothing(self, callback_name, callback_vars): + assert destination_for(callback_name, callback_vars) is None + + @pytest.mark.parametrize( + "callback_name,callback_vars", + [ + ("arize", {"arize_space_id": "s", "arize_api_key": "k"}), + ("weave_otel", {"wandb_api_key": "k", "weave_project_id": "e/p"}), + ("newrelic", {"newrelic_api_key": "k"}), + ], + ) + def test_a_complete_credential_set_resolves(self, callback_name, callback_vars): + assert destination_for(callback_name, callback_vars) is not None + + +@pytest.mark.usefixtures("allow_test_hosts") +class TestCallbackTypeFilter: + @staticmethod + def _auth(callback_type: str | None) -> UserAPIKeyAuth: + return UserAPIKeyAuth( + team_metadata={ + "logging": [ + { + "callback_name": "langfuse_otel", + "callback_type": callback_type, + "callback_vars": { + "langfuse_public_key": "pk", + "langfuse_secret_key": "sk", + "langfuse_host": "http://team.local", + }, + } + ] + } + ) + + @pytest.mark.parametrize("callback_type", ["success", "success_and_failure", None]) + def test_an_entry_that_wants_success_traces_gets_the_whole_trace(self, monkeypatch, callback_type): + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + + assert resolve_tenant_otel_destinations(self._auth(callback_type)) != () + + def test_a_failure_only_entry_does_not_take_over_the_trace(self, monkeypatch): + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + + assert resolve_tenant_otel_destinations(self._auth("failure")) == () + + +class TestEvictionSafety: + def test_evicting_a_processor_does_not_shut_it_down(self): + """``on_end`` hands the caller a processor and then releases the lock, so a + concurrent eviction that shut it down would silently drop that span.""" + from litellm.integrations.otel.plumbing.providers import _MAX_CACHED_DESTINATION_PROCESSORS + + class Recording(SimpleSpanProcessor): + def __init__(self): + super().__init__(InMemorySpanExporter()) + self.shutdown_calls = 0 + + def shutdown(self): + self.shutdown_calls += 1 + + built = [] + + def factory(_destination): + processor = Recording() + built.append(processor) + return processor + + fan_out = TenantFanOutSpanProcessor(processor_factory=factory) + for index in range(_MAX_CACHED_DESTINATION_PROCESSORS + 1): + fan_out._processor_for(LANGFUSE_DEST.model_copy(update={"endpoint": f"http://d{index}/otel"})) + + assert len(built) == _MAX_CACHED_DESTINATION_PROCESSORS + 1 + assert built[0].shutdown_calls == 0 + assert len(fan_out._processors) == _MAX_CACHED_DESTINATION_PROCESSORS + + +class TestTenantHostSsrfGuard: + """Anyone who can mint a key can write ``langfuse_host``, so a tenant-named host + is a user-supplied URL and goes through the proxy's SSRF guard.""" + + @staticmethod + def _reset() -> None: + from litellm.litellm_core_utils.url_utils import _public_host_rejection + + _public_host_rejection.cache_clear() + + @pytest.mark.parametrize("host", ["http://127.0.0.1:9111", "http://169.254.169.254", "http://10.0.0.5:3000"]) + def test_a_tenant_host_on_a_private_address_resolves_to_nothing(self, monkeypatch, host): + monkeypatch.setattr(litellm, "user_url_allowed_hosts", [], raising=False) + monkeypatch.setattr(litellm, "user_url_validation", True, raising=False) + self._reset() + + assert ( + destination_for( + "langfuse_otel", + {"langfuse_public_key": "pk", "langfuse_secret_key": "sk", "langfuse_host": host}, + ) + is None + ) + + def test_the_operator_can_allowlist_its_teams_internal_langfuse(self, monkeypatch): + monkeypatch.setattr(litellm, "user_url_allowed_hosts", ["127.0.0.1:9111"], raising=False) + monkeypatch.setattr(litellm, "user_url_validation", True, raising=False) + self._reset() + + destination = destination_for( + "langfuse_otel", + {"langfuse_public_key": "pk", "langfuse_secret_key": "sk", "langfuse_host": "http://127.0.0.1:9111"}, + ) + + assert destination.endpoint == "http://127.0.0.1:9111/api/public/otel" + + def test_the_master_switch_still_turns_the_guard_off(self, monkeypatch): + monkeypatch.setattr(litellm, "user_url_allowed_hosts", [], raising=False) + monkeypatch.setattr(litellm, "user_url_validation", False, raising=False) + self._reset() + + assert ( + destination_for( + "langfuse_otel", + {"langfuse_public_key": "pk", "langfuse_secret_key": "sk", "langfuse_host": "http://127.0.0.1:9111"}, + ) + is not None + ) + + def test_the_operators_own_internal_host_is_never_blocked(self, monkeypatch): + """The operator configures ``LANGFUSE_HOST`` themselves, so an internal + collector there is a deployment choice rather than caller-supplied input.""" + monkeypatch.setattr(litellm, "user_url_allowed_hosts", [], raising=False) + monkeypatch.setattr(litellm, "user_url_validation", True, raising=False) + monkeypatch.setenv("LANGFUSE_HOST", "http://127.0.0.1:9111") + self._reset() + + destination = destination_for("langfuse_otel", {"langfuse_public_key": "pk", "langfuse_secret_key": "sk"}) + + assert destination.endpoint == "http://127.0.0.1:9111/api/public/otel" From f5fb73f7164061d7899f6f1239faaf6663235450 Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Thu, 3 Sep 2026 16:37:45 -0700 Subject: [PATCH 03/47] refactor(otel v2): reuse the proxy's own destination allowlist for tenant hosts A tenant-supplied Langfuse host is the same threat as a URL-valued `model`, so it now goes through `is_url_destination_allowed_by_host` against `provider_url_destination_allowed_hosts` instead of a second, DNS-based check of its own. The DNS lookup would have blocked the asyncio auth path on a hostname the caller picked, and its cached verdicts could blackhole a real host after one resolver blip. Evicting a destination processor now retires it to drain rather than shutting it down, since `on_end` hands a processor back and exports outside the lock. The retirees are capped so they cannot accumulate a thread each. `credential_gated_exporters` tells the synthesized stdout placeholder from a real exporter by transport rather than by the literal kind `console`, so an unrecognized kind is not mistaken for a configured collector, and an exporter the operator did configure survives. That also stops a weave test's env writes from making this look like a real OTLP exporter later in the same CI worker. --- .../integrations/otel/plumbing/providers.py | 28 ++- .../integrations/otel/presets/destinations.py | 107 ++++----- litellm/integrations/otel/presets/utils.py | 27 ++- litellm/litellm_core_utils/url_utils.py | 49 ---- .../otel/test_otel_v2_destinations.py | 215 +++++++++++++----- 5 files changed, 253 insertions(+), 173 deletions(-) diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index 232827e3d47..a9dc0798c07 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -207,6 +207,7 @@ def _processor_for(exporter: SpanExporter, use_simple: bool | None) -> SpanProce #: Distinct tenant destinations whose exporters stay alive. Each holds a connection #: pool and a batch thread, so the cache is bounded and evicts least-recently-used. _MAX_CACHED_DESTINATION_PROCESSORS: Final = 32 +_MAX_RETIRED_DESTINATION_PROCESSORS: Final = 8 class _ResourceWrappedReadableSpan(ReadableSpan): @@ -254,6 +255,7 @@ class TenantFanOutSpanProcessor(SpanProcessor): self._lock: Final = threading.Lock() self._build: Final = processor_factory if processor_factory is not None else _destination_processor self._processors: OrderedDict[object, SpanProcessor] = OrderedDict() # mutable-ok: bounded LRU + self._retired: OrderedDict[object, SpanProcessor] = OrderedDict() # mutable-ok: bounded drain list def on_start(self, span: SDKSpan, parent_context: Context | None = None) -> None: return None @@ -279,6 +281,7 @@ class TenantFanOutSpanProcessor(SpanProcessor): verbose_logger.debug("OTel V2 fan-out: processor shutdown failed: %s", exc) with self._lock: self._processors.clear() + self._retired.clear() def force_flush(self, timeout_millis: int = 30000) -> bool: results: Final = tuple(self._flush_one(processor, timeout_millis) for processor in self._snapshot()) @@ -286,7 +289,7 @@ class TenantFanOutSpanProcessor(SpanProcessor): def _snapshot(self) -> tuple[SpanProcessor, ...]: with self._lock: - return tuple(self._processors.values()) + return (*self._processors.values(), *self._retired.values()) @staticmethod def _flush_one(processor: SpanProcessor, timeout_millis: int) -> bool: @@ -312,13 +315,26 @@ class TenantFanOutSpanProcessor(SpanProcessor): _shutdown_quietly(built) return existing self._processors[key] = built - if len(self._processors) > _MAX_CACHED_DESTINATION_PROCESSORS: - # Evict without shutting down: another thread may be inside ``on_end`` - # holding the victim, and a shut-down BatchSpanProcessor drops spans - # silently. Same rule as ArizePhoenixLogger's per-project cache. - self._processors.popitem(last=False) + overflowed: Final = self._retired_on_overflow_locked() + if overflowed is not None: + _shutdown_quietly(overflowed) return built + def _retired_on_overflow_locked(self) -> SpanProcessor | None: + """Drop the LRU processor past the cap; return one only once it is safe to close. + + ``on_end`` hands a processor back and then exports outside the lock, so shutting + an evicted one down there loses that span. Evictions retire to drain instead, and + the retirees are capped so they cannot accumulate a thread each. + """ + if len(self._processors) <= _MAX_CACHED_DESTINATION_PROCESSORS: + return None + _, evicted = self._processors.popitem(last=False) + self._retired[id(evicted)] = evicted + if len(self._retired) <= _MAX_RETIRED_DESTINATION_PROCESSORS: + return None + return self._retired.popitem(last=False)[1] + def _destination_processor(destination: "OtelDestination") -> SpanProcessor | None: """A batching OTLP processor aimed at ``destination``, or ``None`` if unbuildable.""" diff --git a/litellm/integrations/otel/presets/destinations.py b/litellm/integrations/otel/presets/destinations.py index 2589f5dbf3e..815e8575a76 100644 --- a/litellm/integrations/otel/presets/destinations.py +++ b/litellm/integrations/otel/presets/destinations.py @@ -7,23 +7,39 @@ did; only the endpoint and transport need a per-backend rule. import os from collections.abc import Callable, Mapping +from functools import lru_cache from types import MappingProxyType from typing import Final +import litellm from litellm._logging import verbose_logger from litellm.integrations.otel.model.destination import OtelDestination -from litellm.litellm_core_utils.url_utils import SSRFError, assert_public_url +from litellm.litellm_core_utils.url_utils import is_url_destination_allowed_by_host from litellm.types.utils import StandardCallbackDynamicParams +#: An endpoint plus the OTLP transport to reach it with, or ``None`` when the backend +#: names no destination. The transport is ``None`` where the backend has only one. +_Destination = tuple[str, str | None] -def _langfuse_endpoint(params: StandardCallbackDynamicParams) -> str | None: + +@lru_cache(maxsize=128) +def _warn_host_not_allowlisted(host: str) -> None: + """Cached so one misconfigured team logs once rather than once per request.""" + verbose_logger.warning( + "OTel V2: not exporting to key/team Langfuse host '%s'. Add it to " + "litellm_settings.provider_url_destination_allowed_hosts to permit it", + host, + ) + + +def _langfuse_destination(params: StandardCallbackDynamicParams) -> "_Destination | None": """The tenant's own Langfuse host, else the operator's, else Langfuse US cloud. - A host the tenant named goes through the proxy's SSRF guard first. Anyone who can - mint a key can write it, so without the check it points the exporter, and the - tenant credentials it carries, at any address the proxy can reach. The operator's - own ``LANGFUSE_HOST`` is not checked: an internal collector is a normal - deployment and the operator is the one configuring it. + A host the tenant named has to be allowlisted by the operator, the same way a + URL-valued ``model`` is: anyone who can mint a key can write it, and it becomes an + endpoint the proxy posts the request's whole trace to, carrying the tenant's own + credentials. The operator's own ``LANGFUSE_HOST`` is not checked, since an internal + collector there is a deployment choice. """ from litellm.integrations.langfuse.langfuse_otel import ( LANGFUSE_CLOUD_US_ENDPOINT, @@ -33,65 +49,50 @@ def _langfuse_endpoint(params: StandardCallbackDynamicParams) -> str | None: tenant_host: Final = params.get("langfuse_host") or None host: Final = tenant_host or LangfuseOtelLogger._get_langfuse_otel_host() # pyright: ignore[reportPrivateUsage] # reuse the backend's own env host resolver rather than duplicating it if not host: - return LANGFUSE_CLOUD_US_ENDPOINT + return (LANGFUSE_CLOUD_US_ENDPOINT, None) normalized: Final = host if host.startswith("http") else f"https://{host}" endpoint: Final = f"{normalized.rstrip('/')}/api/public/otel" if tenant_host is None: - return endpoint - try: - assert_public_url(endpoint) - except SSRFError as exc: - verbose_logger.warning( - "OTel V2: not exporting to key/team Langfuse host '%s' (%s). " - "Add it to general_settings.user_url_allowed_hosts to permit it", - host, - exc, - ) + return (endpoint, None) + if not is_url_destination_allowed_by_host(endpoint, litellm.provider_url_destination_allowed_hosts): + _warn_host_not_allowlisted(host) return None - return endpoint + return (endpoint, None) -def _arize_endpoint(params: StandardCallbackDynamicParams) -> str | None: +def _arize_destination(params: StandardCallbackDynamicParams) -> "_Destination | None": from litellm.integrations.arize.arize import ArizeLogger - return ArizeLogger.get_arize_config().endpoint + config: Final = ArizeLogger.get_arize_config() + return (config.endpoint, config.protocol) -def _arize_protocol(params: StandardCallbackDynamicParams) -> str | None: - from litellm.integrations.arize.arize import ArizeLogger - - return ArizeLogger.get_arize_config().protocol - - -def _weave_endpoint(params: StandardCallbackDynamicParams) -> str | None: +def _weave_destination(params: StandardCallbackDynamicParams) -> "_Destination | None": from litellm.integrations.weave.weave_otel import weave_otel_endpoint - return weave_otel_endpoint(os.environ.get("WANDB_HOST")) + return (weave_otel_endpoint(os.environ.get("WANDB_HOST")), None) -def _newrelic_endpoint(params: StandardCallbackDynamicParams) -> str | None: +def _newrelic_destination(params: StandardCallbackDynamicParams) -> "_Destination | None": from litellm.integrations.otel.presets.newrelic import newrelic_dynamic_endpoint - return newrelic_dynamic_endpoint(params) + endpoint: Final = newrelic_dynamic_endpoint(params) + return (endpoint, None) if endpoint else None -#: Callback name -> endpoint resolver. A backend is destination-capable exactly +#: Callback name -> destination resolver. A backend is destination-capable exactly #: when it appears here AND in ``DYNAMIC_HEADERS_BY_CALLBACK``: without a header #: builder the destination would carry no tenant credentials, and the exporter #: would post the tenant's traffic to the operator's account. -_ENDPOINT_BY_CALLBACK: Final[Mapping[str, Callable[[StandardCallbackDynamicParams], str | None]]] = MappingProxyType( - { - "langfuse_otel": _langfuse_endpoint, - "arize": _arize_endpoint, - "weave_otel": _weave_endpoint, - "newrelic": _newrelic_endpoint, - } -) - -_PROTOCOL_BY_CALLBACK: Final[Mapping[str, Callable[[StandardCallbackDynamicParams], str | None]]] = MappingProxyType( - { - "arize": _arize_protocol, - } +_DESTINATION_BY_CALLBACK: Final[Mapping[str, Callable[[StandardCallbackDynamicParams], "_Destination | None"]]] = ( + MappingProxyType( + { + "langfuse_otel": _langfuse_destination, + "arize": _arize_destination, + "weave_otel": _weave_destination, + "newrelic": _newrelic_destination, + } + ) ) #: Headers a destination must carry to authenticate. Several dynamic-header builders @@ -114,7 +115,7 @@ def destination_capable_backends() -> frozenset[str]: """Backends a key or team can point at its own account.""" from litellm.integrations.otel.presets import DYNAMIC_HEADERS_BY_CALLBACK - return frozenset(_ENDPOINT_BY_CALLBACK) & frozenset(DYNAMIC_HEADERS_BY_CALLBACK) + return frozenset(_DESTINATION_BY_CALLBACK) & frozenset(DYNAMIC_HEADERS_BY_CALLBACK) def destination_for(callback_name: str, params: StandardCallbackDynamicParams) -> OtelDestination | None: @@ -126,20 +127,20 @@ def destination_for(callback_name: str, params: StandardCallbackDynamicParams) - from litellm.integrations.otel.presets import DYNAMIC_HEADERS_BY_CALLBACK header_builder: Final = DYNAMIC_HEADERS_BY_CALLBACK.get(callback_name) - endpoint_builder: Final = _ENDPOINT_BY_CALLBACK.get(callback_name) - if header_builder is None or endpoint_builder is None: + destination_builder: Final = _DESTINATION_BY_CALLBACK.get(callback_name) + if header_builder is None or destination_builder is None: return None headers: Final = header_builder(params) - if not _REQUIRED_HEADERS_BY_CALLBACK.get(callback_name, frozenset()) <= frozenset(headers): + if not headers or not _REQUIRED_HEADERS_BY_CALLBACK[callback_name] <= frozenset(headers): return None - endpoint: Final = endpoint_builder(params) - if not endpoint: + resolved: Final = destination_builder(params) + if resolved is None: return None - protocol_builder: Final = _PROTOCOL_BY_CALLBACK.get(callback_name) + endpoint, protocol = resolved return OtelDestination( endpoint=endpoint, headers=MappingProxyType(dict(headers)), # mutable-ok: MappingProxyType needs a concrete mapping to wrap resource_attributes=_NO_ATTRS, callback_name=callback_name, - protocol=protocol_builder(params) if protocol_builder is not None else None, + protocol=protocol, ) diff --git a/litellm/integrations/otel/presets/utils.py b/litellm/integrations/otel/presets/utils.py index 002d5894e64..71ab2057fda 100644 --- a/litellm/integrations/otel/presets/utils.py +++ b/litellm/integrations/otel/presets/utils.py @@ -32,17 +32,30 @@ def credential_gated_exporters( override filter still recognises which backend this provider speaks for. """ return ( - *(spec for spec in exporters if not _prints_to_stdout(spec)), + *(spec for spec in exporters if not _is_stdout_placeholder(spec)), ExporterSpec(owner=owner, requires_headers=True), ) -def _prints_to_stdout(spec: "ExporterSpec") -> bool: +#: The fields ``OpenTelemetryV2Config._normalize`` fills the synthesized spec from. +_SHORTHAND_FIELDS: Final = frozenset({"kind", "endpoint", "headers"}) + + +def _is_stdout_placeholder(spec: "ExporterSpec") -> bool: """Whether ``spec`` is the placeholder ``_normalize`` folds in for an empty list. - Identified by what it does rather than by equality with a default instance: - ``OpenTelemetryV2Config`` reads the standard ``OTEL_EXPORTER_OTLP_*`` env vars, so - the shorthand it synthesizes is a real operator destination whenever any of them - is set, and only a console exporter with no endpoint prints every span. + Two conditions. It must have nowhere to send a span, which is what + ``exporter_transport`` answers: an unrecognized or misspelled kind falls back to the + console exporter, so comparing against the literal ``"console"`` would miss it. And + every non-shorthand field must still be at its default, which is what says the + operator did not ask for it: an exporter they configured survives, and so does the + gated spec this module appends, which would otherwise eat itself when one preset + layers onto another. """ - return spec.kind == "console" and spec.endpoint is None + from litellm.integrations.otel.plumbing.providers import exporter_transport + + return ( + exporter_transport(spec.kind) == "headerless" + and spec.endpoint is None + and spec.model_dump(exclude_defaults=True).keys() <= _SHORTHAND_FIELDS + ) diff --git a/litellm/litellm_core_utils/url_utils.py b/litellm/litellm_core_utils/url_utils.py index cee248b8384..1e43117933d 100644 --- a/litellm/litellm_core_utils/url_utils.py +++ b/litellm/litellm_core_utils/url_utils.py @@ -20,7 +20,6 @@ Admins can opt out via two ``litellm`` globals (wired from proxy config): """ import socket -from functools import lru_cache from ipaddress import ip_address, ip_network from typing import Any, Final, Protocol from urllib.parse import quote, urlparse, urlunparse @@ -364,54 +363,6 @@ def validate_url(url: str) -> tuple[str, str]: return rewritten, host_header -def assert_public_url(url: str) -> None: - """Raise ``SSRFError`` unless ``url``'s host resolves only to public addresses. - - The validation half of :func:`validate_url`, for callers that must keep the - original hostname on the wire (TLS SNI, vendor-side Host routing) and so cannot - use its IP-rewriting form. It honours the same ``litellm.user_url_validation`` - master switch and ``litellm.user_url_allowed_hosts`` allowlist. Because the - caller still connects by name, this rejects a host that resolves somewhere - private; it does not close a DNS rebind between the check and the connection. - """ - if not getattr(litellm, "user_url_validation", True): - return - rejection: Final = _public_host_rejection(url, tuple(getattr(litellm, "user_url_allowed_hosts", None) or ())) - if rejection is not None: - raise SSRFError(rejection) - - -@lru_cache(maxsize=512) -def _public_host_rejection(url: str, allowed_hosts: tuple[str, ...]) -> str | None: - """Why ``url`` is not safe to reach, or ``None``. - - A verdict rather than an exception so both outcomes are cached: callers check - the same handful of destinations on every request and ``getaddrinfo`` blocks. - ``allowed_hosts`` is part of the key so a config reload takes effect. - """ - parsed: Final = urlparse(url) - if parsed.scheme not in _ALLOWED_SCHEMES: - return f"URL scheme '{parsed.scheme}' is not allowed" - - hostname: Final = parsed.hostname - if not hostname: - return "URL has no hostname" - - effective_port: Final = parsed.port if parsed.port is not None else _default_port_for_scheme(parsed.scheme) - if _is_host_allowlisted(hostname, effective_port): - return None - - try: - addrinfo: Final = socket.getaddrinfo(hostname, effective_port, proto=socket.IPPROTO_TCP) - except socket.gaierror as e: - return f"DNS resolution failed for '{hostname}': {e}" - - blocked: Final = tuple( - address for address in (_sockaddr_host(info[4]) for info in addrinfo) if _is_blocked_ip(address) - ) - return f"'{hostname}' resolves to a non-public address ({blocked[0]})" if blocked else None - - def assert_same_origin(candidate_url: str, expected_url: str) -> None: """Verify ``candidate_url`` shares scheme, host, and port with ``expected_url``. diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index c64ef05ce77..0e6c940fb67 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -44,16 +44,12 @@ LANGFUSE_DEST = OtelDestination( @pytest.fixture def allow_test_hosts(monkeypatch): - """Hosts named by these fixtures do not resolve, and a tenant-supplied host now - goes through the SSRF guard. Allowlist them so the resolution tests stay about - resolution; ``TestTenantHostSsrfGuard`` covers the guard itself.""" - from litellm.litellm_core_utils.url_utils import _public_host_rejection - - monkeypatch.setattr(litellm, "user_url_validation", True, raising=False) - monkeypatch.setattr(litellm, "user_url_allowed_hosts", ["team.local", "key.local", "x"], raising=False) - _public_host_rejection.cache_clear() - yield - _public_host_rejection.cache_clear() + """A tenant-supplied host must be allowlisted by the operator. Allowlist the ones + these fixtures name so the resolution tests stay about resolution; + ``TestTenantHostSsrfGuard`` covers the guard itself.""" + monkeypatch.setattr( + litellm, "provider_url_destination_allowed_hosts", ["team.local", "key.local", "x"], raising=False + ) def in_fresh_context(fn, *args): @@ -512,9 +508,44 @@ class TestCallbackTypeFilter: class TestEvictionSafety: - def test_evicting_a_processor_does_not_shut_it_down(self): - """``on_end`` hands the caller a processor and then releases the lock, so a - concurrent eviction that shut it down would silently drop that span.""" + def test_an_evicted_processor_is_retired_rather_than_shut_down(self): + """``on_end`` hands a processor back and exports outside the lock, so shutting + an evicted one down there loses that span. Retirees are capped so they cannot + accumulate a thread each.""" + from litellm.integrations.otel.plumbing.providers import ( + _MAX_CACHED_DESTINATION_PROCESSORS, + _MAX_RETIRED_DESTINATION_PROCESSORS, + ) + + class Recording(SimpleSpanProcessor): + def __init__(self): + super().__init__(InMemorySpanExporter()) + self.shutdown_calls = 0 + + def shutdown(self): + self.shutdown_calls += 1 + + built = [] + + def factory(_destination): + built.append(Recording()) + return built[-1] + + fan_out = TenantFanOutSpanProcessor(processor_factory=factory) + for index in range(_MAX_CACHED_DESTINATION_PROCESSORS + _MAX_RETIRED_DESTINATION_PROCESSORS): + fan_out._processor_for(LANGFUSE_DEST.model_copy(update={"endpoint": f"http://d{index}/otel"})) + + assert [p.shutdown_calls for p in built] == [0] * len(built) + assert len(fan_out._processors) == _MAX_CACHED_DESTINATION_PROCESSORS + + for index in range(2): + fan_out._processor_for(LANGFUSE_DEST.model_copy(update={"endpoint": f"http://late{index}/otel"})) + + assert [p.shutdown_calls for p in built[:2]] == [1, 1] + assert built[2].shutdown_calls == 0 + assert len(fan_out._processors) == _MAX_CACHED_DESTINATION_PROCESSORS + + def test_a_retired_processor_is_still_flushed_and_closed_on_shutdown(self): from litellm.integrations.otel.plumbing.providers import _MAX_CACHED_DESTINATION_PROCESSORS class Recording(SimpleSpanProcessor): @@ -528,76 +559,144 @@ class TestEvictionSafety: built = [] def factory(_destination): - processor = Recording() - built.append(processor) - return processor + built.append(Recording()) + return built[-1] fan_out = TenantFanOutSpanProcessor(processor_factory=factory) for index in range(_MAX_CACHED_DESTINATION_PROCESSORS + 1): fan_out._processor_for(LANGFUSE_DEST.model_copy(update={"endpoint": f"http://d{index}/otel"})) + fan_out.shutdown() - assert len(built) == _MAX_CACHED_DESTINATION_PROCESSORS + 1 - assert built[0].shutdown_calls == 0 - assert len(fan_out._processors) == _MAX_CACHED_DESTINATION_PROCESSORS + assert built[0].shutdown_calls == 1 + + +class TestCredentialGatedExporters: + def test_layering_a_second_preset_does_not_eat_the_first_gated_exporter(self, monkeypatch): + """``base.Preset`` advertises ``config_overrides`` layering, and the gated spec + is itself a console exporter with no endpoint.""" + credential_less_proxy(monkeypatch) + from litellm.integrations.otel.presets.utils import credential_gated_exporters + + once = credential_gated_exporters((), ExporterOwner.LANGFUSE_OTEL) + twice = credential_gated_exporters(once, ExporterOwner.WEAVE_OTEL) + + assert [spec.owner for spec in twice] == [ExporterOwner.LANGFUSE_OTEL, ExporterOwner.WEAVE_OTEL] + + def test_an_exporter_the_operator_configured_survives(self): + from litellm.integrations.otel.presets.utils import credential_gated_exporters + + operator_console = ExporterSpec(kind="console", use_simple_processor=True) + + kept = credential_gated_exporters((operator_console,), ExporterOwner.LANGFUSE_OTEL) + + assert kept[0] == operator_console + + def test_an_otlp_exporter_on_its_default_endpoint_survives(self): + """``OTEL_EXPORTER=otlp_http`` with no endpoint is a real collector on the SDK's + default port, not the placeholder, so the transport is what tells them apart.""" + from litellm.integrations.otel.presets.utils import credential_gated_exporters + + operator_otlp = ExporterSpec(kind="otlp_http", endpoint=None, headers=None) + + kept = credential_gated_exporters((operator_otlp,), ExporterOwner.LANGFUSE_OTEL) + + assert kept[0] == operator_otlp + + def test_the_synthesized_stdout_placeholder_is_dropped(self): + from litellm.integrations.otel.presets.utils import credential_gated_exporters + + placeholder = ExporterSpec(kind="console", endpoint=None, headers=None) + + kept = credential_gated_exporters((placeholder,), ExporterOwner.LANGFUSE_OTEL) + + assert [spec.owner for spec in kept] == [ExporterOwner.LANGFUSE_OTEL] class TestTenantHostSsrfGuard: - """Anyone who can mint a key can write ``langfuse_host``, so a tenant-named host - is a user-supplied URL and goes through the proxy's SSRF guard.""" + """Anyone who can mint a key can write ``langfuse_host``, so the host it names has + to be one the operator approved.""" + + @pytest.fixture(autouse=True) + def _guard_on(self, monkeypatch): + from litellm.integrations.otel.presets.destinations import _warn_host_not_allowlisted + + monkeypatch.setattr(litellm, "provider_url_destination_allowed_hosts", [], raising=False) + _warn_host_not_allowlisted.cache_clear() + yield + _warn_host_not_allowlisted.cache_clear() @staticmethod - def _reset() -> None: - from litellm.litellm_core_utils.url_utils import _public_host_rejection + def _langfuse(host: str) -> Mapping[str, str]: + return {"langfuse_public_key": "pk", "langfuse_secret_key": "sk", "langfuse_host": host} - _public_host_rejection.cache_clear() + @pytest.mark.parametrize( + "host", + [ + "http://127.0.0.1:9111", + "http://169.254.169.254", + "http://10.0.0.5:3000", + "https://collector.example.com", + "https://langfuse.corp:99999", + "ftp://collector.example.com", + ], + ) + def test_a_host_the_operator_never_approved_resolves_to_nothing(self, host): + assert destination_for("langfuse_otel", self._langfuse(host)) is None - @pytest.mark.parametrize("host", ["http://127.0.0.1:9111", "http://169.254.169.254", "http://10.0.0.5:3000"]) - def test_a_tenant_host_on_a_private_address_resolves_to_nothing(self, monkeypatch, host): - monkeypatch.setattr(litellm, "user_url_allowed_hosts", [], raising=False) - monkeypatch.setattr(litellm, "user_url_validation", True, raising=False) - self._reset() + def test_userinfo_naming_an_allowlisted_host_does_not_smuggle_a_second_one(self, monkeypatch): + """``https://allowed@10.0.0.5`` reads as the allowlisted host to the eye and + posts to 10.0.0.5 on the wire.""" + monkeypatch.setattr(litellm, "provider_url_destination_allowed_hosts", ["collector.example.com"], raising=False) - assert ( - destination_for( - "langfuse_otel", - {"langfuse_public_key": "pk", "langfuse_secret_key": "sk", "langfuse_host": host}, - ) - is None + assert destination_for("langfuse_otel", self._langfuse("https://collector.example.com@10.0.0.5")) is None + + def test_a_malformed_host_does_not_take_the_other_backends_with_it(self, monkeypatch): + """``urlparse(...).port`` raises a bare ValueError, which would escape + ``destination_for`` and kill the whole resolution.""" + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + monkeypatch.setenv("NEW_RELIC_OTEL_ENDPOINT", "https://otlp.nr-data.net") + monkeypatch.setattr(litellm, "provider_url_destination_allowed_hosts", ["collector.example.com"], raising=False) + is_otel_v2_enabled.cache_clear() + auth = UserAPIKeyAuth( + token="hashed", + team_metadata={ + "logging": [ + {"callback_name": "langfuse_otel", "callback_vars": self._langfuse("https://lf.corp:99999")}, + {"callback_name": "newrelic", "callback_vars": {"newrelic_api_key": "nr"}}, + ] + }, ) + assert [d.callback_name for d in resolve_tenant_otel_destinations(auth)] == ["newrelic"] + def test_the_operator_can_allowlist_its_teams_internal_langfuse(self, monkeypatch): - monkeypatch.setattr(litellm, "user_url_allowed_hosts", ["127.0.0.1:9111"], raising=False) - monkeypatch.setattr(litellm, "user_url_validation", True, raising=False) - self._reset() + monkeypatch.setattr(litellm, "provider_url_destination_allowed_hosts", ["127.0.0.1:9111"], raising=False) - destination = destination_for( - "langfuse_otel", - {"langfuse_public_key": "pk", "langfuse_secret_key": "sk", "langfuse_host": "http://127.0.0.1:9111"}, - ) + destination = destination_for("langfuse_otel", self._langfuse("http://127.0.0.1:9111")) assert destination.endpoint == "http://127.0.0.1:9111/api/public/otel" - def test_the_master_switch_still_turns_the_guard_off(self, monkeypatch): - monkeypatch.setattr(litellm, "user_url_allowed_hosts", [], raising=False) - monkeypatch.setattr(litellm, "user_url_validation", False, raising=False) - self._reset() - - assert ( - destination_for( - "langfuse_otel", - {"langfuse_public_key": "pk", "langfuse_secret_key": "sk", "langfuse_host": "http://127.0.0.1:9111"}, - ) - is not None - ) - def test_the_operators_own_internal_host_is_never_blocked(self, monkeypatch): """The operator configures ``LANGFUSE_HOST`` themselves, so an internal collector there is a deployment choice rather than caller-supplied input.""" - monkeypatch.setattr(litellm, "user_url_allowed_hosts", [], raising=False) - monkeypatch.setattr(litellm, "user_url_validation", True, raising=False) monkeypatch.setenv("LANGFUSE_HOST", "http://127.0.0.1:9111") - self._reset() destination = destination_for("langfuse_otel", {"langfuse_public_key": "pk", "langfuse_secret_key": "sk"}) assert destination.endpoint == "http://127.0.0.1:9111/api/public/otel" + + def test_an_allowlisted_host_is_taken_without_resolving_it(self, monkeypatch): + """The check runs on the asyncio auth path, so it must not block on a name the + caller chose. ``.invalid`` never resolves, and it is still accepted.""" + monkeypatch.setattr(litellm, "provider_url_destination_allowed_hosts", ["lf.invalid"], raising=False) + + destination = destination_for("langfuse_otel", self._langfuse("https://lf.invalid")) + + assert destination.endpoint == "https://lf.invalid/api/public/otel" + + def test_a_rejected_host_is_warned_about_once(self, caplog): + with caplog.at_level("WARNING", logger="LiteLLM"): + for _ in range(3): + destination_for("langfuse_otel", self._langfuse("http://10.0.0.5:3000")) + + assert sum("provider_url_destination_allowed_hosts" in record.message for record in caplog.records) == 1 From 1779fcf4a73c66c91b9807965c32b3ef88cc4bd2 Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Thu, 3 Sep 2026 17:15:19 -0700 Subject: [PATCH 04/47] fix(otel v2): read the tenant's stored callback config the way the sibling parser does Three divergences between the destination resolver and `convert_key_logging_metadata_to_callback`, which read the same stored config: - A key whose callbacks are disabled stores an empty list, and `or` treated that as "the key configured nothing", so the request inherited the team's destination. The sibling parser treats an empty list as configured. - Two entries naming one backend now merge their `callback_vars` last-wins, matching the sibling, instead of the resolver taking the first entry and the per-request tracer routing taking the last. - `credential_gated_exporters` dropped any exporter whose kind had no transport, which also dropped an `in_memory` exporter the operator asked for. The placeholder is the spec with every field still at its default, so that is what the predicate now says. Arize's `allow_missing_credentials` branch was unreachable: `get_arize_config` resolves every credential with `os.environ.get` and always supplies an endpoint, so it never raises. Dropped it and corrected the protocol docstring. --- litellm/integrations/otel/presets/arize.py | 17 +----- litellm/integrations/otel/presets/base.py | 6 +- litellm/integrations/otel/presets/utils.py | 37 ++++-------- litellm/proxy/litellm_pre_call_utils.py | 39 +++++++++---- .../otel/test_otel_v2_destinations.py | 57 +++++++++++++++++++ 5 files changed, 100 insertions(+), 56 deletions(-) diff --git a/litellm/integrations/otel/presets/arize.py b/litellm/integrations/otel/presets/arize.py index 63222856b2a..d7ce87f5552 100644 --- a/litellm/integrations/otel/presets/arize.py +++ b/litellm/integrations/otel/presets/arize.py @@ -11,10 +11,7 @@ from litellm.integrations.otel.model.config import ( ExporterSpec, OpenTelemetryV2Config, ) -from litellm.integrations.otel.presets.utils import ( - credential_gated_exporters, - ensure_mappers, -) +from litellm.integrations.otel.presets.utils import ensure_mappers from litellm.types.utils import StandardCallbackDynamicParams @@ -33,17 +30,7 @@ def arize_preset( ) -> OpenTelemetryV2Config: base: Final = config_overrides or OpenTelemetryV2Config() mappers: Final = ensure_mappers(base.mapper_names, "openinference") - try: - arize_cfg: Final = _V1ArizeLogger.get_arize_config() - except Exception: - if not allow_missing_credentials: - raise - return base.model_copy( - update={ # mutable-ok: pydantic model_copy takes a plain update mapping - "exporters": credential_gated_exporters(base.exporters, ExporterOwner.ARIZE_AX), - "mapper_names": mappers, - } - ) + arize_cfg: Final = _V1ArizeLogger.get_arize_config() headers: Final = _arize_headers(arize_cfg) return base.model_copy( update={ diff --git a/litellm/integrations/otel/presets/base.py b/litellm/integrations/otel/presets/base.py index 3b7264abd88..3a768a08a4f 100644 --- a/litellm/integrations/otel/presets/base.py +++ b/litellm/integrations/otel/presets/base.py @@ -19,9 +19,9 @@ class Preset(Protocol): ``config_overrides`` lets one preset layer onto another's config (or onto test-supplied defaults); the factory calls presets with no arguments. - ``allow_missing_credentials`` lets a credential-mandatory backend (langfuse / - arize / weave) degrade to an exporter-less, mapper-only config instead of - raising when the operator set no env credentials of their own. That is a real + ``allow_missing_credentials`` lets a credential-mandatory backend (langfuse and + weave) degrade to an exporter-less, mapper-only config instead of raising when the + operator set no env credentials of their own. That is a real deployment: every team brings its own account and the operator keeps none, and without it the whole V2 path silently falls back to the legacy integration, so no team destination is ever reached. Credential-optional backends ignore it. diff --git a/litellm/integrations/otel/presets/utils.py b/litellm/integrations/otel/presets/utils.py index 71ab2057fda..e132b8854ab 100644 --- a/litellm/integrations/otel/presets/utils.py +++ b/litellm/integrations/otel/presets/utils.py @@ -19,9 +19,7 @@ def ensure_mappers(mapper_names: Iterable[str], *names: str) -> list[str]: return result -def credential_gated_exporters( - exporters: "Iterable[ExporterSpec]", owner: "ExporterOwner" -) -> "tuple[ExporterSpec, ...]": +def credential_gated_exporters(exporters: "Iterable[ExporterSpec]", owner: "ExporterOwner") -> "list[ExporterSpec]": """``exporters`` with the operator's destination replaced by a header-gated one. Used when a credential-mandatory backend is asked to build without the operator's @@ -31,31 +29,18 @@ def credential_gated_exporters( span would be printed to stdout, and the gated spec keeps the owner so the override filter still recognises which backend this provider speaks for. """ - return ( - *(spec for spec in exporters if not _is_stdout_placeholder(spec)), + return [ + *(spec for spec in exporters if not _is_unconfigured_placeholder(spec)), ExporterSpec(owner=owner, requires_headers=True), - ) + ] -#: The fields ``OpenTelemetryV2Config._normalize`` fills the synthesized spec from. -_SHORTHAND_FIELDS: Final = frozenset({"kind", "endpoint", "headers"}) +def _is_unconfigured_placeholder(spec: "ExporterSpec") -> bool: + """Whether ``spec`` is the one ``_normalize`` folds in when nothing was configured. - -def _is_stdout_placeholder(spec: "ExporterSpec") -> bool: - """Whether ``spec`` is the placeholder ``_normalize`` folds in for an empty list. - - Two conditions. It must have nowhere to send a span, which is what - ``exporter_transport`` answers: an unrecognized or misspelled kind falls back to the - console exporter, so comparing against the literal ``"console"`` would miss it. And - every non-shorthand field must still be at its default, which is what says the - operator did not ask for it: an exporter they configured survives, and so does the - gated spec this module appends, which would otherwise eat itself when one preset - layers onto another. + Every field at its default is what says the operator asked for nothing: an exporter + they did configure survives, whatever its kind, and so does the gated spec this + module appends, which would otherwise eat itself when one preset layers onto + another. """ - from litellm.integrations.otel.plumbing.providers import exporter_transport - - return ( - exporter_transport(spec.kind) == "headerless" - and spec.endpoint is None - and spec.model_dump(exclude_defaults=True).keys() <= _SHORTHAND_FIELDS - ) + return not spec.model_dump(exclude_defaults=True) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index dc78e636e37..924203e9d8a 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -994,9 +994,14 @@ def resolve_tenant_otel_destinations( Key settings win over team settings outright, the same precedence ``_get_dynamic_logging_metadata`` applies, so one caller never exports the same - backend to two accounts. Returns empty when OTEL V2 is off, when neither level - named a destination-capable backend, or when the config is incomplete, and the - request then keeps the operator's own exporters. + backend to two accounts. An empty key-level list counts as configured, since that + is what disabling a key's callbacks writes. Returns empty when OTEL V2 is off, when + neither level named a destination-capable backend, or when the config is + incomplete, and the request then keeps the operator's own exporters. + + Two entries naming the same backend merge their ``callback_vars`` last-wins, the + way ``convert_key_logging_metadata_to_callback`` merges them, so the destination + and the per-request tracer routing cannot read one config two ways. A ``failure``-only entry is skipped: a destination is resolved during auth, before the request has an outcome, so honouring the filter would mean holding every span @@ -1009,23 +1014,33 @@ def resolve_tenant_otel_destinations( if not is_otel_v2_enabled(): return () - entries: Final = KeyAndTeamLoggingSettings.get_key_dynamic_logging_settings( - user_api_key_dict - ) or KeyAndTeamLoggingSettings.get_team_dynamic_logging_settings(user_api_key_dict) + key_entries: Final = KeyAndTeamLoggingSettings.get_key_dynamic_logging_settings(user_api_key_dict) + entries: Final = ( + key_entries + if key_entries is not None + else KeyAndTeamLoggingSettings.get_team_dynamic_logging_settings(user_api_key_dict) + ) if not entries: return () - resolved: Final = tuple( - destination + callbacks: Final = tuple( + callback for item in entries if (callback := _get_validated_callback_metadata(item=item, source="otel-destination")) is not None if callback.callback_type != "failure" - if (destination := destination_for(callback.callback_name, _tenant_otel_params(callback.callback_vars))) - is not None ) + merged: Final = { + name: { + var: value + for callback in callbacks + if callback.callback_name == name + for var, value in callback.callback_vars.items() + } + for name in dict.fromkeys(callback.callback_name for callback in callbacks) + } return tuple( destination - for index, destination in enumerate(resolved) - if destination.callback_name not in tuple(earlier.callback_name for earlier in resolved[:index]) + for name, callback_vars in merged.items() + if (destination := destination_for(name, _tenant_otel_params(callback_vars))) is not None ) diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index 0e6c940fb67..c9778bbb2ec 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -507,6 +507,52 @@ class TestCallbackTypeFilter: assert resolve_tenant_otel_destinations(self._auth("failure")) == () +class TestTenantConfigAgreement: + """The destination resolver and ``convert_key_logging_metadata_to_callback`` read + the same stored config, so they must not read it two different ways.""" + + @pytest.fixture(autouse=True) + def _v2_on(self, monkeypatch): + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + monkeypatch.setattr( + litellm, "provider_url_destination_allowed_hosts", ["team.local", "key.local"], raising=False + ) + is_otel_v2_enabled.cache_clear() + yield + is_otel_v2_enabled.cache_clear() + + @staticmethod + def _entry(host, **extra): + return { + "callback_name": "langfuse_otel", + "callback_vars": {"langfuse_public_key": "pk", "langfuse_secret_key": "sk", "langfuse_host": host, **extra}, + } + + def test_a_key_that_disabled_its_callbacks_does_not_fall_back_to_the_team(self): + """Disabling a key's callbacks stores an empty list, which the sibling parser + reads as 'the key configured none'.""" + auth = UserAPIKeyAuth( + metadata={"logging": []}, + team_metadata={"logging": [self._entry("http://team.local")]}, + ) + + assert resolve_tenant_otel_destinations(auth) == () + + def test_two_entries_for_one_backend_merge_their_vars_last_wins(self): + auth = UserAPIKeyAuth( + team_metadata={ + "logging": [ + self._entry("http://team.local"), + {"callback_name": "langfuse_otel", "callback_vars": {"langfuse_host": "http://key.local"}}, + ] + } + ) + + destinations = resolve_tenant_otel_destinations(auth) + + assert [d.endpoint for d in destinations] == ["http://key.local/api/public/otel"] + + class TestEvictionSafety: def test_an_evicted_processor_is_retired_rather_than_shut_down(self): """``on_end`` hands a processor back and exports outside the lock, so shutting @@ -602,6 +648,17 @@ class TestCredentialGatedExporters: assert kept[0] == operator_otlp + def test_an_in_memory_exporter_the_operator_asked_for_survives(self): + """``OTEL_EXPORTER=in_memory`` stores spans, so it is a destination the operator + chose, not the placeholder that stands in for choosing nothing.""" + from litellm.integrations.otel.presets.utils import credential_gated_exporters + + operator_memory = ExporterSpec(kind="in_memory", endpoint=None, headers=None) + + kept = credential_gated_exporters((operator_memory,), ExporterOwner.LANGFUSE_OTEL) + + assert kept[0] == operator_memory + def test_the_synthesized_stdout_placeholder_is_dropped(self): from litellm.integrations.otel.presets.utils import credential_gated_exporters From e9df4458c7928490d5ec1dc917908107a98bba70 Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Thu, 3 Sep 2026 17:17:38 -0700 Subject: [PATCH 05/47] fix(otel v2): keep the destination merge immutable The per-backend var merge seeded a plain dict and the gated exporter list a plain list, both of which the LIT budget counts. Wrap the merge in MappingProxyType and hand the exporters back as a tuple. --- litellm/integrations/otel/presets/utils.py | 8 ++++--- litellm/proxy/litellm_pre_call_utils.py | 28 +++++++++++++--------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/litellm/integrations/otel/presets/utils.py b/litellm/integrations/otel/presets/utils.py index e132b8854ab..2d7598b63c3 100644 --- a/litellm/integrations/otel/presets/utils.py +++ b/litellm/integrations/otel/presets/utils.py @@ -19,7 +19,9 @@ def ensure_mappers(mapper_names: Iterable[str], *names: str) -> list[str]: return result -def credential_gated_exporters(exporters: "Iterable[ExporterSpec]", owner: "ExporterOwner") -> "list[ExporterSpec]": +def credential_gated_exporters( + exporters: "Iterable[ExporterSpec]", owner: "ExporterOwner" +) -> "tuple[ExporterSpec, ...]": """``exporters`` with the operator's destination replaced by a header-gated one. Used when a credential-mandatory backend is asked to build without the operator's @@ -29,10 +31,10 @@ def credential_gated_exporters(exporters: "Iterable[ExporterSpec]", owner: "Expo span would be printed to stdout, and the gated spec keeps the owner so the override filter still recognises which backend this provider speaks for. """ - return [ + return ( *(spec for spec in exporters if not _is_unconfigured_placeholder(spec)), ExporterSpec(owner=owner, requires_headers=True), - ] + ) def _is_unconfigured_placeholder(spec: "ExporterSpec") -> bool: diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 924203e9d8a..aed9933424b 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -1028,19 +1028,25 @@ def resolve_tenant_otel_destinations( if (callback := _get_validated_callback_metadata(item=item, source="otel-destination")) is not None if callback.callback_type != "failure" ) - merged: Final = { - name: { - var: value - for callback in callbacks - if callback.callback_name == name - for var, value in callback.callback_vars.items() - } - for name in dict.fromkeys(callback.callback_name for callback in callbacks) - } return tuple( destination - for name, callback_vars in merged.items() - if (destination := destination_for(name, _tenant_otel_params(callback_vars))) is not None + for name in dict.fromkeys(callback.callback_name for callback in callbacks) + if ( + destination := destination_for( + name, + _tenant_otel_params( + MappingProxyType( + { + var: value + for callback in callbacks + if callback.callback_name == name + for var, value in callback.callback_vars.items() + } + ) + ), + ) + ) + is not None ) From fb3b32d22dc5a4d374f6c3391e424434a65504fe Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Thu, 3 Sep 2026 17:38:08 -0700 Subject: [PATCH 06/47] fix(otel v2): scope the fan-out to its own backend and close shed processors off the export path Three problems in the fan-out, two of them in the eviction added last round: - Every v2 logger carries its own provider and emits its own copy of a gen-AI span, so a proxy running two of them handed the tenant the same model call twice. A provider now forwards only destinations for the backend it speaks for; the tenant's own backend always has a logger, since naming it in the key or team config is what builds one. Reproduced live against a self-hosted Langfuse on an arize-only proxy and on the bare `otel` callback. - Eviction could close a processor another thread was still exporting through, which drops that span. Exports are now counted, and a retired processor is closed only once its count reaches zero. - That close ran inside `on_end`, where `shutdown` flushes over the network, so one unreachable tenant collector stalled every other tenant's spans. It now runs on a short-lived thread, which also retires the retiree cap: a retiree drains as soon as its export finishes. --- litellm/integrations/otel/logger.py | 2 +- .../integrations/otel/plumbing/providers.py | 84 +++++++--- .../otel/test_otel_v2_destinations.py | 153 ++++++++++++------ 3 files changed, 170 insertions(+), 69 deletions(-) diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index ec3c3eaa004..7348d913b35 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -182,7 +182,7 @@ class OpenTelemetryV2(CustomLogger): self._tracer_provider: TracerProvider = ( tracer_provider if tracer_provider is not None - else build_tracer_provider(self.config, tenant_overrides=True) + else build_tracer_provider(self.config, tenant_overrides=True, tenant_callback_name=callback_name) ) self.tracer: Tracer = get_tracer(self._tracer_provider, LITELLM_TRACER_NAME) self._metrics_recorder = self._init_metrics(meter_provider) diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index a9dc0798c07..20649332f40 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -207,7 +207,6 @@ def _processor_for(exporter: SpanExporter, use_simple: bool | None) -> SpanProce #: Distinct tenant destinations whose exporters stay alive. Each holds a connection #: pool and a batch thread, so the cache is bounded and evicts least-recently-used. _MAX_CACHED_DESTINATION_PROCESSORS: Final = 32 -_MAX_RETIRED_DESTINATION_PROCESSORS: Final = 8 class _ResourceWrappedReadableSpan(ReadableSpan): @@ -246,29 +245,42 @@ class TenantFanOutSpanProcessor(SpanProcessor): Destinations ride a request-scoped ``ContextVar`` set during auth, so concurrent requests stay isolated. The forwarded view keeps the original trace and parent ids, so the tenant gets the same tree the operator would have received. + + ``callback_name`` scopes the fan-out to the backend this provider speaks for. Every + v2 logger carries its own provider and emits its own copy of a gen-AI span, so a + proxy running two of them would otherwise deliver the tenant two copies of the same + model call. The tenant's own backend always has a logger, since naming it in the + key or team config is what builds one. """ def __init__( self, + callback_name: str | None = None, processor_factory: 'Callable[["OtelDestination"], SpanProcessor | None] | None' = None, ) -> None: self._lock: Final = threading.Lock() + self._callback_name: Final = callback_name self._build: Final = processor_factory if processor_factory is not None else _destination_processor self._processors: OrderedDict[object, SpanProcessor] = OrderedDict() # mutable-ok: bounded LRU - self._retired: OrderedDict[object, SpanProcessor] = OrderedDict() # mutable-ok: bounded drain list + self._retired: OrderedDict[int, SpanProcessor] = OrderedDict() # mutable-ok: drains as exports finish + self._exporting: dict[int, int] = {} # mutable-ok: per-processor in-flight export count def on_start(self, span: SDKSpan, parent_context: Context | None = None) -> None: return None def on_end(self, span: ReadableSpan) -> None: for destination in request_destinations(): - processor = self._processor_for(destination) # rebind-ok: loop variable; pyright forbids Final in a loop + if destination.callback_name != self._callback_name: + continue + processor = self._acquire(destination) # rebind-ok: loop variable; pyright forbids Final in a loop if processor is None: continue try: processor.on_end(_with_destination_resource(span, destination)) except Exception as exc: # noqa: BLE001 # one destination's failure must not cost the others their span verbose_logger.debug("OTel V2 fan-out: forwarding to %s failed: %s", destination.endpoint, exc) + finally: + self._release(processor) def shutdown(self) -> None: # Snapshot first: ``on_end`` mutates the cache on whichever thread ends a span @@ -282,6 +294,7 @@ class TenantFanOutSpanProcessor(SpanProcessor): with self._lock: self._processors.clear() self._retired.clear() + self._exporting.clear() def force_flush(self, timeout_millis: int = 30000) -> bool: results: Final = tuple(self._flush_one(processor, timeout_millis) for processor in self._snapshot()) @@ -298,12 +311,14 @@ class TenantFanOutSpanProcessor(SpanProcessor): except Exception: # noqa: BLE001 # one exporter's flush failure must not fail the whole flush return False - def _processor_for(self, destination: "OtelDestination") -> SpanProcessor | None: + def _acquire(self, destination: "OtelDestination") -> SpanProcessor | None: + """The processor for ``destination``, marked busy until ``_release``.""" key: Final = destination.cache_key() with self._lock: - cached: Final = self._processors.get(key) + cached = self._processors.get(key) # rebind-ok: reassigned after the build below if cached is not None: self._processors.move_to_end(key) + self._exporting[id(cached)] = self._exporting.get(id(cached), 0) + 1 return cached built: Final = self._build(destination) if built is None: @@ -312,28 +327,44 @@ class TenantFanOutSpanProcessor(SpanProcessor): existing: Final = self._processors.get(key) if existing is not None: # Another thread won the race; drop ours rather than leak its thread. - _shutdown_quietly(built) + _drain_in_background(built) + self._exporting[id(existing)] = self._exporting.get(id(existing), 0) + 1 return existing self._processors[key] = built - overflowed: Final = self._retired_on_overflow_locked() - if overflowed is not None: - _shutdown_quietly(overflowed) + self._exporting[id(built)] = 1 + self._retire_overflow_locked() + drained: Final = self._drainable_locked() + for processor in drained: + _drain_in_background(processor) return built - def _retired_on_overflow_locked(self) -> SpanProcessor | None: - """Drop the LRU processor past the cap; return one only once it is safe to close. + def _release(self, processor: SpanProcessor) -> None: + with self._lock: + remaining: Final = self._exporting.get(id(processor), 1) - 1 + if remaining > 0: + self._exporting[id(processor)] = remaining + else: + self._exporting.pop(id(processor), None) + drained: Final = self._drainable_locked() + for retired in drained: + _drain_in_background(retired) - ``on_end`` hands a processor back and then exports outside the lock, so shutting - an evicted one down there loses that span. Evictions retire to drain instead, and - the retirees are capped so they cannot accumulate a thread each. - """ + def _retire_overflow_locked(self) -> None: + """Move the LRU processor out of the cache once it is past the cap.""" if len(self._processors) <= _MAX_CACHED_DESTINATION_PROCESSORS: - return None + return _, evicted = self._processors.popitem(last=False) self._retired[id(evicted)] = evicted - if len(self._retired) <= _MAX_RETIRED_DESTINATION_PROCESSORS: - return None - return self._retired.popitem(last=False)[1] + + def _drainable_locked(self) -> tuple[SpanProcessor, ...]: + """Retired processors no thread is exporting through, removed from the list. + + ``on_end`` holds a processor across an export, so closing an evicted one there + drops the span it is holding. A retiree is out of the cache and can never be + handed out again, so once its export count reaches zero it stays there. + """ + idle: Final = tuple(key for key in self._retired if self._exporting.get(key, 0) == 0) + return tuple(self._retired.pop(key) for key in idle) def _destination_processor(destination: "OtelDestination") -> SpanProcessor | None: @@ -351,6 +382,18 @@ def _destination_processor(destination: "OtelDestination") -> SpanProcessor | No return None +def _drain_in_background(processor: SpanProcessor) -> None: + """Close a shed processor off the span-export path. + + ``shutdown`` flushes over the network and is reached from ``on_end``, so closing + one inline would let a single unreachable tenant collector stall every other + tenant's spans behind it. + """ + threading.Thread( + target=_shutdown_quietly, args=(processor,), daemon=True, name="litellm-otel-destination-drain" + ).start() + + def _shutdown_quietly(processor: SpanProcessor) -> None: try: processor.shutdown() @@ -629,6 +672,7 @@ def build_tracer_provider( baggage_processor: SpanProcessor | None = None, use_simple_processor: bool | None = None, tenant_overrides: bool = False, + tenant_callback_name: str | None = None, ) -> TracerProvider: """Build the shared :class:`TracerProvider`. @@ -668,7 +712,7 @@ def build_tracer_provider( _OverriddenBackendFilter(processor, owner) if tenant_overrides and owner is not None else processor ) if tenant_overrides: - provider.add_span_processor(TenantFanOutSpanProcessor()) + provider.add_span_processor(TenantFanOutSpanProcessor(tenant_callback_name)) return provider diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index c9778bbb2ec..71b1164bfc0 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -1,14 +1,15 @@ """Key/team OTLP destinations override the operator's exporters for that backend.""" import contextvars +import time from collections.abc import Mapping -import litellm import pytest from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import SimpleSpanProcessor from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +import litellm from litellm.integrations.otel.model.config import ( ExporterOwner, ExporterSpec, @@ -67,7 +68,7 @@ def wired_provider(dest_exporter: InMemorySpanExporter, global_exporter: InMemor provider = TracerProvider() provider.add_span_processor(_OverriddenBackendFilter(SimpleSpanProcessor(global_exporter), "langfuse_otel")) provider.add_span_processor( - TenantFanOutSpanProcessor(processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter)) + TenantFanOutSpanProcessor("langfuse_otel", processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter)) ) return provider @@ -100,7 +101,7 @@ class TestOverrideSuppression: provider = TracerProvider() provider.add_span_processor(_OverriddenBackendFilter(SimpleSpanProcessor(arize_exporter), "arize")) provider.add_span_processor( - TenantFanOutSpanProcessor(processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter)) + TenantFanOutSpanProcessor("langfuse_otel", processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter)) ) def run(): @@ -118,7 +119,7 @@ class TestFanOut: dest_exporter = InMemorySpanExporter() provider = TracerProvider() provider.add_span_processor( - TenantFanOutSpanProcessor(processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter)) + TenantFanOutSpanProcessor("langfuse_otel", processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter)) ) tracer = get_tracer(provider, "litellm") @@ -140,12 +141,16 @@ class TestFanOut: for child in ("auth /v1/chat/completions", "chat gpt-4"): assert by_name[child].parent.span_id == root.context.span_id - def test_two_destinations_each_receive_their_own_copy(self): - first, second = InMemorySpanExporter(), InMemorySpanExporter() - by_endpoint = {"http://a.local": first, "http://b.local": second} + def test_a_destination_for_another_backend_is_left_to_that_backends_provider(self): + """Every v2 logger has its own provider and emits its own copy of a gen-AI + span, so a proxy running two of them would hand the tenant the same model call + twice if each provider forwarded every destination.""" + langfuse, arize = InMemorySpanExporter(), InMemorySpanExporter() + by_endpoint = {"http://a.local": langfuse, "http://b.local": arize} provider = TracerProvider() provider.add_span_processor( TenantFanOutSpanProcessor( + "langfuse_otel", processor_factory=lambda d: SimpleSpanProcessor(by_endpoint[d.endpoint]), ) ) @@ -161,8 +166,25 @@ class TestFanOut: in_fresh_context(run) - assert [s.name for s in first.get_finished_spans()] == ["chat gpt-4"] - assert [s.name for s in second.get_finished_spans()] == ["chat gpt-4"] + assert [s.name for s in langfuse.get_finished_spans()] == ["chat gpt-4"] + assert arize.get_finished_spans() == () + + def test_a_provider_that_speaks_for_no_backend_forwards_nothing(self): + """The bare ``otel`` callback has no backend of its own; forwarding from it + would duplicate whatever the tenant's own backend provider already sent.""" + dest = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor( + TenantFanOutSpanProcessor(None, processor_factory=lambda _d: SimpleSpanProcessor(dest)) + ) + + def run(): + set_request_destinations((LANGFUSE_DEST,)) + emit(provider) + + in_fresh_context(run) + + assert dest.get_finished_spans() == () def test_a_destination_that_cannot_build_a_processor_is_skipped_quietly(self): """An unbuildable destination must not cost the caller its request.""" @@ -173,7 +195,7 @@ class TestFanOut: attempts.append(destination.endpoint) provider = TracerProvider() - provider.add_span_processor(TenantFanOutSpanProcessor(processor_factory=factory)) + provider.add_span_processor(TenantFanOutSpanProcessor("langfuse_otel", processor_factory=factory)) def run(): set_request_destinations((LANGFUSE_DEST,)) @@ -194,7 +216,7 @@ class TestFanOut: return processor provider = TracerProvider() - provider.add_span_processor(TenantFanOutSpanProcessor(processor_factory=factory)) + provider.add_span_processor(TenantFanOutSpanProcessor("langfuse_otel", processor_factory=factory)) def run(): set_request_destinations((LANGFUSE_DEST,)) @@ -554,66 +576,101 @@ class TestTenantConfigAgreement: class TestEvictionSafety: - def test_an_evicted_processor_is_retired_rather_than_shut_down(self): - """``on_end`` hands a processor back and exports outside the lock, so shutting - an evicted one down there loses that span. Retirees are capped so they cannot - accumulate a thread each.""" - from litellm.integrations.otel.plumbing.providers import ( - _MAX_CACHED_DESTINATION_PROCESSORS, - _MAX_RETIRED_DESTINATION_PROCESSORS, - ) + class Recording(SimpleSpanProcessor): + def __init__(self): + super().__init__(InMemorySpanExporter()) + self.shutdown_calls = 0 - class Recording(SimpleSpanProcessor): - def __init__(self): - super().__init__(InMemorySpanExporter()) - self.shutdown_calls = 0 - - def shutdown(self): - self.shutdown_calls += 1 + def shutdown(self): + self.shutdown_calls += 1 + def _fan_out(self): built = [] def factory(_destination): - built.append(Recording()) + built.append(self.Recording()) return built[-1] - fan_out = TenantFanOutSpanProcessor(processor_factory=factory) - for index in range(_MAX_CACHED_DESTINATION_PROCESSORS + _MAX_RETIRED_DESTINATION_PROCESSORS): - fan_out._processor_for(LANGFUSE_DEST.model_copy(update={"endpoint": f"http://d{index}/otel"})) + return TenantFanOutSpanProcessor("langfuse_otel", processor_factory=factory), built - assert [p.shutdown_calls for p in built] == [0] * len(built) - assert len(fan_out._processors) == _MAX_CACHED_DESTINATION_PROCESSORS + @staticmethod + def _dest(index): + return LANGFUSE_DEST.model_copy(update={"endpoint": f"http://d{index}/otel"}) - for index in range(2): - fan_out._processor_for(LANGFUSE_DEST.model_copy(update={"endpoint": f"http://late{index}/otel"})) + @staticmethod + def _settle(fan_out): + for _ in range(50): + if not fan_out._retired: + return + time.sleep(0.02) - assert [p.shutdown_calls for p in built[:2]] == [1, 1] - assert built[2].shutdown_calls == 0 - assert len(fan_out._processors) == _MAX_CACHED_DESTINATION_PROCESSORS - - def test_a_retired_processor_is_still_flushed_and_closed_on_shutdown(self): + def test_a_processor_still_exporting_a_span_is_not_closed_under_it(self): + """``on_end`` holds a processor across the export, so closing an evicted one + there drops the span it is holding.""" from litellm.integrations.otel.plumbing.providers import _MAX_CACHED_DESTINATION_PROCESSORS - class Recording(SimpleSpanProcessor): - def __init__(self): - super().__init__(InMemorySpanExporter()) - self.shutdown_calls = 0 + fan_out, built = self._fan_out() + held = fan_out._acquire(self._dest(0)) + for index in range(1, _MAX_CACHED_DESTINATION_PROCESSORS + 1): + fan_out._acquire(self._dest(index)) + fan_out._release(built[-1]) + assert held.shutdown_calls == 0 + assert id(held) in fan_out._retired + + fan_out._release(held) + self._settle(fan_out) + + assert held.shutdown_calls == 1 + + def test_an_idle_evicted_processor_is_closed_off_the_export_path(self): + from litellm.integrations.otel.plumbing.providers import _MAX_CACHED_DESTINATION_PROCESSORS + + fan_out, built = self._fan_out() + for index in range(_MAX_CACHED_DESTINATION_PROCESSORS + 1): + fan_out._acquire(self._dest(index)) + fan_out._release(built[-1]) + self._settle(fan_out) + + assert built[0].shutdown_calls == 1 + assert len(fan_out._processors) == _MAX_CACHED_DESTINATION_PROCESSORS + + def test_a_slow_collector_does_not_hold_up_the_export_path(self): + """``shutdown`` flushes over the network and is reached from ``on_end``, so + closing a shed processor inline lets one unreachable tenant collector stall + every other tenant's spans.""" + from litellm.integrations.otel.plumbing.providers import _MAX_CACHED_DESTINATION_PROCESSORS + + class Slow(self.Recording): def shutdown(self): - self.shutdown_calls += 1 + time.sleep(3) + super().shutdown() built = [] def factory(_destination): - built.append(Recording()) + built.append(Slow()) return built[-1] - fan_out = TenantFanOutSpanProcessor(processor_factory=factory) + fan_out = TenantFanOutSpanProcessor("langfuse_otel", processor_factory=factory) + started = time.monotonic() for index in range(_MAX_CACHED_DESTINATION_PROCESSORS + 1): - fan_out._processor_for(LANGFUSE_DEST.model_copy(update={"endpoint": f"http://d{index}/otel"})) + fan_out._acquire(self._dest(index)) + fan_out._release(built[-1]) + + assert time.monotonic() - started < 2 + + def test_a_retired_processor_is_still_closed_on_shutdown(self): + from litellm.integrations.otel.plumbing.providers import _MAX_CACHED_DESTINATION_PROCESSORS + + fan_out, built = self._fan_out() + held = fan_out._acquire(self._dest(0)) + for index in range(1, _MAX_CACHED_DESTINATION_PROCESSORS + 1): + fan_out._acquire(self._dest(index)) + fan_out._release(built[-1]) fan_out.shutdown() - assert built[0].shutdown_calls == 1 + assert held.shutdown_calls == 1 class TestCredentialGatedExporters: From f1cab32449fabf39cfd62bda077e952a7f959cc8 Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Thu, 3 Sep 2026 18:21:34 -0700 Subject: [PATCH 07/47] fix(otel v2): deliver tenant destinations from the published global provider Scoping the fan-out by callback name in the previous commit left every backend that is not the canonical logger with a one-span trace: only the published global provider sees the FastAPI server span, the auth span and the post-call database spans, so an arize-only proxy handed a team's Langfuse just the model call. Attach the fan-out once, to that provider, and let it forward every destination. An overridden backend now skips per-request tracer routing outright rather than only clearing its credential headers, since a key or team otel_service_name was still enough to detach the model call onto a second provider. The destination carries that service name as a resource attribute instead. Shed processors drain on a two-thread pool rather than a thread each, so a tenant cycling its destination config cannot spawn threads as fast as it sends requests. --- litellm/integrations/otel/logger.py | 8 +- .../integrations/otel/plumbing/providers.py | 62 ++++-- litellm/integrations/otel/plumbing/routing.py | 11 +- .../integrations/otel/presets/destinations.py | 12 +- litellm/proxy/litellm_pre_call_utils.py | 21 ++ .../otel/test_otel_v2_destinations.py | 198 +++++++++++++++--- 6 files changed, 252 insertions(+), 60 deletions(-) diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index 7348d913b35..3014a334eeb 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -63,6 +63,7 @@ from litellm.integrations.otel.plumbing.metrics import ( create_genai_metrics, ) from litellm.integrations.otel.plumbing.providers import ( + attach_tenant_fan_out, build_tracer_provider, get_event_logger, get_meter, @@ -182,7 +183,7 @@ class OpenTelemetryV2(CustomLogger): self._tracer_provider: TracerProvider = ( tracer_provider if tracer_provider is not None - else build_tracer_provider(self.config, tenant_overrides=True, tenant_callback_name=callback_name) + else build_tracer_provider(self.config, tenant_overrides=True) ) self.tracer: Tracer = get_tracer(self._tracer_provider, LITELLM_TRACER_NAME) self._metrics_recorder = self._init_metrics(meter_provider) @@ -865,8 +866,13 @@ def publish_global_otel_v2_provider( ``opentelemetry.trace.set_tracer_provider``) are injected so the publish step is unit-testable without reading or mutating real global OTel state. Returns the logger whose provider was published. + + The published provider is also the one that fans spans out to key/team + destinations, because it is the only provider the whole request tree passes + through; see :func:`attach_tenant_fan_out`. """ logger: Final = select_global_otel_v2_logger(in_memory_loggers, registered=registered) + attach_tenant_fan_out(logger._tracer_provider) set_global_provider(logger._tracer_provider) return logger diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index 20649332f40..0ec570eb9e3 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -3,6 +3,8 @@ import threading from collections import OrderedDict from collections.abc import Callable, Iterable +from concurrent.futures import ThreadPoolExecutor +from functools import lru_cache from typing import TYPE_CHECKING, Any, Final, Literal from opentelemetry import _logs, baggage, metrics @@ -246,20 +248,20 @@ class TenantFanOutSpanProcessor(SpanProcessor): requests stay isolated. The forwarded view keeps the original trace and parent ids, so the tenant gets the same tree the operator would have received. - ``callback_name`` scopes the fan-out to the backend this provider speaks for. Every - v2 logger carries its own provider and emits its own copy of a gen-AI span, so a - proxy running two of them would otherwise deliver the tenant two copies of the same - model call. The tenant's own backend always has a logger, since naming it in the - key or team config is what builds one. + Exactly one provider carries this processor, the one published as the OTel global + (see :func:`attach_tenant_fan_out`). That provider is the only one every span + passes through: the FastAPI server span, the auth span and the post-call database + spans are emitted on the global, while a second v2 logger's provider sees only + that logger's own gen-AI span. Attaching the fan-out per logger would hand a + tenant a one-span trace whenever its backend is not the global one, and two + copies of the model call whenever it is. """ def __init__( self, - callback_name: str | None = None, processor_factory: 'Callable[["OtelDestination"], SpanProcessor | None] | None' = None, ) -> None: self._lock: Final = threading.Lock() - self._callback_name: Final = callback_name self._build: Final = processor_factory if processor_factory is not None else _destination_processor self._processors: OrderedDict[object, SpanProcessor] = OrderedDict() # mutable-ok: bounded LRU self._retired: OrderedDict[int, SpanProcessor] = OrderedDict() # mutable-ok: drains as exports finish @@ -270,8 +272,6 @@ class TenantFanOutSpanProcessor(SpanProcessor): def on_end(self, span: ReadableSpan) -> None: for destination in request_destinations(): - if destination.callback_name != self._callback_name: - continue processor = self._acquire(destination) # rebind-ok: loop variable; pyright forbids Final in a loop if processor is None: continue @@ -387,11 +387,16 @@ def _drain_in_background(processor: SpanProcessor) -> None: ``shutdown`` flushes over the network and is reached from ``on_end``, so closing one inline would let a single unreachable tenant collector stall every other - tenant's spans behind it. + tenant's spans behind it. The work goes to a two-thread pool rather than a thread + per processor, so a tenant cycling its destination config cannot spawn threads as + fast as it can send requests; slow shutdowns queue behind each other instead. """ - threading.Thread( - target=_shutdown_quietly, args=(processor,), daemon=True, name="litellm-otel-destination-drain" - ).start() + _drain_pool().submit(_shutdown_quietly, processor) + + +@lru_cache(maxsize=1) +def _drain_pool() -> ThreadPoolExecutor: + return ThreadPoolExecutor(max_workers=2, thread_name_prefix="litellm-otel-destination-drain") def _shutdown_quietly(processor: SpanProcessor) -> None: @@ -672,7 +677,6 @@ def build_tracer_provider( baggage_processor: SpanProcessor | None = None, use_simple_processor: bool | None = None, tenant_overrides: bool = False, - tenant_callback_name: str | None = None, ) -> TracerProvider: """Build the shared :class:`TracerProvider`. @@ -682,11 +686,12 @@ def build_tracer_provider( backends. ``exporter`` and ``use_simple_processor`` are explicit overrides: pass a single exporter to attach exactly that one (used by tests). - ``tenant_overrides`` belongs to the operator-level provider alone: it wraps each - owned exporter so a request that pointed that backend at a key's or team's own - account skips it, and adds the fan-out processor that delivers to that account - instead. The per-tenant providers this same function builds must leave it off, - or they would filter out the very spans they exist to carry. + ``tenant_overrides`` wraps each owned exporter so a request that pointed that + backend at a key's or team's own account skips it. Every v2 logger's provider + wants it, since any of them may own the overridden backend; delivering to the + tenant is a separate job, done once by :func:`attach_tenant_fan_out`. The + per-tenant providers this same function builds must leave it off, or they would + filter out the very spans they exist to carry. """ provider: Final = TracerProvider(resource=build_resource(config)) if baggage_processor is None: @@ -711,11 +716,26 @@ def build_tracer_provider( provider.add_span_processor( _OverriddenBackendFilter(processor, owner) if tenant_overrides and owner is not None else processor ) - if tenant_overrides: - provider.add_span_processor(TenantFanOutSpanProcessor(tenant_callback_name)) return provider +def attach_tenant_fan_out(provider: TracerProvider) -> None: + """Give ``provider`` the fan-out that delivers spans to key/team destinations. + + Called on the one provider published as the OTel global, and idempotent so a + second publish (a test, a re-initialized proxy) cannot double-export. + """ + if any(isinstance(processor, TenantFanOutSpanProcessor) for processor in _attached_processors(provider)): + return + provider.add_span_processor(TenantFanOutSpanProcessor()) + + +def _attached_processors(provider: TracerProvider) -> "tuple[SpanProcessor, ...]": + """The processors already on ``provider``, or empty when the SDK hides them.""" + multi: Final = getattr(provider, "_active_span_processor", None) + return tuple(getattr(multi, "_span_processors", ())) + + def get_tracer(provider: TracerProvider, name: str = "litellm") -> Tracer: # Stamp the instrumentation scope with the LiteLLM package version so every # emitted span carries a deterministic ``scope.version`` (the standard OTel diff --git a/litellm/integrations/otel/plumbing/routing.py b/litellm/integrations/otel/plumbing/routing.py index 710c51d4942..a0831db2eea 100644 --- a/litellm/integrations/otel/plumbing/routing.py +++ b/litellm/integrations/otel/plumbing/routing.py @@ -233,13 +233,12 @@ class TenantTracerCache: the caller's span start. The caller must ``release`` it exactly once. """ # An overridden backend is delivered by the fan-out processor, which carries the - # whole trace. Routing here too would detach this span onto a second provider, + # whole trace and already carries this tenant's credentials and service name. + # Routing here too would detach this span onto a second provider, # so the tenant would get the request tree plus a stray one-span trace. - credential_headers: Final = ( - _NO_HEADERS - if self._callback_name is not None and self._callback_name in overridden_backends() - else self._credential_headers(dynamic_params) - ) + if self._callback_name is not None and self._callback_name in overridden_backends(): + return TenantRoute(tracer=default, detached=False) + credential_headers: Final = self._credential_headers(dynamic_params) project_headers: Final = self._project_headers(auth_metadata) service_name: Final = tenant_service_name(auth_metadata) if not credential_headers and not project_headers and service_name is None: diff --git a/litellm/integrations/otel/presets/destinations.py b/litellm/integrations/otel/presets/destinations.py index 815e8575a76..2bf9bfa5261 100644 --- a/litellm/integrations/otel/presets/destinations.py +++ b/litellm/integrations/otel/presets/destinations.py @@ -118,11 +118,17 @@ def destination_capable_backends() -> frozenset[str]: return frozenset(_DESTINATION_BY_CALLBACK) & frozenset(DYNAMIC_HEADERS_BY_CALLBACK) -def destination_for(callback_name: str, params: StandardCallbackDynamicParams) -> OtelDestination | None: +def destination_for( + callback_name: str, + params: StandardCallbackDynamicParams, + service_name: str | None = None, +) -> OtelDestination | None: """The destination ``params`` names for ``callback_name``, or ``None``. ``None`` means the caller configured nothing usable for this backend, so the - request keeps the operator's global exporters. + request keeps the operator's global exporters. ``service_name`` is the key's or + team's ``otel_service_name``, which the per-request tracer route applies when the + backend is not overridden and the destination has to apply once it is. """ from litellm.integrations.otel.presets import DYNAMIC_HEADERS_BY_CALLBACK @@ -140,7 +146,7 @@ def destination_for(callback_name: str, params: StandardCallbackDynamicParams) - return OtelDestination( endpoint=endpoint, headers=MappingProxyType(dict(headers)), # mutable-ok: MappingProxyType needs a concrete mapping to wrap - resource_attributes=_NO_ATTRS, + resource_attributes=MappingProxyType({"service.name": service_name}) if service_name else _NO_ATTRS, callback_name=callback_name, protocol=protocol, ) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index aed9933424b..5c6af78aad7 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -1044,12 +1044,33 @@ def resolve_tenant_otel_destinations( } ) ), + _tenant_service_name(user_api_key_dict), ) ) is not None ) +def _tenant_service_name(user_api_key_dict: UserAPIKeyAuth) -> str | None: + """The ``service.name`` this key or team configured, the key winning over its team. + + Same fields and same precedence the request-metadata build applies, read straight + off the auth object because destinations resolve during auth, before that metadata + is assembled. + """ + sources: Final = (user_api_key_dict.metadata, user_api_key_dict.team_metadata) + return next( + ( + stripped + for source in sources + if source + for field in OTEL_SERVICE_NAME_METADATA_KEYS + if isinstance(value := source.get(field), str) and (stripped := value.strip()) + ), + None, + ) + + def clean_headers( headers: Headers, litellm_key_header_name: str | None = None, diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index 71b1164bfc0..9da2e25312b 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -17,6 +17,10 @@ from litellm.integrations.otel.model.config import ( is_otel_v2_enabled, ) from litellm.integrations.otel.model.destination import OtelDestination +from litellm.integrations.otel.logger import ( + OpenTelemetryV2, + publish_global_otel_v2_provider, +) from litellm.integrations.otel.plumbing.context import ( overridden_backends, request_destinations, @@ -68,7 +72,7 @@ def wired_provider(dest_exporter: InMemorySpanExporter, global_exporter: InMemor provider = TracerProvider() provider.add_span_processor(_OverriddenBackendFilter(SimpleSpanProcessor(global_exporter), "langfuse_otel")) provider.add_span_processor( - TenantFanOutSpanProcessor("langfuse_otel", processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter)) + TenantFanOutSpanProcessor(processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter)) ) return provider @@ -101,7 +105,7 @@ class TestOverrideSuppression: provider = TracerProvider() provider.add_span_processor(_OverriddenBackendFilter(SimpleSpanProcessor(arize_exporter), "arize")) provider.add_span_processor( - TenantFanOutSpanProcessor("langfuse_otel", processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter)) + TenantFanOutSpanProcessor(processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter)) ) def run(): @@ -119,7 +123,7 @@ class TestFanOut: dest_exporter = InMemorySpanExporter() provider = TracerProvider() provider.add_span_processor( - TenantFanOutSpanProcessor("langfuse_otel", processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter)) + TenantFanOutSpanProcessor(processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter)) ) tracer = get_tracer(provider, "litellm") @@ -141,18 +145,14 @@ class TestFanOut: for child in ("auth /v1/chat/completions", "chat gpt-4"): assert by_name[child].parent.span_id == root.context.span_id - def test_a_destination_for_another_backend_is_left_to_that_backends_provider(self): - """Every v2 logger has its own provider and emits its own copy of a gen-AI - span, so a proxy running two of them would hand the tenant the same model call - twice if each provider forwarded every destination.""" + def test_a_team_naming_two_backends_gets_the_trace_at_both(self): + """The fan-out rides one provider, so it cannot skip a destination on the + grounds that some other backend owns it: nothing else would deliver it.""" langfuse, arize = InMemorySpanExporter(), InMemorySpanExporter() by_endpoint = {"http://a.local": langfuse, "http://b.local": arize} provider = TracerProvider() provider.add_span_processor( - TenantFanOutSpanProcessor( - "langfuse_otel", - processor_factory=lambda d: SimpleSpanProcessor(by_endpoint[d.endpoint]), - ) + TenantFanOutSpanProcessor(processor_factory=lambda d: SimpleSpanProcessor(by_endpoint[d.endpoint])) ) def run(): @@ -167,24 +167,30 @@ class TestFanOut: in_fresh_context(run) assert [s.name for s in langfuse.get_finished_spans()] == ["chat gpt-4"] - assert arize.get_finished_spans() == () + assert [s.name for s in arize.get_finished_spans()] == ["chat gpt-4"] - def test_a_provider_that_speaks_for_no_backend_forwards_nothing(self): - """The bare ``otel`` callback has no backend of its own; forwarding from it - would duplicate whatever the tenant's own backend provider already sent.""" + def test_a_destination_carries_the_tenants_service_name(self): + """An overridden backend skips per-request tracer routing, so the service name + that route used to apply has to travel on the destination instead.""" dest = InMemorySpanExporter() provider = TracerProvider() - provider.add_span_processor( - TenantFanOutSpanProcessor(None, processor_factory=lambda _d: SimpleSpanProcessor(dest)) - ) + provider.add_span_processor(TenantFanOutSpanProcessor(processor_factory=lambda _d: SimpleSpanProcessor(dest))) def run(): - set_request_destinations((LANGFUSE_DEST,)) + set_request_destinations( + ( + OtelDestination( + endpoint="http://a.local", + callback_name="langfuse_otel", + resource_attributes={"service.name": "team-checkout"}, + ), + ) + ) emit(provider) in_fresh_context(run) - assert dest.get_finished_spans() == () + assert {s.resource.attributes["service.name"] for s in dest.get_finished_spans()} == {"team-checkout"} def test_a_destination_that_cannot_build_a_processor_is_skipped_quietly(self): """An unbuildable destination must not cost the caller its request.""" @@ -195,7 +201,7 @@ class TestFanOut: attempts.append(destination.endpoint) provider = TracerProvider() - provider.add_span_processor(TenantFanOutSpanProcessor("langfuse_otel", processor_factory=factory)) + provider.add_span_processor(TenantFanOutSpanProcessor(processor_factory=factory)) def run(): set_request_destinations((LANGFUSE_DEST,)) @@ -216,7 +222,7 @@ class TestFanOut: return processor provider = TracerProvider() - provider.add_span_processor(TenantFanOutSpanProcessor("langfuse_otel", processor_factory=factory)) + provider.add_span_processor(TenantFanOutSpanProcessor(processor_factory=factory)) def run(): set_request_destinations((LANGFUSE_DEST,)) @@ -238,10 +244,35 @@ class TestProviderWiring: return [type(p).__name__ for p in provider._active_span_processor._span_processors] assert "_OverriddenBackendFilter" in kinds(operator) - assert "TenantFanOutSpanProcessor" in kinds(operator) assert "_OverriddenBackendFilter" not in kinds(tenant), "a per-tenant provider must not filter itself out" + assert "TenantFanOutSpanProcessor" not in kinds(operator), "delivery belongs to the published global alone" assert "TenantFanOutSpanProcessor" not in kinds(tenant) + def test_only_the_published_global_provider_delivers_to_tenants(self): + """A second v2 logger's provider never sees the server, auth or database spans, + so fanning out from it would hand the tenant a one-span trace. Publishing is + what picks the one provider the whole request tree passes through.""" + config = OpenTelemetryV2Config(exporters=[ExporterSpec(kind="in_memory", owner=ExporterOwner.ARIZE_AX)]) + published, other = OpenTelemetryV2(config=config, callback_name="arize"), OpenTelemetryV2(config=config) + + publish_global_otel_v2_provider([other], lambda _p: None, registered=published) + + def kinds(logger): + return [type(p).__name__ for p in logger._tracer_provider._active_span_processor._span_processors] + + assert kinds(published).count("TenantFanOutSpanProcessor") == 1 + assert "TenantFanOutSpanProcessor" not in kinds(other) + + def test_publishing_twice_does_not_double_export(self): + config = OpenTelemetryV2Config(exporters=[ExporterSpec(kind="in_memory", owner=ExporterOwner.ARIZE_AX)]) + logger = OpenTelemetryV2(config=config, callback_name="arize") + + publish_global_otel_v2_provider([], lambda _p: None, registered=logger) + publish_global_otel_v2_provider([], lambda _p: None, registered=logger) + + kinds = [type(p).__name__ for p in logger._tracer_provider._active_span_processor._span_processors] + assert kinds.count("TenantFanOutSpanProcessor") == 1 + class TestRouting: def test_an_overridden_backend_is_not_detached_onto_a_second_provider(self): @@ -263,6 +294,27 @@ class TestRouting: assert route.tracer is default assert route.provider is None + def test_an_overridden_backend_does_not_detach_on_a_service_name_either(self): + """A key or team service name is its own reason to build a second provider, so + clearing only the credentials would still take the model call out of the tree.""" + config = OpenTelemetryV2Config( + exporters=[ExporterSpec(kind="otlp_http", endpoint="http://op.local", owner=ExporterOwner.LANGFUSE_OTEL)] + ) + cache = TenantTracerCache(config, "langfuse_otel", "litellm") + default = get_tracer(TracerProvider(), "litellm") + auth_metadata = {"otel_service_name": "team-checkout"} + + assert cache.route_for(default, None, auth_metadata).detached is False + assert cache.route_for(default, None, auth_metadata).tracer is not default + + def run(): + set_request_destinations((LANGFUSE_DEST,)) + return cache.route_for(default, None, auth_metadata) + + route = in_fresh_context(run) + assert route.tracer is default, "the fan-out carries the service name on the destination instead" + assert route.provider is None + @pytest.mark.usefixtures("allow_test_hosts") class TestDestinationResolution: @@ -290,6 +342,57 @@ class TestDestinationResolution: assert [d.endpoint for d in destinations] == ["http://team.local/api/public/otel"] assert destinations[0].callback_name == "langfuse_otel" + def test_a_keys_service_name_outranks_its_teams_on_the_destination(self, monkeypatch): + """The key/team ``otel_service_name`` used to reach the backend through + per-request tracer routing, which an overridden backend skips.""" + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + auth = UserAPIKeyAuth( + metadata={"otel_service_name": "key-svc"}, + team_metadata={ + "otel_service_name": "team-svc", + "logging": [ + { + "callback_name": "langfuse_otel", + "callback_type": "success", + "callback_vars": { + "langfuse_public_key": "pk-team", + "langfuse_secret_key": "sk-team", + "langfuse_host": "http://team.local", + }, + } + ], + }, + ) + + destinations = resolve_tenant_otel_destinations(auth) + + assert dict(destinations[0].resource_attributes) == {"service.name": "key-svc"} + + def test_a_team_that_named_no_service_name_gets_no_resource_override(self, monkeypatch): + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + auth = UserAPIKeyAuth( + team_metadata={ + "otel_service_name": " ", + "logging": [ + { + "callback_name": "langfuse_otel", + "callback_type": "success", + "callback_vars": { + "langfuse_public_key": "pk-team", + "langfuse_secret_key": "sk-team", + "langfuse_host": "http://team.local", + }, + } + ], + } + ) + + destinations = resolve_tenant_otel_destinations(auth) + + assert dict(destinations[0].resource_attributes) == {} + def test_the_key_wins_over_the_team_for_the_same_backend(self, monkeypatch): monkeypatch.setenv("LITELLM_OTEL_V2", "true") is_otel_v2_enabled.cache_clear() @@ -591,16 +694,21 @@ class TestEvictionSafety: built.append(self.Recording()) return built[-1] - return TenantFanOutSpanProcessor("langfuse_otel", processor_factory=factory), built + return TenantFanOutSpanProcessor(processor_factory=factory), built @staticmethod def _dest(index): return LANGFUSE_DEST.model_copy(update={"endpoint": f"http://d{index}/otel"}) @staticmethod - def _settle(fan_out): - for _ in range(50): - if not fan_out._retired: + def _settle(fan_out, processor=None): + """Wait for retirement to clear and, when given, for the drain to run. + + The drain pool is shared and bounded, so a shed processor is closed once a + worker picks it up rather than the moment it is handed over. + """ + for _ in range(500): + if not fan_out._retired and (processor is None or processor.shutdown_calls): return time.sleep(0.02) @@ -630,7 +738,7 @@ class TestEvictionSafety: for index in range(_MAX_CACHED_DESTINATION_PROCESSORS + 1): fan_out._acquire(self._dest(index)) fan_out._release(built[-1]) - self._settle(fan_out) + self._settle(fan_out, built[0]) assert built[0].shutdown_calls == 1 assert len(fan_out._processors) == _MAX_CACHED_DESTINATION_PROCESSORS @@ -652,7 +760,7 @@ class TestEvictionSafety: built.append(Slow()) return built[-1] - fan_out = TenantFanOutSpanProcessor("langfuse_otel", processor_factory=factory) + fan_out = TenantFanOutSpanProcessor(processor_factory=factory) started = time.monotonic() for index in range(_MAX_CACHED_DESTINATION_PROCESSORS + 1): fan_out._acquire(self._dest(index)) @@ -660,6 +768,38 @@ class TestEvictionSafety: assert time.monotonic() - started < 2 + def test_shedding_many_processors_does_not_spawn_a_thread_each(self): + """A tenant that cycles its destination config sheds a processor per request, + so a thread per shed processor is a thread per request against a slow + collector.""" + import threading + + from litellm.integrations.otel.plumbing.providers import _MAX_CACHED_DESTINATION_PROCESSORS + + release = threading.Event() + + class Blocking(self.Recording): + def shutdown(self): + release.wait(timeout=10) + super().shutdown() + + built = [] + + def factory(_destination): + built.append(Blocking()) + return built[-1] + + fan_out = TenantFanOutSpanProcessor(processor_factory=factory) + try: + for index in range(_MAX_CACHED_DESTINATION_PROCESSORS + 30): + fan_out._acquire(self._dest(index)) + fan_out._release(built[-1]) + draining = [t for t in threading.enumerate() if t.name.startswith("litellm-otel-destination-drain")] + assert len(draining) <= 2, f"one drain thread per shed processor: {len(draining)}" + finally: + release.set() + self._settle(fan_out, built[0]) + def test_a_retired_processor_is_still_closed_on_shutdown(self): from litellm.integrations.otel.plumbing.providers import _MAX_CACHED_DESTINATION_PROCESSORS From 1338433b223602ca013c92d48ef699fcae31f95b Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Thu, 3 Sep 2026 21:31:26 -0700 Subject: [PATCH 08/47] fix(otel v2): drain shed destination processors on daemon workers A ThreadPoolExecutor joins its workers at interpreter exit, so one unreachable tenant collector would hold the whole proxy open for its export timeout on the way down. Two long-lived daemon workers off a queue keep the thread count bounded without blocking shutdown. --- .../integrations/otel/plumbing/providers.py | 34 +++++++++++++++---- .../otel/test_otel_v2_destinations.py | 16 ++++++++- 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index 0ec570eb9e3..480abe8bf8c 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -1,9 +1,9 @@ """Provider / exporter factory + the Baggage span processor.""" +import queue import threading from collections import OrderedDict from collections.abc import Callable, Iterable -from concurrent.futures import ThreadPoolExecutor from functools import lru_cache from typing import TYPE_CHECKING, Any, Final, Literal @@ -210,6 +210,10 @@ def _processor_for(exporter: SpanExporter, use_simple: bool | None) -> SpanProce #: pool and a batch thread, so the cache is bounded and evicts least-recently-used. _MAX_CACHED_DESTINATION_PROCESSORS: Final = 32 +#: Workers closing shed destination processors, bounding the threads a tenant can +#: create by cycling its destination config. +_DRAIN_WORKERS: Final = 2 + class _ResourceWrappedReadableSpan(ReadableSpan): """A ``ReadableSpan`` view with an overridden Resource, leaving the original alone.""" @@ -387,16 +391,32 @@ def _drain_in_background(processor: SpanProcessor) -> None: ``shutdown`` flushes over the network and is reached from ``on_end``, so closing one inline would let a single unreachable tenant collector stall every other - tenant's spans behind it. The work goes to a two-thread pool rather than a thread - per processor, so a tenant cycling its destination config cannot spawn threads as - fast as it can send requests; slow shutdowns queue behind each other instead. + tenant's spans behind it. The work goes to two long-lived workers rather than a + thread per processor, so a tenant cycling its destination config cannot spawn + threads as fast as it can send requests; slow shutdowns queue behind each other. """ - _drain_pool().submit(_shutdown_quietly, processor) + _drain_queue().put(processor) @lru_cache(maxsize=1) -def _drain_pool() -> ThreadPoolExecutor: - return ThreadPoolExecutor(max_workers=2, thread_name_prefix="litellm-otel-destination-drain") +def _drain_queue() -> "queue.Queue[SpanProcessor]": + """The shed-processor queue, with its daemon workers started on first use. + + Daemon on purpose. ``ThreadPoolExecutor`` joins its workers at interpreter exit, + so a single unreachable tenant collector would hold the whole proxy open for its + export timeout on the way down. + """ + pending: queue.Queue[SpanProcessor] = queue.Queue() + for _ in range(_DRAIN_WORKERS): + threading.Thread( + target=_drain_forever, args=(pending,), daemon=True, name="litellm-otel-destination-drain" + ).start() + return pending + + +def _drain_forever(pending: "queue.Queue[SpanProcessor]") -> None: + while True: + _shutdown_quietly(pending.get()) def _shutdown_quietly(processor: SpanProcessor) -> None: diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index 9da2e25312b..75ee440e33c 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -727,7 +727,7 @@ class TestEvictionSafety: assert id(held) in fan_out._retired fan_out._release(held) - self._settle(fan_out) + self._settle(fan_out, held) assert held.shutdown_calls == 1 @@ -800,6 +800,20 @@ class TestEvictionSafety: release.set() self._settle(fan_out, built[0]) + def test_the_drain_workers_do_not_hold_the_process_open(self): + """Python joins a ThreadPoolExecutor's workers at interpreter exit, so one + unreachable tenant collector would hold the proxy open for its export + timeout on the way down.""" + import threading + + from litellm.integrations.otel.plumbing.providers import _drain_queue + + _drain_queue() + workers = [t for t in threading.enumerate() if t.name.startswith("litellm-otel-destination-drain")] + + assert workers, "no drain worker was started" + assert all(t.daemon for t in workers), "a non-daemon drain worker blocks interpreter exit" + def test_a_retired_processor_is_still_closed_on_shutdown(self): from litellm.integrations.otel.plumbing.providers import _MAX_CACHED_DESTINATION_PROCESSORS From 3480ec9bef94aba6ce65685c3506127175db4200 Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Thu, 3 Sep 2026 21:52:57 -0700 Subject: [PATCH 09/47] fix(otel v2): give the fan-out its own drain pool instead of a lazy singleton functools.lru_cache does not hold a lock across the call it caches, so concurrent first evictions each finish building a queue and start its workers, and every queue but the winner is abandoned with two daemon threads blocked on it forever. --- .../integrations/otel/plumbing/providers.py | 68 +++++++++---------- .../otel/test_otel_v2_destinations.py | 49 +++++++++++-- 2 files changed, 75 insertions(+), 42 deletions(-) diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index 480abe8bf8c..04866c65fd0 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -4,7 +4,6 @@ import queue import threading from collections import OrderedDict from collections.abc import Callable, Iterable -from functools import lru_cache from typing import TYPE_CHECKING, Any, Final, Literal from opentelemetry import _logs, baggage, metrics @@ -215,6 +214,33 @@ _MAX_CACHED_DESTINATION_PROCESSORS: Final = 32 _DRAIN_WORKERS: Final = 2 +class _DrainPool: + """Closes shed destination processors off the span-export path. + + ``shutdown`` flushes over the network and is reached from ``on_end``, so closing + one inline would let a single unreachable tenant collector stall every other + tenant's spans behind it. A fixed set of workers rather than a thread per + processor means a tenant cycling its destination config cannot spawn threads as + fast as it can send requests; slow shutdowns queue behind each other. + + The workers are daemons and belong to the fan-out that sheds the processors, so + neither an unreachable collector nor a lazily built process-wide singleton can + hold the proxy open on the way down. + """ + + def __init__(self, workers: int = _DRAIN_WORKERS) -> None: + self._pending: Final[queue.Queue[SpanProcessor]] = queue.Queue() + for _ in range(workers): + threading.Thread(target=self._drain_forever, daemon=True, name="litellm-otel-destination-drain").start() + + def submit(self, processor: SpanProcessor) -> None: + self._pending.put(processor) + + def _drain_forever(self) -> None: + while True: + _shutdown_quietly(self._pending.get()) + + class _ResourceWrappedReadableSpan(ReadableSpan): """A ``ReadableSpan`` view with an overridden Resource, leaving the original alone.""" @@ -270,6 +296,7 @@ class TenantFanOutSpanProcessor(SpanProcessor): self._processors: OrderedDict[object, SpanProcessor] = OrderedDict() # mutable-ok: bounded LRU self._retired: OrderedDict[int, SpanProcessor] = OrderedDict() # mutable-ok: drains as exports finish self._exporting: dict[int, int] = {} # mutable-ok: per-processor in-flight export count + self._drain: Final = _DrainPool() def on_start(self, span: SDKSpan, parent_context: Context | None = None) -> None: return None @@ -331,7 +358,7 @@ class TenantFanOutSpanProcessor(SpanProcessor): existing: Final = self._processors.get(key) if existing is not None: # Another thread won the race; drop ours rather than leak its thread. - _drain_in_background(built) + self._drain.submit(built) self._exporting[id(existing)] = self._exporting.get(id(existing), 0) + 1 return existing self._processors[key] = built @@ -339,7 +366,7 @@ class TenantFanOutSpanProcessor(SpanProcessor): self._retire_overflow_locked() drained: Final = self._drainable_locked() for processor in drained: - _drain_in_background(processor) + self._drain.submit(processor) return built def _release(self, processor: SpanProcessor) -> None: @@ -351,7 +378,7 @@ class TenantFanOutSpanProcessor(SpanProcessor): self._exporting.pop(id(processor), None) drained: Final = self._drainable_locked() for retired in drained: - _drain_in_background(retired) + self._drain.submit(retired) def _retire_overflow_locked(self) -> None: """Move the LRU processor out of the cache once it is past the cap.""" @@ -386,39 +413,6 @@ def _destination_processor(destination: "OtelDestination") -> SpanProcessor | No return None -def _drain_in_background(processor: SpanProcessor) -> None: - """Close a shed processor off the span-export path. - - ``shutdown`` flushes over the network and is reached from ``on_end``, so closing - one inline would let a single unreachable tenant collector stall every other - tenant's spans behind it. The work goes to two long-lived workers rather than a - thread per processor, so a tenant cycling its destination config cannot spawn - threads as fast as it can send requests; slow shutdowns queue behind each other. - """ - _drain_queue().put(processor) - - -@lru_cache(maxsize=1) -def _drain_queue() -> "queue.Queue[SpanProcessor]": - """The shed-processor queue, with its daemon workers started on first use. - - Daemon on purpose. ``ThreadPoolExecutor`` joins its workers at interpreter exit, - so a single unreachable tenant collector would hold the whole proxy open for its - export timeout on the way down. - """ - pending: queue.Queue[SpanProcessor] = queue.Queue() - for _ in range(_DRAIN_WORKERS): - threading.Thread( - target=_drain_forever, args=(pending,), daemon=True, name="litellm-otel-destination-drain" - ).start() - return pending - - -def _drain_forever(pending: "queue.Queue[SpanProcessor]") -> None: - while True: - _shutdown_quietly(pending.get()) - - def _shutdown_quietly(processor: SpanProcessor) -> None: try: processor.shutdown() diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index 75ee440e33c..9aee7edf293 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -790,12 +790,13 @@ class TestEvictionSafety: return built[-1] fan_out = TenantFanOutSpanProcessor(processor_factory=factory) + before = self._drain_workers() try: for index in range(_MAX_CACHED_DESTINATION_PROCESSORS + 30): fan_out._acquire(self._dest(index)) fan_out._release(built[-1]) - draining = [t for t in threading.enumerate() if t.name.startswith("litellm-otel-destination-drain")] - assert len(draining) <= 2, f"one drain thread per shed processor: {len(draining)}" + grew = self._drain_workers() - before + assert grew == 0, f"one drain thread per shed processor: {grew} new threads" finally: release.set() self._settle(fan_out, built[0]) @@ -806,14 +807,52 @@ class TestEvictionSafety: timeout on the way down.""" import threading - from litellm.integrations.otel.plumbing.providers import _drain_queue - - _drain_queue() + self._fan_out() workers = [t for t in threading.enumerate() if t.name.startswith("litellm-otel-destination-drain")] assert workers, "no drain worker was started" assert all(t.daemon for t in workers), "a non-daemon drain worker blocks interpreter exit" + def test_a_burst_of_first_evictions_starts_one_set_of_drain_workers(self): + """A drain pool built lazily on first use is not built once: several threads + can each finish the build, and every pool but the winner is left with its + workers blocked on a queue nothing will ever feed again.""" + import threading + + from litellm.integrations.otel.plumbing.providers import ( + _DRAIN_WORKERS, + _MAX_CACHED_DESTINATION_PROCESSORS, + ) + + for _ in range(3): + before = self._drain_workers() + fan_out, built = self._fan_out() + for index in range(_MAX_CACHED_DESTINATION_PROCESSORS): + fan_out._release(fan_out._acquire(self._dest(index))) + barrier = threading.Barrier(16) + + def shed(index, fan_out=fan_out, barrier=barrier): + barrier.wait(timeout=10) + fan_out._release(fan_out._acquire(self._dest(index))) + + threads = [ + threading.Thread(target=shed, args=(_MAX_CACHED_DESTINATION_PROCESSORS + index,)) + for index in range(16) + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=10) + self._settle(fan_out) + + assert self._drain_workers() - before == _DRAIN_WORKERS + + @staticmethod + def _drain_workers(): + import threading + + return len([t for t in threading.enumerate() if t.name.startswith("litellm-otel-destination-drain")]) + def test_a_retired_processor_is_still_closed_on_shutdown(self): from litellm.integrations.otel.plumbing.providers import _MAX_CACHED_DESTINATION_PROCESSORS From 5e98a5d5fcf61a22913159afb9e932ef282f38d7 Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Thu, 3 Sep 2026 22:20:34 -0700 Subject: [PATCH 10/47] fix(otel v2): close no destination processor under a span still in flight The fan-out now refuses new work once shutdown starts and waits out the spans already being forwarded, so teardown neither drops a trace mid-forward nor hands the next caller an exporter nothing will ever close. The wait is bounded so a dead collector cannot hold the proxy open. --- .../integrations/otel/plumbing/providers.py | 31 ++++++++++++-- .../otel/test_otel_v2_destinations.py | 41 +++++++++++++++++++ 2 files changed, 68 insertions(+), 4 deletions(-) diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index 04866c65fd0..b9c842656f4 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -213,6 +213,11 @@ _MAX_CACHED_DESTINATION_PROCESSORS: Final = 32 #: create by cycling its destination config. _DRAIN_WORKERS: Final = 2 +#: How long ``shutdown`` waits for spans already being forwarded, so teardown closes +#: no processor under one. Bounded: an exporter that never returns must not hold the +#: proxy open. +_SHUTDOWN_DRAIN_SECONDS: Final = 5.0 + class _DrainPool: """Closes shed destination processors off the span-export path. @@ -290,8 +295,11 @@ class TenantFanOutSpanProcessor(SpanProcessor): def __init__( self, processor_factory: 'Callable[["OtelDestination"], SpanProcessor | None] | None' = None, + shutdown_drain_seconds: float = _SHUTDOWN_DRAIN_SECONDS, ) -> None: - self._lock: Final = threading.Lock() + self._drain_seconds: Final = shutdown_drain_seconds + self._lock: Final = threading.Condition() + self._closed: Final = threading.Event() self._build: Final = processor_factory if processor_factory is not None else _destination_processor self._processors: OrderedDict[object, SpanProcessor] = OrderedDict() # mutable-ok: bounded LRU self._retired: OrderedDict[int, SpanProcessor] = OrderedDict() # mutable-ok: drains as exports finish @@ -314,9 +322,20 @@ class TenantFanOutSpanProcessor(SpanProcessor): self._release(processor) def shutdown(self) -> None: - # Snapshot first: ``on_end`` mutates the cache on whichever thread ends a span - # and can run concurrently with this SDK-driven shutdown, so iterating the live - # mapping risks a "mutated during iteration" the per-item except cannot catch. + """Close every destination processor, once the spans in flight have landed. + + ``on_end`` runs on whichever thread ends a span and can reach this fan-out + while the SDK is tearing the provider down, so closing blind would drop a + trace mid-forward and would hand the next caller a fresh exporter nothing + will ever close. Refusing new work and then waiting out the in-flight ones + keeps both from happening. + """ + self._closed.set() + with self._lock: + self._lock.wait_for(lambda: not self._exporting, timeout=self._drain_seconds) + # Snapshot: ``on_end`` mutates the cache on whichever thread ends a span, so + # iterating the live mapping risks a "mutated during iteration" the per-item + # except cannot catch. for processor in self._snapshot(): try: processor.shutdown() @@ -344,6 +363,8 @@ class TenantFanOutSpanProcessor(SpanProcessor): def _acquire(self, destination: "OtelDestination") -> SpanProcessor | None: """The processor for ``destination``, marked busy until ``_release``.""" + if self._closed.is_set(): + return None key: Final = destination.cache_key() with self._lock: cached = self._processors.get(key) # rebind-ok: reassigned after the build below @@ -376,6 +397,8 @@ class TenantFanOutSpanProcessor(SpanProcessor): self._exporting[id(processor)] = remaining else: self._exporting.pop(id(processor), None) + if not self._exporting: + self._lock.notify_all() drained: Final = self._drainable_locked() for retired in drained: self._drain.submit(retired) diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index 9aee7edf293..25ed0b1af68 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -853,6 +853,47 @@ class TestEvictionSafety: return len([t for t in threading.enumerate() if t.name.startswith("litellm-otel-destination-drain")]) + def test_shutdown_does_not_close_a_processor_under_an_in_flight_export(self): + """``on_end`` runs on whichever thread ends a span, so it reaches the fan-out + while the SDK tears the provider down.""" + import threading + + fan_out, _ = self._fan_out() + held = fan_out._acquire(self._dest(0)) + closed = threading.Thread(target=fan_out.shutdown) + closed.start() + try: + time.sleep(0.3) + + assert held.shutdown_calls == 0, "closed a processor with a span still being forwarded" + finally: + fan_out._release(held) + closed.join(timeout=10) + + assert held.shutdown_calls == 1 + + def test_a_closed_fan_out_builds_no_new_processor(self): + """A processor built after shutdown is one nothing will ever close, and it + exports to a tenant on a provider the SDK has already torn down.""" + fan_out, built = self._fan_out() + fan_out.shutdown() + + assert fan_out._acquire(self._dest(0)) is None + assert built == [] + + def test_shutdown_gives_up_on_an_export_that_never_finishes(self): + """The wait is bounded: an exporter stuck on a dead collector must not hold + the proxy open on the way down.""" + import threading + + fan_out = TenantFanOutSpanProcessor(processor_factory=lambda _d: self.Recording(), shutdown_drain_seconds=0.2) + fan_out._acquire(self._dest(0)) + closed = threading.Thread(target=fan_out.shutdown) + closed.start() + closed.join(timeout=5) + + assert not closed.is_alive(), "shutdown blocked on an export that never finished" + def test_a_retired_processor_is_still_closed_on_shutdown(self): from litellm.integrations.otel.plumbing.providers import _MAX_CACHED_DESTINATION_PROCESSORS From c5003c88cd77d1263d7c167603d3f6e49241aa6d Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Thu, 3 Sep 2026 22:38:47 -0700 Subject: [PATCH 11/47] fix(otel v2): retire the drain workers with the fan-out that started them A proxy that rebuilds its telemetry builds another fan-out, so workers that outlive the one that started them are two more threads per reload. Shutdown now retires them once everything queued is closed, and a processor shed afterwards is closed inline rather than queued to nobody. --- .../integrations/otel/plumbing/providers.py | 31 ++++++++++++++++--- .../otel/test_otel_v2_destinations.py | 27 ++++++++++++++++ 2 files changed, 54 insertions(+), 4 deletions(-) diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index b9c842656f4..65d9a01b1a5 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -234,16 +234,38 @@ class _DrainPool: """ def __init__(self, workers: int = _DRAIN_WORKERS) -> None: - self._pending: Final[queue.Queue[SpanProcessor]] = queue.Queue() + self._workers: Final = workers + self._closed: Final = threading.Event() + self._pending: Final[queue.Queue[SpanProcessor | None]] = queue.Queue() for _ in range(workers): - threading.Thread(target=self._drain_forever, daemon=True, name="litellm-otel-destination-drain").start() + threading.Thread( + target=self._drain_until_closed, daemon=True, name="litellm-otel-destination-drain" + ).start() def submit(self, processor: SpanProcessor) -> None: + if self._closed.is_set(): + _shutdown_quietly(processor) + return self._pending.put(processor) - def _drain_forever(self) -> None: + def close(self) -> None: + """Retire the workers once they have closed everything already queued. + + A proxy that rebuilds its telemetry builds another fan-out, so workers that + outlive the one that started them are two more threads per reload, forever. + """ + if self._closed.is_set(): + return + self._closed.set() + for _ in range(self._workers): + self._pending.put(None) + + def _drain_until_closed(self) -> None: while True: - _shutdown_quietly(self._pending.get()) + processor: SpanProcessor | None = self._pending.get() # rebind-ok: loop variable + if processor is None: + return + _shutdown_quietly(processor) class _ResourceWrappedReadableSpan(ReadableSpan): @@ -345,6 +367,7 @@ class TenantFanOutSpanProcessor(SpanProcessor): self._processors.clear() self._retired.clear() self._exporting.clear() + self._drain.close() def force_flush(self, timeout_millis: int = 30000) -> bool: results: Final = tuple(self._flush_one(processor, timeout_millis) for processor in self._snapshot()) diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index 25ed0b1af68..e950e5e9f6e 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -894,6 +894,33 @@ class TestEvictionSafety: assert not closed.is_alive(), "shutdown blocked on an export that never finished" + def test_shutdown_retires_the_drain_workers(self): + """A proxy that rebuilds its telemetry builds another fan-out, so workers that + outlive the one that started them are two more threads per reload.""" + from litellm.integrations.otel.plumbing.providers import _DRAIN_WORKERS + + before = self._drain_workers() + fan_out, _ = self._fan_out() + assert self._drain_workers() - before == _DRAIN_WORKERS + + fan_out.shutdown() + for _ in range(500): + if self._drain_workers() == before: + break + time.sleep(0.02) + + assert self._drain_workers() == before, "the drain workers outlived their fan-out" + + def test_a_processor_shed_after_shutdown_is_still_closed(self): + """``close`` retires the workers, so anything handed to the pool afterwards + would sit in a queue nobody reads.""" + fan_out, _ = self._fan_out() + stray = self.Recording() + fan_out.shutdown() + fan_out._drain.submit(stray) + + assert stray.shutdown_calls == 1 + def test_a_retired_processor_is_still_closed_on_shutdown(self): from litellm.integrations.otel.plumbing.providers import _MAX_CACHED_DESTINATION_PROCESSORS From 3ba9133fe8c9ad17d3495e5972633dc8a192d332 Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Fri, 4 Sep 2026 13:43:36 -0700 Subject: [PATCH 12/47] fix(otel v2): guard the fan-out's closed state with the lock that gates it An Event read on its own leaves room for shutdown to run in the gap. A cache miss then inserted a live exporter into a map that had been cleared, and a shed processor landed behind sentinels every drain worker had exited on. The drain pool takes its queue by injection so both interleavings are reachable from a test without patching. --- .../integrations/otel/plumbing/providers.py | 49 ++++++++++----- .../otel/test_otel_v2_destinations.py | 60 +++++++++++++++++++ 2 files changed, 93 insertions(+), 16 deletions(-) diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index 65d9a01b1a5..40aa0d2404b 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -233,20 +233,32 @@ class _DrainPool: hold the proxy open on the way down. """ - def __init__(self, workers: int = _DRAIN_WORKERS) -> None: + def __init__( + self, + workers: int = _DRAIN_WORKERS, + pending: "queue.Queue[SpanProcessor | None] | None" = None, + ) -> None: self._workers: Final = workers - self._closed: Final = threading.Event() - self._pending: Final[queue.Queue[SpanProcessor | None]] = queue.Queue() + self._lock: Final = threading.Lock() + self._closed = False + self._pending: Final[queue.Queue[SpanProcessor | None]] = pending if pending is not None else queue.Queue() for _ in range(workers): threading.Thread( target=self._drain_until_closed, daemon=True, name="litellm-otel-destination-drain" ).start() def submit(self, processor: SpanProcessor) -> None: - if self._closed.is_set(): - _shutdown_quietly(processor) - return - self._pending.put(processor) + """Queue ``processor`` for closing, or close it here once the pool is retired. + + The check and the put share one lock. Reading a closed flag on its own leaves + room for :meth:`close` to run in between, and the processor would land behind + the sentinels every worker has already exited on. + """ + with self._lock: + if not self._closed: + self._pending.put(processor) + return + _shutdown_quietly(processor) def close(self) -> None: """Retire the workers once they have closed everything already queued. @@ -254,11 +266,12 @@ class _DrainPool: A proxy that rebuilds its telemetry builds another fan-out, so workers that outlive the one that started them are two more threads per reload, forever. """ - if self._closed.is_set(): - return - self._closed.set() - for _ in range(self._workers): - self._pending.put(None) + with self._lock: + if self._closed: + return + self._closed = True + for _ in range(self._workers): + self._pending.put(None) def _drain_until_closed(self) -> None: while True: @@ -321,7 +334,7 @@ class TenantFanOutSpanProcessor(SpanProcessor): ) -> None: self._drain_seconds: Final = shutdown_drain_seconds self._lock: Final = threading.Condition() - self._closed: Final = threading.Event() + self._closed = False # guarded by ``_lock``: an unlocked read races the teardown it gates self._build: Final = processor_factory if processor_factory is not None else _destination_processor self._processors: OrderedDict[object, SpanProcessor] = OrderedDict() # mutable-ok: bounded LRU self._retired: OrderedDict[int, SpanProcessor] = OrderedDict() # mutable-ok: drains as exports finish @@ -352,8 +365,8 @@ class TenantFanOutSpanProcessor(SpanProcessor): will ever close. Refusing new work and then waiting out the in-flight ones keeps both from happening. """ - self._closed.set() with self._lock: + self._closed = True self._lock.wait_for(lambda: not self._exporting, timeout=self._drain_seconds) # Snapshot: ``on_end`` mutates the cache on whichever thread ends a span, so # iterating the live mapping risks a "mutated during iteration" the per-item @@ -386,10 +399,10 @@ class TenantFanOutSpanProcessor(SpanProcessor): def _acquire(self, destination: "OtelDestination") -> SpanProcessor | None: """The processor for ``destination``, marked busy until ``_release``.""" - if self._closed.is_set(): - return None key: Final = destination.cache_key() with self._lock: + if self._closed: + return None cached = self._processors.get(key) # rebind-ok: reassigned after the build below if cached is not None: self._processors.move_to_end(key) @@ -399,6 +412,10 @@ class TenantFanOutSpanProcessor(SpanProcessor): if built is None: return None with self._lock: + if self._closed: + # Shutdown ran while this one was being built, so it belongs to nobody. + _shutdown_quietly(built) + return None existing: Final = self._processors.get(key) if existing is not None: # Another thread won the race; drop ours rather than leak its thread. diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index e950e5e9f6e..e47532fcde5 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -921,6 +921,66 @@ class TestEvictionSafety: assert stray.shutdown_calls == 1 + def test_a_processor_built_during_shutdown_is_not_left_in_a_cleared_cache(self): + """The build runs outside the lock, so shutdown can finish inside it. Inserting + afterwards leaves a live exporter, with its batch thread and its connection + pool, in a map nothing will read again.""" + import threading + + built = [] + + def slow(_destination): + time.sleep(0.4) + built.append(self.Recording()) + return built[-1] + + fan_out = TenantFanOutSpanProcessor(processor_factory=slow) + acquired = [] + caller = threading.Thread(target=lambda: acquired.append(fan_out._acquire(self._dest(0)))) + caller.start() + time.sleep(0.1) + fan_out.shutdown() + caller.join(timeout=10) + + assert acquired == [None], "an exporter built after shutdown was handed out" + assert fan_out._processors == {}, "an exporter was left in a cleared cache" + assert built[0].shutdown_calls == 1, "the exporter that lost the race was never closed" + + def test_a_submit_racing_close_is_never_stranded_behind_the_sentinels(self): + """A submit that read the closed state and then let ``close`` run queues its + processor after every sentinel, where the workers have already exited.""" + import queue + import threading + + from litellm.integrations.otel.plumbing.providers import _DrainPool + + at_the_put, close_returned = threading.Event(), threading.Event() + + class Gated(queue.Queue): + def put(self, item, *args, **kwargs): + if item is not None: + at_the_put.set() + close_returned.wait(timeout=1) + super().put(item, *args, **kwargs) + + pool = _DrainPool(pending=Gated()) + submitted = self.Recording() + submitter = threading.Thread(target=pool.submit, args=(submitted,)) + submitter.start() + assert at_the_put.wait(timeout=5) + closer = threading.Thread(target=pool.close) + closer.start() + closer.join(timeout=1.5) + close_returned.set() + submitter.join(timeout=5) + closer.join(timeout=5) + for _ in range(250): + if submitted.shutdown_calls: + break + time.sleep(0.02) + + assert submitted.shutdown_calls == 1, "a processor was queued behind the sentinels and never closed" + def test_a_retired_processor_is_still_closed_on_shutdown(self): from litellm.integrations.otel.plumbing.providers import _MAX_CACHED_DESTINATION_PROCESSORS From e6c5594b8a6444879e510dace74de623589db74d Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Fri, 4 Sep 2026 13:50:10 -0700 Subject: [PATCH 13/47] feat(otel v2): let a tenant destination export alongside the operator's own Override stays the default: a key or team destination replaces the operator's exporter for that backend. Operators running one org-wide backend across every team set litellm_settings.otel_tenant_destination_mode to additive, and the same trace lands in both places. A team that names the operator's own project is still written once, since the fan-out skips a destination the operator's exporter is already sending that span to. --- litellm/__init__.py | 3 + litellm/integrations/otel/logger.py | 2 +- litellm/integrations/otel/plumbing/context.py | 39 ++- .../integrations/otel/plumbing/providers.py | 63 ++++- litellm/integrations/otel/plumbing/routing.py | 10 +- .../otel/test_otel_v2_destinations.py | 233 +++++++++++++++++- 6 files changed, 330 insertions(+), 20 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 42c0ea881fd..dfa72d2aa68 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -325,6 +325,9 @@ ssl_certificate: Optional[str] = None user_url_validation: bool = True user_url_allowed_hosts: List[str] = [] provider_url_destination_allowed_hosts: List[str] = [] +#: "override" (default) or "additive": whether a key or team destination replaces +#: the operator's exporter for that backend or exports alongside it. +otel_tenant_destination_mode: Optional[str] = None ssl_ecdh_curve: Optional[str] = None # Set to 'X25519' to disable PQC and improve performance disable_streaming_logging: bool = False disable_token_counter: bool = False diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index 3014a334eeb..e893fec508c 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -872,7 +872,7 @@ def publish_global_otel_v2_provider( through; see :func:`attach_tenant_fan_out`. """ logger: Final = select_global_otel_v2_logger(in_memory_loggers, registered=registered) - attach_tenant_fan_out(logger._tracer_provider) + attach_tenant_fan_out(logger._tracer_provider, logger.config) set_global_provider(logger._tracer_provider) return logger diff --git a/litellm/integrations/otel/plumbing/context.py b/litellm/integrations/otel/plumbing/context.py index 3d8d8b53388..11b2b66d642 100644 --- a/litellm/integrations/otel/plumbing/context.py +++ b/litellm/integrations/otel/plumbing/context.py @@ -1,5 +1,6 @@ """Trace-context + Baggage helpers.""" +import os from collections.abc import Mapping from contextvars import ContextVar, Token from typing import TYPE_CHECKING, Final @@ -329,12 +330,38 @@ def request_destinations() -> 'tuple["OtelDestination", ...]': return _request_destinations.get() -def overridden_backends() -> frozenset[str]: - """Backends whose global exporters this request must NOT reach. +#: ``litellm_settings: otel_tenant_destination_mode`` and its env equivalent. +ADDITIVE_DESTINATION_MODE: Final = "additive" +OTEL_TENANT_DESTINATION_MODE_ENV: Final = "LITELLM_OTEL_TENANT_DESTINATION_MODE" - A team destination is an override, not an addition: once the request resolved a - destination for a backend, that backend's operator-level exporters are suppressed - for every span of the request, so the tenant's traffic reaches the tenant's - account and nowhere else. + +def tenant_destinations_are_additive() -> bool: + """Whether a tenant destination exports alongside the operator's own exporter. + + Override is the default: the tenant's traffic reaches the tenant's account and + nowhere else. Operators running one org-wide backend across every team set this + to ``additive`` so the same trace lands in both places. + """ + import litellm + + configured: Final = litellm.otel_tenant_destination_mode or os.environ.get(OTEL_TENANT_DESTINATION_MODE_ENV) + return isinstance(configured, str) and configured.strip().lower() == ADDITIVE_DESTINATION_MODE + + +def destination_backends() -> frozenset[str]: + """Backends this request resolved a tenant destination for. + + The fan-out already carries the whole trace to those destinations, so the + per-request tracer route must never send a second copy, in either mode. """ return frozenset(d.callback_name for d in _request_destinations.get() if d.callback_name) + + +def suppressed_backends() -> frozenset[str]: + """Backends whose operator-level exporters this request must NOT reach. + + Empty under ``additive``, where the operator keeps its copy of every span. + """ + if tenant_destinations_are_additive(): + return frozenset() + return destination_backends() diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index 40aa0d2404b..f113876b477 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -3,7 +3,7 @@ import queue import threading from collections import OrderedDict -from collections.abc import Callable, Iterable +from collections.abc import Callable, Iterable, Mapping from typing import TYPE_CHECKING, Any, Final, Literal from opentelemetry import _logs, baggage, metrics @@ -42,8 +42,8 @@ from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2 from litellm.integrations.otel.model.semconv import LiteLLM from litellm.integrations.otel.model.spans import LiteLLMSpanKind from litellm.integrations.otel.plumbing.context import ( - overridden_backends, request_destinations, + suppressed_backends, ) if TYPE_CHECKING: @@ -218,6 +218,9 @@ _DRAIN_WORKERS: Final = 2 #: proxy open. _SHUTDOWN_DRAIN_SECONDS: Final = 5.0 +#: An exporter's account: its normalized endpoint and its credentials. +_SinkKey = tuple[str, tuple[tuple[str, str], ...]] + class _DrainPool: """Closes shed destination processors off the span-export path. @@ -331,7 +334,9 @@ class TenantFanOutSpanProcessor(SpanProcessor): self, processor_factory: 'Callable[["OtelDestination"], SpanProcessor | None] | None' = None, shutdown_drain_seconds: float = _SHUTDOWN_DRAIN_SECONDS, + operator_sinks: frozenset[_SinkKey] = frozenset(), ) -> None: + self._operator_sinks: Final = operator_sinks self._drain_seconds: Final = shutdown_drain_seconds self._lock: Final = threading.Condition() self._closed = False # guarded by ``_lock``: an unlocked read races the teardown it gates @@ -345,7 +350,10 @@ class TenantFanOutSpanProcessor(SpanProcessor): return None def on_end(self, span: ReadableSpan) -> None: + suppressed: Final = suppressed_backends() for destination in request_destinations(): + if self._operator_already_writes(destination, suppressed): + continue processor = self._acquire(destination) # rebind-ok: loop variable; pyright forbids Final in a loop if processor is None: continue @@ -356,6 +364,18 @@ class TenantFanOutSpanProcessor(SpanProcessor): finally: self._release(processor) + def _operator_already_writes(self, destination: "OtelDestination", suppressed: frozenset[str]) -> bool: + """Whether the operator's own exporter is sending this span to the same account. + + Only reachable under ``additive``, where nothing is suppressed: a team that + names the operator's own project would otherwise have every span written + there twice, once by the operator's exporter and once by the fan-out. + """ + return ( + destination.callback_name not in suppressed + and _sink_key(destination.endpoint, destination.headers) in self._operator_sinks + ) + def shutdown(self) -> None: """Close every destination processor, once the spans in flight have landed. @@ -489,6 +509,9 @@ class _OverriddenBackendFilter(SpanProcessor): Wrapping is the only place this works: ``SynchronousMultiSpanProcessor.on_end`` ignores return values, so a sibling processor can never veto the export. + + Under ``additive`` mode nothing is suppressed, so the wrapper passes every span + straight through and the operator keeps its copy. """ def __init__(self, inner: SpanProcessor, owner: str) -> None: @@ -499,7 +522,7 @@ class _OverriddenBackendFilter(SpanProcessor): self._inner.on_start(span, parent_context) def on_end(self, span: ReadableSpan) -> None: - if self._owner in overridden_backends(): + if self._owner in suppressed_backends(): return self._inner.on_end(span) @@ -796,15 +819,43 @@ def build_tracer_provider( return provider -def attach_tenant_fan_out(provider: TracerProvider) -> None: +def attach_tenant_fan_out(provider: TracerProvider, config: OpenTelemetryV2Config | None = None) -> None: """Give ``provider`` the fan-out that delivers spans to key/team destinations. Called on the one provider published as the OTel global, and idempotent so a - second publish (a test, a re-initialized proxy) cannot double-export. + second publish (a test, a re-initialized proxy) cannot double-export. ``config`` + names the operator's own exporters so an additive destination pointing at one of + them is delivered once rather than twice. """ if any(isinstance(processor, TenantFanOutSpanProcessor) for processor in _attached_processors(provider)): return - provider.add_span_processor(TenantFanOutSpanProcessor()) + provider.add_span_processor(TenantFanOutSpanProcessor(operator_sinks=operator_sink_keys(config))) + + +def operator_sink_keys(config: OpenTelemetryV2Config | None) -> frozenset[_SinkKey]: + """The accounts the operator's own exporters write to, in destination terms. + + An exporter with no endpoint of its own resolves one from the environment at + export time, so it has no comparable identity and is left out. + """ + if config is None: + return frozenset() + return frozenset( + key for spec in config.exporters if (key := _sink_key(spec.endpoint, parse_headers(spec.headers))) is not None + ) + + +def _sink_key(endpoint: str | None, headers: Mapping[str, str]) -> "_SinkKey | None": + """The account an exporter writes to, or ``None`` when it has no fixed one. + + Normalized on both counts that make the same account look like two: the operator's + spec carries the signal path a tenant destination leaves for the exporter to + append, and header names survive one round trip lowercased and the other not. + """ + normalized: Final = _otlp_traces_endpoint(endpoint) + if normalized is None: + return None + return (normalized, tuple(sorted((name.lower(), value) for name, value in headers.items()))) def _attached_processors(provider: TracerProvider) -> "tuple[SpanProcessor, ...]": diff --git a/litellm/integrations/otel/plumbing/routing.py b/litellm/integrations/otel/plumbing/routing.py index a0831db2eea..8fd24d8706c 100644 --- a/litellm/integrations/otel/plumbing/routing.py +++ b/litellm/integrations/otel/plumbing/routing.py @@ -25,7 +25,7 @@ from opentelemetry.trace import Tracer from litellm._logging import verbose_logger from litellm.constants import OTEL_SERVICE_NAME_METADATA_KEYS from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config -from litellm.integrations.otel.plumbing.context import overridden_backends +from litellm.integrations.otel.plumbing.context import destination_backends from litellm.integrations.otel.plumbing.providers import ( build_tracer_provider, exporter_transport, @@ -232,11 +232,11 @@ class TenantTracerCache: concurrent overflow eviction can't shut it down between selection and the caller's span start. The caller must ``release`` it exactly once. """ - # An overridden backend is delivered by the fan-out processor, which carries the - # whole trace and already carries this tenant's credentials and service name. - # Routing here too would detach this span onto a second provider, + # A backend with a destination is delivered by the fan-out processor, which + # carries the whole trace and already carries this tenant's credentials and + # service name. Routing here too would detach this span onto a second provider, # so the tenant would get the request tree plus a stray one-span trace. - if self._callback_name is not None and self._callback_name in overridden_backends(): + if self._callback_name is not None and self._callback_name in destination_backends(): return TenantRoute(tracer=default, detached=False) credential_headers: Final = self._credential_headers(dynamic_params) project_headers: Final = self._project_headers(auth_metadata) diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index e47532fcde5..e7e0f716f41 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -3,6 +3,7 @@ import contextvars import time from collections.abc import Mapping +from types import MappingProxyType import pytest from opentelemetry.sdk.trace import TracerProvider @@ -22,14 +23,16 @@ from litellm.integrations.otel.logger import ( publish_global_otel_v2_provider, ) from litellm.integrations.otel.plumbing.context import ( - overridden_backends, + destination_backends, request_destinations, set_request_destinations, ) from litellm.integrations.otel.plumbing.providers import ( TenantFanOutSpanProcessor, _OverriddenBackendFilter, + _sink_key, build_tracer_provider, + operator_sink_keys, ) from litellm.integrations.otel.plumbing.routing import TenantTracerCache, get_tracer from litellm.integrations.otel.presets.destinations import ( @@ -38,6 +41,7 @@ from litellm.integrations.otel.presets.destinations import ( ) from litellm.integrations.otel.presets.langfuse import langfuse_preset from litellm.proxy._types import UserAPIKeyAuth +from litellm.types.utils import StandardCallbackDynamicParams from litellm.proxy.litellm_pre_call_utils import resolve_tenant_otel_destinations LANGFUSE_DEST = OtelDestination( @@ -117,6 +121,231 @@ class TestOverrideSuppression: assert [s.name for s in arize_exporter.get_finished_spans()] == ["chat gpt-4"] +class TestRoutingMode: + """The operator's choice between replacing its own exporter and exporting alongside it. + + One org-wide backend across every team is a real deployment, and losing it the + moment a team configures its own is what ``additive`` exists to prevent. + """ + + OPERATOR_SINK = ("https://cloud.langfuse.com/api/public/otel/v1/traces", (("authorization", "Basic op"),)) + #: What a tenant destination for that same project looks like before normalizing: + #: no signal path yet, and the header name cased the way the backend writes it. + SAME_ACCOUNT_ENDPOINT = "https://cloud.langfuse.com/api/public/otel" + + @staticmethod + def _additive(monkeypatch): + monkeypatch.setattr(litellm, "otel_tenant_destination_mode", "additive", raising=False) + + @staticmethod + def _tree(provider): + tracer = get_tracer(provider, "litellm") + with tracer.start_as_current_span("POST /v1/chat/completions"): + with tracer.start_as_current_span("auth /v1/chat/completions"): + pass + with tracer.start_as_current_span("chat gpt-4"): + pass + + def _run(self, provider, destinations=(LANGFUSE_DEST,)): + def run(): + set_request_destinations(destinations) + self._tree(provider) + + in_fresh_context(run) + + def test_global_only_keeps_every_span_and_delivers_to_nobody(self): + """No team destination resolved, so the operator's backbone is untouched.""" + global_exporter, dest_exporter = InMemorySpanExporter(), InMemorySpanExporter() + provider = wired_provider(dest_exporter, global_exporter) + + self._run(provider, destinations=()) + + assert len(global_exporter.get_finished_spans()) == 3 + assert dest_exporter.get_finished_spans() == () + + def test_team_only_gets_the_whole_tree_with_no_operator_exporter(self): + """A deployment with no operator credentials still gives the team its trace.""" + dest_exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor( + TenantFanOutSpanProcessor(processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter)) + ) + + self._run(provider) + + assert {s.name for s in dest_exporter.get_finished_spans()} == { + "POST /v1/chat/completions", + "auth /v1/chat/completions", + "chat gpt-4", + } + + def test_additive_gives_the_operator_and_the_team_the_same_tree(self, monkeypatch): + self._additive(monkeypatch) + global_exporter, dest_exporter = InMemorySpanExporter(), InMemorySpanExporter() + provider = wired_provider(dest_exporter, global_exporter) + + self._run(provider) + + names = {"POST /v1/chat/completions", "auth /v1/chat/completions", "chat gpt-4"} + assert {s.name for s in global_exporter.get_finished_spans()} == names + assert {s.name for s in dest_exporter.get_finished_spans()} == names + assert len(global_exporter.get_finished_spans()) == 3, "the operator must not get a span twice" + + def test_override_moves_the_tree_off_the_operator(self): + """The default, unchanged: the tenant's traffic reaches the tenant and nowhere else.""" + global_exporter, dest_exporter = InMemorySpanExporter(), InMemorySpanExporter() + provider = wired_provider(dest_exporter, global_exporter) + + self._run(provider) + + assert global_exporter.get_finished_spans() == () + assert len(dest_exporter.get_finished_spans()) == 3 + + def test_a_team_naming_the_operators_own_project_is_written_once(self, monkeypatch): + """Fanning out to two accounts is the point. Writing the same account twice + is a duplicate the operator would see in their own project.""" + self._additive(monkeypatch) + shared = InMemorySpanExporter() + same = OtelDestination( + endpoint=self.SAME_ACCOUNT_ENDPOINT, + headers=MappingProxyType({"Authorization": "Basic op"}), + callback_name="langfuse_otel", + ) + provider = TracerProvider() + provider.add_span_processor(_OverriddenBackendFilter(SimpleSpanProcessor(shared), "langfuse_otel")) + provider.add_span_processor( + TenantFanOutSpanProcessor( + processor_factory=lambda _d: SimpleSpanProcessor(shared), + operator_sinks=frozenset({self.OPERATOR_SINK}), + ) + ) + + self._run(provider, destinations=(same,)) + + assert len(shared.get_finished_spans()) == 3, "the same account received the trace twice" + + def test_in_override_a_team_naming_the_operators_project_still_gets_the_trace(self): + """Override suppresses the operator's own exporter, so the fan-out is the only + thing left delivering. Skipping it on a matching account leaves the team with + nothing at all.""" + shared = InMemorySpanExporter() + same = OtelDestination( + endpoint=self.SAME_ACCOUNT_ENDPOINT, + headers=MappingProxyType({"Authorization": "Basic op"}), + callback_name="langfuse_otel", + ) + provider = TracerProvider() + provider.add_span_processor(_OverriddenBackendFilter(SimpleSpanProcessor(shared), "langfuse_otel")) + provider.add_span_processor( + TenantFanOutSpanProcessor( + processor_factory=lambda _d: SimpleSpanProcessor(shared), + operator_sinks=frozenset({self.OPERATOR_SINK}), + ) + ) + + self._run(provider, destinations=(same,)) + + assert len(shared.get_finished_spans()) == 3, "the team's own destination received nothing" + + def test_a_team_naming_a_different_project_still_gets_its_copy(self, monkeypatch): + """The dedup keys on the account, so a second project is still a second copy.""" + self._additive(monkeypatch) + global_exporter, dest_exporter = InMemorySpanExporter(), InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(_OverriddenBackendFilter(SimpleSpanProcessor(global_exporter), "langfuse_otel")) + provider.add_span_processor( + TenantFanOutSpanProcessor( + processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter), + operator_sinks=frozenset({self.OPERATOR_SINK}), + ) + ) + + self._run(provider) + + assert len(global_exporter.get_finished_spans()) == 3 + assert len(dest_exporter.get_finished_spans()) == 3 + + @pytest.mark.parametrize("additive", [True, False]) + def test_a_failing_team_destination_leaves_the_operator_alone(self, monkeypatch, additive): + """A tenant collector that raises on every span must not cost the operator + its own telemetry, nor take the request down with it.""" + if additive: + self._additive(monkeypatch) + global_exporter, arize_exporter = InMemorySpanExporter(), InMemorySpanExporter() + + class Exploding(SimpleSpanProcessor): + def on_end(self, span): + raise RuntimeError("tenant collector is down") + + provider = TracerProvider() + provider.add_span_processor(_OverriddenBackendFilter(SimpleSpanProcessor(global_exporter), "langfuse_otel")) + provider.add_span_processor(_OverriddenBackendFilter(SimpleSpanProcessor(arize_exporter), "arize")) + provider.add_span_processor( + TenantFanOutSpanProcessor(processor_factory=lambda _d: Exploding(InMemorySpanExporter())) + ) + + self._run(provider) + + assert len(arize_exporter.get_finished_spans()) == 3, "an unrelated backend lost spans" + assert len(global_exporter.get_finished_spans()) == (3 if additive else 0) + + def test_the_env_var_turns_additive_on_without_a_config_file(self, monkeypatch): + monkeypatch.setattr(litellm, "otel_tenant_destination_mode", None, raising=False) + monkeypatch.setenv("LITELLM_OTEL_TENANT_DESTINATION_MODE", "Additive") + global_exporter, dest_exporter = InMemorySpanExporter(), InMemorySpanExporter() + provider = wired_provider(dest_exporter, global_exporter) + + self._run(provider) + + assert len(global_exporter.get_finished_spans()) == 3 + assert len(dest_exporter.get_finished_spans()) == 3 + + def test_an_unrecognized_mode_stays_on_override(self, monkeypatch): + monkeypatch.setattr(litellm, "otel_tenant_destination_mode", "both", raising=False) + global_exporter, dest_exporter = InMemorySpanExporter(), InMemorySpanExporter() + provider = wired_provider(dest_exporter, global_exporter) + + self._run(provider) + + assert global_exporter.get_finished_spans() == () + + def test_operator_sink_keys_skips_an_exporter_with_no_endpoint_of_its_own(self): + """Such an exporter resolves its endpoint from the environment at export + time, so it has no identity to compare a destination against.""" + config = OpenTelemetryV2Config( + exporters=( + ExporterSpec(kind="otlp_http", endpoint=self.OPERATOR_SINK[0], headers="authorization=Basic op"), + ExporterSpec(kind="otlp_http", endpoint=None, headers="authorization=Basic other"), + ) + ) + + assert operator_sink_keys(config) == frozenset({self.OPERATOR_SINK}) + + def test_the_operators_own_langfuse_and_a_team_naming_it_are_one_account(self, monkeypatch): + """The two sides are built by different code that writes the endpoint and the + header names differently, so comparing them raw silently never matches.""" + monkeypatch.setenv("LANGFUSE_HOST", "https://lf.internal") + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-op") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-op") + monkeypatch.setattr(litellm, "provider_url_destination_allowed_hosts", ["lf.internal"], raising=False) + operator = operator_sink_keys(langfuse_preset()) + + def sink(public_key, secret_key): + destination = destination_for( + "langfuse_otel", + StandardCallbackDynamicParams( + langfuse_public_key=public_key, + langfuse_secret_key=secret_key, + langfuse_host="https://lf.internal", + ), + ) + assert destination is not None + return _sink_key(destination.endpoint, destination.headers) + + assert sink("pk-op", "sk-op") in operator, "a team naming the operator's own project" + assert sink("pk-team", "sk-team") not in operator, "a different project on the same server" + + class TestFanOut: def test_every_span_of_the_request_reaches_the_destination_in_one_trace(self): """The whole tree, gen-AI span included, parented as the operator would see it.""" @@ -508,7 +737,7 @@ class TestContextIsolation: def test_destinations_do_not_leak_between_requests(self): def first(): set_request_destinations((LANGFUSE_DEST,)) - return overridden_backends() + return destination_backends() assert in_fresh_context(first) == frozenset({"langfuse_otel"}) assert in_fresh_context(request_destinations) == () From 3c36082c620718a06d497add8824afce3e6c6269 Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Fri, 4 Sep 2026 15:57:07 -0700 Subject: [PATCH 14/47] fix(otel v2): let a straggling export close its own destination processor Shutdown waits out the exports in flight, but the wait has to be bounded or a tenant collector that stops answering holds the proxy open on the way down. Past the bound it closed everything anyway, which is the case it was written to avoid: a processor closed under the span it is carrying loses that span. Keep the bound and retire the stragglers instead. The thread still exporting one closes it through the drain as soon as its export returns, so teardown stays bounded and no span is dropped mid-forward. --- .../integrations/otel/plumbing/providers.py | 21 +++++---- .../otel/test_otel_v2_destinations.py | 45 ++++++++++++++++++- 2 files changed, 56 insertions(+), 10 deletions(-) diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index f113876b477..858cf8a1121 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -383,23 +383,26 @@ class TenantFanOutSpanProcessor(SpanProcessor): while the SDK is tearing the provider down, so closing blind would drop a trace mid-forward and would hand the next caller a fresh exporter nothing will ever close. Refusing new work and then waiting out the in-flight ones - keeps both from happening. + keeps both from happening. The wait has to be bounded, or one destination + whose collector stopped answering would hold the proxy open on the way down, + so a straggler past the bound is retired instead of closed: the thread still + exporting it closes it through the drain as soon as its export returns, and + no span is ever dropped mid-forward. """ with self._lock: self._closed = True self._lock.wait_for(lambda: not self._exporting, timeout=self._drain_seconds) - # Snapshot: ``on_end`` mutates the cache on whichever thread ends a span, so - # iterating the live mapping risks a "mutated during iteration" the per-item - # except cannot catch. - for processor in self._snapshot(): + live: Final = tuple((id(p), p) for p in (*self._processors.values(), *self._retired.values())) + closing: Final = tuple(p for ident, p in live if ident not in self._exporting) + self._processors.clear() + self._retired = OrderedDict( # mutable-ok: the same bounded map, keeping only what is still exporting + (ident, p) for ident, p in live if ident in self._exporting + ) + for processor in closing: try: processor.shutdown() except Exception as exc: # noqa: BLE001 # one processor's shutdown must not abort the rest verbose_logger.debug("OTel V2 fan-out: processor shutdown failed: %s", exc) - with self._lock: - self._processors.clear() - self._retired.clear() - self._exporting.clear() self._drain.close() def force_flush(self, timeout_millis: int = 30000) -> bool: diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index e7e0f716f41..04051e70cb9 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -1150,6 +1150,42 @@ class TestEvictionSafety: assert stray.shutdown_calls == 1 + def test_shutdown_waits_out_an_export_that_lands_inside_the_bound(self): + """Without the wait the closing is left to a daemon thread, which the + interpreter can retire before it runs, so the last spans never reach the + tenant.""" + import threading + + fan_out, built = self._fan_out() + held = fan_out._acquire(self._dest(0)) + threading.Timer(0.2, lambda: fan_out._release(held)).start() + + fan_out.shutdown() + + assert held.shutdown_calls == 1, "shutdown returned before the export it should have waited out" + + def test_a_straggler_past_the_drain_bound_is_closed_by_its_own_thread(self): + """The wait is bounded so one dead collector cannot hold the proxy open, which + means a processor still exporting when it expires has to be left to the thread + holding it rather than closed under the span it is carrying.""" + built = [] + + def factory(_destination): + built.append(self.Recording()) + return built[-1] + + fan_out = TenantFanOutSpanProcessor(processor_factory=factory, shutdown_drain_seconds=0.05) + held = fan_out._acquire(self._dest(0)) + + fan_out.shutdown() + + assert held.shutdown_calls == 0 + + fan_out._release(held) + self._settle(fan_out, held) + + assert held.shutdown_calls == 1 + def test_a_processor_built_during_shutdown_is_not_left_in_a_cleared_cache(self): """The build runs outside the lock, so shutdown can finish inside it. Inserting afterwards leaves a live exporter, with its batch thread and its connection @@ -1210,7 +1246,9 @@ class TestEvictionSafety: assert submitted.shutdown_calls == 1, "a processor was queued behind the sentinels and never closed" - def test_a_retired_processor_is_still_closed_on_shutdown(self): + def test_a_retired_processor_is_still_closed_after_shutdown(self): + """Eviction and shutdown can both land while a span is being forwarded, and the + evicted processor still has to be closed once that export returns.""" from litellm.integrations.otel.plumbing.providers import _MAX_CACHED_DESTINATION_PROCESSORS fan_out, built = self._fan_out() @@ -1220,6 +1258,11 @@ class TestEvictionSafety: fan_out._release(built[-1]) fan_out.shutdown() + assert held.shutdown_calls == 0 + + fan_out._release(held) + self._settle(fan_out, held) + assert held.shutdown_calls == 1 From c9929554bb6bc949b6518f617c85b3c63ae334e0 Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Fri, 4 Sep 2026 15:57:35 -0700 Subject: [PATCH 15/47] fix(otel v2): identify a destination account by its credentials, not its header names Under additive the fan-out skips a destination the operator's own exporter already writes to, so the same account is not written twice. It compared header names as well as values, and one account answers to more than one spelling: the operator's Arize exporter sends space_id where a team destination sends arize-space-id, so every span landed in the operator's own space twice. The credentials are the identity. Compare those and leave the spelling to each backend. --- .../integrations/otel/plumbing/providers.py | 14 ++++++----- .../otel/test_otel_v2_destinations.py | 23 ++++++++++++++++++- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index 858cf8a1121..ef8a35801d8 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -218,8 +218,8 @@ _DRAIN_WORKERS: Final = 2 #: proxy open. _SHUTDOWN_DRAIN_SECONDS: Final = 5.0 -#: An exporter's account: its normalized endpoint and its credentials. -_SinkKey = tuple[str, tuple[tuple[str, str], ...]] +#: An exporter's account: its normalized endpoint and the credentials it presents. +_SinkKey = tuple[str, tuple[str, ...]] class _DrainPool: @@ -851,14 +851,16 @@ def operator_sink_keys(config: OpenTelemetryV2Config | None) -> frozenset[_SinkK def _sink_key(endpoint: str | None, headers: Mapping[str, str]) -> "_SinkKey | None": """The account an exporter writes to, or ``None`` when it has no fixed one. - Normalized on both counts that make the same account look like two: the operator's - spec carries the signal path a tenant destination leaves for the exporter to - append, and header names survive one round trip lowercased and the other not. + The credentials are the identity; the header names are only how each backend + spells them, and one account answers to more than one spelling (Arize takes the + operator's ``space_id`` and a tenant's ``arize-space-id``). The endpoint needs + normalizing too: the operator's spec carries the signal path that a tenant + destination leaves for the exporter to append. """ normalized: Final = _otlp_traces_endpoint(endpoint) if normalized is None: return None - return (normalized, tuple(sorted((name.lower(), value) for name, value in headers.items()))) + return (normalized, tuple(sorted(headers.values()))) def _attached_processors(provider: TracerProvider) -> "tuple[SpanProcessor, ...]": diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index 04051e70cb9..d122f066670 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -39,6 +39,7 @@ from litellm.integrations.otel.presets.destinations import ( destination_capable_backends, destination_for, ) +from litellm.integrations.otel.presets.arize import arize_preset from litellm.integrations.otel.presets.langfuse import langfuse_preset from litellm.proxy._types import UserAPIKeyAuth from litellm.types.utils import StandardCallbackDynamicParams @@ -128,7 +129,7 @@ class TestRoutingMode: moment a team configures its own is what ``additive`` exists to prevent. """ - OPERATOR_SINK = ("https://cloud.langfuse.com/api/public/otel/v1/traces", (("authorization", "Basic op"),)) + OPERATOR_SINK = ("https://cloud.langfuse.com/api/public/otel/v1/traces", ("Basic op",)) #: What a tenant destination for that same project looks like before normalizing: #: no signal path yet, and the header name cased the way the backend writes it. SAME_ACCOUNT_ENDPOINT = "https://cloud.langfuse.com/api/public/otel" @@ -345,6 +346,26 @@ class TestRoutingMode: assert sink("pk-op", "sk-op") in operator, "a team naming the operator's own project" assert sink("pk-team", "sk-team") not in operator, "a different project on the same server" + def test_the_operators_own_arize_space_and_a_team_naming_it_are_one_account(self, monkeypatch): + """One account answers to two header names here: the operator's exporter sends + ``space_id`` and a team destination sends ``arize-space-id``. Keyed on the names, + additive would write the operator's own space twice for every request.""" + monkeypatch.setenv("ARIZE_SPACE_ID", "space-op") + monkeypatch.setenv("ARIZE_API_KEY", "key-op") + monkeypatch.delenv("ARIZE_SPACE_KEY", raising=False) + operator = operator_sink_keys(arize_preset()) + + def sink(space, api_key): + destination = destination_for( + "arize", + StandardCallbackDynamicParams(arize_space_key=space, arize_api_key=api_key), + ) + assert destination is not None + return _sink_key(destination.endpoint, destination.headers) + + assert sink("space-op", "key-op") in operator, "a team naming the operator's own space" + assert sink("space-team", "key-team") not in operator, "a different Arize space" + class TestFanOut: def test_every_span_of_the_request_reaches_the_destination_in_one_trace(self): From c366b917294b669387758dff7b82f76b229b2114 Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Fri, 4 Sep 2026 16:33:53 -0700 Subject: [PATCH 16/47] fix(otel v2): keep the credential's role in a destination's account identity Comparing values alone folds two accounts together whenever they hold the same strings in different roles, and the second team would then get no trace at all. Compare the credential under a normalized name instead, and fold the one alias that actually exists: Arize's space_id and arize-space-id. --- .../integrations/otel/plumbing/providers.py | 24 +++++++++++++------ .../otel/test_otel_v2_destinations.py | 12 +++++++++- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index ef8a35801d8..c30e0e7e5b2 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -4,6 +4,7 @@ import queue import threading from collections import OrderedDict from collections.abc import Callable, Iterable, Mapping +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal from opentelemetry import _logs, baggage, metrics @@ -219,7 +220,11 @@ _DRAIN_WORKERS: Final = 2 _SHUTDOWN_DRAIN_SECONDS: Final = 5.0 #: An exporter's account: its normalized endpoint and the credentials it presents. -_SinkKey = tuple[str, tuple[str, ...]] +_SinkKey = tuple[str, tuple[tuple[str, str], ...]] + +#: Header names that spell one credential two ways. Arize's operator exporter sends +#: ``space_id`` where a tenant destination sends ``arize-space-id``. +_CREDENTIAL_ALIASES: Final = MappingProxyType({"arize_space_id": "space_id"}) class _DrainPool: @@ -851,16 +856,21 @@ def operator_sink_keys(config: OpenTelemetryV2Config | None) -> frozenset[_SinkK def _sink_key(endpoint: str | None, headers: Mapping[str, str]) -> "_SinkKey | None": """The account an exporter writes to, or ``None`` when it has no fixed one. - The credentials are the identity; the header names are only how each backend - spells them, and one account answers to more than one spelling (Arize takes the - operator's ``space_id`` and a tenant's ``arize-space-id``). The endpoint needs - normalizing too: the operator's spec carries the signal path that a tenant - destination leaves for the exporter to append. + Normalized on the three counts that make one account look like two: the operator's + spec carries the signal path a tenant destination leaves for the exporter to + append, header names survive one round trip lowercased and the other not, and one + credential answers to more than one name (see :data:`_CREDENTIAL_ALIASES`). """ normalized: Final = _otlp_traces_endpoint(endpoint) if normalized is None: return None - return (normalized, tuple(sorted(headers.values()))) + return (normalized, tuple(sorted((_credential_name(name), value) for name, value in headers.items()))) + + +def _credential_name(header: str) -> str: + """The credential a header carries, under whichever name the backend spells it.""" + normalized: Final = header.strip().lower().replace("-", "_") + return _CREDENTIAL_ALIASES.get(normalized, normalized) def _attached_processors(provider: TracerProvider) -> "tuple[SpanProcessor, ...]": diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index d122f066670..f51dc27a690 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -129,7 +129,7 @@ class TestRoutingMode: moment a team configures its own is what ``additive`` exists to prevent. """ - OPERATOR_SINK = ("https://cloud.langfuse.com/api/public/otel/v1/traces", ("Basic op",)) + OPERATOR_SINK = ("https://cloud.langfuse.com/api/public/otel/v1/traces", (("authorization", "Basic op"),)) #: What a tenant destination for that same project looks like before normalizing: #: no signal path yet, and the header name cased the way the backend writes it. SAME_ACCOUNT_ENDPOINT = "https://cloud.langfuse.com/api/public/otel" @@ -346,6 +346,16 @@ class TestRoutingMode: assert sink("pk-op", "sk-op") in operator, "a team naming the operator's own project" assert sink("pk-team", "sk-team") not in operator, "a different project on the same server" + def test_two_accounts_holding_the_same_strings_in_different_roles_are_not_one(self): + """The values alone are not the identity. Two accounts can hold the same pair + of strings with the space id and the api key the other way round, and folding + them together would leave the second one's team with no trace at all.""" + endpoint = "https://otlp.arize.com/v1" + + assert _sink_key(endpoint, {"space_id": "a", "api_key": "b"}) != _sink_key( + endpoint, {"space_id": "b", "api_key": "a"} + ) + def test_the_operators_own_arize_space_and_a_team_naming_it_are_one_account(self, monkeypatch): """One account answers to two header names here: the operator's exporter sends ``space_id`` and a team destination sends ``arize-space-id``. Keyed on the names, From 386f2a83eba64d5989c86feca35a269684f001ee Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Fri, 4 Sep 2026 17:05:14 -0700 Subject: [PATCH 17/47] fix(otel v2): build one destination processor per destination, not per racing span Building outside the cache lock meant a cold cache met by a burst of concurrent requests constructed an exporter per thread, kept one, and handed the rest to the drain, so a batch worker and a connection pool per losing thread sat in a queue two workers service. Build under the lock that reads the cache. Opening an exporter connects to nothing, so the lock is held for a constructor, once per destination, and the race it was avoiding stops existing. --- .../integrations/otel/plumbing/providers.py | 42 ++++++++--------- .../otel/test_otel_v2_destinations.py | 46 ++++++++++++++++--- 2 files changed, 59 insertions(+), 29 deletions(-) diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index c30e0e7e5b2..f70f07a8ef8 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -426,36 +426,34 @@ class TenantFanOutSpanProcessor(SpanProcessor): return False def _acquire(self, destination: "OtelDestination") -> SpanProcessor | None: - """The processor for ``destination``, marked busy until ``_release``.""" + """The processor for ``destination``, marked busy until ``_release``. + + The build happens under the same lock that reads the cache, so a cold cache + met by a burst of concurrent requests yields one exporter rather than one per + thread with all but the winner shed. Building an exporter opens no connection, + so the cost of holding the lock is a constructor, once per destination. + """ key: Final = destination.cache_key() with self._lock: if self._closed: return None - cached = self._processors.get(key) # rebind-ok: reassigned after the build below - if cached is not None: + if (cached := self._processors.get(key)) is not None: self._processors.move_to_end(key) - self._exporting[id(cached)] = self._exporting.get(id(cached), 0) + 1 - return cached + processor: Final = cached if cached is not None else self._build_locked(destination, key) + if processor is None: + return None + self._exporting[id(processor)] = self._exporting.get(id(processor), 0) + 1 + drained: Final = self._drainable_locked() + for shed in drained: + self._drain.submit(shed) + return processor + + def _build_locked(self, destination: "OtelDestination", key: object) -> SpanProcessor | None: built: Final = self._build(destination) if built is None: return None - with self._lock: - if self._closed: - # Shutdown ran while this one was being built, so it belongs to nobody. - _shutdown_quietly(built) - return None - existing: Final = self._processors.get(key) - if existing is not None: - # Another thread won the race; drop ours rather than leak its thread. - self._drain.submit(built) - self._exporting[id(existing)] = self._exporting.get(id(existing), 0) + 1 - return existing - self._processors[key] = built - self._exporting[id(built)] = 1 - self._retire_overflow_locked() - drained: Final = self._drainable_locked() - for processor in drained: - self._drain.submit(processor) + self._processors[key] = built + self._retire_overflow_locked() return built def _release(self, processor: SpanProcessor) -> None: diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index f51dc27a690..d27ccedf18c 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -1217,10 +1217,10 @@ class TestEvictionSafety: assert held.shutdown_calls == 1 - def test_a_processor_built_during_shutdown_is_not_left_in_a_cleared_cache(self): - """The build runs outside the lock, so shutdown can finish inside it. Inserting - afterwards leaves a live exporter, with its batch thread and its connection - pool, in a map nothing will read again.""" + def test_a_processor_built_while_shutdown_waits_is_still_closed(self): + """Shutdown cannot slip between the build and the insert, which would leave a + live exporter, with its batch thread and its connection pool, in a map nothing + will read again.""" import threading built = [] @@ -1230,7 +1230,7 @@ class TestEvictionSafety: built.append(self.Recording()) return built[-1] - fan_out = TenantFanOutSpanProcessor(processor_factory=slow) + fan_out = TenantFanOutSpanProcessor(processor_factory=slow, shutdown_drain_seconds=0.05) acquired = [] caller = threading.Thread(target=lambda: acquired.append(fan_out._acquire(self._dest(0)))) caller.start() @@ -1238,9 +1238,41 @@ class TestEvictionSafety: fan_out.shutdown() caller.join(timeout=10) - assert acquired == [None], "an exporter built after shutdown was handed out" + assert acquired == built, "the build shutdown waited out was thrown away" + + fan_out._release(built[0]) + self._settle(fan_out, built[0]) + + assert built[0].shutdown_calls == 1, "the exporter outlived the fan-out" assert fan_out._processors == {}, "an exporter was left in a cleared cache" - assert built[0].shutdown_calls == 1, "the exporter that lost the race was never closed" + + def test_a_cold_cache_met_by_a_burst_builds_one_processor_per_destination(self): + """Building outside the cache lock let every thread of the burst construct its + own exporter, each with a batch thread and a connection pool, and shed all but + one into the drain.""" + import threading + + built = [] + + def factory(_destination): + time.sleep(0.01) + built.append(self.Recording()) + return built[-1] + + fan_out = TenantFanOutSpanProcessor(processor_factory=factory) + ready = threading.Barrier(8) + + def acquire(): + ready.wait() + fan_out._release(fan_out._acquire(self._dest(0))) + + callers = [threading.Thread(target=acquire) for _ in range(8)] + for caller in callers: + caller.start() + for caller in callers: + caller.join(timeout=10) + + assert len(built) == 1, f"one destination, {len(built)} exporters built" def test_a_submit_racing_close_is_never_stranded_behind_the_sentinels(self): """A submit that read the closed state and then let ``close`` run queues its From 594340cdc1046d2f9680295ed7df9a8582abc544 Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Fri, 4 Sep 2026 23:40:42 -0700 Subject: [PATCH 18/47] fix(otel v2): bound the teardown that closes a destination, not the one that never blocks The five-second bound guarded the wait for spans still inside on_end, but a batching processor's on_end only queues the span and returns, so that counter is empty and the bound engaged against nothing. The blocking half was the serial close, which flushes over the network and joins the SDK's own worker thread with no timeout of its own, so a single tenant collector that answers and never finishes held process teardown open for as long as it liked. Hand every close to the drain, whose workers are daemons, and give the whole teardown one deadline. --- .../integrations/otel/plumbing/providers.py | 45 ++++++++++++------- .../otel/test_otel_v2_destinations.py | 28 ++++++++++++ 2 files changed, 58 insertions(+), 15 deletions(-) diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index f70f07a8ef8..651da06b0b5 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -2,6 +2,7 @@ import queue import threading +import time from collections import OrderedDict from collections.abc import Callable, Iterable, Mapping from types import MappingProxyType @@ -250,10 +251,12 @@ class _DrainPool: self._lock: Final = threading.Lock() self._closed = False self._pending: Final[queue.Queue[SpanProcessor | None]] = pending if pending is not None else queue.Queue() - for _ in range(workers): - threading.Thread( - target=self._drain_until_closed, daemon=True, name="litellm-otel-destination-drain" - ).start() + self._threads: Final = tuple( + threading.Thread(target=self._drain_until_closed, daemon=True, name="litellm-otel-destination-drain") + for _ in range(workers) + ) + for worker in self._threads: + worker.start() def submit(self, processor: SpanProcessor) -> None: """Queue ``processor`` for closing, or close it here once the pool is retired. @@ -268,11 +271,15 @@ class _DrainPool: return _shutdown_quietly(processor) - def close(self) -> None: + def close(self, timeout: float | None = None) -> None: """Retire the workers once they have closed everything already queued. A proxy that rebuilds its telemetry builds another fan-out, so workers that outlive the one that started them are two more threads per reload, forever. + + ``timeout`` bounds how long the caller waits for that draining to finish. The + workers are daemons, so whatever is still flushing when it expires is dropped + by the interpreter rather than holding it open. """ with self._lock: if self._closed: @@ -280,6 +287,11 @@ class _DrainPool: self._closed = True for _ in range(self._workers): self._pending.put(None) + if timeout is None: + return + deadline: Final = time.monotonic() + timeout + for worker in self._threads: + worker.join(timeout=max(0.0, deadline - time.monotonic())) def _drain_until_closed(self) -> None: while True: @@ -388,12 +400,18 @@ class TenantFanOutSpanProcessor(SpanProcessor): while the SDK is tearing the provider down, so closing blind would drop a trace mid-forward and would hand the next caller a fresh exporter nothing will ever close. Refusing new work and then waiting out the in-flight ones - keeps both from happening. The wait has to be bounded, or one destination - whose collector stopped answering would hold the proxy open on the way down, - so a straggler past the bound is retired instead of closed: the thread still - exporting it closes it through the drain as soon as its export returns, and - no span is ever dropped mid-forward. + keeps both from happening. A straggler past the bound is retired instead of + closed: the thread still exporting it closes it through the drain as soon as + its export returns, so no span is dropped mid-forward. + + Every close then goes to the drain rather than running here. Closing a + destination processor flushes it over the network and the SDK joins its own + worker with no timeout of its own, so one tenant collector that answers but + never finishes a response would otherwise hold process teardown open for as + long as it likes. The drain's workers are daemons, and the whole teardown + shares one deadline. """ + deadline: Final = time.monotonic() + self._drain_seconds with self._lock: self._closed = True self._lock.wait_for(lambda: not self._exporting, timeout=self._drain_seconds) @@ -404,11 +422,8 @@ class TenantFanOutSpanProcessor(SpanProcessor): (ident, p) for ident, p in live if ident in self._exporting ) for processor in closing: - try: - processor.shutdown() - except Exception as exc: # noqa: BLE001 # one processor's shutdown must not abort the rest - verbose_logger.debug("OTel V2 fan-out: processor shutdown failed: %s", exc) - self._drain.close() + self._drain.submit(processor) + self._drain.close(timeout=max(0.0, deadline - time.monotonic())) def force_flush(self, timeout_millis: int = 30000) -> bool: results: Final = tuple(self._flush_one(processor, timeout_millis) for processor in self._snapshot()) diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index d27ccedf18c..602a2e0b55b 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -1246,6 +1246,34 @@ class TestEvictionSafety: assert built[0].shutdown_calls == 1, "the exporter outlived the fan-out" assert fan_out._processors == {}, "an exporter was left in a cleared cache" + def test_shutdown_returns_when_a_destination_never_finishes_closing(self): + """Closing an exporter flushes over the network and the SDK joins its own + worker with no timeout, so a tenant collector that answers but never finishes + a response would hold process teardown open for as long as it likes.""" + import threading + + never = threading.Event() + + class Stuck(self.Recording): + def shutdown(self): + never.wait() + + built = [] + + def factory(_destination): + built.append(Stuck()) + return built[-1] + + fan_out = TenantFanOutSpanProcessor(processor_factory=factory, shutdown_drain_seconds=0.3) + fan_out._release(fan_out._acquire(self._dest(0))) + returned = threading.Event() + threading.Thread(target=lambda: (fan_out.shutdown(), returned.set()), daemon=True).start() + + came_back = returned.wait(timeout=8) + never.set() + + assert came_back, "shutdown never returned while a collector held its exporter open" + def test_a_cold_cache_met_by_a_burst_builds_one_processor_per_destination(self): """Building outside the cache lock let every thread of the burst construct its own exporter, each with a batch thread and a connection pool, and shed all but From 29765dc8c798b432346b924d5267b289a545c70d Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Sat, 5 Sep 2026 01:45:26 -0700 Subject: [PATCH 19/47] fix(otel v2): preserve operator spans on destination failure --- litellm/integrations/otel/logger.py | 9 +- .../integrations/otel/plumbing/providers.py | 89 ++++++- litellm/litellm_core_utils/litellm_logging.py | 26 +- litellm/proxy/auth/user_api_key_auth.py | 14 +- .../otel/test_otel_v2_destinations.py | 247 +++++++++++++++++- 5 files changed, 362 insertions(+), 23 deletions(-) diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index e893fec508c..0d4b91fa049 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -198,6 +198,11 @@ class OpenTelemetryV2(CustomLogger): self._open_llm_calls: OrderedDict[str, _LLMCallSpan] = OrderedDict() self._init_otel_logger_on_litellm_proxy() + @property + def tracer_provider(self) -> TracerProvider: + """The provider this logger emits through, read-only to its callers.""" + return self._tracer_provider + def _init_metrics(self, meter_provider: "MeterProvider | None") -> "GenAIMetricRecorder | None": """Create the six GenAI histograms when metrics are enabled, else ``None``. @@ -872,8 +877,8 @@ def publish_global_otel_v2_provider( through; see :func:`attach_tenant_fan_out`. """ logger: Final = select_global_otel_v2_logger(in_memory_loggers, registered=registered) - attach_tenant_fan_out(logger._tracer_provider, logger.config) - set_global_provider(logger._tracer_provider) + attach_tenant_fan_out(logger.tracer_provider, logger.config) + set_global_provider(logger.tracer_provider) return logger diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index 651da06b0b5..8422f1963f4 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -8,7 +8,7 @@ from collections.abc import Callable, Iterable, Mapping from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal -from opentelemetry import _logs, baggage, metrics +from opentelemetry import _logs, baggage, metrics, trace from opentelemetry._events import EventLogger from opentelemetry._logs import LoggerProvider, NoOpLoggerProvider from opentelemetry.context import Context @@ -259,17 +259,28 @@ class _DrainPool: worker.start() def submit(self, processor: SpanProcessor) -> None: - """Queue ``processor`` for closing, or close it here once the pool is retired. + """Queue ``processor`` for closing, or hand it off once the pool is retired. The check and the put share one lock. Reading a closed flag on its own leaves room for :meth:`close` to run in between, and the processor would land behind the sentinels every worker has already exited on. + + Past close there is no worker left to take it, and the caller is whichever + thread just ended a span, so closing it inline would park that thread on a + network flush the shutdown deadline has already stopped waiting for. The extra + thread is bounded by the same close: the fan-out stops handing processors out + at that point, so only the ones already exporting when it happened arrive here. """ with self._lock: if not self._closed: self._pending.put(processor) return - _shutdown_quietly(processor) + threading.Thread( + target=_shutdown_quietly, + args=(processor,), + daemon=True, + name="litellm-otel-destination-drain-straggler", + ).start() def close(self, timeout: float | None = None) -> None: """Retire the workers once they have closed everything already queued. @@ -440,6 +451,29 @@ class TenantFanOutSpanProcessor(SpanProcessor): except Exception: # noqa: BLE001 # one exporter's flush failure must not fail the whole flush return False + def deliverable(self, destinations: Iterable["OtelDestination"]) -> tuple["OtelDestination", ...]: + """The subset of ``destinations`` this fan-out can actually export to. + + A destination whose exporter will not build (a protocol whose package is not + installed, a malformed endpoint) has to be dropped before the request anchors + it, not when its first span ends. By then the operator's own exporter has been + told to hold that backend's spans back for this request, so dropping there + loses the span outright instead of leaving it where it would have gone with no + override at all. + """ + return tuple(destination for destination in destinations if self._buildable(destination)) + + def _buildable(self, destination: "OtelDestination") -> bool: + """Whether a processor for ``destination`` exists or can be built right now.""" + with self._lock: + if self._closed: + return False + built: Final = self._cached_or_built_locked(destination) + drained: Final = self._drainable_locked() + for shed in drained: + self._drain.submit(shed) + return built is not None + def _acquire(self, destination: "OtelDestination") -> SpanProcessor | None: """The processor for ``destination``, marked busy until ``_release``. @@ -448,13 +482,10 @@ class TenantFanOutSpanProcessor(SpanProcessor): thread with all but the winner shed. Building an exporter opens no connection, so the cost of holding the lock is a constructor, once per destination. """ - key: Final = destination.cache_key() with self._lock: if self._closed: return None - if (cached := self._processors.get(key)) is not None: - self._processors.move_to_end(key) - processor: Final = cached if cached is not None else self._build_locked(destination, key) + processor: Final = self._cached_or_built_locked(destination) if processor is None: return None self._exporting[id(processor)] = self._exporting.get(id(processor), 0) + 1 @@ -463,6 +494,13 @@ class TenantFanOutSpanProcessor(SpanProcessor): self._drain.submit(shed) return processor + def _cached_or_built_locked(self, destination: "OtelDestination") -> SpanProcessor | None: + key: Final = destination.cache_key() + if (cached := self._processors.get(key)) is not None: + self._processors.move_to_end(key) + return cached + return self._build_locked(destination, key) + def _build_locked(self, destination: "OtelDestination", key: object) -> SpanProcessor | None: built: Final = self._build(destination) if built is None: @@ -853,19 +891,50 @@ def attach_tenant_fan_out(provider: TracerProvider, config: OpenTelemetryV2Confi provider.add_span_processor(TenantFanOutSpanProcessor(operator_sinks=operator_sink_keys(config))) +def deliverable_destinations( + destinations: Iterable["OtelDestination"], + provider: trace.TracerProvider | None = None, +) -> tuple["OtelDestination", ...]: + """The destinations a request can anchor, given what is published to carry them. + + Anchoring a destination is what tells the operator's own exporter to stand down + for that backend, so one nothing can deliver has to be dropped here: with no + fan-out attached, or with an exporter that will not build, the request keeps + exactly the routing it would have had without any override. + """ + fan_out: Final = next( + ( + processor + for processor in _attached_processors(provider if provider is not None else trace.get_tracer_provider()) + if isinstance(processor, TenantFanOutSpanProcessor) + ), + None, + ) + return fan_out.deliverable(destinations) if fan_out is not None else () + + def operator_sink_keys(config: OpenTelemetryV2Config | None) -> frozenset[_SinkKey]: """The accounts the operator's own exporters write to, in destination terms. An exporter with no endpoint of its own resolves one from the environment at - export time, so it has no comparable identity and is left out. + export time, so it has no comparable identity and is left out, and so is one + that never reaches the wire: a console kind ignores the endpoint, and a + header-gated spec with no credentials is skipped when the provider is built. """ if config is None: return frozenset() return frozenset( - key for spec in config.exporters if (key := _sink_key(spec.endpoint, parse_headers(spec.headers))) is not None + key + for spec in config.exporters + if _exports_to_the_wire(spec) and (key := _sink_key(spec.endpoint, parse_headers(spec.headers))) is not None ) +def _exports_to_the_wire(spec: ExporterSpec) -> bool: + """Whether ``build_tracer_provider`` gives ``spec`` an exporter that sends OTLP.""" + return exporter_transport(spec.kind) != "headerless" and not (spec.requires_headers and not spec.headers) + + def _sink_key(endpoint: str | None, headers: Mapping[str, str]) -> "_SinkKey | None": """The account an exporter writes to, or ``None`` when it has no fixed one. @@ -886,7 +955,7 @@ def _credential_name(header: str) -> str: return _CREDENTIAL_ALIASES.get(normalized, normalized) -def _attached_processors(provider: TracerProvider) -> "tuple[SpanProcessor, ...]": +def _attached_processors(provider: trace.TracerProvider) -> "tuple[SpanProcessor, ...]": """The processors already on ``provider``, or empty when the SDK hides them.""" multi: Final = getattr(provider, "_active_span_processor", None) return tuple(getattr(multi, "_span_processors", ())) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index c7e8678be68..8e6e2108dfc 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -200,6 +200,7 @@ if TYPE_CHECKING: from mcp.types import EmbeddedResource, ImageContent, TextContent from litellm.integrations.otel.logger import OpenTelemetryV2 + from litellm.integrations.otel.model.config import OpenTelemetryV2Config from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig try: from litellm_enterprise.enterprise_callbacks.callback_controls import ( @@ -4800,27 +4801,41 @@ def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[Custom Returns ``None`` when V2 is off OR when there's no preset registered for ``callback_name`` — callers should then fall through to the legacy path. + + A preset that needs operator credentials it cannot find is allowed to build + anyway, exporting nowhere, only while this request has a key/team destination + for that backend: the exporter-less logger exists to let the fan-out carry those + spans without a second detached copy. With no such destination the preset raises + as it always did and the caller falls through to the legacy path, so the proxy + never publishes a provider that exports nowhere for a backend the operator + configured and no tenant can use. """ from litellm.integrations.otel.model.config import is_otel_v2_enabled if not is_otel_v2_enabled(): return None from litellm.integrations.otel.logger import OpenTelemetryV2, build_otel_v2_logger + from litellm.integrations.otel.plumbing.context import destination_backends from litellm.integrations.otel.presets import PRESET_BY_CALLBACK preset_fn: Final = PRESET_BY_CALLBACK.get(callback_name) if preset_fn is None: return None + serves_a_destination: Final = callback_name in destination_backends() for callback in _in_memory_loggers: - if isinstance(callback, OpenTelemetryV2) and getattr(callback, "callback_name", None) == callback_name: + if ( + isinstance(callback, OpenTelemetryV2) + and getattr(callback, "callback_name", None) == callback_name + and (serves_a_destination or not _exports_nowhere(callback.config)) + ): return callback try: - config: Final = preset_fn(allow_missing_credentials=True) + config: Final = preset_fn(allow_missing_credentials=serves_a_destination) except Exception: # If env vars are missing or the preset raises, defer to the legacy path # so customers get the same error story they had before V2 landed. return None - if all(spec.requires_headers and not spec.headers for spec in config.exporters): + if _exports_nowhere(config): verbose_logger.warning( "OTel V2: no operator credentials for '%s'; only key/team destinations will receive its traces", callback_name, @@ -4830,6 +4845,11 @@ def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[Custom return v2_logger +def _exports_nowhere(config: "OpenTelemetryV2Config") -> bool: + """Whether every exporter in ``config`` is waiting on credentials it never got.""" + return all(spec.requires_headers and not spec.headers for spec in config.exporters) + + def _maybe_auto_initialize_arize_phoenix(_in_memory_loggers: list[CustomLogger]) -> None: """ Auto-initialize ArizePhoenixLogger when Phoenix env vars are detected. diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 4a0257311d1..680d994bee6 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -2852,17 +2852,23 @@ def _seed_request_destinations(user_api_key_dict: UserAPIKeyAuth) -> None: as well, and on the request task so the ``ContextVar`` is inherited by the logging tasks that close the LLM span. Best-effort: trace routing must never fail auth. - The two ``postgres`` spans under ``auth`` close before this runs, because they are - the reads that resolve the identity being read here, so they keep going to the - operator's backend alone. + Only destinations the published fan-out can build are anchored. Anchoring one is + what tells the operator's exporter to hold that backend's spans back under + ``override``, so an unbuildable one would leave the span with nowhere to go. + + The ``postgres`` spans under ``auth`` close before this runs, because they are the + reads that resolve the identity being read here. They never reach the tenant's + account, and they are never withheld from the operator's backend, whichever mode + is set. """ try: from litellm.integrations.otel.plumbing.context import set_request_destinations + from litellm.integrations.otel.plumbing.providers import deliverable_destinations from litellm.proxy.litellm_pre_call_utils import ( resolve_tenant_otel_destinations, ) - set_request_destinations(resolve_tenant_otel_destinations(user_api_key_dict)) + set_request_destinations(deliverable_destinations(resolve_tenant_otel_destinations(user_api_key_dict))) except Exception as exc: # noqa: BLE001 # telemetry routing is best-effort and must never break authentication verbose_proxy_logger.debug("OTel V2: tenant destination resolution failed: %s", exc) diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index 602a2e0b55b..e2d358b5e31 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -32,6 +32,7 @@ from litellm.integrations.otel.plumbing.providers import ( _OverriddenBackendFilter, _sink_key, build_tracer_provider, + deliverable_destinations, operator_sink_keys, ) from litellm.integrations.otel.plumbing.routing import TenantTracerCache, get_tracer @@ -322,6 +323,48 @@ class TestRoutingMode: assert operator_sink_keys(config) == frozenset({self.OPERATOR_SINK}) + def test_operator_sink_keys_skips_exporters_that_never_reach_the_wire(self): + """A console kind ignores the endpoint and a header-gated spec with no + credentials is dropped when the provider is built, so treating either as an + account the operator writes to would silently withhold a team's own spans + under additive.""" + config = OpenTelemetryV2Config( + exporters=( + ExporterSpec(kind="otlp_http", endpoint=self.OPERATOR_SINK[0], headers="authorization=Basic op"), + ExporterSpec(kind="console", endpoint="http://team.local/v1/traces"), + ExporterSpec(kind="otlp_http", endpoint="http://gated.local/v1/traces", requires_headers=True), + ) + ) + + assert operator_sink_keys(config) == frozenset({self.OPERATOR_SINK}) + + def test_a_team_pointing_at_a_credential_less_operator_exporter_still_gets_its_spans(self, monkeypatch): + """Under additive the fan-out skips a destination the operator already writes + to. An exporter the provider never built writes nothing, so skipping it would + cost the team every span.""" + monkeypatch.setenv("LITELLM_OTEL_TENANT_DESTINATION_MODE", "additive") + gated_endpoint = "http://gated.local/v1/traces" + destination = OtelDestination(endpoint=gated_endpoint, callback_name="newrelic") + config = OpenTelemetryV2Config( + exporters=(ExporterSpec(kind="otlp_http", endpoint=gated_endpoint, requires_headers=True),) + ) + dest_exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor( + TenantFanOutSpanProcessor( + processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter), + operator_sinks=operator_sink_keys(config), + ) + ) + + def run(): + set_request_destinations((destination,)) + emit(provider) + + in_fresh_context(run) + + assert [s.name for s in dest_exporter.get_finished_spans()] == ["chat gpt-4"] + def test_the_operators_own_langfuse_and_a_team_naming_it_are_one_account(self, monkeypatch): """The two sides are built by different code that writes the endpoint and the header names differently, so comparing them raw silently never matches.""" @@ -473,6 +516,82 @@ class TestFanOut: assert attempts == [LANGFUSE_DEST.endpoint] assert reached_the_end == [True] + def test_an_unbuildable_destination_leaves_the_span_with_the_operator(self): + """Anchoring the destination is what makes the operator's exporter stand down + for the backend, so a destination nothing can deliver to must never be anchored, + or the span reaches neither account.""" + global_exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(_OverriddenBackendFilter(SimpleSpanProcessor(global_exporter), "langfuse_otel")) + provider.add_span_processor(TenantFanOutSpanProcessor(processor_factory=lambda _d: None)) + + def run(): + set_request_destinations(deliverable_destinations((LANGFUSE_DEST,), provider)) + emit(provider) + return request_destinations() + + anchored = in_fresh_context(run) + + assert anchored == () + assert [s.name for s in global_exporter.get_finished_spans()] == ["chat gpt-4"] + + def test_a_buildable_destination_is_still_anchored_and_still_overrides(self): + global_exporter, dest_exporter = InMemorySpanExporter(), InMemorySpanExporter() + provider = wired_provider(dest_exporter, global_exporter) + + def run(): + set_request_destinations(deliverable_destinations((LANGFUSE_DEST,), provider)) + emit(provider) + return request_destinations() + + anchored = in_fresh_context(run) + + assert anchored == (LANGFUSE_DEST,) + assert global_exporter.get_finished_spans() == () + assert [s.name for s in dest_exporter.get_finished_spans()] == ["chat gpt-4"] + + def test_only_the_unbuildable_destination_is_dropped_from_a_mixed_set(self): + dest_exporter = InMemorySpanExporter() + other = LANGFUSE_DEST.model_copy(update={"endpoint": "http://broken.local/otel"}) + fan_out = TenantFanOutSpanProcessor( + processor_factory=lambda d: None if d.endpoint == other.endpoint else SimpleSpanProcessor(dest_exporter) + ) + + assert fan_out.deliverable((other, LANGFUSE_DEST)) == (LANGFUSE_DEST,) + + def test_no_fan_out_means_nothing_is_anchored(self): + """With nothing to carry the spans to the tenant, anchoring would only stop the + operator's exporter from writing them.""" + provider = TracerProvider() + + assert deliverable_destinations((LANGFUSE_DEST,), provider) == () + + def test_a_closed_fan_out_anchors_nothing(self): + fan_out = TenantFanOutSpanProcessor(processor_factory=lambda _d: SimpleSpanProcessor(InMemorySpanExporter())) + provider = TracerProvider() + provider.add_span_processor(fan_out) + fan_out.shutdown() + + assert deliverable_destinations((LANGFUSE_DEST,), provider) == () + + def test_the_processor_built_to_check_deliverability_is_the_one_that_exports(self): + built = [] + + def factory(_destination): + built.append(SimpleSpanProcessor(InMemorySpanExporter())) + return built[-1] + + provider = TracerProvider() + provider.add_span_processor(TenantFanOutSpanProcessor(processor_factory=factory)) + + def run(): + set_request_destinations(deliverable_destinations((LANGFUSE_DEST,), provider)) + emit(provider) + + in_fresh_context(run) + + assert len(built) == 1 + def test_one_processor_is_reused_across_spans_of_the_same_destination(self): built = [] @@ -750,18 +869,91 @@ class TestPresetDegradation: with pytest.raises(ValueError, match="LANGFUSE_PUBLIC_KEY"): langfuse_preset() - def test_a_credential_less_proxy_still_builds_the_v2_logger(self, monkeypatch): + def test_a_credential_less_proxy_builds_the_v2_logger_for_a_team_destination(self, monkeypatch): + from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 + + credential_less_proxy(monkeypatch) + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + + def run(): + set_request_destinations((LANGFUSE_DEST,)) + return _maybe_construct_otel_v2("langfuse_otel", []) + + is_otel_v2_enabled.cache_clear() + logger = in_fresh_context(run) + is_otel_v2_enabled.cache_clear() + + assert logger is not None, "team-only deployments must not fall back to the legacy integration" + assert all(spec.requires_headers and not spec.headers for spec in logger.config.exporters) + + def test_a_credential_less_proxy_with_no_destinations_falls_back_to_the_legacy_path(self, monkeypatch): + """Nothing can use a credential-less langfuse here, so the operator has to get + the same story as before v2: the legacy integration, not a global provider + that exports nowhere.""" from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 credential_less_proxy(monkeypatch) monkeypatch.setenv("LITELLM_OTEL_V2", "true") is_otel_v2_enabled.cache_clear() - logger = _maybe_construct_otel_v2("langfuse_otel", []) + logger = in_fresh_context(_maybe_construct_otel_v2, "langfuse_otel", []) is_otel_v2_enabled.cache_clear() - assert logger is not None, "team-only deployments must not fall back to the legacy integration" - assert all(spec.requires_headers and not spec.headers for spec in logger.config.exporters) + assert logger is None + + def test_a_destination_for_one_backend_does_not_degrade_another(self, monkeypatch): + from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 + + credential_less_proxy(monkeypatch) + monkeypatch.delenv("WANDB_API_KEY", raising=False) + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + + def run(): + set_request_destinations((LANGFUSE_DEST,)) + return _maybe_construct_otel_v2("weave_otel", []) + + is_otel_v2_enabled.cache_clear() + logger = in_fresh_context(run) + is_otel_v2_enabled.cache_clear() + + assert logger is None + + def test_the_exporter_less_logger_is_not_reused_by_a_request_without_destinations(self, monkeypatch): + """Reusing it would let one team's destination decide how every later request + without one is logged, long after the degrade was justified.""" + from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 + + credential_less_proxy(monkeypatch) + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + loggers = [] + + def with_destination(): + set_request_destinations((LANGFUSE_DEST,)) + return _maybe_construct_otel_v2("langfuse_otel", loggers) + + is_otel_v2_enabled.cache_clear() + degraded = in_fresh_context(with_destination) + plain = in_fresh_context(_maybe_construct_otel_v2, "langfuse_otel", loggers) + is_otel_v2_enabled.cache_clear() + + assert degraded is not None + assert plain is None + + def test_a_credentialed_logger_is_still_reused_across_requests(self, monkeypatch): + from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 + + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-lf-1") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-lf-1") + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + loggers = [] + + is_otel_v2_enabled.cache_clear() + first = in_fresh_context(_maybe_construct_otel_v2, "langfuse_otel", loggers) + second = in_fresh_context(_maybe_construct_otel_v2, "langfuse_otel", loggers) + is_otel_v2_enabled.cache_clear() + + assert first is not None + assert second is first class TestContextIsolation: @@ -991,6 +1183,21 @@ class TestEvictionSafety: assert held.shutdown_calls == 1 + def test_a_recently_used_destination_is_not_the_one_evicted(self): + """Without the refresh the cache sheds by insertion order, so the busiest + destination is the one whose exporter is rebuilt on every overflow.""" + from litellm.integrations.otel.plumbing.providers import _MAX_CACHED_DESTINATION_PROCESSORS + + fan_out, built = self._fan_out() + for index in range(_MAX_CACHED_DESTINATION_PROCESSORS): + fan_out._release(fan_out._acquire(self._dest(index))) + fan_out._release(fan_out._acquire(self._dest(0))) + fan_out._release(fan_out._acquire(self._dest(_MAX_CACHED_DESTINATION_PROCESSORS))) + self._settle(fan_out, built[1]) + + assert built[1].shutdown_calls == 1 + assert built[0].shutdown_calls == 0, "the destination used most recently was the one shed" + def test_an_idle_evicted_processor_is_closed_off_the_export_path(self): from litellm.integrations.otel.plumbing.providers import _MAX_CACHED_DESTINATION_PROCESSORS @@ -1179,8 +1386,40 @@ class TestEvictionSafety: fan_out.shutdown() fan_out._drain.submit(stray) + for _ in range(500): + if stray.shutdown_calls: + break + time.sleep(0.02) + assert stray.shutdown_calls == 1 + def test_releasing_a_straggler_after_shutdown_does_not_block_the_span_thread(self): + """The teardown deadline has already expired by then, so closing the straggler + inline would park whichever thread just ended a span on the very flush the + deadline gave up waiting for.""" + import threading + + never = threading.Event() + + class Stuck(self.Recording): + def shutdown(self): + never.wait() + + def factory(_destination): + return Stuck() + + fan_out = TenantFanOutSpanProcessor(processor_factory=factory, shutdown_drain_seconds=0.05) + held = fan_out._acquire(self._dest(0)) + fan_out.shutdown() + + released = threading.Event() + caller = threading.Thread(target=lambda: (fan_out._release(held), released.set()), daemon=True) + caller.start() + came_back = released.wait(timeout=5) + never.set() + + assert came_back, "the thread that ended the span was left holding a stuck teardown" + def test_shutdown_waits_out_an_export_that_lands_inside_the_bound(self): """Without the wait the closing is left to a daemon thread, which the interpreter can retire before it runs, so the last spans never reach the From e8cea3e7353054e6c5b970a2415aded6757ee315 Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Sat, 5 Sep 2026 02:16:10 -0700 Subject: [PATCH 20/47] fix(otel v2): anchor destinations off the published provider, refuse headerless tenant transports set_tracer_provider keeps the first provider it is handed, so a process whose OTel global was claimed before the proxy published (auto-instrumentation, a legacy logger) had no fan-out on the global and auth anchored no destination. Auth now reads the fan-out off the registered logger's own provider. A destination whose protocol maps to a headerless exporter kind is no longer buildable: the console fallback would drop the tenant's credentials and print the spans to stdout while the operator's exporter stood down for them. --- litellm/integrations/otel/logger.py | 14 ++++ .../integrations/otel/plumbing/providers.py | 13 +++- litellm/proxy/auth/user_api_key_auth.py | 5 +- .../otel/test_otel_v2_destinations.py | 76 +++++++++++++++++++ 4 files changed, 105 insertions(+), 3 deletions(-) diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index 0d4b91fa049..36b186cdb96 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -16,9 +16,11 @@ from opentelemetry.trace import ( Span, Tracer, get_current_span, + get_tracer_provider, set_span_in_context, use_span, ) +from opentelemetry.trace import TracerProvider as ApiTracerProvider import litellm from litellm._logging import verbose_logger @@ -917,6 +919,18 @@ def seed_request_identity(user_api_key_dict: object, model: str | None = None) - logger.seed_request_identity(user_api_key_dict, model=model) +def fan_out_provider() -> ApiTracerProvider: + """The provider :func:`publish_global_otel_v2_provider` gave the tenant fan-out. + + That is the registered logger's own provider, which stays the carrier even when + the OTel global was claimed before the proxy published (auto-instrumentation, a + legacy logger): ``set_tracer_provider`` keeps the first provider it was given, so + reading the global there would find no fan-out and drop every destination. + """ + logger: Final = _registered_v2_logger() + return logger.tracer_provider if logger is not None else get_tracer_provider() + + @contextmanager def phase_span(name: str) -> "Iterator[Span | None]": logger: Final = _registered_v2_logger() diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index 8422f1963f4..350566d1985 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -541,10 +541,19 @@ class TenantFanOutSpanProcessor(SpanProcessor): def _destination_processor(destination: "OtelDestination") -> SpanProcessor | None: - """A batching OTLP processor aimed at ``destination``, or ``None`` if unbuildable.""" + """A batching OTLP processor aimed at ``destination``, or ``None`` if unbuildable. + + A protocol that resolves to a headerless exporter is unbuildable too: the + console fallback would swallow the tenant's credentials and print its spans to + the proxy's stdout while the operator's exporter stands down for them. + """ + kind: Final = destination.protocol or "otlp_http" + if exporter_transport(kind) == "headerless": + verbose_logger.debug("OTel V2 fan-out: no OTLP transport for protocol %r at %s", kind, destination.endpoint) + return None try: spec: Final = ExporterSpec( - kind=destination.protocol or "otlp_http", + kind=kind, endpoint=destination.endpoint, headers=destination.header_string(), owner=None, diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 680d994bee6..ba92d710cc8 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -2862,13 +2862,16 @@ def _seed_request_destinations(user_api_key_dict: UserAPIKeyAuth) -> None: is set. """ try: + from litellm.integrations.otel.logger import fan_out_provider from litellm.integrations.otel.plumbing.context import set_request_destinations from litellm.integrations.otel.plumbing.providers import deliverable_destinations from litellm.proxy.litellm_pre_call_utils import ( resolve_tenant_otel_destinations, ) - set_request_destinations(deliverable_destinations(resolve_tenant_otel_destinations(user_api_key_dict))) + set_request_destinations( + deliverable_destinations(resolve_tenant_otel_destinations(user_api_key_dict), fan_out_provider()) + ) except Exception as exc: # noqa: BLE001 # telemetry routing is best-effort and must never break authentication verbose_proxy_logger.debug("OTel V2: tenant destination resolution failed: %s", exc) diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index e2d358b5e31..d0e01fcba89 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -566,6 +566,17 @@ class TestFanOut: assert deliverable_destinations((LANGFUSE_DEST,), provider) == () + def test_a_protocol_with_no_otlp_transport_is_not_deliverable(self): + """An unknown exporter kind falls back to the console exporter, which ignores the + tenant's credentials and prints its spans to the proxy's stdout. Treating that as + deliverable would stand the operator's exporter down for spans nobody stores.""" + typo = LANGFUSE_DEST.model_copy(update={"protocol": "consle"}) + fan_out = TenantFanOutSpanProcessor() + try: + assert fan_out.deliverable((typo, LANGFUSE_DEST)) == (LANGFUSE_DEST,) + finally: + fan_out.shutdown() + def test_a_closed_fan_out_anchors_nothing(self): fan_out = TenantFanOutSpanProcessor(processor_factory=lambda _d: SimpleSpanProcessor(InMemorySpanExporter())) provider = TracerProvider() @@ -652,6 +663,71 @@ class TestProviderWiring: kinds = [type(p).__name__ for p in logger._tracer_provider._active_span_processor._span_processors] assert kinds.count("TenantFanOutSpanProcessor") == 1 + def test_anchoring_reads_the_fan_out_off_the_registered_logger_not_the_otel_global(self, monkeypatch): + """``set_tracer_provider`` keeps the first provider it was handed. When + auto-instrumentation or a legacy logger claimed it before the proxy published, + the OTel global carries no fan-out, so reading it there would refuse every + destination while the registered logger's provider would have delivered them.""" + from litellm.integrations.otel.logger import fan_out_provider + from litellm.proxy import proxy_server + + config = OpenTelemetryV2Config(exporters=[ExporterSpec(kind="in_memory", owner=ExporterOwner.LANGFUSE_OTEL)]) + logger = OpenTelemetryV2(config=config, callback_name="langfuse_otel") + publish_global_otel_v2_provider([], lambda _p: None, registered=logger) + monkeypatch.setattr(proxy_server, "open_telemetry_logger", logger) + claimed_first = TracerProvider() + + assert fan_out_provider() is logger.tracer_provider + assert deliverable_destinations((LANGFUSE_DEST,), claimed_first) == () + assert deliverable_destinations((LANGFUSE_DEST,), fan_out_provider()) == (LANGFUSE_DEST,) + + def test_without_a_registered_logger_anchoring_falls_back_to_the_otel_global(self, monkeypatch): + from opentelemetry import trace + + from litellm.integrations.otel.logger import fan_out_provider + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "open_telemetry_logger", None) + + assert fan_out_provider() is trace.get_tracer_provider() + + def test_auth_seeds_the_request_with_destinations_the_registered_logger_can_deliver( + self, monkeypatch, allow_test_hosts + ): + from litellm.proxy import proxy_server + from litellm.proxy.auth.user_api_key_auth import _seed_request_destinations + + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + config = OpenTelemetryV2Config(exporters=[ExporterSpec(kind="in_memory", owner=ExporterOwner.LANGFUSE_OTEL)]) + logger = OpenTelemetryV2(config=config, callback_name="langfuse_otel") + publish_global_otel_v2_provider([], lambda _p: None, registered=logger) + monkeypatch.setattr(proxy_server, "open_telemetry_logger", logger) + auth = UserAPIKeyAuth( + team_metadata={ + "logging": [ + { + "callback_name": "langfuse_otel", + "callback_type": "success", + "callback_vars": { + "langfuse_public_key": "pk-team", + "langfuse_secret_key": "sk-team", + "langfuse_host": "http://team.local", + }, + } + ] + } + ) + expected = resolve_tenant_otel_destinations(auth) + assert expected, "the fixture must resolve to a destination for the test to mean anything" + + def run(): + _seed_request_destinations(auth) + return request_destinations() + + assert deliverable_destinations(expected, TracerProvider()) == () + assert in_fresh_context(run) == expected + class TestRouting: def test_an_overridden_backend_is_not_detached_onto_a_second_provider(self): From 37c964b0a2b5d0b69e4e89e2739b2c1d8ba4545c Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Sat, 5 Sep 2026 02:42:29 -0700 Subject: [PATCH 21/47] fix(otel): anchor tenant fan-out to the published provider A legacy v1 logger can occupy proxy_server.open_telemetry_logger, in which case the proxy publishes with registered=None and the fan-out lands on a v2 logger taken from _in_memory_loggers. Reading the registered slot found no v2 logger and the OTel global belonged to v1, so auth refused every tenant destination. --- litellm/integrations/otel/logger.py | 21 ++++++++---- .../otel/test_otel_v2_destinations.py | 34 ++++++++++++++++--- .../integrations/otel/test_otel_v2_logger.py | 4 ++- 3 files changed, 46 insertions(+), 13 deletions(-) diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index 36b186cdb96..21e1feb4022 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -88,6 +88,7 @@ if TYPE_CHECKING: ) LITELLM_TRACER_NAME: Final = "litellm" +_published_v2_provider: ApiTracerProvider | None = None def _span_error_from_exception( @@ -876,11 +877,17 @@ def publish_global_otel_v2_provider( The published provider is also the one that fans spans out to key/team destinations, because it is the only provider the whole request tree passes - through; see :func:`attach_tenant_fan_out`. + through; see :func:`attach_tenant_fan_out`. It is remembered for + :func:`fan_out_provider` because neither the OTel global (``set_tracer_provider`` + keeps the first provider it was ever handed) nor + ``proxy_server.open_telemetry_logger`` (a legacy v1 logger can hold that slot) + reliably leads back to it. """ + global _published_v2_provider logger: Final = select_global_otel_v2_logger(in_memory_loggers, registered=registered) attach_tenant_fan_out(logger.tracer_provider, logger.config) set_global_provider(logger.tracer_provider) + _published_v2_provider = logger.tracer_provider # rebind-ok: startup records the one provider carrying the fan-out return logger @@ -922,13 +929,13 @@ def seed_request_identity(user_api_key_dict: object, model: str | None = None) - def fan_out_provider() -> ApiTracerProvider: """The provider :func:`publish_global_otel_v2_provider` gave the tenant fan-out. - That is the registered logger's own provider, which stays the carrier even when - the OTel global was claimed before the proxy published (auto-instrumentation, a - legacy logger): ``set_tracer_provider`` keeps the first provider it was given, so - reading the global there would find no fan-out and drop every destination. + Read off the publish itself, not the OTel global and not the registered logger: + the global keeps whichever provider claimed it first (auto-instrumentation, a + legacy logger), and the registered slot can hold a v1 logger while the publish + picked a v2 one from ``_in_memory_loggers``. Either detour lands on a provider + with no fan-out and drops every destination at auth. """ - logger: Final = _registered_v2_logger() - return logger.tracer_provider if logger is not None else get_tracer_provider() + return _published_v2_provider if _published_v2_provider is not None else get_tracer_provider() @contextmanager diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index d0e01fcba89..706916c69fb 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -18,8 +18,10 @@ from litellm.integrations.otel.model.config import ( is_otel_v2_enabled, ) from litellm.integrations.otel.model.destination import OtelDestination +from litellm.integrations.otel import logger as otel_logger from litellm.integrations.otel.logger import ( OpenTelemetryV2, + fan_out_provider, publish_global_otel_v2_provider, ) from litellm.integrations.otel.plumbing.context import ( @@ -63,6 +65,13 @@ def allow_test_hosts(monkeypatch): ) +@pytest.fixture(autouse=True) +def isolate_published_provider(monkeypatch): + """Publishing records the fan-out carrier in module state; one test's publish must + not become the next test's provider.""" + monkeypatch.setattr(otel_logger, "_published_v2_provider", None) + + def in_fresh_context(fn, *args): """Run ``fn`` in its own context so one test's destinations never leak.""" return contextvars.copy_context().run(fn, *args) @@ -663,12 +672,11 @@ class TestProviderWiring: kinds = [type(p).__name__ for p in logger._tracer_provider._active_span_processor._span_processors] assert kinds.count("TenantFanOutSpanProcessor") == 1 - def test_anchoring_reads_the_fan_out_off_the_registered_logger_not_the_otel_global(self, monkeypatch): + def test_anchoring_reads_the_fan_out_off_the_published_provider_not_the_otel_global(self, monkeypatch): """``set_tracer_provider`` keeps the first provider it was handed. When auto-instrumentation or a legacy logger claimed it before the proxy published, the OTel global carries no fan-out, so reading it there would refuse every - destination while the registered logger's provider would have delivered them.""" - from litellm.integrations.otel.logger import fan_out_provider + destination the published provider delivers.""" from litellm.proxy import proxy_server config = OpenTelemetryV2Config(exporters=[ExporterSpec(kind="in_memory", owner=ExporterOwner.LANGFUSE_OTEL)]) @@ -681,10 +689,26 @@ class TestProviderWiring: assert deliverable_destinations((LANGFUSE_DEST,), claimed_first) == () assert deliverable_destinations((LANGFUSE_DEST,), fan_out_provider()) == (LANGFUSE_DEST,) - def test_without_a_registered_logger_anchoring_falls_back_to_the_otel_global(self, monkeypatch): + def test_a_legacy_v1_logger_holding_the_registered_slot_does_not_hide_the_fan_out(self, monkeypatch): + """The proxy publishes with ``registered=None`` when ``open_telemetry_logger`` + holds a v1 logger, so the fan-out lands on a v2 logger taken from + ``_in_memory_loggers``. Reading the registered slot finds no v2 logger there and + the OTel global belongs to v1, so both detours refuse every destination the + published provider delivers.""" + from litellm.integrations.opentelemetry import OpenTelemetry + from litellm.proxy import proxy_server + + config = OpenTelemetryV2Config(exporters=[ExporterSpec(kind="in_memory", owner=ExporterOwner.LANGFUSE_OTEL)]) + v2 = OpenTelemetryV2(config=config, callback_name="langfuse_otel") + publish_global_otel_v2_provider([v2], lambda _p: None, registered=None) + monkeypatch.setattr(proxy_server, "open_telemetry_logger", OpenTelemetry()) + + assert fan_out_provider() is v2.tracer_provider + assert deliverable_destinations((LANGFUSE_DEST,), fan_out_provider()) == (LANGFUSE_DEST,) + + def test_without_a_publish_anchoring_falls_back_to_the_otel_global(self, monkeypatch): from opentelemetry import trace - from litellm.integrations.otel.logger import fan_out_provider from litellm.proxy import proxy_server monkeypatch.setattr(proxy_server, "open_telemetry_logger", None) diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py index b735abaf7bf..2869c804c07 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py @@ -2041,7 +2041,7 @@ def test_select_global_otel_v2_logger_builds_one_when_none_registered(): assert isinstance(chosen, OpenTelemetryV2) -def test_publish_global_otel_v2_provider_sets_selected_logger_provider(): +def test_publish_global_otel_v2_provider_sets_selected_logger_provider(monkeypatch): """The startup publish must set the OTel global provider to the *selected* logger's provider (the preset logger that owns every exporter), so the FastAPI server span and the gen-ai spans share one provider and one trace. @@ -2051,8 +2051,10 @@ def test_publish_global_otel_v2_provider_sets_selected_logger_provider(): test would otherwise miss: that the published provider is the selected logger's, not some other. """ + from litellm.integrations.otel import logger as otel_logger from litellm.integrations.otel.logger import publish_global_otel_v2_provider + monkeypatch.setattr(otel_logger, "_published_v2_provider", None) cfg = OpenTelemetryV2Config(exporter="in_memory") tp = providers.build_tracer_provider(cfg) preset_logger = OpenTelemetryV2( From 8689e76eaf29c60f5753a2d2ac5bb8ca4708e59e Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Sat, 5 Sep 2026 02:51:20 -0700 Subject: [PATCH 22/47] fix(otel): preserve registered provider fallback --- litellm/integrations/otel/logger.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index 21e1feb4022..107532f6b55 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -935,7 +935,11 @@ def fan_out_provider() -> ApiTracerProvider: picked a v2 one from ``_in_memory_loggers``. Either detour lands on a provider with no fan-out and drops every destination at auth. """ - return _published_v2_provider if _published_v2_provider is not None else get_tracer_provider() + published: Final = _published_v2_provider + if published is not None: + return published + logger: Final = _registered_v2_logger() + return logger.tracer_provider if logger is not None else get_tracer_provider() @contextmanager From 63a5b1cbd8e464d45ee02f5b38de7aca8a207761 Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Sat, 5 Sep 2026 02:53:04 -0700 Subject: [PATCH 23/47] test(otel): cover pre-publish provider fallback --- .../integrations/otel/test_otel_v2_destinations.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index 706916c69fb..ec8834f5ec7 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -706,6 +706,15 @@ class TestProviderWiring: assert fan_out_provider() is v2.tracer_provider assert deliverable_destinations((LANGFUSE_DEST,), fan_out_provider()) == (LANGFUSE_DEST,) + def test_without_a_publish_anchoring_uses_a_registered_v2_logger(self, monkeypatch): + from litellm.proxy import proxy_server + + config = OpenTelemetryV2Config(exporters=[ExporterSpec(kind="in_memory", owner=ExporterOwner.LANGFUSE_OTEL)]) + logger = OpenTelemetryV2(config=config, callback_name="langfuse_otel") + monkeypatch.setattr(proxy_server, "open_telemetry_logger", logger) + + assert fan_out_provider() is logger.tracer_provider + def test_without_a_publish_anchoring_falls_back_to_the_otel_global(self, monkeypatch): from opentelemetry import trace From 836d31babc4102ef9d0c5e2acb2267fd52a8f823 Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Sat, 5 Sep 2026 03:03:03 -0700 Subject: [PATCH 24/47] fix(otel): attach fan-out on fallback provider --- litellm/integrations/otel/logger.py | 5 ++++- .../integrations/otel/test_otel_v2_destinations.py | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index 107532f6b55..2353472aefa 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -939,7 +939,10 @@ def fan_out_provider() -> ApiTracerProvider: if published is not None: return published logger: Final = _registered_v2_logger() - return logger.tracer_provider if logger is not None else get_tracer_provider() + if logger is not None: + attach_tenant_fan_out(logger.tracer_provider, logger.config) + return logger.tracer_provider + return get_tracer_provider() @contextmanager diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index ec8834f5ec7..fbda890da88 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -706,7 +706,7 @@ class TestProviderWiring: assert fan_out_provider() is v2.tracer_provider assert deliverable_destinations((LANGFUSE_DEST,), fan_out_provider()) == (LANGFUSE_DEST,) - def test_without_a_publish_anchoring_uses_a_registered_v2_logger(self, monkeypatch): + def test_without_a_publish_anchoring_attaches_fan_out_to_registered_v2_logger(self, monkeypatch): from litellm.proxy import proxy_server config = OpenTelemetryV2Config(exporters=[ExporterSpec(kind="in_memory", owner=ExporterOwner.LANGFUSE_OTEL)]) @@ -714,6 +714,7 @@ class TestProviderWiring: monkeypatch.setattr(proxy_server, "open_telemetry_logger", logger) assert fan_out_provider() is logger.tracer_provider + assert deliverable_destinations((LANGFUSE_DEST,), fan_out_provider()) == (LANGFUSE_DEST,) def test_without_a_publish_anchoring_falls_back_to_the_otel_global(self, monkeypatch): from opentelemetry import trace From 0cce313796d6f876e46928791e91f87549a2d42f Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Sat, 5 Sep 2026 03:18:52 -0700 Subject: [PATCH 25/47] fix(otel): serialize first fan-out attach --- litellm/__init__.py | 2 +- .../integrations/otel/plumbing/providers.py | 17 +++++++---- .../otel/test_otel_v2_destinations.py | 29 +++++++++++++++++++ 3 files changed, 41 insertions(+), 7 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index dfa72d2aa68..14327d54897 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -327,7 +327,7 @@ user_url_allowed_hosts: List[str] = [] provider_url_destination_allowed_hosts: List[str] = [] #: "override" (default) or "additive": whether a key or team destination replaces #: the operator's exporter for that backend or exports alongside it. -otel_tenant_destination_mode: Optional[str] = None +otel_tenant_destination_mode: str | None = None ssl_ecdh_curve: Optional[str] = None # Set to 'X25519' to disable PQC and improve performance disable_streaming_logging: bool = False disable_token_counter: bool = False diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index 350566d1985..ff1da6c5f4b 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -887,17 +887,22 @@ def build_tracer_provider( return provider +_FAN_OUT_ATTACH_LOCK: Final = threading.Lock() + + def attach_tenant_fan_out(provider: TracerProvider, config: OpenTelemetryV2Config | None = None) -> None: """Give ``provider`` the fan-out that delivers spans to key/team destinations. Called on the one provider published as the OTel global, and idempotent so a - second publish (a test, a re-initialized proxy) cannot double-export. ``config`` - names the operator's own exporters so an additive destination pointing at one of - them is delivered once rather than twice. + second publish (a test, a re-initialized proxy) cannot double-export. Concurrent + first calls (requests racing to anchor before any publish) serialize on one lock + so exactly one fan-out lands. ``config`` names the operator's own exporters so an + additive destination pointing at one of them is delivered once rather than twice. """ - if any(isinstance(processor, TenantFanOutSpanProcessor) for processor in _attached_processors(provider)): - return - provider.add_span_processor(TenantFanOutSpanProcessor(operator_sinks=operator_sink_keys(config))) + with _FAN_OUT_ATTACH_LOCK: + if any(isinstance(processor, TenantFanOutSpanProcessor) for processor in _attached_processors(provider)): + return + provider.add_span_processor(TenantFanOutSpanProcessor(operator_sinks=operator_sink_keys(config))) def deliverable_destinations( diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index fbda890da88..a603990f5e7 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -716,6 +716,35 @@ class TestProviderWiring: assert fan_out_provider() is logger.tracer_provider assert deliverable_destinations((LANGFUSE_DEST,), fan_out_provider()) == (LANGFUSE_DEST,) + def test_concurrent_anchoring_attaches_exactly_one_fan_out(self): + """Requests race to anchor when the startup publish never ran, and a fan-out + attached twice delivers every tenant span twice.""" + import threading + + from litellm.integrations.otel.plumbing.providers import attach_tenant_fan_out + + class SlowAttachProvider(TracerProvider): + def add_span_processor(self, span_processor): + time.sleep(0.05) + super().add_span_processor(span_processor) + + provider = SlowAttachProvider() + config = OpenTelemetryV2Config(exporters=[ExporterSpec(kind="in_memory", owner=ExporterOwner.LANGFUSE_OTEL)]) + barrier = threading.Barrier(8) + + def anchor(): + barrier.wait(timeout=10) + attach_tenant_fan_out(provider, config) + + threads = [threading.Thread(target=anchor) for _ in range(8)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=10) + + kinds = [type(p).__name__ for p in provider._active_span_processor._span_processors] + assert kinds.count("TenantFanOutSpanProcessor") == 1, f"one fan-out per provider, got {kinds}" + def test_without_a_publish_anchoring_falls_back_to_the_otel_global(self, monkeypatch): from opentelemetry import trace From 9bb728f354c8c84ac6b5a99d5dd5a1abdfe7d445 Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Sat, 5 Sep 2026 05:14:05 -0700 Subject: [PATCH 26/47] fix(otel): keep the operator's database endpoint out of tenant traces A database span forwarded to a key or team destination carried the proxy's own Postgres host, port and schema, and on failure the Prisma error text naming them. The fan-out now hands tenants a view of each database span without those keys, its events or its status text, while the operator's own copy is untouched and model endpoints such as server.address on the LLM span still travel --- .../integrations/otel/plumbing/providers.py | 74 +++++++++++++---- .../otel/test_otel_v2_destinations.py | 81 ++++++++++++++++--- 2 files changed, 129 insertions(+), 26 deletions(-) diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index ff1da6c5f4b..64a45151ab5 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -4,7 +4,7 @@ import queue import threading import time from collections import OrderedDict -from collections.abc import Callable, Iterable, Mapping +from collections.abc import Callable, Iterable, Mapping, Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal @@ -24,7 +24,7 @@ from opentelemetry.sdk._logs.export import ( ) from opentelemetry.sdk.metrics import MeterProvider as SDKMeterProvider from opentelemetry.sdk.resources import Resource -from opentelemetry.sdk.trace import ReadableSpan, SpanProcessor, TracerProvider +from opentelemetry.sdk.trace import Event, ReadableSpan, SpanProcessor, TracerProvider from opentelemetry.sdk.trace import Span as SDKSpan from opentelemetry.sdk.trace.export import ( BatchSpanProcessor, @@ -35,13 +35,14 @@ from opentelemetry.sdk.trace.export import ( from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( InMemorySpanExporter, ) -from opentelemetry.trace import Span, SpanKind, Tracer +from opentelemetry.trace import Span, SpanKind, Status, Tracer from opentelemetry.util.re import parse_env_headers +from opentelemetry.util.types import Attributes, AttributeValue from litellm._logging import verbose_logger from litellm._version import version as litellm_version from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config -from litellm.integrations.otel.model.semconv import LiteLLM +from litellm.integrations.otel.model.semconv import DB, Error, LiteLLM, LiteLLMError, Server from litellm.integrations.otel.model.spans import LiteLLMSpanKind from litellm.integrations.otel.plumbing.context import ( request_destinations, @@ -312,34 +313,75 @@ class _DrainPool: _shutdown_quietly(processor) -class _ResourceWrappedReadableSpan(ReadableSpan): - """A ``ReadableSpan`` view with an overridden Resource, leaving the original alone.""" +_NO_ATTRIBUTES: Final[Mapping[str, AttributeValue]] = MappingProxyType({}) +_DB_SYSTEM_KEYS: Final = frozenset({DB.SYSTEM_NAME, DB.SYSTEM_LEGACY}) +# Keys on a database span that describe the proxy's own datastore: its host, its +# port, its schema, and the Prisma error text that spells the first two out again. +_OPERATOR_INFRASTRUCTURE_KEYS: Final = frozenset( + {Server.ADDRESS, Server.PORT, DB.NAMESPACE, Error.MESSAGE, LiteLLMError.STACK_TRACE} +) - def __init__(self, inner: ReadableSpan, resource: Resource) -> None: + +class _TenantSpanView(ReadableSpan): + """A ``ReadableSpan`` view for one destination, leaving the operator's own span alone.""" + + def __init__( + self, + inner: ReadableSpan, + resource: Resource, + attributes: Attributes, + events: Sequence[Event], + status: Status, + ) -> None: super().__init__( name=inner.name, context=inner.context, parent=inner.parent, resource=resource, - attributes=inner.attributes, - events=inner.events, + attributes=attributes, + events=events, links=inner.links, kind=inner.kind, - status=inner.status, + status=status, start_time=inner.start_time, end_time=inner.end_time, instrumentation_scope=inner.instrumentation_scope, ) -def _with_destination_resource(span: ReadableSpan, destination: "OtelDestination") -> ReadableSpan: +def _is_database_span(span: ReadableSpan) -> bool: + attributes: Final = span.attributes or _NO_ATTRIBUTES + return any(key in attributes for key in _DB_SYSTEM_KEYS) + + +def _for_destination(span: ReadableSpan, destination: "OtelDestination") -> ReadableSpan: + """The view of ``span`` a tenant destination receives. + + A database span describes the operator's own Postgres rather than the tenant's + request, so its endpoint and its error text come off on the way out. The span + itself stays, so the tenant still gets the whole trace tree. + """ extra: Final = destination.resource_attributes - if not extra: + redacted: Final = _is_database_span(span) + if not extra and not redacted: return span - merged: Final = Resource.create( - {**dict(span.resource.attributes), **dict(extra)} # mutable-ok: the OTel SDK takes a concrete attribute mapping + resource: Final = ( + Resource.create( + {**dict(span.resource.attributes), **dict(extra)} # mutable-ok: the OTel SDK takes a concrete mapping + ) + if extra + else span.resource + ) + if not redacted: + return _TenantSpanView(span, resource, span.attributes, span.events, span.status) + attributes: Final = span.attributes or _NO_ATTRIBUTES + return _TenantSpanView( + span, + resource, + MappingProxyType({key: value for key, value in attributes.items() if key not in _OPERATOR_INFRASTRUCTURE_KEYS}), + (), + Status(span.status.status_code), ) - return _ResourceWrappedReadableSpan(span, merged) class TenantFanOutSpanProcessor(SpanProcessor): @@ -386,7 +428,7 @@ class TenantFanOutSpanProcessor(SpanProcessor): if processor is None: continue try: - processor.on_end(_with_destination_resource(span, destination)) + processor.on_end(_for_destination(span, destination)) except Exception as exc: # noqa: BLE001 # one destination's failure must not cost the others their span verbose_logger.debug("OTel V2 fan-out: forwarding to %s failed: %s", destination.endpoint, exc) finally: diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index a603990f5e7..513e0512903 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -9,8 +9,15 @@ import pytest from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import SimpleSpanProcessor from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import Status, StatusCode import litellm +from litellm.integrations.otel import logger as otel_logger +from litellm.integrations.otel.logger import ( + OpenTelemetryV2, + fan_out_provider, + publish_global_otel_v2_provider, +) from litellm.integrations.otel.model.config import ( ExporterOwner, ExporterSpec, @@ -18,12 +25,6 @@ from litellm.integrations.otel.model.config import ( is_otel_v2_enabled, ) from litellm.integrations.otel.model.destination import OtelDestination -from litellm.integrations.otel import logger as otel_logger -from litellm.integrations.otel.logger import ( - OpenTelemetryV2, - fan_out_provider, - publish_global_otel_v2_provider, -) from litellm.integrations.otel.plumbing.context import ( destination_backends, request_destinations, @@ -38,15 +39,15 @@ from litellm.integrations.otel.plumbing.providers import ( operator_sink_keys, ) from litellm.integrations.otel.plumbing.routing import TenantTracerCache, get_tracer +from litellm.integrations.otel.presets.arize import arize_preset from litellm.integrations.otel.presets.destinations import ( destination_capable_backends, destination_for, ) -from litellm.integrations.otel.presets.arize import arize_preset from litellm.integrations.otel.presets.langfuse import langfuse_preset from litellm.proxy._types import UserAPIKeyAuth -from litellm.types.utils import StandardCallbackDynamicParams from litellm.proxy.litellm_pre_call_utils import resolve_tenant_otel_destinations +from litellm.types.utils import StandardCallbackDynamicParams LANGFUSE_DEST = OtelDestination( endpoint="http://tenant.local/api/public/otel", @@ -504,6 +505,67 @@ class TestFanOut: assert {s.resource.attributes["service.name"] for s in dest.get_finished_spans()} == {"team-checkout"} + def test_the_operators_database_endpoint_does_not_ride_along_to_the_tenant(self): + """A database span describes the proxy's own Postgres, so the tenant gets the + span and its timing without the host, the port, the schema or the error text + that names them. The operator's own copy keeps everything.""" + dest_exporter, operator_exporter = InMemorySpanExporter(), InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(operator_exporter)) + provider.add_span_processor( + TenantFanOutSpanProcessor(processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter)) + ) + tracer = get_tracer(provider, "litellm") + unreachable = "Can't reach database server at db.internal.example:15400" + + def run(): + set_request_destinations((LANGFUSE_DEST,)) + with tracer.start_as_current_span("postgres get_data") as db_span: + db_span.set_attributes( + { + "db.system.name": "postgresql", + "db.system": "postgresql", + "db.operation.name": "get_data", + "server.address": "db.internal.example", + "server.port": 15400, + "db.namespace": "litellm", + "error.type": "PrismaError", + "error.message": unreachable, + "litellm.provider.error.stack_trace": f"Traceback: {unreachable}", + } + ) + db_span.add_event("exception", {"exception.message": unreachable}) + db_span.set_status(Status(StatusCode.ERROR, unreachable)) + with tracer.start_as_current_span("chat claude-haiku") as llm_span: + llm_span.set_attribute("server.address", "api.anthropic.com") + + in_fresh_context(run) + + tenant = {s.name: s for s in dest_exporter.get_finished_spans()} + operator = {s.name: s for s in operator_exporter.get_finished_spans()} + assert set(tenant) == {"postgres get_data", "chat claude-haiku"}, "the tenant keeps the whole tree" + tenant_db = tenant["postgres get_data"] + assert dict(tenant_db.attributes) == { + "db.system.name": "postgresql", + "db.system": "postgresql", + "db.operation.name": "get_data", + "error.type": "PrismaError", + } + assert list(tenant_db.events) == [] + assert tenant_db.status.status_code is StatusCode.ERROR, "the tenant still sees that the call failed" + assert tenant_db.status.description is None + assert "db.internal.example" not in tenant_db.to_json() + assert tenant["chat claude-haiku"].attributes["server.address"] == "api.anthropic.com", ( + "only the operator's datastore is redacted, never the model endpoint" + ) + operator_db = operator["postgres get_data"] + assert operator_db.attributes["server.address"] == "db.internal.example" + assert operator_db.attributes["server.port"] == 15400 + assert operator_db.attributes["db.namespace"] == "litellm" + assert operator_db.attributes["error.message"] == unreachable + assert operator_db.status.description == unreachable + assert [event.name for event in operator_db.events] == ["exception"] + def test_a_destination_that_cannot_build_a_processor_is_skipped_quietly(self): """An unbuildable destination must not cost the caller its request.""" attempts = [] @@ -1442,8 +1504,7 @@ class TestEvictionSafety: fan_out._release(fan_out._acquire(self._dest(index))) threads = [ - threading.Thread(target=shed, args=(_MAX_CACHED_DESTINATION_PROCESSORS + index,)) - for index in range(16) + threading.Thread(target=shed, args=(_MAX_CACHED_DESTINATION_PROCESSORS + index,)) for index in range(16) ] for thread in threads: thread.start() From 7b8e233975d15a955bab2e2d68cf5cd41e90698f Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Sat, 5 Sep 2026 05:55:21 -0700 Subject: [PATCH 27/47] fix(otel): keep relabelled spans in the fan-out and honour disabled callbacks for destinations A key or team otel_service_name used to move a backend's span onto a second provider even when another backend had a destination, so the fan-out never saw the model call and the tenant's trace lost it. A service name alone now stays on the published provider whenever the request has a destination; credential and project routing to a tenant's own account is unchanged Destinations now skip a backend the request disabled dynamically, reading the x-litellm-disable-callbacks header and the key's litellm_disabled_callbacks with the same precedence and premium gate dispatch applies, so a disabled backend is neither delivered to nor withheld from the operator --- litellm/integrations/otel/plumbing/routing.py | 9 +- litellm/proxy/auth/user_api_key_auth.py | 12 ++- litellm/proxy/litellm_pre_call_utils.py | 41 ++++++++ .../otel/test_otel_v2_destinations.py | 94 +++++++++++++++++++ 4 files changed, 151 insertions(+), 5 deletions(-) diff --git a/litellm/integrations/otel/plumbing/routing.py b/litellm/integrations/otel/plumbing/routing.py index 8fd24d8706c..f78d18d943c 100644 --- a/litellm/integrations/otel/plumbing/routing.py +++ b/litellm/integrations/otel/plumbing/routing.py @@ -241,7 +241,12 @@ class TenantTracerCache: credential_headers: Final = self._credential_headers(dynamic_params) project_headers: Final = self._project_headers(auth_metadata) service_name: Final = tenant_service_name(auth_metadata) - if not credential_headers and not project_headers and service_name is None: + tenant_account: Final = bool(credential_headers) or bool(project_headers) + # A service name on its own only relabels the operator's own backend, so moving + # the span to a second provider for it while some other backend has a + # destination would drop the model call out of the trace the fan-out delivers. + # The destination stamps the same service name itself. + if not tenant_account and (service_name is None or destination_backends()): return TenantRoute(tracer=default, detached=False) # A fixed per-integration region endpoint (New Relic us/eu), never a # caller-supplied host; ``None`` keeps the preset's own endpoint. @@ -262,7 +267,7 @@ class TenantTracerCache: _shutdown_provider(evicted) return TenantRoute( tracer=get_tracer(provider, self._tracer_name), - detached=bool(project_headers) or bool(credential_headers), + detached=tenant_account, provider=provider, ) diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index ba92d710cc8..0ec28b62a3a 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -2845,13 +2845,16 @@ async def _authorize_authenticated_request( @tracer.wrap() -def _seed_request_destinations(user_api_key_dict: UserAPIKeyAuth) -> None: +def _seed_request_destinations(user_api_key_dict: UserAPIKeyAuth, request: Request | None = None) -> None: """Anchor the OTLP destinations this key or team overrides its traces to. Called inside the ``auth`` phase span so that span reaches the tenant's account as well, and on the request task so the ``ContextVar`` is inherited by the logging tasks that close the LLM span. Best-effort: trace routing must never fail auth. + ``request`` carries the headers, so a backend this request disabled with + ``x-litellm-disable-callbacks`` resolves to no destination. + Only destinations the published fan-out can build are anchored. Anchoring one is what tells the operator's exporter to hold that backend's spans back under ``override``, so an unbuildable one would leave the span with nowhere to go. @@ -2870,7 +2873,10 @@ def _seed_request_destinations(user_api_key_dict: UserAPIKeyAuth) -> None: ) set_request_destinations( - deliverable_destinations(resolve_tenant_otel_destinations(user_api_key_dict), fan_out_provider()) + deliverable_destinations( + resolve_tenant_otel_destinations(user_api_key_dict, _safe_get_request_headers(request)), + fan_out_provider(), + ) ) except Exception as exc: # noqa: BLE001 # telemetry routing is best-effort and must never break authentication verbose_proxy_logger.debug("OTel V2: tenant destination resolution failed: %s", exc) @@ -2922,7 +2928,7 @@ async def user_api_key_auth( raise body_parse_exception raise user_api_key_auth_obj.budget_reservation = None - _seed_request_destinations(user_api_key_auth_obj) + _seed_request_destinations(user_api_key_auth_obj, request) # A body that never parsed is authenticated (so the trace carries identity # and this ``auth`` span) but not authorized: there is no model to check it diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 5c6af78aad7..bc76f630769 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -27,6 +27,7 @@ from litellm.constants import ( SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, SESSION_ID_GENERATED_METADATA_KEY, SESSION_ID_OMITTED_METADATA_KEY, + X_LITELLM_DISABLE_CALLBACKS, ) from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( @@ -987,8 +988,40 @@ def _tenant_otel_params(callback_vars: Mapping[str, str]) -> StandardCallbackDyn return StandardCallbackDynamicParams() +_NO_REQUEST_HEADERS: Final[Mapping[str, str]] = MappingProxyType({}) + + +def _dynamically_disabled_backends( + user_api_key_dict: UserAPIKeyAuth, + request_headers: Mapping[str, str] | None, +) -> frozenset[str]: + """The callbacks this request turned off, read the way dispatch reads them. + + Same sources, precedence, and premium gate ``EnterpriseCallbackControls`` applies + before it skips a callback: the ``x-litellm-disable-callbacks`` header wins over the + key's stored list, team settings are not a source, and a non-premium proxy honours + neither. A destination has to agree with that decision, or a backend the key turned + off would still be exported to, now through the fan-out instead of the callback. + """ + from litellm.proxy.proxy_server import premium_user + + if litellm.allow_dynamic_callback_disabling is not True or not premium_user: + return frozenset() + header: Final = (request_headers if request_headers is not None else _NO_REQUEST_HEADERS).get( + X_LITELLM_DISABLE_CALLBACKS + ) + if header is not None: + return frozenset(name.strip().lower() for name in header.split(",")) + metadata: Final = user_api_key_dict.metadata + disabled: Final = metadata.get("litellm_disabled_callbacks") if metadata else None + if not isinstance(disabled, list): + return frozenset() + return frozenset(name.lower() for name in disabled if isinstance(name, str)) + + def resolve_tenant_otel_destinations( user_api_key_dict: UserAPIKeyAuth, + request_headers: Mapping[str, str] | None = None, ) -> "tuple[OtelDestination, ...]": """The OTLP destinations this request's key or team config overrides its traces to. @@ -1008,6 +1041,12 @@ def resolve_tenant_otel_destinations( back until the call finishes. Those entries keep today's behaviour instead, where the tenant's credentials reach the backend through per-request tracer routing and the operator's exporter is left alone. + + A backend the request disabled dynamically, through the key's + ``litellm_disabled_callbacks`` or the ``x-litellm-disable-callbacks`` header in + ``request_headers``, resolves to no destination: dispatch skips that callback, so + the request keeps the operator's exporters for it exactly as it did before + destinations existed. """ from litellm.integrations.otel.model.config import is_otel_v2_enabled from litellm.integrations.otel.presets.destinations import destination_for @@ -1022,11 +1061,13 @@ def resolve_tenant_otel_destinations( ) if not entries: return () + disabled: Final = _dynamically_disabled_backends(user_api_key_dict, request_headers) callbacks: Final = tuple( callback for item in entries if (callback := _get_validated_callback_metadata(item=item, source="otel-destination")) is not None if callback.callback_type != "failure" + if callback.callback_name.lower() not in disabled ) return tuple( destination diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index 513e0512903..6b9ff19b418 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -895,6 +895,50 @@ class TestRouting: assert route.tracer is default, "the fan-out carries the service name on the destination instead" assert route.provider is None + @pytest.mark.parametrize("callback_name", ["arize", None]) + def test_a_service_name_does_not_detach_a_backend_the_destination_does_not_name(self, callback_name): + """The fan-out only sees spans on the published provider, so relabelling this + logger's span onto a second provider would drop the model call out of the + trace another backend's destination receives.""" + config = OpenTelemetryV2Config( + exporters=[ExporterSpec(kind="otlp_http", endpoint="http://op.local", owner=ExporterOwner.ARIZE_AX)] + ) + cache = TenantTracerCache(config, callback_name, "litellm") + default = get_tracer(TracerProvider(), "litellm") + auth_metadata = {"otel_service_name": "team-checkout"} + + relabelled = cache.route_for(default, None, auth_metadata) + assert relabelled.tracer is not default + cache.release(relabelled.provider) + + def run(): + set_request_destinations((LANGFUSE_DEST,)) + return cache.route_for(default, None, auth_metadata) + + route = in_fresh_context(run) + assert route.tracer is default + assert route.detached is False + assert route.provider is None + + def test_a_backend_with_its_own_credentials_still_routes_next_to_another_backend_destination(self): + """Credentials name the tenant's own account for this backend, which the other + backend's destination cannot stand in for.""" + config = OpenTelemetryV2Config( + exporters=[ExporterSpec(kind="otlp_http", endpoint="http://op.local", owner=ExporterOwner.ARIZE_AX)] + ) + cache = TenantTracerCache(config, "arize", "litellm") + default = get_tracer(TracerProvider(), "litellm") + params = {"arize_space_key": "space", "arize_api_key": "key"} + + def run(): + set_request_destinations((LANGFUSE_DEST,)) + return cache.route_for(default, params, {"otel_service_name": "team-checkout"}) + + route = in_fresh_context(run) + assert route.tracer is not default + assert route.detached is True + cache.release(route.provider) + @pytest.mark.usefixtures("allow_test_hosts") class TestDestinationResolution: @@ -1330,6 +1374,56 @@ class TestTenantConfigAgreement: assert [d.endpoint for d in destinations] == ["http://key.local/api/public/otel"] + @pytest.fixture + def premium(self, monkeypatch): + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "premium_user", True) + monkeypatch.setattr(litellm, "allow_dynamic_callback_disabling", True) + + @pytest.mark.usefixtures("premium") + def test_a_backend_the_key_disabled_resolves_to_no_destination(self): + """Dispatch skips a callback named in the key's ``litellm_disabled_callbacks``, + so the fan-out must not deliver to it either.""" + auth = UserAPIKeyAuth( + metadata={"litellm_disabled_callbacks": ["Langfuse_OTEL"]}, + team_metadata={"logging": [self._entry("http://team.local")]}, + ) + + assert resolve_tenant_otel_destinations(auth) == () + + @pytest.mark.usefixtures("premium") + @pytest.mark.parametrize( + ("header", "resolved"), + [ + ("langfuse_otel", False), + (" LANGFUSE_OTEL ,arize", False), + ("arize", True), + ], + ) + def test_the_disable_header_wins_over_the_key_list(self, header, resolved): + """Same precedence as dispatch: a header that names other backends re-enables + the one the key stored.""" + auth = UserAPIKeyAuth( + metadata={"litellm_disabled_callbacks": ["langfuse_otel"]}, + team_metadata={"logging": [self._entry("http://team.local")]}, + ) + + destinations = resolve_tenant_otel_destinations(auth, {"x-litellm-disable-callbacks": header}) + + assert bool(destinations) is resolved + + def test_a_non_premium_proxy_ignores_the_disabled_list_like_dispatch_does(self, monkeypatch): + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "premium_user", False) + auth = UserAPIKeyAuth( + metadata={"litellm_disabled_callbacks": ["langfuse_otel"]}, + team_metadata={"logging": [self._entry("http://team.local")]}, + ) + + assert resolve_tenant_otel_destinations(auth, {"x-litellm-disable-callbacks": "langfuse_otel"}) != () + class TestEvictionSafety: class Recording(SimpleSpanProcessor): From 8fc2ff1f27ed7398f3ab53418a191ed25e5b4780 Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Sat, 5 Sep 2026 05:57:27 -0700 Subject: [PATCH 28/47] test(otel): project routing survives a sibling backend destination --- .../otel/test_otel_v2_destinations.py | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index 6b9ff19b418..7e40f333e07 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -920,19 +920,27 @@ class TestRouting: assert route.detached is False assert route.provider is None - def test_a_backend_with_its_own_credentials_still_routes_next_to_another_backend_destination(self): - """Credentials name the tenant's own account for this backend, which the other - backend's destination cannot stand in for.""" + @pytest.mark.parametrize( + ("owner", "params", "auth_metadata"), + [ + (ExporterOwner.ARIZE_AX, {"arize_space_key": "space", "arize_api_key": "key"}, {}), + (ExporterOwner.ARIZE_PHOENIX, None, {"phoenix_project_name": "team-project"}), + ], + ) + def test_a_backend_pointed_at_its_own_account_still_routes_next_to_another_backend_destination( + self, owner, params, auth_metadata + ): + """Credentials or a project name the tenant's own account for this backend, which + the other backend's destination cannot stand in for.""" config = OpenTelemetryV2Config( - exporters=[ExporterSpec(kind="otlp_http", endpoint="http://op.local", owner=ExporterOwner.ARIZE_AX)] + exporters=[ExporterSpec(kind="otlp_http", endpoint="http://op.local", owner=owner)] ) - cache = TenantTracerCache(config, "arize", "litellm") + cache = TenantTracerCache(config, owner.value, "litellm") default = get_tracer(TracerProvider(), "litellm") - params = {"arize_space_key": "space", "arize_api_key": "key"} def run(): set_request_destinations((LANGFUSE_DEST,)) - return cache.route_for(default, params, {"otel_service_name": "team-checkout"}) + return cache.route_for(default, params, {"otel_service_name": "team-checkout", **auth_metadata}) route = in_fresh_context(run) assert route.tracer is not default From 9f871f6895d20344eeecf4cee0bbe4e77f575ba9 Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Sat, 5 Sep 2026 06:15:20 -0700 Subject: [PATCH 29/47] docs(otel): state why a disabled backend still routes its own span --- litellm/proxy/litellm_pre_call_utils.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index bc76f630769..0ced2294248 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -1044,9 +1044,11 @@ def resolve_tenant_otel_destinations( A backend the request disabled dynamically, through the key's ``litellm_disabled_callbacks`` or the ``x-litellm-disable-callbacks`` header in - ``request_headers``, resolves to no destination: dispatch skips that callback, so - the request keeps the operator's exporters for it exactly as it did before - destinations existed. + ``request_headers``, resolves to no destination, so the fan-out never carries the + request tree to that account and the operator's exporter is never suppressed for + it. That leaves the request exactly where it stood before destinations existed: + the OTel V2 logger itself is not on the disable list's class registry, so its own + span still routes to the tenant's credentials the way it did then. """ from litellm.integrations.otel.model.config import is_otel_v2_enabled from litellm.integrations.otel.presets.destinations import destination_for From 22217d03ea71c44e47a0d43577e71573effd951c Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Sat, 5 Sep 2026 06:26:33 -0700 Subject: [PATCH 30/47] fix(otel): keep a degraded backend's spans off a collector another v2 logger already serves --- litellm/litellm_core_utils/litellm_logging.py | 31 +++++++++++++++-- .../otel/test_otel_v2_destinations.py | 34 +++++++++++++++++++ 2 files changed, 62 insertions(+), 3 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 8e6e2108dfc..29a29d376dc 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -200,7 +200,7 @@ if TYPE_CHECKING: from mcp.types import EmbeddedResource, ImageContent, TextContent from litellm.integrations.otel.logger import OpenTelemetryV2 - from litellm.integrations.otel.model.config import OpenTelemetryV2Config + from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig try: from litellm_enterprise.enterprise_callbacks.callback_controls import ( @@ -4809,6 +4809,11 @@ def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[Custom as it always did and the caller falls through to the legacy path, so the proxy never publishes a provider that exports nowhere for a backend the operator configured and no tenant can use. + + The degraded preset keeps the operator's generic OTLP collector, so a proxy whose + only v2 backend is that preset still reaches it. Beside another v2 logger the + collector is already that logger's, and a second copy of every model span from + this one would land there too, so only the credential-gated exporter is kept. """ from litellm.integrations.otel.model.config import is_otel_v2_enabled @@ -4830,11 +4835,16 @@ def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[Custom ): return callback try: - config: Final = preset_fn(allow_missing_credentials=serves_a_destination) + built: Final = preset_fn(allow_missing_credentials=serves_a_destination) except Exception: # If env vars are missing or the preset raises, defer to the legacy path # so customers get the same error story they had before V2 landed. return None + config: Final = ( + _only_the_gated_exporter(built) + if _is_credential_gated(built) and any(isinstance(callback, OpenTelemetryV2) for callback in _in_memory_loggers) + else built + ) if _exports_nowhere(config): verbose_logger.warning( "OTel V2: no operator credentials for '%s'; only key/team destinations will receive its traces", @@ -4847,7 +4857,22 @@ def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[Custom def _exports_nowhere(config: "OpenTelemetryV2Config") -> bool: """Whether every exporter in ``config`` is waiting on credentials it never got.""" - return all(spec.requires_headers and not spec.headers for spec in config.exporters) + return all(_is_gated(spec) for spec in config.exporters) + + +def _is_credential_gated(config: "OpenTelemetryV2Config") -> bool: + """Whether the preset built without the operator's own credentials for its backend.""" + return any(_is_gated(spec) for spec in config.exporters) + + +def _only_the_gated_exporter(config: "OpenTelemetryV2Config") -> "OpenTelemetryV2Config": + return config.model_copy( + update={"exporters": [spec for spec in config.exporters if _is_gated(spec)]} # mutable-ok: model_copy update + ) + + +def _is_gated(spec: "ExporterSpec") -> bool: + return spec.requires_headers and not spec.headers def _maybe_auto_initialize_arize_phoenix(_in_memory_loggers: list[CustomLogger]) -> None: diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index 7e40f333e07..4e793b480fb 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -15,6 +15,7 @@ import litellm from litellm.integrations.otel import logger as otel_logger from litellm.integrations.otel.logger import ( OpenTelemetryV2, + build_otel_v2_logger, fan_out_provider, publish_global_otel_v2_provider, ) @@ -1208,6 +1209,39 @@ class TestPresetDegradation: assert first is not None assert second is first + @staticmethod + def _degraded_langfuse_beside(loggers, monkeypatch): + from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 + + credential_less_proxy(monkeypatch) + monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.local:4318") + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + + def run(): + set_request_destinations((LANGFUSE_DEST,)) + return _maybe_construct_otel_v2("langfuse_otel", loggers) + + is_otel_v2_enabled.cache_clear() + logger = in_fresh_context(run) + is_otel_v2_enabled.cache_clear() + assert logger is not None + return logger + + def test_a_degraded_logger_beside_another_v2_logger_leaves_the_collector_to_it(self, monkeypatch): + """The other logger's provider already exports every span to the operator's + collector, so a second model span from this one would land there twice.""" + collector_logger = build_otel_v2_logger(OpenTelemetryV2Config(exporter="in_memory")) + + logger = self._degraded_langfuse_beside([collector_logger], monkeypatch) + + assert [spec.endpoint for spec in logger.config.exporters] == [None] + assert all(spec.requires_headers and not spec.headers for spec in logger.config.exporters) + + def test_a_degraded_logger_on_its_own_keeps_the_operator_collector(self, monkeypatch): + logger = self._degraded_langfuse_beside([], monkeypatch) + + assert [spec.endpoint for spec in logger.config.exporters] == ["http://collector.local:4318", None] + class TestContextIsolation: def test_destinations_do_not_leak_between_requests(self): From c46341a1fd20cf305540a7e6100ec2010551301c Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Sat, 5 Sep 2026 06:30:11 -0700 Subject: [PATCH 31/47] test(otel): a credentialed preset beside another v2 logger keeps every exporter --- .../otel/test_otel_v2_destinations.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index 4e793b480fb..28360d4f30a 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -1242,6 +1242,28 @@ class TestPresetDegradation: assert [spec.endpoint for spec in logger.config.exporters] == ["http://collector.local:4318", None] + def test_a_credentialed_logger_beside_another_v2_logger_keeps_every_exporter(self, monkeypatch): + """Only a degraded preset gives the collector up; an operator who configured + both the backend and the collector still exports to both, as on base.""" + from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 + + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-lf-1") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-lf-1") + monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.local:4318") + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + collector_logger = build_otel_v2_logger(OpenTelemetryV2Config(exporter="in_memory")) + + is_otel_v2_enabled.cache_clear() + logger = in_fresh_context(_maybe_construct_otel_v2, "langfuse_otel", [collector_logger]) + is_otel_v2_enabled.cache_clear() + + assert logger is not None + assert [spec.endpoint for spec in logger.config.exporters] == [ + "http://collector.local:4318", + "https://cloud.langfuse.com/api/public/otel", + ] + assert all(spec.headers for spec in logger.config.exporters if spec.requires_headers) + class TestContextIsolation: def test_destinations_do_not_leak_between_requests(self): From c12da6dc905906128a32cf33e1f2d313780c7db6 Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Sat, 5 Sep 2026 06:56:39 -0700 Subject: [PATCH 32/47] fix(otel): keep credentialless fallback on base path --- litellm/litellm_core_utils/litellm_logging.py | 25 ++++++----------- .../otel/test_otel_v2_destinations.py | 27 ++++++++++++++----- 2 files changed, 28 insertions(+), 24 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 29a29d376dc..d6f80e89c66 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -4803,17 +4803,11 @@ def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[Custom ``callback_name`` — callers should then fall through to the legacy path. A preset that needs operator credentials it cannot find is allowed to build - anyway, exporting nowhere, only while this request has a key/team destination - for that backend: the exporter-less logger exists to let the fan-out carry those - spans without a second detached copy. With no such destination the preset raises - as it always did and the caller falls through to the legacy path, so the proxy - never publishes a provider that exports nowhere for a backend the operator - configured and no tenant can use. - - The degraded preset keeps the operator's generic OTLP collector, so a proxy whose - only v2 backend is that preset still reaches it. Beside another v2 logger the - collector is already that logger's, and a second copy of every model span from - this one would land there too, so only the credential-gated exporter is kept. + only when this request has a key/team destination for that backend and another + V2 logger is already registered to carry the fan-out. The resulting logger keeps + only its credential-gated exporter, while the registered logger owns operator + delivery. Without that carrier, the preset raises as it always did and the + caller falls through to the legacy path. """ from litellm.integrations.otel.model.config import is_otel_v2_enabled @@ -4827,6 +4821,7 @@ def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[Custom if preset_fn is None: return None serves_a_destination: Final = callback_name in destination_backends() + has_v2_logger: Final = any(isinstance(callback, OpenTelemetryV2) for callback in _in_memory_loggers) for callback in _in_memory_loggers: if ( isinstance(callback, OpenTelemetryV2) @@ -4835,16 +4830,12 @@ def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[Custom ): return callback try: - built: Final = preset_fn(allow_missing_credentials=serves_a_destination) + built: Final = preset_fn(allow_missing_credentials=serves_a_destination and has_v2_logger) except Exception: # If env vars are missing or the preset raises, defer to the legacy path # so customers get the same error story they had before V2 landed. return None - config: Final = ( - _only_the_gated_exporter(built) - if _is_credential_gated(built) and any(isinstance(callback, OpenTelemetryV2) for callback in _in_memory_loggers) - else built - ) + config: Final = _only_the_gated_exporter(built) if _is_credential_gated(built) else built if _exports_nowhere(config): verbose_logger.warning( "OTel V2: no operator credentials for '%s'; only key/team destinations will receive its traces", diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index 28360d4f30a..bea83ff2407 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -1123,21 +1123,22 @@ class TestPresetDegradation: with pytest.raises(ValueError, match="LANGFUSE_PUBLIC_KEY"): langfuse_preset() - def test_a_credential_less_proxy_builds_the_v2_logger_for_a_team_destination(self, monkeypatch): + def test_a_credential_less_proxy_builds_the_gated_logger_beside_a_v2_carrier(self, monkeypatch): from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 credential_less_proxy(monkeypatch) monkeypatch.setenv("LITELLM_OTEL_V2", "true") + carrier = build_otel_v2_logger(OpenTelemetryV2Config(exporter="in_memory")) def run(): set_request_destinations((LANGFUSE_DEST,)) - return _maybe_construct_otel_v2("langfuse_otel", []) + return _maybe_construct_otel_v2("langfuse_otel", [carrier]) is_otel_v2_enabled.cache_clear() logger = in_fresh_context(run) is_otel_v2_enabled.cache_clear() - assert logger is not None, "team-only deployments must not fall back to the legacy integration" + assert logger is not None assert all(spec.requires_headers and not spec.headers for spec in logger.config.exporters) def test_a_credential_less_proxy_with_no_destinations_falls_back_to_the_legacy_path(self, monkeypatch): @@ -1179,7 +1180,7 @@ class TestPresetDegradation: credential_less_proxy(monkeypatch) monkeypatch.setenv("LITELLM_OTEL_V2", "true") - loggers = [] + loggers = [build_otel_v2_logger(OpenTelemetryV2Config(exporter="in_memory"))] def with_destination(): set_request_destinations((LANGFUSE_DEST,)) @@ -1237,10 +1238,21 @@ class TestPresetDegradation: assert [spec.endpoint for spec in logger.config.exporters] == [None] assert all(spec.requires_headers and not spec.headers for spec in logger.config.exporters) - def test_a_degraded_logger_on_its_own_keeps_the_operator_collector(self, monkeypatch): - logger = self._degraded_langfuse_beside([], monkeypatch) + def test_a_credential_less_proxy_with_a_destination_but_no_v2_carrier_falls_back(self, monkeypatch): + from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 - assert [spec.endpoint for spec in logger.config.exporters] == ["http://collector.local:4318", None] + credential_less_proxy(monkeypatch) + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + + def run(): + set_request_destinations((LANGFUSE_DEST,)) + return _maybe_construct_otel_v2("langfuse_otel", []) + + is_otel_v2_enabled.cache_clear() + logger = in_fresh_context(run) + is_otel_v2_enabled.cache_clear() + + assert logger is None def test_a_credentialed_logger_beside_another_v2_logger_keeps_every_exporter(self, monkeypatch): """Only a degraded preset gives the collector up; an operator who configured @@ -1249,6 +1261,7 @@ class TestPresetDegradation: monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-lf-1") monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-lf-1") + monkeypatch.setenv("LANGFUSE_HOST", "https://cloud.langfuse.com") monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.local:4318") monkeypatch.setenv("LITELLM_OTEL_V2", "true") collector_logger = build_otel_v2_logger(OpenTelemetryV2Config(exporter="in_memory")) From e5a985040903b09a5ee18100d7986a1fc63586d4 Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Sat, 5 Sep 2026 06:57:50 -0700 Subject: [PATCH 33/47] test(otel): cover legacy callback carrier rejection --- .../integrations/otel/test_otel_v2_destinations.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index bea83ff2407..0a608ee1099 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -12,6 +12,7 @@ from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanE from opentelemetry.trace import Status, StatusCode import litellm +from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.otel import logger as otel_logger from litellm.integrations.otel.logger import ( OpenTelemetryV2, @@ -1238,7 +1239,11 @@ class TestPresetDegradation: assert [spec.endpoint for spec in logger.config.exporters] == [None] assert all(spec.requires_headers and not spec.headers for spec in logger.config.exporters) - def test_a_credential_less_proxy_with_a_destination_but_no_v2_carrier_falls_back(self, monkeypatch): + @pytest.mark.parametrize("registered", [(), (CustomLogger(),)]) + def test_a_credential_less_proxy_with_a_destination_but_no_v2_carrier_falls_back(self, monkeypatch, registered): + """Only a V2 logger publishes the provider the fan-out rides on, so a legacy + callback beside this one leaves the destination just as unreachable as no + callback at all, and the operator keeps the pre-V2 story.""" from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 credential_less_proxy(monkeypatch) @@ -1246,7 +1251,7 @@ class TestPresetDegradation: def run(): set_request_destinations((LANGFUSE_DEST,)) - return _maybe_construct_otel_v2("langfuse_otel", []) + return _maybe_construct_otel_v2("langfuse_otel", list(registered)) is_otel_v2_enabled.cache_clear() logger = in_fresh_context(run) From 9e3d9dc4665d440f0a6d31a1625f5303284f2d99 Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Sat, 5 Sep 2026 07:40:38 -0700 Subject: [PATCH 34/47] fix(otel): preserve valid exporter beside gated preset --- litellm/litellm_core_utils/litellm_logging.py | 5 ++++- .../otel/test_otel_v2_destinations.py | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index d6f80e89c66..d9a78156f4b 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -4835,7 +4835,10 @@ def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[Custom # If env vars are missing or the preset raises, defer to the legacy path # so customers get the same error story they had before V2 landed. return None - config: Final = _only_the_gated_exporter(built) if _is_credential_gated(built) else built + gated: Final = _is_credential_gated(built) + if _exports_nowhere(built) and not (serves_a_destination and has_v2_logger): + return None + config: Final = _only_the_gated_exporter(built) if gated and serves_a_destination and has_v2_logger else built if _exports_nowhere(config): verbose_logger.warning( "OTel V2: no operator credentials for '%s'; only key/team destinations will receive its traces", diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index 0a608ee1099..30e067df9f9 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -1157,6 +1157,23 @@ class TestPresetDegradation: assert logger is None + def test_a_valid_newrelic_base_exporter_survives_without_a_license_key(self, monkeypatch): + from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 + + monkeypatch.delenv("NEW_RELIC_LICENSE_KEY", raising=False) + monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.local:4318") + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + + is_otel_v2_enabled.cache_clear() + logger = in_fresh_context(_maybe_construct_otel_v2, "newrelic", []) + is_otel_v2_enabled.cache_clear() + + assert logger is not None + assert [spec.endpoint for spec in logger.config.exporters] == [ + "http://collector.local:4318", + "https://otlp.nr-data.net", + ] + def test_a_destination_for_one_backend_does_not_degrade_another(self, monkeypatch): from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 From 88526c39c6b42ef9c96a9f37005203bb00715555 Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Sat, 5 Sep 2026 08:01:09 -0700 Subject: [PATCH 35/47] fix(otel): avoid console export without operator destination --- litellm/litellm_core_utils/litellm_logging.py | 5 +++++ .../otel/test_otel_v2_destinations.py | 15 +++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index d9a78156f4b..c155092fdbd 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -4836,8 +4836,13 @@ def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[Custom # so customers get the same error story they had before V2 landed. return None gated: Final = _is_credential_gated(built) + has_operator_exporter: Final = any( + not _is_gated(spec) and bool(spec.model_dump(exclude_defaults=True)) for spec in built.exporters + ) if _exports_nowhere(built) and not (serves_a_destination and has_v2_logger): return None + if gated and not has_operator_exporter and not (serves_a_destination and has_v2_logger): + return None config: Final = _only_the_gated_exporter(built) if gated and serves_a_destination and has_v2_logger else built if _exports_nowhere(config): verbose_logger.warning( diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index 30e067df9f9..d52183638b8 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -1174,6 +1174,21 @@ class TestPresetDegradation: "https://otlp.nr-data.net", ] + def test_a_credentialless_newrelic_without_a_base_exporter_falls_back(self, monkeypatch, capsys): + from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 + + monkeypatch.delenv("NEW_RELIC_LICENSE_KEY", raising=False) + for name in _OTEL_SHORTHAND_ENV: + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + + is_otel_v2_enabled.cache_clear() + logger = in_fresh_context(_maybe_construct_otel_v2, "newrelic", []) + is_otel_v2_enabled.cache_clear() + + assert logger is None + assert capsys.readouterr().out == "" + def test_a_destination_for_one_backend_does_not_degrade_another(self, monkeypatch): from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 From 4372884404fbbb465f35612a4b89de993e2315b0 Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Sat, 5 Sep 2026 08:18:46 -0700 Subject: [PATCH 36/47] refactor(otel): share the console placeholder check with the presets --- litellm/integrations/otel/presets/utils.py | 4 ++-- litellm/litellm_core_utils/litellm_logging.py | 24 +++++++++++-------- .../otel/test_otel_v2_destinations.py | 3 +-- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/litellm/integrations/otel/presets/utils.py b/litellm/integrations/otel/presets/utils.py index 2d7598b63c3..19bcdd71aff 100644 --- a/litellm/integrations/otel/presets/utils.py +++ b/litellm/integrations/otel/presets/utils.py @@ -32,12 +32,12 @@ def credential_gated_exporters( override filter still recognises which backend this provider speaks for. """ return ( - *(spec for spec in exporters if not _is_unconfigured_placeholder(spec)), + *(spec for spec in exporters if not is_unconfigured_placeholder(spec)), ExporterSpec(owner=owner, requires_headers=True), ) -def _is_unconfigured_placeholder(spec: "ExporterSpec") -> bool: +def is_unconfigured_placeholder(spec: "ExporterSpec") -> bool: """Whether ``spec`` is the one ``_normalize`` folds in when nothing was configured. Every field at its default is what says the operator asked for nothing: an exporter diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index c155092fdbd..2b669abcfd7 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -4806,8 +4806,9 @@ def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[Custom only when this request has a key/team destination for that backend and another V2 logger is already registered to carry the fan-out. The resulting logger keeps only its credential-gated exporter, while the registered logger owns operator - delivery. Without that carrier, the preset raises as it always did and the - caller falls through to the legacy path. + delivery. Without that carrier, a preset that raises or that ends up with nothing + but its gated exporter and the default console placeholder returns ``None``, so the + caller falls through to the legacy path exactly as before V2 landed. """ from litellm.integrations.otel.model.config import is_otel_v2_enabled @@ -4822,6 +4823,7 @@ def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[Custom return None serves_a_destination: Final = callback_name in destination_backends() has_v2_logger: Final = any(isinstance(callback, OpenTelemetryV2) for callback in _in_memory_loggers) + carried: Final = serves_a_destination and has_v2_logger for callback in _in_memory_loggers: if ( isinstance(callback, OpenTelemetryV2) @@ -4830,20 +4832,15 @@ def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[Custom ): return callback try: - built: Final = preset_fn(allow_missing_credentials=serves_a_destination and has_v2_logger) + built: Final = preset_fn(allow_missing_credentials=carried) except Exception: # If env vars are missing or the preset raises, defer to the legacy path # so customers get the same error story they had before V2 landed. return None gated: Final = _is_credential_gated(built) - has_operator_exporter: Final = any( - not _is_gated(spec) and bool(spec.model_dump(exclude_defaults=True)) for spec in built.exporters - ) - if _exports_nowhere(built) and not (serves_a_destination and has_v2_logger): + if gated and not carried and not _has_operator_exporter(built): return None - if gated and not has_operator_exporter and not (serves_a_destination and has_v2_logger): - return None - config: Final = _only_the_gated_exporter(built) if gated and serves_a_destination and has_v2_logger else built + config: Final = _only_the_gated_exporter(built) if gated and carried else built if _exports_nowhere(config): verbose_logger.warning( "OTel V2: no operator credentials for '%s'; only key/team destinations will receive its traces", @@ -4864,6 +4861,13 @@ def _is_credential_gated(config: "OpenTelemetryV2Config") -> bool: return any(_is_gated(spec) for spec in config.exporters) +def _has_operator_exporter(config: "OpenTelemetryV2Config") -> bool: + """Whether the operator configured somewhere real to export, beyond the default console placeholder.""" + from litellm.integrations.otel.presets.utils import is_unconfigured_placeholder + + return any(not _is_gated(spec) and not is_unconfigured_placeholder(spec) for spec in config.exporters) + + def _only_the_gated_exporter(config: "OpenTelemetryV2Config") -> "OpenTelemetryV2Config": return config.model_copy( update={"exporters": [spec for spec in config.exporters if _is_gated(spec)]} # mutable-ok: model_copy update diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index d52183638b8..f051c6dfcb0 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -1174,7 +1174,7 @@ class TestPresetDegradation: "https://otlp.nr-data.net", ] - def test_a_credentialless_newrelic_without_a_base_exporter_falls_back(self, monkeypatch, capsys): + def test_a_credentialless_newrelic_without_a_base_exporter_falls_back(self, monkeypatch): from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 monkeypatch.delenv("NEW_RELIC_LICENSE_KEY", raising=False) @@ -1187,7 +1187,6 @@ class TestPresetDegradation: is_otel_v2_enabled.cache_clear() assert logger is None - assert capsys.readouterr().out == "" def test_a_destination_for_one_backend_does_not_degrade_another(self, monkeypatch): from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 From 093a9912c81738f355eed373444b8a90f16142ba Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Sat, 5 Sep 2026 08:31:57 -0700 Subject: [PATCH 37/47] fix(otel): bound shed destination processors waiting on a dead collector --- .../integrations/otel/plumbing/providers.py | 37 ++++++++++++++++- .../otel/test_otel_v2_destinations.py | 40 ++++++++++++++++++- 2 files changed, 75 insertions(+), 2 deletions(-) diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index 64a45151ab5..b60bcde2b5a 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -216,6 +216,12 @@ _MAX_CACHED_DESTINATION_PROCESSORS: Final = 32 #: create by cycling its destination config. _DRAIN_WORKERS: Final = 2 +#: Shed processors waiting to be closed before the fan-out stops building new ones. +#: Each still owns a batch thread until its close returns, and a collector that never +#: answers makes every close take the exporter's full timeout, so past this many the +#: operator's exporter keeps the span instead (see ``deliverable``). +_MAX_PENDING_DRAINS: Final = 64 + #: How long ``shutdown`` waits for spans already being forwarded, so teardown closes #: no processor under one. Bounded: an exporter that never returns must not hold the #: proxy open. @@ -247,10 +253,13 @@ class _DrainPool: self, workers: int = _DRAIN_WORKERS, pending: "queue.Queue[SpanProcessor | None] | None" = None, + capacity: int = _MAX_PENDING_DRAINS, ) -> None: self._workers: Final = workers + self._capacity: Final = capacity self._lock: Final = threading.Lock() self._closed = False + self._backlog = 0 # guarded by ``_lock``: submitted processors whose close has not returned self._pending: Final[queue.Queue[SpanProcessor | None]] = pending if pending is not None else queue.Queue() self._threads: Final = tuple( threading.Thread(target=self._drain_until_closed, daemon=True, name="litellm-otel-destination-drain") @@ -274,6 +283,7 @@ class _DrainPool: """ with self._lock: if not self._closed: + self._backlog += 1 self._pending.put(processor) return threading.Thread( @@ -283,6 +293,18 @@ class _DrainPool: name="litellm-otel-destination-drain-straggler", ).start() + def saturated(self) -> bool: + """Whether enough closes are outstanding that building another processor must wait. + + The workers close in order and each close blocks for as long as its exporter + does, so a collector that stopped answering would otherwise turn every new + destination into one more batch thread parked behind them, for as long as the + tenants keep rotating. Holding the count here rather than reading the queue + keeps the two processors a worker is mid-close on in the total. + """ + with self._lock: + return self._backlog >= self._capacity + def close(self, timeout: float | None = None) -> None: """Retire the workers once they have closed everything already queued. @@ -311,6 +333,8 @@ class _DrainPool: if processor is None: return _shutdown_quietly(processor) + with self._lock: + self._backlog -= 1 _NO_ATTRIBUTES: Final[Mapping[str, AttributeValue]] = MappingProxyType({}) @@ -405,6 +429,7 @@ class TenantFanOutSpanProcessor(SpanProcessor): processor_factory: 'Callable[["OtelDestination"], SpanProcessor | None] | None' = None, shutdown_drain_seconds: float = _SHUTDOWN_DRAIN_SECONDS, operator_sinks: frozenset[_SinkKey] = frozenset(), + pending_drains: int = _MAX_PENDING_DRAINS, ) -> None: self._operator_sinks: Final = operator_sinks self._drain_seconds: Final = shutdown_drain_seconds @@ -414,7 +439,7 @@ class TenantFanOutSpanProcessor(SpanProcessor): self._processors: OrderedDict[object, SpanProcessor] = OrderedDict() # mutable-ok: bounded LRU self._retired: OrderedDict[int, SpanProcessor] = OrderedDict() # mutable-ok: drains as exports finish self._exporting: dict[int, int] = {} # mutable-ok: per-processor in-flight export count - self._drain: Final = _DrainPool() + self._drain: Final = _DrainPool(capacity=pending_drains) def on_start(self, span: SDKSpan, parent_context: Context | None = None) -> None: return None @@ -544,6 +569,16 @@ class TenantFanOutSpanProcessor(SpanProcessor): return self._build_locked(destination, key) def _build_locked(self, destination: "OtelDestination", key: object) -> SpanProcessor | None: + """Build and cache a processor for ``destination``, unless the drain is saturated. + + Every build past the cache cap sheds one processor into the drain, so while the + shed ones are stuck closing against a collector that stopped answering, a new + destination is refused rather than parked behind them: ``deliverable`` then + leaves its spans with the operator's exporter until the drain catches up. + """ + if self._drain.saturated(): + verbose_logger.debug("OTel V2 fan-out: drain saturated, not building for %s", destination.endpoint) + return None built: Final = self._build(destination) if built is None: return None diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index f051c6dfcb0..1787dd68ad8 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -1676,7 +1676,45 @@ class TestEvictionSafety: release.set() self._settle(fan_out, built[0]) - def test_the_drain_workers_do_not_hold_the_process_open(self): + def test_a_saturated_drain_leaves_new_destinations_with_the_operator(self): + """A shed processor keeps its batch thread until its close returns, and against + a collector that never answers every close waits out the exporter's timeout. + Tenants rotating past the cache cap would otherwise queue one more processor, + and one more thread, per request for as long as the outage lasts.""" + import threading + + from litellm.integrations.otel.plumbing.providers import _MAX_CACHED_DESTINATION_PROCESSORS + + release = threading.Event() + + class Blocking(self.Recording): + def shutdown(self): + release.wait(timeout=10) + super().shutdown() + + built = [] + + def factory(_destination): + built.append(Blocking()) + return built[-1] + + fan_out = TenantFanOutSpanProcessor(processor_factory=factory, pending_drains=3) + try: + for index in range(_MAX_CACHED_DESTINATION_PROCESSORS + 40): + processor = fan_out._acquire(self._dest(index)) + if processor is not None: + fan_out._release(processor) + + assert len(built) == _MAX_CACHED_DESTINATION_PROCESSORS + 3, "a processor per request during the outage" + assert fan_out.deliverable((self._dest(999),)) == (), "the span would vanish instead of staying with the operator" + finally: + release.set() + for _ in range(500): + if not fan_out._drain.saturated(): + break + time.sleep(0.02) + + assert fan_out.deliverable((self._dest(999),)) == (self._dest(999),), "the fan-out never recovered" """Python joins a ThreadPoolExecutor's workers at interpreter exit, so one unreachable tenant collector would hold the proxy open for its export timeout on the way down.""" From 3b2635674c216ad3c568a1e8d6929991277e39c8 Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Sat, 5 Sep 2026 08:49:30 -0700 Subject: [PATCH 38/47] fix(otel): preserve explicit console exporters --- litellm/integrations/otel/model/config.py | 6 ++- litellm/integrations/otel/presets/utils.py | 10 ++-- .../otel/test_otel_v2_destinations.py | 46 +++++++++++++++++-- 3 files changed, 51 insertions(+), 11 deletions(-) diff --git a/litellm/integrations/otel/model/config.py b/litellm/integrations/otel/model/config.py index 9e3064c2bff..d8439045a12 100644 --- a/litellm/integrations/otel/model/config.py +++ b/litellm/integrations/otel/model/config.py @@ -253,7 +253,9 @@ class OpenTelemetryV2Config(BaseSettings): if self.endpoint and self.exporter == "console": self.exporter = "otlp_http" # When no explicit destinations are given, fold the single-destination - # shorthand into one spec so the provider always has a destination. + # shorthand into one spec so the provider always has a destination. A spec + # with no fields set is how the presets tell "nothing configured" from an + # operator who asked for the console by name. if not self.exporters: self.exporters = [ ExporterSpec( @@ -261,6 +263,8 @@ class OpenTelemetryV2Config(BaseSettings): endpoint=self.endpoint, headers=self.headers, ) + if self.model_fields_set & {"exporter", "endpoint", "headers"} + else ExporterSpec() ] # Ensure ``genai`` is always present and first. names = list(self.mapper_names) diff --git a/litellm/integrations/otel/presets/utils.py b/litellm/integrations/otel/presets/utils.py index 19bcdd71aff..1270c41e77b 100644 --- a/litellm/integrations/otel/presets/utils.py +++ b/litellm/integrations/otel/presets/utils.py @@ -40,9 +40,9 @@ def credential_gated_exporters( def is_unconfigured_placeholder(spec: "ExporterSpec") -> bool: """Whether ``spec`` is the one ``_normalize`` folds in when nothing was configured. - Every field at its default is what says the operator asked for nothing: an exporter - they did configure survives, whatever its kind, and so does the gated spec this - module appends, which would otherwise eat itself when one preset layers onto - another. + No field set is what says the operator asked for nothing: an exporter they did + configure survives, even ``OTEL_EXPORTER=console`` whose value matches the default, + and so does the gated spec this module appends, which would otherwise eat itself + when one preset layers onto another. """ - return not spec.model_dump(exclude_defaults=True) + return not spec.model_fields_set diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index 1787dd68ad8..4cdc1d93e78 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -1103,18 +1103,18 @@ def credential_less_proxy(monkeypatch) -> None: class TestPresetDegradation: - def test_a_credential_less_langfuse_exports_nowhere_instead_of_to_the_console(self, monkeypatch, capsys): + def test_a_credential_less_langfuse_exports_nowhere_instead_of_to_the_console(self, monkeypatch, capfd): """``_normalize`` folds a console exporter in for an empty list, which would print every span on a proxy whose teams bring their own credentials.""" credential_less_proxy(monkeypatch) config = langfuse_preset(allow_missing_credentials=True) provider = build_tracer_provider(config, tenant_overrides=True) - capsys.readouterr() + capfd.readouterr() in_fresh_context(emit, provider) provider.force_flush() - assert capsys.readouterr().out == "" + assert '"name": "chat gpt-4"' not in capfd.readouterr().out assert "langfuse" in config.mapper_names def test_langfuse_still_raises_for_a_global_callback_with_no_credentials(self, monkeypatch): @@ -1188,6 +1188,26 @@ class TestPresetDegradation: assert logger is None + def test_an_explicit_console_exporter_keeps_a_credentialless_preset_on_v2(self, monkeypatch, capfd): + """``OTEL_EXPORTER=console`` reads exactly like the placeholder ``_normalize`` + folds in, but the operator asked for it, so a credential-less New Relic keeps + the V2 logger and its spans reach stdout instead of the legacy path.""" + from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 + + monkeypatch.delenv("NEW_RELIC_LICENSE_KEY", raising=False) + for name in _OTEL_SHORTHAND_ENV: + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("OTEL_EXPORTER", "console") + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + + is_otel_v2_enabled.cache_clear() + logger = in_fresh_context(_maybe_construct_otel_v2, "newrelic", []) + is_otel_v2_enabled.cache_clear() + + assert logger is not None + assert logger.config.exporters[0].kind == "console" + assert not logger.config.exporters[0].requires_headers + def test_a_destination_for_one_backend_does_not_degrade_another(self, monkeypatch): from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 @@ -2084,15 +2104,31 @@ class TestCredentialGatedExporters: assert kept[0] == operator_memory - def test_the_synthesized_stdout_placeholder_is_dropped(self): + def test_the_synthesized_stdout_placeholder_is_dropped(self, monkeypatch): from litellm.integrations.otel.presets.utils import credential_gated_exporters - placeholder = ExporterSpec(kind="console", endpoint=None, headers=None) + for name in _OTEL_SHORTHAND_ENV: + monkeypatch.delenv(name, raising=False) + placeholder = OpenTelemetryV2Config().exporters[0] kept = credential_gated_exporters((placeholder,), ExporterOwner.LANGFUSE_OTEL) assert [spec.owner for spec in kept] == [ExporterOwner.LANGFUSE_OTEL] + def test_a_console_exporter_the_operator_named_survives(self, monkeypatch): + """Same kind, endpoint and headers as the placeholder; only the fact that the + operator set ``OTEL_EXPORTER`` tells them apart.""" + from litellm.integrations.otel.presets.utils import credential_gated_exporters + + for name in _OTEL_SHORTHAND_ENV: + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("OTEL_EXPORTER", "console") + operator_console = OpenTelemetryV2Config().exporters[0] + + kept = credential_gated_exporters((operator_console,), ExporterOwner.LANGFUSE_OTEL) + + assert kept[0] is operator_console + class TestTenantHostSsrfGuard: """Anyone who can mint a key can write ``langfuse_host``, so the host it names has From 6a1563de121d1d8db4c12b28b3fc8ab36c6157a1 Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Sat, 5 Sep 2026 08:50:52 -0700 Subject: [PATCH 39/47] fix(otel): avoid mutable field-set construction --- litellm/integrations/otel/model/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/integrations/otel/model/config.py b/litellm/integrations/otel/model/config.py index d8439045a12..f0f5befe430 100644 --- a/litellm/integrations/otel/model/config.py +++ b/litellm/integrations/otel/model/config.py @@ -263,7 +263,7 @@ class OpenTelemetryV2Config(BaseSettings): endpoint=self.endpoint, headers=self.headers, ) - if self.model_fields_set & {"exporter", "endpoint", "headers"} + if not self.model_fields_set.isdisjoint(("exporter", "endpoint", "headers")) else ExporterSpec() ] # Ensure ``genai`` is always present and first. From 4306513f4152d8eab27077902f3583fb73b0e863 Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Sat, 5 Sep 2026 08:59:02 -0700 Subject: [PATCH 40/47] fix(otel): close drain saturation race --- .../integrations/otel/plumbing/providers.py | 19 +++---- .../otel/test_otel_v2_destinations.py | 52 +++++++++++++++++++ 2 files changed, 62 insertions(+), 9 deletions(-) diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index b60bcde2b5a..bb1abb493db 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -430,6 +430,7 @@ class TenantFanOutSpanProcessor(SpanProcessor): shutdown_drain_seconds: float = _SHUTDOWN_DRAIN_SECONDS, operator_sinks: frozenset[_SinkKey] = frozenset(), pending_drains: int = _MAX_PENDING_DRAINS, + drain_pool: _DrainPool | None = None, ) -> None: self._operator_sinks: Final = operator_sinks self._drain_seconds: Final = shutdown_drain_seconds @@ -439,7 +440,7 @@ class TenantFanOutSpanProcessor(SpanProcessor): self._processors: OrderedDict[object, SpanProcessor] = OrderedDict() # mutable-ok: bounded LRU self._retired: OrderedDict[int, SpanProcessor] = OrderedDict() # mutable-ok: drains as exports finish self._exporting: dict[int, int] = {} # mutable-ok: per-processor in-flight export count - self._drain: Final = _DrainPool(capacity=pending_drains) + self._drain: Final = drain_pool if drain_pool is not None else _DrainPool(capacity=pending_drains) def on_start(self, span: SDKSpan, parent_context: Context | None = None) -> None: return None @@ -537,9 +538,9 @@ class TenantFanOutSpanProcessor(SpanProcessor): return False built: Final = self._cached_or_built_locked(destination) drained: Final = self._drainable_locked() - for shed in drained: - self._drain.submit(shed) - return built is not None + for shed in drained: + self._drain.submit(shed) + return built is not None def _acquire(self, destination: "OtelDestination") -> SpanProcessor | None: """The processor for ``destination``, marked busy until ``_release``. @@ -557,9 +558,9 @@ class TenantFanOutSpanProcessor(SpanProcessor): return None self._exporting[id(processor)] = self._exporting.get(id(processor), 0) + 1 drained: Final = self._drainable_locked() - for shed in drained: - self._drain.submit(shed) - return processor + for shed in drained: + self._drain.submit(shed) + return processor def _cached_or_built_locked(self, destination: "OtelDestination") -> SpanProcessor | None: key: Final = destination.cache_key() @@ -596,8 +597,8 @@ class TenantFanOutSpanProcessor(SpanProcessor): if not self._exporting: self._lock.notify_all() drained: Final = self._drainable_locked() - for retired in drained: - self._drain.submit(retired) + for retired in drained: + self._drain.submit(retired) def _retire_overflow_locked(self) -> None: """Move the LRU processor out of the cache once it is past the cap.""" diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index 4cdc1d93e78..410c4ae4cc6 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -1735,6 +1735,58 @@ class TestEvictionSafety: time.sleep(0.02) assert fan_out.deliverable((self._dest(999),)) == (self._dest(999),), "the fan-out never recovered" + + def test_concurrent_eviction_cannot_build_between_retirement_and_drain_submission(self): + """A second request cannot build while the first eviction is being handed to + the drain, or concurrent churn can outrun the pending-drain limit.""" + import threading + + from litellm.integrations.otel.plumbing.providers import ( + _DrainPool, + _MAX_CACHED_DESTINATION_PROCESSORS, + ) + + class GatedDrain(_DrainPool): + def __init__(self): + super().__init__(workers=0) + self.started = threading.Event() + self.release = threading.Event() + + def saturated(self): + return False + + def submit(self, processor): + if not self.started.is_set(): + self.started.set() + self.release.wait(timeout=5) + + built = [] + + def factory(_destination): + built.append(self.Recording()) + return built[-1] + + drain = GatedDrain() + fan_out = TenantFanOutSpanProcessor(processor_factory=factory, drain_pool=drain) + for index in range(_MAX_CACHED_DESTINATION_PROCESSORS): + fan_out._release(fan_out._acquire(self._dest(index))) + + first = threading.Thread(target=lambda: fan_out._release(fan_out._acquire(self._dest(32)))) + first.start() + assert drain.started.wait(timeout=5) + second = threading.Thread(target=lambda: fan_out._release(fan_out._acquire(self._dest(33)))) + second.start() + time.sleep(0.1) + + assert len(built) == _MAX_CACHED_DESTINATION_PROCESSORS + 1 + + drain.release.set() + first.join(timeout=5) + second.join(timeout=5) + assert not first.is_alive() and not second.is_alive() + assert len(built) == _MAX_CACHED_DESTINATION_PROCESSORS + 2 + + def test_drain_workers_are_daemons(self): """Python joins a ThreadPoolExecutor's workers at interpreter exit, so one unreachable tenant collector would hold the proxy open for its export timeout on the way down.""" From 21353577d084a3d51dd8346139e9129e7ca87714 Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Sat, 5 Sep 2026 10:02:02 -0700 Subject: [PATCH 41/47] fix(otel): drop captured request headers from tenant spans --- .../integrations/otel/plumbing/providers.py | 38 ++++++++++-------- .../otel/test_otel_v2_destinations.py | 40 +++++++++++++++++++ 2 files changed, 62 insertions(+), 16 deletions(-) diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index bb1abb493db..df8e449bc42 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -344,6 +344,10 @@ _DB_SYSTEM_KEYS: Final = frozenset({DB.SYSTEM_NAME, DB.SYSTEM_LEGACY}) _OPERATOR_INFRASTRUCTURE_KEYS: Final = frozenset( {Server.ADDRESS, Server.PORT, DB.NAMESPACE, Error.MESSAGE, LiteLLMError.STACK_TRACE} ) +# Attribute prefixes the FastAPI instrumentor uses for headers the operator opted to +# capture (``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_*``). The request +# side carries the caller's bearer token verbatim. +_CAPTURED_HEADER_PREFIXES: Final = ("http.request.header.", "http.response.header.") class _TenantSpanView(ReadableSpan): @@ -373,21 +377,30 @@ class _TenantSpanView(ReadableSpan): ) -def _is_database_span(span: ReadableSpan) -> bool: - attributes: Final = span.attributes or _NO_ATTRIBUTES +def _is_database_span(attributes: Mapping[str, AttributeValue]) -> bool: return any(key in attributes for key in _DB_SYSTEM_KEYS) +def _tenant_visible(key: str, database: bool) -> bool: + if key.startswith(_CAPTURED_HEADER_PREFIXES): + return False + return not database or key not in _OPERATOR_INFRASTRUCTURE_KEYS + + def _for_destination(span: ReadableSpan, destination: "OtelDestination") -> ReadableSpan: """The view of ``span`` a tenant destination receives. A database span describes the operator's own Postgres rather than the tenant's - request, so its endpoint and its error text come off on the way out. The span - itself stays, so the tenant still gets the whole trace tree. + request, so its endpoint and its error text come off on the way out. Headers the + operator captures on the server span come off every span too, since the request + side holds the caller's bearer token. The span itself stays, so the tenant still + gets the whole trace tree. """ extra: Final = destination.resource_attributes - redacted: Final = _is_database_span(span) - if not extra and not redacted: + attributes: Final = span.attributes or _NO_ATTRIBUTES + database: Final = _is_database_span(attributes) + kept: Final = MappingProxyType({key: value for key, value in attributes.items() if _tenant_visible(key, database)}) + if not extra and not database and len(kept) == len(attributes): return span resource: Final = ( Resource.create( @@ -396,16 +409,9 @@ def _for_destination(span: ReadableSpan, destination: "OtelDestination") -> Read if extra else span.resource ) - if not redacted: - return _TenantSpanView(span, resource, span.attributes, span.events, span.status) - attributes: Final = span.attributes or _NO_ATTRIBUTES - return _TenantSpanView( - span, - resource, - MappingProxyType({key: value for key, value in attributes.items() if key not in _OPERATOR_INFRASTRUCTURE_KEYS}), - (), - Status(span.status.status_code), - ) + if not database: + return _TenantSpanView(span, resource, kept, span.events, span.status) + return _TenantSpanView(span, resource, kept, (), Status(span.status.status_code)) class TenantFanOutSpanProcessor(SpanProcessor): diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index 410c4ae4cc6..0f74cb4d9d0 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -568,6 +568,46 @@ class TestFanOut: assert operator_db.status.description == unreachable assert [event.name for event in operator_db.events] == ["exception"] + def test_captured_request_headers_do_not_ride_along_to_the_tenant(self): + """With ``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST`` set, the + server span carries the caller's bearer token. A team admin's collector must + not receive it, while the operator's own copy keeps it and the tenant keeps the + rest of the span, its events and its status.""" + dest_exporter, operator_exporter = InMemorySpanExporter(), InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(operator_exporter)) + provider.add_span_processor( + TenantFanOutSpanProcessor(processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter)) + ) + tracer = get_tracer(provider, "litellm") + bearer = "Bearer sk-another-members-virtual-key" + + def run(): + set_request_destinations((LANGFUSE_DEST,)) + with tracer.start_as_current_span("POST /v1/chat/completions") as server_span: + server_span.set_attributes( + { + "http.request.method": "POST", + "http.route": "/v1/chat/completions", + "http.request.header.authorization": (bearer,), + "http.request.header.x_litellm_api_key": (bearer,), + "http.response.header.set_cookie": ("session=abc",), + } + ) + server_span.add_event("request.received") + server_span.set_status(Status(StatusCode.ERROR, "rate limited")) + + in_fresh_context(run) + + tenant = dest_exporter.get_finished_spans()[0] + assert dict(tenant.attributes) == {"http.request.method": "POST", "http.route": "/v1/chat/completions"} + assert bearer not in tenant.to_json() + assert [event.name for event in tenant.events] == ["request.received"] + assert tenant.status.description == "rate limited", "only the header capture comes off a server span" + operator = operator_exporter.get_finished_spans()[0] + assert operator.attributes["http.request.header.authorization"] == (bearer,) + assert operator.attributes["http.response.header.set_cookie"] == ("session=abc",) + def test_a_destination_that_cannot_build_a_processor_is_skipped_quietly(self): """An unbuildable destination must not cost the caller its request.""" attempts = [] From 1970f3b1e73ea8d732bd314e74c4ecbd80e1ac28 Mon Sep 17 00:00:00 2001 From: yucheng Date: Sat, 5 Sep 2026 17:12:21 +0000 Subject: [PATCH 42/47] test(otel v2): give the newrelic dispatch tests operator credentials, since a credential-less preset now falls back Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../litellm_core_utils/test_litellm_logging.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 0f44cbbb2a7..3b2b503b044 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -5953,15 +5953,17 @@ def test_prompt_hooks_skip_prompt_managers_when_no_prompt_id(logging_obj, tmp_pa for hook in [cb for cb in litellm.callbacks if isinstance(cb, VectorStorePreCallHook)]: litellm.logging_callback_manager.remove_callback_from_list_by_object(litellm.callbacks, hook) def test_newrelic_dispatch_prefers_otel_v2_when_flag_on(monkeypatch): - """With LITELLM_OTEL_V2 on, the "newrelic" callback builds the OTel v2 - logger (per-team credential routing); with the flag off (default) it keeps - the legacy agent-based logger, so existing deployments are untouched.""" + """With LITELLM_OTEL_V2 on and operator credentials present, the "newrelic" + callback builds the OTel v2 logger (per-team credential routing); with the + flag off (default) it keeps the legacy agent-based logger, so existing + deployments are untouched.""" from litellm.integrations.otel.logger import OpenTelemetryV2 from litellm.integrations.otel.model.config import is_otel_v2_enabled from litellm.litellm_core_utils import litellm_logging as logging_module logging_module._in_memory_loggers.clear() monkeypatch.setenv("LITELLM_OTEL_V2", "true") + monkeypatch.setenv("NEW_RELIC_LICENSE_KEY", "test-license-key") is_otel_v2_enabled.cache_clear() try: v2_logger = logging_module._init_custom_logger_compatible_class( @@ -6016,6 +6018,7 @@ def test_get_custom_logger_compatible_class_finds_v2_newrelic(monkeypatch): logging_module._in_memory_loggers.clear() monkeypatch.setenv("LITELLM_OTEL_V2", "true") + monkeypatch.setenv("NEW_RELIC_LICENSE_KEY", "test-license-key") is_otel_v2_enabled.cache_clear() try: created = logging_module._init_custom_logger_compatible_class( From 9b3d1febfba59bd72d187f2b70f5929ac10f4492 Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Sat, 5 Sep 2026 10:43:06 -0700 Subject: [PATCH 43/47] fix(otel): rebuild an anchored destination's processor past drain saturation A destination deliverable() accepted at auth can be evicted by other tenants' auths before its request's spans end, and that eviction is what tips the drain over. The saturation gate then refused the rebuild at on_end, and with the operator's exporter already stood down for that backend the span went nowhere. The gate now applies only while a request decides whether to anchor --- .../integrations/otel/plumbing/providers.py | 31 ++++++---- .../otel/test_otel_v2_destinations.py | 60 +++++++++++++++++-- 2 files changed, 74 insertions(+), 17 deletions(-) diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index df8e449bc42..0c95fcb3d61 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -542,7 +542,7 @@ class TenantFanOutSpanProcessor(SpanProcessor): with self._lock: if self._closed: return False - built: Final = self._cached_or_built_locked(destination) + built: Final = self._cached_or_built_locked(destination, anchored=False) drained: Final = self._drainable_locked() for shed in drained: self._drain.submit(shed) @@ -559,7 +559,7 @@ class TenantFanOutSpanProcessor(SpanProcessor): with self._lock: if self._closed: return None - processor: Final = self._cached_or_built_locked(destination) + processor: Final = self._cached_or_built_locked(destination, anchored=True) if processor is None: return None self._exporting[id(processor)] = self._exporting.get(id(processor), 0) + 1 @@ -568,24 +568,29 @@ class TenantFanOutSpanProcessor(SpanProcessor): self._drain.submit(shed) return processor - def _cached_or_built_locked(self, destination: "OtelDestination") -> SpanProcessor | None: + def _cached_or_built_locked(self, destination: "OtelDestination", *, anchored: bool) -> SpanProcessor | None: + """The cached processor for ``destination``, or a new one if the drain can take it. + + Every build past the cache cap sheds one processor into the drain, so while the + shed ones are stuck closing against a collector that stopped answering, a + destination that is not yet anchored is refused rather than parked behind them: + ``deliverable`` then leaves its spans with the operator's exporter until the + drain catches up. One the request already anchored is rebuilt regardless. The + operator's exporter has stood down for it, so refusing here would drop the span, + and other tenants' auths can evict it in the meantime, with that eviction being + what tips the drain over. Those rebuilds are bounded by the requests in flight, + since anchoring itself stops once the drain is full. + """ key: Final = destination.cache_key() if (cached := self._processors.get(key)) is not None: self._processors.move_to_end(key) return cached + if not anchored and self._drain.saturated(): + verbose_logger.debug("OTel V2 fan-out: drain saturated, not building for %s", destination.endpoint) + return None return self._build_locked(destination, key) def _build_locked(self, destination: "OtelDestination", key: object) -> SpanProcessor | None: - """Build and cache a processor for ``destination``, unless the drain is saturated. - - Every build past the cache cap sheds one processor into the drain, so while the - shed ones are stuck closing against a collector that stopped answering, a new - destination is refused rather than parked behind them: ``deliverable`` then - leaves its spans with the operator's exporter until the drain catches up. - """ - if self._drain.saturated(): - verbose_logger.debug("OTel V2 fan-out: drain saturated, not building for %s", destination.endpoint) - return None built: Final = self._build(destination) if built is None: return None diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index 0f74cb4d9d0..54946febf1d 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -1760,12 +1760,12 @@ class TestEvictionSafety: fan_out = TenantFanOutSpanProcessor(processor_factory=factory, pending_drains=3) try: - for index in range(_MAX_CACHED_DESTINATION_PROCESSORS + 40): - processor = fan_out._acquire(self._dest(index)) - if processor is not None: - fan_out._release(processor) + anchored = tuple( + fan_out.deliverable((self._dest(index),)) for index in range(_MAX_CACHED_DESTINATION_PROCESSORS + 40) + ) assert len(built) == _MAX_CACHED_DESTINATION_PROCESSORS + 3, "a processor per request during the outage" + assert sum(1 for accepted in anchored if accepted) == len(built), "anchored what it could not build" assert fan_out.deliverable((self._dest(999),)) == (), "the span would vanish instead of staying with the operator" finally: release.set() @@ -1776,6 +1776,58 @@ class TestEvictionSafety: assert fan_out.deliverable((self._dest(999),)) == (self._dest(999),), "the fan-out never recovered" + def test_an_anchored_destination_evicted_under_a_saturated_drain_still_gets_the_span(self): + """``deliverable`` accepted the destination, so the operator's exporter has stood + down for it. Other tenants' auths can then evict it, and the eviction is what + tips the drain into saturation, so refusing the rebuild at ``on_end`` would drop + the span outright.""" + import threading + + from litellm.integrations.otel.plumbing.providers import _MAX_CACHED_DESTINATION_PROCESSORS + + release = threading.Event() + + class Blocking(self.Recording): + def shutdown(self): + release.wait(timeout=10) + super().shutdown() + + built = [] + + def factory(_destination): + built.append(Blocking()) + return built[-1] + + fan_out = TenantFanOutSpanProcessor( + processor_factory=factory, pending_drains=_MAX_CACHED_DESTINATION_PROCESSORS + 1 + ) + provider = TracerProvider() + provider.add_span_processor(fan_out) + tracer = get_tracer(provider, "litellm") + anchored = self._dest(0) + try: + for index in range(1, _MAX_CACHED_DESTINATION_PROCESSORS + 1): + assert fan_out.deliverable((self._dest(index),)) + assert fan_out.deliverable((anchored,)) == (anchored,) + first = built[-1] + for index in range(_MAX_CACHED_DESTINATION_PROCESSORS + 1, 2 * _MAX_CACHED_DESTINATION_PROCESSORS + 1): + assert fan_out.deliverable((self._dest(index),)) + assert fan_out._drain.saturated(), "the anchored destination's own eviction saturates the drain" + assert first not in fan_out._processors.values(), "the anchored destination was not evicted" + + def run(): + set_request_destinations((anchored,)) + with tracer.start_as_current_span("chat anthropic"): + pass + + before = len(built) + in_fresh_context(run) + assert len(built) == before + 1, "the anchored destination was not rebuilt, so its span went nowhere" + assert [span.name for span in built[-1].span_exporter.get_finished_spans()] == ["chat anthropic"] + assert first.span_exporter.get_finished_spans() == (), "the shed processor was handed out again" + finally: + release.set() + def test_concurrent_eviction_cannot_build_between_retirement_and_drain_submission(self): """A second request cannot build while the first eviction is being handed to the drain, or concurrent churn can outrun the pending-drain limit.""" From 97e1c8f9afb7df3b0b0ec19f2ed4cca5a16c5e6a Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Sat, 5 Sep 2026 11:50:57 -0700 Subject: [PATCH 44/47] fix(otel): hold destination eviction while the drain is saturated An anchored destination evicted by other tenants' auths is rebuilt on its next span, and that rebuild evicted another anchored one, so with more destinations in flight than the cache holds every span cost one more processor, one more batch thread and one more close queued behind a collector that never answers. Eviction now holds while the drain is saturated, so the cache keeps one entry per destination in flight and trims back to its cap on the next hit or build once the drain has room --- .../integrations/otel/plumbing/providers.py | 19 ++++- .../otel/test_otel_v2_destinations.py | 80 +++++++++++++++++++ 2 files changed, 95 insertions(+), 4 deletions(-) diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index 0c95fcb3d61..35ca0b7085d 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -578,12 +578,14 @@ class TenantFanOutSpanProcessor(SpanProcessor): drain catches up. One the request already anchored is rebuilt regardless. The operator's exporter has stood down for it, so refusing here would drop the span, and other tenants' auths can evict it in the meantime, with that eviction being - what tips the drain over. Those rebuilds are bounded by the requests in flight, - since anchoring itself stops once the drain is full. + what tips the drain over. Eviction holds while the drain is saturated, so such a + rebuild costs the cache one entry rather than shedding another processor, and + the total stays at one per destination in flight. """ key: Final = destination.cache_key() if (cached := self._processors.get(key)) is not None: self._processors.move_to_end(key) + self._retire_overflow_locked() return cached if not anchored and self._drain.saturated(): verbose_logger.debug("OTel V2 fan-out: drain saturated, not building for %s", destination.endpoint) @@ -612,8 +614,17 @@ class TenantFanOutSpanProcessor(SpanProcessor): self._drain.submit(retired) def _retire_overflow_locked(self) -> None: - """Move the LRU processor out of the cache once it is past the cap.""" - if len(self._processors) <= _MAX_CACHED_DESTINATION_PROCESSORS: + """Move the LRU processor out of the cache once it is past the cap, drain permitting. + + Eviction is what feeds the drain, and a destination a request already anchored + is rebuilt on its next span, which would shed another one. While the shed ones + are stuck closing against a collector that stopped answering, evicting would + churn the cache at one more processor, and one more batch thread, per span. + Holding above the cap instead keeps the total at one processor per destination + in flight, since ``deliverable`` anchors no new destination while the drain is + saturated. Once it has room again, every hit and build trims one entry. + """ + if len(self._processors) <= _MAX_CACHED_DESTINATION_PROCESSORS or self._drain.saturated(): return _, evicted = self._processors.popitem(last=False) self._retired[id(evicted)] = evicted diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index 54946febf1d..268acd85a11 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -1828,6 +1828,86 @@ class TestEvictionSafety: finally: release.set() + def _saturated_by_anchoring(self, pending_drains, extra): + """A fan-out whose drain ``extra`` anchorings past the cache cap have saturated. + + Returns it with the processors built, the destinations that anchored, and the + event that lets the blocked closes finish. + """ + import threading + + from litellm.integrations.otel.plumbing.providers import _MAX_CACHED_DESTINATION_PROCESSORS + + release = threading.Event() + + class Blocking(self.Recording): + def shutdown(self): + release.wait(timeout=10) + super().shutdown() + + built = [] + + def factory(_destination): + built.append(Blocking()) + return built[-1] + + fan_out = TenantFanOutSpanProcessor(processor_factory=factory, pending_drains=pending_drains) + destinations = tuple(self._dest(index) for index in range(_MAX_CACHED_DESTINATION_PROCESSORS + extra)) + anchored = tuple(destination for destination in destinations if fan_out.deliverable((destination,))) + assert fan_out._drain.saturated(), "anchoring past the cap did not saturate the drain" + assert len(anchored) > _MAX_CACHED_DESTINATION_PROCESSORS, "not enough destinations in flight to churn" + return fan_out, built, anchored, release + + def test_anchored_rebuilds_under_a_saturated_drain_do_not_grow_with_the_spans(self): + """Every anchored rebuild past the cap evicts another anchored destination, whose + next span rebuilds it in turn. With more destinations in flight than the cache + holds, each span would then cost one more processor, one more batch thread and + one more close queued behind a collector that never answers.""" + from litellm.integrations.otel.plumbing.providers import _MAX_CACHED_DESTINATION_PROCESSORS + + fan_out, built, anchored, release = self._saturated_by_anchoring(pending_drains=4, extra=8) + try: + after_anchoring = len(built) + for _ in range(5): + for destination in anchored: + fan_out._release(fan_out._acquire(destination)) + + rebuilt = len(built) - after_anchoring + assert rebuilt == len(anchored) - _MAX_CACHED_DESTINATION_PROCESSORS, ( + f"{rebuilt} rebuilds over 5 rounds of {len(anchored)} anchored destinations: one per evicted one expected" + ) + assert len(fan_out._processors) == len(anchored), "an anchored destination was shed under a saturated drain" + assert all(destination in fan_out.deliverable((destination,)) for destination in anchored) + finally: + release.set() + + def test_the_cache_returns_to_its_cap_once_the_drain_has_room(self): + """Holding above the cap is for the outage only: with the drain caught up, the + entries kept for the destinations in flight are the ones to shed.""" + from litellm.integrations.otel.plumbing.providers import _MAX_CACHED_DESTINATION_PROCESSORS + + fan_out, built, anchored, release = self._saturated_by_anchoring(pending_drains=4, extra=8) + for destination in anchored: + fan_out._release(fan_out._acquire(destination)) + assert len(fan_out._processors) > _MAX_CACHED_DESTINATION_PROCESSORS + + release.set() + for _ in range(500): + for destination in anchored[-4:]: + fan_out._release(fan_out._acquire(destination)) + if len(fan_out._processors) <= _MAX_CACHED_DESTINATION_PROCESSORS: + break + time.sleep(0.02) + + assert len(fan_out._processors) == _MAX_CACHED_DESTINATION_PROCESSORS, "the cache never came back to its cap" + shed = len(built) - _MAX_CACHED_DESTINATION_PROCESSORS + for _ in range(500): + if sum(processor.shutdown_calls for processor in built) == shed: + break + time.sleep(0.02) + + assert sum(processor.shutdown_calls for processor in built) == shed, "a shed processor was never closed" + def test_concurrent_eviction_cannot_build_between_retirement_and_drain_submission(self): """A second request cannot build while the first eviction is being handed to the drain, or concurrent churn can outrun the pending-drain limit.""" From c9decf061544166de918946287e10f420346cba9 Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Sat, 5 Sep 2026 13:25:05 -0700 Subject: [PATCH 45/47] fix(otel): keep the proxy's own error text out of tenant traces A tenant destination received every span the request produced, error text included, so a Prisma failure during auth handed a team admin's collector the operator's Postgres endpoint, and the exception event on any failed span carried a stack trace naming the proxy's install paths. Spans the tenant's own call produced (the model call, MCP, guardrails) keep their error text. Every other span keeps the failure without the prose: its type, its provider error code and its status code, with the message, the events and the status description dropped. Stack traces come off every span, attribute and event alike. A destination's resource attributes now merge onto the span's resource instead of rebuilding one per span, which was re-running resource detection on every export. --- .../integrations/otel/plumbing/providers.py | 82 +++++++++---- .../otel/test_otel_v2_destinations.py | 110 +++++++++++++++++- 2 files changed, 162 insertions(+), 30 deletions(-) diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index 35ca0b7085d..db06a467080 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -42,7 +42,16 @@ from opentelemetry.util.types import Attributes, AttributeValue from litellm._logging import verbose_logger from litellm._version import version as litellm_version from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config -from litellm.integrations.otel.model.semconv import DB, Error, LiteLLM, LiteLLMError, Server +from litellm.integrations.otel.model.semconv import ( + DB, + MCP, + Error, + ExceptionEvent, + GenAI, + LiteLLM, + LiteLLMError, + Server, +) from litellm.integrations.otel.model.spans import LiteLLMSpanKind from litellm.integrations.otel.plumbing.context import ( request_destinations, @@ -340,10 +349,12 @@ class _DrainPool: _NO_ATTRIBUTES: Final[Mapping[str, AttributeValue]] = MappingProxyType({}) _DB_SYSTEM_KEYS: Final = frozenset({DB.SYSTEM_NAME, DB.SYSTEM_LEGACY}) # Keys on a database span that describe the proxy's own datastore: its host, its -# port, its schema, and the Prisma error text that spells the first two out again. -_OPERATOR_INFRASTRUCTURE_KEYS: Final = frozenset( - {Server.ADDRESS, Server.PORT, DB.NAMESPACE, Error.MESSAGE, LiteLLMError.STACK_TRACE} -) +# port, and its schema. +_DATASTORE_ENDPOINT_KEYS: Final = frozenset({Server.ADDRESS, Server.PORT, DB.NAMESPACE}) +# A span carrying one of these describes the tenant's own call (the model call, the +# MCP call, the guardrail), so its error text is theirs to see. Every other span is +# the proxy's own work, whose error text names the operator's infrastructure. +_TENANT_OWNED_KEYS: Final = frozenset({GenAI.OPERATION_NAME, MCP.METHOD_NAME, LiteLLM.GUARDRAIL_NAME}) # Attribute prefixes the FastAPI instrumentor uses for headers the operator opted to # capture (``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_*``). The request # side carries the caller's bearer token verbatim. @@ -381,37 +392,58 @@ def _is_database_span(attributes: Mapping[str, AttributeValue]) -> bool: return any(key in attributes for key in _DB_SYSTEM_KEYS) -def _tenant_visible(key: str, database: bool) -> bool: - if key.startswith(_CAPTURED_HEADER_PREFIXES): +def _is_tenant_owned_span(attributes: Mapping[str, AttributeValue]) -> bool: + return any(key in attributes for key in _TENANT_OWNED_KEYS) + + +def _tenant_visible(key: str, database: bool, owned: bool) -> bool: + if key.startswith(_CAPTURED_HEADER_PREFIXES) or key == LiteLLMError.STACK_TRACE: return False - return not database or key not in _OPERATOR_INFRASTRUCTURE_KEYS + if database and key in _DATASTORE_ENDPOINT_KEYS: + return False + return owned or key != Error.MESSAGE + + +def _without_stack_trace(event: Event) -> Event: + attributes: Final = event.attributes or _NO_ATTRIBUTES + if ExceptionEvent.STACKTRACE not in attributes: + return event + return Event( + name=event.name, + attributes=MappingProxyType( + {key: value for key, value in attributes.items() if key != ExceptionEvent.STACKTRACE} + ), + timestamp=event.timestamp, + ) def _for_destination(span: ReadableSpan, destination: "OtelDestination") -> ReadableSpan: """The view of ``span`` a tenant destination receives. - A database span describes the operator's own Postgres rather than the tenant's - request, so its endpoint and its error text come off on the way out. Headers the - operator captures on the server span come off every span too, since the request - side holds the caller's bearer token. The span itself stays, so the tenant still - gets the whole trace tree. + A span the tenant's own call produced keeps its error text. Every other span is + the proxy's own work (the request root, auth, the database), and its error text, + its events and its status description come off, since a Prisma failure there + spells out the operator's Postgres endpoint. A database span loses that endpoint + too. Stack traces walk the operator's install and come off every span, as do the + headers the operator captures on the server span, whose request side holds the + caller's bearer token. The span itself stays, so the tenant still gets the whole + trace tree. """ extra: Final = destination.resource_attributes attributes: Final = span.attributes or _NO_ATTRIBUTES database: Final = _is_database_span(attributes) - kept: Final = MappingProxyType({key: value for key, value in attributes.items() if _tenant_visible(key, database)}) - if not extra and not database and len(kept) == len(attributes): - return span - resource: Final = ( - Resource.create( - {**dict(span.resource.attributes), **dict(extra)} # mutable-ok: the OTel SDK takes a concrete mapping - ) - if extra - else span.resource + owned: Final = _is_tenant_owned_span(attributes) + kept: Final = MappingProxyType( + {key: value for key, value in attributes.items() if _tenant_visible(key, database, owned)} ) - if not database: - return _TenantSpanView(span, resource, kept, span.events, span.status) - return _TenantSpanView(span, resource, kept, (), Status(span.status.status_code)) + recorded: Final = span.events + events: Final = tuple(_without_stack_trace(event) for event in recorded) if owned else () + unchanged: Final = owned and len(kept) == len(attributes) and all(a is b for a, b in zip(events, recorded)) + if not extra and unchanged: + return span + resource: Final = span.resource.merge(Resource(extra)) if extra else span.resource + status: Final = span.status if owned else Status(span.status.status_code) + return _TenantSpanView(span, resource, kept, events, status) class TenantFanOutSpanProcessor(SpanProcessor): diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index 268acd85a11..212b646d8fa 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -6,6 +6,7 @@ from collections.abc import Mapping from types import MappingProxyType import pytest +from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import SimpleSpanProcessor from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter @@ -572,7 +573,7 @@ class TestFanOut: """With ``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST`` set, the server span carries the caller's bearer token. A team admin's collector must not receive it, while the operator's own copy keeps it and the tenant keeps the - rest of the span, its events and its status.""" + rest of the span.""" dest_exporter, operator_exporter = InMemorySpanExporter(), InMemorySpanExporter() provider = TracerProvider() provider.add_span_processor(SimpleSpanProcessor(operator_exporter)) @@ -594,20 +595,119 @@ class TestFanOut: "http.response.header.set_cookie": ("session=abc",), } ) - server_span.add_event("request.received") - server_span.set_status(Status(StatusCode.ERROR, "rate limited")) + server_span.set_status(Status(StatusCode.ERROR)) in_fresh_context(run) tenant = dest_exporter.get_finished_spans()[0] assert dict(tenant.attributes) == {"http.request.method": "POST", "http.route": "/v1/chat/completions"} assert bearer not in tenant.to_json() - assert [event.name for event in tenant.events] == ["request.received"] - assert tenant.status.description == "rate limited", "only the header capture comes off a server span" + assert tenant.status.status_code is StatusCode.ERROR operator = operator_exporter.get_finished_spans()[0] assert operator.attributes["http.request.header.authorization"] == (bearer,) assert operator.attributes["http.response.header.set_cookie"] == ("session=abc",) + def test_the_proxys_own_error_text_does_not_ride_along_to_the_tenant(self): + """Postgres failing during auth surfaces as a ``ProxyException`` whose message + quotes the Prisma error, so the auth span and the request root carry the + operator's database endpoint in ``error.message``, in the exception event and + in the status description. None of it is the tenant's, so it all comes off, + while the failure itself (its type, its code, its status) stays. The tenant's + own model call keeps its error text, less the stack trace that walks the + operator's install. The operator's copy keeps everything.""" + dest_exporter, operator_exporter = InMemorySpanExporter(), InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(operator_exporter)) + provider.add_span_processor( + TenantFanOutSpanProcessor(processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter)) + ) + tracer = get_tracer(provider, "litellm") + unreachable = "Authentication Error, Can't reach database server at db.internal.example:15400" + install = "/srv/litellm/.venv/lib/python3.13/site-packages/opentelemetry/trace/__init__.py" + provider_error = "AnthropicException - invalid x-api-key" + + def fail(span, message: str) -> None: + span.set_attributes( + { + "error.type": "ProxyException", + "error.message": message, + "litellm.provider.error.code": "500", + "litellm.provider.error.stack_trace": f"Traceback\n File {install}\n{message}", + } + ) + span.add_event( + "exception", + {"exception.type": "ProxyException", "exception.message": message, "exception.stacktrace": install}, + ) + span.set_status(Status(StatusCode.ERROR, message)) + + def run(): + set_request_destinations((LANGFUSE_DEST,)) + with tracer.start_as_current_span("POST /v1/chat/completions") as root: + with tracer.start_as_current_span("auth /v1/chat/completions") as auth: + fail(auth, unreachable) + with tracer.start_as_current_span("chat claude-haiku") as llm: + llm.set_attribute("gen_ai.operation.name", "chat") + fail(llm, provider_error) + fail(root, unreachable) + + in_fresh_context(run) + + tenant = {s.name: s for s in dest_exporter.get_finished_spans()} + operator = {s.name: s for s in operator_exporter.get_finished_spans()} + assert set(tenant) == {"POST /v1/chat/completions", "auth /v1/chat/completions", "chat claude-haiku"} + for name in ("POST /v1/chat/completions", "auth /v1/chat/completions"): + proxy_span = tenant[name] + assert dict(proxy_span.attributes) == {"error.type": "ProxyException", "litellm.provider.error.code": "500"} + assert list(proxy_span.events) == [] + assert proxy_span.status.status_code is StatusCode.ERROR + assert proxy_span.status.description is None + assert "db.internal.example" not in proxy_span.to_json() + assert install not in proxy_span.to_json() + llm_span = tenant["chat claude-haiku"] + assert llm_span.attributes["error.message"] == provider_error, "the tenant's own call keeps its error text" + assert "litellm.provider.error.stack_trace" not in llm_span.attributes + assert llm_span.status.description == provider_error + assert [dict(event.attributes) for event in llm_span.events] == [ + {"exception.type": "ProxyException", "exception.message": provider_error} + ] + assert install not in llm_span.to_json() + for name, message in (("auth /v1/chat/completions", unreachable), ("chat claude-haiku", provider_error)): + assert operator[name].attributes["error.message"] == message + assert install in operator[name].attributes["litellm.provider.error.stack_trace"] + assert operator[name].events[0].attributes["exception.stacktrace"] == install + assert operator[name].status.description == message + + def test_a_tenants_service_name_is_layered_onto_the_operators_resource(self): + """The destination's ``service.name`` replaces the operator's on the tenant's + copy and every other resource attribute travels unchanged. Nothing is detected + afresh per span, so no attribute the operator did not configure appears.""" + dest = InMemorySpanExporter() + provider = TracerProvider( + resource=Resource({"service.name": "litellm-proxy", "deployment.environment.name": "prod"}) + ) + provider.add_span_processor(TenantFanOutSpanProcessor(processor_factory=lambda _d: SimpleSpanProcessor(dest))) + + def run(): + set_request_destinations( + ( + OtelDestination( + endpoint="http://a.local", + callback_name="langfuse_otel", + resource_attributes={"service.name": "team-checkout"}, + ), + ) + ) + emit(provider) + + in_fresh_context(run) + + (span,) = dest.get_finished_spans() + assert dict(span.resource.attributes) == { + "service.name": "team-checkout", + "deployment.environment.name": "prod", + } + def test_a_destination_that_cannot_build_a_processor_is_skipped_quietly(self): """An unbuildable destination must not cost the caller its request.""" attempts = [] From 1e33b2fede1cb972743dd605c14b01b4657e1cab Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Sat, 5 Sep 2026 14:25:49 -0700 Subject: [PATCH 46/47] fix(otel): redact tenant URL query parameters --- litellm/integrations/otel/mappers/legacy.py | 3 +- litellm/integrations/otel/model/semconv.py | 3 + .../integrations/otel/plumbing/providers.py | 29 +++++++-- .../otel/test_otel_v2_destinations.py | 59 +++++++++++++++++++ 4 files changed, 87 insertions(+), 7 deletions(-) diff --git a/litellm/integrations/otel/mappers/legacy.py b/litellm/integrations/otel/mappers/legacy.py index 37475acb8f7..d25c25cd127 100644 --- a/litellm/integrations/otel/mappers/legacy.py +++ b/litellm/integrations/otel/mappers/legacy.py @@ -23,6 +23,7 @@ from litellm.integrations.otel.model.payloads import ( ServiceSpanData, ToolDefinition, ) +from litellm.integrations.otel.model.semconv import Error # Attribute keys in the semconv-ai / Traceloop vocabulary. _LEGACY_SYSTEM: Final = "gen_ai.system" @@ -36,7 +37,7 @@ _LEGACY_PRESENCE_PENALTY: Final = "llm.presence_penalty" _LEGACY_STOP_SEQUENCES: Final = "llm.chat.stop_sequences" _LEGACY_SERVICE: Final = "service" _LEGACY_CALL_TYPE: Final = "call_type" -_LEGACY_ERROR: Final = "error" +_LEGACY_ERROR: Final = Error.MESSAGE_LEGACY class LegacyMapper: diff --git a/litellm/integrations/otel/model/semconv.py b/litellm/integrations/otel/model/semconv.py index af5327cbd41..d3628005bac 100644 --- a/litellm/integrations/otel/model/semconv.py +++ b/litellm/integrations/otel/model/semconv.py @@ -204,6 +204,9 @@ class Error: TYPE: Final = "error.type" MESSAGE: Final = "error.message" + # The same text under the bare key the semconv-ai / Traceloop vocabulary uses + # (see ``LegacyMapper``), so anything reading or redacting error text covers both. + MESSAGE_LEGACY: Final = "error" class LiteLLMError: diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index db06a467080..70948f466ff 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -355,10 +355,16 @@ _DATASTORE_ENDPOINT_KEYS: Final = frozenset({Server.ADDRESS, Server.PORT, DB.NAM # MCP call, the guardrail), so its error text is theirs to see. Every other span is # the proxy's own work, whose error text names the operator's infrastructure. _TENANT_OWNED_KEYS: Final = frozenset({GenAI.OPERATION_NAME, MCP.METHOD_NAME, LiteLLM.GUARDRAIL_NAME}) +_PROXY_ERROR_TEXT_KEYS: Final = frozenset({Error.MESSAGE, Error.MESSAGE_LEGACY}) # Attribute prefixes the FastAPI instrumentor uses for headers the operator opted to # capture (``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_*``). The request # side carries the caller's bearer token verbatim. _CAPTURED_HEADER_PREFIXES: Final = ("http.request.header.", "http.response.header.") +# The instrumentor stamps the request URL on the server span with its query string, +# under the old convention and the new one, and litellm accepts a virtual key as a +# ``?key=`` query parameter. +_URL_KEYS: Final = frozenset({"http.url", "http.target", "url.full"}) +_URL_QUERY_KEY: Final = "url.query" class _TenantSpanView(ReadableSpan): @@ -397,11 +403,21 @@ def _is_tenant_owned_span(attributes: Mapping[str, AttributeValue]) -> bool: def _tenant_visible(key: str, database: bool, owned: bool) -> bool: - if key.startswith(_CAPTURED_HEADER_PREFIXES) or key == LiteLLMError.STACK_TRACE: + if key.startswith(_CAPTURED_HEADER_PREFIXES) or key in (LiteLLMError.STACK_TRACE, _URL_QUERY_KEY): return False if database and key in _DATASTORE_ENDPOINT_KEYS: return False - return owned or key != Error.MESSAGE + return owned or key not in _PROXY_ERROR_TEXT_KEYS + + +def _without_query(key: str, value: AttributeValue) -> AttributeValue: + if key not in _URL_KEYS or not isinstance(value, str): + return value + return value.partition("?")[0] + + +def _same_attributes(kept: Mapping[str, AttributeValue], attributes: Mapping[str, AttributeValue]) -> bool: + return len(kept) == len(attributes) and all(kept[key] is value for key, value in attributes.items()) def _without_stack_trace(event: Event) -> Event: @@ -426,19 +442,20 @@ def _for_destination(span: ReadableSpan, destination: "OtelDestination") -> Read spells out the operator's Postgres endpoint. A database span loses that endpoint too. Stack traces walk the operator's install and come off every span, as do the headers the operator captures on the server span, whose request side holds the - caller's bearer token. The span itself stays, so the tenant still gets the whole - trace tree. + caller's bearer token, and the query string of the request URL, which can hold + the same key. The span itself stays, so the tenant still gets the whole trace + tree. """ extra: Final = destination.resource_attributes attributes: Final = span.attributes or _NO_ATTRIBUTES database: Final = _is_database_span(attributes) owned: Final = _is_tenant_owned_span(attributes) kept: Final = MappingProxyType( - {key: value for key, value in attributes.items() if _tenant_visible(key, database, owned)} + {key: _without_query(key, value) for key, value in attributes.items() if _tenant_visible(key, database, owned)} ) recorded: Final = span.events events: Final = tuple(_without_stack_trace(event) for event in recorded) if owned else () - unchanged: Final = owned and len(kept) == len(attributes) and all(a is b for a, b in zip(events, recorded)) + unchanged: Final = owned and _same_attributes(kept, attributes) and all(a is b for a, b in zip(events, recorded)) if not extra and unchanged: return span resource: Final = span.resource.merge(Resource(extra)) if extra else span.resource diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index 212b646d8fa..c98144d1776 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -534,6 +534,7 @@ class TestFanOut: "db.namespace": "litellm", "error.type": "PrismaError", "error.message": unreachable, + "error": unreachable, "litellm.provider.error.stack_trace": f"Traceback: {unreachable}", } ) @@ -566,9 +567,67 @@ class TestFanOut: assert operator_db.attributes["server.port"] == 15400 assert operator_db.attributes["db.namespace"] == "litellm" assert operator_db.attributes["error.message"] == unreachable + assert operator_db.attributes["error"] == unreachable assert operator_db.status.description == unreachable assert [event.name for event in operator_db.events] == ["exception"] + def test_the_callers_key_in_the_query_string_does_not_ride_along_to_the_tenant(self): + """A Google AI Studio style request authenticates with ``?key=``, + and the instrumentor stamps the full request URL on the server span. The + tenant keeps the URL up to the query string, and the operator's copy keeps it + whole.""" + dest_exporter, operator_exporter = InMemorySpanExporter(), InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(operator_exporter)) + provider.add_span_processor( + TenantFanOutSpanProcessor(processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter)) + ) + tracer = get_tracer(provider, "litellm") + path = "/v1beta/models/gemini-2.5-flash:generateContent" + query = "key=sk-another-members-virtual-key&alt=sse" + + def run(): + set_request_destinations((LANGFUSE_DEST,)) + with tracer.start_as_current_span(f"POST {path}") as server_span: + server_span.set_attributes( + { + "http.method": "POST", + "http.route": path, + "http.target": f"{path}?{query}", + "http.url": f"http://proxy.example:4000{path}?{query}", + "url.path": path, + "url.query": query, + "http.status_code": 200, + } + ) + with tracer.start_as_current_span("generate_content gemini-2.5-flash") as llm_span: + llm_span.set_attributes( + { + "gen_ai.operation.name": "generate_content", + "url.full": f"https://generativelanguage.googleapis.com{path}?key=AIza-operator-provider-key", + } + ) + + in_fresh_context(run) + + tenant = {s.name: s for s in dest_exporter.get_finished_spans()} + assert dict(tenant[f"POST {path}"].attributes) == { + "http.method": "POST", + "http.route": path, + "http.target": path, + "http.url": f"http://proxy.example:4000{path}", + "url.path": path, + "http.status_code": 200, + } + assert "sk-another-members-virtual-key" not in tenant[f"POST {path}"].to_json() + assert tenant["generate_content gemini-2.5-flash"].attributes["url.full"] == ( + f"https://generativelanguage.googleapis.com{path}" + ), "the tenant's own span keeps its error text, and still loses a query string" + operator = {s.name: s for s in operator_exporter.get_finished_spans()} + assert operator[f"POST {path}"].attributes["http.url"] == f"http://proxy.example:4000{path}?{query}" + assert operator[f"POST {path}"].attributes["url.query"] == query + assert "AIza-operator-provider-key" in operator["generate_content gemini-2.5-flash"].to_json() + def test_captured_request_headers_do_not_ride_along_to_the_tenant(self): """With ``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST`` set, the server span carries the caller's bearer token. A team admin's collector must From 7680c355c622138ed117620507fb44368310ebca Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Sat, 5 Sep 2026 17:28:59 -0700 Subject: [PATCH 47/47] fix(otel): close final tenant routing gaps --- litellm/integrations/otel/logger.py | 12 +- .../integrations/otel/plumbing/providers.py | 54 +++++--- .../otel/test_otel_v2_destinations.py | 116 ++++++++++++++++++ 3 files changed, 163 insertions(+), 19 deletions(-) diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index 2353472aefa..630aa313dc9 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -885,12 +885,22 @@ def publish_global_otel_v2_provider( """ global _published_v2_provider logger: Final = select_global_otel_v2_logger(in_memory_loggers, registered=registered) - attach_tenant_fan_out(logger.tracer_provider, logger.config) + attach_tenant_fan_out(logger.tracer_provider, *_v2_configs(in_memory_loggers, logger)) set_global_provider(logger.tracer_provider) _published_v2_provider = logger.tracer_provider # rebind-ok: startup records the one provider carrying the fan-out return logger +def _v2_configs(in_memory_loggers: Sequence[object], logger: "OpenTelemetryV2") -> tuple[OpenTelemetryV2Config, ...]: + """Every v2 logger's config, the published logger's first. + + Each preset keeps its own provider and exporters, so the accounts the operator + writes to are spread over all of them, not held by the published logger alone. + """ + others: Final = tuple(cb.config for cb in in_memory_loggers if isinstance(cb, OpenTelemetryV2) and cb is not logger) + return (logger.config, *others) + + def _registered_v2_logger() -> "OpenTelemetryV2 | None": try: from litellm.proxy import proxy_server diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index 70948f466ff..afd502962cf 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -356,6 +356,10 @@ _DATASTORE_ENDPOINT_KEYS: Final = frozenset({Server.ADDRESS, Server.PORT, DB.NAM # the proxy's own work, whose error text names the operator's infrastructure. _TENANT_OWNED_KEYS: Final = frozenset({GenAI.OPERATION_NAME, MCP.METHOD_NAME, LiteLLM.GUARDRAIL_NAME}) _PROXY_ERROR_TEXT_KEYS: Final = frozenset({Error.MESSAGE, Error.MESSAGE_LEGACY}) +# A guardrail that never answered carries the exception it raised as its response, +# which names the operator's guardrail endpoint. The second spelling is the legacy +# status the request-level logger still maps. +_GUARDRAIL_UNREACHABLE_STATUSES: Final = frozenset({"guardrail_failed_to_respond", "failure"}) # Attribute prefixes the FastAPI instrumentor uses for headers the operator opted to # capture (``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_*``). The request # side carries the caller's bearer token verbatim. @@ -402,11 +406,17 @@ def _is_tenant_owned_span(attributes: Mapping[str, AttributeValue]) -> bool: return any(key in attributes for key in _TENANT_OWNED_KEYS) -def _tenant_visible(key: str, database: bool, owned: bool) -> bool: +def _guardrail_unreachable(attributes: Mapping[str, AttributeValue]) -> bool: + return attributes.get(LiteLLM.GUARDRAIL_STATUS) in _GUARDRAIL_UNREACHABLE_STATUSES + + +def _tenant_visible(key: str, database: bool, owned: bool, unreachable_guardrail: bool) -> bool: if key.startswith(_CAPTURED_HEADER_PREFIXES) or key in (LiteLLMError.STACK_TRACE, _URL_QUERY_KEY): return False if database and key in _DATASTORE_ENDPOINT_KEYS: return False + if unreachable_guardrail and key == LiteLLM.GUARDRAIL_RESPONSE: + return False return owned or key not in _PROXY_ERROR_TEXT_KEYS @@ -440,18 +450,24 @@ def _for_destination(span: ReadableSpan, destination: "OtelDestination") -> Read the proxy's own work (the request root, auth, the database), and its error text, its events and its status description come off, since a Prisma failure there spells out the operator's Postgres endpoint. A database span loses that endpoint - too. Stack traces walk the operator's install and come off every span, as do the - headers the operator captures on the server span, whose request side holds the - caller's bearer token, and the query string of the request URL, which can hold - the same key. The span itself stays, so the tenant still gets the whole trace - tree. + too, and a guardrail that failed to respond loses its response text, which is the + exception it raised and names the operator's guardrail endpoint. Stack traces walk + the operator's install and come off every span, as do the headers the operator + captures on the server span, whose request side holds the caller's bearer token, + and the query string of the request URL, which can hold the same key. The span + itself stays, so the tenant still gets the whole trace tree. """ extra: Final = destination.resource_attributes attributes: Final = span.attributes or _NO_ATTRIBUTES database: Final = _is_database_span(attributes) owned: Final = _is_tenant_owned_span(attributes) + unreachable: Final = _guardrail_unreachable(attributes) kept: Final = MappingProxyType( - {key: _without_query(key, value) for key, value in attributes.items() if _tenant_visible(key, database, owned)} + { + key: _without_query(key, value) + for key, value in attributes.items() + if _tenant_visible(key, database, owned, unreachable) + } ) recorded: Final = span.events events: Final = tuple(_without_stack_trace(event) for event in recorded) if owned else () @@ -1039,19 +1055,21 @@ def build_tracer_provider( _FAN_OUT_ATTACH_LOCK: Final = threading.Lock() -def attach_tenant_fan_out(provider: TracerProvider, config: OpenTelemetryV2Config | None = None) -> None: +def attach_tenant_fan_out(provider: TracerProvider, *configs: OpenTelemetryV2Config) -> None: """Give ``provider`` the fan-out that delivers spans to key/team destinations. Called on the one provider published as the OTel global, and idempotent so a second publish (a test, a re-initialized proxy) cannot double-export. Concurrent first calls (requests racing to anchor before any publish) serialize on one lock - so exactly one fan-out lands. ``config`` names the operator's own exporters so an - additive destination pointing at one of them is delivered once rather than twice. + so exactly one fan-out lands. ``configs`` name the operator's own exporters, one + config per v2 logger since each keeps its own provider and still writes its + account, so an additive destination pointing at any of them is delivered once + rather than twice. """ with _FAN_OUT_ATTACH_LOCK: if any(isinstance(processor, TenantFanOutSpanProcessor) for processor in _attached_processors(provider)): return - provider.add_span_processor(TenantFanOutSpanProcessor(operator_sinks=operator_sink_keys(config))) + provider.add_span_processor(TenantFanOutSpanProcessor(operator_sinks=operator_sink_keys(*configs))) def deliverable_destinations( @@ -1076,18 +1094,18 @@ def deliverable_destinations( return fan_out.deliverable(destinations) if fan_out is not None else () -def operator_sink_keys(config: OpenTelemetryV2Config | None) -> frozenset[_SinkKey]: +def operator_sink_keys(*configs: OpenTelemetryV2Config) -> frozenset[_SinkKey]: """The accounts the operator's own exporters write to, in destination terms. - An exporter with no endpoint of its own resolves one from the environment at - export time, so it has no comparable identity and is left out, and so is one - that never reaches the wire: a console kind ignores the endpoint, and a - header-gated spec with no credentials is skipped when the provider is built. + Every v2 logger's config counts, since each logger exports through its own + provider. An exporter with no endpoint of its own resolves one from the + environment at export time, so it has no comparable identity and is left out, + and so is one that never reaches the wire: a console kind ignores the endpoint, + and a header-gated spec with no credentials is skipped when the provider is built. """ - if config is None: - return frozenset() return frozenset( key + for config in configs for spec in config.exporters if _exports_to_the_wire(spec) and (key := _sink_key(spec.endpoint, parse_headers(spec.headers))) is not None ) diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index c98144d1776..1799381bada 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -28,6 +28,7 @@ from litellm.integrations.otel.model.config import ( is_otel_v2_enabled, ) from litellm.integrations.otel.model.destination import OtelDestination +from litellm.integrations.otel.plumbing import providers as otel_providers from litellm.integrations.otel.plumbing.context import ( destination_backends, request_destinations, @@ -351,6 +352,31 @@ class TestRoutingMode: assert operator_sink_keys(config) == frozenset({self.OPERATOR_SINK}) + def test_operator_sink_keys_spans_every_config_it_is_handed(self): + first = OpenTelemetryV2Config( + exporters=( + ExporterSpec( + kind="otlp_http", + endpoint=self.OPERATOR_SINK[0], + headers="authorization=Basic op", + ), + ) + ) + second = OpenTelemetryV2Config( + exporters=( + ExporterSpec( + kind="otlp_http", + endpoint="https://otlp.arize.com/v1/traces", + headers="space_id=s,api_key=k", + ), + ) + ) + + assert operator_sink_keys(first, second) == { + self.OPERATOR_SINK, + _sink_key("https://otlp.arize.com/v1/traces", {"space_id": "s", "api_key": "k"}), + } + def test_a_team_pointing_at_a_credential_less_operator_exporter_still_gets_its_spans(self, monkeypatch): """Under additive the fan-out skips a destination the operator already writes to. An exporter the provider never built writes nothing, so skipping it would @@ -571,6 +597,50 @@ class TestFanOut: assert operator_db.status.description == unreachable assert [event.name for event in operator_db.events] == ["exception"] + @pytest.mark.parametrize("failure_status", ["guardrail_failed_to_respond", "failure"]) + def test_a_guardrails_failure_text_does_not_ride_along_to_the_tenant(self, failure_status): + dest_exporter, operator_exporter = InMemorySpanExporter(), InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(operator_exporter)) + provider.add_span_processor( + TenantFanOutSpanProcessor(processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter)) + ) + tracer = get_tracer(provider, "litellm") + unreachable = "Cannot connect to host guardrail.internal.example:9000" + verdict = '{"action": "block", "categories": ["pii"]}' + + def run(): + set_request_destinations((LANGFUSE_DEST,)) + with tracer.start_as_current_span("POST /v1/chat/completions"): + with tracer.start_as_current_span("execute_guardrail pii") as down: + down.set_attributes( + { + "litellm.guardrail.name": "pii", + "litellm.guardrail.status": failure_status, + "litellm.guardrail.response": unreachable, + } + ) + with tracer.start_as_current_span("execute_guardrail toxicity") as up: + up.set_attributes( + { + "litellm.guardrail.name": "toxicity", + "litellm.guardrail.status": "guardrail_intervened", + "litellm.guardrail.response": verdict, + } + ) + + in_fresh_context(run) + + tenant = {s.name: s for s in dest_exporter.get_finished_spans()} + operator = {s.name: s for s in operator_exporter.get_finished_spans()} + assert dict(tenant["execute_guardrail pii"].attributes) == { + "litellm.guardrail.name": "pii", + "litellm.guardrail.status": failure_status, + } + assert "guardrail.internal.example" not in tenant["execute_guardrail pii"].to_json() + assert tenant["execute_guardrail toxicity"].attributes["litellm.guardrail.response"] == verdict + assert operator["execute_guardrail pii"].attributes["litellm.guardrail.response"] == unreachable + def test_the_callers_key_in_the_query_string_does_not_ride_along_to_the_tenant(self): """A Google AI Studio style request authenticates with ``?key=``, and the instrumentor stamps the full request URL on the server span. The @@ -925,6 +995,52 @@ class TestProviderWiring: assert kinds(published).count("TenantFanOutSpanProcessor") == 1 assert "TenantFanOutSpanProcessor" not in kinds(other) + @pytest.mark.parametrize("canonical", ["langfuse_otel", "arize"]) + def test_publishing_tells_the_fan_out_about_every_v2_loggers_account(self, monkeypatch, canonical): + monkeypatch.setenv("LITELLM_OTEL_TENANT_DESTINATION_MODE", "additive") + shared = InMemorySpanExporter() + monkeypatch.setattr(otel_providers, "_destination_processor", lambda _d: SimpleSpanProcessor(shared)) + accounts = { + "langfuse_otel": ( + "https://cloud.langfuse.com/api/public/otel/v1/traces", + "authorization=Basic op", + ), + "arize": ( + "https://otlp.arize.com/v1/traces", + "space_id=space-op,api_key=key-op", + ), + } + loggers = { + name: OpenTelemetryV2( + config=OpenTelemetryV2Config( + exporters=(ExporterSpec(kind="otlp_http", endpoint=endpoint, headers=headers),) + ), + callback_name=name, + tracer_provider=TracerProvider(), + ) + for name, (endpoint, headers) in accounts.items() + } + other = "arize" if canonical == "langfuse_otel" else "langfuse_otel" + published = publish_global_otel_v2_provider( + [loggers[other]], + lambda _p: None, + registered=loggers[canonical], + ) + + def destination(name, headers): + return OtelDestination(endpoint=accounts[name][0], headers=headers, callback_name=name) + + def run(destinations): + set_request_destinations(destinations) + emit(published.tracer_provider) + + in_fresh_context(run, (destination(canonical, dict(pair.split("=") for pair in accounts[canonical][1].split(","))),)) + in_fresh_context(run, (destination(other, dict(pair.split("=") for pair in accounts[other][1].split(","))),)) + assert shared.get_finished_spans() == (), "an account the operator already writes to was written twice" + + in_fresh_context(run, (destination(other, {"authorization": "Basic team"}),)) + assert [s.name for s in shared.get_finished_spans()] == ["chat gpt-4"] + def test_publishing_twice_does_not_double_export(self): config = OpenTelemetryV2Config(exporters=[ExporterSpec(kind="in_memory", owner=ExporterOwner.ARIZE_AX)]) logger = OpenTelemetryV2(config=config, callback_name="arize")