diff --git a/litellm/integrations/otel/langfuse_logger.py b/litellm/integrations/otel/langfuse_logger.py index f8fd417392f..c377a0ceeaf 100644 --- a/litellm/integrations/otel/langfuse_logger.py +++ b/litellm/integrations/otel/langfuse_logger.py @@ -1,6 +1,9 @@ from collections.abc import AsyncGenerator, AsyncIterator, Callable, Mapping from typing import TYPE_CHECKING, Final +from opentelemetry.sdk.trace import ReadableSpan +from opentelemetry.trace import Span + from litellm._logging import verbose_logger from litellm.integrations.otel.logger import OpenTelemetryV2 from litellm.integrations.otel.mappers.langfuse import ( @@ -19,15 +22,22 @@ if TYPE_CHECKING: class LangfuseOpenTelemetryV2(OpenTelemetryV2): """Names the trace from the request. Langfuse reads ``langfuse.trace.name`` off the root observation, - and the proxy's root span is still recording when the LLM call starts.""" + and the proxy's root span is still recording when the LLM call starts. Only a root created by this + logger's own provider is stamped; any other root is exported to destinations that are not Langfuse.""" def log_pre_api_call(self, model: str, messages: object, kwargs: Mapping[str, object]) -> None: - root: Final = request_root_span() + root: Final = self._owned_recording_root() name: Final = caller_trace_name(kwargs) - if root is not None and root.is_recording() and name is not None: + if root is not None and name is not None: root.set_attribute(LANGFUSE_TRACE_NAME, name) super().log_pre_api_call(model, messages, kwargs) + def _owned_recording_root(self) -> Span | None: + root: Final = request_root_span() + if root is None or not root.is_recording() or not isinstance(root, ReadableSpan): + return None + return root if root.resource is self.tracer_provider.resource else None + class LangfuseContentOpenTelemetryV2(LangfuseOpenTelemetryV2): """Stamps the request's input and output on the root observation while it is still recording. @@ -59,8 +69,8 @@ class LangfuseContentOpenTelemetryV2(LangfuseOpenTelemetryV2): self._stamp_root_io(request_data, lambda: stream_output(tuple(relayed), request_data)) def _stamp_root_io(self, data: Mapping[str, object], render_output: Callable[[], str | None]) -> None: - root: Final = request_root_span() - if root is None or not root.is_recording(): + root: Final = self._owned_recording_root() + if root is None: return try: output: Final = render_output() diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index 9ac748b231c..0403270e8dd 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -259,9 +259,15 @@ class OpenTelemetryV2(CustomLogger): self._register_in_callback_list(litellm._async_failure_callback) except Exception: pass - if getattr(proxy_server, "open_telemetry_logger", None) is None: + if self._outranks_for_proxy_slot(getattr(proxy_server, "open_telemetry_logger", None)): setattr(proxy_server, "open_telemetry_logger", self) + def _outranks_for_proxy_slot(self, holder: object) -> bool: + """``otel`` owns the slot whenever configured; a preset holds it only until ``otel`` is built.""" + if holder is None: + return True + return self.callback_name is None and isinstance(holder, OpenTelemetryV2) and holder.callback_name is not None + # ====================================================================== # # LLM-call callbacks — the span is opened at the ``pre_call`` boundary and # closed here. See ``log_pre_api_call``. @@ -842,10 +848,12 @@ def select_global_otel_v2_logger( ``proxy_server.open_telemetry_logger``), and every other v2 entry point — guardrail, identity seeding, phase spans — already routes through that same ``registered`` owner. Reuse it here too so the global provider has one source - of truth instead of a second, independently-derived guess; this is the logger - a preset (arize, langfuse, …) folds the ``OTEL_*`` base exporter and its own - exporter into, so the FastAPI server span and the gen-ai spans share one - provider and one trace. + of truth instead of a second, independently-derived guess. Each logger exports + only to the destination its callback owns (``otel`` serves ``OTEL_*``, a preset + serves its own backend), so the FastAPI server span and the request root land + at the ``otel`` callback's collector when one is configured and at the first + preset's backend otherwise, whatever the callback order; a preset stamps that + root only when its own provider created it. Fall back to ``in_memory_loggers`` for the SDK path, where no proxy global is set (selecting from there, not ``service_callback``, which a preset logger does @@ -855,8 +863,11 @@ def select_global_otel_v2_logger( """ if registered is not None: return registered - existing: Final = next((cb for cb in in_memory_loggers if isinstance(cb, OpenTelemetryV2)), None) - return existing if existing is not None else OpenTelemetryV2() + v2_loggers: Final = tuple(cb for cb in in_memory_loggers if isinstance(cb, OpenTelemetryV2)) + generic: Final = next((cb for cb in v2_loggers if cb.callback_name is None), None) + if generic is not None: + return generic + return v2_loggers[0] if v2_loggers else OpenTelemetryV2() def publish_global_otel_v2_provider( diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index 9f815b76aa6..81f22c8c642 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -41,7 +41,7 @@ from opentelemetry.util.types import Attributes, AttributeValue from litellm._logging import verbose_logger from litellm._version import version as litellm_version -from litellm.integrations.otel.model.config import ExporterOwner, ExporterSpec, OpenTelemetryV2Config +from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config from litellm.integrations.otel.model.semconv import ( DB, MCP, @@ -377,7 +377,6 @@ _CAPTURED_HEADER_PREFIXES: Final = ("http.request.header.", "http.response.heade # ``?key=`` query parameter. _URL_KEYS: Final = frozenset({"http.url", "http.target", "url.full"}) _URL_QUERY_KEY: Final = "url.query" -_LANGFUSE_ATTRIBUTE_PREFIX: Final = "langfuse." class _TenantSpanView(ReadableSpan): @@ -419,11 +418,9 @@ def _guardrail_unreachable(attributes: Mapping[str, AttributeValue]) -> bool: return attributes.get(LiteLLM.GUARDRAIL_STATUS) in _GUARDRAIL_UNREACHABLE_STATUSES -def _tenant_visible(key: str, database: bool, owned: bool, unreachable_guardrail: bool, langfuse: bool) -> bool: +def _tenant_visible(key: str, database: bool, owned: bool, unreachable_guardrail: bool) -> bool: if key.startswith(_CAPTURED_HEADER_PREFIXES) or key in (LiteLLMError.STACK_TRACE, _URL_QUERY_KEY): return False - if not langfuse and key.startswith(_LANGFUSE_ATTRIBUTE_PREFIX): - return False if database and key in _DATASTORE_ENDPOINT_KEYS: return False if unreachable_guardrail and key == LiteLLM.GUARDRAIL_RESPONSE: @@ -473,12 +470,11 @@ def _for_destination(span: ReadableSpan, destination: "OtelDestination") -> Read database: Final = _is_database_span(attributes) owned: Final = _is_tenant_owned_span(attributes) unreachable: Final = _guardrail_unreachable(attributes) - langfuse: Final = destination.callback_name == ExporterOwner.LANGFUSE_OTEL.value kept: Final = MappingProxyType( { key: _without_query(key, value) for key, value in attributes.items() - if _tenant_visible(key, database, owned, unreachable, langfuse) + if _tenant_visible(key, database, owned, unreachable) } ) recorded: Final = span.events @@ -491,24 +487,6 @@ def _for_destination(span: ReadableSpan, destination: "OtelDestination") -> Read return _TenantSpanView(span, resource, kept, events, status) -def _without_langfuse_attributes(span: ReadableSpan) -> ReadableSpan: - attributes: Final = span.attributes or _NO_ATTRIBUTES - kept: Final = MappingProxyType( - {key: value for key, value in attributes.items() if not key.startswith(_LANGFUSE_ATTRIBUTE_PREFIX)} - ) - if len(kept) == len(attributes): - return span - return _TenantSpanView(span, span.resource, kept, span.events, span.status) - - -def _reads_langfuse_attributes(spec: ExporterSpec, config: OpenTelemetryV2Config) -> bool: - if spec.owner is not None: - return spec.owner is ExporterOwner.LANGFUSE_OTEL - return "langfuse" in config.mapper_names and all( - other.owner is not ExporterOwner.LANGFUSE_OTEL for other in config.exporters - ) - - class TenantFanOutSpanProcessor(SpanProcessor): """Export every finished span to each destination this request resolved. @@ -796,23 +774,6 @@ class _OverriddenBackendFilter(SpanProcessor): return self._inner.force_flush(timeout_millis) -class _LangfuseAttributeFilter(SpanProcessor): - def __init__(self, inner: SpanProcessor) -> None: - self._inner: Final = inner - - def on_start(self, span: SDKSpan, parent_context: Context | None = None) -> None: - self._inner.on_start(span, parent_context) - - def on_end(self, span: ReadableSpan) -> None: - self._inner.on_end(_without_langfuse_attributes(span)) - - def shutdown(self) -> None: - self._inner.shutdown() - - def force_flush(self, timeout_millis: int = 30000) -> bool: - return self._inner.force_flush(timeout_millis) - - def build_span_exporter(config: OpenTelemetryV2Config) -> SpanExporter: """Build a single exporter from the top-level config fields. @@ -1099,10 +1060,9 @@ def build_tracer_provider( exp, (spec.use_simple_processor if spec.use_simple_processor is not None else use_simple_processor), ) - sink = processor if _reads_langfuse_attributes(spec, config) else _LangfuseAttributeFilter(processor) owner = spec.owner.value if spec.owner is not None else None provider.add_span_processor( - _OverriddenBackendFilter(sink, owner) if tenant_overrides and owner is not None else sink + _OverriddenBackendFilter(processor, owner) if tenant_overrides and owner is not None else processor ) return provider diff --git a/litellm/integrations/otel/presets/agentops.py b/litellm/integrations/otel/presets/agentops.py index 965213f2ee4..f6d85dd80d0 100644 --- a/litellm/integrations/otel/presets/agentops.py +++ b/litellm/integrations/otel/presets/agentops.py @@ -53,7 +53,6 @@ def agentops_preset( return base.model_copy( update={ "exporters": [ - *base.exporters, ExporterSpec( kind=_AGENTOPS_EXPORTER_KIND, endpoint=_AGENTOPS_ENDPOINT, diff --git a/litellm/integrations/otel/presets/arize.py b/litellm/integrations/otel/presets/arize.py index d7ce87f5552..6ac2c1a2d20 100644 --- a/litellm/integrations/otel/presets/arize.py +++ b/litellm/integrations/otel/presets/arize.py @@ -35,7 +35,6 @@ def arize_preset( return base.model_copy( update={ "exporters": [ - *base.exporters, ExporterSpec( kind=arize_cfg.protocol or "otlp_grpc", endpoint=arize_cfg.endpoint or "https://otlp.arize.com/v1", diff --git a/litellm/integrations/otel/presets/langfuse.py b/litellm/integrations/otel/presets/langfuse.py index 9149e0c0d94..bea050d1d77 100644 --- a/litellm/integrations/otel/presets/langfuse.py +++ b/litellm/integrations/otel/presets/langfuse.py @@ -10,10 +10,7 @@ from litellm.integrations.otel.model.config import ( ExporterSpec, OpenTelemetryV2Config, ) -from litellm.integrations.otel.presets.utils import ( - credential_gated_exporters, - ensure_mappers, -) +from litellm.integrations.otel.presets.utils import ensure_mappers from litellm.types.utils import StandardCallbackDynamicParams @@ -31,7 +28,7 @@ def langfuse_preset( raise return base.model_copy( update={ # mutable-ok: pydantic model_copy takes a plain update mapping - "exporters": credential_gated_exporters(base.exporters, ExporterOwner.LANGFUSE_OTEL), + "exporters": [ExporterSpec(owner=ExporterOwner.LANGFUSE_OTEL, requires_headers=True)], "mapper_names": mappers, } ) @@ -39,7 +36,6 @@ def langfuse_preset( return base.model_copy( update={ "exporters": [ - *base.exporters, ExporterSpec( kind=kind, endpoint=cfg.endpoint, diff --git a/litellm/integrations/otel/presets/levo.py b/litellm/integrations/otel/presets/levo.py index c1580cf7a5b..e1ea44f585f 100644 --- a/litellm/integrations/otel/presets/levo.py +++ b/litellm/integrations/otel/presets/levo.py @@ -20,7 +20,6 @@ def levo_preset( return base.model_copy( update={ "exporters": [ - *base.exporters, ExporterSpec( kind="otlp_http", endpoint=cfg.endpoint, diff --git a/litellm/integrations/otel/presets/newrelic.py b/litellm/integrations/otel/presets/newrelic.py index 771b3f643c8..298eee2e86c 100644 --- a/litellm/integrations/otel/presets/newrelic.py +++ b/litellm/integrations/otel/presets/newrelic.py @@ -54,7 +54,6 @@ def newrelic_preset( return base.model_copy( update={ "exporters": [ - *base.exporters, ExporterSpec( kind="otlp_http", endpoint=endpoint, diff --git a/litellm/integrations/otel/presets/phoenix.py b/litellm/integrations/otel/presets/phoenix.py index f4f34ee7525..e3273c6b9c2 100644 --- a/litellm/integrations/otel/presets/phoenix.py +++ b/litellm/integrations/otel/presets/phoenix.py @@ -69,7 +69,6 @@ def phoenix_preset( return base.model_copy( update={ "exporters": [ - *base.exporters, ExporterSpec( kind=cfg.protocol if hasattr(cfg, "protocol") else "otlp_http", endpoint=cfg.endpoint, diff --git a/litellm/integrations/otel/presets/utils.py b/litellm/integrations/otel/presets/utils.py index 1270c41e77b..328569d3daf 100644 --- a/litellm/integrations/otel/presets/utils.py +++ b/litellm/integrations/otel/presets/utils.py @@ -3,8 +3,6 @@ from collections.abc import Iterable from typing import Final -from litellm.integrations.otel.model.config import ExporterOwner, ExporterSpec - def ensure_mappers(mapper_names: Iterable[str], *names: str) -> list[str]: """Return ``mapper_names`` with each of ``names`` appended if not already present. @@ -17,32 +15,3 @@ def ensure_mappers(mapper_names: Iterable[str], *names: str) -> list[str]: if name not in result: result.append(name) return result - - -def credential_gated_exporters( - exporters: "Iterable[ExporterSpec]", owner: "ExporterOwner" -) -> "tuple[ExporterSpec, ...]": - """``exporters`` with the operator's destination replaced by a header-gated one. - - Used when a credential-mandatory backend is asked to build without the operator's - own credentials, so only key/team destinations receive spans. Two things have to - happen for that to mean "export nowhere": the placeholder console spec that - ``OpenTelemetryV2Config`` folds in for an empty exporter list is dropped, or every - span would be printed to stdout, and the gated spec keeps the owner so the - override filter still recognises which backend this provider speaks for. - """ - return ( - *(spec for spec in exporters if not is_unconfigured_placeholder(spec)), - ExporterSpec(owner=owner, requires_headers=True), - ) - - -def is_unconfigured_placeholder(spec: "ExporterSpec") -> bool: - """Whether ``spec`` is the one ``_normalize`` folds in when nothing was configured. - - No field set is what says the operator asked for nothing: an exporter they did - configure survives, even ``OTEL_EXPORTER=console`` whose value matches the default, - and so does the gated spec this module appends, which would otherwise eat itself - when one preset layers onto another. - """ - return not spec.model_fields_set diff --git a/litellm/integrations/otel/presets/weave.py b/litellm/integrations/otel/presets/weave.py index 644cd39ad36..41567758d7e 100644 --- a/litellm/integrations/otel/presets/weave.py +++ b/litellm/integrations/otel/presets/weave.py @@ -7,13 +7,10 @@ from litellm.integrations.otel.model.config import ( ExporterSpec, OpenTelemetryV2Config, ) -from litellm.integrations.otel.presets.utils import ( - credential_gated_exporters, - ensure_mappers, -) +from litellm.integrations.otel.presets.utils import ensure_mappers from litellm.integrations.weave.weave_otel import ( _get_weave_authorization_header, - get_weave_otel_config, + read_weave_otel_config, ) from litellm.types.utils import StandardCallbackDynamicParams @@ -26,20 +23,19 @@ def weave_preset( base: Final = config_overrides or OpenTelemetryV2Config() mappers: Final = ensure_mappers(base.mapper_names, "openinference", "weave") try: - weave_cfg: Final = get_weave_otel_config() + weave_cfg: Final = read_weave_otel_config() except Exception: if not allow_missing_credentials: raise return base.model_copy( update={ # mutable-ok: pydantic model_copy takes a plain update mapping - "exporters": credential_gated_exporters(base.exporters, ExporterOwner.WEAVE_OTEL), + "exporters": [ExporterSpec(owner=ExporterOwner.WEAVE_OTEL, requires_headers=True)], "mapper_names": mappers, } ) return base.model_copy( update={ "exporters": [ - *base.exporters, ExporterSpec( kind=weave_cfg.protocol or "otlp_http", endpoint=weave_cfg.endpoint, diff --git a/litellm/integrations/weave/weave_otel.py b/litellm/integrations/weave/weave_otel.py index 50289263f38..bb68a867068 100644 --- a/litellm/integrations/weave/weave_otel.py +++ b/litellm/integrations/weave/weave_otel.py @@ -125,17 +125,8 @@ def weave_otel_endpoint(host: str | None) -> str: return normalized.rstrip("/") + WEAVE_OTEL_ENDPOINT -def get_weave_otel_config() -> WeaveOtelConfig: - """ - Retrieves the Weave OpenTelemetry configuration based on environment variables. - - Environment Variables: - WANDB_API_KEY: Required. W&B API key for authentication. - WANDB_PROJECT_ID: Required. Project ID in format /. - WANDB_HOST: Optional. Custom Weave host URL. Defaults to cloud endpoint. - - Returns: - WeaveOtelConfig: A Pydantic model containing Weave OTEL configuration. +def read_weave_otel_config() -> WeaveOtelConfig: + """Weave OTLP settings from ``WANDB_API_KEY``, ``WANDB_PROJECT_ID`` and optional ``WANDB_HOST``. Raises: ValueError: If required environment variables are missing. @@ -158,10 +149,6 @@ def get_weave_otel_config() -> WeaveOtelConfig: auth_header: Final = _get_weave_authorization_header(api_key=api_key) otlp_auth_headers: Final = f"Authorization={auth_header},project_id={project_id}" - # Set standard OTEL environment variables - os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = endpoint - os.environ["OTEL_EXPORTER_OTLP_HEADERS"] = otlp_auth_headers - return WeaveOtelConfig( otlp_auth_headers=otlp_auth_headers, endpoint=endpoint, @@ -170,6 +157,14 @@ def get_weave_otel_config() -> WeaveOtelConfig: ) +def get_weave_otel_config() -> WeaveOtelConfig: + """``read_weave_otel_config`` plus the v1 side effect of publishing it as the process-wide ``OTEL_*`` env.""" + config: Final = read_weave_otel_config() + os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = config.endpoint + os.environ["OTEL_EXPORTER_OTLP_HEADERS"] = config.otlp_auth_headers + return config + + def set_weave_otel_attributes(span: Span, kwargs: Mapping[str, object], response_obj: object): """ Sets OpenTelemetry span attributes for Weave observability. diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 2d3a99abe81..5def8368794 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -4499,7 +4499,7 @@ def _init_custom_logger_compatible_class( from litellm.integrations.otel.model.config import OpenTelemetryV2Config for callback in _in_memory_loggers: - if isinstance(callback, OpenTelemetryV2): + if isinstance(callback, OpenTelemetryV2) and callback.callback_name is None: return callback otel_settings: Final = _get_custom_logger_settings_from_proxy_server(callback_name=logging_integration) otel_logger_v2: Final = build_otel_v2_logger( @@ -4862,13 +4862,13 @@ def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[Custom Returns ``None`` when V2 is off OR when there's no preset registered for ``callback_name`` — callers should then fall through to the legacy path. - A preset that needs operator credentials it cannot find is allowed to build - only when this request has a key/team destination for that backend and another - V2 logger is already registered to carry the fan-out. The resulting logger keeps - only its credential-gated exporter, while the registered logger owns operator - delivery. Without that carrier, a preset that raises or that ends up with nothing - but its gated exporter and the default console placeholder returns ``None``, so the - caller falls through to the legacy path exactly as before V2 landed. + Each preset exports only to the destination it owns; the operator's ``OTEL_*`` + collector is served by the ``otel`` callback alone. A preset that needs operator + credentials it cannot find is allowed to build only when this request has a + key/team destination for that backend and another V2 logger is already registered + to carry the fan-out. Without that carrier, a preset that raises or that has nothing + but its credential-gated exporter returns ``None``, so the caller falls through to + the legacy path exactly as before V2 landed. """ from litellm.integrations.otel.model.config import is_otel_v2_enabled @@ -4887,21 +4887,20 @@ def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[Custom for callback in _in_memory_loggers: if ( isinstance(callback, OpenTelemetryV2) - and getattr(callback, "callback_name", None) == callback_name + and callback.callback_name == callback_name and (serves_a_destination or not _exports_nowhere(callback.config)) ): return callback try: - built: Final = preset_fn(allow_missing_credentials=carried) + config: Final = preset_fn(allow_missing_credentials=carried) except Exception: # If env vars are missing or the preset raises, defer to the legacy path # so customers get the same error story they had before V2 landed. return None - gated: Final = _is_credential_gated(built) - if gated and not carried and not _has_operator_exporter(built): + exports_nowhere: Final = _exports_nowhere(config) + if exports_nowhere and not carried: return None - config: Final = _only_the_gated_exporter(built) if gated and carried else built - if _exports_nowhere(config): + if exports_nowhere: verbose_logger.warning( "OTel V2: no operator credentials for '%s'; only key/team destinations will receive its traces", callback_name, @@ -4916,24 +4915,6 @@ def _exports_nowhere(config: "OpenTelemetryV2Config") -> bool: return all(_is_gated(spec) for spec in config.exporters) -def _is_credential_gated(config: "OpenTelemetryV2Config") -> bool: - """Whether the preset built without the operator's own credentials for its backend.""" - return any(_is_gated(spec) for spec in config.exporters) - - -def _has_operator_exporter(config: "OpenTelemetryV2Config") -> bool: - """Whether the operator configured somewhere real to export, beyond the default console placeholder.""" - from litellm.integrations.otel.presets.utils import is_unconfigured_placeholder - - return any(not _is_gated(spec) and not is_unconfigured_placeholder(spec) for spec in config.exporters) - - -def _only_the_gated_exporter(config: "OpenTelemetryV2Config") -> "OpenTelemetryV2Config": - return config.model_copy( - update={"exporters": [spec for spec in config.exporters if _is_gated(spec)]} # mutable-ok: model_copy update - ) - - def _is_gated(spec: "ExporterSpec") -> bool: return spec.requires_headers and not spec.headers diff --git a/tests/test_litellm/integrations/otel/test_langfuse_logger.py b/tests/test_litellm/integrations/otel/test_langfuse_logger.py index 8db84b090a0..65302963d31 100644 --- a/tests/test_litellm/integrations/otel/test_langfuse_logger.py +++ b/tests/test_litellm/integrations/otel/test_langfuse_logger.py @@ -374,6 +374,53 @@ def test_unnamed_request_leaves_the_trace_name_off_both_spans(): assert TRACE_NAME_ATTR not in root_attrs and TRACE_NAME_ATTR not in generation_attrs +def test_a_root_owned_by_another_loggers_provider_is_never_stamped_with_langfuse_vocabulary(): + """``callbacks: [otel, langfuse_otel]``: the request root belongs to the ``otel`` logger and exports to the + operator's collector, so Langfuse's trace name and root input/output must not be written on it.""" + generic, generic_exporter = _logger(mappers=("genai",)) + langfuse, langfuse_exporter = _logger() + response = ModelResponse(choices=[Choices(message=Message(role="assistant", content="pong"))]) + named: Final = {"proxy_server_request": {"headers": {"langfuse_trace_name": "private-name"}}} + + root = _start_root(generic) + asyncio.run(langfuse.async_pre_call_hook(UserAPIKeyAuth(), DualCache(), CHAT_DATA, "acompletion")) + langfuse.log_pre_api_call( + model="gpt-5.4-mini", messages=[], kwargs={"litellm_call_id": "c1", "litellm_params": named} + ) + asyncio.run( + langfuse.async_post_call_success_hook(data=CHAT_DATA, user_api_key_dict=UserAPIKeyAuth(), response=response) + ) + root.end() + + root_attrs = _root_attrs(generic_exporter) + assert not any(key.startswith("langfuse.") for key in root_attrs) + assert all(span.name != LITELLM_PROXY_REQUEST_SPAN_NAME for span in langfuse_exporter.get_finished_spans()) + + +def test_a_root_owned_by_the_langfuse_provider_is_still_named_and_given_its_io_next_to_another_logger(): + """``callbacks: [langfuse_otel]`` with a second v2 logger alongside: the root was created by the Langfuse + provider, so the ownership check must not stop the trace name and root input/output from reaching Langfuse.""" + _logger(mappers=("genai",)) + langfuse, langfuse_exporter = _logger() + response = ModelResponse(choices=[Choices(message=Message(role="assistant", content="pong"))]) + named: Final = {"proxy_server_request": {"headers": {"langfuse_trace_name": "private-name"}}} + + root = _start_root(langfuse) + asyncio.run(langfuse.async_pre_call_hook(UserAPIKeyAuth(), DualCache(), CHAT_DATA, "acompletion")) + langfuse.log_pre_api_call( + model="gpt-5.4-mini", messages=[], kwargs={"litellm_call_id": "c1", "litellm_params": named} + ) + asyncio.run( + langfuse.async_post_call_success_hook(data=CHAT_DATA, user_api_key_dict=UserAPIKeyAuth(), response=response) + ) + root.end() + + root_attrs = _root_attrs(langfuse_exporter) + assert root_attrs[TRACE_NAME_ATTR] == "private-name" + assert json.loads(root_attrs[INPUT_ATTR]) == CHAT_DATA["messages"] + assert "pong" in root_attrs[OUTPUT_ATTR] + + @pytest.mark.parametrize( ("capture", "mappers"), [("no_content", ("genai", "langfuse")), ("span_only", ("genai",))], diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index b2c79d43f29..f0dad6a5927 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -3,14 +3,14 @@ import contextvars import time from base64 import b64encode -from collections.abc import Mapping +from collections.abc import Callable, Mapping from functools import reduce from types import MappingProxyType import pytest from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export import SimpleSpanProcessor, SpanExporter from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter from opentelemetry.trace import Status, StatusCode @@ -22,6 +22,7 @@ from litellm.integrations.otel.logger import ( build_otel_v2_logger, fan_out_provider, publish_global_otel_v2_provider, + select_global_otel_v2_logger, ) from litellm.integrations.otel.model.config import ( ExporterOwner, @@ -51,6 +52,11 @@ from litellm.integrations.otel.presets.destinations import ( destination_for, ) from litellm.integrations.otel.presets.langfuse import langfuse_preset +from litellm.litellm_core_utils import litellm_logging +from litellm.litellm_core_utils.litellm_logging import ( + _init_custom_logger_compatible_class, + _maybe_construct_otel_v2, +) from litellm.proxy._types import AddTeamCallback, UserAPIKeyAuth from litellm.proxy.litellm_pre_call_utils import ( convert_key_logging_metadata_to_callback, @@ -92,6 +98,12 @@ def emit(provider: TracerProvider, name: str = "chat gpt-4") -> None: pass +def swap_exporter_factories(monkeypatch, **factories: Callable[[ExporterSpec], SpanExporter]) -> None: + """Swap the exporter built for each ``kind`` for the duration of one test.""" + for kind, factory in factories.items(): + monkeypatch.setitem(otel_providers._EXPORTER_FACTORIES, kind, factory) + + def wired_provider(dest_exporter: InMemorySpanExporter, global_exporter: InMemorySpanExporter) -> TracerProvider: """The operator's provider: one owned exporter plus the tenant fan-out.""" provider = TracerProvider() @@ -516,34 +528,6 @@ class TestFanOut: assert [s.name for s in langfuse.get_finished_spans()] == ["chat gpt-4"] assert [s.name for s in arize.get_finished_spans()] == ["chat gpt-4"] - def test_only_the_langfuse_destination_receives_the_langfuse_attributes(self): - langfuse, arize = InMemorySpanExporter(), InMemorySpanExporter() - by_endpoint = {"http://a.local": langfuse, "http://b.local": arize} - provider = TracerProvider() - provider.add_span_processor( - TenantFanOutSpanProcessor(processor_factory=lambda d: SimpleSpanProcessor(by_endpoint[d.endpoint])) - ) - - def run(): - set_request_destinations( - ( - OtelDestination(endpoint="http://a.local", callback_name="langfuse_otel"), - OtelDestination(endpoint="http://b.local", callback_name="arize"), - ) - ) - span = get_tracer(provider, "litellm").start_span("chat gpt-4") - span.set_attribute("gen_ai.request.model", "gpt-4") - span.set_attribute("langfuse.trace.name", "private-langfuse-only-name") - span.end() - - in_fresh_context(run) - - (langfuse_span,) = langfuse.get_finished_spans() - (arize_span,) = arize.get_finished_spans() - assert langfuse_span.attributes["langfuse.trace.name"] == "private-langfuse-only-name" - assert arize_span.attributes["gen_ai.request.model"] == "gpt-4" - assert [k for k in arize_span.attributes if k.startswith("langfuse.")] == [] - def test_a_destination_carries_the_tenants_service_name(self): """An overridden backend skips per-request tracer routing, so the service name that route used to apply has to travel on the destination instead.""" @@ -1473,8 +1457,6 @@ class TestPresetDegradation: langfuse_preset() def test_a_credential_less_proxy_builds_the_gated_logger_beside_a_v2_carrier(self, monkeypatch): - from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 - credential_less_proxy(monkeypatch) monkeypatch.setenv("LITELLM_OTEL_V2", "true") carrier = build_otel_v2_logger(OpenTelemetryV2Config(exporter="in_memory")) @@ -1494,8 +1476,6 @@ class TestPresetDegradation: """Nothing can use a credential-less langfuse here, so the operator has to get the same story as before v2: the legacy integration, not a global provider that exports nowhere.""" - from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 - credential_less_proxy(monkeypatch) monkeypatch.setenv("LITELLM_OTEL_V2", "true") @@ -1505,9 +1485,10 @@ class TestPresetDegradation: assert logger is None - def test_a_valid_newrelic_base_exporter_survives_without_a_license_key(self, monkeypatch): - from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 - + def test_a_credentialless_newrelic_does_not_take_over_the_operator_collector(self, monkeypatch): + """The ``OTEL_*`` collector belongs to the ``otel`` callback. A New Relic + entry with no license key has nowhere of its own to export, so it takes + the legacy path instead of shipping New Relic-shaped spans to the collector.""" monkeypatch.delenv("NEW_RELIC_LICENSE_KEY", raising=False) monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.local:4318") monkeypatch.setenv("LITELLM_OTEL_V2", "true") @@ -1516,15 +1497,21 @@ class TestPresetDegradation: logger = in_fresh_context(_maybe_construct_otel_v2, "newrelic", []) is_otel_v2_enabled.cache_clear() + assert logger is None + + def test_a_credentialed_newrelic_exports_only_to_new_relic(self, monkeypatch): + monkeypatch.setenv("NEW_RELIC_LICENSE_KEY", "nr-license") + monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.local:4318") + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + + is_otel_v2_enabled.cache_clear() + logger = in_fresh_context(_maybe_construct_otel_v2, "newrelic", []) + is_otel_v2_enabled.cache_clear() + assert logger is not None - assert [spec.endpoint for spec in logger.config.exporters] == [ - "http://collector.local:4318", - "https://otlp.nr-data.net", - ] + assert [spec.endpoint for spec in logger.config.exporters] == ["https://otlp.nr-data.net"] def test_a_credentialless_newrelic_without_a_base_exporter_falls_back(self, monkeypatch): - from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 - monkeypatch.delenv("NEW_RELIC_LICENSE_KEY", raising=False) for name in _OTEL_SHORTHAND_ENV: monkeypatch.delenv(name, raising=False) @@ -1536,12 +1523,10 @@ class TestPresetDegradation: assert logger is None - def test_an_explicit_console_exporter_keeps_a_credentialless_preset_on_v2(self, monkeypatch, capfd): - """``OTEL_EXPORTER=console`` reads exactly like the placeholder ``_normalize`` - folds in, but the operator asked for it, so a credential-less New Relic keeps - the V2 logger and its spans reach stdout instead of the legacy path.""" - from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 - + def test_an_explicit_console_exporter_is_served_by_otel_not_by_a_credentialless_preset(self, monkeypatch): + """``OTEL_EXPORTER=console`` is the operator's generic destination. Only the + ``otel`` callback prints there; a credential-less New Relic has nothing of + its own to export to and takes the legacy path.""" monkeypatch.delenv("NEW_RELIC_LICENSE_KEY", raising=False) for name in _OTEL_SHORTHAND_ENV: monkeypatch.delenv(name, raising=False) @@ -1550,15 +1535,13 @@ class TestPresetDegradation: is_otel_v2_enabled.cache_clear() logger = in_fresh_context(_maybe_construct_otel_v2, "newrelic", []) + generic = build_otel_v2_logger(OpenTelemetryV2Config()) is_otel_v2_enabled.cache_clear() - assert logger is not None - assert logger.config.exporters[0].kind == "console" - assert not logger.config.exporters[0].requires_headers + assert logger is None + assert [spec.kind for spec in generic.config.exporters] == ["console"] def test_a_destination_for_one_backend_does_not_degrade_another(self, monkeypatch): - from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 - credential_less_proxy(monkeypatch) monkeypatch.delenv("WANDB_API_KEY", raising=False) monkeypatch.setenv("LITELLM_OTEL_V2", "true") @@ -1576,8 +1559,6 @@ class TestPresetDegradation: def test_the_exporter_less_logger_is_not_reused_by_a_request_without_destinations(self, monkeypatch): """Reusing it would let one team's destination decide how every later request without one is logged, long after the degrade was justified.""" - from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 - credential_less_proxy(monkeypatch) monkeypatch.setenv("LITELLM_OTEL_V2", "true") loggers = [build_otel_v2_logger(OpenTelemetryV2Config(exporter="in_memory"))] @@ -1595,8 +1576,6 @@ class TestPresetDegradation: assert plain is None def test_a_credentialed_logger_is_still_reused_across_requests(self, monkeypatch): - from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 - monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-lf-1") monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-lf-1") monkeypatch.setenv("LITELLM_OTEL_V2", "true") @@ -1612,8 +1591,6 @@ class TestPresetDegradation: @staticmethod def _degraded_langfuse_beside(loggers, monkeypatch): - from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 - credential_less_proxy(monkeypatch) monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.local:4318") monkeypatch.setenv("LITELLM_OTEL_V2", "true") @@ -1643,8 +1620,6 @@ class TestPresetDegradation: """Only a V2 logger publishes the provider the fan-out rides on, so a legacy callback beside this one leaves the destination just as unreachable as no callback at all, and the operator keeps the pre-V2 story.""" - from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 - credential_less_proxy(monkeypatch) monkeypatch.setenv("LITELLM_OTEL_V2", "true") @@ -1658,28 +1633,24 @@ class TestPresetDegradation: assert logger is None - def test_a_credentialed_logger_beside_another_v2_logger_keeps_every_exporter(self, monkeypatch): - """Only a degraded preset gives the collector up; an operator who configured - both the backend and the collector still exports to both, as on base.""" - from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 - + @pytest.mark.parametrize("beside", [(), ("collector",)]) + def test_a_credentialed_langfuse_exports_only_to_langfuse(self, monkeypatch, beside): + """The operator's collector is the ``otel`` callback's destination whether or + not that callback is registered; Langfuse never inherits it.""" monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-lf-1") monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-lf-1") monkeypatch.setenv("LANGFUSE_HOST", "https://cloud.langfuse.com") monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.local:4318") monkeypatch.setenv("LITELLM_OTEL_V2", "true") - collector_logger = build_otel_v2_logger(OpenTelemetryV2Config(exporter="in_memory")) + loggers = [build_otel_v2_logger(OpenTelemetryV2Config(exporter="in_memory")) for _ in beside] is_otel_v2_enabled.cache_clear() - logger = in_fresh_context(_maybe_construct_otel_v2, "langfuse_otel", [collector_logger]) + logger = in_fresh_context(_maybe_construct_otel_v2, "langfuse_otel", loggers) is_otel_v2_enabled.cache_clear() assert logger is not None - assert [spec.endpoint for spec in logger.config.exporters] == [ - "http://collector.local:4318", - "https://cloud.langfuse.com/api/public/otel", - ] - assert all(spec.headers for spec in logger.config.exporters if spec.requires_headers) + assert [spec.endpoint for spec in logger.config.exporters] == ["https://cloud.langfuse.com/api/public/otel"] + assert all(spec.headers for spec in logger.config.exporters) class TestContextIsolation: @@ -1692,26 +1663,184 @@ class TestContextIsolation: assert in_fresh_context(request_destinations) == () -class TestOperatorShorthandSurvivesDegradation: - def test_a_generic_otlp_collector_keeps_receiving_when_langfuse_has_no_credentials(self, monkeypatch): - """Only the stdout placeholder is dropped. An operator who set the standard - OTLP env vars configured a real destination and must keep it.""" +class TestOperatorShorthandStaysWithOtel: + def test_a_credential_less_langfuse_leaves_the_operator_collector_to_the_otel_callback(self, monkeypatch): monkeypatch.delenv("LANGFUSE_PUBLIC_KEY", raising=False) monkeypatch.delenv("LANGFUSE_SECRET_KEY", raising=False) monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.local:4318") config = langfuse_preset(allow_missing_credentials=True) - assert [spec.endpoint for spec in config.exporters] == ["http://collector.local:4318", None] - assert [spec.kind for spec in config.exporters] == ["otlp_http", "console"] + assert [spec.endpoint for spec in config.exporters] == [None] + assert [spec.owner for spec in config.exporters] == [ExporterOwner.LANGFUSE_OTEL] - def test_the_stdout_placeholder_is_still_dropped_when_it_is_the_only_exporter(self, monkeypatch): + def test_the_stdout_placeholder_is_never_inherited_by_a_credential_less_preset(self, monkeypatch): credential_less_proxy(monkeypatch) config = langfuse_preset(allow_missing_credentials=True) assert all(spec.requires_headers and not spec.headers for spec in config.exporters) + def test_a_lone_credentialed_preset_without_otel_env_prints_nothing_to_stdout(self, monkeypatch): + """``callbacks: [langfuse_otel]`` and no ``OTEL_*`` at all: the base config's stdout placeholder + belongs to ``otel``, so the Langfuse logger must not dump every span as JSON on the proxy log.""" + for name in _OTEL_SHORTHAND_ENV: + monkeypatch.delenv(name, raising=False) + for name, value in _VENDOR_ENV["langfuse_otel"].items(): + monkeypatch.setenv(name, value) + langfuse_sink, stdout = InMemorySpanExporter(), InMemorySpanExporter() + swap_exporter_factories(monkeypatch, otlp_http=lambda spec: langfuse_sink, console=lambda spec: stdout) + + logger = build_otel_v2_logger(langfuse_preset(), callback_name="langfuse_otel") + emit(logger.tracer_provider, "vendor only") + logger.tracer_provider.force_flush() + + assert [s.name for s in langfuse_sink.get_finished_spans()] == ["vendor only"] + assert stdout.get_finished_spans() == () + + +_COLLECTOR = "http://collector.local:4318" + +#: Env that gives each endpoint-owning preset its own credentials. +_VENDOR_ENV = MappingProxyType( + { + "langfuse_otel": { + "LANGFUSE_PUBLIC_KEY": "pk", + "LANGFUSE_SECRET_KEY": "sk", + "LANGFUSE_HOST": "https://lf.local", + }, + "arize": {"ARIZE_SPACE_ID": "space", "ARIZE_API_KEY": "key"}, + "arize_phoenix": { + "PHOENIX_API_KEY": "key", + "PHOENIX_COLLECTOR_HTTP_ENDPOINT": "https://phoenix.local/v1/traces", + }, + "newrelic": {"NEW_RELIC_LICENSE_KEY": "license"}, + "agentops": {"AGENTOPS_API_KEY": "ao-key"}, + "levo": { + "LEVOAI_API_KEY": "k", + "LEVOAI_ORG_ID": "org", + "LEVOAI_WORKSPACE_ID": "ws", + "LEVOAI_COLLECTOR_URL": "https://levo.local/v1/traces", + }, + "weave_otel": {"WANDB_API_KEY": "wandb", "WANDB_PROJECT_ID": "entity/project"}, + } +) + + +class TestDestinationOwnership: + """A callback exports only to the destination it owns: ``otel`` serves the operator's + ``OTEL_*`` collector and a preset serves its own backend. The leak this closes was every + preset inheriting the collector from the base config, so its vendor-vocabulary spans + (``langfuse.*``, OpenInference, ...) landed on the operator's collector too.""" + + @staticmethod + def _capture_exporters(monkeypatch) -> dict[str, InMemorySpanExporter]: + sinks: dict[str, InMemorySpanExporter] = {} + + def factory(spec: ExporterSpec) -> InMemorySpanExporter: + return sinks.setdefault(spec.endpoint or "", InMemorySpanExporter()) + + swap_exporter_factories(monkeypatch, otlp_http=factory, otlp_grpc=factory, agentops=factory) + return sinks + + @staticmethod + def _operator_with(monkeypatch, vendor: str) -> None: + for name in ("LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY", *_OTEL_SHORTHAND_ENV): + monkeypatch.delenv(name, raising=False) + for name, value in _VENDOR_ENV[vendor].items(): + monkeypatch.setenv(name, value) + monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", _COLLECTOR) + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + + @pytest.mark.parametrize("vendor", sorted(_VENDOR_ENV)) + def test_a_presets_spans_never_reach_the_operator_collector(self, monkeypatch, vendor): + sinks = self._capture_exporters(monkeypatch) + self._operator_with(monkeypatch, vendor) + generic = build_otel_v2_logger(OpenTelemetryV2Config()) + preset = in_fresh_context(_maybe_construct_otel_v2, vendor, [generic]) + is_otel_v2_enabled.cache_clear() + assert preset is not None + + emit(generic.tracer_provider, "from otel") + emit(preset.tracer_provider, "from preset") + generic.tracer_provider.force_flush() + preset.tracer_provider.force_flush() + + assert [s.name for s in sinks[_COLLECTOR].get_finished_spans()] == ["from otel"] + vendor_sinks = {endpoint: sink for endpoint, sink in sinks.items() if endpoint != _COLLECTOR} + assert [[s.name for s in sink.get_finished_spans()] for sink in vendor_sinks.values()] == [["from preset"]] + + @pytest.mark.parametrize("vendor", sorted(_VENDOR_ENV)) + def test_a_preset_built_first_does_not_point_otel_at_its_own_backend(self, monkeypatch, vendor): + """``callbacks: [, otel]``: reading the vendor's credentials must not rewrite the + process-wide ``OTEL_*`` env, or the ``otel`` built next inherits the vendor's endpoint and auth.""" + sinks = self._capture_exporters(monkeypatch) + self._operator_with(monkeypatch, vendor) + preset = in_fresh_context(_maybe_construct_otel_v2, vendor, []) + generic = build_otel_v2_logger(OpenTelemetryV2Config()) + is_otel_v2_enabled.cache_clear() + assert preset is not None + + emit(generic.tracer_provider, "from otel") + generic.tracer_provider.force_flush() + + assert [(spec.endpoint, spec.headers) for spec in generic.config.exporters] == [(_COLLECTOR, None)] + assert [s.name for s in sinks[_COLLECTOR].get_finished_spans()] == ["from otel"] + + @pytest.mark.parametrize("otel_first", [True, False]) + def test_the_otel_entry_never_reuses_a_preset_logger_whatever_the_order(self, monkeypatch, otel_first): + from litellm.proxy import proxy_server + + self._operator_with(monkeypatch, "langfuse_otel") + monkeypatch.setattr(proxy_server, "open_telemetry_logger", None) + loggers: list[CustomLogger] = [] + monkeypatch.setattr(litellm_logging, "_in_memory_loggers", loggers) + + def build(name): + if name == "otel": + return _init_custom_logger_compatible_class("otel", None, None) + return _maybe_construct_otel_v2(name, loggers) + + order = ("otel", "langfuse_otel") if otel_first else ("langfuse_otel", "otel") + built = {name: in_fresh_context(build, name) for name in order} + is_otel_v2_enabled.cache_clear() + + generic, preset = built["otel"], built["langfuse_otel"] + assert isinstance(generic, OpenTelemetryV2) and isinstance(preset, OpenTelemetryV2) + assert generic is not preset + assert generic.callback_name is None and preset.callback_name == "langfuse_otel" + assert [spec.endpoint for spec in generic.config.exporters] == [_COLLECTOR] + assert [spec.endpoint for spec in preset.config.exporters] == ["https://lf.local/api/public/otel"] + assert select_global_otel_v2_logger(loggers) is generic + assert proxy_server.open_telemetry_logger is generic + + @pytest.mark.parametrize("otel_first", [True, False]) + def test_the_otel_callback_holds_the_proxy_slot_whatever_the_order(self, monkeypatch, otel_first): + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "open_telemetry_logger", None) + config = OpenTelemetryV2Config(exporters=[ExporterSpec(kind="in_memory")]) + + def build(name): + return build_otel_v2_logger(config, callback_name=None if name == "otel" else name) + + order = ("otel", "langfuse_otel") if otel_first else ("langfuse_otel", "otel") + built = {name: build(name) for name in order} + + assert proxy_server.open_telemetry_logger is built["otel"] + + def test_a_lone_preset_keeps_the_proxy_slot_and_the_global(self, monkeypatch): + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "open_telemetry_logger", None) + config = OpenTelemetryV2Config(exporters=[ExporterSpec(kind="in_memory")]) + first = build_otel_v2_logger(config, callback_name="langfuse_otel") + second = build_otel_v2_logger(config, callback_name="arize") + + assert proxy_server.open_telemetry_logger is first + assert select_global_otel_v2_logger([first, second]) is first + class TestBackendEndpointParity: def test_arize_follows_its_own_http_endpoint_instead_of_the_grpc_default(self, monkeypatch): @@ -2613,75 +2742,6 @@ class TestEvictionSafety: assert held.shutdown_calls == 1 -class TestCredentialGatedExporters: - def test_layering_a_second_preset_does_not_eat_the_first_gated_exporter(self, monkeypatch): - """``base.Preset`` advertises ``config_overrides`` layering, and the gated spec - is itself a console exporter with no endpoint.""" - credential_less_proxy(monkeypatch) - from litellm.integrations.otel.presets.utils import credential_gated_exporters - - once = credential_gated_exporters((), ExporterOwner.LANGFUSE_OTEL) - twice = credential_gated_exporters(once, ExporterOwner.WEAVE_OTEL) - - assert [spec.owner for spec in twice] == [ExporterOwner.LANGFUSE_OTEL, ExporterOwner.WEAVE_OTEL] - - def test_an_exporter_the_operator_configured_survives(self): - from litellm.integrations.otel.presets.utils import credential_gated_exporters - - operator_console = ExporterSpec(kind="console", use_simple_processor=True) - - kept = credential_gated_exporters((operator_console,), ExporterOwner.LANGFUSE_OTEL) - - assert kept[0] == operator_console - - def test_an_otlp_exporter_on_its_default_endpoint_survives(self): - """``OTEL_EXPORTER=otlp_http`` with no endpoint is a real collector on the SDK's - default port, not the placeholder, so the transport is what tells them apart.""" - from litellm.integrations.otel.presets.utils import credential_gated_exporters - - operator_otlp = ExporterSpec(kind="otlp_http", endpoint=None, headers=None) - - kept = credential_gated_exporters((operator_otlp,), ExporterOwner.LANGFUSE_OTEL) - - assert kept[0] == operator_otlp - - def test_an_in_memory_exporter_the_operator_asked_for_survives(self): - """``OTEL_EXPORTER=in_memory`` stores spans, so it is a destination the operator - chose, not the placeholder that stands in for choosing nothing.""" - from litellm.integrations.otel.presets.utils import credential_gated_exporters - - operator_memory = ExporterSpec(kind="in_memory", endpoint=None, headers=None) - - kept = credential_gated_exporters((operator_memory,), ExporterOwner.LANGFUSE_OTEL) - - assert kept[0] == operator_memory - - def test_the_synthesized_stdout_placeholder_is_dropped(self, monkeypatch): - from litellm.integrations.otel.presets.utils import credential_gated_exporters - - for name in _OTEL_SHORTHAND_ENV: - monkeypatch.delenv(name, raising=False) - placeholder = OpenTelemetryV2Config().exporters[0] - - kept = credential_gated_exporters((placeholder,), ExporterOwner.LANGFUSE_OTEL) - - assert [spec.owner for spec in kept] == [ExporterOwner.LANGFUSE_OTEL] - - def test_a_console_exporter_the_operator_named_survives(self, monkeypatch): - """Same kind, endpoint and headers as the placeholder; only the fact that the - operator set ``OTEL_EXPORTER`` tells them apart.""" - from litellm.integrations.otel.presets.utils import credential_gated_exporters - - for name in _OTEL_SHORTHAND_ENV: - monkeypatch.delenv(name, raising=False) - monkeypatch.setenv("OTEL_EXPORTER", "console") - operator_console = OpenTelemetryV2Config().exporters[0] - - kept = credential_gated_exporters((operator_console,), ExporterOwner.LANGFUSE_OTEL) - - assert kept[0] is operator_console - - class TestTenantHostSsrfGuard: """Anyone who can mint a key can write ``langfuse_host``, so the host it names has to be one the operator approved.""" diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_multibackend.py b/tests/test_litellm/integrations/otel/test_otel_v2_multibackend.py index 92b14324023..033835a676b 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_multibackend.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_multibackend.py @@ -1,9 +1,9 @@ -"""Multi-backend fan-out: one TracerProvider, *N* SpanProcessors. +"""Multi-backend fan-out within one callback: one TracerProvider, *N* SpanProcessors. -V1 needed a separate ``TracerProvider`` per integration to avoid stepping on -the global. V2 attaches a ``SpanProcessor`` per exporter to the *same* -provider, so the same trace ID lights up every backend — no duplicate spans, -no per-integration provider caches. +A single ``OpenTelemetryV2Config`` may list several exporters (say two ``OTEL_*`` +collectors), and its provider attaches one ``SpanProcessor`` per exporter so the same +span reaches each of them. Which exporters a config lists is decided per callback: a +preset only lists the destination it owns, see ``TestDestinationOwnership``. """ import pytest @@ -14,8 +14,8 @@ from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( InMemorySpanExporter, ) -from litellm.integrations.otel.model.config import ExporterOwner, ExporterSpec, OpenTelemetryV2Config -from litellm.integrations.otel.plumbing.providers import build_tracer_provider, register_exporter_factory +from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config +from litellm.integrations.otel.plumbing.providers import build_tracer_provider def test_two_exporters_receive_the_same_span(): @@ -54,52 +54,6 @@ def test_two_exporters_receive_the_same_span(): assert spans_a[0].context.span_id == spans_b[0].context.span_id -def _capturing_kind(kind): - exporter = InMemorySpanExporter() - register_exporter_factory(kind, lambda _spec: exporter) - return exporter - - -def _emit_langfuse_stamped_span(provider): - span = provider.get_tracer("test").start_span("chat proof-model") - span.set_attribute("gen_ai.request.model", "proof-model") - span.set_attribute("langfuse.trace.name", "private-langfuse-only-name") - span.set_attribute("langfuse.observation.input", '[{"role":"user","content":"hi"}]') - span.end() - return span.context.span_id - - -def test_generic_exporter_does_not_receive_langfuse_attributes_the_langfuse_exporter_keeps(): - generic = _capturing_kind("capture_generic_collector") - langfuse = _capturing_kind("capture_langfuse_sink") - cfg = OpenTelemetryV2Config( - exporters=[ - ExporterSpec(kind="capture_generic_collector"), - ExporterSpec(kind="capture_langfuse_sink", owner=ExporterOwner.LANGFUSE_OTEL), - ] - ) - span_id = _emit_langfuse_stamped_span(build_tracer_provider(cfg, tenant_overrides=True)) - - (generic_span,) = generic.get_finished_spans() - (langfuse_span,) = langfuse.get_finished_spans() - assert generic_span.context.span_id == span_id == langfuse_span.context.span_id - assert generic_span.attributes["gen_ai.request.model"] == "proof-model" - assert [k for k in generic_span.attributes if k.startswith("langfuse.")] == [] - assert langfuse_span.attributes["langfuse.trace.name"] == "private-langfuse-only-name" - assert langfuse_span.attributes["langfuse.observation.input"] == '[{"role":"user","content":"hi"}]' - - -def test_langfuse_mapper_on_a_plain_otlp_exporter_keeps_langfuse_attributes(): - sink = _capturing_kind("capture_langfuse_mapper_sink") - cfg = OpenTelemetryV2Config( - mapper_names=["langfuse"], exporters=[ExporterSpec(kind="capture_langfuse_mapper_sink")] - ) - _emit_langfuse_stamped_span(build_tracer_provider(cfg)) - - (span,) = sink.get_finished_spans() - assert span.attributes["langfuse.trace.name"] == "private-langfuse-only-name" - - def test_resource_attributes_apply_to_all_exporters(): """``resource_attributes`` flow through the shared TracerProvider.""" cfg = OpenTelemetryV2Config(