fix(otel v2): each callback exports only to the destination it owns

Presets (langfuse_otel, arize, phoenix, weave_otel, newrelic, agentops, levo) no longer fold the operator's OTEL_* exporter into their own exporter list, so vendor-mapped spans reach only the vendor backend. The otel callback alone serves OTEL_*, is the only logger a second otel entry reuses, and takes the proxy global slot over a preset whatever the callback order. The Langfuse logger stamps trace name and content on the request root only when its own provider created that root. The langfuse.* attribute filter is removed since no exporter can receive another callback's vocabulary any more. Credentialless presets keep only their header-gated exporter instead of inheriting the console placeholder, and the v2 Weave preset reads its config without publishing it as process-wide OTEL_* env

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
yucheng 2026-09-13 00:19:19 +00:00
parent 4a075423a9
commit 210cbf36e9
16 changed files with 330 additions and 356 deletions

View file

@ -1,6 +1,9 @@
from collections.abc import AsyncGenerator, AsyncIterator, Callable, Mapping
from typing import TYPE_CHECKING, Final
from opentelemetry.sdk.trace import ReadableSpan
from opentelemetry.trace import Span
from litellm._logging import verbose_logger
from litellm.integrations.otel.logger import OpenTelemetryV2
from litellm.integrations.otel.mappers.langfuse import (
@ -19,15 +22,22 @@ if TYPE_CHECKING:
class LangfuseOpenTelemetryV2(OpenTelemetryV2):
"""Names the trace from the request. Langfuse reads ``langfuse.trace.name`` off the root observation,
and the proxy's root span is still recording when the LLM call starts."""
and the proxy's root span is still recording when the LLM call starts. Only a root created by this
logger's own provider is stamped; any other root is exported to destinations that are not Langfuse."""
def log_pre_api_call(self, model: str, messages: object, kwargs: Mapping[str, object]) -> None:
root: Final = request_root_span()
root: Final = self._owned_recording_root()
name: Final = caller_trace_name(kwargs)
if root is not None and root.is_recording() and name is not None:
if root is not None and name is not None:
root.set_attribute(LANGFUSE_TRACE_NAME, name)
super().log_pre_api_call(model, messages, kwargs)
def _owned_recording_root(self) -> Span | None:
root: Final = request_root_span()
if root is None or not root.is_recording() or not isinstance(root, ReadableSpan):
return None
return root if root.resource is self.tracer_provider.resource else None
class LangfuseContentOpenTelemetryV2(LangfuseOpenTelemetryV2):
"""Stamps the request's input and output on the root observation while it is still recording.
@ -59,8 +69,8 @@ class LangfuseContentOpenTelemetryV2(LangfuseOpenTelemetryV2):
self._stamp_root_io(request_data, lambda: stream_output(tuple(relayed), request_data))
def _stamp_root_io(self, data: Mapping[str, object], render_output: Callable[[], str | None]) -> None:
root: Final = request_root_span()
if root is None or not root.is_recording():
root: Final = self._owned_recording_root()
if root is None:
return
try:
output: Final = render_output()

View file

@ -259,9 +259,15 @@ class OpenTelemetryV2(CustomLogger):
self._register_in_callback_list(litellm._async_failure_callback)
except Exception:
pass
if getattr(proxy_server, "open_telemetry_logger", None) is None:
if self._outranks_for_proxy_slot(getattr(proxy_server, "open_telemetry_logger", None)):
setattr(proxy_server, "open_telemetry_logger", self)
def _outranks_for_proxy_slot(self, holder: object) -> bool:
"""``otel`` owns the slot whenever configured; a preset holds it only until ``otel`` is built."""
if holder is None:
return True
return self.callback_name is None and isinstance(holder, OpenTelemetryV2) and holder.callback_name is not None
# ====================================================================== #
# LLM-call callbacks — the span is opened at the ``pre_call`` boundary and
# closed here. See ``log_pre_api_call``.
@ -842,10 +848,12 @@ def select_global_otel_v2_logger(
``proxy_server.open_telemetry_logger``), and every other v2 entry point —
guardrail, identity seeding, phase spans — already routes through that same
``registered`` owner. Reuse it here too so the global provider has one source
of truth instead of a second, independently-derived guess; this is the logger
a preset (arize, langfuse, …) folds the ``OTEL_*`` base exporter and its own
exporter into, so the FastAPI server span and the gen-ai spans share one
provider and one trace.
of truth instead of a second, independently-derived guess. Each logger exports
only to the destination its callback owns (``otel`` serves ``OTEL_*``, a preset
serves its own backend), so the FastAPI server span and the request root land
at the ``otel`` callback's collector when one is configured and at the first
preset's backend otherwise, whatever the callback order; a preset stamps that
root only when its own provider created it.
Fall back to ``in_memory_loggers`` for the SDK path, where no proxy global is
set (selecting from there, not ``service_callback``, which a preset logger does
@ -855,8 +863,11 @@ def select_global_otel_v2_logger(
"""
if registered is not None:
return registered
existing: Final = next((cb for cb in in_memory_loggers if isinstance(cb, OpenTelemetryV2)), None)
return existing if existing is not None else OpenTelemetryV2()
v2_loggers: Final = tuple(cb for cb in in_memory_loggers if isinstance(cb, OpenTelemetryV2))
generic: Final = next((cb for cb in v2_loggers if cb.callback_name is None), None)
if generic is not None:
return generic
return v2_loggers[0] if v2_loggers else OpenTelemetryV2()
def publish_global_otel_v2_provider(

View file

@ -41,7 +41,7 @@ from opentelemetry.util.types import Attributes, AttributeValue
from litellm._logging import verbose_logger
from litellm._version import version as litellm_version
from litellm.integrations.otel.model.config import ExporterOwner, ExporterSpec, OpenTelemetryV2Config
from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config
from litellm.integrations.otel.model.semconv import (
DB,
MCP,
@ -377,7 +377,6 @@ _CAPTURED_HEADER_PREFIXES: Final = ("http.request.header.", "http.response.heade
# ``?key=`` query parameter.
_URL_KEYS: Final = frozenset({"http.url", "http.target", "url.full"})
_URL_QUERY_KEY: Final = "url.query"
_LANGFUSE_ATTRIBUTE_PREFIX: Final = "langfuse."
class _TenantSpanView(ReadableSpan):
@ -419,11 +418,9 @@ def _guardrail_unreachable(attributes: Mapping[str, AttributeValue]) -> bool:
return attributes.get(LiteLLM.GUARDRAIL_STATUS) in _GUARDRAIL_UNREACHABLE_STATUSES
def _tenant_visible(key: str, database: bool, owned: bool, unreachable_guardrail: bool, langfuse: bool) -> bool:
def _tenant_visible(key: str, database: bool, owned: bool, unreachable_guardrail: bool) -> bool:
if key.startswith(_CAPTURED_HEADER_PREFIXES) or key in (LiteLLMError.STACK_TRACE, _URL_QUERY_KEY):
return False
if not langfuse and key.startswith(_LANGFUSE_ATTRIBUTE_PREFIX):
return False
if database and key in _DATASTORE_ENDPOINT_KEYS:
return False
if unreachable_guardrail and key == LiteLLM.GUARDRAIL_RESPONSE:
@ -473,12 +470,11 @@ def _for_destination(span: ReadableSpan, destination: "OtelDestination") -> Read
database: Final = _is_database_span(attributes)
owned: Final = _is_tenant_owned_span(attributes)
unreachable: Final = _guardrail_unreachable(attributes)
langfuse: Final = destination.callback_name == ExporterOwner.LANGFUSE_OTEL.value
kept: Final = MappingProxyType(
{
key: _without_query(key, value)
for key, value in attributes.items()
if _tenant_visible(key, database, owned, unreachable, langfuse)
if _tenant_visible(key, database, owned, unreachable)
}
)
recorded: Final = span.events
@ -491,24 +487,6 @@ def _for_destination(span: ReadableSpan, destination: "OtelDestination") -> Read
return _TenantSpanView(span, resource, kept, events, status)
def _without_langfuse_attributes(span: ReadableSpan) -> ReadableSpan:
attributes: Final = span.attributes or _NO_ATTRIBUTES
kept: Final = MappingProxyType(
{key: value for key, value in attributes.items() if not key.startswith(_LANGFUSE_ATTRIBUTE_PREFIX)}
)
if len(kept) == len(attributes):
return span
return _TenantSpanView(span, span.resource, kept, span.events, span.status)
def _reads_langfuse_attributes(spec: ExporterSpec, config: OpenTelemetryV2Config) -> bool:
if spec.owner is not None:
return spec.owner is ExporterOwner.LANGFUSE_OTEL
return "langfuse" in config.mapper_names and all(
other.owner is not ExporterOwner.LANGFUSE_OTEL for other in config.exporters
)
class TenantFanOutSpanProcessor(SpanProcessor):
"""Export every finished span to each destination this request resolved.
@ -796,23 +774,6 @@ class _OverriddenBackendFilter(SpanProcessor):
return self._inner.force_flush(timeout_millis)
class _LangfuseAttributeFilter(SpanProcessor):
def __init__(self, inner: SpanProcessor) -> None:
self._inner: Final = inner
def on_start(self, span: SDKSpan, parent_context: Context | None = None) -> None:
self._inner.on_start(span, parent_context)
def on_end(self, span: ReadableSpan) -> None:
self._inner.on_end(_without_langfuse_attributes(span))
def shutdown(self) -> None:
self._inner.shutdown()
def force_flush(self, timeout_millis: int = 30000) -> bool:
return self._inner.force_flush(timeout_millis)
def build_span_exporter(config: OpenTelemetryV2Config) -> SpanExporter:
"""Build a single exporter from the top-level config fields.
@ -1099,10 +1060,9 @@ def build_tracer_provider(
exp,
(spec.use_simple_processor if spec.use_simple_processor is not None else use_simple_processor),
)
sink = processor if _reads_langfuse_attributes(spec, config) else _LangfuseAttributeFilter(processor)
owner = spec.owner.value if spec.owner is not None else None
provider.add_span_processor(
_OverriddenBackendFilter(sink, owner) if tenant_overrides and owner is not None else sink
_OverriddenBackendFilter(processor, owner) if tenant_overrides and owner is not None else processor
)
return provider

View file

@ -53,7 +53,6 @@ def agentops_preset(
return base.model_copy(
update={
"exporters": [
*base.exporters,
ExporterSpec(
kind=_AGENTOPS_EXPORTER_KIND,
endpoint=_AGENTOPS_ENDPOINT,

View file

@ -35,7 +35,6 @@ def arize_preset(
return base.model_copy(
update={
"exporters": [
*base.exporters,
ExporterSpec(
kind=arize_cfg.protocol or "otlp_grpc",
endpoint=arize_cfg.endpoint or "https://otlp.arize.com/v1",

View file

@ -10,10 +10,7 @@ from litellm.integrations.otel.model.config import (
ExporterSpec,
OpenTelemetryV2Config,
)
from litellm.integrations.otel.presets.utils import (
credential_gated_exporters,
ensure_mappers,
)
from litellm.integrations.otel.presets.utils import ensure_mappers
from litellm.types.utils import StandardCallbackDynamicParams
@ -31,7 +28,7 @@ def langfuse_preset(
raise
return base.model_copy(
update={ # mutable-ok: pydantic model_copy takes a plain update mapping
"exporters": credential_gated_exporters(base.exporters, ExporterOwner.LANGFUSE_OTEL),
"exporters": [ExporterSpec(owner=ExporterOwner.LANGFUSE_OTEL, requires_headers=True)],
"mapper_names": mappers,
}
)
@ -39,7 +36,6 @@ def langfuse_preset(
return base.model_copy(
update={
"exporters": [
*base.exporters,
ExporterSpec(
kind=kind,
endpoint=cfg.endpoint,

View file

@ -20,7 +20,6 @@ def levo_preset(
return base.model_copy(
update={
"exporters": [
*base.exporters,
ExporterSpec(
kind="otlp_http",
endpoint=cfg.endpoint,

View file

@ -54,7 +54,6 @@ def newrelic_preset(
return base.model_copy(
update={
"exporters": [
*base.exporters,
ExporterSpec(
kind="otlp_http",
endpoint=endpoint,

View file

@ -69,7 +69,6 @@ def phoenix_preset(
return base.model_copy(
update={
"exporters": [
*base.exporters,
ExporterSpec(
kind=cfg.protocol if hasattr(cfg, "protocol") else "otlp_http",
endpoint=cfg.endpoint,

View file

@ -3,8 +3,6 @@
from collections.abc import Iterable
from typing import Final
from litellm.integrations.otel.model.config import ExporterOwner, ExporterSpec
def ensure_mappers(mapper_names: Iterable[str], *names: str) -> list[str]:
"""Return ``mapper_names`` with each of ``names`` appended if not already present.
@ -17,32 +15,3 @@ def ensure_mappers(mapper_names: Iterable[str], *names: str) -> list[str]:
if name not in result:
result.append(name)
return result
def credential_gated_exporters(
exporters: "Iterable[ExporterSpec]", owner: "ExporterOwner"
) -> "tuple[ExporterSpec, ...]":
"""``exporters`` with the operator's destination replaced by a header-gated one.
Used when a credential-mandatory backend is asked to build without the operator's
own credentials, so only key/team destinations receive spans. Two things have to
happen for that to mean "export nowhere": the placeholder console spec that
``OpenTelemetryV2Config`` folds in for an empty exporter list is dropped, or every
span would be printed to stdout, and the gated spec keeps the owner so the
override filter still recognises which backend this provider speaks for.
"""
return (
*(spec for spec in exporters if not is_unconfigured_placeholder(spec)),
ExporterSpec(owner=owner, requires_headers=True),
)
def is_unconfigured_placeholder(spec: "ExporterSpec") -> bool:
"""Whether ``spec`` is the one ``_normalize`` folds in when nothing was configured.
No field set is what says the operator asked for nothing: an exporter they did
configure survives, even ``OTEL_EXPORTER=console`` whose value matches the default,
and so does the gated spec this module appends, which would otherwise eat itself
when one preset layers onto another.
"""
return not spec.model_fields_set

View file

@ -7,13 +7,10 @@ from litellm.integrations.otel.model.config import (
ExporterSpec,
OpenTelemetryV2Config,
)
from litellm.integrations.otel.presets.utils import (
credential_gated_exporters,
ensure_mappers,
)
from litellm.integrations.otel.presets.utils import ensure_mappers
from litellm.integrations.weave.weave_otel import (
_get_weave_authorization_header,
get_weave_otel_config,
read_weave_otel_config,
)
from litellm.types.utils import StandardCallbackDynamicParams
@ -26,20 +23,19 @@ def weave_preset(
base: Final = config_overrides or OpenTelemetryV2Config()
mappers: Final = ensure_mappers(base.mapper_names, "openinference", "weave")
try:
weave_cfg: Final = get_weave_otel_config()
weave_cfg: Final = read_weave_otel_config()
except Exception:
if not allow_missing_credentials:
raise
return base.model_copy(
update={ # mutable-ok: pydantic model_copy takes a plain update mapping
"exporters": credential_gated_exporters(base.exporters, ExporterOwner.WEAVE_OTEL),
"exporters": [ExporterSpec(owner=ExporterOwner.WEAVE_OTEL, requires_headers=True)],
"mapper_names": mappers,
}
)
return base.model_copy(
update={
"exporters": [
*base.exporters,
ExporterSpec(
kind=weave_cfg.protocol or "otlp_http",
endpoint=weave_cfg.endpoint,

View file

@ -125,17 +125,8 @@ def weave_otel_endpoint(host: str | None) -> str:
return normalized.rstrip("/") + WEAVE_OTEL_ENDPOINT
def get_weave_otel_config() -> WeaveOtelConfig:
"""
Retrieves the Weave OpenTelemetry configuration based on environment variables.
Environment Variables:
WANDB_API_KEY: Required. W&B API key for authentication.
WANDB_PROJECT_ID: Required. Project ID in format <entity>/<project_name>.
WANDB_HOST: Optional. Custom Weave host URL. Defaults to cloud endpoint.
Returns:
WeaveOtelConfig: A Pydantic model containing Weave OTEL configuration.
def read_weave_otel_config() -> WeaveOtelConfig:
"""Weave OTLP settings from ``WANDB_API_KEY``, ``WANDB_PROJECT_ID`` and optional ``WANDB_HOST``.
Raises:
ValueError: If required environment variables are missing.
@ -158,10 +149,6 @@ def get_weave_otel_config() -> WeaveOtelConfig:
auth_header: Final = _get_weave_authorization_header(api_key=api_key)
otlp_auth_headers: Final = f"Authorization={auth_header},project_id={project_id}"
# Set standard OTEL environment variables
os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = endpoint
os.environ["OTEL_EXPORTER_OTLP_HEADERS"] = otlp_auth_headers
return WeaveOtelConfig(
otlp_auth_headers=otlp_auth_headers,
endpoint=endpoint,
@ -170,6 +157,14 @@ def get_weave_otel_config() -> WeaveOtelConfig:
)
def get_weave_otel_config() -> WeaveOtelConfig:
"""``read_weave_otel_config`` plus the v1 side effect of publishing it as the process-wide ``OTEL_*`` env."""
config: Final = read_weave_otel_config()
os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = config.endpoint
os.environ["OTEL_EXPORTER_OTLP_HEADERS"] = config.otlp_auth_headers
return config
def set_weave_otel_attributes(span: Span, kwargs: Mapping[str, object], response_obj: object):
"""
Sets OpenTelemetry span attributes for Weave observability.

View file

@ -4499,7 +4499,7 @@ def _init_custom_logger_compatible_class(
from litellm.integrations.otel.model.config import OpenTelemetryV2Config
for callback in _in_memory_loggers:
if isinstance(callback, OpenTelemetryV2):
if isinstance(callback, OpenTelemetryV2) and callback.callback_name is None:
return callback
otel_settings: Final = _get_custom_logger_settings_from_proxy_server(callback_name=logging_integration)
otel_logger_v2: Final = build_otel_v2_logger(
@ -4862,13 +4862,13 @@ def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[Custom
Returns ``None`` when V2 is off OR when there's no preset registered for
``callback_name`` — callers should then fall through to the legacy path.
A preset that needs operator credentials it cannot find is allowed to build
only when this request has a key/team destination for that backend and another
V2 logger is already registered to carry the fan-out. The resulting logger keeps
only its credential-gated exporter, while the registered logger owns operator
delivery. Without that carrier, a preset that raises or that ends up with nothing
but its gated exporter and the default console placeholder returns ``None``, so the
caller falls through to the legacy path exactly as before V2 landed.
Each preset exports only to the destination it owns; the operator's ``OTEL_*``
collector is served by the ``otel`` callback alone. A preset that needs operator
credentials it cannot find is allowed to build only when this request has a
key/team destination for that backend and another V2 logger is already registered
to carry the fan-out. Without that carrier, a preset that raises or that has nothing
but its credential-gated exporter returns ``None``, so the caller falls through to
the legacy path exactly as before V2 landed.
"""
from litellm.integrations.otel.model.config import is_otel_v2_enabled
@ -4887,21 +4887,20 @@ def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[Custom
for callback in _in_memory_loggers:
if (
isinstance(callback, OpenTelemetryV2)
and getattr(callback, "callback_name", None) == callback_name
and callback.callback_name == callback_name
and (serves_a_destination or not _exports_nowhere(callback.config))
):
return callback
try:
built: Final = preset_fn(allow_missing_credentials=carried)
config: Final = preset_fn(allow_missing_credentials=carried)
except Exception:
# If env vars are missing or the preset raises, defer to the legacy path
# so customers get the same error story they had before V2 landed.
return None
gated: Final = _is_credential_gated(built)
if gated and not carried and not _has_operator_exporter(built):
exports_nowhere: Final = _exports_nowhere(config)
if exports_nowhere and not carried:
return None
config: Final = _only_the_gated_exporter(built) if gated and carried else built
if _exports_nowhere(config):
if exports_nowhere:
verbose_logger.warning(
"OTel V2: no operator credentials for '%s'; only key/team destinations will receive its traces",
callback_name,
@ -4916,24 +4915,6 @@ def _exports_nowhere(config: "OpenTelemetryV2Config") -> bool:
return all(_is_gated(spec) for spec in config.exporters)
def _is_credential_gated(config: "OpenTelemetryV2Config") -> bool:
"""Whether the preset built without the operator's own credentials for its backend."""
return any(_is_gated(spec) for spec in config.exporters)
def _has_operator_exporter(config: "OpenTelemetryV2Config") -> bool:
"""Whether the operator configured somewhere real to export, beyond the default console placeholder."""
from litellm.integrations.otel.presets.utils import is_unconfigured_placeholder
return any(not _is_gated(spec) and not is_unconfigured_placeholder(spec) for spec in config.exporters)
def _only_the_gated_exporter(config: "OpenTelemetryV2Config") -> "OpenTelemetryV2Config":
return config.model_copy(
update={"exporters": [spec for spec in config.exporters if _is_gated(spec)]} # mutable-ok: model_copy update
)
def _is_gated(spec: "ExporterSpec") -> bool:
return spec.requires_headers and not spec.headers

View file

@ -374,6 +374,53 @@ def test_unnamed_request_leaves_the_trace_name_off_both_spans():
assert TRACE_NAME_ATTR not in root_attrs and TRACE_NAME_ATTR not in generation_attrs
def test_a_root_owned_by_another_loggers_provider_is_never_stamped_with_langfuse_vocabulary():
"""``callbacks: [otel, langfuse_otel]``: the request root belongs to the ``otel`` logger and exports to the
operator's collector, so Langfuse's trace name and root input/output must not be written on it."""
generic, generic_exporter = _logger(mappers=("genai",))
langfuse, langfuse_exporter = _logger()
response = ModelResponse(choices=[Choices(message=Message(role="assistant", content="pong"))])
named: Final = {"proxy_server_request": {"headers": {"langfuse_trace_name": "private-name"}}}
root = _start_root(generic)
asyncio.run(langfuse.async_pre_call_hook(UserAPIKeyAuth(), DualCache(), CHAT_DATA, "acompletion"))
langfuse.log_pre_api_call(
model="gpt-5.4-mini", messages=[], kwargs={"litellm_call_id": "c1", "litellm_params": named}
)
asyncio.run(
langfuse.async_post_call_success_hook(data=CHAT_DATA, user_api_key_dict=UserAPIKeyAuth(), response=response)
)
root.end()
root_attrs = _root_attrs(generic_exporter)
assert not any(key.startswith("langfuse.") for key in root_attrs)
assert all(span.name != LITELLM_PROXY_REQUEST_SPAN_NAME for span in langfuse_exporter.get_finished_spans())
def test_a_root_owned_by_the_langfuse_provider_is_still_named_and_given_its_io_next_to_another_logger():
"""``callbacks: [langfuse_otel]`` with a second v2 logger alongside: the root was created by the Langfuse
provider, so the ownership check must not stop the trace name and root input/output from reaching Langfuse."""
_logger(mappers=("genai",))
langfuse, langfuse_exporter = _logger()
response = ModelResponse(choices=[Choices(message=Message(role="assistant", content="pong"))])
named: Final = {"proxy_server_request": {"headers": {"langfuse_trace_name": "private-name"}}}
root = _start_root(langfuse)
asyncio.run(langfuse.async_pre_call_hook(UserAPIKeyAuth(), DualCache(), CHAT_DATA, "acompletion"))
langfuse.log_pre_api_call(
model="gpt-5.4-mini", messages=[], kwargs={"litellm_call_id": "c1", "litellm_params": named}
)
asyncio.run(
langfuse.async_post_call_success_hook(data=CHAT_DATA, user_api_key_dict=UserAPIKeyAuth(), response=response)
)
root.end()
root_attrs = _root_attrs(langfuse_exporter)
assert root_attrs[TRACE_NAME_ATTR] == "private-name"
assert json.loads(root_attrs[INPUT_ATTR]) == CHAT_DATA["messages"]
assert "pong" in root_attrs[OUTPUT_ATTR]
@pytest.mark.parametrize(
("capture", "mappers"),
[("no_content", ("genai", "langfuse")), ("span_only", ("genai",))],

View file

@ -3,14 +3,14 @@
import contextvars
import time
from base64 import b64encode
from collections.abc import Mapping
from collections.abc import Callable, Mapping
from functools import reduce
from types import MappingProxyType
import pytest
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
from opentelemetry.sdk.trace.export import SimpleSpanProcessor, SpanExporter
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
from opentelemetry.trace import Status, StatusCode
@ -22,6 +22,7 @@ from litellm.integrations.otel.logger import (
build_otel_v2_logger,
fan_out_provider,
publish_global_otel_v2_provider,
select_global_otel_v2_logger,
)
from litellm.integrations.otel.model.config import (
ExporterOwner,
@ -51,6 +52,11 @@ from litellm.integrations.otel.presets.destinations import (
destination_for,
)
from litellm.integrations.otel.presets.langfuse import langfuse_preset
from litellm.litellm_core_utils import litellm_logging
from litellm.litellm_core_utils.litellm_logging import (
_init_custom_logger_compatible_class,
_maybe_construct_otel_v2,
)
from litellm.proxy._types import AddTeamCallback, UserAPIKeyAuth
from litellm.proxy.litellm_pre_call_utils import (
convert_key_logging_metadata_to_callback,
@ -92,6 +98,12 @@ def emit(provider: TracerProvider, name: str = "chat gpt-4") -> None:
pass
def swap_exporter_factories(monkeypatch, **factories: Callable[[ExporterSpec], SpanExporter]) -> None:
"""Swap the exporter built for each ``kind`` for the duration of one test."""
for kind, factory in factories.items():
monkeypatch.setitem(otel_providers._EXPORTER_FACTORIES, kind, factory)
def wired_provider(dest_exporter: InMemorySpanExporter, global_exporter: InMemorySpanExporter) -> TracerProvider:
"""The operator's provider: one owned exporter plus the tenant fan-out."""
provider = TracerProvider()
@ -516,34 +528,6 @@ class TestFanOut:
assert [s.name for s in langfuse.get_finished_spans()] == ["chat gpt-4"]
assert [s.name for s in arize.get_finished_spans()] == ["chat gpt-4"]
def test_only_the_langfuse_destination_receives_the_langfuse_attributes(self):
langfuse, arize = InMemorySpanExporter(), InMemorySpanExporter()
by_endpoint = {"http://a.local": langfuse, "http://b.local": arize}
provider = TracerProvider()
provider.add_span_processor(
TenantFanOutSpanProcessor(processor_factory=lambda d: SimpleSpanProcessor(by_endpoint[d.endpoint]))
)
def run():
set_request_destinations(
(
OtelDestination(endpoint="http://a.local", callback_name="langfuse_otel"),
OtelDestination(endpoint="http://b.local", callback_name="arize"),
)
)
span = get_tracer(provider, "litellm").start_span("chat gpt-4")
span.set_attribute("gen_ai.request.model", "gpt-4")
span.set_attribute("langfuse.trace.name", "private-langfuse-only-name")
span.end()
in_fresh_context(run)
(langfuse_span,) = langfuse.get_finished_spans()
(arize_span,) = arize.get_finished_spans()
assert langfuse_span.attributes["langfuse.trace.name"] == "private-langfuse-only-name"
assert arize_span.attributes["gen_ai.request.model"] == "gpt-4"
assert [k for k in arize_span.attributes if k.startswith("langfuse.")] == []
def test_a_destination_carries_the_tenants_service_name(self):
"""An overridden backend skips per-request tracer routing, so the service name
that route used to apply has to travel on the destination instead."""
@ -1473,8 +1457,6 @@ class TestPresetDegradation:
langfuse_preset()
def test_a_credential_less_proxy_builds_the_gated_logger_beside_a_v2_carrier(self, monkeypatch):
from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2
credential_less_proxy(monkeypatch)
monkeypatch.setenv("LITELLM_OTEL_V2", "true")
carrier = build_otel_v2_logger(OpenTelemetryV2Config(exporter="in_memory"))
@ -1494,8 +1476,6 @@ class TestPresetDegradation:
"""Nothing can use a credential-less langfuse here, so the operator has to get
the same story as before v2: the legacy integration, not a global provider
that exports nowhere."""
from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2
credential_less_proxy(monkeypatch)
monkeypatch.setenv("LITELLM_OTEL_V2", "true")
@ -1505,9 +1485,10 @@ class TestPresetDegradation:
assert logger is None
def test_a_valid_newrelic_base_exporter_survives_without_a_license_key(self, monkeypatch):
from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2
def test_a_credentialless_newrelic_does_not_take_over_the_operator_collector(self, monkeypatch):
"""The ``OTEL_*`` collector belongs to the ``otel`` callback. A New Relic
entry with no license key has nowhere of its own to export, so it takes
the legacy path instead of shipping New Relic-shaped spans to the collector."""
monkeypatch.delenv("NEW_RELIC_LICENSE_KEY", raising=False)
monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.local:4318")
monkeypatch.setenv("LITELLM_OTEL_V2", "true")
@ -1516,15 +1497,21 @@ class TestPresetDegradation:
logger = in_fresh_context(_maybe_construct_otel_v2, "newrelic", [])
is_otel_v2_enabled.cache_clear()
assert logger is None
def test_a_credentialed_newrelic_exports_only_to_new_relic(self, monkeypatch):
monkeypatch.setenv("NEW_RELIC_LICENSE_KEY", "nr-license")
monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.local:4318")
monkeypatch.setenv("LITELLM_OTEL_V2", "true")
is_otel_v2_enabled.cache_clear()
logger = in_fresh_context(_maybe_construct_otel_v2, "newrelic", [])
is_otel_v2_enabled.cache_clear()
assert logger is not None
assert [spec.endpoint for spec in logger.config.exporters] == [
"http://collector.local:4318",
"https://otlp.nr-data.net",
]
assert [spec.endpoint for spec in logger.config.exporters] == ["https://otlp.nr-data.net"]
def test_a_credentialless_newrelic_without_a_base_exporter_falls_back(self, monkeypatch):
from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2
monkeypatch.delenv("NEW_RELIC_LICENSE_KEY", raising=False)
for name in _OTEL_SHORTHAND_ENV:
monkeypatch.delenv(name, raising=False)
@ -1536,12 +1523,10 @@ class TestPresetDegradation:
assert logger is None
def test_an_explicit_console_exporter_keeps_a_credentialless_preset_on_v2(self, monkeypatch, capfd):
"""``OTEL_EXPORTER=console`` reads exactly like the placeholder ``_normalize``
folds in, but the operator asked for it, so a credential-less New Relic keeps
the V2 logger and its spans reach stdout instead of the legacy path."""
from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2
def test_an_explicit_console_exporter_is_served_by_otel_not_by_a_credentialless_preset(self, monkeypatch):
"""``OTEL_EXPORTER=console`` is the operator's generic destination. Only the
``otel`` callback prints there; a credential-less New Relic has nothing of
its own to export to and takes the legacy path."""
monkeypatch.delenv("NEW_RELIC_LICENSE_KEY", raising=False)
for name in _OTEL_SHORTHAND_ENV:
monkeypatch.delenv(name, raising=False)
@ -1550,15 +1535,13 @@ class TestPresetDegradation:
is_otel_v2_enabled.cache_clear()
logger = in_fresh_context(_maybe_construct_otel_v2, "newrelic", [])
generic = build_otel_v2_logger(OpenTelemetryV2Config())
is_otel_v2_enabled.cache_clear()
assert logger is not None
assert logger.config.exporters[0].kind == "console"
assert not logger.config.exporters[0].requires_headers
assert logger is None
assert [spec.kind for spec in generic.config.exporters] == ["console"]
def test_a_destination_for_one_backend_does_not_degrade_another(self, monkeypatch):
from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2
credential_less_proxy(monkeypatch)
monkeypatch.delenv("WANDB_API_KEY", raising=False)
monkeypatch.setenv("LITELLM_OTEL_V2", "true")
@ -1576,8 +1559,6 @@ class TestPresetDegradation:
def test_the_exporter_less_logger_is_not_reused_by_a_request_without_destinations(self, monkeypatch):
"""Reusing it would let one team's destination decide how every later request
without one is logged, long after the degrade was justified."""
from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2
credential_less_proxy(monkeypatch)
monkeypatch.setenv("LITELLM_OTEL_V2", "true")
loggers = [build_otel_v2_logger(OpenTelemetryV2Config(exporter="in_memory"))]
@ -1595,8 +1576,6 @@ class TestPresetDegradation:
assert plain is None
def test_a_credentialed_logger_is_still_reused_across_requests(self, monkeypatch):
from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2
monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-lf-1")
monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-lf-1")
monkeypatch.setenv("LITELLM_OTEL_V2", "true")
@ -1612,8 +1591,6 @@ class TestPresetDegradation:
@staticmethod
def _degraded_langfuse_beside(loggers, monkeypatch):
from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2
credential_less_proxy(monkeypatch)
monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.local:4318")
monkeypatch.setenv("LITELLM_OTEL_V2", "true")
@ -1643,8 +1620,6 @@ class TestPresetDegradation:
"""Only a V2 logger publishes the provider the fan-out rides on, so a legacy
callback beside this one leaves the destination just as unreachable as no
callback at all, and the operator keeps the pre-V2 story."""
from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2
credential_less_proxy(monkeypatch)
monkeypatch.setenv("LITELLM_OTEL_V2", "true")
@ -1658,28 +1633,24 @@ class TestPresetDegradation:
assert logger is None
def test_a_credentialed_logger_beside_another_v2_logger_keeps_every_exporter(self, monkeypatch):
"""Only a degraded preset gives the collector up; an operator who configured
both the backend and the collector still exports to both, as on base."""
from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2
@pytest.mark.parametrize("beside", [(), ("collector",)])
def test_a_credentialed_langfuse_exports_only_to_langfuse(self, monkeypatch, beside):
"""The operator's collector is the ``otel`` callback's destination whether or
not that callback is registered; Langfuse never inherits it."""
monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-lf-1")
monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-lf-1")
monkeypatch.setenv("LANGFUSE_HOST", "https://cloud.langfuse.com")
monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.local:4318")
monkeypatch.setenv("LITELLM_OTEL_V2", "true")
collector_logger = build_otel_v2_logger(OpenTelemetryV2Config(exporter="in_memory"))
loggers = [build_otel_v2_logger(OpenTelemetryV2Config(exporter="in_memory")) for _ in beside]
is_otel_v2_enabled.cache_clear()
logger = in_fresh_context(_maybe_construct_otel_v2, "langfuse_otel", [collector_logger])
logger = in_fresh_context(_maybe_construct_otel_v2, "langfuse_otel", loggers)
is_otel_v2_enabled.cache_clear()
assert logger is not None
assert [spec.endpoint for spec in logger.config.exporters] == [
"http://collector.local:4318",
"https://cloud.langfuse.com/api/public/otel",
]
assert all(spec.headers for spec in logger.config.exporters if spec.requires_headers)
assert [spec.endpoint for spec in logger.config.exporters] == ["https://cloud.langfuse.com/api/public/otel"]
assert all(spec.headers for spec in logger.config.exporters)
class TestContextIsolation:
@ -1692,26 +1663,184 @@ class TestContextIsolation:
assert in_fresh_context(request_destinations) == ()
class TestOperatorShorthandSurvivesDegradation:
def test_a_generic_otlp_collector_keeps_receiving_when_langfuse_has_no_credentials(self, monkeypatch):
"""Only the stdout placeholder is dropped. An operator who set the standard
OTLP env vars configured a real destination and must keep it."""
class TestOperatorShorthandStaysWithOtel:
def test_a_credential_less_langfuse_leaves_the_operator_collector_to_the_otel_callback(self, monkeypatch):
monkeypatch.delenv("LANGFUSE_PUBLIC_KEY", raising=False)
monkeypatch.delenv("LANGFUSE_SECRET_KEY", raising=False)
monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.local:4318")
config = langfuse_preset(allow_missing_credentials=True)
assert [spec.endpoint for spec in config.exporters] == ["http://collector.local:4318", None]
assert [spec.kind for spec in config.exporters] == ["otlp_http", "console"]
assert [spec.endpoint for spec in config.exporters] == [None]
assert [spec.owner for spec in config.exporters] == [ExporterOwner.LANGFUSE_OTEL]
def test_the_stdout_placeholder_is_still_dropped_when_it_is_the_only_exporter(self, monkeypatch):
def test_the_stdout_placeholder_is_never_inherited_by_a_credential_less_preset(self, monkeypatch):
credential_less_proxy(monkeypatch)
config = langfuse_preset(allow_missing_credentials=True)
assert all(spec.requires_headers and not spec.headers for spec in config.exporters)
def test_a_lone_credentialed_preset_without_otel_env_prints_nothing_to_stdout(self, monkeypatch):
"""``callbacks: [langfuse_otel]`` and no ``OTEL_*`` at all: the base config's stdout placeholder
belongs to ``otel``, so the Langfuse logger must not dump every span as JSON on the proxy log."""
for name in _OTEL_SHORTHAND_ENV:
monkeypatch.delenv(name, raising=False)
for name, value in _VENDOR_ENV["langfuse_otel"].items():
monkeypatch.setenv(name, value)
langfuse_sink, stdout = InMemorySpanExporter(), InMemorySpanExporter()
swap_exporter_factories(monkeypatch, otlp_http=lambda spec: langfuse_sink, console=lambda spec: stdout)
logger = build_otel_v2_logger(langfuse_preset(), callback_name="langfuse_otel")
emit(logger.tracer_provider, "vendor only")
logger.tracer_provider.force_flush()
assert [s.name for s in langfuse_sink.get_finished_spans()] == ["vendor only"]
assert stdout.get_finished_spans() == ()
_COLLECTOR = "http://collector.local:4318"
#: Env that gives each endpoint-owning preset its own credentials.
_VENDOR_ENV = MappingProxyType(
{
"langfuse_otel": {
"LANGFUSE_PUBLIC_KEY": "pk",
"LANGFUSE_SECRET_KEY": "sk",
"LANGFUSE_HOST": "https://lf.local",
},
"arize": {"ARIZE_SPACE_ID": "space", "ARIZE_API_KEY": "key"},
"arize_phoenix": {
"PHOENIX_API_KEY": "key",
"PHOENIX_COLLECTOR_HTTP_ENDPOINT": "https://phoenix.local/v1/traces",
},
"newrelic": {"NEW_RELIC_LICENSE_KEY": "license"},
"agentops": {"AGENTOPS_API_KEY": "ao-key"},
"levo": {
"LEVOAI_API_KEY": "k",
"LEVOAI_ORG_ID": "org",
"LEVOAI_WORKSPACE_ID": "ws",
"LEVOAI_COLLECTOR_URL": "https://levo.local/v1/traces",
},
"weave_otel": {"WANDB_API_KEY": "wandb", "WANDB_PROJECT_ID": "entity/project"},
}
)
class TestDestinationOwnership:
"""A callback exports only to the destination it owns: ``otel`` serves the operator's
``OTEL_*`` collector and a preset serves its own backend. The leak this closes was every
preset inheriting the collector from the base config, so its vendor-vocabulary spans
(``langfuse.*``, OpenInference, ...) landed on the operator's collector too."""
@staticmethod
def _capture_exporters(monkeypatch) -> dict[str, InMemorySpanExporter]:
sinks: dict[str, InMemorySpanExporter] = {}
def factory(spec: ExporterSpec) -> InMemorySpanExporter:
return sinks.setdefault(spec.endpoint or "", InMemorySpanExporter())
swap_exporter_factories(monkeypatch, otlp_http=factory, otlp_grpc=factory, agentops=factory)
return sinks
@staticmethod
def _operator_with(monkeypatch, vendor: str) -> None:
for name in ("LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY", *_OTEL_SHORTHAND_ENV):
monkeypatch.delenv(name, raising=False)
for name, value in _VENDOR_ENV[vendor].items():
monkeypatch.setenv(name, value)
monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", _COLLECTOR)
monkeypatch.setenv("LITELLM_OTEL_V2", "true")
is_otel_v2_enabled.cache_clear()
@pytest.mark.parametrize("vendor", sorted(_VENDOR_ENV))
def test_a_presets_spans_never_reach_the_operator_collector(self, monkeypatch, vendor):
sinks = self._capture_exporters(monkeypatch)
self._operator_with(monkeypatch, vendor)
generic = build_otel_v2_logger(OpenTelemetryV2Config())
preset = in_fresh_context(_maybe_construct_otel_v2, vendor, [generic])
is_otel_v2_enabled.cache_clear()
assert preset is not None
emit(generic.tracer_provider, "from otel")
emit(preset.tracer_provider, "from preset")
generic.tracer_provider.force_flush()
preset.tracer_provider.force_flush()
assert [s.name for s in sinks[_COLLECTOR].get_finished_spans()] == ["from otel"]
vendor_sinks = {endpoint: sink for endpoint, sink in sinks.items() if endpoint != _COLLECTOR}
assert [[s.name for s in sink.get_finished_spans()] for sink in vendor_sinks.values()] == [["from preset"]]
@pytest.mark.parametrize("vendor", sorted(_VENDOR_ENV))
def test_a_preset_built_first_does_not_point_otel_at_its_own_backend(self, monkeypatch, vendor):
"""``callbacks: [<vendor>, otel]``: reading the vendor's credentials must not rewrite the
process-wide ``OTEL_*`` env, or the ``otel`` built next inherits the vendor's endpoint and auth."""
sinks = self._capture_exporters(monkeypatch)
self._operator_with(monkeypatch, vendor)
preset = in_fresh_context(_maybe_construct_otel_v2, vendor, [])
generic = build_otel_v2_logger(OpenTelemetryV2Config())
is_otel_v2_enabled.cache_clear()
assert preset is not None
emit(generic.tracer_provider, "from otel")
generic.tracer_provider.force_flush()
assert [(spec.endpoint, spec.headers) for spec in generic.config.exporters] == [(_COLLECTOR, None)]
assert [s.name for s in sinks[_COLLECTOR].get_finished_spans()] == ["from otel"]
@pytest.mark.parametrize("otel_first", [True, False])
def test_the_otel_entry_never_reuses_a_preset_logger_whatever_the_order(self, monkeypatch, otel_first):
from litellm.proxy import proxy_server
self._operator_with(monkeypatch, "langfuse_otel")
monkeypatch.setattr(proxy_server, "open_telemetry_logger", None)
loggers: list[CustomLogger] = []
monkeypatch.setattr(litellm_logging, "_in_memory_loggers", loggers)
def build(name):
if name == "otel":
return _init_custom_logger_compatible_class("otel", None, None)
return _maybe_construct_otel_v2(name, loggers)
order = ("otel", "langfuse_otel") if otel_first else ("langfuse_otel", "otel")
built = {name: in_fresh_context(build, name) for name in order}
is_otel_v2_enabled.cache_clear()
generic, preset = built["otel"], built["langfuse_otel"]
assert isinstance(generic, OpenTelemetryV2) and isinstance(preset, OpenTelemetryV2)
assert generic is not preset
assert generic.callback_name is None and preset.callback_name == "langfuse_otel"
assert [spec.endpoint for spec in generic.config.exporters] == [_COLLECTOR]
assert [spec.endpoint for spec in preset.config.exporters] == ["https://lf.local/api/public/otel"]
assert select_global_otel_v2_logger(loggers) is generic
assert proxy_server.open_telemetry_logger is generic
@pytest.mark.parametrize("otel_first", [True, False])
def test_the_otel_callback_holds_the_proxy_slot_whatever_the_order(self, monkeypatch, otel_first):
from litellm.proxy import proxy_server
monkeypatch.setattr(proxy_server, "open_telemetry_logger", None)
config = OpenTelemetryV2Config(exporters=[ExporterSpec(kind="in_memory")])
def build(name):
return build_otel_v2_logger(config, callback_name=None if name == "otel" else name)
order = ("otel", "langfuse_otel") if otel_first else ("langfuse_otel", "otel")
built = {name: build(name) for name in order}
assert proxy_server.open_telemetry_logger is built["otel"]
def test_a_lone_preset_keeps_the_proxy_slot_and_the_global(self, monkeypatch):
from litellm.proxy import proxy_server
monkeypatch.setattr(proxy_server, "open_telemetry_logger", None)
config = OpenTelemetryV2Config(exporters=[ExporterSpec(kind="in_memory")])
first = build_otel_v2_logger(config, callback_name="langfuse_otel")
second = build_otel_v2_logger(config, callback_name="arize")
assert proxy_server.open_telemetry_logger is first
assert select_global_otel_v2_logger([first, second]) is first
class TestBackendEndpointParity:
def test_arize_follows_its_own_http_endpoint_instead_of_the_grpc_default(self, monkeypatch):
@ -2613,75 +2742,6 @@ class TestEvictionSafety:
assert held.shutdown_calls == 1
class TestCredentialGatedExporters:
def test_layering_a_second_preset_does_not_eat_the_first_gated_exporter(self, monkeypatch):
"""``base.Preset`` advertises ``config_overrides`` layering, and the gated spec
is itself a console exporter with no endpoint."""
credential_less_proxy(monkeypatch)
from litellm.integrations.otel.presets.utils import credential_gated_exporters
once = credential_gated_exporters((), ExporterOwner.LANGFUSE_OTEL)
twice = credential_gated_exporters(once, ExporterOwner.WEAVE_OTEL)
assert [spec.owner for spec in twice] == [ExporterOwner.LANGFUSE_OTEL, ExporterOwner.WEAVE_OTEL]
def test_an_exporter_the_operator_configured_survives(self):
from litellm.integrations.otel.presets.utils import credential_gated_exporters
operator_console = ExporterSpec(kind="console", use_simple_processor=True)
kept = credential_gated_exporters((operator_console,), ExporterOwner.LANGFUSE_OTEL)
assert kept[0] == operator_console
def test_an_otlp_exporter_on_its_default_endpoint_survives(self):
"""``OTEL_EXPORTER=otlp_http`` with no endpoint is a real collector on the SDK's
default port, not the placeholder, so the transport is what tells them apart."""
from litellm.integrations.otel.presets.utils import credential_gated_exporters
operator_otlp = ExporterSpec(kind="otlp_http", endpoint=None, headers=None)
kept = credential_gated_exporters((operator_otlp,), ExporterOwner.LANGFUSE_OTEL)
assert kept[0] == operator_otlp
def test_an_in_memory_exporter_the_operator_asked_for_survives(self):
"""``OTEL_EXPORTER=in_memory`` stores spans, so it is a destination the operator
chose, not the placeholder that stands in for choosing nothing."""
from litellm.integrations.otel.presets.utils import credential_gated_exporters
operator_memory = ExporterSpec(kind="in_memory", endpoint=None, headers=None)
kept = credential_gated_exporters((operator_memory,), ExporterOwner.LANGFUSE_OTEL)
assert kept[0] == operator_memory
def test_the_synthesized_stdout_placeholder_is_dropped(self, monkeypatch):
from litellm.integrations.otel.presets.utils import credential_gated_exporters
for name in _OTEL_SHORTHAND_ENV:
monkeypatch.delenv(name, raising=False)
placeholder = OpenTelemetryV2Config().exporters[0]
kept = credential_gated_exporters((placeholder,), ExporterOwner.LANGFUSE_OTEL)
assert [spec.owner for spec in kept] == [ExporterOwner.LANGFUSE_OTEL]
def test_a_console_exporter_the_operator_named_survives(self, monkeypatch):
"""Same kind, endpoint and headers as the placeholder; only the fact that the
operator set ``OTEL_EXPORTER`` tells them apart."""
from litellm.integrations.otel.presets.utils import credential_gated_exporters
for name in _OTEL_SHORTHAND_ENV:
monkeypatch.delenv(name, raising=False)
monkeypatch.setenv("OTEL_EXPORTER", "console")
operator_console = OpenTelemetryV2Config().exporters[0]
kept = credential_gated_exporters((operator_console,), ExporterOwner.LANGFUSE_OTEL)
assert kept[0] is operator_console
class TestTenantHostSsrfGuard:
"""Anyone who can mint a key can write ``langfuse_host``, so the host it names has
to be one the operator approved."""

View file

@ -1,9 +1,9 @@
"""Multi-backend fan-out: one TracerProvider, *N* SpanProcessors.
"""Multi-backend fan-out within one callback: one TracerProvider, *N* SpanProcessors.
V1 needed a separate ``TracerProvider`` per integration to avoid stepping on
the global. V2 attaches a ``SpanProcessor`` per exporter to the *same*
provider, so the same trace ID lights up every backend — no duplicate spans,
no per-integration provider caches.
A single ``OpenTelemetryV2Config`` may list several exporters (say two ``OTEL_*``
collectors), and its provider attaches one ``SpanProcessor`` per exporter so the same
span reaches each of them. Which exporters a config lists is decided per callback: a
preset only lists the destination it owns, see ``TestDestinationOwnership``.
"""
import pytest
@ -14,8 +14,8 @@ from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
InMemorySpanExporter,
)
from litellm.integrations.otel.model.config import ExporterOwner, ExporterSpec, OpenTelemetryV2Config
from litellm.integrations.otel.plumbing.providers import build_tracer_provider, register_exporter_factory
from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config
from litellm.integrations.otel.plumbing.providers import build_tracer_provider
def test_two_exporters_receive_the_same_span():
@ -54,52 +54,6 @@ def test_two_exporters_receive_the_same_span():
assert spans_a[0].context.span_id == spans_b[0].context.span_id
def _capturing_kind(kind):
exporter = InMemorySpanExporter()
register_exporter_factory(kind, lambda _spec: exporter)
return exporter
def _emit_langfuse_stamped_span(provider):
span = provider.get_tracer("test").start_span("chat proof-model")
span.set_attribute("gen_ai.request.model", "proof-model")
span.set_attribute("langfuse.trace.name", "private-langfuse-only-name")
span.set_attribute("langfuse.observation.input", '[{"role":"user","content":"hi"}]')
span.end()
return span.context.span_id
def test_generic_exporter_does_not_receive_langfuse_attributes_the_langfuse_exporter_keeps():
generic = _capturing_kind("capture_generic_collector")
langfuse = _capturing_kind("capture_langfuse_sink")
cfg = OpenTelemetryV2Config(
exporters=[
ExporterSpec(kind="capture_generic_collector"),
ExporterSpec(kind="capture_langfuse_sink", owner=ExporterOwner.LANGFUSE_OTEL),
]
)
span_id = _emit_langfuse_stamped_span(build_tracer_provider(cfg, tenant_overrides=True))
(generic_span,) = generic.get_finished_spans()
(langfuse_span,) = langfuse.get_finished_spans()
assert generic_span.context.span_id == span_id == langfuse_span.context.span_id
assert generic_span.attributes["gen_ai.request.model"] == "proof-model"
assert [k for k in generic_span.attributes if k.startswith("langfuse.")] == []
assert langfuse_span.attributes["langfuse.trace.name"] == "private-langfuse-only-name"
assert langfuse_span.attributes["langfuse.observation.input"] == '[{"role":"user","content":"hi"}]'
def test_langfuse_mapper_on_a_plain_otlp_exporter_keeps_langfuse_attributes():
sink = _capturing_kind("capture_langfuse_mapper_sink")
cfg = OpenTelemetryV2Config(
mapper_names=["langfuse"], exporters=[ExporterSpec(kind="capture_langfuse_mapper_sink")]
)
_emit_langfuse_stamped_span(build_tracer_provider(cfg))
(span,) = sink.get_finished_spans()
assert span.attributes["langfuse.trace.name"] == "private-langfuse-only-name"
def test_resource_attributes_apply_to_all_exporters():
"""``resource_attributes`` flow through the shared TracerProvider."""
cfg = OpenTelemetryV2Config(