fix(otel v2): keep langfuse.* span attributes off exporters that are not Langfuse sinks

The Langfuse preset builds its exporter list as [*base.exporters, langfuse], and both
the Langfuse mapper and the Langfuse logger's log_pre_api_call stamp langfuse.*
attributes onto spans every exporter of the provider shares, so the operator's generic
OTLP collector received Langfuse-only attributes whenever otel and langfuse_otel ran
together. Wrap every exporter that is not a Langfuse sink in a processor that hands it
a span view without langfuse.* attributes, and apply the same rule to per-request
tenant destinations in the fan-out.

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
yucheng 2026-09-12 19:02:57 +00:00
parent 27f8d2a9ba
commit e46858b5ee
3 changed files with 135 additions and 6 deletions

View file

@ -41,7 +41,7 @@ from opentelemetry.util.types import Attributes, AttributeValue
from litellm._logging import verbose_logger
from litellm._version import version as litellm_version
from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config
from litellm.integrations.otel.model.config import ExporterOwner, ExporterSpec, OpenTelemetryV2Config
from litellm.integrations.otel.model.semconv import (
DB,
MCP,
@ -377,6 +377,7 @@ _CAPTURED_HEADER_PREFIXES: Final = ("http.request.header.", "http.response.heade
# ``?key=`` query parameter.
_URL_KEYS: Final = frozenset({"http.url", "http.target", "url.full"})
_URL_QUERY_KEY: Final = "url.query"
_LANGFUSE_ATTRIBUTE_PREFIX: Final = "langfuse."
class _TenantSpanView(ReadableSpan):
@ -418,9 +419,11 @@ def _guardrail_unreachable(attributes: Mapping[str, AttributeValue]) -> bool:
return attributes.get(LiteLLM.GUARDRAIL_STATUS) in _GUARDRAIL_UNREACHABLE_STATUSES
def _tenant_visible(key: str, database: bool, owned: bool, unreachable_guardrail: bool) -> bool:
def _tenant_visible(key: str, database: bool, owned: bool, unreachable_guardrail: bool, langfuse: bool) -> bool:
if key.startswith(_CAPTURED_HEADER_PREFIXES) or key in (LiteLLMError.STACK_TRACE, _URL_QUERY_KEY):
return False
if not langfuse and key.startswith(_LANGFUSE_ATTRIBUTE_PREFIX):
return False
if database and key in _DATASTORE_ENDPOINT_KEYS:
return False
if unreachable_guardrail and key == LiteLLM.GUARDRAIL_RESPONSE:
@ -470,11 +473,12 @@ def _for_destination(span: ReadableSpan, destination: "OtelDestination") -> Read
database: Final = _is_database_span(attributes)
owned: Final = _is_tenant_owned_span(attributes)
unreachable: Final = _guardrail_unreachable(attributes)
langfuse: Final = destination.callback_name == ExporterOwner.LANGFUSE_OTEL.value
kept: Final = MappingProxyType(
{
key: _without_query(key, value)
for key, value in attributes.items()
if _tenant_visible(key, database, owned, unreachable)
if _tenant_visible(key, database, owned, unreachable, langfuse)
}
)
recorded: Final = span.events
@ -487,6 +491,30 @@ def _for_destination(span: ReadableSpan, destination: "OtelDestination") -> Read
return _TenantSpanView(span, resource, kept, events, status)
def _without_langfuse_attributes(span: ReadableSpan) -> ReadableSpan:
attributes: Final = span.attributes or _NO_ATTRIBUTES
kept: Final = MappingProxyType(
{key: value for key, value in attributes.items() if not key.startswith(_LANGFUSE_ATTRIBUTE_PREFIX)}
)
if len(kept) == len(attributes):
return span
return _TenantSpanView(span, span.resource, kept, span.events, span.status)
def _reads_langfuse_attributes(spec: ExporterSpec, config: OpenTelemetryV2Config) -> bool:
"""Whether ``spec`` is the sink the ``langfuse.*`` attributes on a span are meant for.
An unowned exporter is the operator's generic collector, unless the config runs the
``langfuse`` mapper with no Langfuse-owned exporter to deliver it (the mapper pointed
at ``OTEL_EXPORTER_*`` directly).
"""
if spec.owner is not None:
return spec.owner is ExporterOwner.LANGFUSE_OTEL
return "langfuse" in config.mapper_names and all(
other.owner is not ExporterOwner.LANGFUSE_OTEL for other in config.exporters
)
class TenantFanOutSpanProcessor(SpanProcessor):
"""Export every finished span to each destination this request resolved.
@ -774,6 +802,26 @@ class _OverriddenBackendFilter(SpanProcessor):
return self._inner.force_flush(timeout_millis)
class _LangfuseAttributeFilter(SpanProcessor):
"""Strip the ``langfuse.*`` attributes a Langfuse callback stamps on the shared spans
before they reach an exporter that is not a Langfuse sink."""
def __init__(self, inner: SpanProcessor) -> None:
self._inner: Final = inner
def on_start(self, span: SDKSpan, parent_context: Context | None = None) -> None:
self._inner.on_start(span, parent_context)
def on_end(self, span: ReadableSpan) -> None:
self._inner.on_end(_without_langfuse_attributes(span))
def shutdown(self) -> None:
self._inner.shutdown()
def force_flush(self, timeout_millis: int = 30000) -> bool:
return self._inner.force_flush(timeout_millis)
def build_span_exporter(config: OpenTelemetryV2Config) -> SpanExporter:
"""Build a single exporter from the top-level config fields.
@ -1060,9 +1108,10 @@ def build_tracer_provider(
exp,
(spec.use_simple_processor if spec.use_simple_processor is not None else use_simple_processor),
)
sink = processor if _reads_langfuse_attributes(spec, config) else _LangfuseAttributeFilter(processor)
owner = spec.owner.value if spec.owner is not None else None
provider.add_span_processor(
_OverriddenBackendFilter(processor, owner) if tenant_overrides and owner is not None else processor
_OverriddenBackendFilter(sink, owner) if tenant_overrides and owner is not None else sink
)
return provider

View file

@ -516,6 +516,34 @@ class TestFanOut:
assert [s.name for s in langfuse.get_finished_spans()] == ["chat gpt-4"]
assert [s.name for s in arize.get_finished_spans()] == ["chat gpt-4"]
def test_only_the_langfuse_destination_receives_the_langfuse_attributes(self):
langfuse, arize = InMemorySpanExporter(), InMemorySpanExporter()
by_endpoint = {"http://a.local": langfuse, "http://b.local": arize}
provider = TracerProvider()
provider.add_span_processor(
TenantFanOutSpanProcessor(processor_factory=lambda d: SimpleSpanProcessor(by_endpoint[d.endpoint]))
)
def run():
set_request_destinations(
(
OtelDestination(endpoint="http://a.local", callback_name="langfuse_otel"),
OtelDestination(endpoint="http://b.local", callback_name="arize"),
)
)
span = get_tracer(provider, "litellm").start_span("chat gpt-4")
span.set_attribute("gen_ai.request.model", "gpt-4")
span.set_attribute("langfuse.trace.name", "private-langfuse-only-name")
span.end()
in_fresh_context(run)
(langfuse_span,) = langfuse.get_finished_spans()
(arize_span,) = arize.get_finished_spans()
assert langfuse_span.attributes["langfuse.trace.name"] == "private-langfuse-only-name"
assert arize_span.attributes["gen_ai.request.model"] == "gpt-4"
assert [k for k in arize_span.attributes if k.startswith("langfuse.")] == []
def test_a_destination_carries_the_tenants_service_name(self):
"""An overridden backend skips per-request tracer routing, so the service name
that route used to apply has to travel on the destination instead."""

View file

@ -14,8 +14,8 @@ from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
InMemorySpanExporter,
)
from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config
from litellm.integrations.otel.plumbing.providers import build_tracer_provider
from litellm.integrations.otel.model.config import ExporterOwner, ExporterSpec, OpenTelemetryV2Config
from litellm.integrations.otel.plumbing.providers import build_tracer_provider, register_exporter_factory
def test_two_exporters_receive_the_same_span():
@ -54,6 +54,58 @@ def test_two_exporters_receive_the_same_span():
assert spans_a[0].context.span_id == spans_b[0].context.span_id
def _capturing_kind(kind):
"""Register ``kind`` so every ExporterSpec of that kind exports into one buffer."""
exporter = InMemorySpanExporter()
register_exporter_factory(kind, lambda _spec: exporter)
return exporter
def _emit_langfuse_stamped_span(provider):
"""The shared span both a generic ``otel`` and a ``langfuse_otel`` callback write to."""
span = provider.get_tracer("test").start_span("chat proof-model")
span.set_attribute("gen_ai.request.model", "proof-model")
span.set_attribute("langfuse.trace.name", "private-langfuse-only-name")
span.set_attribute("langfuse.observation.input", '[{"role":"user","content":"hi"}]')
span.end()
return span.context.span_id
def test_generic_exporter_does_not_receive_langfuse_attributes_the_langfuse_exporter_keeps():
"""``otel`` + ``langfuse_otel`` together: the operator's own collector must see the
OTel GenAI vocabulary only, while the Langfuse sink keeps its trace name and I/O."""
generic = _capturing_kind("capture_generic_collector")
langfuse = _capturing_kind("capture_langfuse_sink")
cfg = OpenTelemetryV2Config(
exporters=[
ExporterSpec(kind="capture_generic_collector"),
ExporterSpec(kind="capture_langfuse_sink", owner=ExporterOwner.LANGFUSE_OTEL),
]
)
span_id = _emit_langfuse_stamped_span(build_tracer_provider(cfg, tenant_overrides=True))
(generic_span,) = generic.get_finished_spans()
(langfuse_span,) = langfuse.get_finished_spans()
assert generic_span.context.span_id == span_id == langfuse_span.context.span_id
assert generic_span.attributes["gen_ai.request.model"] == "proof-model"
assert [k for k in generic_span.attributes if k.startswith("langfuse.")] == []
assert langfuse_span.attributes["langfuse.trace.name"] == "private-langfuse-only-name"
assert langfuse_span.attributes["langfuse.observation.input"] == '[{"role":"user","content":"hi"}]'
def test_langfuse_mapper_on_a_plain_otlp_exporter_keeps_langfuse_attributes():
"""``otel`` alone with ``mapper_names: [langfuse]`` points the mapper at the operator's
exporter on purpose, so that exporter is the Langfuse sink and nothing is stripped."""
sink = _capturing_kind("capture_langfuse_mapper_sink")
cfg = OpenTelemetryV2Config(
mapper_names=["langfuse"], exporters=[ExporterSpec(kind="capture_langfuse_mapper_sink")]
)
_emit_langfuse_stamped_span(build_tracer_provider(cfg))
(span,) = sink.get_finished_spans()
assert span.attributes["langfuse.trace.name"] == "private-langfuse-only-name"
def test_resource_attributes_apply_to_all_exporters():
"""``resource_attributes`` flow through the shared TracerProvider."""
cfg = OpenTelemetryV2Config(