From e46858b5ee90eb853db5858864667cbaefe7d3f7 Mon Sep 17 00:00:00 2001 From: yucheng Date: Sat, 12 Sep 2026 19:02:57 +0000 Subject: [PATCH] fix(otel v2): keep langfuse.* span attributes off exporters that are not Langfuse sinks The Langfuse preset builds its exporter list as [*base.exporters, langfuse], and both the Langfuse mapper and the Langfuse logger's log_pre_api_call stamp langfuse.* attributes onto spans every exporter of the provider shares, so the operator's generic OTLP collector received Langfuse-only attributes whenever otel and langfuse_otel ran together. Wrap every exporter that is not a Langfuse sink in a processor that hands it a span view without langfuse.* attributes, and apply the same rule to per-request tenant destinations in the fan-out. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../integrations/otel/plumbing/providers.py | 57 +++++++++++++++++-- .../otel/test_otel_v2_destinations.py | 28 +++++++++ .../otel/test_otel_v2_multibackend.py | 56 +++++++++++++++++- 3 files changed, 135 insertions(+), 6 deletions(-) diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index 81f22c8c642..24595a66628 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -41,7 +41,7 @@ from opentelemetry.util.types import Attributes, AttributeValue from litellm._logging import verbose_logger from litellm._version import version as litellm_version -from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config +from litellm.integrations.otel.model.config import ExporterOwner, ExporterSpec, OpenTelemetryV2Config from litellm.integrations.otel.model.semconv import ( DB, MCP, @@ -377,6 +377,7 @@ _CAPTURED_HEADER_PREFIXES: Final = ("http.request.header.", "http.response.heade # ``?key=`` query parameter. _URL_KEYS: Final = frozenset({"http.url", "http.target", "url.full"}) _URL_QUERY_KEY: Final = "url.query" +_LANGFUSE_ATTRIBUTE_PREFIX: Final = "langfuse." class _TenantSpanView(ReadableSpan): @@ -418,9 +419,11 @@ def _guardrail_unreachable(attributes: Mapping[str, AttributeValue]) -> bool: return attributes.get(LiteLLM.GUARDRAIL_STATUS) in _GUARDRAIL_UNREACHABLE_STATUSES -def _tenant_visible(key: str, database: bool, owned: bool, unreachable_guardrail: bool) -> bool: +def _tenant_visible(key: str, database: bool, owned: bool, unreachable_guardrail: bool, langfuse: bool) -> bool: if key.startswith(_CAPTURED_HEADER_PREFIXES) or key in (LiteLLMError.STACK_TRACE, _URL_QUERY_KEY): return False + if not langfuse and key.startswith(_LANGFUSE_ATTRIBUTE_PREFIX): + return False if database and key in _DATASTORE_ENDPOINT_KEYS: return False if unreachable_guardrail and key == LiteLLM.GUARDRAIL_RESPONSE: @@ -470,11 +473,12 @@ def _for_destination(span: ReadableSpan, destination: "OtelDestination") -> Read database: Final = _is_database_span(attributes) owned: Final = _is_tenant_owned_span(attributes) unreachable: Final = _guardrail_unreachable(attributes) + langfuse: Final = destination.callback_name == ExporterOwner.LANGFUSE_OTEL.value kept: Final = MappingProxyType( { key: _without_query(key, value) for key, value in attributes.items() - if _tenant_visible(key, database, owned, unreachable) + if _tenant_visible(key, database, owned, unreachable, langfuse) } ) recorded: Final = span.events @@ -487,6 +491,30 @@ def _for_destination(span: ReadableSpan, destination: "OtelDestination") -> Read return _TenantSpanView(span, resource, kept, events, status) +def _without_langfuse_attributes(span: ReadableSpan) -> ReadableSpan: + attributes: Final = span.attributes or _NO_ATTRIBUTES + kept: Final = MappingProxyType( + {key: value for key, value in attributes.items() if not key.startswith(_LANGFUSE_ATTRIBUTE_PREFIX)} + ) + if len(kept) == len(attributes): + return span + return _TenantSpanView(span, span.resource, kept, span.events, span.status) + + +def _reads_langfuse_attributes(spec: ExporterSpec, config: OpenTelemetryV2Config) -> bool: + """Whether ``spec`` is the sink the ``langfuse.*`` attributes on a span are meant for. + + An unowned exporter is the operator's generic collector, unless the config runs the + ``langfuse`` mapper with no Langfuse-owned exporter to deliver it (the mapper pointed + at ``OTEL_EXPORTER_*`` directly). + """ + if spec.owner is not None: + return spec.owner is ExporterOwner.LANGFUSE_OTEL + return "langfuse" in config.mapper_names and all( + other.owner is not ExporterOwner.LANGFUSE_OTEL for other in config.exporters + ) + + class TenantFanOutSpanProcessor(SpanProcessor): """Export every finished span to each destination this request resolved. @@ -774,6 +802,26 @@ class _OverriddenBackendFilter(SpanProcessor): return self._inner.force_flush(timeout_millis) +class _LangfuseAttributeFilter(SpanProcessor): + """Strip the ``langfuse.*`` attributes a Langfuse callback stamps on the shared spans + before they reach an exporter that is not a Langfuse sink.""" + + def __init__(self, inner: SpanProcessor) -> None: + self._inner: Final = inner + + def on_start(self, span: SDKSpan, parent_context: Context | None = None) -> None: + self._inner.on_start(span, parent_context) + + def on_end(self, span: ReadableSpan) -> None: + self._inner.on_end(_without_langfuse_attributes(span)) + + def shutdown(self) -> None: + self._inner.shutdown() + + def force_flush(self, timeout_millis: int = 30000) -> bool: + return self._inner.force_flush(timeout_millis) + + def build_span_exporter(config: OpenTelemetryV2Config) -> SpanExporter: """Build a single exporter from the top-level config fields. @@ -1060,9 +1108,10 @@ def build_tracer_provider( exp, (spec.use_simple_processor if spec.use_simple_processor is not None else use_simple_processor), ) + sink = processor if _reads_langfuse_attributes(spec, config) else _LangfuseAttributeFilter(processor) owner = spec.owner.value if spec.owner is not None else None provider.add_span_processor( - _OverriddenBackendFilter(processor, owner) if tenant_overrides and owner is not None else processor + _OverriddenBackendFilter(sink, owner) if tenant_overrides and owner is not None else sink ) return provider diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index 67695d5aed8..b2c79d43f29 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -516,6 +516,34 @@ class TestFanOut: assert [s.name for s in langfuse.get_finished_spans()] == ["chat gpt-4"] assert [s.name for s in arize.get_finished_spans()] == ["chat gpt-4"] + def test_only_the_langfuse_destination_receives_the_langfuse_attributes(self): + langfuse, arize = InMemorySpanExporter(), InMemorySpanExporter() + by_endpoint = {"http://a.local": langfuse, "http://b.local": arize} + provider = TracerProvider() + provider.add_span_processor( + TenantFanOutSpanProcessor(processor_factory=lambda d: SimpleSpanProcessor(by_endpoint[d.endpoint])) + ) + + def run(): + set_request_destinations( + ( + OtelDestination(endpoint="http://a.local", callback_name="langfuse_otel"), + OtelDestination(endpoint="http://b.local", callback_name="arize"), + ) + ) + span = get_tracer(provider, "litellm").start_span("chat gpt-4") + span.set_attribute("gen_ai.request.model", "gpt-4") + span.set_attribute("langfuse.trace.name", "private-langfuse-only-name") + span.end() + + in_fresh_context(run) + + (langfuse_span,) = langfuse.get_finished_spans() + (arize_span,) = arize.get_finished_spans() + assert langfuse_span.attributes["langfuse.trace.name"] == "private-langfuse-only-name" + assert arize_span.attributes["gen_ai.request.model"] == "gpt-4" + assert [k for k in arize_span.attributes if k.startswith("langfuse.")] == [] + def test_a_destination_carries_the_tenants_service_name(self): """An overridden backend skips per-request tracer routing, so the service name that route used to apply has to travel on the destination instead.""" diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_multibackend.py b/tests/test_litellm/integrations/otel/test_otel_v2_multibackend.py index e879766c5c7..dc5489db3f2 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_multibackend.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_multibackend.py @@ -14,8 +14,8 @@ from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( InMemorySpanExporter, ) -from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config -from litellm.integrations.otel.plumbing.providers import build_tracer_provider +from litellm.integrations.otel.model.config import ExporterOwner, ExporterSpec, OpenTelemetryV2Config +from litellm.integrations.otel.plumbing.providers import build_tracer_provider, register_exporter_factory def test_two_exporters_receive_the_same_span(): @@ -54,6 +54,58 @@ def test_two_exporters_receive_the_same_span(): assert spans_a[0].context.span_id == spans_b[0].context.span_id +def _capturing_kind(kind): + """Register ``kind`` so every ExporterSpec of that kind exports into one buffer.""" + exporter = InMemorySpanExporter() + register_exporter_factory(kind, lambda _spec: exporter) + return exporter + + +def _emit_langfuse_stamped_span(provider): + """The shared span both a generic ``otel`` and a ``langfuse_otel`` callback write to.""" + span = provider.get_tracer("test").start_span("chat proof-model") + span.set_attribute("gen_ai.request.model", "proof-model") + span.set_attribute("langfuse.trace.name", "private-langfuse-only-name") + span.set_attribute("langfuse.observation.input", '[{"role":"user","content":"hi"}]') + span.end() + return span.context.span_id + + +def test_generic_exporter_does_not_receive_langfuse_attributes_the_langfuse_exporter_keeps(): + """``otel`` + ``langfuse_otel`` together: the operator's own collector must see the + OTel GenAI vocabulary only, while the Langfuse sink keeps its trace name and I/O.""" + generic = _capturing_kind("capture_generic_collector") + langfuse = _capturing_kind("capture_langfuse_sink") + cfg = OpenTelemetryV2Config( + exporters=[ + ExporterSpec(kind="capture_generic_collector"), + ExporterSpec(kind="capture_langfuse_sink", owner=ExporterOwner.LANGFUSE_OTEL), + ] + ) + span_id = _emit_langfuse_stamped_span(build_tracer_provider(cfg, tenant_overrides=True)) + + (generic_span,) = generic.get_finished_spans() + (langfuse_span,) = langfuse.get_finished_spans() + assert generic_span.context.span_id == span_id == langfuse_span.context.span_id + assert generic_span.attributes["gen_ai.request.model"] == "proof-model" + assert [k for k in generic_span.attributes if k.startswith("langfuse.")] == [] + assert langfuse_span.attributes["langfuse.trace.name"] == "private-langfuse-only-name" + assert langfuse_span.attributes["langfuse.observation.input"] == '[{"role":"user","content":"hi"}]' + + +def test_langfuse_mapper_on_a_plain_otlp_exporter_keeps_langfuse_attributes(): + """``otel`` alone with ``mapper_names: [langfuse]`` points the mapper at the operator's + exporter on purpose, so that exporter is the Langfuse sink and nothing is stripped.""" + sink = _capturing_kind("capture_langfuse_mapper_sink") + cfg = OpenTelemetryV2Config( + mapper_names=["langfuse"], exporters=[ExporterSpec(kind="capture_langfuse_mapper_sink")] + ) + _emit_langfuse_stamped_span(build_tracer_provider(cfg)) + + (span,) = sink.get_finished_spans() + assert span.attributes["langfuse.trace.name"] == "private-langfuse-only-name" + + def test_resource_attributes_apply_to_all_exporters(): """``resource_attributes`` flow through the shared TracerProvider.""" cfg = OpenTelemetryV2Config(