fix(otel v2): send only the already-exported end user as user.id outside Langfuse

A caller's trace_user_id stays a Langfuse-only control, so non-Langfuse
collectors get no value they did not already receive as
litellm.metadata.user_api_key_end_user_id. user.id also sheds first at the
span attribute limit, so it never costs an indexed message attribute.

litellm/integrations/otel/emitter.py

@@ -13,6 +13,7 @@ from opentelemetry.trace.status import Status, StatusCode
 
 from litellm.integrations.otel.mappers import resolve_mappers
 from litellm.integrations.otel.mappers.base import AttributeMapper, AttrValue, SpanData
+from litellm.integrations.otel.mappers.langfuse import LangfuseMapper
 from litellm.integrations.otel.mappers.openinference import fit_indexed_messages
 from litellm.integrations.otel.model.config import OpenTelemetryV2Config
 from litellm.integrations.otel.model.payloads import (
@@ -23,7 +24,7 @@ from litellm.integrations.otel.model.payloads import (
     ServiceSpanData,
     SpanError,
 )
-from litellm.integrations.otel.model.semconv import Error, ExceptionEvent, LiteLLMError
+from litellm.integrations.otel.model.semconv import Error, ExceptionEvent, LiteLLMError, User
 from litellm.integrations.otel.model.spans import (
     SPAN_REGISTRY,
     SpanRole,
@@ -171,6 +172,11 @@ class SpanEmitter:
         self._mappers: list[AttributeMapper] = (
             list(mappers) if mappers is not None else resolve_mappers(config.mapper_names)
         )
+        self._shed_first: Final = (
+            frozenset[str]()
+            if any(isinstance(mapper, LangfuseMapper) for mapper in self._mappers)
+            else frozenset({User.ID})
+        )
         # Bounded LRU (ordered by insertion / most-recent touch). Storing keys
         # only — the value is unused — so it behaves like a capped set.
         self._emitted: OrderedDict[tuple[str, SpanRole], None] = OrderedDict()
@@ -303,7 +309,8 @@ class SpanEmitter:
         )
         stamped_later: Final = error_attributes(error) if error else _NO_ATTRIBUTES
         reserved: Final = len(stamped_later.keys() - mapped.keys())
-        for key, value in fit_indexed_messages(mapped, attribute_budget(span, reserved, frozenset(mapped))).items():
+        budget: Final = attribute_budget(span, reserved, frozenset(mapped))
+        for key, value in fit_indexed_messages(mapped, budget, self._shed_first).items():
             span.set_attribute(key, value)
         if error:
             stamped: Final = stamp_error(span, error)

litellm/integrations/otel/mappers/genai.py

@@ -39,6 +39,7 @@ from litellm.integrations.otel.model.semconv import (
     User,
 )
 from litellm.integrations.otel.model.spans import postgres_operation
+from litellm.integrations.otel.model.trace_controls import END_USER_METADATA_KEY
 
 
 class GenAIMapper:
@@ -47,7 +48,7 @@ class GenAIMapper:
         GenAI.PROVIDER_NAME: lambda d: d.provider or None,
         GenAI.OUTPUT_TYPE: lambda d: d.output_type.value if d.output_type else None,
         GenAI.CONVERSATION_ID: lambda d: d.session_id,
-        User.ID: lambda d: d.trace.user_id,
+        User.ID: lambda d: d.promoted_metadata.get(END_USER_METADATA_KEY),
         GenAI.REQUEST_MODEL: lambda d: d.request_model or None,
         GenAI.REQUEST_TEMPERATURE: lambda d: d.request_params.temperature,
         GenAI.REQUEST_TOP_P: lambda d: d.request_params.top_p,

litellm/integrations/otel/mappers/openinference.py

@@ -153,7 +153,9 @@ def _shed_order(groups: Mapping[tuple[str, int, int], tuple[str, ...]]) -> tuple
 _METADATA_KEY: Final = "metadata"
 
 
-def fit_indexed_messages(attrs: Mapping[str, AttrValue], budget: int | None) -> Mapping[str, AttrValue]:
+def fit_indexed_messages(
+    attrs: Mapping[str, AttrValue], budget: int | None, shed_first: frozenset[str] = frozenset()
+) -> Mapping[str, AttrValue]:
     """``attrs`` with indexed message attributes shed, least valuable first, until at most ``budget`` keys remain.
 
     ``None`` means the span has no attribute count limit. Every message still rides the ``input.value`` and
@@ -161,12 +163,14 @@ def fit_indexed_messages(attrs: Mapping[str, AttrValue], budget: int | None) ->
     after every indexed message attribute: message attributes are the indexed, queryable view (the blobs
     carry no per-index keys), so a count-squeezed span keeps them and the single metadata key absorbs only
     the residual shortfall. Shedding happens here, before ``span.set_attribute``, so the fit is exact and
-    the SDK's dropped-attributes counter never silently masks the choice.
+    the SDK's dropped-attributes counter never silently masks the choice. ``shed_first`` keys shed one at a
+    time before anything else, so they never cost the span any other attribute.
     """
     if budget is None or len(attrs) <= budget:
         return attrs
     groups: Final = _message_key_groups(attrs)
-    sheddable: Final = (*(groups[group] for group in _shed_order(groups)),)
+    leading: Final = tuple((key,) for key in attrs if key in shed_first)
+    sheddable: Final = (*leading, *(groups[group] for group in _shed_order(groups)))
     flex: Final = (_METADATA_KEY,) if _METADATA_KEY in attrs else ()
     candidates: Final = (*sheddable, flex) if flex else sheddable
     running: Final = tuple(accumulate(len(candidate) for candidate in candidates))

litellm/integrations/otel/model/trace_controls.py

@@ -19,9 +19,8 @@ _ITEMS: Final = TypeAdapter(tuple[object, ...])
 class TraceControls:
     """The caller's trace-level Langfuse controls: ``metadata.trace_name`` / ``trace_user_id`` / ``session_id`` /
     ``tags`` on the request (SDK or proxy body), with the proxy's ``langfuse_<control>`` headers winning over the
-    body for the scalar ones. ``request_trace_controls`` fills a missing ``user_id`` with the proxy end user, which
-    every v2 callback then sends as ``user.id``. Mutation controls (``trace_id``, ``existing_trace_id``,
-    ``update_trace_keys``) are deliberately not carried."""
+    body for the scalar ones. ``request_trace_controls`` fills a missing ``user_id`` with the proxy end user. Mutation
+    controls (``trace_id``, ``existing_trace_id``, ``update_trace_keys``) are deliberately not carried."""
 
     name: str | None = None
     user_id: str | None = None

tests/integration/observability/test_otel_v2_user_id_every_callback.py

@@ -15,6 +15,7 @@ _OPENINFERENCE_CALLBACKS: Final = ("arize", "arize_phoenix", "weave_otel")
 _GENAI_CALLBACKS: Final = ("otel", "langfuse_otel", "langtrace", "signoz", "levo", "newrelic", "agentops")
 _UNUSED_ENDPOINT: Final[dict[str, JsonValue]] = {"otel": {"exporter": "http/protobuf", "endpoint": "unused"}}
 _METADATA_ALLOWLIST: Final = "LITELLM_OTEL_BAGGAGE_METADATA_KEYS"
+_LANGFUSE_CALLBACK: Final = "langfuse_otel"
 
 
 def _settings(callback: str) -> Mapping[str, JsonValue] | None:
@@ -65,6 +66,11 @@ def _model_call_spans(destination: Wire, markers: tuple[str, ...]) -> Mapping[st
     return eventually(by_marker, lambda found: all(found.values()), seconds=30)
 
 
+def _carrying(spans: Mapping[str, tuple[dict[str, str], ...]], value: str) -> tuple[str, ...]:
+    model_call_spans: Final = itertools.chain.from_iterable(spans.values())
+    return tuple(key for span in model_call_spans for key, attribute in span.items() if value in str(attribute))
+
+
 def _chat(rig: Rig, marker: str, key: str, headers: Mapping[str, str], extra: Mapping[str, JsonValue]) -> None:
     response: Final = rig.proxy.request(
         "POST",
@@ -81,7 +87,7 @@ def test_otel_v2_callback_sends_the_request_user_as_user_id(callback: str, gatew
     run: Final = uuid.uuid4().hex
     end_user: Final = f"end-user-{run}"
     caller: Final = f"caller-{run}"
-    markers: Final = (f"end-{run}", f"caller-{run}-marker", f"owner-{run}")
+    markers: Final = (f"end-{run}", f"trace-{run}", f"owner-{run}")
     with (
         _rig(
             gateway,
@@ -106,9 +112,11 @@ def test_otel_v2_callback_sends_the_request_user_as_user_id(callback: str, gatew
             _chat(rig, marker, key, headers, extra)
         spans: Final = _model_call_spans(rig.destination, markers)
     users: Final = {marker: {attributes.get("user.id") for attributes in spans[marker]} for marker in markers}
-    assert users == {markers[0]: {end_user}, markers[1]: {caller}, markers[2]: {None}}, spans
+    caller_user: Final = caller if callback == _LANGFUSE_CALLBACK else end_user
+    assert users == {markers[0]: {end_user}, markers[1]: {caller_user}, markers[2]: {None}}, spans
     sessions: Final = {attributes.get("session.id") for attributes in itertools.chain.from_iterable(spans.values())}
     assert end_user not in sessions, spans
+    assert callback == _LANGFUSE_CALLBACK or not _carrying(spans, caller), spans
 
 
 @pytest.mark.parametrize("callback", ("arize", "otel", "langfuse_otel"))
@@ -118,7 +126,7 @@ def test_otel_v2_callback_withholds_an_end_user_the_metadata_allowlist_drops(
     run: Final = uuid.uuid4().hex
     end_user: Final = f"end-user-{run}"
     caller: Final = f"caller-{run}"
-    markers: Final = (f"end-{run}", f"caller-{run}-marker")
+    markers: Final = (f"end-{run}", f"trace-{run}")
     with (
         _rig(
             gateway,
@@ -137,7 +145,6 @@ def test_otel_v2_callback_withholds_an_end_user_the_metadata_allowlist_drops(
         )
         spans: Final = _model_call_spans(rig.destination, markers)
     users: Final = {marker: {attributes.get("user.id") for attributes in spans[marker]} for marker in markers}
-    assert users == {markers[0]: {None}, markers[1]: {caller}}, spans
-    model_call_spans: Final = tuple(itertools.chain.from_iterable(spans.values()))
-    values: Final = tuple(itertools.chain.from_iterable(span.values() for span in model_call_spans))
-    assert not [value for value in values if end_user in str(value)], spans
+    assert users == {markers[0]: {None}, markers[1]: {caller if callback == _LANGFUSE_CALLBACK else None}}, spans
+    assert not _carrying(spans, end_user), spans
+    assert callback == _LANGFUSE_CALLBACK or not _carrying(spans, caller), spans

tests/unit/integrations/otel/test_otel_v2_logger.py

@@ -45,6 +45,7 @@ from litellm.integrations.otel import (  # noqa: E402
     OpenTelemetryV2Config,
 )
 from litellm.integrations.otel.logger import OpenTelemetryV2  # noqa: E402
+from litellm.integrations.otel.mappers.genai import GenAIMapper  # noqa: E402
 from litellm.integrations.otel.model.config import CaptureMessageContent, ExporterSpec  # noqa: E402
 from litellm.integrations.otel.model.destination import OtelDestination  # noqa: E402
 from litellm.integrations.otel.model.semconv import User  # noqa: E402
@@ -302,38 +303,80 @@ def test_replayed_llm_call_span_does_not_take_the_payloads_session_id():
     assert GenAI.CONVERSATION_ID not in span.attributes
 
 
+def _user_kwargs(metadata, messages=None, call_id="call_1"):
+    payload: Final = _payload(metadata={**_payload()["metadata"], **metadata}, litellm_call_id=call_id)
+    if messages is not None:
+        payload["messages"] = messages
+    return {**_kwargs(payload), "litellm_params": {"metadata": metadata}, "messages": messages or []}
+
+
+def _logger_with(**config):
+    cfg: Final = OpenTelemetryV2Config(exporter="in_memory", **config)
+    exporter: Final = InMemorySpanExporter()
+    return OpenTelemetryV2(
+        config=cfg, tracer_provider=providers.build_tracer_provider(cfg, exporter=exporter)
+    ), exporter
+
+
 @pytest.mark.parametrize(
     ("metadata", "expected"),
     [
         ({"user_api_key_end_user_id": "end-1"}, "end-1"),
-        ({"trace_user_id": "caller-1", "user_api_key_end_user_id": "end-1"}, "caller-1"),
+        ({"trace_user_id": "caller-1", "user_api_key_end_user_id": "end-1"}, "end-1"),
+        ({"trace_user_id": "caller-1"}, None),
         ({"user_api_key_user_id": "internal-1"}, None),
     ],
-    ids=["proxy-end-user", "caller-trace-user-wins", "internal-user-never-sent"],
+    ids=["proxy-end-user", "caller-trace-user-not-sent", "caller-trace-user-alone", "internal-user-never-sent"],
 )
-def test_llm_call_span_carries_the_request_user_as_user_id(metadata, expected):
+def test_llm_call_span_carries_the_end_user_as_user_id(metadata, expected):
     logger, exporter = _logger()
-    _emit_llm(logger, {**_kwargs(), "litellm_params": {"metadata": metadata}})
+    _emit_llm(logger, _user_kwargs(metadata))
     (span,) = exporter.get_finished_spans()
     assert span.attributes.get(User.ID) == expected
+    assert "caller-1" not in {str(value) for value in span.attributes.values()}
 
 
 @pytest.mark.parametrize(
-    ("metadata", "expected"),
-    [
-        ({"user_api_key_end_user_id": "end-1"}, None),
-        ({"trace_user_id": "caller-1", "user_api_key_end_user_id": "end-1"}, "caller-1"),
-    ],
-    ids=["end-user-withheld", "caller-trace-user-still-sent"],
+    "metadata",
+    [{"user_api_key_end_user_id": "end-1"}, {"trace_user_id": "caller-1", "user_api_key_end_user_id": "end-1"}],
+    ids=["end-user", "end-user-and-caller-trace-user"],
 )
-def test_llm_call_span_withholds_an_end_user_the_metadata_allowlist_drops(metadata, expected):
-    cfg: Final = OpenTelemetryV2Config(exporter="in_memory", baggage_metadata_keys=["user_api_key_org_id"])
-    exporter: Final = InMemorySpanExporter()
-    logger: Final = OpenTelemetryV2(config=cfg, tracer_provider=providers.build_tracer_provider(cfg, exporter=exporter))
-    _emit_llm(logger, {**_kwargs(), "litellm_params": {"metadata": metadata}})
+def test_llm_call_span_withholds_an_end_user_the_metadata_allowlist_drops(metadata):
+    logger, exporter = _logger_with(baggage_metadata_keys=["user_api_key_org_id"])
+    _emit_llm(logger, _user_kwargs(metadata))
     (span,) = exporter.get_finished_spans()
-    assert span.attributes.get(User.ID) == expected
-    assert "end-1" not in {str(value) for value in span.attributes.values()}
+    assert User.ID not in span.attributes
+    assert not {"end-1", "caller-1"} & {str(value) for value in span.attributes.values()}
+
+
+def test_user_id_costs_no_other_attribute_at_the_span_limit(monkeypatch):
+    def spans() -> list:
+        logger, exporter = _logger_with(
+            mapper_names=["genai", "openinference"], capture_message_content=CaptureMessageContent.SPAN_ONLY
+        )
+        for count in range(40, 70):
+            messages = [{"role": "user", "content": f"m{index}"} for index in range(count)]
+            _emit_llm(logger, _user_kwargs({"user_api_key_end_user_id": "end-1"}, messages, f"call_{count}"))
+        return list(exporter.get_finished_spans())
+
+    with_user_id: Final = spans()
+    monkeypatch.delitem(GenAIMapper._LLM_CALL_ATTRS, User.ID)
+    without_user_id: Final = spans()
+    assert {span.attributes.get(User.ID) for span in with_user_id} == {"end-1", None}
+    for with_user, without in zip(with_user_id, without_user_id, strict=True):
+        assert set(with_user.attributes) - {User.ID} == set(without.attributes)
+        assert with_user.dropped_attributes == without.dropped_attributes
+
+
+def test_langfuse_vocabulary_keeps_the_caller_trace_user_at_the_span_limit():
+    logger, exporter = _logger_with(
+        mapper_names=["genai", "openinference", "langfuse"], capture_message_content=CaptureMessageContent.SPAN_ONLY
+    )
+    messages: Final = [{"role": "user", "content": f"m{index}"} for index in range(80)]
+    _emit_llm(logger, _user_kwargs({"trace_user_id": "caller-1"}, messages))
+    (span,) = exporter.get_finished_spans()
+    assert span.attributes[User.ID] == "caller-1"
+    assert span.dropped_attributes == 0
 
 
 def test_streaming_span_carries_time_to_first_chunk():

tests/unit/integrations/otel/test_otel_v2_sources_of_truth.py

@@ -1546,34 +1546,28 @@ def test_llm_span_stamps_gen_ai_conversation_id_only_when_the_caller_sent_one():
     assert GenAI.CONVERSATION_ID not in GenAIMapper().map(without)
 
 
-def test_llm_span_stamps_user_id_only_when_the_request_names_a_user():
-    with_user: Final = LLMCallSpanData.from_standard_logging_payload(
-        _sample_payload(), trace=TraceControls(user_id="end-1")
-    )
-    assert GenAIMapper().map(with_user)[User.ID] == "end-1"
-
-    without: Final = LLMCallSpanData.from_standard_logging_payload(_sample_payload())
-    assert User.ID not in GenAIMapper().map(without)
+def _end_user_payload():
+    payload: Final = _sample_payload()
+    return {**payload, "metadata": {**payload["metadata"], "user_api_key_end_user_id": "end-1"}}
 
 
 @pytest.mark.parametrize(
-    ("trace", "metadata_keys", "expected"),
+    ("payload", "trace", "metadata_keys", "expected"),
     [
-        (TraceControls(user_id="end-1", user_id_is_end_user=True), ("user_api_key_end_user_id",), "end-1"),
-        (TraceControls(user_id="end-1", user_id_is_end_user=True), ("user_api_key_org_id",), None),
-        (TraceControls(user_id="caller-1"), ("user_api_key_org_id",), "caller-1"),
+        (_end_user_payload(), TraceControls(), ("user_api_key_end_user_id",), "end-1"),
+        (_end_user_payload(), TraceControls(), ("user_api_key_org_id",), None),
+        (_end_user_payload(), TraceControls(user_id="caller-1"), ("user_api_key_end_user_id",), "end-1"),
+        (_sample_payload(), TraceControls(user_id="caller-1"), ("user_api_key_end_user_id",), None),
     ],
-    ids=["end-user-allowlisted", "end-user-withheld", "caller-trace-user-always-sent"],
+    ids=["end-user-allowlisted", "end-user-withheld", "caller-trace-user-not-sent", "no-end-user"],
 )
-def test_llm_span_sends_the_end_user_only_when_the_metadata_allowlist_releases_it(trace, metadata_keys, expected):
-    data: Final = LLMCallSpanData.from_standard_logging_payload(
-        _sample_payload(), trace=trace, metadata_keys=metadata_keys
-    )
+def test_llm_span_sends_only_the_allowlisted_end_user_as_user_id(payload, trace, metadata_keys, expected):
+    data: Final = LLMCallSpanData.from_standard_logging_payload(payload, trace=trace, metadata_keys=metadata_keys)
     assert GenAIMapper().map(data).get(User.ID) == expected
 
 
 @pytest.mark.parametrize("callback_name", ["otel", *sorted(PRESET_BY_CALLBACK)])
-def test_every_v2_callback_sends_the_request_user_as_user_id(callback_name, monkeypatch):
+def test_every_v2_callback_sends_the_end_user_and_only_langfuse_the_caller_trace_user(callback_name, monkeypatch):
     for name in ("LEVOAI_API_KEY", "LEVOAI_ORG_ID", "LEVOAI_WORKSPACE_ID"):
         monkeypatch.setenv(name, "levo-test")
     monkeypatch.setenv("LEVOAI_COLLECTOR_URL", "http://127.0.0.1:4318")
@@ -1582,11 +1576,16 @@ def test_every_v2_callback_sends_the_request_user_as_user_id(callback_name, monk
         if callback_name == "otel"
         else PRESET_BY_CALLBACK[callback_name](allow_missing_credentials=True)
     )
-    data: Final = LLMCallSpanData.from_standard_logging_payload(_sample_payload(), trace=TraceControls(user_id="end-1"))
-    attrs: Final = {
-        key: value for mapper in resolve_mappers(config.mapper_names) for key, value in mapper.map(data).items()
-    }
-    assert attrs[User.ID] == "end-1"
+    mappers: Final = resolve_mappers(config.mapper_names)
+
+    def user_id(trace: TraceControls) -> object:
+        data: Final = LLMCallSpanData.from_standard_logging_payload(
+            _end_user_payload(), trace=trace, metadata_keys=tuple(config.baggage_metadata_keys)
+        )
+        return {key: value for mapper in mappers for key, value in mapper.map(data).items()}.get(User.ID)
+
+    assert user_id(TraceControls(user_id="end-1", user_id_is_end_user=True)) == "end-1"
+    assert user_id(TraceControls(user_id="caller-1")) == ("caller-1" if callback_name == "langfuse_otel" else "end-1")
 
 
 def test_llm_span_carries_proxy_request_route():