Merge c45d05ca23 into bd31a196a1

litellm/integrations/otel/emitter.py

@@ -13,6 +13,7 @@ from opentelemetry.trace.status import Status, StatusCode
 
 from litellm.integrations.otel.mappers import resolve_mappers
 from litellm.integrations.otel.mappers.base import AttributeMapper, AttrValue, SpanData
+from litellm.integrations.otel.mappers.langfuse import LangfuseMapper
 from litellm.integrations.otel.mappers.openinference import fit_indexed_messages
 from litellm.integrations.otel.model.config import OpenTelemetryV2Config
 from litellm.integrations.otel.model.payloads import (
@@ -23,7 +24,7 @@ from litellm.integrations.otel.model.payloads import (
     ServiceSpanData,
     SpanError,
 )
-from litellm.integrations.otel.model.semconv import Error, ExceptionEvent, LiteLLMError
+from litellm.integrations.otel.model.semconv import Error, ExceptionEvent, LiteLLMError, User
 from litellm.integrations.otel.model.spans import (
     SPAN_REGISTRY,
     SpanRole,
@@ -171,6 +172,11 @@ class SpanEmitter:
         self._mappers: list[AttributeMapper] = (
             list(mappers) if mappers is not None else resolve_mappers(config.mapper_names)
         )
+        self._shed_first: Final = (
+            frozenset[str]()
+            if any(isinstance(mapper, LangfuseMapper) for mapper in self._mappers)
+            else frozenset({User.ID})
+        )
         # Bounded LRU (ordered by insertion / most-recent touch). Storing keys
         # only — the value is unused — so it behaves like a capped set.
         self._emitted: OrderedDict[tuple[str, SpanRole], None] = OrderedDict()
@@ -303,7 +309,8 @@ class SpanEmitter:
         )
         stamped_later: Final = error_attributes(error) if error else _NO_ATTRIBUTES
         reserved: Final = len(stamped_later.keys() - mapped.keys())
-        for key, value in fit_indexed_messages(mapped, attribute_budget(span, reserved, frozenset(mapped))).items():
+        budget: Final = attribute_budget(span, reserved, frozenset(mapped))
+        for key, value in fit_indexed_messages(mapped, budget, self._shed_first).items():
             span.set_attribute(key, value)
         if error:
             stamped: Final = stamp_error(span, error)

litellm/integrations/otel/langfuse_logger.py

@@ -31,7 +31,11 @@ class LangfuseOpenTelemetryV2(OpenTelemetryV2):
     def log_pre_api_call(self, model: str, messages: object, kwargs: Mapping[str, object]) -> None:
         root: Final = request_root_span()
         if root is not None and root.is_recording():
-            controls: Final = request_trace_controls(kwargs) if self._started(root) else caller_trace_controls(kwargs)
+            controls: Final = (
+                request_trace_controls(kwargs).released(self.config.baggage_metadata_keys)
+                if self._started(root)
+                else caller_trace_controls(kwargs)
+            )
             root.set_attributes(LangfuseMapper.trace_attributes(controls))
         super().log_pre_api_call(model, messages, kwargs)
 

litellm/integrations/otel/mappers/genai.py

@@ -36,8 +36,10 @@ from litellm.integrations.otel.model.semconv import (
     LiteLLM,
     RpcSystem,
     Server,
+    User,
 )
 from litellm.integrations.otel.model.spans import postgres_operation
+from litellm.integrations.otel.model.trace_controls import END_USER_METADATA_KEY
 
 
 class GenAIMapper:
@@ -46,6 +48,7 @@ class GenAIMapper:
         GenAI.PROVIDER_NAME: lambda d: d.provider or None,
         GenAI.OUTPUT_TYPE: lambda d: d.output_type.value if d.output_type else None,
         GenAI.CONVERSATION_ID: lambda d: d.session_id,
+        User.ID: lambda d: d.promoted_metadata.get(END_USER_METADATA_KEY),
         GenAI.REQUEST_MODEL: lambda d: d.request_model or None,
         GenAI.REQUEST_TEMPERATURE: lambda d: d.request_params.temperature,
         GenAI.REQUEST_TOP_P: lambda d: d.request_params.top_p,

litellm/integrations/otel/mappers/langfuse.py

@@ -27,12 +27,13 @@ from litellm.integrations.otel.model.payloads import (
     LLMRequestParams,
     LLMUsage,
 )
+from litellm.integrations.otel.model.semconv import User
 from litellm.integrations.otel.model.trace_controls import TraceControls
 
 LANGFUSE_OBSERVATION_INPUT: Final = "langfuse.observation.input"
 LANGFUSE_OBSERVATION_OUTPUT: Final = "langfuse.observation.output"
 LANGFUSE_TRACE_NAME: Final = "langfuse.trace.name"
-LANGFUSE_TRACE_USER_ID: Final = "user.id"
+LANGFUSE_TRACE_USER_ID: Final = User.ID
 LANGFUSE_TRACE_SESSION_ID: Final = "session.id"
 LANGFUSE_TRACE_TAGS: Final = "langfuse.trace.tags"
 

litellm/integrations/otel/mappers/openinference.py

@@ -153,7 +153,9 @@ def _shed_order(groups: Mapping[tuple[str, int, int], tuple[str, ...]]) -> tuple
 _METADATA_KEY: Final = "metadata"
 
 
-def fit_indexed_messages(attrs: Mapping[str, AttrValue], budget: int | None) -> Mapping[str, AttrValue]:
+def fit_indexed_messages(
+    attrs: Mapping[str, AttrValue], budget: int | None, shed_first: frozenset[str] = frozenset()
+) -> Mapping[str, AttrValue]:
     """``attrs`` with indexed message attributes shed, least valuable first, until at most ``budget`` keys remain.
 
     ``None`` means the span has no attribute count limit. Every message still rides the ``input.value`` and
@@ -161,12 +163,14 @@ def fit_indexed_messages(attrs: Mapping[str, AttrValue], budget: int | None) ->
     after every indexed message attribute: message attributes are the indexed, queryable view (the blobs
     carry no per-index keys), so a count-squeezed span keeps them and the single metadata key absorbs only
     the residual shortfall. Shedding happens here, before ``span.set_attribute``, so the fit is exact and
-    the SDK's dropped-attributes counter never silently masks the choice.
+    the SDK's dropped-attributes counter never silently masks the choice. ``shed_first`` keys shed one at a
+    time before anything else, so they never cost the span any other attribute.
     """
     if budget is None or len(attrs) <= budget:
         return attrs
     groups: Final = _message_key_groups(attrs)
-    sheddable: Final = (*(groups[group] for group in _shed_order(groups)),)
+    leading: Final = tuple((key,) for key in attrs if key in shed_first)
+    sheddable: Final = (*leading, *(groups[group] for group in _shed_order(groups)))
     flex: Final = (_METADATA_KEY,) if _METADATA_KEY in attrs else ()
     candidates: Final = (*sheddable, flex) if flex else sheddable
     running: Final = tuple(accumulate(len(candidate) for candidate in candidates))

litellm/integrations/otel/model/payloads.py

@@ -503,7 +503,7 @@ class LLMCallSpanData:
             call_type=call_type or None,
             request_route=request_route or context.identity.request_route,
             request_purpose=request_purpose,
-            trace=trace or TraceControls(),
+            trace=(trace or TraceControls()).released(metadata_keys),
             session_id=session_id or None,
             embedding_output=embedding_output if capture_content else None,
             routing_attributes=routing_decision_attributes(decision),

litellm/integrations/otel/model/semconv.py

@@ -254,6 +254,12 @@ class Server:
     PORT: Final = "server.port"
 
 
+class User:
+    """OTel ``user.*`` registry keys. OpenInference and Langfuse read the same ``user.id``."""
+
+    ID: Final = "user.id"
+
+
 class DB:
     """Database / cache client-span keys (OTel ``db.*`` semconv).
 

litellm/integrations/otel/model/trace_controls.py

@@ -1,8 +1,8 @@
-"""Langfuse trace controls, parsed from the live callback kwargs."""
+"""The request's trace controls, parsed from the live callback kwargs."""
 
 from __future__ import annotations
 
-from collections.abc import Mapping
+from collections.abc import Collection, Mapping
 from dataclasses import dataclass, replace
 from typing import Final
 
@@ -11,6 +11,7 @@ from pydantic import TypeAdapter, ValidationError
 from litellm.integrations.otel.model.utils import as_str, as_str_mapping
 
 LANGFUSE_HEADER_PREFIX: Final = "langfuse_"
+END_USER_METADATA_KEY: Final = "user_api_key_end_user_id"
 _ITEMS: Final = TypeAdapter(tuple[object, ...])
 
 
@@ -25,6 +26,14 @@ class TraceControls:
     user_id: str | None = None
     session_id: str | None = None
     tags: tuple[str, ...] = ()
+    user_id_is_end_user: bool = False
+
+    def released(self, metadata_keys: Collection[str]) -> TraceControls:
+        """These controls minus an end-user ``user_id`` when the operator's ``baggage_metadata_keys`` keeps the
+        end-user id in the proxy. A caller's own ``trace_user_id`` is always kept."""
+        if self.user_id_is_end_user and END_USER_METADATA_KEY not in metadata_keys:
+            return replace(self, user_id=None, user_id_is_end_user=False)
+        return self
 
 
 def caller_trace_controls(kwargs: Mapping[str, object]) -> TraceControls:
@@ -57,10 +66,10 @@ def request_trace_controls(kwargs: Mapping[str, object]) -> TraceControls:
     if request is None:
         return controls
     end_user: Final = next(
-        (value for body in _metadata_bodies(request) if (value := as_str(body.get("user_api_key_end_user_id")))),
+        (value for body in _metadata_bodies(request) if (value := as_str(body.get(END_USER_METADATA_KEY)))),
         None,
     )
-    return replace(controls, user_id=end_user)
+    return replace(controls, user_id=end_user, user_id_is_end_user=end_user is not None)
 
 
 def _metadata_bodies(request: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:

tests/integration/observability/test_otel_v2_user_id_every_callback.py

@@ -0,0 +1,150 @@
+import itertools
+import json
+import uuid
+from collections.abc import Mapping
+from pathlib import Path
+from typing import Final
+
+import pytest
+from _openinference_support import Rig, _chat_plain_response, _rig, _spans
+from integration._support.client import Gateway, eventually
+from integration._support.wire import Reply, Request, Wire
+from pydantic import JsonValue
+
+_OPENINFERENCE_CALLBACKS: Final = ("arize", "arize_phoenix", "weave_otel")
+_GENAI_CALLBACKS: Final = ("otel", "langfuse_otel", "langtrace", "signoz", "levo", "newrelic", "agentops")
+_UNUSED_ENDPOINT: Final[dict[str, JsonValue]] = {"otel": {"exporter": "http/protobuf", "endpoint": "unused"}}
+_METADATA_ALLOWLIST: Final = "LITELLM_OTEL_BAGGAGE_METADATA_KEYS"
+_LANGFUSE_CALLBACK: Final = "langfuse_otel"
+
+
+def _settings(callback: str) -> Mapping[str, JsonValue] | None:
+    if callback == "arize":
+        return None
+    if callback in _OPENINFERENCE_CALLBACKS:
+        return _UNUSED_ENDPOINT
+    return {"otel": {"exporter": "http/protobuf", "endpoint": "unused", "mapper_names": ["genai"]}}
+
+
+def _environment(callback: str) -> dict[str, str]:
+    return {
+        "PHOENIX_PROJECT_NAME": "integration",
+        "WANDB_BASE_URL": "http://127.0.0.1",
+        "LANGFUSE_HOST": "http://127.0.0.1",
+        **(
+            {"HTTPS_PROXY": "http://127.0.0.1:0", "https_proxy": "http://127.0.0.1:0", "NO_PROXY": "", "no_proxy": ""}
+            if callback == "agentops"
+            else {}
+        ),
+    }
+
+
+def _removed_environment(callback: str) -> tuple[str, ...]:
+    if callback == "newrelic":
+        return ("NEW_RELIC_LICENSE_KEY",)
+    if callback == "agentops":
+        return ("AGENTOPS_API_KEY",)
+    return ()
+
+
+def _upstream(request: Request) -> Reply:
+    marker: Final = json.loads(request.body)["messages"][0]["content"]
+    return _chat_plain_response(marker, "ok")
+
+
+def _model_call_spans(destination: Wire, markers: tuple[str, ...]) -> Mapping[str, tuple[dict[str, str], ...]]:
+    seen: Final[list[Request]] = []
+
+    def by_marker() -> Mapping[str, tuple[dict[str, str], ...]]:
+        seen.extend(destination.drain())
+        spans: Final = tuple(attributes for attributes in _spans(tuple(seen)) if "gen_ai.response.id" in attributes)
+        return {
+            marker: tuple(attributes for attributes in spans if attributes["gen_ai.response.id"] == marker)
+            for marker in markers
+        }
+
+    return eventually(by_marker, lambda found: all(found.values()), seconds=30)
+
+
+def _carrying(spans: Mapping[str, tuple[dict[str, str], ...]], value: str) -> tuple[str, ...]:
+    model_call_spans: Final = itertools.chain.from_iterable(spans.values())
+    return tuple(key for span in model_call_spans for key, attribute in span.items() if value in str(attribute))
+
+
+def _chat(rig: Rig, marker: str, key: str, headers: Mapping[str, str], extra: Mapping[str, JsonValue]) -> None:
+    response: Final = rig.proxy.request(
+        "POST",
+        "/v1/chat/completions",
+        {"model": rig.model, "messages": [{"role": "user", "content": marker}], "cache": {"no-cache": True}, **extra},
+        key=key,
+        headers=headers,
+    )
+    assert response.status_code == 200, response.text
+
+
+@pytest.mark.parametrize("callback", (*_OPENINFERENCE_CALLBACKS, *_GENAI_CALLBACKS))
+def test_otel_v2_callback_sends_the_request_user_as_user_id(callback: str, gateway: Gateway, tmp_path: Path) -> None:
+    run: Final = uuid.uuid4().hex
+    end_user: Final = f"end-user-{run}"
+    caller: Final = f"caller-{run}"
+    markers: Final = (f"end-{run}", f"trace-{run}", f"owner-{run}")
+    with (
+        _rig(
+            gateway,
+            tmp_path,
+            _upstream,
+            callbacks=(callback,),
+            callback_settings=_settings(callback),
+            environment=_environment(callback),
+            remove_environment=(*_removed_environment(callback), _METADATA_ALLOWLIST),
+            disabled_environment=(_METADATA_ALLOWLIST,),
+        ) as rig,
+        rig.proxy.scenario() as scenario,
+    ):
+        service_key: Final = scenario.key()
+        owned_key: Final = scenario.key(user_id=scenario.user())
+        sent: Final = (
+            (markers[0], service_key, {"x-litellm-end-user-id": end_user}, {}),
+            (markers[1], service_key, {"x-litellm-end-user-id": end_user}, {"metadata": {"trace_user_id": caller}}),
+            (markers[2], owned_key, {}, {}),
+        )
+        for marker, key, headers, extra in sent:
+            _chat(rig, marker, key, headers, extra)
+        spans: Final = _model_call_spans(rig.destination, markers)
+    users: Final = {marker: {attributes.get("user.id") for attributes in spans[marker]} for marker in markers}
+    caller_user: Final = caller if callback == _LANGFUSE_CALLBACK else end_user
+    assert users == {markers[0]: {end_user}, markers[1]: {caller_user}, markers[2]: {None}}, spans
+    sessions: Final = {attributes.get("session.id") for attributes in itertools.chain.from_iterable(spans.values())}
+    assert end_user not in sessions, spans
+    assert callback == _LANGFUSE_CALLBACK or not _carrying(spans, caller), spans
+
+
+@pytest.mark.parametrize("callback", ("arize", "otel", "langfuse_otel"))
+def test_otel_v2_callback_withholds_an_end_user_the_metadata_allowlist_drops(
+    callback: str, gateway: Gateway, tmp_path: Path
+) -> None:
+    run: Final = uuid.uuid4().hex
+    end_user: Final = f"end-user-{run}"
+    caller: Final = f"caller-{run}"
+    markers: Final = (f"end-{run}", f"trace-{run}")
+    with (
+        _rig(
+            gateway,
+            tmp_path,
+            _upstream,
+            callbacks=(callback,),
+            callback_settings=_settings(callback),
+            environment={**_environment(callback), _METADATA_ALLOWLIST: "user_api_key_org_id"},
+        ) as rig,
+        rig.proxy.scenario() as scenario,
+    ):
+        service_key: Final = scenario.key()
+        _chat(rig, markers[0], service_key, {"x-litellm-end-user-id": end_user}, {})
+        _chat(
+            rig, markers[1], service_key, {"x-litellm-end-user-id": end_user}, {"metadata": {"trace_user_id": caller}}
+        )
+        spans: Final = _model_call_spans(rig.destination, markers)
+    users: Final = {marker: {attributes.get("user.id") for attributes in spans[marker]} for marker in markers}
+    assert users == {markers[0]: {None}, markers[1]: {caller if callback == _LANGFUSE_CALLBACK else None}}, spans
+    assert not _carrying(spans, end_user), spans
+    assert callback == _LANGFUSE_CALLBACK or not _carrying(spans, caller), spans

tests/unit/integrations/otel/test_langfuse_logger.py

@@ -466,6 +466,31 @@ def test_a_callers_trace_user_id_still_wins_over_the_proxy_end_user():
     assert generation_attrs["user.id"] == "caller-1"
 
 
+@pytest.mark.parametrize(
+    ("metadata", "expected_user"),
+    [
+        ({"user_api_key_end_user_id": "end-1"}, None),
+        ({"trace_user_id": "caller-1", "user_api_key_end_user_id": "end-1"}, "caller-1"),
+    ],
+    ids=["end-user-withheld", "caller-trace-user-still-sent"],
+)
+def test_an_end_user_the_metadata_allowlist_drops_stays_off_the_root_and_the_generation(metadata, expected_user):
+    cfg: Final = OpenTelemetryV2Config(
+        exporter="in_memory", mapper_names=["genai", "langfuse"], baggage_metadata_keys=["user_api_key_org_id"]
+    )
+    exporter: Final = InMemorySpanExporter()
+    logger: Final = build_otel_v2_logger(
+        config=cfg, tracer_provider=providers.build_tracer_provider(cfg, exporter=exporter)
+    )
+
+    root_attrs, generation_attrs = _run_named_request(
+        logger, exporter, {"metadata": metadata, "proxy_server_request": {"headers": {}}}
+    )
+
+    assert root_attrs.get("user.id") == expected_user
+    assert generation_attrs.get("user.id") == expected_user
+
+
 @pytest.mark.parametrize(
     ("metadata", "expected_root_user"),
     [

tests/unit/integrations/otel/test_otel_v2_logger.py

@@ -45,8 +45,10 @@ from litellm.integrations.otel import (  # noqa: E402
     OpenTelemetryV2Config,
 )
 from litellm.integrations.otel.logger import OpenTelemetryV2  # noqa: E402
+from litellm.integrations.otel.mappers.genai import GenAIMapper  # noqa: E402
 from litellm.integrations.otel.model.config import CaptureMessageContent, ExporterSpec  # noqa: E402
 from litellm.integrations.otel.model.destination import OtelDestination  # noqa: E402
+from litellm.integrations.otel.model.semconv import User  # noqa: E402
 from litellm.integrations.otel.model.spans import (  # noqa: E402
     LITELLM_PROXY_REQUEST_SPAN_NAME,
     SpanRole,
@@ -301,6 +303,82 @@ def test_replayed_llm_call_span_does_not_take_the_payloads_session_id():
     assert GenAI.CONVERSATION_ID not in span.attributes
 
 
+def _user_kwargs(metadata, messages=None, call_id="call_1"):
+    payload: Final = _payload(metadata={**_payload()["metadata"], **metadata}, litellm_call_id=call_id)
+    if messages is not None:
+        payload["messages"] = messages
+    return {**_kwargs(payload), "litellm_params": {"metadata": metadata}, "messages": messages or []}
+
+
+def _logger_with(**config):
+    cfg: Final = OpenTelemetryV2Config(exporter="in_memory", **config)
+    exporter: Final = InMemorySpanExporter()
+    return OpenTelemetryV2(
+        config=cfg, tracer_provider=providers.build_tracer_provider(cfg, exporter=exporter)
+    ), exporter
+
+
+@pytest.mark.parametrize(
+    ("metadata", "expected"),
+    [
+        ({"user_api_key_end_user_id": "end-1"}, "end-1"),
+        ({"trace_user_id": "caller-1", "user_api_key_end_user_id": "end-1"}, "end-1"),
+        ({"trace_user_id": "caller-1"}, None),
+        ({"user_api_key_user_id": "internal-1"}, None),
+    ],
+    ids=["proxy-end-user", "caller-trace-user-not-sent", "caller-trace-user-alone", "internal-user-never-sent"],
+)
+def test_llm_call_span_carries_the_end_user_as_user_id(metadata, expected):
+    logger, exporter = _logger()
+    _emit_llm(logger, _user_kwargs(metadata))
+    (span,) = exporter.get_finished_spans()
+    assert span.attributes.get(User.ID) == expected
+    assert "caller-1" not in {str(value) for value in span.attributes.values()}
+
+
+@pytest.mark.parametrize(
+    "metadata",
+    [{"user_api_key_end_user_id": "end-1"}, {"trace_user_id": "caller-1", "user_api_key_end_user_id": "end-1"}],
+    ids=["end-user", "end-user-and-caller-trace-user"],
+)
+def test_llm_call_span_withholds_an_end_user_the_metadata_allowlist_drops(metadata):
+    logger, exporter = _logger_with(baggage_metadata_keys=["user_api_key_org_id"])
+    _emit_llm(logger, _user_kwargs(metadata))
+    (span,) = exporter.get_finished_spans()
+    assert User.ID not in span.attributes
+    assert not {"end-1", "caller-1"} & {str(value) for value in span.attributes.values()}
+
+
+def test_user_id_costs_no_other_attribute_at_the_span_limit(monkeypatch):
+    def spans() -> list:
+        logger, exporter = _logger_with(
+            mapper_names=["genai", "openinference"], capture_message_content=CaptureMessageContent.SPAN_ONLY
+        )
+        for count in range(40, 70):
+            messages = [{"role": "user", "content": f"m{index}"} for index in range(count)]
+            _emit_llm(logger, _user_kwargs({"user_api_key_end_user_id": "end-1"}, messages, f"call_{count}"))
+        return list(exporter.get_finished_spans())
+
+    with_user_id: Final = spans()
+    monkeypatch.delitem(GenAIMapper._LLM_CALL_ATTRS, User.ID)
+    without_user_id: Final = spans()
+    assert {span.attributes.get(User.ID) for span in with_user_id} == {"end-1", None}
+    for with_user, without in zip(with_user_id, without_user_id, strict=True):
+        assert set(with_user.attributes) - {User.ID} == set(without.attributes)
+        assert with_user.dropped_attributes == without.dropped_attributes
+
+
+def test_langfuse_vocabulary_keeps_the_caller_trace_user_at_the_span_limit():
+    logger, exporter = _logger_with(
+        mapper_names=["genai", "openinference", "langfuse"], capture_message_content=CaptureMessageContent.SPAN_ONLY
+    )
+    messages: Final = [{"role": "user", "content": f"m{index}"} for index in range(80)]
+    _emit_llm(logger, _user_kwargs({"trace_user_id": "caller-1"}, messages))
+    (span,) = exporter.get_finished_spans()
+    assert span.attributes[User.ID] == "caller-1"
+    assert span.dropped_attributes == 0
+
+
 def test_streaming_span_carries_time_to_first_chunk():
     logger, exporter = _logger()
     kwargs = {

tests/unit/integrations/otel/test_otel_v2_sources_of_truth.py

@@ -29,6 +29,7 @@ from litellm.integrations.otel import (
     resolve_output_type,
     resolve_provider,
 )
+from litellm.integrations.otel.mappers import resolve_mappers
 from litellm.integrations.otel.mappers.genai import GenAIMapper
 from litellm.integrations.otel.model import spans as spans_mod
 from litellm.integrations.otel.model.metadata import LLMCallEvent
@@ -38,6 +39,7 @@ from litellm.integrations.otel.model.payloads import (
     RequestIdentity,
     _upstream_address_port,
 )
+from litellm.integrations.otel.model.semconv import User
 from litellm.integrations.otel.model.spans import (
     SPAN_REGISTRY,
     LiteLLMSpanKind,
@@ -51,6 +53,7 @@ from litellm.integrations.otel.model.trace_controls import (
     caller_trace_controls,
     request_trace_controls,
 )
+from litellm.integrations.otel.presets import PRESET_BY_CALLBACK
 
 
 @pytest.fixture(autouse=True)
@@ -1355,8 +1358,11 @@ def test_caller_trace_controls_carry_user_session_and_tags(request_data, expecte
 @pytest.mark.parametrize(
     ("request_data", "expected"),
     [
-        ({"metadata": {"user_api_key_end_user_id": "end-1"}}, TraceControls(user_id="end-1")),
-        ({"litellm_metadata": {"user_api_key_end_user_id": "end-2"}}, TraceControls(user_id="end-2")),
+        ({"metadata": {"user_api_key_end_user_id": "end-1"}}, TraceControls(user_id="end-1", user_id_is_end_user=True)),
+        (
+            {"litellm_metadata": {"user_api_key_end_user_id": "end-2"}},
+            TraceControls(user_id="end-2", user_id_is_end_user=True),
+        ),
         (
             {"metadata": {"trace_user_id": "caller-1", "user_api_key_end_user_id": "end-1"}},
             TraceControls(user_id="caller-1"),
@@ -1370,7 +1376,7 @@ def test_caller_trace_controls_carry_user_session_and_tags(request_data, expecte
         ),
         (
             {"metadata": {"user_api_key_end_user_id": "end-1", "session_id": "s-1"}},
-            TraceControls(user_id="end-1", session_id="s-1"),
+            TraceControls(user_id="end-1", session_id="s-1", user_id_is_end_user=True),
         ),
         ({"metadata": {"user_api_key_user_id": "internal-1"}}, TraceControls()),
         ({}, TraceControls()),
@@ -1540,6 +1546,48 @@ def test_llm_span_stamps_gen_ai_conversation_id_only_when_the_caller_sent_one():
     assert GenAI.CONVERSATION_ID not in GenAIMapper().map(without)
 
 
+def _end_user_payload():
+    payload: Final = _sample_payload()
+    return {**payload, "metadata": {**payload["metadata"], "user_api_key_end_user_id": "end-1"}}
+
+
+@pytest.mark.parametrize(
+    ("payload", "trace", "metadata_keys", "expected"),
+    [
+        (_end_user_payload(), TraceControls(), ("user_api_key_end_user_id",), "end-1"),
+        (_end_user_payload(), TraceControls(), ("user_api_key_org_id",), None),
+        (_end_user_payload(), TraceControls(user_id="caller-1"), ("user_api_key_end_user_id",), "end-1"),
+        (_sample_payload(), TraceControls(user_id="caller-1"), ("user_api_key_end_user_id",), None),
+    ],
+    ids=["end-user-allowlisted", "end-user-withheld", "caller-trace-user-not-sent", "no-end-user"],
+)
+def test_llm_span_sends_only_the_allowlisted_end_user_as_user_id(payload, trace, metadata_keys, expected):
+    data: Final = LLMCallSpanData.from_standard_logging_payload(payload, trace=trace, metadata_keys=metadata_keys)
+    assert GenAIMapper().map(data).get(User.ID) == expected
+
+
+@pytest.mark.parametrize("callback_name", ["otel", *sorted(PRESET_BY_CALLBACK)])
+def test_every_v2_callback_sends_the_end_user_and_only_langfuse_the_caller_trace_user(callback_name, monkeypatch):
+    for name in ("LEVOAI_API_KEY", "LEVOAI_ORG_ID", "LEVOAI_WORKSPACE_ID"):
+        monkeypatch.setenv(name, "levo-test")
+    monkeypatch.setenv("LEVOAI_COLLECTOR_URL", "http://127.0.0.1:4318")
+    config: Final = (
+        OpenTelemetryV2Config()
+        if callback_name == "otel"
+        else PRESET_BY_CALLBACK[callback_name](allow_missing_credentials=True)
+    )
+    mappers: Final = resolve_mappers(config.mapper_names)
+
+    def user_id(trace: TraceControls) -> object:
+        data: Final = LLMCallSpanData.from_standard_logging_payload(
+            _end_user_payload(), trace=trace, metadata_keys=tuple(config.baggage_metadata_keys)
+        )
+        return {key: value for mapper in mappers for key, value in mapper.map(data).items()}.get(User.ID)
+
+    assert user_id(TraceControls(user_id="end-1", user_id_is_end_user=True)) == "end-1"
+    assert user_id(TraceControls(user_id="caller-1")) == ("caller-1" if callback_name == "langfuse_otel" else "end-1")
+
+
 def test_llm_span_carries_proxy_request_route():
     """The LLM span records the proxy route the request arrived on, so it can be
     filtered by endpoint (``/v1/responses`` vs ``/v1/chat/completions``) without