fix(otel)!: print OTel v2 spans to stdout only when the operator asks (#48723)

* fix(otel): stop OTel v2 vendor presets from printing every span to stdout

With LITELLM_OTEL_V2 on and a vendor callback such as langfuse_otel or arize
holding operator credentials, the proxy printed every span as JSON to stdout,
prompts included, on the request path. Nobody set OTEL_EXPORTER, but the
console placeholder that OpenTelemetryV2Config folds in for an empty exporter
list stayed next to the vendor's own exporter.

The provider now skips that placeholder when the config has any other
exporter. A console exporter the operator names with OTEL_EXPORTER=console is
kept, and so is the placeholder when it is the only exporter.

* test(otel): type the console-placeholder tests and drop inherited OTEL_* env from the stdout check

* fix(otel): print the console default only for the otel callback

The proxy's background V2 logger (published when no callback built one, which is
what key-only and team-only Langfuse destinations run under) also kept the
unconfigured console exporter, so it printed every span with prompts to stdout.
The console default now prints only for a listed otel callback or an exporter the
operator named.

* fix(otel): keep the otel callback's console output on service-name routes

A key or team otel_service_name moves the call onto a provider the tenant
tracer cache builds, which never got the console choice, so the plain otel
callback stopped printing those calls. The no-operator-keys integration rig
also drops inherited Langfuse keys.

litellm/integrations/otel/logger.py

@@ -198,6 +198,7 @@ class OpenTelemetryV2(CustomLogger):
         tracer_provider: TracerProvider | None = None,
         logger_provider: LoggerProvider | None = None,
         meter_provider: "MeterProvider | None" = None,
+        console_default: bool = False,
         **kwargs: Unpack[_CustomLoggerOptions],
     ) -> None:
         super().__init__(**kwargs)
@@ -206,7 +207,7 @@ class OpenTelemetryV2(CustomLogger):
         self._tracer_provider: TracerProvider = (
             tracer_provider
             if tracer_provider is not None
-            else build_tracer_provider(self.config, tenant_overrides=True)
+            else build_tracer_provider(self.config, tenant_overrides=True, console_default=console_default)
         )
         self.tracer: Tracer = get_tracer(self._tracer_provider, LITELLM_TRACER_NAME)
         self._metrics_recorder = self._init_metrics(meter_provider)
@@ -217,7 +218,9 @@ class OpenTelemetryV2(CustomLogger):
             mappers=resolve_mappers(self.config.mapper_names),
             event_recorder=self._init_events(logger_provider),
         )
-        self._tenant_tracers = TenantTracerCache(self.config, callback_name, LITELLM_TRACER_NAME)
+        self._tenant_tracers = TenantTracerCache(
+            self.config, callback_name, LITELLM_TRACER_NAME, console_default=console_default
+        )
         self._open_llm_calls: OrderedDict[str, _LLMCallSpan] = OrderedDict()
         self._init_otel_logger_on_litellm_proxy()
 
@@ -1089,6 +1092,7 @@ def build_otel_v2_logger(
     logger_provider: LoggerProvider | None = None,
     meter_provider: "MeterProvider | None" = None,
     settings: Mapping[str, object] = MappingProxyType({}),
+    console_default: bool = False,
 ) -> OpenTelemetryV2:
     return _logger_class(config)(
         config=config,
@@ -1096,6 +1100,7 @@ def build_otel_v2_logger(
         tracer_provider=tracer_provider,
         logger_provider=logger_provider,
         meter_provider=meter_provider,
+        console_default=console_default,
         **settings,
     )
 

litellm/integrations/otel/model/config.py

@@ -121,6 +121,17 @@ class ExporterSpec(LiteLLMBaseModel):
     )
 
 
+def is_unconfigured_placeholder(spec: ExporterSpec) -> bool:
+    """Whether ``spec`` is the one ``_normalize`` folds in when nothing was configured.
+
+    No field set is what says the operator asked for nothing: an exporter they did
+    configure survives, even ``OTEL_EXPORTER=console`` whose value matches the default,
+    and so does the gated spec ``credential_gated_exporters`` appends, which would
+    otherwise eat itself when one preset layers onto another.
+    """
+    return not spec.model_fields_set
+
+
 class _EnvWithoutBareExcludedServices(PydanticBaseSettingsSource):
     def __init__(self, settings_cls: type[BaseSettings], env_settings: PydanticBaseSettingsSource) -> None:
         super().__init__(settings_cls)

litellm/integrations/otel/plumbing/providers.py

@@ -56,6 +56,7 @@ from litellm.integrations.otel.model.config import (
     OpenTelemetryV2Config,
     capture_message_content_from_env,
     captures_span_content,
+    is_unconfigured_placeholder,
 )
 from litellm.integrations.otel.model.semconv import (
     DB,
@@ -1551,6 +1552,7 @@ def build_tracer_provider(
     use_simple_processor: bool | None = None,
     tenant_overrides: bool = False,
     strip_message_content: bool = False,
+    console_default: bool = False,
 ) -> TracerProvider:
     """Build the shared :class:`TracerProvider`.
 
@@ -1575,6 +1577,10 @@ def build_tracer_provider(
     ``strip_message_content`` removes prompt and response content before every
     exporter, for a routed provider whose spans can carry content another
     destination asked for.
+
+    The console spec ``config._normalize`` folds in when nothing was configured
+    prints only under ``console_default``, which the ``otel`` callback passes: a
+    vendor preset or the proxy's background global logger was never asked to print.
     """
     provider: Final = TracerProvider(resource=build_resource(config))
     provider.sampler = _RequestExclusionSampler(provider.sampler)
@@ -1592,6 +1598,8 @@ def build_tracer_provider(
     for spec in config.exporters:
         if spec.requires_headers and not spec.headers:
             continue
+        if is_unconfigured_placeholder(spec) and not console_default:
+            continue
         processor = _spec_processor(
             spec,
             use_simple_processor,

litellm/integrations/otel/plumbing/routing.py

@@ -163,10 +163,12 @@ class TenantTracerCache:
         config: OpenTelemetryV2Config,
         callback_name: str | None,
         tracer_name: str,
+        console_default: bool = False,
     ) -> None:
         self._config = config
         self._callback_name = callback_name
         self._tracer_name = tracer_name
+        self._console_default = console_default
         # Guards the three mutable structures below: ``pre_call`` can run on
         # thread-pool workers concurrently with the event loop, so cache
         # updates, span counts, and retirement must be atomic.
@@ -284,6 +286,7 @@ class TenantTracerCache:
         built: Final = build_tracer_provider(
             self._routed_config(credential_headers, project_headers, endpoint, service_name),
             strip_message_content=not self._config.capture_span_content,
+            console_default=self._console_default,
         )
         self._providers[cache_key] = built
         return built

litellm/integrations/otel/presets/utils.py

@@ -3,7 +3,7 @@
 from collections.abc import Iterable
 from typing import Final
 
-from litellm.integrations.otel.model.config import ExporterOwner, ExporterSpec
+from litellm.integrations.otel.model.config import ExporterOwner, ExporterSpec, is_unconfigured_placeholder
 
 
 def ensure_mappers(mapper_names: Iterable[str], *names: str) -> list[str]:
@@ -35,14 +35,3 @@ def credential_gated_exporters(
         *(spec for spec in exporters if not is_unconfigured_placeholder(spec)),
         ExporterSpec(owner=owner, requires_headers=True),
     )
-
-
-def is_unconfigured_placeholder(spec: "ExporterSpec") -> bool:
-    """Whether ``spec`` is the one ``_normalize`` folds in when nothing was configured.
-
-    No field set is what says the operator asked for nothing: an exporter they did
-    configure survives, even ``OTEL_EXPORTER=console`` whose value matches the default,
-    and so does the gated spec this module appends, which would otherwise eat itself
-    when one preset layers onto another.
-    """
-    return not spec.model_fields_set

litellm/litellm_core_utils/litellm_logging.py

@@ -5623,7 +5623,7 @@ def build_custom_logger_compatible_class(
                     return callback
             otel_settings: Final = _get_custom_logger_settings_from_proxy_server(callback_name=logging_integration)
             otel_logger_v2: Final = build_otel_v2_logger(
-                config=OpenTelemetryV2Config(**otel_settings), settings=otel_settings
+                config=OpenTelemetryV2Config(**otel_settings), settings=otel_settings, console_default=True
             )
             _in_memory_loggers.append(otel_logger_v2)
             _maybe_auto_initialize_arize_phoenix(_in_memory_loggers)

tests/integration/observability/_langfuse_otel_metadata_support.py

@@ -210,6 +210,8 @@ def langfuse_rig(
     sink_reply: Callable[[Request], Reply] | None = None,
     overrides: Mapping[str, str] | None = None,
     model_list: Callable[[str], Sequence[Mapping[str, object]]] | None = None,
+    remove_environment: tuple[str, ...] = (),
+    operator_keys: bool = True,
 ) -> Iterator[LangfuseRig]:
     def upstream(request: Request) -> Reply:
         if request.method == "GET":
@@ -253,8 +255,11 @@ def langfuse_rig(
             gateway,
             tmp_path,
             {
-                "LANGFUSE_PUBLIC_KEY": "pk-lf-integration",
-                "LANGFUSE_SECRET_KEY": "sk-lf-integration",
+                **(
+                    {"LANGFUSE_PUBLIC_KEY": "pk-lf-integration", "LANGFUSE_SECRET_KEY": "sk-lf-integration"}
+                    if operator_keys
+                    else {}
+                ),
                 "LANGFUSE_HOST": collector.url,
                 "LITELLM_OTEL_V2": "1",
                 **{
@@ -264,6 +269,11 @@ def langfuse_rig(
             },
             config=path,
             workers=2,
+            remove_environment=(
+                remove_environment
+                if operator_keys
+                else (*remove_environment, "LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY")
+            ),
         ) as owned:
             yield LangfuseRig(owned.gateway, provider, collector, owned)
 

tests/integration/observability/test_langfuse_otel_console_output.py

@@ -0,0 +1,98 @@
+import json
+import os
+import uuid
+from collections.abc import Mapping
+from pathlib import Path
+from typing import Final
+
+import pytest
+from _langfuse_otel_metadata_support import exported_llm_span, langfuse_rig, scenario_identity, watched_spans
+from integration._support.client import Gateway, eventually
+
+CONSOLE_SPAN: Final = '"span_id": "0x'
+INHERITED_OTEL_ENV: Final = tuple(name for name in os.environ if name.startswith("OTEL_"))
+
+
+@pytest.mark.parametrize(
+    ("overrides", "printed"),
+    [({}, False), ({"OTEL_EXPORTER": "console"}, True)],
+    ids=["exporter-unset", "operator-asked-for-console"],
+)
+def test_langfuse_otel_prints_spans_to_stdout_only_when_the_operator_asks(
+    gateway: Gateway, tmp_path: Path, overrides: Mapping[str, str], printed: bool
+) -> None:
+    marker: Final = "lf-console-" + uuid.uuid4().hex
+    with (
+        langfuse_rig(
+            gateway,
+            tmp_path,
+            {},
+            marker,
+            overrides={"OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT": "span_only", **overrides},
+            remove_environment=INHERITED_OTEL_ENV,
+        ) as rig,
+        rig.candidate.scenario() as scenario,
+    ):
+        key, identity = scenario_identity(scenario, rig.provider, team=False)
+        response: Final = rig.candidate.request(
+            "POST",
+            "/v1/chat/completions",
+            {"model": identity["model"], "messages": [{"role": "user", "content": marker}]},
+            key=key,
+        )
+        assert response.status_code == 200, response.text
+
+        exported_llm_span(rig.collector, response.json()["id"])
+
+        on_stdout: Final = CONSOLE_SPAN in rig.proxy.log.read_text()
+        assert on_stdout is printed, f"span JSON on proxy stdout: {on_stdout}, see {rig.proxy.log}"
+
+
+@pytest.mark.parametrize("callbacks", [[], ["langfuse_otel"]], ids=["key-only", "callback-listed"])
+def test_key_langfuse_without_operator_keys_prints_no_spans(
+    gateway: Gateway, tmp_path: Path, callbacks: list[str]
+) -> None:
+    marker: Final = "lf-console-key-" + uuid.uuid4().hex
+    with (
+        langfuse_rig(
+            gateway,
+            tmp_path,
+            {"callbacks": callbacks},
+            marker,
+            overrides={"OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT": "span_only"},
+            remove_environment=INHERITED_OTEL_ENV,
+            operator_keys=False,
+        ) as rig,
+        rig.candidate.scenario() as scenario,
+    ):
+        key: Final = scenario.key(
+            key_alias="lf-console-key-" + uuid.uuid4().hex,
+            metadata={
+                "logging": [
+                    {
+                        "callback_name": "langfuse_otel",
+                        "callback_type": "success",
+                        "callback_vars": {
+                            "langfuse_public_key": "pk-lf-key",
+                            "langfuse_secret_key": "sk-lf-key",
+                            "langfuse_host": rig.collector.url,
+                        },
+                    }
+                ]
+            },
+        )
+        model: Final = scenario.model(api_base=rig.provider.url + "/v1")
+        response: Final = rig.candidate.request(
+            "POST",
+            "/v1/chat/completions",
+            {"model": model, "messages": [{"role": "user", "content": marker}]},
+            key=key,
+        )
+        assert response.status_code == 200, response.text
+
+        eventually(
+            watched_spans(rig.collector), lambda spans: any(marker in json.dumps(span) for span in spans), seconds=25
+        )
+
+        on_stdout: Final = CONSOLE_SPAN in rig.proxy.log.read_text()
+        assert not on_stdout, f"span JSON on proxy stdout, see {rig.proxy.log}"

tests/unit/integrations/otel/test_otel_v2_destinations.py

@@ -3352,8 +3352,7 @@ class TestArizeTenantOnly:
 
         assert capture.received() == {
             "space_id=space-operator,api_key=key-operator": ("chat gpt-4",),
-            None: ("chat gpt-4",),
-        }, "the operator's space, plus the stdout placeholder every credentialed preset keeps today"
+        }, "the operator's space, with no stdout copy the operator never asked for"
 
     def test_the_standard_otlp_headers_still_reach_the_operators_space(self, monkeypatch):
         credential_less_arize(monkeypatch)
@@ -3366,8 +3365,7 @@ class TestArizeTenantOnly:
 
         assert capture.received() == {
             "space_id=space-operator,api_key=key-operator": ("chat gpt-4",),
-            None: ("chat gpt-4",),
-        }, "the operator's space, plus the stdout placeholder every credentialed preset keeps today"
+        }, "the operator's space, with no stdout copy the operator never asked for"
 
     def test_a_credential_less_arize_still_delivers_a_team_destination(self, monkeypatch):
         from litellm.integrations.otel.plumbing.providers import attach_tenant_fan_out
@@ -4825,6 +4823,115 @@ class TestCredentialGatedExporters:
         assert kept[0] is operator_console
 
 
+class TestConsolePlaceholder:
+    """The console spec ``_normalize`` folds in for an empty exporter list prints spans
+    only for the ``otel`` callback; no other logger was asked to print."""
+
+    @pytest.fixture
+    def capture(self, monkeypatch: pytest.MonkeyPatch) -> "_ExporterCapture":
+        return TestArizeTenantOnly._capture_exporters(monkeypatch)
+
+    @staticmethod
+    def _received(capture: "_ExporterCapture", provider: TracerProvider) -> Mapping[str | None, tuple[str, ...]]:
+        emit(provider)
+        provider.force_flush()
+        return capture.received()
+
+    @staticmethod
+    def _no_otel_env(monkeypatch: pytest.MonkeyPatch) -> None:
+        for name in _OTEL_SHORTHAND_ENV:
+            monkeypatch.delenv(name, raising=False)
+
+    @staticmethod
+    def _operator_langfuse(monkeypatch: pytest.MonkeyPatch) -> None:
+        TestConsolePlaceholder._no_otel_env(monkeypatch)
+        monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-lf-operator")
+        monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-lf-operator")
+
+    @staticmethod
+    def _isolated_proxy_registration(monkeypatch: pytest.MonkeyPatch) -> None:
+        for name in ("service_callback", "input_callback", "_async_success_callback", "_async_failure_callback"):
+            monkeypatch.setattr(litellm, name, [])
+        monkeypatch.setattr("litellm.proxy.proxy_server.open_telemetry_logger", None)
+
+    def test_a_credentialed_langfuse_exports_to_langfuse_alone(
+        self, monkeypatch: pytest.MonkeyPatch, capture: "_ExporterCapture"
+    ) -> None:
+        self._operator_langfuse(monkeypatch)
+        config: Final = langfuse_preset()
+        langfuse_headers: Final = next(
+            spec.headers for spec in config.exporters if spec.owner is ExporterOwner.LANGFUSE_OTEL
+        )
+
+        assert self._received(capture, build_tracer_provider(config)) == {langfuse_headers: ("chat gpt-4",)}
+
+    def test_a_console_exporter_the_operator_named_still_prints(
+        self, monkeypatch: pytest.MonkeyPatch, capture: "_ExporterCapture"
+    ) -> None:
+        self._operator_langfuse(monkeypatch)
+        monkeypatch.setenv("OTEL_EXPORTER", "console")
+
+        assert self._received(capture, build_tracer_provider(langfuse_preset()))[None] == ("chat gpt-4",)
+
+    @pytest.fixture
+    def otel_callback(self, monkeypatch: pytest.MonkeyPatch, capture: "_ExporterCapture") -> OpenTelemetryV2:
+        """The plain ``otel`` callback with nothing else configured, built the way the proxy builds it."""
+        from litellm.litellm_core_utils import litellm_logging as logging_module
+
+        self._no_otel_env(monkeypatch)
+        self._isolated_proxy_registration(monkeypatch)
+        monkeypatch.setenv("LITELLM_OTEL_V2", "true")
+        monkeypatch.setattr(logging_module, "_in_memory_loggers", [])
+        is_otel_v2_enabled.cache_clear()
+        built: Final = logging_module.build_custom_logger_compatible_class(
+            "otel", internal_usage_cache=None, llm_router=None
+        )
+        assert isinstance(built, OpenTelemetryV2)
+        return built
+
+    def test_the_otel_callback_prints_when_nothing_else_is_configured(
+        self, otel_callback: OpenTelemetryV2, capture: "_ExporterCapture"
+    ) -> None:
+        assert self._received(capture, otel_callback.tracer_provider) == {None: ("chat gpt-4",)}
+
+    def test_the_otel_callback_prints_a_call_for_a_key_with_its_own_service_name(
+        self, otel_callback: OpenTelemetryV2, capture: "_ExporterCapture"
+    ) -> None:
+        """A key or team ``otel_service_name`` moves the call onto a second provider built for that name."""
+        kwargs: Final = {
+            **_closed_chat_call_kwargs(),
+            "litellm_params": {"metadata": {"user_api_key_auth_metadata": {"otel_service_name": "payments-gateway"}}},
+        }
+
+        otel_callback.log_pre_api_call(model="gpt-4o", messages=[], kwargs=kwargs)
+        asyncio.run(otel_callback.async_log_success_event(kwargs, None, None, None))
+
+        assert capture.received() == {None: ("chat gpt-4o",)}
+
+    @pytest.mark.parametrize(
+        ("env", "expected"),
+        [({}, {}), ({"OTEL_EXPORTER": "console"}, {None: ("chat gpt-4",)})],
+        ids=["nothing-configured", "operator-asked-for-console"],
+    )
+    def test_the_background_global_logger_prints_only_when_asked(
+        self,
+        monkeypatch: pytest.MonkeyPatch,
+        capture: "_ExporterCapture",
+        env: Mapping[str, str],
+        expected: Mapping[str | None, tuple[str, ...]],
+    ) -> None:
+        """The proxy publishes a generic logger even when no callback built one, which is
+        what key-only and team-only destinations run under."""
+        from litellm.integrations.otel.logger import select_global_otel_v2_logger
+
+        self._no_otel_env(monkeypatch)
+        self._isolated_proxy_registration(monkeypatch)
+        for name, value in env.items():
+            monkeypatch.setenv(name, value)
+
+        assert self._received(capture, select_global_otel_v2_logger([]).tracer_provider) == expected
+
+
 class TestTenantHostSsrfGuard:
     """Anyone who can mint a key can write ``langfuse_host``, so the host it names has
     to be one the operator approved."""