From ffcc4f7a70148df3905b6b064857c32b17d3ea71 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Thu, 29 Oct 2026 12:20:07 -0700 Subject: [PATCH] fix(otel)!: print OTel v2 spans to stdout only when the operator asks (#48723) * fix(otel): stop OTel v2 vendor presets from printing every span to stdout With LITELLM_OTEL_V2 on and a vendor callback such as langfuse_otel or arize holding operator credentials, the proxy printed every span as JSON to stdout, prompts included, on the request path. Nobody set OTEL_EXPORTER, but the console placeholder that OpenTelemetryV2Config folds in for an empty exporter list stayed next to the vendor's own exporter. The provider now skips that placeholder when the config has any other exporter. A console exporter the operator names with OTEL_EXPORTER=console is kept, and so is the placeholder when it is the only exporter. * test(otel): type the console-placeholder tests and drop inherited OTEL_* env from the stdout check * fix(otel): print the console default only for the otel callback The proxy's background V2 logger (published when no callback built one, which is what key-only and team-only Langfuse destinations run under) also kept the unconfigured console exporter, so it printed every span with prompts to stdout. The console default now prints only for a listed otel callback or an exporter the operator named. * fix(otel): keep the otel callback's console output on service-name routes A key or team otel_service_name moves the call onto a provider the tenant tracer cache builds, which never got the console choice, so the plain otel callback stopped printing those calls. The no-operator-keys integration rig also drops inherited Langfuse keys. --- litellm/integrations/otel/logger.py | 9 +- litellm/integrations/otel/model/config.py | 11 ++ .../integrations/otel/plumbing/providers.py | 8 ++ litellm/integrations/otel/plumbing/routing.py | 3 + litellm/integrations/otel/presets/utils.py | 13 +- litellm/litellm_core_utils/litellm_logging.py | 2 +- .../_langfuse_otel_metadata_support.py | 14 ++- .../test_langfuse_otel_console_output.py | 98 +++++++++++++++ .../otel/test_otel_v2_destinations.py | 115 +++++++++++++++++- 9 files changed, 252 insertions(+), 21 deletions(-) create mode 100644 tests/integration/observability/test_langfuse_otel_console_output.py diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index 4c9618800ab..b5b65d299e0 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -198,6 +198,7 @@ class OpenTelemetryV2(CustomLogger): tracer_provider: TracerProvider | None = None, logger_provider: LoggerProvider | None = None, meter_provider: "MeterProvider | None" = None, + console_default: bool = False, **kwargs: Unpack[_CustomLoggerOptions], ) -> None: super().__init__(**kwargs) @@ -206,7 +207,7 @@ class OpenTelemetryV2(CustomLogger): self._tracer_provider: TracerProvider = ( tracer_provider if tracer_provider is not None - else build_tracer_provider(self.config, tenant_overrides=True) + else build_tracer_provider(self.config, tenant_overrides=True, console_default=console_default) ) self.tracer: Tracer = get_tracer(self._tracer_provider, LITELLM_TRACER_NAME) self._metrics_recorder = self._init_metrics(meter_provider) @@ -217,7 +218,9 @@ class OpenTelemetryV2(CustomLogger): mappers=resolve_mappers(self.config.mapper_names), event_recorder=self._init_events(logger_provider), ) - self._tenant_tracers = TenantTracerCache(self.config, callback_name, LITELLM_TRACER_NAME) + self._tenant_tracers = TenantTracerCache( + self.config, callback_name, LITELLM_TRACER_NAME, console_default=console_default + ) self._open_llm_calls: OrderedDict[str, _LLMCallSpan] = OrderedDict() self._init_otel_logger_on_litellm_proxy() @@ -1089,6 +1092,7 @@ def build_otel_v2_logger( logger_provider: LoggerProvider | None = None, meter_provider: "MeterProvider | None" = None, settings: Mapping[str, object] = MappingProxyType({}), + console_default: bool = False, ) -> OpenTelemetryV2: return _logger_class(config)( config=config, @@ -1096,6 +1100,7 @@ def build_otel_v2_logger( tracer_provider=tracer_provider, logger_provider=logger_provider, meter_provider=meter_provider, + console_default=console_default, **settings, ) diff --git a/litellm/integrations/otel/model/config.py b/litellm/integrations/otel/model/config.py index dd150082252..b50e927f977 100644 --- a/litellm/integrations/otel/model/config.py +++ b/litellm/integrations/otel/model/config.py @@ -121,6 +121,17 @@ class ExporterSpec(LiteLLMBaseModel): ) +def is_unconfigured_placeholder(spec: ExporterSpec) -> bool: + """Whether ``spec`` is the one ``_normalize`` folds in when nothing was configured. + + No field set is what says the operator asked for nothing: an exporter they did + configure survives, even ``OTEL_EXPORTER=console`` whose value matches the default, + and so does the gated spec ``credential_gated_exporters`` appends, which would + otherwise eat itself when one preset layers onto another. + """ + return not spec.model_fields_set + + class _EnvWithoutBareExcludedServices(PydanticBaseSettingsSource): def __init__(self, settings_cls: type[BaseSettings], env_settings: PydanticBaseSettingsSource) -> None: super().__init__(settings_cls) diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index 39e78194228..b5a5b85fc11 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -56,6 +56,7 @@ from litellm.integrations.otel.model.config import ( OpenTelemetryV2Config, capture_message_content_from_env, captures_span_content, + is_unconfigured_placeholder, ) from litellm.integrations.otel.model.semconv import ( DB, @@ -1551,6 +1552,7 @@ def build_tracer_provider( use_simple_processor: bool | None = None, tenant_overrides: bool = False, strip_message_content: bool = False, + console_default: bool = False, ) -> TracerProvider: """Build the shared :class:`TracerProvider`. @@ -1575,6 +1577,10 @@ def build_tracer_provider( ``strip_message_content`` removes prompt and response content before every exporter, for a routed provider whose spans can carry content another destination asked for. + + The console spec ``config._normalize`` folds in when nothing was configured + prints only under ``console_default``, which the ``otel`` callback passes: a + vendor preset or the proxy's background global logger was never asked to print. """ provider: Final = TracerProvider(resource=build_resource(config)) provider.sampler = _RequestExclusionSampler(provider.sampler) @@ -1592,6 +1598,8 @@ def build_tracer_provider( for spec in config.exporters: if spec.requires_headers and not spec.headers: continue + if is_unconfigured_placeholder(spec) and not console_default: + continue processor = _spec_processor( spec, use_simple_processor, diff --git a/litellm/integrations/otel/plumbing/routing.py b/litellm/integrations/otel/plumbing/routing.py index eb90b9035c5..9cf96a1e5e3 100644 --- a/litellm/integrations/otel/plumbing/routing.py +++ b/litellm/integrations/otel/plumbing/routing.py @@ -163,10 +163,12 @@ class TenantTracerCache: config: OpenTelemetryV2Config, callback_name: str | None, tracer_name: str, + console_default: bool = False, ) -> None: self._config = config self._callback_name = callback_name self._tracer_name = tracer_name + self._console_default = console_default # Guards the three mutable structures below: ``pre_call`` can run on # thread-pool workers concurrently with the event loop, so cache # updates, span counts, and retirement must be atomic. @@ -284,6 +286,7 @@ class TenantTracerCache: built: Final = build_tracer_provider( self._routed_config(credential_headers, project_headers, endpoint, service_name), strip_message_content=not self._config.capture_span_content, + console_default=self._console_default, ) self._providers[cache_key] = built return built diff --git a/litellm/integrations/otel/presets/utils.py b/litellm/integrations/otel/presets/utils.py index 1270c41e77b..677f09a1190 100644 --- a/litellm/integrations/otel/presets/utils.py +++ b/litellm/integrations/otel/presets/utils.py @@ -3,7 +3,7 @@ from collections.abc import Iterable from typing import Final -from litellm.integrations.otel.model.config import ExporterOwner, ExporterSpec +from litellm.integrations.otel.model.config import ExporterOwner, ExporterSpec, is_unconfigured_placeholder def ensure_mappers(mapper_names: Iterable[str], *names: str) -> list[str]: @@ -35,14 +35,3 @@ def credential_gated_exporters( *(spec for spec in exporters if not is_unconfigured_placeholder(spec)), ExporterSpec(owner=owner, requires_headers=True), ) - - -def is_unconfigured_placeholder(spec: "ExporterSpec") -> bool: - """Whether ``spec`` is the one ``_normalize`` folds in when nothing was configured. - - No field set is what says the operator asked for nothing: an exporter they did - configure survives, even ``OTEL_EXPORTER=console`` whose value matches the default, - and so does the gated spec this module appends, which would otherwise eat itself - when one preset layers onto another. - """ - return not spec.model_fields_set diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 81695925831..34c98efc22f 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -5623,7 +5623,7 @@ def build_custom_logger_compatible_class( return callback otel_settings: Final = _get_custom_logger_settings_from_proxy_server(callback_name=logging_integration) otel_logger_v2: Final = build_otel_v2_logger( - config=OpenTelemetryV2Config(**otel_settings), settings=otel_settings + config=OpenTelemetryV2Config(**otel_settings), settings=otel_settings, console_default=True ) _in_memory_loggers.append(otel_logger_v2) _maybe_auto_initialize_arize_phoenix(_in_memory_loggers) diff --git a/tests/integration/observability/_langfuse_otel_metadata_support.py b/tests/integration/observability/_langfuse_otel_metadata_support.py index 65e1196591a..260498cfb46 100644 --- a/tests/integration/observability/_langfuse_otel_metadata_support.py +++ b/tests/integration/observability/_langfuse_otel_metadata_support.py @@ -210,6 +210,8 @@ def langfuse_rig( sink_reply: Callable[[Request], Reply] | None = None, overrides: Mapping[str, str] | None = None, model_list: Callable[[str], Sequence[Mapping[str, object]]] | None = None, + remove_environment: tuple[str, ...] = (), + operator_keys: bool = True, ) -> Iterator[LangfuseRig]: def upstream(request: Request) -> Reply: if request.method == "GET": @@ -253,8 +255,11 @@ def langfuse_rig( gateway, tmp_path, { - "LANGFUSE_PUBLIC_KEY": "pk-lf-integration", - "LANGFUSE_SECRET_KEY": "sk-lf-integration", + **( + {"LANGFUSE_PUBLIC_KEY": "pk-lf-integration", "LANGFUSE_SECRET_KEY": "sk-lf-integration"} + if operator_keys + else {} + ), "LANGFUSE_HOST": collector.url, "LITELLM_OTEL_V2": "1", **{ @@ -264,6 +269,11 @@ def langfuse_rig( }, config=path, workers=2, + remove_environment=( + remove_environment + if operator_keys + else (*remove_environment, "LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY") + ), ) as owned: yield LangfuseRig(owned.gateway, provider, collector, owned) diff --git a/tests/integration/observability/test_langfuse_otel_console_output.py b/tests/integration/observability/test_langfuse_otel_console_output.py new file mode 100644 index 00000000000..5f056b77227 --- /dev/null +++ b/tests/integration/observability/test_langfuse_otel_console_output.py @@ -0,0 +1,98 @@ +import json +import os +import uuid +from collections.abc import Mapping +from pathlib import Path +from typing import Final + +import pytest +from _langfuse_otel_metadata_support import exported_llm_span, langfuse_rig, scenario_identity, watched_spans +from integration._support.client import Gateway, eventually + +CONSOLE_SPAN: Final = '"span_id": "0x' +INHERITED_OTEL_ENV: Final = tuple(name for name in os.environ if name.startswith("OTEL_")) + + +@pytest.mark.parametrize( + ("overrides", "printed"), + [({}, False), ({"OTEL_EXPORTER": "console"}, True)], + ids=["exporter-unset", "operator-asked-for-console"], +) +def test_langfuse_otel_prints_spans_to_stdout_only_when_the_operator_asks( + gateway: Gateway, tmp_path: Path, overrides: Mapping[str, str], printed: bool +) -> None: + marker: Final = "lf-console-" + uuid.uuid4().hex + with ( + langfuse_rig( + gateway, + tmp_path, + {}, + marker, + overrides={"OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT": "span_only", **overrides}, + remove_environment=INHERITED_OTEL_ENV, + ) as rig, + rig.candidate.scenario() as scenario, + ): + key, identity = scenario_identity(scenario, rig.provider, team=False) + response: Final = rig.candidate.request( + "POST", + "/v1/chat/completions", + {"model": identity["model"], "messages": [{"role": "user", "content": marker}]}, + key=key, + ) + assert response.status_code == 200, response.text + + exported_llm_span(rig.collector, response.json()["id"]) + + on_stdout: Final = CONSOLE_SPAN in rig.proxy.log.read_text() + assert on_stdout is printed, f"span JSON on proxy stdout: {on_stdout}, see {rig.proxy.log}" + + +@pytest.mark.parametrize("callbacks", [[], ["langfuse_otel"]], ids=["key-only", "callback-listed"]) +def test_key_langfuse_without_operator_keys_prints_no_spans( + gateway: Gateway, tmp_path: Path, callbacks: list[str] +) -> None: + marker: Final = "lf-console-key-" + uuid.uuid4().hex + with ( + langfuse_rig( + gateway, + tmp_path, + {"callbacks": callbacks}, + marker, + overrides={"OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT": "span_only"}, + remove_environment=INHERITED_OTEL_ENV, + operator_keys=False, + ) as rig, + rig.candidate.scenario() as scenario, + ): + key: Final = scenario.key( + key_alias="lf-console-key-" + uuid.uuid4().hex, + metadata={ + "logging": [ + { + "callback_name": "langfuse_otel", + "callback_type": "success", + "callback_vars": { + "langfuse_public_key": "pk-lf-key", + "langfuse_secret_key": "sk-lf-key", + "langfuse_host": rig.collector.url, + }, + } + ] + }, + ) + model: Final = scenario.model(api_base=rig.provider.url + "/v1") + response: Final = rig.candidate.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": marker}]}, + key=key, + ) + assert response.status_code == 200, response.text + + eventually( + watched_spans(rig.collector), lambda spans: any(marker in json.dumps(span) for span in spans), seconds=25 + ) + + on_stdout: Final = CONSOLE_SPAN in rig.proxy.log.read_text() + assert not on_stdout, f"span JSON on proxy stdout, see {rig.proxy.log}" diff --git a/tests/unit/integrations/otel/test_otel_v2_destinations.py b/tests/unit/integrations/otel/test_otel_v2_destinations.py index 2739239e3fa..2bac85d43d7 100644 --- a/tests/unit/integrations/otel/test_otel_v2_destinations.py +++ b/tests/unit/integrations/otel/test_otel_v2_destinations.py @@ -3352,8 +3352,7 @@ class TestArizeTenantOnly: assert capture.received() == { "space_id=space-operator,api_key=key-operator": ("chat gpt-4",), - None: ("chat gpt-4",), - }, "the operator's space, plus the stdout placeholder every credentialed preset keeps today" + }, "the operator's space, with no stdout copy the operator never asked for" def test_the_standard_otlp_headers_still_reach_the_operators_space(self, monkeypatch): credential_less_arize(monkeypatch) @@ -3366,8 +3365,7 @@ class TestArizeTenantOnly: assert capture.received() == { "space_id=space-operator,api_key=key-operator": ("chat gpt-4",), - None: ("chat gpt-4",), - }, "the operator's space, plus the stdout placeholder every credentialed preset keeps today" + }, "the operator's space, with no stdout copy the operator never asked for" def test_a_credential_less_arize_still_delivers_a_team_destination(self, monkeypatch): from litellm.integrations.otel.plumbing.providers import attach_tenant_fan_out @@ -4825,6 +4823,115 @@ class TestCredentialGatedExporters: assert kept[0] is operator_console +class TestConsolePlaceholder: + """The console spec ``_normalize`` folds in for an empty exporter list prints spans + only for the ``otel`` callback; no other logger was asked to print.""" + + @pytest.fixture + def capture(self, monkeypatch: pytest.MonkeyPatch) -> "_ExporterCapture": + return TestArizeTenantOnly._capture_exporters(monkeypatch) + + @staticmethod + def _received(capture: "_ExporterCapture", provider: TracerProvider) -> Mapping[str | None, tuple[str, ...]]: + emit(provider) + provider.force_flush() + return capture.received() + + @staticmethod + def _no_otel_env(monkeypatch: pytest.MonkeyPatch) -> None: + for name in _OTEL_SHORTHAND_ENV: + monkeypatch.delenv(name, raising=False) + + @staticmethod + def _operator_langfuse(monkeypatch: pytest.MonkeyPatch) -> None: + TestConsolePlaceholder._no_otel_env(monkeypatch) + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-lf-operator") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-lf-operator") + + @staticmethod + def _isolated_proxy_registration(monkeypatch: pytest.MonkeyPatch) -> None: + for name in ("service_callback", "input_callback", "_async_success_callback", "_async_failure_callback"): + monkeypatch.setattr(litellm, name, []) + monkeypatch.setattr("litellm.proxy.proxy_server.open_telemetry_logger", None) + + def test_a_credentialed_langfuse_exports_to_langfuse_alone( + self, monkeypatch: pytest.MonkeyPatch, capture: "_ExporterCapture" + ) -> None: + self._operator_langfuse(monkeypatch) + config: Final = langfuse_preset() + langfuse_headers: Final = next( + spec.headers for spec in config.exporters if spec.owner is ExporterOwner.LANGFUSE_OTEL + ) + + assert self._received(capture, build_tracer_provider(config)) == {langfuse_headers: ("chat gpt-4",)} + + def test_a_console_exporter_the_operator_named_still_prints( + self, monkeypatch: pytest.MonkeyPatch, capture: "_ExporterCapture" + ) -> None: + self._operator_langfuse(monkeypatch) + monkeypatch.setenv("OTEL_EXPORTER", "console") + + assert self._received(capture, build_tracer_provider(langfuse_preset()))[None] == ("chat gpt-4",) + + @pytest.fixture + def otel_callback(self, monkeypatch: pytest.MonkeyPatch, capture: "_ExporterCapture") -> OpenTelemetryV2: + """The plain ``otel`` callback with nothing else configured, built the way the proxy builds it.""" + from litellm.litellm_core_utils import litellm_logging as logging_module + + self._no_otel_env(monkeypatch) + self._isolated_proxy_registration(monkeypatch) + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + monkeypatch.setattr(logging_module, "_in_memory_loggers", []) + is_otel_v2_enabled.cache_clear() + built: Final = logging_module.build_custom_logger_compatible_class( + "otel", internal_usage_cache=None, llm_router=None + ) + assert isinstance(built, OpenTelemetryV2) + return built + + def test_the_otel_callback_prints_when_nothing_else_is_configured( + self, otel_callback: OpenTelemetryV2, capture: "_ExporterCapture" + ) -> None: + assert self._received(capture, otel_callback.tracer_provider) == {None: ("chat gpt-4",)} + + def test_the_otel_callback_prints_a_call_for_a_key_with_its_own_service_name( + self, otel_callback: OpenTelemetryV2, capture: "_ExporterCapture" + ) -> None: + """A key or team ``otel_service_name`` moves the call onto a second provider built for that name.""" + kwargs: Final = { + **_closed_chat_call_kwargs(), + "litellm_params": {"metadata": {"user_api_key_auth_metadata": {"otel_service_name": "payments-gateway"}}}, + } + + otel_callback.log_pre_api_call(model="gpt-4o", messages=[], kwargs=kwargs) + asyncio.run(otel_callback.async_log_success_event(kwargs, None, None, None)) + + assert capture.received() == {None: ("chat gpt-4o",)} + + @pytest.mark.parametrize( + ("env", "expected"), + [({}, {}), ({"OTEL_EXPORTER": "console"}, {None: ("chat gpt-4",)})], + ids=["nothing-configured", "operator-asked-for-console"], + ) + def test_the_background_global_logger_prints_only_when_asked( + self, + monkeypatch: pytest.MonkeyPatch, + capture: "_ExporterCapture", + env: Mapping[str, str], + expected: Mapping[str | None, tuple[str, ...]], + ) -> None: + """The proxy publishes a generic logger even when no callback built one, which is + what key-only and team-only destinations run under.""" + from litellm.integrations.otel.logger import select_global_otel_v2_logger + + self._no_otel_env(monkeypatch) + self._isolated_proxy_registration(monkeypatch) + for name, value in env.items(): + monkeypatch.setenv(name, value) + + assert self._received(capture, select_global_otel_v2_logger([]).tracer_provider) == expected + + class TestTenantHostSsrfGuard: """Anyone who can mint a key can write ``langfuse_host``, so the host it names has to be one the operator approved."""