From 1963a42bac3d88778dc8c931d8e77bf6d2870f85 Mon Sep 17 00:00:00 2001 From: yucheng Date: Wed, 28 Oct 2026 01:32:37 -0700 Subject: [PATCH] fix(otel): stop a key's Langfuse logger from re-sending spans to the operator's collector With langfuse_otel only in key or team logging, the proxy publishes the generic V2 logger at startup and it already exports to the operator's OTLP collector. The Langfuse logger a key's request builds kept the same base exporter, so the collector got every model span twice. A logger built after the publish now keeps only its preset's own exporters, as it already did beside a configured V2 logger, and a later call that would rebuild one exporting nowhere reuses it. --- litellm/integrations/otel/logger.py | 5 ++ litellm/litellm_core_utils/litellm_logging.py | 21 +++++---- .../test_langfuse_otel_tenant_only.py | 1 + .../otel/test_otel_v2_destinations.py | 46 +++++++++++++++++++ 4 files changed, 64 insertions(+), 9 deletions(-) diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index e09226ac9cd..21414e80c00 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -1038,6 +1038,11 @@ def seed_request_identity(user_api_key_dict: object, model: str | None = None) - logger.seed_request_identity(user_api_key_dict, model=model) +def v2_provider_published() -> bool: + """Whether the proxy published a v2 provider at startup, which delivers to the operator's own backends.""" + return _published_v2_provider is not None + + def fan_out_provider() -> ApiTracerProvider: """The provider :func:`publish_global_otel_v2_provider` gave the tenant fan-out. diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 48e8a44b9a4..69e9d338ba7 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -5971,11 +5971,12 @@ def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[Custom Returns ``None`` when V2 is off OR when there's no preset registered for ``callback_name`` — callers should then fall through to the legacy path. - A logger built while another V2 logger is already registered keeps only the - exporters its own preset contributed, whether or not the operator holds - credentials for that backend and whether or not a destination is anchored: the - registered logger owns operator delivery, so a copy of the operator's base OTLP - exporters here would emit every LLM call a second time into the operator's sink. + A logger built while another V2 logger is already registered, or after the proxy + published one at startup, keeps only the exporters its own preset contributed, + whether or not the operator holds credentials for that backend and whether or not + a destination is anchored: that logger owns operator delivery, so a copy of the + operator's base OTLP exporters here would emit every LLM call a second time into + the operator's sink. A preset that contributes no exporter of its own (a mapper over the operator's collector) keeps the base exporters, since it has nothing else to deliver through. A preset that needs operator credentials it cannot find is allowed to build only @@ -5986,7 +5987,7 @@ def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[Custom until a key/team destination appears: the proxy builds the operator's callback at startup, before any request has resolved a destination, and the legacy path there would post every non-team request to the backend keyless. That logger is reused by - later calls as long as the preset would again build exporting nowhere; one that + later calls as long as a rebuild would again export nowhere; one that was degraded for a destination while the preset raises without one is not, since the degrade was justified by that destination alone. """ @@ -5994,7 +5995,7 @@ def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[Custom if not is_otel_v2_enabled(): return None - from litellm.integrations.otel.logger import OpenTelemetryV2, build_otel_v2_logger + from litellm.integrations.otel.logger import OpenTelemetryV2, build_otel_v2_logger, v2_provider_published from litellm.integrations.otel.plumbing.context import destination_backends from litellm.integrations.otel.presets import PRESET_BY_CALLBACK @@ -6022,9 +6023,11 @@ def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[Custom return None if _is_credential_gated(built) and not carried and _only_the_placeholder_would_export(built): return None - if existing is not None and _exports_nowhere(built): + config: Final = ( + _only_the_presets_own_exporters(built, callback_name) if has_v2_logger or v2_provider_published() else built + ) + if existing is not None and _exports_nowhere(config): return existing - config: Final = _only_the_presets_own_exporters(built, callback_name) if has_v2_logger else built if _exports_nowhere(config): verbose_logger.warning( "OTel V2: no operator credentials for '%s'; only key/team destinations will receive its traces", diff --git a/tests/integration/observability/test_langfuse_otel_tenant_only.py b/tests/integration/observability/test_langfuse_otel_tenant_only.py index 29ae7f4c990..9ea50faf439 100644 --- a/tests/integration/observability/test_langfuse_otel_tenant_only.py +++ b/tests/integration/observability/test_langfuse_otel_tenant_only.py @@ -128,4 +128,5 @@ def test_a_tenant_langfuse_project_gets_user_session_and_tags_without_operator_k for span in tenant ] == [{"user.id": "user-42", "session.id": session, "langfuse.trace.tags": TAGS}], [s["name"] for s in tenant] assert "langfuse.observation.metadata" in tenant[0]["attributes"], sorted(tenant[0]["attributes"]) + assert len(operator) == 1, [span["name"] for span in operator] assert "langfuse.observation.metadata" not in operator[0]["attributes"], sorted(operator[0]["attributes"]) diff --git a/tests/unit/integrations/otel/test_otel_v2_destinations.py b/tests/unit/integrations/otel/test_otel_v2_destinations.py index 03c545334df..cd2ae99e93c 100644 --- a/tests/unit/integrations/otel/test_otel_v2_destinations.py +++ b/tests/unit/integrations/otel/test_otel_v2_destinations.py @@ -3522,6 +3522,52 @@ class TestLangfuseTenantOnly: assert later is at_startup assert loggers == [at_startup] + @staticmethod + def _key_logger_beside_the_generic_startup_logger( + monkeypatch: pytest.MonkeyPatch, loggers: list[CustomLogger] + ) -> OpenTelemetryV2: + """``langfuse_otel`` only in a key's logging: the proxy published the generic logger, + which exports to the operator's collector, and a key's request builds the Langfuse one.""" + from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 + + credential_less_proxy(monkeypatch) + monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.local:4318") + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + publish_global_otel_v2_provider(loggers, lambda _p: None) + + def with_destination() -> OpenTelemetryV2 | None: + set_request_destinations((LANGFUSE_DEST,)) + return _maybe_construct_otel_v2("langfuse_otel", loggers) + + is_otel_v2_enabled.cache_clear() + logger: Final = in_fresh_context(with_destination) + is_otel_v2_enabled.cache_clear() + assert isinstance(logger, OpenTelemetryV2) + return logger + + def test_a_key_destinations_logger_leaves_the_operators_collector_to_the_published_logger( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The published logger already sends every span to the collector; a copy here doubles each one.""" + logger: Final = self._key_logger_beside_the_generic_startup_logger(monkeypatch, []) + + assert all(spec.requires_headers and not spec.headers for spec in logger.config.exporters) + + def test_a_key_destinations_logger_is_reused_by_later_requests_without_a_destination( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 + + loggers: Final[list[CustomLogger]] = [] + for_the_key: Final = self._key_logger_beside_the_generic_startup_logger(monkeypatch, loggers) + + is_otel_v2_enabled.cache_clear() + later: Final = in_fresh_context(_maybe_construct_otel_v2, "langfuse_otel", loggers) + is_otel_v2_enabled.cache_clear() + + assert later is for_the_key + assert loggers == [for_the_key] + def test_a_key_destination_gets_the_langfuse_fields_from_the_generic_startup_logger( self, monkeypatch: pytest.MonkeyPatch ) -> None: