diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 8e6e2108dfc..29a29d376dc 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -200,7 +200,7 @@ if TYPE_CHECKING: from mcp.types import EmbeddedResource, ImageContent, TextContent from litellm.integrations.otel.logger import OpenTelemetryV2 - from litellm.integrations.otel.model.config import OpenTelemetryV2Config + from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig try: from litellm_enterprise.enterprise_callbacks.callback_controls import ( @@ -4809,6 +4809,11 @@ def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[Custom as it always did and the caller falls through to the legacy path, so the proxy never publishes a provider that exports nowhere for a backend the operator configured and no tenant can use. + + The degraded preset keeps the operator's generic OTLP collector, so a proxy whose + only v2 backend is that preset still reaches it. Beside another v2 logger the + collector is already that logger's, and a second copy of every model span from + this one would land there too, so only the credential-gated exporter is kept. """ from litellm.integrations.otel.model.config import is_otel_v2_enabled @@ -4830,11 +4835,16 @@ def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[Custom ): return callback try: - config: Final = preset_fn(allow_missing_credentials=serves_a_destination) + built: Final = preset_fn(allow_missing_credentials=serves_a_destination) except Exception: # If env vars are missing or the preset raises, defer to the legacy path # so customers get the same error story they had before V2 landed. return None + config: Final = ( + _only_the_gated_exporter(built) + if _is_credential_gated(built) and any(isinstance(callback, OpenTelemetryV2) for callback in _in_memory_loggers) + else built + ) if _exports_nowhere(config): verbose_logger.warning( "OTel V2: no operator credentials for '%s'; only key/team destinations will receive its traces", @@ -4847,7 +4857,22 @@ def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[Custom def _exports_nowhere(config: "OpenTelemetryV2Config") -> bool: """Whether every exporter in ``config`` is waiting on credentials it never got.""" - return all(spec.requires_headers and not spec.headers for spec in config.exporters) + return all(_is_gated(spec) for spec in config.exporters) + + +def _is_credential_gated(config: "OpenTelemetryV2Config") -> bool: + """Whether the preset built without the operator's own credentials for its backend.""" + return any(_is_gated(spec) for spec in config.exporters) + + +def _only_the_gated_exporter(config: "OpenTelemetryV2Config") -> "OpenTelemetryV2Config": + return config.model_copy( + update={"exporters": [spec for spec in config.exporters if _is_gated(spec)]} # mutable-ok: model_copy update + ) + + +def _is_gated(spec: "ExporterSpec") -> bool: + return spec.requires_headers and not spec.headers def _maybe_auto_initialize_arize_phoenix(_in_memory_loggers: list[CustomLogger]) -> None: diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index 7e40f333e07..4e793b480fb 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -15,6 +15,7 @@ import litellm from litellm.integrations.otel import logger as otel_logger from litellm.integrations.otel.logger import ( OpenTelemetryV2, + build_otel_v2_logger, fan_out_provider, publish_global_otel_v2_provider, ) @@ -1208,6 +1209,39 @@ class TestPresetDegradation: assert first is not None assert second is first + @staticmethod + def _degraded_langfuse_beside(loggers, monkeypatch): + from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 + + credential_less_proxy(monkeypatch) + monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.local:4318") + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + + def run(): + set_request_destinations((LANGFUSE_DEST,)) + return _maybe_construct_otel_v2("langfuse_otel", loggers) + + is_otel_v2_enabled.cache_clear() + logger = in_fresh_context(run) + is_otel_v2_enabled.cache_clear() + assert logger is not None + return logger + + def test_a_degraded_logger_beside_another_v2_logger_leaves_the_collector_to_it(self, monkeypatch): + """The other logger's provider already exports every span to the operator's + collector, so a second model span from this one would land there twice.""" + collector_logger = build_otel_v2_logger(OpenTelemetryV2Config(exporter="in_memory")) + + logger = self._degraded_langfuse_beside([collector_logger], monkeypatch) + + assert [spec.endpoint for spec in logger.config.exporters] == [None] + assert all(spec.requires_headers and not spec.headers for spec in logger.config.exporters) + + def test_a_degraded_logger_on_its_own_keeps_the_operator_collector(self, monkeypatch): + logger = self._degraded_langfuse_beside([], monkeypatch) + + assert [spec.endpoint for spec in logger.config.exporters] == ["http://collector.local:4318", None] + class TestContextIsolation: def test_destinations_do_not_leak_between_requests(self):