diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index 36b186cdb96..21e1feb4022 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -88,6 +88,7 @@ if TYPE_CHECKING: ) LITELLM_TRACER_NAME: Final = "litellm" +_published_v2_provider: ApiTracerProvider | None = None def _span_error_from_exception( @@ -876,11 +877,17 @@ def publish_global_otel_v2_provider( The published provider is also the one that fans spans out to key/team destinations, because it is the only provider the whole request tree passes - through; see :func:`attach_tenant_fan_out`. + through; see :func:`attach_tenant_fan_out`. It is remembered for + :func:`fan_out_provider` because neither the OTel global (``set_tracer_provider`` + keeps the first provider it was ever handed) nor + ``proxy_server.open_telemetry_logger`` (a legacy v1 logger can hold that slot) + reliably leads back to it. """ + global _published_v2_provider logger: Final = select_global_otel_v2_logger(in_memory_loggers, registered=registered) attach_tenant_fan_out(logger.tracer_provider, logger.config) set_global_provider(logger.tracer_provider) + _published_v2_provider = logger.tracer_provider # rebind-ok: startup records the one provider carrying the fan-out return logger @@ -922,13 +929,13 @@ def seed_request_identity(user_api_key_dict: object, model: str | None = None) - def fan_out_provider() -> ApiTracerProvider: """The provider :func:`publish_global_otel_v2_provider` gave the tenant fan-out. - That is the registered logger's own provider, which stays the carrier even when - the OTel global was claimed before the proxy published (auto-instrumentation, a - legacy logger): ``set_tracer_provider`` keeps the first provider it was given, so - reading the global there would find no fan-out and drop every destination. + Read off the publish itself, not the OTel global and not the registered logger: + the global keeps whichever provider claimed it first (auto-instrumentation, a + legacy logger), and the registered slot can hold a v1 logger while the publish + picked a v2 one from ``_in_memory_loggers``. Either detour lands on a provider + with no fan-out and drops every destination at auth. """ - logger: Final = _registered_v2_logger() - return logger.tracer_provider if logger is not None else get_tracer_provider() + return _published_v2_provider if _published_v2_provider is not None else get_tracer_provider() @contextmanager diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index d0e01fcba89..706916c69fb 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -18,8 +18,10 @@ from litellm.integrations.otel.model.config import ( is_otel_v2_enabled, ) from litellm.integrations.otel.model.destination import OtelDestination +from litellm.integrations.otel import logger as otel_logger from litellm.integrations.otel.logger import ( OpenTelemetryV2, + fan_out_provider, publish_global_otel_v2_provider, ) from litellm.integrations.otel.plumbing.context import ( @@ -63,6 +65,13 @@ def allow_test_hosts(monkeypatch): ) +@pytest.fixture(autouse=True) +def isolate_published_provider(monkeypatch): + """Publishing records the fan-out carrier in module state; one test's publish must + not become the next test's provider.""" + monkeypatch.setattr(otel_logger, "_published_v2_provider", None) + + def in_fresh_context(fn, *args): """Run ``fn`` in its own context so one test's destinations never leak.""" return contextvars.copy_context().run(fn, *args) @@ -663,12 +672,11 @@ class TestProviderWiring: kinds = [type(p).__name__ for p in logger._tracer_provider._active_span_processor._span_processors] assert kinds.count("TenantFanOutSpanProcessor") == 1 - def test_anchoring_reads_the_fan_out_off_the_registered_logger_not_the_otel_global(self, monkeypatch): + def test_anchoring_reads_the_fan_out_off_the_published_provider_not_the_otel_global(self, monkeypatch): """``set_tracer_provider`` keeps the first provider it was handed. When auto-instrumentation or a legacy logger claimed it before the proxy published, the OTel global carries no fan-out, so reading it there would refuse every - destination while the registered logger's provider would have delivered them.""" - from litellm.integrations.otel.logger import fan_out_provider + destination the published provider delivers.""" from litellm.proxy import proxy_server config = OpenTelemetryV2Config(exporters=[ExporterSpec(kind="in_memory", owner=ExporterOwner.LANGFUSE_OTEL)]) @@ -681,10 +689,26 @@ class TestProviderWiring: assert deliverable_destinations((LANGFUSE_DEST,), claimed_first) == () assert deliverable_destinations((LANGFUSE_DEST,), fan_out_provider()) == (LANGFUSE_DEST,) - def test_without_a_registered_logger_anchoring_falls_back_to_the_otel_global(self, monkeypatch): + def test_a_legacy_v1_logger_holding_the_registered_slot_does_not_hide_the_fan_out(self, monkeypatch): + """The proxy publishes with ``registered=None`` when ``open_telemetry_logger`` + holds a v1 logger, so the fan-out lands on a v2 logger taken from + ``_in_memory_loggers``. Reading the registered slot finds no v2 logger there and + the OTel global belongs to v1, so both detours refuse every destination the + published provider delivers.""" + from litellm.integrations.opentelemetry import OpenTelemetry + from litellm.proxy import proxy_server + + config = OpenTelemetryV2Config(exporters=[ExporterSpec(kind="in_memory", owner=ExporterOwner.LANGFUSE_OTEL)]) + v2 = OpenTelemetryV2(config=config, callback_name="langfuse_otel") + publish_global_otel_v2_provider([v2], lambda _p: None, registered=None) + monkeypatch.setattr(proxy_server, "open_telemetry_logger", OpenTelemetry()) + + assert fan_out_provider() is v2.tracer_provider + assert deliverable_destinations((LANGFUSE_DEST,), fan_out_provider()) == (LANGFUSE_DEST,) + + def test_without_a_publish_anchoring_falls_back_to_the_otel_global(self, monkeypatch): from opentelemetry import trace - from litellm.integrations.otel.logger import fan_out_provider from litellm.proxy import proxy_server monkeypatch.setattr(proxy_server, "open_telemetry_logger", None) diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py index b735abaf7bf..2869c804c07 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py @@ -2041,7 +2041,7 @@ def test_select_global_otel_v2_logger_builds_one_when_none_registered(): assert isinstance(chosen, OpenTelemetryV2) -def test_publish_global_otel_v2_provider_sets_selected_logger_provider(): +def test_publish_global_otel_v2_provider_sets_selected_logger_provider(monkeypatch): """The startup publish must set the OTel global provider to the *selected* logger's provider (the preset logger that owns every exporter), so the FastAPI server span and the gen-ai spans share one provider and one trace. @@ -2051,8 +2051,10 @@ def test_publish_global_otel_v2_provider_sets_selected_logger_provider(): test would otherwise miss: that the published provider is the selected logger's, not some other. """ + from litellm.integrations.otel import logger as otel_logger from litellm.integrations.otel.logger import publish_global_otel_v2_provider + monkeypatch.setattr(otel_logger, "_published_v2_provider", None) cfg = OpenTelemetryV2Config(exporter="in_memory") tp = providers.build_tracer_provider(cfg) preset_logger = OpenTelemetryV2(