fix(otel): anchor tenant fan-out to the published provider

A legacy v1 logger can occupy proxy_server.open_telemetry_logger, in which case
the proxy publishes with registered=None and the fan-out lands on a v2 logger
taken from _in_memory_loggers. Reading the registered slot found no v2 logger
and the OTel global belonged to v1, so auth refused every tenant destination.
This commit is contained in:
Yucheng He 2026-09-05 02:42:29 -07:00
parent e8cea3e735
commit 37c964b0a2
3 changed files with 46 additions and 13 deletions

View file

@ -88,6 +88,7 @@ if TYPE_CHECKING:
)
LITELLM_TRACER_NAME: Final = "litellm"
_published_v2_provider: ApiTracerProvider | None = None
def _span_error_from_exception(
@ -876,11 +877,17 @@ def publish_global_otel_v2_provider(
The published provider is also the one that fans spans out to key/team
destinations, because it is the only provider the whole request tree passes
through; see :func:`attach_tenant_fan_out`.
through; see :func:`attach_tenant_fan_out`. It is remembered for
:func:`fan_out_provider` because neither the OTel global (``set_tracer_provider``
keeps the first provider it was ever handed) nor
``proxy_server.open_telemetry_logger`` (a legacy v1 logger can hold that slot)
reliably leads back to it.
"""
global _published_v2_provider
logger: Final = select_global_otel_v2_logger(in_memory_loggers, registered=registered)
attach_tenant_fan_out(logger.tracer_provider, logger.config)
set_global_provider(logger.tracer_provider)
_published_v2_provider = logger.tracer_provider # rebind-ok: startup records the one provider carrying the fan-out
return logger
@ -922,13 +929,13 @@ def seed_request_identity(user_api_key_dict: object, model: str | None = None) -
def fan_out_provider() -> ApiTracerProvider:
"""The provider :func:`publish_global_otel_v2_provider` gave the tenant fan-out.
That is the registered logger's own provider, which stays the carrier even when
the OTel global was claimed before the proxy published (auto-instrumentation, a
legacy logger): ``set_tracer_provider`` keeps the first provider it was given, so
reading the global there would find no fan-out and drop every destination.
Read off the publish itself, not the OTel global and not the registered logger:
the global keeps whichever provider claimed it first (auto-instrumentation, a
legacy logger), and the registered slot can hold a v1 logger while the publish
picked a v2 one from ``_in_memory_loggers``. Either detour lands on a provider
with no fan-out and drops every destination at auth.
"""
logger: Final = _registered_v2_logger()
return logger.tracer_provider if logger is not None else get_tracer_provider()
return _published_v2_provider if _published_v2_provider is not None else get_tracer_provider()
@contextmanager

View file

@ -18,8 +18,10 @@ from litellm.integrations.otel.model.config import (
is_otel_v2_enabled,
)
from litellm.integrations.otel.model.destination import OtelDestination
from litellm.integrations.otel import logger as otel_logger
from litellm.integrations.otel.logger import (
OpenTelemetryV2,
fan_out_provider,
publish_global_otel_v2_provider,
)
from litellm.integrations.otel.plumbing.context import (
@ -63,6 +65,13 @@ def allow_test_hosts(monkeypatch):
)
@pytest.fixture(autouse=True)
def isolate_published_provider(monkeypatch):
"""Publishing records the fan-out carrier in module state; one test's publish must
not become the next test's provider."""
monkeypatch.setattr(otel_logger, "_published_v2_provider", None)
def in_fresh_context(fn, *args):
"""Run ``fn`` in its own context so one test's destinations never leak."""
return contextvars.copy_context().run(fn, *args)
@ -663,12 +672,11 @@ class TestProviderWiring:
kinds = [type(p).__name__ for p in logger._tracer_provider._active_span_processor._span_processors]
assert kinds.count("TenantFanOutSpanProcessor") == 1
def test_anchoring_reads_the_fan_out_off_the_registered_logger_not_the_otel_global(self, monkeypatch):
def test_anchoring_reads_the_fan_out_off_the_published_provider_not_the_otel_global(self, monkeypatch):
"""``set_tracer_provider`` keeps the first provider it was handed. When
auto-instrumentation or a legacy logger claimed it before the proxy published,
the OTel global carries no fan-out, so reading it there would refuse every
destination while the registered logger's provider would have delivered them."""
from litellm.integrations.otel.logger import fan_out_provider
destination the published provider delivers."""
from litellm.proxy import proxy_server
config = OpenTelemetryV2Config(exporters=[ExporterSpec(kind="in_memory", owner=ExporterOwner.LANGFUSE_OTEL)])
@ -681,10 +689,26 @@ class TestProviderWiring:
assert deliverable_destinations((LANGFUSE_DEST,), claimed_first) == ()
assert deliverable_destinations((LANGFUSE_DEST,), fan_out_provider()) == (LANGFUSE_DEST,)
def test_without_a_registered_logger_anchoring_falls_back_to_the_otel_global(self, monkeypatch):
def test_a_legacy_v1_logger_holding_the_registered_slot_does_not_hide_the_fan_out(self, monkeypatch):
"""The proxy publishes with ``registered=None`` when ``open_telemetry_logger``
holds a v1 logger, so the fan-out lands on a v2 logger taken from
``_in_memory_loggers``. Reading the registered slot finds no v2 logger there and
the OTel global belongs to v1, so both detours refuse every destination the
published provider delivers."""
from litellm.integrations.opentelemetry import OpenTelemetry
from litellm.proxy import proxy_server
config = OpenTelemetryV2Config(exporters=[ExporterSpec(kind="in_memory", owner=ExporterOwner.LANGFUSE_OTEL)])
v2 = OpenTelemetryV2(config=config, callback_name="langfuse_otel")
publish_global_otel_v2_provider([v2], lambda _p: None, registered=None)
monkeypatch.setattr(proxy_server, "open_telemetry_logger", OpenTelemetry())
assert fan_out_provider() is v2.tracer_provider
assert deliverable_destinations((LANGFUSE_DEST,), fan_out_provider()) == (LANGFUSE_DEST,)
def test_without_a_publish_anchoring_falls_back_to_the_otel_global(self, monkeypatch):
from opentelemetry import trace
from litellm.integrations.otel.logger import fan_out_provider
from litellm.proxy import proxy_server
monkeypatch.setattr(proxy_server, "open_telemetry_logger", None)

View file

@ -2041,7 +2041,7 @@ def test_select_global_otel_v2_logger_builds_one_when_none_registered():
assert isinstance(chosen, OpenTelemetryV2)
def test_publish_global_otel_v2_provider_sets_selected_logger_provider():
def test_publish_global_otel_v2_provider_sets_selected_logger_provider(monkeypatch):
"""The startup publish must set the OTel global provider to the *selected*
logger's provider (the preset logger that owns every exporter), so the FastAPI
server span and the gen-ai spans share one provider and one trace.
@ -2051,8 +2051,10 @@ def test_publish_global_otel_v2_provider_sets_selected_logger_provider():
test would otherwise miss: that the published provider is the selected logger's,
not some other.
"""
from litellm.integrations.otel import logger as otel_logger
from litellm.integrations.otel.logger import publish_global_otel_v2_provider
monkeypatch.setattr(otel_logger, "_published_v2_provider", None)
cfg = OpenTelemetryV2Config(exporter="in_memory")
tp = providers.build_tracer_provider(cfg)
preset_logger = OpenTelemetryV2(