mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-10 03:28:53 +00:00
fix(otel): anchor tenant fan-out to the published provider
A legacy v1 logger can occupy proxy_server.open_telemetry_logger, in which case the proxy publishes with registered=None and the fan-out lands on a v2 logger taken from _in_memory_loggers. Reading the registered slot found no v2 logger and the OTel global belonged to v1, so auth refused every tenant destination.
This commit is contained in:
parent
e8cea3e735
commit
37c964b0a2
3 changed files with 46 additions and 13 deletions
|
|
@ -88,6 +88,7 @@ if TYPE_CHECKING:
|
|||
)
|
||||
|
||||
LITELLM_TRACER_NAME: Final = "litellm"
|
||||
_published_v2_provider: ApiTracerProvider | None = None
|
||||
|
||||
|
||||
def _span_error_from_exception(
|
||||
|
|
@ -876,11 +877,17 @@ def publish_global_otel_v2_provider(
|
|||
|
||||
The published provider is also the one that fans spans out to key/team
|
||||
destinations, because it is the only provider the whole request tree passes
|
||||
through; see :func:`attach_tenant_fan_out`.
|
||||
through; see :func:`attach_tenant_fan_out`. It is remembered for
|
||||
:func:`fan_out_provider` because neither the OTel global (``set_tracer_provider``
|
||||
keeps the first provider it was ever handed) nor
|
||||
``proxy_server.open_telemetry_logger`` (a legacy v1 logger can hold that slot)
|
||||
reliably leads back to it.
|
||||
"""
|
||||
global _published_v2_provider
|
||||
logger: Final = select_global_otel_v2_logger(in_memory_loggers, registered=registered)
|
||||
attach_tenant_fan_out(logger.tracer_provider, logger.config)
|
||||
set_global_provider(logger.tracer_provider)
|
||||
_published_v2_provider = logger.tracer_provider # rebind-ok: startup records the one provider carrying the fan-out
|
||||
return logger
|
||||
|
||||
|
||||
|
|
@ -922,13 +929,13 @@ def seed_request_identity(user_api_key_dict: object, model: str | None = None) -
|
|||
def fan_out_provider() -> ApiTracerProvider:
|
||||
"""The provider :func:`publish_global_otel_v2_provider` gave the tenant fan-out.
|
||||
|
||||
That is the registered logger's own provider, which stays the carrier even when
|
||||
the OTel global was claimed before the proxy published (auto-instrumentation, a
|
||||
legacy logger): ``set_tracer_provider`` keeps the first provider it was given, so
|
||||
reading the global there would find no fan-out and drop every destination.
|
||||
Read off the publish itself, not the OTel global and not the registered logger:
|
||||
the global keeps whichever provider claimed it first (auto-instrumentation, a
|
||||
legacy logger), and the registered slot can hold a v1 logger while the publish
|
||||
picked a v2 one from ``_in_memory_loggers``. Either detour lands on a provider
|
||||
with no fan-out and drops every destination at auth.
|
||||
"""
|
||||
logger: Final = _registered_v2_logger()
|
||||
return logger.tracer_provider if logger is not None else get_tracer_provider()
|
||||
return _published_v2_provider if _published_v2_provider is not None else get_tracer_provider()
|
||||
|
||||
|
||||
@contextmanager
|
||||
|
|
|
|||
|
|
@ -18,8 +18,10 @@ from litellm.integrations.otel.model.config import (
|
|||
is_otel_v2_enabled,
|
||||
)
|
||||
from litellm.integrations.otel.model.destination import OtelDestination
|
||||
from litellm.integrations.otel import logger as otel_logger
|
||||
from litellm.integrations.otel.logger import (
|
||||
OpenTelemetryV2,
|
||||
fan_out_provider,
|
||||
publish_global_otel_v2_provider,
|
||||
)
|
||||
from litellm.integrations.otel.plumbing.context import (
|
||||
|
|
@ -63,6 +65,13 @@ def allow_test_hosts(monkeypatch):
|
|||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def isolate_published_provider(monkeypatch):
|
||||
"""Publishing records the fan-out carrier in module state; one test's publish must
|
||||
not become the next test's provider."""
|
||||
monkeypatch.setattr(otel_logger, "_published_v2_provider", None)
|
||||
|
||||
|
||||
def in_fresh_context(fn, *args):
|
||||
"""Run ``fn`` in its own context so one test's destinations never leak."""
|
||||
return contextvars.copy_context().run(fn, *args)
|
||||
|
|
@ -663,12 +672,11 @@ class TestProviderWiring:
|
|||
kinds = [type(p).__name__ for p in logger._tracer_provider._active_span_processor._span_processors]
|
||||
assert kinds.count("TenantFanOutSpanProcessor") == 1
|
||||
|
||||
def test_anchoring_reads_the_fan_out_off_the_registered_logger_not_the_otel_global(self, monkeypatch):
|
||||
def test_anchoring_reads_the_fan_out_off_the_published_provider_not_the_otel_global(self, monkeypatch):
|
||||
"""``set_tracer_provider`` keeps the first provider it was handed. When
|
||||
auto-instrumentation or a legacy logger claimed it before the proxy published,
|
||||
the OTel global carries no fan-out, so reading it there would refuse every
|
||||
destination while the registered logger's provider would have delivered them."""
|
||||
from litellm.integrations.otel.logger import fan_out_provider
|
||||
destination the published provider delivers."""
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
config = OpenTelemetryV2Config(exporters=[ExporterSpec(kind="in_memory", owner=ExporterOwner.LANGFUSE_OTEL)])
|
||||
|
|
@ -681,10 +689,26 @@ class TestProviderWiring:
|
|||
assert deliverable_destinations((LANGFUSE_DEST,), claimed_first) == ()
|
||||
assert deliverable_destinations((LANGFUSE_DEST,), fan_out_provider()) == (LANGFUSE_DEST,)
|
||||
|
||||
def test_without_a_registered_logger_anchoring_falls_back_to_the_otel_global(self, monkeypatch):
|
||||
def test_a_legacy_v1_logger_holding_the_registered_slot_does_not_hide_the_fan_out(self, monkeypatch):
|
||||
"""The proxy publishes with ``registered=None`` when ``open_telemetry_logger``
|
||||
holds a v1 logger, so the fan-out lands on a v2 logger taken from
|
||||
``_in_memory_loggers``. Reading the registered slot finds no v2 logger there and
|
||||
the OTel global belongs to v1, so both detours refuse every destination the
|
||||
published provider delivers."""
|
||||
from litellm.integrations.opentelemetry import OpenTelemetry
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
config = OpenTelemetryV2Config(exporters=[ExporterSpec(kind="in_memory", owner=ExporterOwner.LANGFUSE_OTEL)])
|
||||
v2 = OpenTelemetryV2(config=config, callback_name="langfuse_otel")
|
||||
publish_global_otel_v2_provider([v2], lambda _p: None, registered=None)
|
||||
monkeypatch.setattr(proxy_server, "open_telemetry_logger", OpenTelemetry())
|
||||
|
||||
assert fan_out_provider() is v2.tracer_provider
|
||||
assert deliverable_destinations((LANGFUSE_DEST,), fan_out_provider()) == (LANGFUSE_DEST,)
|
||||
|
||||
def test_without_a_publish_anchoring_falls_back_to_the_otel_global(self, monkeypatch):
|
||||
from opentelemetry import trace
|
||||
|
||||
from litellm.integrations.otel.logger import fan_out_provider
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
monkeypatch.setattr(proxy_server, "open_telemetry_logger", None)
|
||||
|
|
|
|||
|
|
@ -2041,7 +2041,7 @@ def test_select_global_otel_v2_logger_builds_one_when_none_registered():
|
|||
assert isinstance(chosen, OpenTelemetryV2)
|
||||
|
||||
|
||||
def test_publish_global_otel_v2_provider_sets_selected_logger_provider():
|
||||
def test_publish_global_otel_v2_provider_sets_selected_logger_provider(monkeypatch):
|
||||
"""The startup publish must set the OTel global provider to the *selected*
|
||||
logger's provider (the preset logger that owns every exporter), so the FastAPI
|
||||
server span and the gen-ai spans share one provider and one trace.
|
||||
|
|
@ -2051,8 +2051,10 @@ def test_publish_global_otel_v2_provider_sets_selected_logger_provider():
|
|||
test would otherwise miss: that the published provider is the selected logger's,
|
||||
not some other.
|
||||
"""
|
||||
from litellm.integrations.otel import logger as otel_logger
|
||||
from litellm.integrations.otel.logger import publish_global_otel_v2_provider
|
||||
|
||||
monkeypatch.setattr(otel_logger, "_published_v2_provider", None)
|
||||
cfg = OpenTelemetryV2Config(exporter="in_memory")
|
||||
tp = providers.build_tracer_provider(cfg)
|
||||
preset_logger = OpenTelemetryV2(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue