mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-10 08:28:25 +00:00
fix(otel)!: keep a credential-less langfuse_otel on OTel v2 so key destinations get the Langfuse mapper
With LITELLM_OTEL_V2 on, langfuse_otel in callbacks and no operator LANGFUSE_PUBLIC_KEY / LANGFUSE_SECRET_KEY, the preset raised at startup and the callback fell back to the legacy logger. Key and team destinations were then served by a generic v2 logger without the Langfuse mapper, so their traces lost tags, user and cost, and the legacy logger added an orphaned raw_gen_ai_request span. The preset now always degrades to the credential-gated exporter when the operator has no keys, the same way arize_preset does, so the startup logger stays on v2, exports nowhere for traffic without a destination, and carries the Langfuse mapper to every key and team destination.
This commit is contained in:
parent
c7c895db45
commit
1c46a69889
3 changed files with 100 additions and 24 deletions
|
|
@ -19,12 +19,13 @@ class Preset(Protocol):
|
|||
``config_overrides`` lets one preset layer onto another's config (or onto
|
||||
test-supplied defaults); the factory calls presets with no arguments.
|
||||
|
||||
``allow_missing_credentials`` lets a credential-mandatory backend (langfuse and
|
||||
weave) degrade to an exporter-less, mapper-only config instead of raising when the
|
||||
operator set no env credentials of their own. That is a real
|
||||
``allow_missing_credentials`` lets a credential-mandatory backend (weave) degrade
|
||||
to an exporter-less, mapper-only config instead of raising when the operator set
|
||||
no env credentials of their own. That is a real
|
||||
deployment: every team brings its own account and the operator keeps none, and
|
||||
without it the whole V2 path silently falls back to the legacy integration, so
|
||||
no team destination is ever reached. Credential-optional backends ignore it.
|
||||
no team destination is ever reached. Backends that always degrade without
|
||||
operator credentials (arize, langfuse) and credential-optional ones ignore it.
|
||||
"""
|
||||
|
||||
def __call__(
|
||||
|
|
|
|||
|
|
@ -26,9 +26,7 @@ def langfuse_preset(
|
|||
mappers: Final = ensure_mappers(base.mapper_names, "langfuse")
|
||||
try:
|
||||
cfg: Final = _V1Langfuse.get_langfuse_otel_config()
|
||||
except Exception:
|
||||
if not allow_missing_credentials:
|
||||
raise
|
||||
except ValueError:
|
||||
return base.model_copy(
|
||||
update={
|
||||
"exporters": credential_gated_exporters(base.exporters, ExporterOwner.LANGFUSE_OTEL),
|
||||
|
|
|
|||
|
|
@ -2945,8 +2945,20 @@ def credential_less_proxy(monkeypatch) -> None:
|
|||
"""An operator with no Langfuse account and no generic OTLP collector."""
|
||||
for name in ("LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY", *_OTEL_SHORTHAND_ENV):
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
with pytest.raises(ValueError, match="LANGFUSE_PUBLIC_KEY"):
|
||||
langfuse_preset()
|
||||
assert all(spec.requires_headers and not spec.headers for spec in langfuse_preset().exporters)
|
||||
|
||||
|
||||
def credential_less_weave(monkeypatch) -> None:
|
||||
"""An operator with no Weave account and no generic OTLP collector."""
|
||||
for name in ("WANDB_API_KEY", "WANDB_PROJECT_ID", *_OTEL_SHORTHAND_ENV):
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
|
||||
|
||||
WEAVE_DEST = OtelDestination(
|
||||
endpoint="https://trace.wandb.ai/otel/v1/traces",
|
||||
headers={"Authorization": "Basic dGVuYW50"},
|
||||
callback_name="weave_otel",
|
||||
)
|
||||
|
||||
|
||||
def _closed_chat_call_kwargs() -> dict[str, object]:
|
||||
|
|
@ -2986,12 +2998,14 @@ class TestPresetDegradation:
|
|||
assert '"name": "chat gpt-4"' not in capfd.readouterr().out
|
||||
assert "langfuse" in config.mapper_names
|
||||
|
||||
def test_langfuse_still_raises_for_a_global_callback_with_no_credentials(self, monkeypatch):
|
||||
def test_a_global_langfuse_callback_with_no_credentials_degrades_instead_of_raising(self, monkeypatch):
|
||||
monkeypatch.delenv("LANGFUSE_PUBLIC_KEY", raising=False)
|
||||
monkeypatch.delenv("LANGFUSE_SECRET_KEY", raising=False)
|
||||
|
||||
with pytest.raises(ValueError, match="LANGFUSE_PUBLIC_KEY"):
|
||||
langfuse_preset()
|
||||
config = langfuse_preset()
|
||||
|
||||
assert any(spec.owner == ExporterOwner.LANGFUSE_OTEL and spec.requires_headers for spec in config.exporters)
|
||||
assert "langfuse" in config.mapper_names
|
||||
|
||||
def test_a_credential_less_proxy_builds_the_gated_logger_beside_a_v2_carrier(self, monkeypatch):
|
||||
from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2
|
||||
|
|
@ -3011,10 +3025,9 @@ class TestPresetDegradation:
|
|||
assert logger is not None
|
||||
assert all(spec.requires_headers and not spec.headers for spec in logger.config.exporters)
|
||||
|
||||
def test_a_credential_less_proxy_with_no_destinations_falls_back_to_the_legacy_path(self, monkeypatch):
|
||||
"""Nothing can use a credential-less langfuse here, so the operator has to get
|
||||
the same story as before v2: the legacy integration, not a global provider
|
||||
that exports nowhere."""
|
||||
def test_a_credential_less_proxy_with_no_destinations_stays_on_v2_and_exports_nowhere(self, monkeypatch):
|
||||
"""The proxy builds the operator's callback at startup, before any key or team
|
||||
destination has resolved, so this is the logger those destinations ride on."""
|
||||
from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2
|
||||
|
||||
credential_less_proxy(monkeypatch)
|
||||
|
|
@ -3024,7 +3037,9 @@ class TestPresetDegradation:
|
|||
logger = in_fresh_context(_maybe_construct_otel_v2, "langfuse_otel", [])
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
|
||||
assert logger is None
|
||||
assert isinstance(logger, OpenTelemetryV2), "None hands 'langfuse_otel' to the legacy logger"
|
||||
assert "langfuse" in logger.config.mapper_names
|
||||
assert all(spec.requires_headers and not spec.headers for spec in logger.config.exporters)
|
||||
|
||||
def test_a_valid_newrelic_base_exporter_survives_without_a_license_key(self, monkeypatch):
|
||||
from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2
|
||||
|
|
@ -3099,17 +3114,17 @@ class TestPresetDegradation:
|
|||
without one is logged, long after the degrade was justified."""
|
||||
from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2
|
||||
|
||||
credential_less_proxy(monkeypatch)
|
||||
credential_less_weave(monkeypatch)
|
||||
monkeypatch.setenv("LITELLM_OTEL_V2", "true")
|
||||
loggers = [build_otel_v2_logger(OpenTelemetryV2Config(exporter="in_memory"))]
|
||||
|
||||
def with_destination():
|
||||
set_request_destinations((LANGFUSE_DEST,))
|
||||
return _maybe_construct_otel_v2("langfuse_otel", loggers)
|
||||
set_request_destinations((WEAVE_DEST,))
|
||||
return _maybe_construct_otel_v2("weave_otel", loggers)
|
||||
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
degraded = in_fresh_context(with_destination)
|
||||
plain = in_fresh_context(_maybe_construct_otel_v2, "langfuse_otel", loggers)
|
||||
plain = in_fresh_context(_maybe_construct_otel_v2, "weave_otel", loggers)
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
|
||||
assert degraded is not None
|
||||
|
|
@ -3166,12 +3181,12 @@ class TestPresetDegradation:
|
|||
callback at all, and the operator keeps the pre-V2 story."""
|
||||
from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2
|
||||
|
||||
credential_less_proxy(monkeypatch)
|
||||
credential_less_weave(monkeypatch)
|
||||
monkeypatch.setenv("LITELLM_OTEL_V2", "true")
|
||||
|
||||
def run():
|
||||
set_request_destinations((LANGFUSE_DEST,))
|
||||
return _maybe_construct_otel_v2("langfuse_otel", list(registered))
|
||||
set_request_destinations((WEAVE_DEST,))
|
||||
return _maybe_construct_otel_v2("weave_otel", list(registered))
|
||||
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
logger = in_fresh_context(run)
|
||||
|
|
@ -3435,6 +3450,68 @@ class TestArizeTenantOnly:
|
|||
assert loggers == [at_startup]
|
||||
|
||||
|
||||
class TestLangfuseTenantOnly:
|
||||
"""An operator whose keys and teams each bring their own Langfuse project keeps no
|
||||
Langfuse credentials of their own; the startup logger then exports nowhere for
|
||||
traffic without a destination and carries the Langfuse mapper to the destinations."""
|
||||
|
||||
@staticmethod
|
||||
def _startup_logger(monkeypatch, loggers: list) -> OpenTelemetryV2:
|
||||
from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2
|
||||
|
||||
credential_less_proxy(monkeypatch)
|
||||
monkeypatch.setenv("LITELLM_OTEL_V2", "true")
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
logger = in_fresh_context(_maybe_construct_otel_v2, "langfuse_otel", loggers)
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
assert isinstance(logger, OpenTelemetryV2)
|
||||
return logger
|
||||
|
||||
def test_the_published_provider_is_the_langfuse_logger(self, monkeypatch):
|
||||
"""The published provider carries the key/team fan-out, so its mappers are the
|
||||
attributes every destination receives."""
|
||||
loggers = []
|
||||
logger = self._startup_logger(monkeypatch, loggers)
|
||||
|
||||
published = publish_global_otel_v2_provider(loggers, lambda _p: None)
|
||||
|
||||
assert published is logger
|
||||
assert "langfuse" in published.config.mapper_names
|
||||
|
||||
def test_the_startup_logger_delivers_a_key_destination_and_nothing_else(self, monkeypatch):
|
||||
from litellm.integrations.otel.plumbing.providers import attach_tenant_fan_out
|
||||
|
||||
capture = _ExporterCapture()
|
||||
for kind in ("otlp_grpc", "otlp_http", "console"):
|
||||
monkeypatch.setitem(otel_providers._EXPORTER_FACTORIES, kind, capture.build)
|
||||
logger = self._startup_logger(monkeypatch, [])
|
||||
provider = logger.tracer_provider
|
||||
attach_tenant_fan_out(provider, logger.config)
|
||||
|
||||
def with_destination():
|
||||
set_request_destinations(deliverable_destinations((LANGFUSE_DEST,), provider))
|
||||
emit(provider)
|
||||
|
||||
in_fresh_context(with_destination)
|
||||
in_fresh_context(emit, provider, "chat without a destination")
|
||||
provider.force_flush()
|
||||
|
||||
assert capture.received() == {LANGFUSE_DEST.header_string(): ("chat gpt-4",)}
|
||||
|
||||
def test_the_startup_logger_is_reused_by_later_requests_without_a_destination(self, monkeypatch):
|
||||
from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2
|
||||
|
||||
loggers = []
|
||||
at_startup = self._startup_logger(monkeypatch, loggers)
|
||||
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
later = in_fresh_context(_maybe_construct_otel_v2, "langfuse_otel", loggers)
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
|
||||
assert later is at_startup
|
||||
assert loggers == [at_startup]
|
||||
|
||||
|
||||
class TestContextIsolation:
|
||||
def test_destinations_do_not_leak_between_requests(self):
|
||||
def first():
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue