mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-10 03:28:53 +00:00
fix(otel): keep a degraded backend's spans off a collector another v2 logger already serves
This commit is contained in:
parent
9f871f6895
commit
22217d03ea
2 changed files with 62 additions and 3 deletions
|
|
@ -200,7 +200,7 @@ if TYPE_CHECKING:
|
|||
from mcp.types import EmbeddedResource, ImageContent, TextContent
|
||||
|
||||
from litellm.integrations.otel.logger import OpenTelemetryV2
|
||||
from litellm.integrations.otel.model.config import OpenTelemetryV2Config
|
||||
from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config
|
||||
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
|
||||
try:
|
||||
from litellm_enterprise.enterprise_callbacks.callback_controls import (
|
||||
|
|
@ -4809,6 +4809,11 @@ def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[Custom
|
|||
as it always did and the caller falls through to the legacy path, so the proxy
|
||||
never publishes a provider that exports nowhere for a backend the operator
|
||||
configured and no tenant can use.
|
||||
|
||||
The degraded preset keeps the operator's generic OTLP collector, so a proxy whose
|
||||
only v2 backend is that preset still reaches it. Beside another v2 logger the
|
||||
collector is already that logger's, and a second copy of every model span from
|
||||
this one would land there too, so only the credential-gated exporter is kept.
|
||||
"""
|
||||
from litellm.integrations.otel.model.config import is_otel_v2_enabled
|
||||
|
||||
|
|
@ -4830,11 +4835,16 @@ def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[Custom
|
|||
):
|
||||
return callback
|
||||
try:
|
||||
config: Final = preset_fn(allow_missing_credentials=serves_a_destination)
|
||||
built: Final = preset_fn(allow_missing_credentials=serves_a_destination)
|
||||
except Exception:
|
||||
# If env vars are missing or the preset raises, defer to the legacy path
|
||||
# so customers get the same error story they had before V2 landed.
|
||||
return None
|
||||
config: Final = (
|
||||
_only_the_gated_exporter(built)
|
||||
if _is_credential_gated(built) and any(isinstance(callback, OpenTelemetryV2) for callback in _in_memory_loggers)
|
||||
else built
|
||||
)
|
||||
if _exports_nowhere(config):
|
||||
verbose_logger.warning(
|
||||
"OTel V2: no operator credentials for '%s'; only key/team destinations will receive its traces",
|
||||
|
|
@ -4847,7 +4857,22 @@ def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[Custom
|
|||
|
||||
def _exports_nowhere(config: "OpenTelemetryV2Config") -> bool:
|
||||
"""Whether every exporter in ``config`` is waiting on credentials it never got."""
|
||||
return all(spec.requires_headers and not spec.headers for spec in config.exporters)
|
||||
return all(_is_gated(spec) for spec in config.exporters)
|
||||
|
||||
|
||||
def _is_credential_gated(config: "OpenTelemetryV2Config") -> bool:
|
||||
"""Whether the preset built without the operator's own credentials for its backend."""
|
||||
return any(_is_gated(spec) for spec in config.exporters)
|
||||
|
||||
|
||||
def _only_the_gated_exporter(config: "OpenTelemetryV2Config") -> "OpenTelemetryV2Config":
|
||||
return config.model_copy(
|
||||
update={"exporters": [spec for spec in config.exporters if _is_gated(spec)]} # mutable-ok: model_copy update
|
||||
)
|
||||
|
||||
|
||||
def _is_gated(spec: "ExporterSpec") -> bool:
|
||||
return spec.requires_headers and not spec.headers
|
||||
|
||||
|
||||
def _maybe_auto_initialize_arize_phoenix(_in_memory_loggers: list[CustomLogger]) -> None:
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ import litellm
|
|||
from litellm.integrations.otel import logger as otel_logger
|
||||
from litellm.integrations.otel.logger import (
|
||||
OpenTelemetryV2,
|
||||
build_otel_v2_logger,
|
||||
fan_out_provider,
|
||||
publish_global_otel_v2_provider,
|
||||
)
|
||||
|
|
@ -1208,6 +1209,39 @@ class TestPresetDegradation:
|
|||
assert first is not None
|
||||
assert second is first
|
||||
|
||||
@staticmethod
|
||||
def _degraded_langfuse_beside(loggers, monkeypatch):
|
||||
from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2
|
||||
|
||||
credential_less_proxy(monkeypatch)
|
||||
monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.local:4318")
|
||||
monkeypatch.setenv("LITELLM_OTEL_V2", "true")
|
||||
|
||||
def run():
|
||||
set_request_destinations((LANGFUSE_DEST,))
|
||||
return _maybe_construct_otel_v2("langfuse_otel", loggers)
|
||||
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
logger = in_fresh_context(run)
|
||||
is_otel_v2_enabled.cache_clear()
|
||||
assert logger is not None
|
||||
return logger
|
||||
|
||||
def test_a_degraded_logger_beside_another_v2_logger_leaves_the_collector_to_it(self, monkeypatch):
|
||||
"""The other logger's provider already exports every span to the operator's
|
||||
collector, so a second model span from this one would land there twice."""
|
||||
collector_logger = build_otel_v2_logger(OpenTelemetryV2Config(exporter="in_memory"))
|
||||
|
||||
logger = self._degraded_langfuse_beside([collector_logger], monkeypatch)
|
||||
|
||||
assert [spec.endpoint for spec in logger.config.exporters] == [None]
|
||||
assert all(spec.requires_headers and not spec.headers for spec in logger.config.exporters)
|
||||
|
||||
def test_a_degraded_logger_on_its_own_keeps_the_operator_collector(self, monkeypatch):
|
||||
logger = self._degraded_langfuse_beside([], monkeypatch)
|
||||
|
||||
assert [spec.endpoint for spec in logger.config.exporters] == ["http://collector.local:4318", None]
|
||||
|
||||
|
||||
class TestContextIsolation:
|
||||
def test_destinations_do_not_leak_between_requests(self):
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue