fix(otel): build the otel logger after preset callbacks and validate the exclusion env at boot

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

litellm/integrations/otel/model/config.py

@@ -1,5 +1,6 @@
 """Typed configuration for the OpenTelemetry instrumentation."""
 
+import os
 from collections.abc import Mapping
 from enum import Enum
 from functools import lru_cache
@@ -369,6 +370,22 @@ def _db_system_for_excluded_service(service: str) -> str:
     return resolved
 
 
+def validate_otel_v2_excluded_services_env() -> None:
+    """Validate ``LITELLM_OTEL_EXCLUDED_SERVICES`` at boot even with no ``otel`` callback.
+
+    Preset-only deployments build env-only configs through a path that swallows
+    init errors, so a bogus value would otherwise degrade to the legacy callback
+    silently. Splitting and normalizing here raises the same ``ValueError`` the
+    field raises.
+    """
+    if not is_otel_v2_enabled():
+        return
+    raw: Final = os.environ.get("LITELLM_OTEL_EXCLUDED_SERVICES")
+    if not raw:
+        return
+    _normalize_excluded_services(frozenset(item.strip() for item in raw.split(",") if item.strip()))
+
+
 def validate_otel_v2_callback_settings(settings: object) -> None:
     """Parse ``callback_settings.otel`` so a malformed block fails proxy boot.
 

litellm/litellm_core_utils/litellm_logging.py

@@ -4764,7 +4764,7 @@ def _init_custom_logger_compatible_class(
                 from litellm.integrations.otel.model.config import OpenTelemetryV2Config
 
                 for callback in _in_memory_loggers:
-                    if isinstance(callback, OpenTelemetryV2):
+                    if isinstance(callback, OpenTelemetryV2) and callback.callback_name == "otel":
                         return callback
                 otel_settings: Final = _get_custom_logger_settings_from_proxy_server(callback_name=logging_integration)
                 otel_logger_v2: Final = build_otel_v2_logger(

litellm/proxy/common_utils/callback_utils.py

@@ -179,6 +179,10 @@ def initialize_callbacks_on_proxy(
 
                 validate_otel_v2_callback_settings(callback_specific_params.get("otel"))
 
+            from litellm.integrations.otel.model.config import validate_otel_v2_excluded_services_env
+
+            validate_otel_v2_excluded_services_env()
+
             # check if callback is a custom logger compatible callback
             if isinstance(callback, str):
                 callback = LoggingCallbackManager._add_custom_callback_generic_api_str(callback)

tests/integration/observability/test_otel_excluded_services.py

@@ -245,6 +245,33 @@ def test_config_excluded_services_wins_over_env(
         )
 
 
+def test_excluded_services_applies_with_preset_ordered_first(
+    gateway: Gateway,
+    audit_sinks: SpanSinks,
+    otel_audit_config: AuditConfigWriter,
+    langfuse_vars: dict[str, JsonValue],
+    tmp_path: Path,
+) -> None:
+    def preset_first(config: dict) -> None:
+        config["litellm_settings"]["callbacks"] = ["langfuse_otel", "otel"]
+
+    config: Final = _config_with(
+        tmp_path, otel_audit_config, otel={"excluded_services": ["postgres"]}, extra=preset_first
+    )
+    with owned_proxy(gateway, tmp_path, {"LITELLM_OTEL_V2": "1"}, config=config, workers=2) as candidate:
+        start, _ = recorded_spans(audit_sinks.tenant)
+        traffic: Final = _drive(candidate, langfuse_vars)
+        tenant_trace: Final = _trace_id(audit_sinks.tenant, traffic)
+        _await_db_span(audit_sinks.tenant, tenant_trace, "redis", seconds=60)
+        tenant_spans: Final = _trace_spans(audit_sinks.tenant, tenant_trace, seconds=15)
+        _, all_tenant = recorded_spans(audit_sinks.tenant, start)
+        systems: Final = _db_systems(tenant_spans)
+        assert "redis" in systems, f"redis spans missing at tenant: {systems}"
+        assert "postgresql" not in _db_systems(all_tenant), (
+            f"postgresql spans reached tenant: {_db_systems(all_tenant)}"
+        )
+
+
 def test_bogus_excluded_service_fails_proxy_start(
     gateway: Gateway, otel_audit_config: AuditConfigWriter, tmp_path: Path
 ) -> None:
@@ -261,6 +288,26 @@ def test_bogus_excluded_service_fails_proxy_start(
     assert "postgres, redis" in text, text[-3000:]
 
 
+def test_bogus_excluded_services_env_fails_proxy_start_without_otel_callback(
+    gateway: Gateway, otel_audit_config: AuditConfigWriter, tmp_path: Path
+) -> None:
+    def presets_only(config: dict) -> None:
+        config["litellm_settings"]["callbacks"] = ["langfuse_otel"]
+
+    config: Final = _config_with(tmp_path, otel_audit_config, extra=presets_only)
+    log_dir: Final = Path(os.environ.get("INTEGRATION_RESULTS_DIR", str(tmp_path)))
+    before: Final = frozenset(log_dir.glob("owned-proxy-*.log"))
+    with pytest.raises(AssertionError, match="readiness"):
+        with owned_proxy_process(
+            gateway, tmp_path, {"LITELLM_OTEL_V2": "1", "LITELLM_OTEL_EXCLUDED_SERVICES": "auth"}, config=config, workers=2
+        ):
+            pass
+    logs: Final = [path.read_text() for path in frozenset(log_dir.glob("owned-proxy-*.log")) - before]
+    assert logs, "no owned proxy log written"
+    text: Final = "\n".join(logs)
+    assert "'auth' is not a datastore service" in text, text[-3000:]
+
+
 def test_postgres_exclusion_covers_batch_write_to_db(
     gateway: Gateway,
     audit_sinks: SpanSinks,

tests/unit/integrations/otel/test_otel_v2_config_baggage_parenting_guardrails.py

@@ -120,6 +120,31 @@ def test_excluded_services_rejects_a_non_datastore_service():
         OpenTelemetryV2Config(excluded_services=["auth"])
 
 
+def test_excluded_services_env_is_validated_at_boot_when_enabled(monkeypatch):
+    from litellm.integrations.otel.model.config import is_otel_v2_enabled, validate_otel_v2_excluded_services_env
+
+    monkeypatch.setenv("LITELLM_OTEL_V2", "1")
+    monkeypatch.setenv("LITELLM_OTEL_EXCLUDED_SERVICES", "auth")
+    is_otel_v2_enabled.cache_clear()
+    try:
+        with pytest.raises(ValueError, match="'auth' is not a datastore service; allowed: postgres, redis"):
+            validate_otel_v2_excluded_services_env()
+    finally:
+        is_otel_v2_enabled.cache_clear()
+
+
+def test_excluded_services_env_validation_accepts_datastore_names(monkeypatch):
+    from litellm.integrations.otel.model.config import is_otel_v2_enabled, validate_otel_v2_excluded_services_env
+
+    monkeypatch.setenv("LITELLM_OTEL_V2", "1")
+    monkeypatch.setenv("LITELLM_OTEL_EXCLUDED_SERVICES", "redis, postgres")
+    is_otel_v2_enabled.cache_clear()
+    try:
+        validate_otel_v2_excluded_services_env()
+    finally:
+        is_otel_v2_enabled.cache_clear()
+
+
 # --------------------------------------------------------------------------- #
 #  Area 2 — pass-through LLM span parents to the ambient server span
 # --------------------------------------------------------------------------- #

tests/unit/integrations/otel/test_otel_v2_destinations.py

@@ -1106,6 +1106,44 @@ class TestProviderWiring:
         )
         assert fan_out._excluded_db_systems == frozenset({"redis"})
 
+    def test_otel_callback_builds_its_own_logger_after_a_preset(self, monkeypatch):
+        """With ``callbacks: [langfuse_otel, otel]`` the otel branch reused any
+        V2 logger, so ``callback_settings.otel`` (excluded_services) was dropped
+        onto the preset's env-only config."""
+        from litellm.integrations.otel.logger import _excluded_db_systems
+        from litellm.litellm_core_utils import litellm_logging as logging_module
+
+        logging_module._in_memory_loggers.clear()
+        monkeypatch.setenv("LITELLM_OTEL_V2", "true")
+        monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk")
+        monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk")
+        monkeypatch.delenv("LITELLM_OTEL_EXCLUDED_SERVICES", raising=False)
+        is_otel_v2_enabled.cache_clear()
+        monkeypatch.setattr(litellm, "callback_settings", {"otel": {"excluded_services": ["postgres"]}}, raising=False)
+        try:
+
+            def init(name: str) -> CustomLogger | None:
+                return logging_module._init_custom_logger_compatible_class(
+                    logging_integration=name,  # type: ignore[arg-type]  # test passes a literal callback name
+                    internal_usage_cache=None,
+                    llm_router=None,
+                    custom_logger_init_args={},
+                )
+
+            preset = init("langfuse_otel")
+            otel_cb = init("otel")
+
+            assert otel_cb is not None and otel_cb is not preset
+            v2_names = {
+                cb.callback_name for cb in logging_module._in_memory_loggers if isinstance(cb, OpenTelemetryV2)
+            }
+            assert {"langfuse_otel", "otel"} <= v2_names, v2_names
+            resolved = _excluded_db_systems(logging_module._in_memory_loggers, otel_cb)
+            assert resolved == frozenset({"postgresql"}), resolved
+        finally:
+            logging_module._in_memory_loggers.clear()
+            is_otel_v2_enabled.cache_clear()
+
     @pytest.mark.parametrize("canonical", ["langfuse_otel", "arize"])
     def test_publishing_tells_the_fan_out_about_every_v2_loggers_account(self, monkeypatch, canonical):
         monkeypatch.setenv("LITELLM_OTEL_TENANT_DESTINATION_MODE", "additive")