diff --git a/litellm/integrations/otel/model/config.py b/litellm/integrations/otel/model/config.py index ee169f968b5..5b7b9e5d352 100644 --- a/litellm/integrations/otel/model/config.py +++ b/litellm/integrations/otel/model/config.py @@ -1,5 +1,6 @@ """Typed configuration for the OpenTelemetry instrumentation.""" +import os from collections.abc import Mapping from enum import Enum from functools import lru_cache @@ -369,6 +370,22 @@ def _db_system_for_excluded_service(service: str) -> str: return resolved +def validate_otel_v2_excluded_services_env() -> None: + """Validate ``LITELLM_OTEL_EXCLUDED_SERVICES`` at boot even with no ``otel`` callback. + + Preset-only deployments build env-only configs through a path that swallows + init errors, so a bogus value would otherwise degrade to the legacy callback + silently. Splitting and normalizing here raises the same ``ValueError`` the + field raises. + """ + if not is_otel_v2_enabled(): + return + raw: Final = os.environ.get("LITELLM_OTEL_EXCLUDED_SERVICES") + if not raw: + return + _normalize_excluded_services(frozenset(item.strip() for item in raw.split(",") if item.strip())) + + def validate_otel_v2_callback_settings(settings: object) -> None: """Parse ``callback_settings.otel`` so a malformed block fails proxy boot. diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 6815323725e..e3d7eefb4bb 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -4764,7 +4764,7 @@ def _init_custom_logger_compatible_class( from litellm.integrations.otel.model.config import OpenTelemetryV2Config for callback in _in_memory_loggers: - if isinstance(callback, OpenTelemetryV2): + if isinstance(callback, OpenTelemetryV2) and callback.callback_name == "otel": return callback otel_settings: Final = _get_custom_logger_settings_from_proxy_server(callback_name=logging_integration) otel_logger_v2: Final = build_otel_v2_logger( diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index cbe5dca2edc..97d608884b9 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -179,6 +179,10 @@ def initialize_callbacks_on_proxy( validate_otel_v2_callback_settings(callback_specific_params.get("otel")) + from litellm.integrations.otel.model.config import validate_otel_v2_excluded_services_env + + validate_otel_v2_excluded_services_env() + # check if callback is a custom logger compatible callback if isinstance(callback, str): callback = LoggingCallbackManager._add_custom_callback_generic_api_str(callback) diff --git a/tests/integration/observability/test_otel_excluded_services.py b/tests/integration/observability/test_otel_excluded_services.py index 30e4ee64531..df71dbc3fa4 100644 --- a/tests/integration/observability/test_otel_excluded_services.py +++ b/tests/integration/observability/test_otel_excluded_services.py @@ -245,6 +245,33 @@ def test_config_excluded_services_wins_over_env( ) +def test_excluded_services_applies_with_preset_ordered_first( + gateway: Gateway, + audit_sinks: SpanSinks, + otel_audit_config: AuditConfigWriter, + langfuse_vars: dict[str, JsonValue], + tmp_path: Path, +) -> None: + def preset_first(config: dict) -> None: + config["litellm_settings"]["callbacks"] = ["langfuse_otel", "otel"] + + config: Final = _config_with( + tmp_path, otel_audit_config, otel={"excluded_services": ["postgres"]}, extra=preset_first + ) + with owned_proxy(gateway, tmp_path, {"LITELLM_OTEL_V2": "1"}, config=config, workers=2) as candidate: + start, _ = recorded_spans(audit_sinks.tenant) + traffic: Final = _drive(candidate, langfuse_vars) + tenant_trace: Final = _trace_id(audit_sinks.tenant, traffic) + _await_db_span(audit_sinks.tenant, tenant_trace, "redis", seconds=60) + tenant_spans: Final = _trace_spans(audit_sinks.tenant, tenant_trace, seconds=15) + _, all_tenant = recorded_spans(audit_sinks.tenant, start) + systems: Final = _db_systems(tenant_spans) + assert "redis" in systems, f"redis spans missing at tenant: {systems}" + assert "postgresql" not in _db_systems(all_tenant), ( + f"postgresql spans reached tenant: {_db_systems(all_tenant)}" + ) + + def test_bogus_excluded_service_fails_proxy_start( gateway: Gateway, otel_audit_config: AuditConfigWriter, tmp_path: Path ) -> None: @@ -261,6 +288,26 @@ def test_bogus_excluded_service_fails_proxy_start( assert "postgres, redis" in text, text[-3000:] +def test_bogus_excluded_services_env_fails_proxy_start_without_otel_callback( + gateway: Gateway, otel_audit_config: AuditConfigWriter, tmp_path: Path +) -> None: + def presets_only(config: dict) -> None: + config["litellm_settings"]["callbacks"] = ["langfuse_otel"] + + config: Final = _config_with(tmp_path, otel_audit_config, extra=presets_only) + log_dir: Final = Path(os.environ.get("INTEGRATION_RESULTS_DIR", str(tmp_path))) + before: Final = frozenset(log_dir.glob("owned-proxy-*.log")) + with pytest.raises(AssertionError, match="readiness"): + with owned_proxy_process( + gateway, tmp_path, {"LITELLM_OTEL_V2": "1", "LITELLM_OTEL_EXCLUDED_SERVICES": "auth"}, config=config, workers=2 + ): + pass + logs: Final = [path.read_text() for path in frozenset(log_dir.glob("owned-proxy-*.log")) - before] + assert logs, "no owned proxy log written" + text: Final = "\n".join(logs) + assert "'auth' is not a datastore service" in text, text[-3000:] + + def test_postgres_exclusion_covers_batch_write_to_db( gateway: Gateway, audit_sinks: SpanSinks, diff --git a/tests/unit/integrations/otel/test_otel_v2_config_baggage_parenting_guardrails.py b/tests/unit/integrations/otel/test_otel_v2_config_baggage_parenting_guardrails.py index 2184db533a5..117989e8326 100644 --- a/tests/unit/integrations/otel/test_otel_v2_config_baggage_parenting_guardrails.py +++ b/tests/unit/integrations/otel/test_otel_v2_config_baggage_parenting_guardrails.py @@ -120,6 +120,31 @@ def test_excluded_services_rejects_a_non_datastore_service(): OpenTelemetryV2Config(excluded_services=["auth"]) +def test_excluded_services_env_is_validated_at_boot_when_enabled(monkeypatch): + from litellm.integrations.otel.model.config import is_otel_v2_enabled, validate_otel_v2_excluded_services_env + + monkeypatch.setenv("LITELLM_OTEL_V2", "1") + monkeypatch.setenv("LITELLM_OTEL_EXCLUDED_SERVICES", "auth") + is_otel_v2_enabled.cache_clear() + try: + with pytest.raises(ValueError, match="'auth' is not a datastore service; allowed: postgres, redis"): + validate_otel_v2_excluded_services_env() + finally: + is_otel_v2_enabled.cache_clear() + + +def test_excluded_services_env_validation_accepts_datastore_names(monkeypatch): + from litellm.integrations.otel.model.config import is_otel_v2_enabled, validate_otel_v2_excluded_services_env + + monkeypatch.setenv("LITELLM_OTEL_V2", "1") + monkeypatch.setenv("LITELLM_OTEL_EXCLUDED_SERVICES", "redis, postgres") + is_otel_v2_enabled.cache_clear() + try: + validate_otel_v2_excluded_services_env() + finally: + is_otel_v2_enabled.cache_clear() + + # --------------------------------------------------------------------------- # # Area 2 — pass-through LLM span parents to the ambient server span # --------------------------------------------------------------------------- # diff --git a/tests/unit/integrations/otel/test_otel_v2_destinations.py b/tests/unit/integrations/otel/test_otel_v2_destinations.py index eee7d24a475..104cee5166c 100644 --- a/tests/unit/integrations/otel/test_otel_v2_destinations.py +++ b/tests/unit/integrations/otel/test_otel_v2_destinations.py @@ -1106,6 +1106,44 @@ class TestProviderWiring: ) assert fan_out._excluded_db_systems == frozenset({"redis"}) + def test_otel_callback_builds_its_own_logger_after_a_preset(self, monkeypatch): + """With ``callbacks: [langfuse_otel, otel]`` the otel branch reused any + V2 logger, so ``callback_settings.otel`` (excluded_services) was dropped + onto the preset's env-only config.""" + from litellm.integrations.otel.logger import _excluded_db_systems + from litellm.litellm_core_utils import litellm_logging as logging_module + + logging_module._in_memory_loggers.clear() + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk") + monkeypatch.delenv("LITELLM_OTEL_EXCLUDED_SERVICES", raising=False) + is_otel_v2_enabled.cache_clear() + monkeypatch.setattr(litellm, "callback_settings", {"otel": {"excluded_services": ["postgres"]}}, raising=False) + try: + + def init(name: str) -> CustomLogger | None: + return logging_module._init_custom_logger_compatible_class( + logging_integration=name, # type: ignore[arg-type] # test passes a literal callback name + internal_usage_cache=None, + llm_router=None, + custom_logger_init_args={}, + ) + + preset = init("langfuse_otel") + otel_cb = init("otel") + + assert otel_cb is not None and otel_cb is not preset + v2_names = { + cb.callback_name for cb in logging_module._in_memory_loggers if isinstance(cb, OpenTelemetryV2) + } + assert {"langfuse_otel", "otel"} <= v2_names, v2_names + resolved = _excluded_db_systems(logging_module._in_memory_loggers, otel_cb) + assert resolved == frozenset({"postgresql"}), resolved + finally: + logging_module._in_memory_loggers.clear() + is_otel_v2_enabled.cache_clear() + @pytest.mark.parametrize("canonical", ["langfuse_otel", "arize"]) def test_publishing_tells_the_fan_out_about_every_v2_loggers_account(self, monkeypatch, canonical): monkeypatch.setenv("LITELLM_OTEL_TENANT_DESTINATION_MODE", "additive")