diff --git a/litellm/integrations/otel/model/config.py b/litellm/integrations/otel/model/config.py index 5b7b9e5d352..4ba1357a3d8 100644 --- a/litellm/integrations/otel/model/config.py +++ b/litellm/integrations/otel/model/config.py @@ -370,16 +370,19 @@ def _db_system_for_excluded_service(service: str) -> str: return resolved -def validate_otel_v2_excluded_services_env() -> None: +def validate_otel_v2_excluded_services_env(settings: object) -> None: """Validate ``LITELLM_OTEL_EXCLUDED_SERVICES`` at boot even with no ``otel`` callback. Preset-only deployments build env-only configs through a path that swallows init errors, so a bogus value would otherwise degrade to the legacy callback silently. Splitting and normalizing here raises the same ``ValueError`` the - field raises. + field raises. An explicit ``callback_settings.otel.excluded_services`` wins + over the env var, so a bad env value is inert then and must not block boot. """ if not is_otel_v2_enabled(): return + if isinstance(settings, Mapping) and "excluded_services" in settings: + return raw: Final = os.environ.get("LITELLM_OTEL_EXCLUDED_SERVICES") if not raw: return diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index 97d608884b9..88b5f556977 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -181,7 +181,7 @@ def initialize_callbacks_on_proxy( from litellm.integrations.otel.model.config import validate_otel_v2_excluded_services_env - validate_otel_v2_excluded_services_env() + validate_otel_v2_excluded_services_env(callback_specific_params.get("otel") if "otel" in value else None) # check if callback is a custom logger compatible callback if isinstance(callback, str): diff --git a/tests/integration/observability/test_otel_excluded_services.py b/tests/integration/observability/test_otel_excluded_services.py index df71dbc3fa4..a6646cb4242 100644 --- a/tests/integration/observability/test_otel_excluded_services.py +++ b/tests/integration/observability/test_otel_excluded_services.py @@ -162,6 +162,7 @@ def _guardrail_block(config: dict) -> None: ] +@pytest.mark.timeout(180) def test_excluded_services_drops_db_spans_at_tenant_only( gateway: Gateway, audit_sinks: SpanSinks, @@ -288,6 +289,35 @@ def test_bogus_excluded_service_fails_proxy_start( assert "postgres, redis" in text, text[-3000:] +def test_valid_config_excluded_services_tolerates_bogus_env( + gateway: Gateway, + audit_sinks: SpanSinks, + otel_audit_config: AuditConfigWriter, + langfuse_vars: dict[str, JsonValue], + tmp_path: Path, +) -> None: + def with_langfuse_otel(config: dict) -> None: + config["litellm_settings"]["callbacks"] = ["otel", "langfuse_otel"] + + config: Final = _config_with( + tmp_path, otel_audit_config, otel={"excluded_services": ["postgres"]}, extra=with_langfuse_otel + ) + with owned_proxy( + gateway, tmp_path, {"LITELLM_OTEL_V2": "1", "LITELLM_OTEL_EXCLUDED_SERVICES": "auth"}, config=config, workers=2 + ) as candidate: + start, _ = recorded_spans(audit_sinks.tenant) + traffic: Final = _drive(candidate, langfuse_vars) + tenant_trace: Final = _trace_id(audit_sinks.tenant, traffic) + _await_db_span(audit_sinks.tenant, tenant_trace, "redis", seconds=60) + tenant_spans: Final = _trace_spans(audit_sinks.tenant, tenant_trace, seconds=15) + _, all_tenant = recorded_spans(audit_sinks.tenant, start) + systems: Final = _db_systems(tenant_spans) + assert "redis" in systems, f"redis spans missing at tenant: {systems}" + assert "postgresql" not in _db_systems(all_tenant), ( + f"postgresql spans reached tenant: {_db_systems(all_tenant)}" + ) + + def test_bogus_excluded_services_env_fails_proxy_start_without_otel_callback( gateway: Gateway, otel_audit_config: AuditConfigWriter, tmp_path: Path ) -> None: diff --git a/tests/unit/integrations/otel/test_otel_v2_config_baggage_parenting_guardrails.py b/tests/unit/integrations/otel/test_otel_v2_config_baggage_parenting_guardrails.py index 117989e8326..c85394e8233 100644 --- a/tests/unit/integrations/otel/test_otel_v2_config_baggage_parenting_guardrails.py +++ b/tests/unit/integrations/otel/test_otel_v2_config_baggage_parenting_guardrails.py @@ -128,7 +128,7 @@ def test_excluded_services_env_is_validated_at_boot_when_enabled(monkeypatch): is_otel_v2_enabled.cache_clear() try: with pytest.raises(ValueError, match="'auth' is not a datastore service; allowed: postgres, redis"): - validate_otel_v2_excluded_services_env() + validate_otel_v2_excluded_services_env(None) finally: is_otel_v2_enabled.cache_clear() @@ -140,7 +140,19 @@ def test_excluded_services_env_validation_accepts_datastore_names(monkeypatch): monkeypatch.setenv("LITELLM_OTEL_EXCLUDED_SERVICES", "redis, postgres") is_otel_v2_enabled.cache_clear() try: - validate_otel_v2_excluded_services_env() + validate_otel_v2_excluded_services_env(None) + finally: + is_otel_v2_enabled.cache_clear() + + +def test_excluded_services_env_bad_value_is_inert_when_config_wins(monkeypatch): + from litellm.integrations.otel.model.config import is_otel_v2_enabled, validate_otel_v2_excluded_services_env + + monkeypatch.setenv("LITELLM_OTEL_V2", "1") + monkeypatch.setenv("LITELLM_OTEL_EXCLUDED_SERVICES", "auth") + is_otel_v2_enabled.cache_clear() + try: + validate_otel_v2_excluded_services_env({"excluded_services": ["postgres"]}) finally: is_otel_v2_enabled.cache_clear()