fix(otel): keep relabelled spans in the fan-out and honour disabled callbacks for destinations

A key or team otel_service_name used to move a backend's span onto a second
provider even when another backend had a destination, so the fan-out never saw
the model call and the tenant's trace lost it. A service name alone now stays on
the published provider whenever the request has a destination; credential and
project routing to a tenant's own account is unchanged

Destinations now skip a backend the request disabled dynamically, reading the
x-litellm-disable-callbacks header and the key's litellm_disabled_callbacks with
the same precedence and premium gate dispatch applies, so a disabled backend is
neither delivered to nor withheld from the operator

litellm/integrations/otel/plumbing/routing.py

@@ -241,7 +241,12 @@ class TenantTracerCache:
         credential_headers: Final = self._credential_headers(dynamic_params)
         project_headers: Final = self._project_headers(auth_metadata)
         service_name: Final = tenant_service_name(auth_metadata)
-        if not credential_headers and not project_headers and service_name is None:
+        tenant_account: Final = bool(credential_headers) or bool(project_headers)
+        # A service name on its own only relabels the operator's own backend, so moving
+        # the span to a second provider for it while some other backend has a
+        # destination would drop the model call out of the trace the fan-out delivers.
+        # The destination stamps the same service name itself.
+        if not tenant_account and (service_name is None or destination_backends()):
             return TenantRoute(tracer=default, detached=False)
         # A fixed per-integration region endpoint (New Relic us/eu), never a
         # caller-supplied host; ``None`` keeps the preset's own endpoint.
@@ -262,7 +267,7 @@ class TenantTracerCache:
             _shutdown_provider(evicted)
         return TenantRoute(
             tracer=get_tracer(provider, self._tracer_name),
-            detached=bool(project_headers) or bool(credential_headers),
+            detached=tenant_account,
             provider=provider,
         )
 

litellm/proxy/auth/user_api_key_auth.py

@@ -2845,13 +2845,16 @@ async def _authorize_authenticated_request(
 
 
 @tracer.wrap()
-def _seed_request_destinations(user_api_key_dict: UserAPIKeyAuth) -> None:
+def _seed_request_destinations(user_api_key_dict: UserAPIKeyAuth, request: Request | None = None) -> None:
     """Anchor the OTLP destinations this key or team overrides its traces to.
 
     Called inside the ``auth`` phase span so that span reaches the tenant's account
     as well, and on the request task so the ``ContextVar`` is inherited by the logging
     tasks that close the LLM span. Best-effort: trace routing must never fail auth.
 
+    ``request`` carries the headers, so a backend this request disabled with
+    ``x-litellm-disable-callbacks`` resolves to no destination.
+
     Only destinations the published fan-out can build are anchored. Anchoring one is
     what tells the operator's exporter to hold that backend's spans back under
     ``override``, so an unbuildable one would leave the span with nowhere to go.
@@ -2870,7 +2873,10 @@ def _seed_request_destinations(user_api_key_dict: UserAPIKeyAuth) -> None:
         )
 
         set_request_destinations(
-            deliverable_destinations(resolve_tenant_otel_destinations(user_api_key_dict), fan_out_provider())
+            deliverable_destinations(
+                resolve_tenant_otel_destinations(user_api_key_dict, _safe_get_request_headers(request)),
+                fan_out_provider(),
+            )
         )
     except Exception as exc:  # noqa: BLE001  # telemetry routing is best-effort and must never break authentication
         verbose_proxy_logger.debug("OTel V2: tenant destination resolution failed: %s", exc)
@@ -2922,7 +2928,7 @@ async def user_api_key_auth(
                 raise body_parse_exception
             raise
         user_api_key_auth_obj.budget_reservation = None
-        _seed_request_destinations(user_api_key_auth_obj)
+        _seed_request_destinations(user_api_key_auth_obj, request)
 
         # A body that never parsed is authenticated (so the trace carries identity
         # and this ``auth`` span) but not authorized: there is no model to check it

litellm/proxy/litellm_pre_call_utils.py

@@ -27,6 +27,7 @@ from litellm.constants import (
     SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
     SESSION_ID_GENERATED_METADATA_KEY,
     SESSION_ID_OMITTED_METADATA_KEY,
+    X_LITELLM_DISABLE_CALLBACKS,
 )
 from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
 from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
@@ -987,8 +988,40 @@ def _tenant_otel_params(callback_vars: Mapping[str, str]) -> StandardCallbackDyn
         return StandardCallbackDynamicParams()
 
 
+_NO_REQUEST_HEADERS: Final[Mapping[str, str]] = MappingProxyType({})
+
+
+def _dynamically_disabled_backends(
+    user_api_key_dict: UserAPIKeyAuth,
+    request_headers: Mapping[str, str] | None,
+) -> frozenset[str]:
+    """The callbacks this request turned off, read the way dispatch reads them.
+
+    Same sources, precedence, and premium gate ``EnterpriseCallbackControls`` applies
+    before it skips a callback: the ``x-litellm-disable-callbacks`` header wins over the
+    key's stored list, team settings are not a source, and a non-premium proxy honours
+    neither. A destination has to agree with that decision, or a backend the key turned
+    off would still be exported to, now through the fan-out instead of the callback.
+    """
+    from litellm.proxy.proxy_server import premium_user
+
+    if litellm.allow_dynamic_callback_disabling is not True or not premium_user:
+        return frozenset()
+    header: Final = (request_headers if request_headers is not None else _NO_REQUEST_HEADERS).get(
+        X_LITELLM_DISABLE_CALLBACKS
+    )
+    if header is not None:
+        return frozenset(name.strip().lower() for name in header.split(","))
+    metadata: Final = user_api_key_dict.metadata
+    disabled: Final = metadata.get("litellm_disabled_callbacks") if metadata else None
+    if not isinstance(disabled, list):
+        return frozenset()
+    return frozenset(name.lower() for name in disabled if isinstance(name, str))
+
+
 def resolve_tenant_otel_destinations(
     user_api_key_dict: UserAPIKeyAuth,
+    request_headers: Mapping[str, str] | None = None,
 ) -> "tuple[OtelDestination, ...]":
     """The OTLP destinations this request's key or team config overrides its traces to.
 
@@ -1008,6 +1041,12 @@ def resolve_tenant_otel_destinations(
     back until the call finishes. Those entries keep today's behaviour instead, where
     the tenant's credentials reach the backend through per-request tracer routing and
     the operator's exporter is left alone.
+
+    A backend the request disabled dynamically, through the key's
+    ``litellm_disabled_callbacks`` or the ``x-litellm-disable-callbacks`` header in
+    ``request_headers``, resolves to no destination: dispatch skips that callback, so
+    the request keeps the operator's exporters for it exactly as it did before
+    destinations existed.
     """
     from litellm.integrations.otel.model.config import is_otel_v2_enabled
     from litellm.integrations.otel.presets.destinations import destination_for
@@ -1022,11 +1061,13 @@ def resolve_tenant_otel_destinations(
     )
     if not entries:
         return ()
+    disabled: Final = _dynamically_disabled_backends(user_api_key_dict, request_headers)
     callbacks: Final = tuple(
         callback
         for item in entries
         if (callback := _get_validated_callback_metadata(item=item, source="otel-destination")) is not None
         if callback.callback_type != "failure"
+        if callback.callback_name.lower() not in disabled
     )
     return tuple(
         destination

tests/test_litellm/integrations/otel/test_otel_v2_destinations.py

@@ -895,6 +895,50 @@ class TestRouting:
         assert route.tracer is default, "the fan-out carries the service name on the destination instead"
         assert route.provider is None
 
+    @pytest.mark.parametrize("callback_name", ["arize", None])
+    def test_a_service_name_does_not_detach_a_backend_the_destination_does_not_name(self, callback_name):
+        """The fan-out only sees spans on the published provider, so relabelling this
+        logger's span onto a second provider would drop the model call out of the
+        trace another backend's destination receives."""
+        config = OpenTelemetryV2Config(
+            exporters=[ExporterSpec(kind="otlp_http", endpoint="http://op.local", owner=ExporterOwner.ARIZE_AX)]
+        )
+        cache = TenantTracerCache(config, callback_name, "litellm")
+        default = get_tracer(TracerProvider(), "litellm")
+        auth_metadata = {"otel_service_name": "team-checkout"}
+
+        relabelled = cache.route_for(default, None, auth_metadata)
+        assert relabelled.tracer is not default
+        cache.release(relabelled.provider)
+
+        def run():
+            set_request_destinations((LANGFUSE_DEST,))
+            return cache.route_for(default, None, auth_metadata)
+
+        route = in_fresh_context(run)
+        assert route.tracer is default
+        assert route.detached is False
+        assert route.provider is None
+
+    def test_a_backend_with_its_own_credentials_still_routes_next_to_another_backend_destination(self):
+        """Credentials name the tenant's own account for this backend, which the other
+        backend's destination cannot stand in for."""
+        config = OpenTelemetryV2Config(
+            exporters=[ExporterSpec(kind="otlp_http", endpoint="http://op.local", owner=ExporterOwner.ARIZE_AX)]
+        )
+        cache = TenantTracerCache(config, "arize", "litellm")
+        default = get_tracer(TracerProvider(), "litellm")
+        params = {"arize_space_key": "space", "arize_api_key": "key"}
+
+        def run():
+            set_request_destinations((LANGFUSE_DEST,))
+            return cache.route_for(default, params, {"otel_service_name": "team-checkout"})
+
+        route = in_fresh_context(run)
+        assert route.tracer is not default
+        assert route.detached is True
+        cache.release(route.provider)
+
 
 @pytest.mark.usefixtures("allow_test_hosts")
 class TestDestinationResolution:
@@ -1330,6 +1374,56 @@ class TestTenantConfigAgreement:
 
         assert [d.endpoint for d in destinations] == ["http://key.local/api/public/otel"]
 
+    @pytest.fixture
+    def premium(self, monkeypatch):
+        from litellm.proxy import proxy_server
+
+        monkeypatch.setattr(proxy_server, "premium_user", True)
+        monkeypatch.setattr(litellm, "allow_dynamic_callback_disabling", True)
+
+    @pytest.mark.usefixtures("premium")
+    def test_a_backend_the_key_disabled_resolves_to_no_destination(self):
+        """Dispatch skips a callback named in the key's ``litellm_disabled_callbacks``,
+        so the fan-out must not deliver to it either."""
+        auth = UserAPIKeyAuth(
+            metadata={"litellm_disabled_callbacks": ["Langfuse_OTEL"]},
+            team_metadata={"logging": [self._entry("http://team.local")]},
+        )
+
+        assert resolve_tenant_otel_destinations(auth) == ()
+
+    @pytest.mark.usefixtures("premium")
+    @pytest.mark.parametrize(
+        ("header", "resolved"),
+        [
+            ("langfuse_otel", False),
+            (" LANGFUSE_OTEL ,arize", False),
+            ("arize", True),
+        ],
+    )
+    def test_the_disable_header_wins_over_the_key_list(self, header, resolved):
+        """Same precedence as dispatch: a header that names other backends re-enables
+        the one the key stored."""
+        auth = UserAPIKeyAuth(
+            metadata={"litellm_disabled_callbacks": ["langfuse_otel"]},
+            team_metadata={"logging": [self._entry("http://team.local")]},
+        )
+
+        destinations = resolve_tenant_otel_destinations(auth, {"x-litellm-disable-callbacks": header})
+
+        assert bool(destinations) is resolved
+
+    def test_a_non_premium_proxy_ignores_the_disabled_list_like_dispatch_does(self, monkeypatch):
+        from litellm.proxy import proxy_server
+
+        monkeypatch.setattr(proxy_server, "premium_user", False)
+        auth = UserAPIKeyAuth(
+            metadata={"litellm_disabled_callbacks": ["langfuse_otel"]},
+            team_metadata={"logging": [self._entry("http://team.local")]},
+        )
+
+        assert resolve_tenant_otel_destinations(auth, {"x-litellm-disable-callbacks": "langfuse_otel"}) != ()
+
 
 class TestEvictionSafety:
     class Recording(SimpleSpanProcessor):