diff --git a/litellm/integrations/otel/plumbing/routing.py b/litellm/integrations/otel/plumbing/routing.py index 8fd24d8706c..f78d18d943c 100644 --- a/litellm/integrations/otel/plumbing/routing.py +++ b/litellm/integrations/otel/plumbing/routing.py @@ -241,7 +241,12 @@ class TenantTracerCache: credential_headers: Final = self._credential_headers(dynamic_params) project_headers: Final = self._project_headers(auth_metadata) service_name: Final = tenant_service_name(auth_metadata) - if not credential_headers and not project_headers and service_name is None: + tenant_account: Final = bool(credential_headers) or bool(project_headers) + # A service name on its own only relabels the operator's own backend, so moving + # the span to a second provider for it while some other backend has a + # destination would drop the model call out of the trace the fan-out delivers. + # The destination stamps the same service name itself. + if not tenant_account and (service_name is None or destination_backends()): return TenantRoute(tracer=default, detached=False) # A fixed per-integration region endpoint (New Relic us/eu), never a # caller-supplied host; ``None`` keeps the preset's own endpoint. @@ -262,7 +267,7 @@ class TenantTracerCache: _shutdown_provider(evicted) return TenantRoute( tracer=get_tracer(provider, self._tracer_name), - detached=bool(project_headers) or bool(credential_headers), + detached=tenant_account, provider=provider, ) diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index ba92d710cc8..0ec28b62a3a 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -2845,13 +2845,16 @@ async def _authorize_authenticated_request( @tracer.wrap() -def _seed_request_destinations(user_api_key_dict: UserAPIKeyAuth) -> None: +def _seed_request_destinations(user_api_key_dict: UserAPIKeyAuth, request: Request | None = None) -> None: """Anchor the OTLP destinations this key or team overrides its traces to. Called inside the ``auth`` phase span so that span reaches the tenant's account as well, and on the request task so the ``ContextVar`` is inherited by the logging tasks that close the LLM span. Best-effort: trace routing must never fail auth. + ``request`` carries the headers, so a backend this request disabled with + ``x-litellm-disable-callbacks`` resolves to no destination. + Only destinations the published fan-out can build are anchored. Anchoring one is what tells the operator's exporter to hold that backend's spans back under ``override``, so an unbuildable one would leave the span with nowhere to go. @@ -2870,7 +2873,10 @@ def _seed_request_destinations(user_api_key_dict: UserAPIKeyAuth) -> None: ) set_request_destinations( - deliverable_destinations(resolve_tenant_otel_destinations(user_api_key_dict), fan_out_provider()) + deliverable_destinations( + resolve_tenant_otel_destinations(user_api_key_dict, _safe_get_request_headers(request)), + fan_out_provider(), + ) ) except Exception as exc: # noqa: BLE001 # telemetry routing is best-effort and must never break authentication verbose_proxy_logger.debug("OTel V2: tenant destination resolution failed: %s", exc) @@ -2922,7 +2928,7 @@ async def user_api_key_auth( raise body_parse_exception raise user_api_key_auth_obj.budget_reservation = None - _seed_request_destinations(user_api_key_auth_obj) + _seed_request_destinations(user_api_key_auth_obj, request) # A body that never parsed is authenticated (so the trace carries identity # and this ``auth`` span) but not authorized: there is no model to check it diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 5c6af78aad7..bc76f630769 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -27,6 +27,7 @@ from litellm.constants import ( SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, SESSION_ID_GENERATED_METADATA_KEY, SESSION_ID_OMITTED_METADATA_KEY, + X_LITELLM_DISABLE_CALLBACKS, ) from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( @@ -987,8 +988,40 @@ def _tenant_otel_params(callback_vars: Mapping[str, str]) -> StandardCallbackDyn return StandardCallbackDynamicParams() +_NO_REQUEST_HEADERS: Final[Mapping[str, str]] = MappingProxyType({}) + + +def _dynamically_disabled_backends( + user_api_key_dict: UserAPIKeyAuth, + request_headers: Mapping[str, str] | None, +) -> frozenset[str]: + """The callbacks this request turned off, read the way dispatch reads them. + + Same sources, precedence, and premium gate ``EnterpriseCallbackControls`` applies + before it skips a callback: the ``x-litellm-disable-callbacks`` header wins over the + key's stored list, team settings are not a source, and a non-premium proxy honours + neither. A destination has to agree with that decision, or a backend the key turned + off would still be exported to, now through the fan-out instead of the callback. + """ + from litellm.proxy.proxy_server import premium_user + + if litellm.allow_dynamic_callback_disabling is not True or not premium_user: + return frozenset() + header: Final = (request_headers if request_headers is not None else _NO_REQUEST_HEADERS).get( + X_LITELLM_DISABLE_CALLBACKS + ) + if header is not None: + return frozenset(name.strip().lower() for name in header.split(",")) + metadata: Final = user_api_key_dict.metadata + disabled: Final = metadata.get("litellm_disabled_callbacks") if metadata else None + if not isinstance(disabled, list): + return frozenset() + return frozenset(name.lower() for name in disabled if isinstance(name, str)) + + def resolve_tenant_otel_destinations( user_api_key_dict: UserAPIKeyAuth, + request_headers: Mapping[str, str] | None = None, ) -> "tuple[OtelDestination, ...]": """The OTLP destinations this request's key or team config overrides its traces to. @@ -1008,6 +1041,12 @@ def resolve_tenant_otel_destinations( back until the call finishes. Those entries keep today's behaviour instead, where the tenant's credentials reach the backend through per-request tracer routing and the operator's exporter is left alone. + + A backend the request disabled dynamically, through the key's + ``litellm_disabled_callbacks`` or the ``x-litellm-disable-callbacks`` header in + ``request_headers``, resolves to no destination: dispatch skips that callback, so + the request keeps the operator's exporters for it exactly as it did before + destinations existed. """ from litellm.integrations.otel.model.config import is_otel_v2_enabled from litellm.integrations.otel.presets.destinations import destination_for @@ -1022,11 +1061,13 @@ def resolve_tenant_otel_destinations( ) if not entries: return () + disabled: Final = _dynamically_disabled_backends(user_api_key_dict, request_headers) callbacks: Final = tuple( callback for item in entries if (callback := _get_validated_callback_metadata(item=item, source="otel-destination")) is not None if callback.callback_type != "failure" + if callback.callback_name.lower() not in disabled ) return tuple( destination diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index 513e0512903..6b9ff19b418 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -895,6 +895,50 @@ class TestRouting: assert route.tracer is default, "the fan-out carries the service name on the destination instead" assert route.provider is None + @pytest.mark.parametrize("callback_name", ["arize", None]) + def test_a_service_name_does_not_detach_a_backend_the_destination_does_not_name(self, callback_name): + """The fan-out only sees spans on the published provider, so relabelling this + logger's span onto a second provider would drop the model call out of the + trace another backend's destination receives.""" + config = OpenTelemetryV2Config( + exporters=[ExporterSpec(kind="otlp_http", endpoint="http://op.local", owner=ExporterOwner.ARIZE_AX)] + ) + cache = TenantTracerCache(config, callback_name, "litellm") + default = get_tracer(TracerProvider(), "litellm") + auth_metadata = {"otel_service_name": "team-checkout"} + + relabelled = cache.route_for(default, None, auth_metadata) + assert relabelled.tracer is not default + cache.release(relabelled.provider) + + def run(): + set_request_destinations((LANGFUSE_DEST,)) + return cache.route_for(default, None, auth_metadata) + + route = in_fresh_context(run) + assert route.tracer is default + assert route.detached is False + assert route.provider is None + + def test_a_backend_with_its_own_credentials_still_routes_next_to_another_backend_destination(self): + """Credentials name the tenant's own account for this backend, which the other + backend's destination cannot stand in for.""" + config = OpenTelemetryV2Config( + exporters=[ExporterSpec(kind="otlp_http", endpoint="http://op.local", owner=ExporterOwner.ARIZE_AX)] + ) + cache = TenantTracerCache(config, "arize", "litellm") + default = get_tracer(TracerProvider(), "litellm") + params = {"arize_space_key": "space", "arize_api_key": "key"} + + def run(): + set_request_destinations((LANGFUSE_DEST,)) + return cache.route_for(default, params, {"otel_service_name": "team-checkout"}) + + route = in_fresh_context(run) + assert route.tracer is not default + assert route.detached is True + cache.release(route.provider) + @pytest.mark.usefixtures("allow_test_hosts") class TestDestinationResolution: @@ -1330,6 +1374,56 @@ class TestTenantConfigAgreement: assert [d.endpoint for d in destinations] == ["http://key.local/api/public/otel"] + @pytest.fixture + def premium(self, monkeypatch): + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "premium_user", True) + monkeypatch.setattr(litellm, "allow_dynamic_callback_disabling", True) + + @pytest.mark.usefixtures("premium") + def test_a_backend_the_key_disabled_resolves_to_no_destination(self): + """Dispatch skips a callback named in the key's ``litellm_disabled_callbacks``, + so the fan-out must not deliver to it either.""" + auth = UserAPIKeyAuth( + metadata={"litellm_disabled_callbacks": ["Langfuse_OTEL"]}, + team_metadata={"logging": [self._entry("http://team.local")]}, + ) + + assert resolve_tenant_otel_destinations(auth) == () + + @pytest.mark.usefixtures("premium") + @pytest.mark.parametrize( + ("header", "resolved"), + [ + ("langfuse_otel", False), + (" LANGFUSE_OTEL ,arize", False), + ("arize", True), + ], + ) + def test_the_disable_header_wins_over_the_key_list(self, header, resolved): + """Same precedence as dispatch: a header that names other backends re-enables + the one the key stored.""" + auth = UserAPIKeyAuth( + metadata={"litellm_disabled_callbacks": ["langfuse_otel"]}, + team_metadata={"logging": [self._entry("http://team.local")]}, + ) + + destinations = resolve_tenant_otel_destinations(auth, {"x-litellm-disable-callbacks": header}) + + assert bool(destinations) is resolved + + def test_a_non_premium_proxy_ignores_the_disabled_list_like_dispatch_does(self, monkeypatch): + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "premium_user", False) + auth = UserAPIKeyAuth( + metadata={"litellm_disabled_callbacks": ["langfuse_otel"]}, + team_metadata={"logging": [self._entry("http://team.local")]}, + ) + + assert resolve_tenant_otel_destinations(auth, {"x-litellm-disable-callbacks": "langfuse_otel"}) != () + class TestEvictionSafety: class Recording(SimpleSpanProcessor):