diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index bb1abb493db..df8e449bc42 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -344,6 +344,10 @@ _DB_SYSTEM_KEYS: Final = frozenset({DB.SYSTEM_NAME, DB.SYSTEM_LEGACY}) _OPERATOR_INFRASTRUCTURE_KEYS: Final = frozenset( {Server.ADDRESS, Server.PORT, DB.NAMESPACE, Error.MESSAGE, LiteLLMError.STACK_TRACE} ) +# Attribute prefixes the FastAPI instrumentor uses for headers the operator opted to +# capture (``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_*``). The request +# side carries the caller's bearer token verbatim. +_CAPTURED_HEADER_PREFIXES: Final = ("http.request.header.", "http.response.header.") class _TenantSpanView(ReadableSpan): @@ -373,21 +377,30 @@ class _TenantSpanView(ReadableSpan): ) -def _is_database_span(span: ReadableSpan) -> bool: - attributes: Final = span.attributes or _NO_ATTRIBUTES +def _is_database_span(attributes: Mapping[str, AttributeValue]) -> bool: return any(key in attributes for key in _DB_SYSTEM_KEYS) +def _tenant_visible(key: str, database: bool) -> bool: + if key.startswith(_CAPTURED_HEADER_PREFIXES): + return False + return not database or key not in _OPERATOR_INFRASTRUCTURE_KEYS + + def _for_destination(span: ReadableSpan, destination: "OtelDestination") -> ReadableSpan: """The view of ``span`` a tenant destination receives. A database span describes the operator's own Postgres rather than the tenant's - request, so its endpoint and its error text come off on the way out. The span - itself stays, so the tenant still gets the whole trace tree. + request, so its endpoint and its error text come off on the way out. Headers the + operator captures on the server span come off every span too, since the request + side holds the caller's bearer token. The span itself stays, so the tenant still + gets the whole trace tree. """ extra: Final = destination.resource_attributes - redacted: Final = _is_database_span(span) - if not extra and not redacted: + attributes: Final = span.attributes or _NO_ATTRIBUTES + database: Final = _is_database_span(attributes) + kept: Final = MappingProxyType({key: value for key, value in attributes.items() if _tenant_visible(key, database)}) + if not extra and not database and len(kept) == len(attributes): return span resource: Final = ( Resource.create( @@ -396,16 +409,9 @@ def _for_destination(span: ReadableSpan, destination: "OtelDestination") -> Read if extra else span.resource ) - if not redacted: - return _TenantSpanView(span, resource, span.attributes, span.events, span.status) - attributes: Final = span.attributes or _NO_ATTRIBUTES - return _TenantSpanView( - span, - resource, - MappingProxyType({key: value for key, value in attributes.items() if key not in _OPERATOR_INFRASTRUCTURE_KEYS}), - (), - Status(span.status.status_code), - ) + if not database: + return _TenantSpanView(span, resource, kept, span.events, span.status) + return _TenantSpanView(span, resource, kept, (), Status(span.status.status_code)) class TenantFanOutSpanProcessor(SpanProcessor): diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index 410c4ae4cc6..0f74cb4d9d0 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -568,6 +568,46 @@ class TestFanOut: assert operator_db.status.description == unreachable assert [event.name for event in operator_db.events] == ["exception"] + def test_captured_request_headers_do_not_ride_along_to_the_tenant(self): + """With ``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST`` set, the + server span carries the caller's bearer token. A team admin's collector must + not receive it, while the operator's own copy keeps it and the tenant keeps the + rest of the span, its events and its status.""" + dest_exporter, operator_exporter = InMemorySpanExporter(), InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(operator_exporter)) + provider.add_span_processor( + TenantFanOutSpanProcessor(processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter)) + ) + tracer = get_tracer(provider, "litellm") + bearer = "Bearer sk-another-members-virtual-key" + + def run(): + set_request_destinations((LANGFUSE_DEST,)) + with tracer.start_as_current_span("POST /v1/chat/completions") as server_span: + server_span.set_attributes( + { + "http.request.method": "POST", + "http.route": "/v1/chat/completions", + "http.request.header.authorization": (bearer,), + "http.request.header.x_litellm_api_key": (bearer,), + "http.response.header.set_cookie": ("session=abc",), + } + ) + server_span.add_event("request.received") + server_span.set_status(Status(StatusCode.ERROR, "rate limited")) + + in_fresh_context(run) + + tenant = dest_exporter.get_finished_spans()[0] + assert dict(tenant.attributes) == {"http.request.method": "POST", "http.route": "/v1/chat/completions"} + assert bearer not in tenant.to_json() + assert [event.name for event in tenant.events] == ["request.received"] + assert tenant.status.description == "rate limited", "only the header capture comes off a server span" + operator = operator_exporter.get_finished_spans()[0] + assert operator.attributes["http.request.header.authorization"] == (bearer,) + assert operator.attributes["http.response.header.set_cookie"] == ("session=abc",) + def test_a_destination_that_cannot_build_a_processor_is_skipped_quietly(self): """An unbuildable destination must not cost the caller its request.""" attempts = []