diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index ef8a35801d8..c30e0e7e5b2 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -4,6 +4,7 @@ import queue import threading from collections import OrderedDict from collections.abc import Callable, Iterable, Mapping +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal from opentelemetry import _logs, baggage, metrics @@ -219,7 +220,11 @@ _DRAIN_WORKERS: Final = 2 _SHUTDOWN_DRAIN_SECONDS: Final = 5.0 #: An exporter's account: its normalized endpoint and the credentials it presents. -_SinkKey = tuple[str, tuple[str, ...]] +_SinkKey = tuple[str, tuple[tuple[str, str], ...]] + +#: Header names that spell one credential two ways. Arize's operator exporter sends +#: ``space_id`` where a tenant destination sends ``arize-space-id``. +_CREDENTIAL_ALIASES: Final = MappingProxyType({"arize_space_id": "space_id"}) class _DrainPool: @@ -851,16 +856,21 @@ def operator_sink_keys(config: OpenTelemetryV2Config | None) -> frozenset[_SinkK def _sink_key(endpoint: str | None, headers: Mapping[str, str]) -> "_SinkKey | None": """The account an exporter writes to, or ``None`` when it has no fixed one. - The credentials are the identity; the header names are only how each backend - spells them, and one account answers to more than one spelling (Arize takes the - operator's ``space_id`` and a tenant's ``arize-space-id``). The endpoint needs - normalizing too: the operator's spec carries the signal path that a tenant - destination leaves for the exporter to append. + Normalized on the three counts that make one account look like two: the operator's + spec carries the signal path a tenant destination leaves for the exporter to + append, header names survive one round trip lowercased and the other not, and one + credential answers to more than one name (see :data:`_CREDENTIAL_ALIASES`). """ normalized: Final = _otlp_traces_endpoint(endpoint) if normalized is None: return None - return (normalized, tuple(sorted(headers.values()))) + return (normalized, tuple(sorted((_credential_name(name), value) for name, value in headers.items()))) + + +def _credential_name(header: str) -> str: + """The credential a header carries, under whichever name the backend spells it.""" + normalized: Final = header.strip().lower().replace("-", "_") + return _CREDENTIAL_ALIASES.get(normalized, normalized) def _attached_processors(provider: TracerProvider) -> "tuple[SpanProcessor, ...]": diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index d122f066670..f51dc27a690 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -129,7 +129,7 @@ class TestRoutingMode: moment a team configures its own is what ``additive`` exists to prevent. """ - OPERATOR_SINK = ("https://cloud.langfuse.com/api/public/otel/v1/traces", ("Basic op",)) + OPERATOR_SINK = ("https://cloud.langfuse.com/api/public/otel/v1/traces", (("authorization", "Basic op"),)) #: What a tenant destination for that same project looks like before normalizing: #: no signal path yet, and the header name cased the way the backend writes it. SAME_ACCOUNT_ENDPOINT = "https://cloud.langfuse.com/api/public/otel" @@ -346,6 +346,16 @@ class TestRoutingMode: assert sink("pk-op", "sk-op") in operator, "a team naming the operator's own project" assert sink("pk-team", "sk-team") not in operator, "a different project on the same server" + def test_two_accounts_holding_the_same_strings_in_different_roles_are_not_one(self): + """The values alone are not the identity. Two accounts can hold the same pair + of strings with the space id and the api key the other way round, and folding + them together would leave the second one's team with no trace at all.""" + endpoint = "https://otlp.arize.com/v1" + + assert _sink_key(endpoint, {"space_id": "a", "api_key": "b"}) != _sink_key( + endpoint, {"space_id": "b", "api_key": "a"} + ) + def test_the_operators_own_arize_space_and_a_team_naming_it_are_one_account(self, monkeypatch): """One account answers to two header names here: the operator's exporter sends ``space_id`` and a team destination sends ``arize-space-id``. Keyed on the names,