From c366b917294b669387758dff7b82f76b229b2114 Mon Sep 17 00:00:00 2001 From: Yucheng He Date: Fri, 4 Sep 2026 16:33:53 -0700 Subject: [PATCH] fix(otel v2): keep the credential's role in a destination's account identity Comparing values alone folds two accounts together whenever they hold the same strings in different roles, and the second team would then get no trace at all. Compare the credential under a normalized name instead, and fold the one alias that actually exists: Arize's space_id and arize-space-id. --- .../integrations/otel/plumbing/providers.py | 24 +++++++++++++------ .../otel/test_otel_v2_destinations.py | 12 +++++++++- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index ef8a35801d8..c30e0e7e5b2 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -4,6 +4,7 @@ import queue import threading from collections import OrderedDict from collections.abc import Callable, Iterable, Mapping +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal from opentelemetry import _logs, baggage, metrics @@ -219,7 +220,11 @@ _DRAIN_WORKERS: Final = 2 _SHUTDOWN_DRAIN_SECONDS: Final = 5.0 #: An exporter's account: its normalized endpoint and the credentials it presents. -_SinkKey = tuple[str, tuple[str, ...]] +_SinkKey = tuple[str, tuple[tuple[str, str], ...]] + +#: Header names that spell one credential two ways. Arize's operator exporter sends +#: ``space_id`` where a tenant destination sends ``arize-space-id``. +_CREDENTIAL_ALIASES: Final = MappingProxyType({"arize_space_id": "space_id"}) class _DrainPool: @@ -851,16 +856,21 @@ def operator_sink_keys(config: OpenTelemetryV2Config | None) -> frozenset[_SinkK def _sink_key(endpoint: str | None, headers: Mapping[str, str]) -> "_SinkKey | None": """The account an exporter writes to, or ``None`` when it has no fixed one. - The credentials are the identity; the header names are only how each backend - spells them, and one account answers to more than one spelling (Arize takes the - operator's ``space_id`` and a tenant's ``arize-space-id``). The endpoint needs - normalizing too: the operator's spec carries the signal path that a tenant - destination leaves for the exporter to append. + Normalized on the three counts that make one account look like two: the operator's + spec carries the signal path a tenant destination leaves for the exporter to + append, header names survive one round trip lowercased and the other not, and one + credential answers to more than one name (see :data:`_CREDENTIAL_ALIASES`). """ normalized: Final = _otlp_traces_endpoint(endpoint) if normalized is None: return None - return (normalized, tuple(sorted(headers.values()))) + return (normalized, tuple(sorted((_credential_name(name), value) for name, value in headers.items()))) + + +def _credential_name(header: str) -> str: + """The credential a header carries, under whichever name the backend spells it.""" + normalized: Final = header.strip().lower().replace("-", "_") + return _CREDENTIAL_ALIASES.get(normalized, normalized) def _attached_processors(provider: TracerProvider) -> "tuple[SpanProcessor, ...]": diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index d122f066670..f51dc27a690 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -129,7 +129,7 @@ class TestRoutingMode: moment a team configures its own is what ``additive`` exists to prevent. """ - OPERATOR_SINK = ("https://cloud.langfuse.com/api/public/otel/v1/traces", ("Basic op",)) + OPERATOR_SINK = ("https://cloud.langfuse.com/api/public/otel/v1/traces", (("authorization", "Basic op"),)) #: What a tenant destination for that same project looks like before normalizing: #: no signal path yet, and the header name cased the way the backend writes it. SAME_ACCOUNT_ENDPOINT = "https://cloud.langfuse.com/api/public/otel" @@ -346,6 +346,16 @@ class TestRoutingMode: assert sink("pk-op", "sk-op") in operator, "a team naming the operator's own project" assert sink("pk-team", "sk-team") not in operator, "a different project on the same server" + def test_two_accounts_holding_the_same_strings_in_different_roles_are_not_one(self): + """The values alone are not the identity. Two accounts can hold the same pair + of strings with the space id and the api key the other way round, and folding + them together would leave the second one's team with no trace at all.""" + endpoint = "https://otlp.arize.com/v1" + + assert _sink_key(endpoint, {"space_id": "a", "api_key": "b"}) != _sink_key( + endpoint, {"space_id": "b", "api_key": "a"} + ) + def test_the_operators_own_arize_space_and_a_team_naming_it_are_one_account(self, monkeypatch): """One account answers to two header names here: the operator's exporter sends ``space_id`` and a team destination sends ``arize-space-id``. Keyed on the names,