fix(otel): decide per-team Arize sampling from the trace id so every pod agrees

This commit is contained in:
yucheng-berri 2026-10-19 16:13:50 -07:00
parent ca0ec627b6
commit 14bd4649f7
4 changed files with 357 additions and 61 deletions

View file

@ -42,8 +42,8 @@ class OtelDestination(LiteLLMBaseModel):
success_sampling_rate: float | None = Field(
default=None,
description=(
"Share of the request trees forwarded, 0.0..1.0, drawn once per request when its root span ends; "
"``None`` forwards every one."
"Share of the request trees forwarded, 0.0..1.0, decided from the trace id so every process "
"serving the trace agrees; ``None`` forwards every one."
),
)
error_sampling_rate: float | None = Field(

View file

@ -1,11 +1,11 @@
"""Provider / exporter factory + the Baggage span processor."""
import queue
import random
import threading
import time
from collections import OrderedDict
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
from hashlib import sha256
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal
@ -512,9 +512,9 @@ def _is_sampled(destination: "OtelDestination") -> bool:
return destination.success_sampling_rate is not None or destination.error_sampling_rate is not None
#: A verdict is one request tree's draw for one destination at its rates: the exporter
#: A verdict is one trace's keep or drop for one destination at its rates: the exporter
#: ``cache_key`` leaves the rates out, so two destinations sharing an exporter with
#: different rates would otherwise share one draw.
#: different rates would otherwise share one verdict.
_VerdictKey = tuple[int, object, float | None, float | None]
@ -522,15 +522,19 @@ def _verdict_key(trace_id: int, destination: "OtelDestination") -> _VerdictKey:
return (trace_id, destination.cache_key(), destination.success_sampling_rate, destination.error_sampling_rate)
def _keeps_tree(destination: "OtelDestination", failed: bool, draw: Callable[[], float]) -> bool:
"""Whether one draw against the destination's rate keeps a request tree, read the
way the legacy Arize callback reads ``arize_success_sampling_rate`` /
``arize_error_sampling_rate``: a tree with a failed span answers to the error
rate, an unset rate keeps everything and ``0.0`` keeps nothing."""
def _keeps_tree(destination: "OtelDestination", failed: bool, trace_id: int) -> bool:
"""Whether the destination's rate keeps a request tree of ``trace_id``: a tree with
a failed span answers to ``arize_error_sampling_rate``, the rest to
``arize_success_sampling_rate``, an unset rate keeps everything and ``0.0`` keeps
nothing.
The first eight bytes of a SHA-256 of the trace id are read against the rate, so
every process serving a span of the trace reaches the same verdict."""
rate: Final = destination.error_sampling_rate if failed else destination.success_sampling_rate
if rate is None:
return True
return rate > 0.0 and draw() <= rate
digest: Final = sha256(trace_id.to_bytes(16, "big")).digest()
return int.from_bytes(digest[:8], "big") < round(rate * 2**64)
def _in_scope(span: ReadableSpan, scope: "OtelSpanScope") -> bool:
@ -667,10 +671,8 @@ class TenantFanOutSpanProcessor(SpanProcessor):
default_capture_by_backend: Mapping[str, str] = MappingProxyType({}),
pending_drains: int = _MAX_PENDING_DRAINS,
drain_pool: _DrainPool | None = None,
sampling_draw: Callable[[], float] | None = None,
) -> None:
self._operator_sinks: Final = operator_sinks
self._draw: Final = sampling_draw if sampling_draw is not None else random.random
self._excluded_db_systems: Final = excluded_db_systems
self._default_capture: Final = default_capture
self._default_capture_by_backend: Final = default_capture_by_backend
@ -750,7 +752,7 @@ class TenantFanOutSpanProcessor(SpanProcessor):
tree is held for the rest of its trace, or what the bounds made the fan-out decide
early (a tree that outgrew its cap is decided on what it has, and so is the oldest
tree once too many are waiting, so a span that never ends holds nothing back for
ever). A failed span marks its tree before any such early draw. Once shutdown
ever). A failed span marks its tree before any such early decision. Once shutdown
has started flushing, nothing is held: a span ending then is decided at once,
since no later flush would reach it.
@ -795,16 +797,16 @@ class TenantFanOutSpanProcessor(SpanProcessor):
return self._decide_locked(trace_id)
def _decide_locked(self, trace_id: int) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]":
"""Draw once per destination for the held tree; the spans the draws keep, and a
verdict remembered for the spans of this trace that start later."""
"""Decide once per destination for the held tree; the spans the verdicts keep, and
a verdict remembered for the spans of this trace that start later."""
tree: Final = self._pending.pop(trace_id, None)
if tree is None:
return ()
verdicts: dict[_VerdictKey, bool] = {} # mutable-ok: one draw per destination of this tree
verdicts: dict[_VerdictKey, bool] = {} # mutable-ok: one verdict per destination of this tree
for _, destination in tree.held:
key = _verdict_key(trace_id, destination)
if key not in verdicts:
verdicts[key] = _keeps_tree(destination, tree.failed, self._draw)
verdicts[key] = _keeps_tree(destination, tree.failed, trace_id)
self._verdicts[key] = verdicts[key]
while len(self._verdicts) > _MAX_REMEMBERED_VERDICTS:
self._verdicts.popitem(last=False)

View file

@ -0,0 +1,248 @@
"""A team's Arize sampling rate decides once per trace, so pods serving calls of one trace agree.
Two proxies share one database, as two pods behind a load balancer do. Each trace sends one call
to each pod under the caller's ``traceparent``; the team's Arize space must receive both calls of a
trace or neither.
"""
import re
import uuid
from collections.abc import Iterator, Mapping
from contextlib import contextmanager
from dataclasses import dataclass, replace
from pathlib import Path
from typing import Final
from urllib.parse import urlparse
import pytest
import yaml
from integration._support.client import (
Gateway,
Scenario,
eventually,
gateway_from_environment,
object_value,
until_asserted,
)
from integration._support.database import scratch_database
from integration._support.otlp_sink import Span, SpanSinks, recorded_requests, recorded_spans
from integration._support.process import owned_proxy_process
from integration._support.wire import Reply, Request, Wire, wire_server
from pydantic import JsonValue
MODEL: Final = "team-sampling"
SERVER: Final = 2
CALL_ID: Final = "litellm.call_id"
OPERATION: Final = "gen_ai.operation.name"
SPACE: Final = "space-sampled"
CREDENTIALS: Final = {"arize_space_key": SPACE, "arize_api_key": "arize-sampled"}
FAILING: Final = re.compile(rb"sampling-fail")
TRACES_PER_CASE: Final = 16
def _upstream(request: Request) -> Reply:
if FAILING.search(request.body) is not None:
return Reply(status=500, body=b'{"error":{"message":"scripted upstream failure","type":"server_error"}}')
return Reply(
body=(
b'{"id":"chatcmpl-sampling","object":"chat.completion","created":1,"model":"gpt-4o-mini",'
b'"choices":[{"index":0,"message":{"role":"assistant","content":"ok"},"finish_reason":"stop"}],'
b'"usage":{"prompt_tokens":7,"completion_tokens":2,"total_tokens":9}}'
)
)
@dataclass(frozen=True, slots=True)
class Sent:
trace_id: str
parent_span_id: str
call_id: str
@dataclass(frozen=True, slots=True)
class Pods:
first: Gateway
second: Gateway
scenario: Scenario
sinks: SpanSinks
def team_key(self, variables: Mapping[str, str]) -> str:
team: Final = self.scenario.team()
attached: Final = self.first.request(
"POST", f"/team/{team}/callback", {"callback_name": "arize", "callback_vars": dict(variables)}
)
assert attached.status_code == 200, attached.text
return self.scenario.key(team_id=team)
def send(self, pod: Gateway, key: str, trace_id: str, content: str = "hello") -> Sent:
parent: Final = uuid.uuid4().hex[:16]
call_id: Final = str(uuid.uuid4())
body: Final[dict[str, JsonValue]] = {"model": MODEL, "messages": [{"role": "user", "content": content}]}
response: Final = pod.client.post(
"/v1/chat/completions",
json=body,
headers={
"Authorization": f"Bearer {key}",
"traceparent": f"00-{trace_id}-{parent}-01",
"x-litellm-call-id": call_id,
},
)
assert response.status_code == (500 if FAILING.search(content.encode()) else 200), response.text
return Sent(trace_id, parent, call_id)
def _space_span_ids(url: str) -> frozenset[str]:
"""Span ids the Arize sink received in export requests sent with the team's space id."""
return frozenset(
span_id
for request in recorded_requests(url)
if {name.lower(): value for name, value in object_value(request.get("headers") or {}).items()}.get(
"arize-space-id"
)
== SPACE
for span_id in request.get("span_ids") or ()
if isinstance(span_id, str)
)
def _roots(spans: tuple[Span, ...], sent: tuple[Sent, ...]) -> frozenset[str]:
"""Call ids of ``sent`` whose server span is among ``spans``."""
by_parent: Final = {(call.trace_id, call.parent_span_id): call.call_id for call in sent}
return frozenset(
by_parent[(span["trace_id"], span["parent_span_id"])]
for span in spans
if span["kind"] == SERVER and (span["trace_id"], span["parent_span_id"]) in by_parent
)
def _team_roots(pods: Pods, since: int, sent: tuple[Sent, ...]) -> frozenset[str]:
owned: Final = _space_span_ids(pods.sinks.arize)
spans: Final = recorded_spans(pods.sinks.arize, since)[1]
return _roots(tuple(span for span in spans if span["span_id"] in owned), sent)
def _logged(pods: Pods, since: int, sent: tuple[Sent, ...]) -> frozenset[str]:
"""Call ids of ``sent`` whose model span reached the operator's unsampled copy."""
spans: Final = recorded_spans(pods.sinks.operator, since)[1]
return frozenset(
str(span["attributes"][CALL_ID])
for span in spans
if OPERATION in span["attributes"] and CALL_ID in span["attributes"]
) & {call.call_id for call in sent}
def _trace_ids(case: str) -> tuple[str, ...]:
"""Fixed RFC 4122 shaped trace ids, like the uuid4 hex a caller mints for ``traceparent``."""
return tuple(
uuid.uuid5(uuid.NAMESPACE_URL, f"team-sampling/{case}/{index}").hex for index in range(TRACES_PER_CASE)
)
@pytest.mark.parametrize(
("variables", "content"),
[({"arize_success_sampling_rate": "0.5"}, "hello"), ({"arize_error_sampling_rate": "0.5"}, "sampling-fail")],
ids=["success-rate", "error-rate"],
)
def test_each_trace_reaches_the_team_space_whole_or_not_at_all(
pods: Pods, variables: Mapping[str, str], content: str
) -> None:
sampled: Final = pods.team_key({**CREDENTIALS, **variables})
barrier: Final = pods.team_key(CREDENTIALS)
operator_since: Final = recorded_spans(pods.sinks.operator)[0]
arize_since: Final = recorded_spans(pods.sinks.arize)[0]
case: Final = "-".join(sorted(variables))
calls: Final = tuple(
pods.send(pod, sampled, trace_id, content) for trace_id in _trace_ids(case) for pod in (pods.first, pods.second)
)
eventually(
lambda: _logged(pods, operator_since, calls),
lambda logged: len(logged) == len(calls),
seconds=60,
)
# The unsampled team shares the sampled team's exporter, which sends spans in the order they were handed to it,
# so its calls arriving means every decided call of the sampled team has been sent too.
fences: Final = tuple(pods.send(pod, barrier, uuid.uuid4().hex) for pod in (pods.first, pods.second))
def whole_or_nothing() -> dict[str, tuple[bool, bool]]:
delivered: Final = _team_roots(pods, arize_since, calls + fences)
assert {fence.call_id for fence in fences} <= delivered, "the unsampled team's calls have not arrived yet"
by_trace: Final = {
trace_id: (calls[2 * index].call_id in delivered, calls[2 * index + 1].call_id in delivered)
for index, trace_id in enumerate(_trace_ids(case))
}
split: Final = {trace_id: kept for trace_id, kept in by_trace.items() if kept[0] != kept[1]}
assert split == {}, f"pods disagreed on {len(split)} of {len(by_trace)} traces (first pod, second pod): {split}"
return by_trace
verdicts: Final = until_asserted(whole_or_nothing, seconds=30)
kept: Final = sum(1 for first, _ in verdicts.values() if first)
assert 0 < kept < TRACES_PER_CASE, f"a rate of 0.5 kept {kept} of {TRACES_PER_CASE} traces"
def _write_config(directory: Path, sinks: SpanSinks, provider: Wire) -> Path:
config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
config["model_list"] = [
*config.get("model_list", []),
{
"model_name": MODEL,
"litellm_params": {
"model": "openai/gpt-4o-mini",
"api_key": "integration-provider-key",
"api_base": provider.url + "/v1",
},
},
]
config["litellm_settings"] = {
**config.get("litellm_settings", {}),
"callbacks": ["langfuse_otel"],
"otel_tenant_destination_mode": "additive",
"provider_url_destination_allowed_hosts": [urlparse(sinks.operator).netloc, urlparse(sinks.arize).netloc],
}
config["router_settings"] = {**config.get("router_settings", {}), "num_retries": 0}
config["general_settings"] = {**config.get("general_settings", {}), "user_api_key_cache_ttl": 2}
path: Final = directory / "team-sampling.yaml"
path.write_text(yaml.safe_dump(config))
return path
@contextmanager
def _two_pods(provider: Wire, sinks: SpanSinks, directory: Path) -> Iterator[Pods]:
environment: Final = {
"LITELLM_OTEL_V2": "1",
"OTEL_BSP_SCHEDULE_DELAY": "300",
"MAPPER_NAMES": "genai,openinference",
"LANGFUSE_HOST": sinks.operator,
"LANGFUSE_PUBLIC_KEY": "pk-lf-operator",
"LANGFUSE_SECRET_KEY": "sk-lf-operator",
"ARIZE_SPACE_KEY": "space-operator",
"ARIZE_API_KEY": "arize-operator",
"ARIZE_HTTP_ENDPOINT": sinks.arize + "/v1/traces",
}
removed: Final = ("LITELLM_OTEL_TENANT_DESTINATION_MODE", "OTEL_TRACES_SAMPLER", "OTEL_TRACES_SAMPLER_ARG")
config: Final = _write_config(directory, sinks, provider)
for pod in ("first", "second"):
(directory / pod).mkdir()
with gateway_from_environment() as shared, scratch_database("team_sampling") as database_url:
overrides: Final = {**environment, "DATABASE_URL": database_url}
with (
owned_proxy_process(
shared, directory / "first", overrides, config=config, remove_environment=removed
) as first,
owned_proxy_process(
shared, directory / "second", overrides, config=config, remove_environment=removed
) as second,
replace(first.gateway, database_url=database_url).scenario() as scenario,
):
yield Pods(first.gateway, second.gateway, scenario, sinks)
@pytest.fixture(scope="module")
def provider() -> Iterator[Wire]:
with wire_server(_upstream) as wire:
yield wire
@pytest.fixture(scope="module")
def pods(provider: Wire, audit_sinks: SpanSinks, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Pods]:
with _two_pods(provider, audit_sinks, tmp_path_factory.mktemp("team-sampling")) as started:
yield started

View file

@ -4,6 +4,7 @@ import asyncio
import contextvars
import threading
import time
import uuid
from base64 import b64encode
from collections.abc import Mapping
from dataclasses import replace
@ -18,6 +19,7 @@ from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import ReadableSpan, TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
from opentelemetry.sdk.trace.id_generator import RandomIdGenerator
from opentelemetry.trace import Status, StatusCode
import litellm
@ -1165,6 +1167,22 @@ def arize_destination(**rates: str) -> OtelDestination:
return destination
#: Trace ids whose SHA-256 reads 0.40 and 0.80 of the way through its first eight bytes,
#: so a rate of 0.5 keeps the first and drops the second.
KEPT_AT_HALF: Final = 4
DROPPED_AT_HALF: Final = 5
class _FixedTraceId(RandomIdGenerator):
"""Every trace the provider starts gets ``trace_id``."""
def __init__(self, trace_id: int) -> None:
self._trace_id = trace_id
def generate_trace_id(self) -> int:
return self._trace_id
class TestTenantSampling:
"""A team's ``arize_success_sampling_rate`` and ``arize_error_sampling_rate`` decide how
much of its traffic the fan-out delivers to its space, the way they already decide what
@ -1173,14 +1191,13 @@ class TestTenantSampling:
NAMES = {"POST /v1/chat/completions", "auth /v1/chat/completions", "chat gpt-4"}
@staticmethod
def _fan_out(dest_exporter, draw=None, global_exporter=None):
provider = TracerProvider()
def _fan_out(dest_exporter, trace_id=None, global_exporter=None):
provider = TracerProvider() if trace_id is None else TracerProvider(id_generator=_FixedTraceId(trace_id))
if global_exporter is not None:
provider.add_span_processor(_OverriddenBackendFilter(SimpleSpanProcessor(global_exporter), "arize"))
kwargs = {"processor_factory": lambda _d: SimpleSpanProcessor(dest_exporter)}
if draw is not None:
kwargs["sampling_draw"] = draw
provider.add_span_processor(TenantFanOutSpanProcessor(**kwargs))
provider.add_span_processor(
TenantFanOutSpanProcessor(processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter))
)
return provider
@staticmethod
@ -1224,7 +1241,7 @@ class TestTenantSampling:
def test_an_unset_rate_exports_everything(self):
dest_exporter = InMemorySpanExporter()
provider = self._fan_out(dest_exporter, draw=lambda: 0.99)
provider = self._fan_out(dest_exporter, trace_id=DROPPED_AT_HALF)
self._run(provider, (arize_destination(),))
@ -1300,10 +1317,10 @@ class TestTenantSampling:
def test_a_span_that_ends_after_the_root_follows_the_requests_verdict(self):
"""A post-call database write that starts after the whole request tree has ended
goes where the rest of the tree went, without a second draw."""
goes where the rest of the tree went, without a second verdict."""
kept_exporter, dropped_exporter = InMemorySpanExporter(), InMemorySpanExporter()
kept = self._fan_out(kept_exporter, draw=lambda: 0.1)
dropped = self._fan_out(dropped_exporter, draw=lambda: 0.9)
kept = self._fan_out(kept_exporter, trace_id=KEPT_AT_HALF)
dropped = self._fan_out(dropped_exporter, trace_id=DROPPED_AT_HALF)
destinations = (arize_destination(arize_success_sampling_rate="0.5"),)
def late_tree(provider):
@ -1330,11 +1347,11 @@ class TestTenantSampling:
def test_a_late_span_whose_verdict_was_forgotten_is_decided_on_its_own_not_stranded(self):
"""Once enough other requests have been decided to evict a request's verdict, a span
of it that starts late (a post-call database write) opens and closes its own count,
so it is decided on a draw of its own as soon as it ends instead of waiting in a tree
nothing would ever close."""
so it is decided on its own as soon as it ends instead of waiting in a tree nothing
would ever close."""
dest_exporter = InMemorySpanExporter()
provider = self._fan_out(dest_exporter, draw=lambda: 0.1)
destinations = (arize_destination(arize_success_sampling_rate="0.5"),)
provider = self._fan_out(dest_exporter)
destinations = (arize_destination(arize_success_sampling_rate="1.0"),)
def run():
set_request_destinations(destinations)
@ -1353,18 +1370,17 @@ class TestTenantSampling:
assert names[0] == "POST /v1/chat/completions"
assert names[-1] == "postgres INSERT LiteLLM_SpendLogs", "exported as it ended, not held for a root"
def test_a_callers_traceparent_neither_steers_the_draw_nor_hides_the_root(self):
"""The draw is random, not read from the trace id a caller may have chosen, and the
server span under a remote parent still closes the tree."""
def test_a_callers_traceparent_decides_by_its_trace_id_and_still_closes_the_tree(self):
"""A request continuing a caller's trace is decided on that trace id, and the server
span under a remote parent still closes the tree."""
dest_exporter = InMemorySpanExporter()
draws = iter([0.9, 0.1])
provider = self._fan_out(dest_exporter, draw=lambda: next(draws))
provider = self._fan_out(dest_exporter)
destinations = (arize_destination(arize_success_sampling_rate="0.5"),)
def run():
set_request_destinations(destinations)
tracer = get_tracer(provider, "litellm")
for trace_id in (1, 2):
for trace_id in (DROPPED_AT_HALF, KEPT_AT_HALF):
remote = trace_api.SpanContext(
trace_id=trace_id, span_id=1, is_remote=True, trace_flags=trace_api.TraceFlags(0x01)
)
@ -1377,12 +1393,12 @@ class TestTenantSampling:
kept = dest_exporter.get_finished_spans()
assert [s.name for s in kept] == ["chat gpt-4", "POST /v1/chat/completions"]
assert all(s.context.trace_id == 2 for s in kept), "the second request, whose draw of 0.1 passed"
assert all(s.context.trace_id == KEPT_AT_HALF for s in kept)
def test_a_root_that_never_ends_holds_the_tree_only_up_to_the_bound(self):
dest_exporter = InMemorySpanExporter()
provider = self._fan_out(dest_exporter, draw=lambda: 0.0)
destinations = (arize_destination(arize_success_sampling_rate="0.5"),)
provider = self._fan_out(dest_exporter)
destinations = (arize_destination(arize_success_sampling_rate="1.0"),)
def run():
set_request_destinations(destinations)
@ -1397,8 +1413,8 @@ class TestTenantSampling:
def test_a_flood_of_waiting_trees_decides_the_oldest_on_what_it_has(self):
dest_exporter = InMemorySpanExporter()
provider = self._fan_out(dest_exporter, draw=lambda: 0.0)
destinations = (arize_destination(arize_success_sampling_rate="0.5"),)
provider = self._fan_out(dest_exporter)
destinations = (arize_destination(arize_success_sampling_rate="1.0"),)
def run():
set_request_destinations(destinations)
@ -1419,8 +1435,8 @@ class TestTenantSampling:
only the sampled ones. A trace pushed out of that count is decided on what it has as
its next span ends, root still open, rather than held until a bound gets to it."""
dest_exporter = InMemorySpanExporter()
provider = self._fan_out(dest_exporter, draw=lambda: 0.0)
destinations = (arize_destination(arize_success_sampling_rate="0.5"),)
provider = self._fan_out(dest_exporter)
destinations = (arize_destination(arize_success_sampling_rate="1.0"),)
def run():
set_request_destinations(destinations)
@ -1443,14 +1459,12 @@ class TestTenantSampling:
def test_shutdown_decides_what_is_still_held(self):
dest_exporter = InMemorySpanExporter()
fan_out = TenantFanOutSpanProcessor(
processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter), sampling_draw=lambda: 0.0
)
fan_out = TenantFanOutSpanProcessor(processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter))
provider = TracerProvider()
provider.add_span_processor(fan_out)
def run():
set_request_destinations((arize_destination(arize_success_sampling_rate="0.5"),))
set_request_destinations((arize_destination(arize_success_sampling_rate="1.0"),))
tracer = get_tracer(provider, "litellm")
with tracer.start_as_current_span("POST /v1/chat/completions"):
with tracer.start_as_current_span("chat gpt-4"):
@ -1476,14 +1490,12 @@ class TestTenantSampling:
straggler.end()
super().on_end(span)
fan_out = TenantFanOutSpanProcessor(
processor_factory=lambda _d: _EndsAStragglerOnExport(dest_exporter), sampling_draw=lambda: 0.0
)
fan_out = TenantFanOutSpanProcessor(processor_factory=lambda _d: _EndsAStragglerOnExport(dest_exporter))
provider = TracerProvider()
provider.add_span_processor(fan_out)
def run():
set_request_destinations((arize_destination(arize_success_sampling_rate="0.5"),))
set_request_destinations((arize_destination(arize_success_sampling_rate="1.0"),))
tracer = get_tracer(provider, "litellm")
root = tracer.start_span("POST /v1/chat/completions", context=trace_api.Context())
tracer.start_span("chat gpt-4", context=trace_api.set_span_in_context(root)).end()
@ -1495,20 +1507,20 @@ class TestTenantSampling:
assert {s.name for s in dest_exporter.get_finished_spans()} == {"chat gpt-4", "embed ada"}
def test_a_zero_rate_drops_even_a_draw_of_zero(self):
def test_a_zero_rate_drops_a_trace_any_other_rate_would_keep(self):
dest_exporter = InMemorySpanExporter()
provider = self._fan_out(dest_exporter, draw=lambda: 0.0)
provider = self._fan_out(dest_exporter, trace_id=KEPT_AT_HALF)
self._run(provider, (arize_destination(arize_success_sampling_rate="0.0"),))
assert dest_exporter.get_finished_spans() == ()
def test_the_draw_is_compared_to_the_rate_the_way_the_legacy_callback_compares_it(self):
def test_the_trace_id_is_read_against_the_rate(self):
dropped_exporter, kept_exporter = InMemorySpanExporter(), InMemorySpanExporter()
destinations = (arize_destination(arize_success_sampling_rate="0.2"),)
destinations = (arize_destination(arize_success_sampling_rate="0.5"),)
self._run(self._fan_out(dropped_exporter, draw=lambda: 0.3), destinations)
self._run(self._fan_out(kept_exporter, draw=lambda: 0.2), destinations)
self._run(self._fan_out(dropped_exporter, trace_id=DROPPED_AT_HALF), destinations)
self._run(self._fan_out(kept_exporter, trace_id=KEPT_AT_HALF), destinations)
assert dropped_exporter.get_finished_spans() == ()
assert {s.name for s in kept_exporter.get_finished_spans()} == self.NAMES
@ -1527,6 +1539,40 @@ class TestTenantSampling:
assert all(names == self.NAMES for names in by_trace.values())
assert 0 < len(by_trace) < 64
@pytest.mark.parametrize(
("rates", "failed"),
[({"arize_success_sampling_rate": "0.5"}, False), ({"arize_error_sampling_rate": "0.5"}, True)],
ids=["success-rate", "error-rate"],
)
def test_two_processes_serving_one_trace_reach_the_same_verdict(self, rates, failed):
"""Two pods behind a load balancer each serve one call of a trace the caller
continued with its ``traceparent``: the team's space gets both calls or neither,
and a rate of 0.5 over UUID-shaped trace ids keeps some traces and drops others."""
exporters = (InMemorySpanExporter(), InMemorySpanExporter())
pods = tuple(self._fan_out(exporter) for exporter in exporters)
destinations = (arize_destination(**rates),)
trace_ids = tuple(uuid.uuid5(uuid.NAMESPACE_URL, f"pod/{index}").int for index in range(32))
def run(provider):
set_request_destinations(destinations)
tracer = get_tracer(provider, "litellm")
for trace_id in trace_ids:
remote = trace_api.SpanContext(
trace_id=trace_id, span_id=1, is_remote=True, trace_flags=trace_api.TraceFlags(0x01)
)
parent = trace_api.set_span_in_context(trace_api.NonRecordingSpan(remote))
with tracer.start_as_current_span("POST /v1/chat/completions", context=parent):
with tracer.start_as_current_span("chat gpt-4") as span:
if failed:
span.set_status(Status(StatusCode.ERROR, "upstream 500"))
for provider in pods:
in_fresh_context(run, provider)
first, second = ({s.context.trace_id for s in exporter.get_finished_spans()} for exporter in exporters)
assert first == second
assert 0 < len(first) < len(trace_ids)
def test_in_additive_mode_the_operators_own_copy_is_not_sampled(self, monkeypatch):
"""The rates are the team's setting for the team's space; the operator's backbone keeps
every span."""
@ -1547,7 +1593,7 @@ class TestTenantSampling:
assert {s.name for s in dest_exporter.get_finished_spans()} == self.NAMES
def test_two_views_of_one_exporter_each_draw_at_their_own_rate(self):
def test_two_views_of_one_exporter_each_answer_to_their_own_rate(self):
"""Two destinations to the same space with different rates share an exporter, not a
verdict: the 1.0 view gets the tree once and the 0.0 view adds nothing."""
dest_exporter = InMemorySpanExporter()
@ -1582,7 +1628,7 @@ class TestTenantSampling:
@pytest.mark.parametrize("bad", ["abc", "-0.1", "1.5", "nan"])
def test_an_unusable_rate_exports_rather_than_dropping(self, bad):
dest_exporter = InMemorySpanExporter()
provider = self._fan_out(dest_exporter, draw=lambda: 0.99)
provider = self._fan_out(dest_exporter, trace_id=DROPPED_AT_HALF)
self._run(provider, (arize_destination(arize_success_sampling_rate=bad),))