fix(otel): decide team sampling per request, not per trace

Calls sharing a caller's trace id are separate requests, keyed by their root span.
Each is decided by the trace hash at the rate for its own outcome, and only its own
late spans follow its verdict, so no pod applies an earlier call's verdict to a later
call of the trace
This commit is contained in:
yucheng-berri 2026-10-19 19:18:18 -07:00
parent debff6eed5
commit 9e8eb03817
3 changed files with 235 additions and 67 deletions

View file

@ -7,7 +7,7 @@ from collections import OrderedDict
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
from hashlib import sha256
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal
from typing import TYPE_CHECKING, Final, Literal, TypeAlias
from opentelemetry import _logs, baggage, metrics, trace
from opentelemetry._logs import Logger, LoggerProvider, NoOpLoggerProvider
@ -486,15 +486,16 @@ def is_llm_call_span(span: ReadableSpan) -> bool:
return GenAI.OPERATION_NAME in attributes and MCP.METHOD_NAME not in attributes
#: Request trees held back for a sampled destination until the last span of the trace
#: Request trees held back for a sampled destination until the last span of the request
#: still open ends, the verdicts kept for the spans that start after that (a post-call
#: database write), and the spans in flight per trace. All bounded, so a span that never
#: ends, or a flood of requests, cannot hold spans for ever: the oldest tree is decided
#: on what it has.
#: database write), the spans in flight per request, and the request each started span
#: belongs to. All bounded, so a span that never ends, or a flood of requests, cannot
#: hold spans for ever: the oldest tree is decided on what it has.
_MAX_PENDING_TREES: Final = 1024
_MAX_PENDING_SPANS_PER_TREE: Final = 512
_MAX_REMEMBERED_VERDICTS: Final = 4096
_MAX_OPEN_TRACES: Final = 16384
_MAX_OPEN_REQUESTS: Final = 16384
_MAX_REMEMBERED_ROOTS: Final = 16384
class _PendingTree:
@ -512,14 +513,19 @@ def _is_sampled(destination: "OtelDestination") -> bool:
return destination.success_sampling_rate is not None or destination.error_sampling_rate is not None
#: A verdict is one trace's keep or drop for one destination at its rates: the exporter
#: A request is its trace id and the id of its root, the span of it whose parent is
#: remote or missing. Calls sharing a caller's trace are separate requests, each decided
#: on its own outcome.
_RequestKey: TypeAlias = tuple[int, int]
#: A verdict is one request's keep or drop for one destination at its rates: the exporter
#: ``cache_key`` leaves the rates out, so two destinations sharing an exporter with
#: different rates would otherwise share one verdict.
_VerdictKey = tuple[int, object, float | None, float | None]
_VerdictKey: TypeAlias = tuple[int, int, object, float | None, float | None]
def _verdict_key(trace_id: int, destination: "OtelDestination") -> _VerdictKey:
return (trace_id, destination.cache_key(), destination.success_sampling_rate, destination.error_sampling_rate)
def _verdict_key(request: _RequestKey, destination: "OtelDestination") -> _VerdictKey:
return (*request, destination.cache_key(), destination.success_sampling_rate, destination.error_sampling_rate)
def _keeps_tree(destination: "OtelDestination", failed: bool, trace_id: int) -> bool:
@ -529,7 +535,7 @@ def _keeps_tree(destination: "OtelDestination", failed: bool, trace_id: int) ->
nothing.
The first eight bytes of a SHA-256 of the trace id are read against the rate, so
every process deciding a tree of the trace with the same outcome reaches the same
every process deciding a request of the trace with the same outcome reaches the same
verdict."""
rate: Final = destination.error_sampling_rate if failed else destination.success_sampling_rate
if rate is None:
@ -685,24 +691,29 @@ class TenantFanOutSpanProcessor(SpanProcessor):
self._processors: OrderedDict[object, SpanProcessor] = OrderedDict() # mutable-ok: bounded LRU
self._retired: OrderedDict[int, SpanProcessor] = OrderedDict() # mutable-ok: drains as exports finish
self._exporting: dict[int, int] = {} # mutable-ok: per-processor in-flight export count
self._pending: OrderedDict[int, _PendingTree] = OrderedDict() # mutable-ok: bounded, by trace id
self._open: OrderedDict[int, int] = OrderedDict() # mutable-ok: bounded, spans in flight by trace id
self._pending: OrderedDict[_RequestKey, _PendingTree] = OrderedDict() # mutable-ok: bounded, by request
self._open: OrderedDict[_RequestKey, int] = OrderedDict() # mutable-ok: bounded, spans in flight by request
self._roots: OrderedDict[int, int] = OrderedDict() # mutable-ok: bounded LRU, root span id by span id
self._verdicts: OrderedDict[_VerdictKey, bool] = OrderedDict() # mutable-ok: bounded LRU
self._drain: Final = drain_pool if drain_pool is not None else _DrainPool(capacity=pending_drains)
def on_start(self, span: SDKSpan, parent_context: Context | None = None) -> None:
context: Final = span.context
trace_id: Final = context.trace_id if context is not None else 0
with self._lock:
self._open[trace_id] = self._open.get(trace_id, 0) + 1
while len(self._open) > _MAX_OPEN_TRACES:
request: Final = self._request_of(span)
if context is not None:
self._roots[context.span_id] = request[1]
while len(self._roots) > _MAX_REMEMBERED_ROOTS:
self._roots.popitem(last=False)
self._open[request] = self._open.get(request, 0) + 1
while len(self._open) > _MAX_OPEN_REQUESTS:
self._open.popitem(last=False)
def on_end(self, span: ReadableSpan) -> None:
suppressed: Final = suppressed_backends()
attributes: Final = span.attributes or _NO_ATTRIBUTES
context: Final = span.context
trace_id: Final = context.trace_id if context is not None else 0
with self._lock:
request: Final = self._request_of(span)
failed: Final = span.status.status_code is StatusCode.ERROR
for destination in request_destinations():
if (
@ -719,11 +730,27 @@ class TenantFanOutSpanProcessor(SpanProcessor):
if not _is_sampled(destination):
self._forward(span, destination)
continue
for held_span, held_destination in self._route(trace_id, span, destination, failed):
for held_span, held_destination in self._route(request, span, destination, failed):
self._forward(held_span, held_destination)
for held_span, held_destination in self._settle(trace_id, failed):
for held_span, held_destination in self._settle(request, failed):
self._forward(held_span, held_destination)
def _request_of(self, span: ReadableSpan) -> _RequestKey:
"""The request ``span`` belongs to, read under ``_lock``: the root recorded when it
started, else the span itself when its parent is remote or missing, else its
parent's root, or the parent itself once that record is gone, which is right for
a late child of the root."""
context: Final = span.context
if context is None:
return (0, 0)
recorded: Final = self._roots.get(context.span_id)
if recorded is not None:
return (context.trace_id, recorded)
parent: Final = span.parent
if parent is None or parent.is_remote:
return (context.trace_id, context.span_id)
return (context.trace_id, self._roots.get(parent.span_id, parent.span_id))
def _default_capture_for(self, destination: "OtelDestination") -> str:
backend: Final = destination.callback_name
return (
@ -746,11 +773,11 @@ class TenantFanOutSpanProcessor(SpanProcessor):
self._release(processor)
def _route(
self, trace_id: int, span: ReadableSpan, destination: "OtelDestination", failed: bool
self, request: _RequestKey, span: ReadableSpan, destination: "OtelDestination", failed: bool
) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]":
"""The spans to forward now for ``span`` on a sampled ``destination``: the span
itself when its tree's verdict is already known and keeps it, nothing while the
tree is held for the rest of its trace, or what the bounds made the fan-out decide
tree is held for the rest of its request, or what the bounds made the fan-out decide
early (a tree that outgrew its cap is decided on what it has, and so is the oldest
tree once too many are waiting, so a span that never ends holds nothing back for
ever). A failed span marks its tree before any such early decision. Once shutdown
@ -761,62 +788,58 @@ class TenantFanOutSpanProcessor(SpanProcessor):
another thread between the two would leave this span in a fresh tree.
"""
with self._lock:
verdict: Final = self._verdicts.get(_verdict_key(trace_id, destination))
verdict: Final = self._verdicts.get(_verdict_key(request, destination))
if verdict is not None:
return ((span, destination),) if verdict else ()
tree: Final = self._pending.setdefault(trace_id, _PendingTree())
tree: Final = self._pending.setdefault(request, _PendingTree())
tree.held.append((span, destination))
if failed:
tree.failed = True
if self._draining or len(tree.held) >= _MAX_PENDING_SPANS_PER_TREE:
return self._decide_locked(trace_id)
return self._decide_locked(request)
if len(self._pending) > _MAX_PENDING_TREES:
oldest: Final = next(iter(self._pending))
return self._decide_locked(oldest)
return ()
def _decide(self, trace_id: int) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]":
def _decide(self, request: _RequestKey) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]":
with self._lock:
return self._decide_locked(trace_id)
return self._decide_locked(request)
def _settle(self, trace_id: int, failed: bool) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]":
"""What the trace's held tree exports once the span that just ended was the last
of its trace still open. The model-call span ends in the post-call callback,
after the server span, so a tree is decided when its trace goes quiet rather than
def _settle(self, request: _RequestKey, failed: bool) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]":
"""What the request's held tree exports once the span that just ended was the last
of its request still open. The model-call span ends in the post-call callback,
after the server span, so a tree is decided when its request goes quiet rather than
when its root ends, and a model call that fails late still answers to the error
rate. A trace whose starts this fan-out never saw (attached mid-flight, or forgotten
by the bound) is decided as each of its spans ends.
rate. A request whose starts this fan-out never saw (attached mid-flight, or
forgotten by the bound) is decided as each of its spans ends.
"""
with self._lock:
tree: Final = self._pending.get(trace_id)
tree: Final = self._pending.get(request)
if tree is not None and failed:
tree.failed = True
still_open: Final = self._open.pop(trace_id, 1) - 1
still_open: Final = self._open.pop(request, 1) - 1
if still_open > 0:
self._open[trace_id] = still_open
self._open[request] = still_open
return ()
return self._decide_locked(trace_id)
return self._decide_locked(request)
def _decide_locked(self, trace_id: int) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]":
"""Decide once per destination for the held tree; the spans the verdicts keep. A
failed tree's verdicts are remembered for the spans of its trace that start later,
such as a post-call database write that would otherwise answer to the success rate.
A successful tree's are not, so a later failed call of the trace answers to the
error rate here as it would on a process that never saw the trace."""
tree: Final = self._pending.pop(trace_id, None)
def _decide_locked(self, request: _RequestKey) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]":
"""Decide once per destination for the held tree; the spans the verdicts keep, and
a verdict remembered for the spans of this request that start later."""
tree: Final = self._pending.pop(request, None)
if tree is None:
return ()
verdicts: dict[_VerdictKey, bool] = {} # mutable-ok: one verdict per destination of this tree
for _, destination in tree.held:
key = _verdict_key(trace_id, destination)
key = _verdict_key(request, destination)
if key not in verdicts:
verdicts[key] = _keeps_tree(destination, tree.failed, trace_id)
if tree.failed:
self._verdicts.update(verdicts)
verdicts[key] = _keeps_tree(destination, tree.failed, request[0])
self._verdicts[key] = verdicts[key]
while len(self._verdicts) > _MAX_REMEMBERED_VERDICTS:
self._verdicts.popitem(last=False)
return tuple(
(span, destination) for span, destination in tree.held if verdicts[_verdict_key(trace_id, destination)]
(span, destination) for span, destination in tree.held if verdicts[_verdict_key(request, destination)]
)
def _operator_already_writes(
@ -857,8 +880,8 @@ class TenantFanOutSpanProcessor(SpanProcessor):
with self._lock:
self._draining = True
undecided: Final = tuple(self._pending)
for trace_id in undecided:
for held_span, held_destination in self._decide(trace_id):
for request in undecided:
for held_span, held_destination in self._decide(request):
self._forward(held_span, held_destination)
deadline: Final = time.monotonic() + self._drain_seconds
with self._lock:

View file

@ -30,7 +30,7 @@ from integration._support.client import (
until_asserted,
)
from integration._support.database import scratch_database
from integration._support.otlp_sink import Span, SpanSinks, recorded_requests, recorded_spans
from integration._support.otlp_sink import Span, SpanSinks, configure_sink, recorded_requests, recorded_spans
from integration._support.process import OwnedProxy, WorkerHealthcheck, owned_proxy_process
from integration._support.wire import Reply, Request, Wire, wire_server
from pydantic import JsonValue, TypeAdapter
@ -477,6 +477,47 @@ def test_a_failed_call_answers_to_the_error_rate_on_a_pod_that_dropped_its_trace
assert any(not kept[0] for kept in verdicts.values()), "no successful call was dropped before its trace failed"
def test_a_successful_call_answers_to_the_success_rate_on_a_pod_that_kept_its_failed_call(pods: Pods) -> None:
"""A pod that already kept a failed call of the trace drops a later successful call of it at a zero
success rate, as the pod that only served the successful call does."""
sampled: Final = pods.team_key({**CREDENTIALS, "arize_success_sampling_rate": "0.0"})
cursors: Final = _cursors(pods)
traces: Final = _trace_ids("succeeded-after-kept")
failed: Final = {
trace_id: pods.send(pods.first, _call(sampled, trace_id, content="sampling-fail")) for trace_id in traces
}
_await_logged(pods, cursors.operator, tuple(failed.values()))
by_trace: Final = {
trace_id: (
failed[trace_id],
pods.send(pods.first, _call(sampled, trace_id)),
pods.send(pods.second, _call(sampled, trace_id)),
)
for trace_id in traces
}
_settled(pods, cursors, by_trace, pods.team_key(CREDENTIALS), holds=lambda kept: kept == (True, False, False))
def test_concurrent_calls_of_one_trace_on_one_worker_each_answer_to_their_own_outcome(pods: Pods) -> None:
"""A failed and a successful call of the trace in flight at once on the single worker of the first pod
are decided apart: the failed one is kept at an unset error rate, the successful one dropped at a zero
success rate."""
sampled: Final = pods.team_key({**CREDENTIALS, "arize_success_sampling_rate": "0.0"})
cursors: Final = _cursors(pods)
traces: Final = _trace_ids("concurrent-mixed")
plan: Final = tuple(product(traces, ("sampling-fail", "hello")))
def send(job: tuple[str, str]) -> Sent:
return pods.send(pods.first, _call(sampled, job[0], content=job[1]))
with ThreadPoolExecutor(max_workers=8) as pool:
sent: Final = tuple(pool.map(send, plan))
by_trace: Final = {trace_id: tuple(call for call in sent if call.trace_id == trace_id) for trace_id in traces}
_settled(pods, cursors, by_trace, pods.team_key(CREDENTIALS), holds=lambda kept: kept == (True, False))
@pytest.mark.parametrize(
("variables", "expected"),
[({"arize_success_sampling_rate": "1.0"}, TRACES), ({"arize_success_sampling_rate": "0.0"}, 0), ({}, TRACES)],
@ -577,6 +618,30 @@ def test_a_concurrent_burst_keeps_every_trace_whole(pods: Pods) -> None:
assert 0 < _kept(verdicts) < len(traces)
def test_a_stalled_team_space_during_a_burst_keeps_serving_and_each_trace_whole(pods: Pods) -> None:
"""The team's Arize endpoint stops answering while a burst of calls runs over both pods: every call is
still served, and once the endpoint answers again each trace arrives whole or not at all."""
sampled: Final = pods.team_key({**CREDENTIALS, "arize_success_sampling_rate": "0.5"})
cursors: Final = _cursors(pods)
traces: Final = _trace_ids("stalled")
plan: Final = tuple(product(traces, (pods.first, pods.second)))
def send(job: tuple[str, OwnedProxy]) -> Sent:
return pods.send(job[1], _call(sampled, job[0]))
configure_sink(pods.sinks.arize, paused=True)
try:
with ThreadPoolExecutor(max_workers=8) as pool:
sent: Final = tuple(pool.map(send, plan))
finally:
configure_sink(pods.sinks.arize, paused=False)
by_trace: Final = {trace_id: tuple(call for call in sent if call.trace_id == trace_id) for trace_id in traces}
verdicts: Final = _settled(pods, cursors, by_trace, pods.team_key(CREDENTIALS))
assert 0 < _kept(verdicts) < len(traces)
def _write_config(directory: Path, sinks: SpanSinks, provider: Wire) -> Path:
config: Final = object_value(
JSON.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))

View file

@ -1321,7 +1321,7 @@ class TestTenantSampling:
def test_a_span_that_ends_after_the_root_follows_the_requests_verdict(self):
"""A post-call database write that starts after the whole request tree has ended
goes where the rest of the tree went."""
goes where the rest of the tree went, without a second verdict."""
kept_exporter, dropped_exporter = InMemorySpanExporter(), InMemorySpanExporter()
kept = self._fan_out(kept_exporter, trace_id=KEPT_AT_HALF)
dropped = self._fan_out(dropped_exporter, trace_id=DROPPED_AT_HALF)
@ -1386,24 +1386,104 @@ class TestTenantSampling:
sorted(self.NAMES)
] * 2
def test_a_late_span_whose_verdict_was_forgotten_is_decided_on_its_own_not_stranded(self):
"""Once enough other failed requests have been decided to evict a failed request's
verdict, a span of it that starts late (a post-call database write) opens and closes
its own count, so it is decided on its own as soon as it ends instead of waiting in a
tree nothing would ever close."""
def test_a_successful_call_after_a_kept_failed_call_of_its_trace_answers_to_the_success_rate(self):
"""A process that kept a failed call of the trace drops a later successful call of it
at a zero success rate, as a process that only served the successful call does."""
served_both, served_success = InMemorySpanExporter(), InMemorySpanExporter()
destinations = (arize_destination(arize_success_sampling_rate="0.0", arize_error_sampling_rate="1.0"),)
both = self._fan_out(served_both, trace_id=KEPT_AT_HALF)
self._run(both, destinations, failed=True)
self._run(both, destinations)
self._run(self._fan_out(served_success, trace_id=KEPT_AT_HALF), destinations)
assert len(served_both.get_finished_spans()) == len(self.NAMES), "only the failed call"
assert all(s.name in self.NAMES for s in served_both.get_finished_spans())
assert served_success.get_finished_spans() == ()
def test_a_late_child_that_fails_after_its_request_was_dropped_is_dropped_with_it(self):
"""A post-call write that fails after its successful request was dropped goes where
the request went, rather than reaching the space on its own at the error rate."""
dest_exporter = InMemorySpanExporter()
provider = self._fan_out(dest_exporter)
destinations = (arize_destination(arize_success_sampling_rate="1.0", arize_error_sampling_rate="1.0"),)
destinations = (arize_destination(arize_success_sampling_rate="0.0", arize_error_sampling_rate="1.0"),)
def run():
set_request_destinations(destinations)
tracer = get_tracer(provider, "litellm")
with tracer.start_as_current_span("POST /v1/chat/completions") as root:
root_context = trace_api.set_span_in_context(root)
with tracer.start_as_current_span("postgres INSERT LiteLLM_SpendLogs", context=root_context) as write:
write.set_status(Status(StatusCode.ERROR, "connection reset"))
in_fresh_context(run)
assert dest_exporter.get_finished_spans() == ()
def test_a_failure_deep_in_a_request_marks_the_whole_request(self):
"""A database span that fails under the auth span puts the whole request, root
included, on the error rate."""
dest_exporter = InMemorySpanExporter()
provider = self._fan_out(dest_exporter)
destinations = (arize_destination(arize_success_sampling_rate="0.0", arize_error_sampling_rate="1.0"),)
def run():
set_request_destinations(destinations)
tracer = get_tracer(provider, "litellm")
with tracer.start_as_current_span("POST /v1/chat/completions"):
with tracer.start_as_current_span("auth /v1/chat/completions"):
with tracer.start_as_current_span("postgres SELECT LiteLLM_VerificationToken") as lookup:
lookup.set_status(Status(StatusCode.ERROR, "connection reset"))
with tracer.start_as_current_span("chat gpt-4"):
pass
in_fresh_context(run)
assert {s.name for s in dest_exporter.get_finished_spans()} == {
*self.NAMES,
"postgres SELECT LiteLLM_VerificationToken",
}
def test_concurrent_calls_of_one_trace_each_answer_to_their_own_outcome(self):
"""Two calls of the caller's trace in flight at once on one process, one failing,
are decided as two processes serving one each would decide them."""
dest_exporter = InMemorySpanExporter()
provider = self._fan_out(dest_exporter)
destinations = (arize_destination(arize_success_sampling_rate="0.0", arize_error_sampling_rate="1.0"),)
remote = trace_api.SpanContext(
trace_id=KEPT_AT_HALF, span_id=1, is_remote=True, trace_flags=trace_api.TraceFlags(0x01)
)
def run():
set_request_destinations(destinations)
tracer = get_tracer(provider, "litellm")
caller = trace_api.set_span_in_context(trace_api.NonRecordingSpan(remote))
with tracer.start_as_current_span("POST failed", context=caller) as failed:
with tracer.start_as_current_span("POST succeeded", context=caller):
pass
failed.set_status(Status(StatusCode.ERROR, "upstream 500"))
in_fresh_context(run)
assert [s.name for s in dest_exporter.get_finished_spans()] == ["POST failed"]
def test_a_late_span_whose_verdict_was_forgotten_is_decided_on_its_own_not_stranded(self):
"""Once enough other requests have been decided to evict a request's verdict, a span
of it that starts late (a post-call database write) opens and closes its own count,
so it is decided on its own as soon as it ends instead of waiting in a tree nothing
would ever close."""
dest_exporter = InMemorySpanExporter()
provider = self._fan_out(dest_exporter)
destinations = (arize_destination(arize_success_sampling_rate="1.0"),)
def run():
set_request_destinations(destinations)
tracer = get_tracer(provider, "litellm")
with tracer.start_as_current_span("POST /v1/chat/completions") as root:
root.set_status(Status(StatusCode.ERROR, "upstream 500"))
root_context = trace_api.set_span_in_context(root)
for index in range(otel_providers._MAX_REMEMBERED_VERDICTS):
with tracer.start_as_current_span(f"POST {index}", context=trace_api.Context()) as other:
other.set_status(Status(StatusCode.ERROR, "upstream 500"))
with tracer.start_as_current_span(f"POST {index}", context=trace_api.Context()):
pass
with tracer.start_as_current_span("postgres INSERT LiteLLM_SpendLogs", context=root_context):
pass
@ -1472,10 +1552,10 @@ class TestTenantSampling:
assert [s.name for s in dest_exporter.get_finished_spans()] == ["chat 0"]
def test_a_flood_of_open_traces_forgets_the_oldest_and_decides_it_as_its_spans_end(self):
"""The fan-out counts open spans for a bounded number of traces, a bound far above
what one process keeps in flight, since every trace of the provider is counted, not
only the sampled ones. A trace pushed out of that count is decided on what it has as
def test_a_flood_of_open_requests_forgets_the_oldest_and_decides_it_as_its_spans_end(self):
"""The fan-out counts open spans for a bounded number of requests, a bound far above
what one process keeps in flight, since every request of the provider is counted, not
only the sampled ones. A request pushed out of that count is decided on what it has as
its next span ends, root still open, rather than held until a bound gets to it."""
dest_exporter = InMemorySpanExporter()
provider = self._fan_out(dest_exporter)
@ -1486,7 +1566,7 @@ class TestTenantSampling:
tracer = get_tracer(provider, "litellm")
root = tracer.start_span("POST /v1/chat/completions", context=trace_api.Context())
tracer.start_span("auth /v1/chat/completions", context=trace_api.set_span_in_context(root)).end()
for index in range(otel_providers._MAX_OPEN_TRACES):
for index in range(otel_providers._MAX_OPEN_REQUESTS):
tracer.start_span(f"POST {index}", context=trace_api.Context())
assert dest_exporter.get_finished_spans() == (), "still held before the count is forgotten"
tracer.start_span("chat gpt-4", context=trace_api.set_span_in_context(root)).end()