fix(otel): forget a span's request root when it ends, not when newer spans start

The root of each open span was kept in a first-in-first-out table of 16384 entries,
so a span two levels below the root that stayed open while the worker started more
spans than that lost its entry and ended under its parent's id. Its request then
never closed and was held from every sampled destination until shutdown.

Each open span now keeps its entry until it ends, and a span starting under a parent
that has already ended reads the root from the parent span itself
This commit is contained in:
yucheng-berri 2026-10-20 12:40:16 -07:00
parent 9e8eb03817
commit f5bfb292e6
2 changed files with 80 additions and 11 deletions

View file

@ -3,6 +3,7 @@
import queue
import threading
import time
import weakref
from collections import OrderedDict
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
from hashlib import sha256
@ -488,14 +489,14 @@ def is_llm_call_span(span: ReadableSpan) -> bool:
#: Request trees held back for a sampled destination until the last span of the request
#: still open ends, the verdicts kept for the spans that start after that (a post-call
#: database write), the spans in flight per request, and the request each started span
#: database write), the spans in flight per request, and the request each open span
#: belongs to. All bounded, so a span that never ends, or a flood of requests, cannot
#: hold spans for ever: the oldest tree is decided on what it has.
_MAX_PENDING_TREES: Final = 1024
_MAX_PENDING_SPANS_PER_TREE: Final = 512
_MAX_REMEMBERED_VERDICTS: Final = 4096
_MAX_OPEN_REQUESTS: Final = 16384
_MAX_REMEMBERED_ROOTS: Final = 16384
_MAX_OPEN_SPANS: Final = 16384
class _PendingTree:
@ -693,17 +694,19 @@ class TenantFanOutSpanProcessor(SpanProcessor):
self._exporting: dict[int, int] = {} # mutable-ok: per-processor in-flight export count
self._pending: OrderedDict[_RequestKey, _PendingTree] = OrderedDict() # mutable-ok: bounded, by request
self._open: OrderedDict[_RequestKey, int] = OrderedDict() # mutable-ok: bounded, spans in flight by request
self._roots: OrderedDict[int, int] = OrderedDict() # mutable-ok: bounded LRU, root span id by span id
self._roots: OrderedDict[int, int] = OrderedDict() # mutable-ok: bounded, root span id by open span id
self._span_roots: weakref.WeakKeyDictionary[SDKSpan, int] = weakref.WeakKeyDictionary()
self._verdicts: OrderedDict[_VerdictKey, bool] = OrderedDict() # mutable-ok: bounded LRU
self._drain: Final = drain_pool if drain_pool is not None else _DrainPool(capacity=pending_drains)
def on_start(self, span: SDKSpan, parent_context: Context | None = None) -> None:
context: Final = span.context
with self._lock:
request: Final = self._request_of(span)
request: Final = self._request_of(span, trace.get_current_span(parent_context))
if context is not None:
self._span_roots[span] = request[1]
self._roots[context.span_id] = request[1]
while len(self._roots) > _MAX_REMEMBERED_ROOTS:
while len(self._roots) > _MAX_OPEN_SPANS:
self._roots.popitem(last=False)
self._open[request] = self._open.get(request, 0) + 1
while len(self._open) > _MAX_OPEN_REQUESTS:
@ -714,6 +717,8 @@ class TenantFanOutSpanProcessor(SpanProcessor):
attributes: Final = span.attributes or _NO_ATTRIBUTES
with self._lock:
request: Final = self._request_of(span)
if span.context is not None:
self._roots.pop(span.context.span_id, None)
failed: Final = span.status.status_code is StatusCode.ERROR
for destination in request_destinations():
if (
@ -735,11 +740,11 @@ class TenantFanOutSpanProcessor(SpanProcessor):
for held_span, held_destination in self._settle(request, failed):
self._forward(held_span, held_destination)
def _request_of(self, span: ReadableSpan) -> _RequestKey:
"""The request ``span`` belongs to, read under ``_lock``: the root recorded when it
started, else the span itself when its parent is remote or missing, else its
parent's root, or the parent itself once that record is gone, which is right for
a late child of the root."""
def _request_of(self, span: ReadableSpan, parent_span: Span | None = None) -> _RequestKey:
"""The request ``span`` belongs to, read under ``_lock``: the root recorded while it
is open, else the span itself when its parent is remote or missing, else its
parent's root, recorded while the parent is open or on ``parent_span`` for a child
starting under a parent that has ended, else the parent itself."""
context: Final = span.context
if context is None:
return (0, 0)
@ -749,7 +754,12 @@ class TenantFanOutSpanProcessor(SpanProcessor):
parent: Final = span.parent
if parent is None or parent.is_remote:
return (context.trace_id, context.span_id)
return (context.trace_id, self._roots.get(parent.span_id, parent.span_id))
root: Final = self._roots.get(parent.span_id)
if root is not None:
return (context.trace_id, root)
if isinstance(parent_span, SDKSpan):
return (context.trace_id, self._span_roots.get(parent_span, parent.span_id))
return (context.trace_id, parent.span_id)
def _default_capture_for(self, destination: "OtelDestination") -> str:
backend: Final = destination.callback_name

View file

@ -1444,6 +1444,65 @@ class TestTenantSampling:
"postgres SELECT LiteLLM_VerificationToken",
}
def test_a_span_two_levels_down_that_outlives_many_other_spans_still_closes_its_request(self):
"""A classifier call under the route span that stays open while the worker starts
more spans than it tracks at once still counts against its own request, so the
request reaches the space when it ends rather than being held."""
dest_exporter = InMemorySpanExporter()
provider = self._fan_out(dest_exporter)
destinations = (arize_destination(arize_success_sampling_rate="1.0"),)
def other_request():
set_request_destinations(())
with get_tracer(provider, "litellm").start_as_current_span("POST /v1/chat/completions"):
pass
def run():
set_request_destinations(destinations)
tracer = get_tracer(provider, "litellm")
with tracer.start_as_current_span("POST /v1/chat/completions"):
with tracer.start_as_current_span("route gpt-4"):
with tracer.start_as_current_span("chat classifier"):
for _ in range(otel_providers._MAX_OPEN_SPANS + 1):
contextvars.Context().run(other_request)
with tracer.start_as_current_span("chat gpt-4"):
pass
in_fresh_context(run)
assert {s.name for s in dest_exporter.get_finished_spans()} == {
"POST /v1/chat/completions",
"route gpt-4",
"chat classifier",
"chat gpt-4",
}
def test_a_span_starting_under_an_ended_span_below_the_root_follows_its_request(self):
"""A post-call write started under the route span after its failed request ended
is kept with the request at the error rate, rather than decided on its own at the
success rate."""
dest_exporter = InMemorySpanExporter()
provider = self._fan_out(dest_exporter)
destinations = (arize_destination(arize_success_sampling_rate="0.0", arize_error_sampling_rate="1.0"),)
def run():
set_request_destinations(destinations)
tracer = get_tracer(provider, "litellm")
with tracer.start_as_current_span("POST /v1/chat/completions") as root:
with tracer.start_as_current_span("route gpt-4") as route:
route_context = trace_api.set_span_in_context(route)
root.set_status(Status(StatusCode.ERROR, "upstream 500"))
with tracer.start_as_current_span("postgres INSERT LiteLLM_SpendLogs", context=route_context):
pass
in_fresh_context(run)
assert {s.name for s in dest_exporter.get_finished_spans()} == {
"POST /v1/chat/completions",
"route gpt-4",
"postgres INSERT LiteLLM_SpendLogs",
}
def test_concurrent_calls_of_one_trace_each_answer_to_their_own_outcome(self):
"""Two calls of the caller's trace in flight at once on one process, one failing,
are decided as two processes serving one each would decide them."""