fix(otel): keep each open span's request until it ends, bounded by the open requests

The span table had its own bound of 16384, so a worker holding more spans open than
that pushed out the oldest, and a failure in such a span no longer reached its
request. Each request now holds the ids of its open spans, and a span's entry in the
lookup goes when the span ends or when its request is pushed out of the open request
table, the same bound main already applies. A failure anywhere in a tracked request
reaches its verdict however many spans are open
This commit is contained in:
yucheng-berri 2026-10-20 16:24:16 -07:00
parent 23f1f9b1d1
commit 368ea7bdac
2 changed files with 57 additions and 22 deletions

View file

@ -496,7 +496,6 @@ _MAX_PENDING_TREES: Final = 1024
_MAX_PENDING_SPANS_PER_TREE: Final = 512
_MAX_REMEMBERED_VERDICTS: Final = 4096
_MAX_OPEN_REQUESTS: Final = 16384
_MAX_OPEN_SPANS: Final = 16384
class _PendingTree:
@ -693,8 +692,8 @@ class TenantFanOutSpanProcessor(SpanProcessor):
self._retired: OrderedDict[int, SpanProcessor] = OrderedDict() # mutable-ok: drains as exports finish
self._exporting: dict[int, int] = {} # mutable-ok: per-processor in-flight export count
self._pending: OrderedDict[_RequestKey, _PendingTree] = OrderedDict() # mutable-ok: bounded, by request
self._open: OrderedDict[_RequestKey, int] = OrderedDict() # mutable-ok: bounded, spans in flight by request
self._roots: OrderedDict[int, _RequestKey] = OrderedDict() # mutable-ok: bounded, request by open span id
self._open: OrderedDict[_RequestKey, set[int]] = OrderedDict() # mutable-ok: bounded, open span ids by request
self._roots: dict[int, _RequestKey] = {} # mutable-ok: holds only the spans ``_open`` holds, request by span id
self._span_roots: weakref.WeakKeyDictionary[SDKSpan, int] = weakref.WeakKeyDictionary()
self._verdicts: OrderedDict[_VerdictKey, bool] = OrderedDict() # mutable-ok: bounded LRU
self._drain: Final = drain_pool if drain_pool is not None else _DrainPool(capacity=pending_drains)
@ -706,11 +705,10 @@ class TenantFanOutSpanProcessor(SpanProcessor):
if context is not None:
self._span_roots[span] = request[1]
self._roots[context.span_id] = request
while len(self._roots) > _MAX_OPEN_SPANS:
self._open.pop(self._roots.popitem(last=False)[1], None)
self._open[request] = self._open.get(request, 0) + 1
self._open.setdefault(request, set()).add(context.span_id)
while len(self._open) > _MAX_OPEN_REQUESTS:
self._open.popitem(last=False)
for forgotten in self._open.popitem(last=False)[1]:
self._roots.pop(forgotten, None)
def on_end(self, span: ReadableSpan) -> None:
suppressed: Final = suppressed_backends()
@ -737,7 +735,7 @@ class TenantFanOutSpanProcessor(SpanProcessor):
continue
for held_span, held_destination in self._route(request, span, destination, failed):
self._forward(held_span, held_destination)
for held_span, held_destination in self._settle(request, failed):
for held_span, held_destination in self._settle(request, span, failed):
self._forward(held_span, held_destination)
def _request_of(self, span: ReadableSpan, parent_span: Span | None = None) -> _RequestKey:
@ -816,7 +814,9 @@ class TenantFanOutSpanProcessor(SpanProcessor):
with self._lock:
return self._decide_locked(request)
def _settle(self, request: _RequestKey, failed: bool) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]":
def _settle(
self, request: _RequestKey, span: ReadableSpan, failed: bool
) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]":
"""What the request's held tree exports once the span that just ended was the last
of its request still open. The model-call span ends in the post-call callback,
after the server span, so a tree is decided when its request goes quiet rather than
@ -828,8 +828,10 @@ class TenantFanOutSpanProcessor(SpanProcessor):
tree: Final = self._pending.get(request)
if tree is not None and failed:
tree.failed = True
still_open: Final = self._open.pop(request, 1) - 1
if still_open > 0:
still_open: Final = self._open.pop(request, set())
if span.context is not None:
still_open.discard(span.context.span_id)
if still_open:
self._open[request] = still_open
return ()
return self._decide_locked(request)

View file

@ -1454,8 +1454,8 @@ class TestTenantSampling:
}
def test_a_span_two_levels_down_that_outlives_many_other_spans_still_closes_its_request(self):
"""A classifier call under the route span that stays open while the worker starts
more spans than it tracks at once still counts against its own request, so the
"""A classifier call under the route span that stays open while the worker serves
more requests than it tracks at once still counts against its own request, so the
request is decided once it ends, on its own outcome."""
dest_exporter: Final = InMemorySpanExporter()
provider: Final = self._fan_out(dest_exporter)
@ -1472,7 +1472,7 @@ class TestTenantSampling:
with tracer.start_as_current_span("POST /v1/chat/completions") as root:
with tracer.start_as_current_span("route gpt-4"):
with tracer.start_as_current_span("chat classifier"):
for _ in range(otel_providers._MAX_OPEN_SPANS + 1):
for _ in range(otel_providers._MAX_OPEN_REQUESTS + 1):
contextvars.Context().run(other_request)
with tracer.start_as_current_span("chat gpt-4"):
pass
@ -1487,13 +1487,13 @@ class TestTenantSampling:
"chat gpt-4",
}
def test_a_request_whose_spans_fall_out_of_the_open_span_bound_is_still_decided(self):
"""A request still open while the worker holds more open spans than it tracks is
decided as each of its spans ends, rather than held for want of a span it can no
longer place."""
def test_a_failure_two_levels_down_reaches_its_request_however_many_spans_are_open(self):
"""A classifier call that fails while another request holds more spans open than
the worker tracks requests marks its own request, which is then kept at the error
rate as a whole."""
dest_exporter: Final = InMemorySpanExporter()
provider: Final = self._fan_out(dest_exporter)
destinations: Final = (arize_destination(arize_success_sampling_rate="1.0"),)
destinations: Final = (arize_destination(arize_success_sampling_rate="0.0", arize_error_sampling_rate="1.0"),)
others: Final = contextvars.Context()
def open_other_request():
@ -1503,7 +1503,7 @@ class TestTenantSampling:
parent: Final = trace_api.set_span_in_context(root)
return (
root,
*(tracer.start_span("chat flood", context=parent) for _ in range(otel_providers._MAX_OPEN_SPANS)),
*(tracer.start_span("chat flood", context=parent) for _ in range(otel_providers._MAX_OPEN_REQUESTS)),
)
def run():
@ -1511,8 +1511,10 @@ class TestTenantSampling:
tracer: Final = get_tracer(provider, "litellm")
with tracer.start_as_current_span("POST /v1/chat/completions"):
with tracer.start_as_current_span("route gpt-4"):
with tracer.start_as_current_span("chat classifier"):
return others.run(open_other_request)
with tracer.start_as_current_span("chat classifier") as classifier:
held_open: Final = others.run(open_other_request)
classifier.set_status(Status(StatusCode.ERROR, "classifier timed out"))
return held_open
held_open: Final = in_fresh_context(run)
@ -1524,6 +1526,37 @@ class TestTenantSampling:
for span in held_open:
span.end()
def test_the_fan_out_forgets_a_span_once_it_ends_and_a_request_once_it_is_pushed_out(self):
"""The request lookup holds only spans still open, and only those of the requests
the fan-out still counts, so neither ended spans nor a flood of open requests grow
it past the open request bound."""
fan_out: Final = TenantFanOutSpanProcessor(
processor_factory=lambda _d: SimpleSpanProcessor(InMemorySpanExporter())
)
provider: Final = TracerProvider()
provider.add_span_processor(fan_out)
def run():
set_request_destinations((arize_destination(arize_success_sampling_rate="1.0"),))
tracer: Final = get_tracer(provider, "litellm")
with tracer.start_as_current_span("POST /v1/chat/completions"):
with tracer.start_as_current_span("chat gpt-4"):
pass
assert (fan_out._roots, dict(fan_out._open)) == ({}, {})
root: Final = tracer.start_span("POST /v1/chat/completions", context=trace_api.Context())
tracer.start_span("route gpt-4", context=trace_api.set_span_in_context(root))
return tuple(
tracer.start_span(f"POST {index}", context=trace_api.Context())
for index in range(otel_providers._MAX_OPEN_REQUESTS)
)
held_open: Final = in_fresh_context(run)
assert len(fan_out._roots) == otel_providers._MAX_OPEN_REQUESTS
for span in held_open:
span.end()
assert (fan_out._roots, dict(fan_out._open)) == ({}, {})
def test_a_span_starting_under_an_ended_span_below_the_root_follows_its_request(self):
"""A post-call write started under the route span after its failed request ended
is kept with the request at the error rate, rather than decided on its own at the