From 368ea7bdace1aead9a6c994749b45f2b009b854d Mon Sep 17 00:00:00 2001 From: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 16:24:16 -0700 Subject: [PATCH] fix(otel): keep each open span's request until it ends, bounded by the open requests The span table had its own bound of 16384, so a worker holding more spans open than that pushed out the oldest, and a failure in such a span no longer reached its request. Each request now holds the ids of its open spans, and a span's entry in the lookup goes when the span ends or when its request is pushed out of the open request table, the same bound main already applies. A failure anywhere in a tracked request reaches its verdict however many spans are open --- .../integrations/otel/plumbing/providers.py | 24 ++++---- .../otel/test_otel_v2_destinations.py | 55 +++++++++++++++---- 2 files changed, 57 insertions(+), 22 deletions(-) diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index b0a017c6c9b..fc1e9b7099d 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -496,7 +496,6 @@ _MAX_PENDING_TREES: Final = 1024 _MAX_PENDING_SPANS_PER_TREE: Final = 512 _MAX_REMEMBERED_VERDICTS: Final = 4096 _MAX_OPEN_REQUESTS: Final = 16384 -_MAX_OPEN_SPANS: Final = 16384 class _PendingTree: @@ -693,8 +692,8 @@ class TenantFanOutSpanProcessor(SpanProcessor): self._retired: OrderedDict[int, SpanProcessor] = OrderedDict() # mutable-ok: drains as exports finish self._exporting: dict[int, int] = {} # mutable-ok: per-processor in-flight export count self._pending: OrderedDict[_RequestKey, _PendingTree] = OrderedDict() # mutable-ok: bounded, by request - self._open: OrderedDict[_RequestKey, int] = OrderedDict() # mutable-ok: bounded, spans in flight by request - self._roots: OrderedDict[int, _RequestKey] = OrderedDict() # mutable-ok: bounded, request by open span id + self._open: OrderedDict[_RequestKey, set[int]] = OrderedDict() # mutable-ok: bounded, open span ids by request + self._roots: dict[int, _RequestKey] = {} # mutable-ok: holds only the spans ``_open`` holds, request by span id self._span_roots: weakref.WeakKeyDictionary[SDKSpan, int] = weakref.WeakKeyDictionary() self._verdicts: OrderedDict[_VerdictKey, bool] = OrderedDict() # mutable-ok: bounded LRU self._drain: Final = drain_pool if drain_pool is not None else _DrainPool(capacity=pending_drains) @@ -706,11 +705,10 @@ class TenantFanOutSpanProcessor(SpanProcessor): if context is not None: self._span_roots[span] = request[1] self._roots[context.span_id] = request - while len(self._roots) > _MAX_OPEN_SPANS: - self._open.pop(self._roots.popitem(last=False)[1], None) - self._open[request] = self._open.get(request, 0) + 1 + self._open.setdefault(request, set()).add(context.span_id) while len(self._open) > _MAX_OPEN_REQUESTS: - self._open.popitem(last=False) + for forgotten in self._open.popitem(last=False)[1]: + self._roots.pop(forgotten, None) def on_end(self, span: ReadableSpan) -> None: suppressed: Final = suppressed_backends() @@ -737,7 +735,7 @@ class TenantFanOutSpanProcessor(SpanProcessor): continue for held_span, held_destination in self._route(request, span, destination, failed): self._forward(held_span, held_destination) - for held_span, held_destination in self._settle(request, failed): + for held_span, held_destination in self._settle(request, span, failed): self._forward(held_span, held_destination) def _request_of(self, span: ReadableSpan, parent_span: Span | None = None) -> _RequestKey: @@ -816,7 +814,9 @@ class TenantFanOutSpanProcessor(SpanProcessor): with self._lock: return self._decide_locked(request) - def _settle(self, request: _RequestKey, failed: bool) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]": + def _settle( + self, request: _RequestKey, span: ReadableSpan, failed: bool + ) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]": """What the request's held tree exports once the span that just ended was the last of its request still open. The model-call span ends in the post-call callback, after the server span, so a tree is decided when its request goes quiet rather than @@ -828,8 +828,10 @@ class TenantFanOutSpanProcessor(SpanProcessor): tree: Final = self._pending.get(request) if tree is not None and failed: tree.failed = True - still_open: Final = self._open.pop(request, 1) - 1 - if still_open > 0: + still_open: Final = self._open.pop(request, set()) + if span.context is not None: + still_open.discard(span.context.span_id) + if still_open: self._open[request] = still_open return () return self._decide_locked(request) diff --git a/tests/unit/integrations/otel/test_otel_v2_destinations.py b/tests/unit/integrations/otel/test_otel_v2_destinations.py index dd720f093a8..782442aad22 100644 --- a/tests/unit/integrations/otel/test_otel_v2_destinations.py +++ b/tests/unit/integrations/otel/test_otel_v2_destinations.py @@ -1454,8 +1454,8 @@ class TestTenantSampling: } def test_a_span_two_levels_down_that_outlives_many_other_spans_still_closes_its_request(self): - """A classifier call under the route span that stays open while the worker starts - more spans than it tracks at once still counts against its own request, so the + """A classifier call under the route span that stays open while the worker serves + more requests than it tracks at once still counts against its own request, so the request is decided once it ends, on its own outcome.""" dest_exporter: Final = InMemorySpanExporter() provider: Final = self._fan_out(dest_exporter) @@ -1472,7 +1472,7 @@ class TestTenantSampling: with tracer.start_as_current_span("POST /v1/chat/completions") as root: with tracer.start_as_current_span("route gpt-4"): with tracer.start_as_current_span("chat classifier"): - for _ in range(otel_providers._MAX_OPEN_SPANS + 1): + for _ in range(otel_providers._MAX_OPEN_REQUESTS + 1): contextvars.Context().run(other_request) with tracer.start_as_current_span("chat gpt-4"): pass @@ -1487,13 +1487,13 @@ class TestTenantSampling: "chat gpt-4", } - def test_a_request_whose_spans_fall_out_of_the_open_span_bound_is_still_decided(self): - """A request still open while the worker holds more open spans than it tracks is - decided as each of its spans ends, rather than held for want of a span it can no - longer place.""" + def test_a_failure_two_levels_down_reaches_its_request_however_many_spans_are_open(self): + """A classifier call that fails while another request holds more spans open than + the worker tracks requests marks its own request, which is then kept at the error + rate as a whole.""" dest_exporter: Final = InMemorySpanExporter() provider: Final = self._fan_out(dest_exporter) - destinations: Final = (arize_destination(arize_success_sampling_rate="1.0"),) + destinations: Final = (arize_destination(arize_success_sampling_rate="0.0", arize_error_sampling_rate="1.0"),) others: Final = contextvars.Context() def open_other_request(): @@ -1503,7 +1503,7 @@ class TestTenantSampling: parent: Final = trace_api.set_span_in_context(root) return ( root, - *(tracer.start_span("chat flood", context=parent) for _ in range(otel_providers._MAX_OPEN_SPANS)), + *(tracer.start_span("chat flood", context=parent) for _ in range(otel_providers._MAX_OPEN_REQUESTS)), ) def run(): @@ -1511,8 +1511,10 @@ class TestTenantSampling: tracer: Final = get_tracer(provider, "litellm") with tracer.start_as_current_span("POST /v1/chat/completions"): with tracer.start_as_current_span("route gpt-4"): - with tracer.start_as_current_span("chat classifier"): - return others.run(open_other_request) + with tracer.start_as_current_span("chat classifier") as classifier: + held_open: Final = others.run(open_other_request) + classifier.set_status(Status(StatusCode.ERROR, "classifier timed out")) + return held_open held_open: Final = in_fresh_context(run) @@ -1524,6 +1526,37 @@ class TestTenantSampling: for span in held_open: span.end() + def test_the_fan_out_forgets_a_span_once_it_ends_and_a_request_once_it_is_pushed_out(self): + """The request lookup holds only spans still open, and only those of the requests + the fan-out still counts, so neither ended spans nor a flood of open requests grow + it past the open request bound.""" + fan_out: Final = TenantFanOutSpanProcessor( + processor_factory=lambda _d: SimpleSpanProcessor(InMemorySpanExporter()) + ) + provider: Final = TracerProvider() + provider.add_span_processor(fan_out) + + def run(): + set_request_destinations((arize_destination(arize_success_sampling_rate="1.0"),)) + tracer: Final = get_tracer(provider, "litellm") + with tracer.start_as_current_span("POST /v1/chat/completions"): + with tracer.start_as_current_span("chat gpt-4"): + pass + assert (fan_out._roots, dict(fan_out._open)) == ({}, {}) + root: Final = tracer.start_span("POST /v1/chat/completions", context=trace_api.Context()) + tracer.start_span("route gpt-4", context=trace_api.set_span_in_context(root)) + return tuple( + tracer.start_span(f"POST {index}", context=trace_api.Context()) + for index in range(otel_providers._MAX_OPEN_REQUESTS) + ) + + held_open: Final = in_fresh_context(run) + + assert len(fan_out._roots) == otel_providers._MAX_OPEN_REQUESTS + for span in held_open: + span.end() + assert (fan_out._roots, dict(fan_out._open)) == ({}, {}) + def test_a_span_starting_under_an_ended_span_below_the_root_follows_its_request(self): """A post-call write started under the route span after its failed request ended is kept with the request at the error rate, rather than decided on its own at the