From f5bfb292e666cf4c7875e8ef42228641b8ce6d8a Mon Sep 17 00:00:00 2001 From: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 12:40:16 -0700 Subject: [PATCH] fix(otel): forget a span's request root when it ends, not when newer spans start The root of each open span was kept in a first-in-first-out table of 16384 entries, so a span two levels below the root that stayed open while the worker started more spans than that lost its entry and ended under its parent's id. Its request then never closed and was held from every sampled destination until shutdown. Each open span now keeps its entry until it ends, and a span starting under a parent that has already ended reads the root from the parent span itself --- .../integrations/otel/plumbing/providers.py | 32 ++++++---- .../otel/test_otel_v2_destinations.py | 59 +++++++++++++++++++ 2 files changed, 80 insertions(+), 11 deletions(-) diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index 86eb3935956..55be035de4f 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -3,6 +3,7 @@ import queue import threading import time +import weakref from collections import OrderedDict from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from hashlib import sha256 @@ -488,14 +489,14 @@ def is_llm_call_span(span: ReadableSpan) -> bool: #: Request trees held back for a sampled destination until the last span of the request #: still open ends, the verdicts kept for the spans that start after that (a post-call -#: database write), the spans in flight per request, and the request each started span +#: database write), the spans in flight per request, and the request each open span #: belongs to. All bounded, so a span that never ends, or a flood of requests, cannot #: hold spans for ever: the oldest tree is decided on what it has. _MAX_PENDING_TREES: Final = 1024 _MAX_PENDING_SPANS_PER_TREE: Final = 512 _MAX_REMEMBERED_VERDICTS: Final = 4096 _MAX_OPEN_REQUESTS: Final = 16384 -_MAX_REMEMBERED_ROOTS: Final = 16384 +_MAX_OPEN_SPANS: Final = 16384 class _PendingTree: @@ -693,17 +694,19 @@ class TenantFanOutSpanProcessor(SpanProcessor): self._exporting: dict[int, int] = {} # mutable-ok: per-processor in-flight export count self._pending: OrderedDict[_RequestKey, _PendingTree] = OrderedDict() # mutable-ok: bounded, by request self._open: OrderedDict[_RequestKey, int] = OrderedDict() # mutable-ok: bounded, spans in flight by request - self._roots: OrderedDict[int, int] = OrderedDict() # mutable-ok: bounded LRU, root span id by span id + self._roots: OrderedDict[int, int] = OrderedDict() # mutable-ok: bounded, root span id by open span id + self._span_roots: weakref.WeakKeyDictionary[SDKSpan, int] = weakref.WeakKeyDictionary() self._verdicts: OrderedDict[_VerdictKey, bool] = OrderedDict() # mutable-ok: bounded LRU self._drain: Final = drain_pool if drain_pool is not None else _DrainPool(capacity=pending_drains) def on_start(self, span: SDKSpan, parent_context: Context | None = None) -> None: context: Final = span.context with self._lock: - request: Final = self._request_of(span) + request: Final = self._request_of(span, trace.get_current_span(parent_context)) if context is not None: + self._span_roots[span] = request[1] self._roots[context.span_id] = request[1] - while len(self._roots) > _MAX_REMEMBERED_ROOTS: + while len(self._roots) > _MAX_OPEN_SPANS: self._roots.popitem(last=False) self._open[request] = self._open.get(request, 0) + 1 while len(self._open) > _MAX_OPEN_REQUESTS: @@ -714,6 +717,8 @@ class TenantFanOutSpanProcessor(SpanProcessor): attributes: Final = span.attributes or _NO_ATTRIBUTES with self._lock: request: Final = self._request_of(span) + if span.context is not None: + self._roots.pop(span.context.span_id, None) failed: Final = span.status.status_code is StatusCode.ERROR for destination in request_destinations(): if ( @@ -735,11 +740,11 @@ class TenantFanOutSpanProcessor(SpanProcessor): for held_span, held_destination in self._settle(request, failed): self._forward(held_span, held_destination) - def _request_of(self, span: ReadableSpan) -> _RequestKey: - """The request ``span`` belongs to, read under ``_lock``: the root recorded when it - started, else the span itself when its parent is remote or missing, else its - parent's root, or the parent itself once that record is gone, which is right for - a late child of the root.""" + def _request_of(self, span: ReadableSpan, parent_span: Span | None = None) -> _RequestKey: + """The request ``span`` belongs to, read under ``_lock``: the root recorded while it + is open, else the span itself when its parent is remote or missing, else its + parent's root, recorded while the parent is open or on ``parent_span`` for a child + starting under a parent that has ended, else the parent itself.""" context: Final = span.context if context is None: return (0, 0) @@ -749,7 +754,12 @@ class TenantFanOutSpanProcessor(SpanProcessor): parent: Final = span.parent if parent is None or parent.is_remote: return (context.trace_id, context.span_id) - return (context.trace_id, self._roots.get(parent.span_id, parent.span_id)) + root: Final = self._roots.get(parent.span_id) + if root is not None: + return (context.trace_id, root) + if isinstance(parent_span, SDKSpan): + return (context.trace_id, self._span_roots.get(parent_span, parent.span_id)) + return (context.trace_id, parent.span_id) def _default_capture_for(self, destination: "OtelDestination") -> str: backend: Final = destination.callback_name diff --git a/tests/unit/integrations/otel/test_otel_v2_destinations.py b/tests/unit/integrations/otel/test_otel_v2_destinations.py index c14ab7023f7..0090cfa05ef 100644 --- a/tests/unit/integrations/otel/test_otel_v2_destinations.py +++ b/tests/unit/integrations/otel/test_otel_v2_destinations.py @@ -1444,6 +1444,65 @@ class TestTenantSampling: "postgres SELECT LiteLLM_VerificationToken", } + def test_a_span_two_levels_down_that_outlives_many_other_spans_still_closes_its_request(self): + """A classifier call under the route span that stays open while the worker starts + more spans than it tracks at once still counts against its own request, so the + request reaches the space when it ends rather than being held.""" + dest_exporter = InMemorySpanExporter() + provider = self._fan_out(dest_exporter) + destinations = (arize_destination(arize_success_sampling_rate="1.0"),) + + def other_request(): + set_request_destinations(()) + with get_tracer(provider, "litellm").start_as_current_span("POST /v1/chat/completions"): + pass + + def run(): + set_request_destinations(destinations) + tracer = get_tracer(provider, "litellm") + with tracer.start_as_current_span("POST /v1/chat/completions"): + with tracer.start_as_current_span("route gpt-4"): + with tracer.start_as_current_span("chat classifier"): + for _ in range(otel_providers._MAX_OPEN_SPANS + 1): + contextvars.Context().run(other_request) + with tracer.start_as_current_span("chat gpt-4"): + pass + + in_fresh_context(run) + + assert {s.name for s in dest_exporter.get_finished_spans()} == { + "POST /v1/chat/completions", + "route gpt-4", + "chat classifier", + "chat gpt-4", + } + + def test_a_span_starting_under_an_ended_span_below_the_root_follows_its_request(self): + """A post-call write started under the route span after its failed request ended + is kept with the request at the error rate, rather than decided on its own at the + success rate.""" + dest_exporter = InMemorySpanExporter() + provider = self._fan_out(dest_exporter) + destinations = (arize_destination(arize_success_sampling_rate="0.0", arize_error_sampling_rate="1.0"),) + + def run(): + set_request_destinations(destinations) + tracer = get_tracer(provider, "litellm") + with tracer.start_as_current_span("POST /v1/chat/completions") as root: + with tracer.start_as_current_span("route gpt-4") as route: + route_context = trace_api.set_span_in_context(route) + root.set_status(Status(StatusCode.ERROR, "upstream 500")) + with tracer.start_as_current_span("postgres INSERT LiteLLM_SpendLogs", context=route_context): + pass + + in_fresh_context(run) + + assert {s.name for s in dest_exporter.get_finished_spans()} == { + "POST /v1/chat/completions", + "route gpt-4", + "postgres INSERT LiteLLM_SpendLogs", + } + def test_concurrent_calls_of_one_trace_each_answer_to_their_own_outcome(self): """Two calls of the caller's trace in flight at once on one process, one failing, are decided as two processes serving one each would decide them."""