diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index 86eb3935956..55be035de4f 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -3,6 +3,7 @@ import queue import threading import time +import weakref from collections import OrderedDict from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from hashlib import sha256 @@ -488,14 +489,14 @@ def is_llm_call_span(span: ReadableSpan) -> bool: #: Request trees held back for a sampled destination until the last span of the request #: still open ends, the verdicts kept for the spans that start after that (a post-call -#: database write), the spans in flight per request, and the request each started span +#: database write), the spans in flight per request, and the request each open span #: belongs to. All bounded, so a span that never ends, or a flood of requests, cannot #: hold spans for ever: the oldest tree is decided on what it has. _MAX_PENDING_TREES: Final = 1024 _MAX_PENDING_SPANS_PER_TREE: Final = 512 _MAX_REMEMBERED_VERDICTS: Final = 4096 _MAX_OPEN_REQUESTS: Final = 16384 -_MAX_REMEMBERED_ROOTS: Final = 16384 +_MAX_OPEN_SPANS: Final = 16384 class _PendingTree: @@ -693,17 +694,19 @@ class TenantFanOutSpanProcessor(SpanProcessor): self._exporting: dict[int, int] = {} # mutable-ok: per-processor in-flight export count self._pending: OrderedDict[_RequestKey, _PendingTree] = OrderedDict() # mutable-ok: bounded, by request self._open: OrderedDict[_RequestKey, int] = OrderedDict() # mutable-ok: bounded, spans in flight by request - self._roots: OrderedDict[int, int] = OrderedDict() # mutable-ok: bounded LRU, root span id by span id + self._roots: OrderedDict[int, int] = OrderedDict() # mutable-ok: bounded, root span id by open span id + self._span_roots: weakref.WeakKeyDictionary[SDKSpan, int] = weakref.WeakKeyDictionary() self._verdicts: OrderedDict[_VerdictKey, bool] = OrderedDict() # mutable-ok: bounded LRU self._drain: Final = drain_pool if drain_pool is not None else _DrainPool(capacity=pending_drains) def on_start(self, span: SDKSpan, parent_context: Context | None = None) -> None: context: Final = span.context with self._lock: - request: Final = self._request_of(span) + request: Final = self._request_of(span, trace.get_current_span(parent_context)) if context is not None: + self._span_roots[span] = request[1] self._roots[context.span_id] = request[1] - while len(self._roots) > _MAX_REMEMBERED_ROOTS: + while len(self._roots) > _MAX_OPEN_SPANS: self._roots.popitem(last=False) self._open[request] = self._open.get(request, 0) + 1 while len(self._open) > _MAX_OPEN_REQUESTS: @@ -714,6 +717,8 @@ class TenantFanOutSpanProcessor(SpanProcessor): attributes: Final = span.attributes or _NO_ATTRIBUTES with self._lock: request: Final = self._request_of(span) + if span.context is not None: + self._roots.pop(span.context.span_id, None) failed: Final = span.status.status_code is StatusCode.ERROR for destination in request_destinations(): if ( @@ -735,11 +740,11 @@ class TenantFanOutSpanProcessor(SpanProcessor): for held_span, held_destination in self._settle(request, failed): self._forward(held_span, held_destination) - def _request_of(self, span: ReadableSpan) -> _RequestKey: - """The request ``span`` belongs to, read under ``_lock``: the root recorded when it - started, else the span itself when its parent is remote or missing, else its - parent's root, or the parent itself once that record is gone, which is right for - a late child of the root.""" + def _request_of(self, span: ReadableSpan, parent_span: Span | None = None) -> _RequestKey: + """The request ``span`` belongs to, read under ``_lock``: the root recorded while it + is open, else the span itself when its parent is remote or missing, else its + parent's root, recorded while the parent is open or on ``parent_span`` for a child + starting under a parent that has ended, else the parent itself.""" context: Final = span.context if context is None: return (0, 0) @@ -749,7 +754,12 @@ class TenantFanOutSpanProcessor(SpanProcessor): parent: Final = span.parent if parent is None or parent.is_remote: return (context.trace_id, context.span_id) - return (context.trace_id, self._roots.get(parent.span_id, parent.span_id)) + root: Final = self._roots.get(parent.span_id) + if root is not None: + return (context.trace_id, root) + if isinstance(parent_span, SDKSpan): + return (context.trace_id, self._span_roots.get(parent_span, parent.span_id)) + return (context.trace_id, parent.span_id) def _default_capture_for(self, destination: "OtelDestination") -> str: backend: Final = destination.callback_name diff --git a/tests/unit/integrations/otel/test_otel_v2_destinations.py b/tests/unit/integrations/otel/test_otel_v2_destinations.py index c14ab7023f7..0090cfa05ef 100644 --- a/tests/unit/integrations/otel/test_otel_v2_destinations.py +++ b/tests/unit/integrations/otel/test_otel_v2_destinations.py @@ -1444,6 +1444,65 @@ class TestTenantSampling: "postgres SELECT LiteLLM_VerificationToken", } + def test_a_span_two_levels_down_that_outlives_many_other_spans_still_closes_its_request(self): + """A classifier call under the route span that stays open while the worker starts + more spans than it tracks at once still counts against its own request, so the + request reaches the space when it ends rather than being held.""" + dest_exporter = InMemorySpanExporter() + provider = self._fan_out(dest_exporter) + destinations = (arize_destination(arize_success_sampling_rate="1.0"),) + + def other_request(): + set_request_destinations(()) + with get_tracer(provider, "litellm").start_as_current_span("POST /v1/chat/completions"): + pass + + def run(): + set_request_destinations(destinations) + tracer = get_tracer(provider, "litellm") + with tracer.start_as_current_span("POST /v1/chat/completions"): + with tracer.start_as_current_span("route gpt-4"): + with tracer.start_as_current_span("chat classifier"): + for _ in range(otel_providers._MAX_OPEN_SPANS + 1): + contextvars.Context().run(other_request) + with tracer.start_as_current_span("chat gpt-4"): + pass + + in_fresh_context(run) + + assert {s.name for s in dest_exporter.get_finished_spans()} == { + "POST /v1/chat/completions", + "route gpt-4", + "chat classifier", + "chat gpt-4", + } + + def test_a_span_starting_under_an_ended_span_below_the_root_follows_its_request(self): + """A post-call write started under the route span after its failed request ended + is kept with the request at the error rate, rather than decided on its own at the + success rate.""" + dest_exporter = InMemorySpanExporter() + provider = self._fan_out(dest_exporter) + destinations = (arize_destination(arize_success_sampling_rate="0.0", arize_error_sampling_rate="1.0"),) + + def run(): + set_request_destinations(destinations) + tracer = get_tracer(provider, "litellm") + with tracer.start_as_current_span("POST /v1/chat/completions") as root: + with tracer.start_as_current_span("route gpt-4") as route: + route_context = trace_api.set_span_in_context(route) + root.set_status(Status(StatusCode.ERROR, "upstream 500")) + with tracer.start_as_current_span("postgres INSERT LiteLLM_SpendLogs", context=route_context): + pass + + in_fresh_context(run) + + assert {s.name for s in dest_exporter.get_finished_spans()} == { + "POST /v1/chat/completions", + "route gpt-4", + "postgres INSERT LiteLLM_SpendLogs", + } + def test_concurrent_calls_of_one_trace_each_answer_to_their_own_outcome(self): """Two calls of the caller's trace in flight at once on one process, one failing, are decided as two processes serving one each would decide them."""