mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-21 10:15:47 +00:00
fix(otel): keep each open span's request until it ends, bounded by the open requests
The span table had its own bound of 16384, so a worker holding more spans open than that pushed out the oldest, and a failure in such a span no longer reached its request. Each request now holds the ids of its open spans, and a span's entry in the lookup goes when the span ends or when its request is pushed out of the open request table, the same bound main already applies. A failure anywhere in a tracked request reaches its verdict however many spans are open
This commit is contained in:
parent
23f1f9b1d1
commit
368ea7bdac
2 changed files with 57 additions and 22 deletions
|
|
@ -496,7 +496,6 @@ _MAX_PENDING_TREES: Final = 1024
|
|||
_MAX_PENDING_SPANS_PER_TREE: Final = 512
|
||||
_MAX_REMEMBERED_VERDICTS: Final = 4096
|
||||
_MAX_OPEN_REQUESTS: Final = 16384
|
||||
_MAX_OPEN_SPANS: Final = 16384
|
||||
|
||||
|
||||
class _PendingTree:
|
||||
|
|
@ -693,8 +692,8 @@ class TenantFanOutSpanProcessor(SpanProcessor):
|
|||
self._retired: OrderedDict[int, SpanProcessor] = OrderedDict() # mutable-ok: drains as exports finish
|
||||
self._exporting: dict[int, int] = {} # mutable-ok: per-processor in-flight export count
|
||||
self._pending: OrderedDict[_RequestKey, _PendingTree] = OrderedDict() # mutable-ok: bounded, by request
|
||||
self._open: OrderedDict[_RequestKey, int] = OrderedDict() # mutable-ok: bounded, spans in flight by request
|
||||
self._roots: OrderedDict[int, _RequestKey] = OrderedDict() # mutable-ok: bounded, request by open span id
|
||||
self._open: OrderedDict[_RequestKey, set[int]] = OrderedDict() # mutable-ok: bounded, open span ids by request
|
||||
self._roots: dict[int, _RequestKey] = {} # mutable-ok: holds only the spans ``_open`` holds, request by span id
|
||||
self._span_roots: weakref.WeakKeyDictionary[SDKSpan, int] = weakref.WeakKeyDictionary()
|
||||
self._verdicts: OrderedDict[_VerdictKey, bool] = OrderedDict() # mutable-ok: bounded LRU
|
||||
self._drain: Final = drain_pool if drain_pool is not None else _DrainPool(capacity=pending_drains)
|
||||
|
|
@ -706,11 +705,10 @@ class TenantFanOutSpanProcessor(SpanProcessor):
|
|||
if context is not None:
|
||||
self._span_roots[span] = request[1]
|
||||
self._roots[context.span_id] = request
|
||||
while len(self._roots) > _MAX_OPEN_SPANS:
|
||||
self._open.pop(self._roots.popitem(last=False)[1], None)
|
||||
self._open[request] = self._open.get(request, 0) + 1
|
||||
self._open.setdefault(request, set()).add(context.span_id)
|
||||
while len(self._open) > _MAX_OPEN_REQUESTS:
|
||||
self._open.popitem(last=False)
|
||||
for forgotten in self._open.popitem(last=False)[1]:
|
||||
self._roots.pop(forgotten, None)
|
||||
|
||||
def on_end(self, span: ReadableSpan) -> None:
|
||||
suppressed: Final = suppressed_backends()
|
||||
|
|
@ -737,7 +735,7 @@ class TenantFanOutSpanProcessor(SpanProcessor):
|
|||
continue
|
||||
for held_span, held_destination in self._route(request, span, destination, failed):
|
||||
self._forward(held_span, held_destination)
|
||||
for held_span, held_destination in self._settle(request, failed):
|
||||
for held_span, held_destination in self._settle(request, span, failed):
|
||||
self._forward(held_span, held_destination)
|
||||
|
||||
def _request_of(self, span: ReadableSpan, parent_span: Span | None = None) -> _RequestKey:
|
||||
|
|
@ -816,7 +814,9 @@ class TenantFanOutSpanProcessor(SpanProcessor):
|
|||
with self._lock:
|
||||
return self._decide_locked(request)
|
||||
|
||||
def _settle(self, request: _RequestKey, failed: bool) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]":
|
||||
def _settle(
|
||||
self, request: _RequestKey, span: ReadableSpan, failed: bool
|
||||
) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]":
|
||||
"""What the request's held tree exports once the span that just ended was the last
|
||||
of its request still open. The model-call span ends in the post-call callback,
|
||||
after the server span, so a tree is decided when its request goes quiet rather than
|
||||
|
|
@ -828,8 +828,10 @@ class TenantFanOutSpanProcessor(SpanProcessor):
|
|||
tree: Final = self._pending.get(request)
|
||||
if tree is not None and failed:
|
||||
tree.failed = True
|
||||
still_open: Final = self._open.pop(request, 1) - 1
|
||||
if still_open > 0:
|
||||
still_open: Final = self._open.pop(request, set())
|
||||
if span.context is not None:
|
||||
still_open.discard(span.context.span_id)
|
||||
if still_open:
|
||||
self._open[request] = still_open
|
||||
return ()
|
||||
return self._decide_locked(request)
|
||||
|
|
|
|||
|
|
@ -1454,8 +1454,8 @@ class TestTenantSampling:
|
|||
}
|
||||
|
||||
def test_a_span_two_levels_down_that_outlives_many_other_spans_still_closes_its_request(self):
|
||||
"""A classifier call under the route span that stays open while the worker starts
|
||||
more spans than it tracks at once still counts against its own request, so the
|
||||
"""A classifier call under the route span that stays open while the worker serves
|
||||
more requests than it tracks at once still counts against its own request, so the
|
||||
request is decided once it ends, on its own outcome."""
|
||||
dest_exporter: Final = InMemorySpanExporter()
|
||||
provider: Final = self._fan_out(dest_exporter)
|
||||
|
|
@ -1472,7 +1472,7 @@ class TestTenantSampling:
|
|||
with tracer.start_as_current_span("POST /v1/chat/completions") as root:
|
||||
with tracer.start_as_current_span("route gpt-4"):
|
||||
with tracer.start_as_current_span("chat classifier"):
|
||||
for _ in range(otel_providers._MAX_OPEN_SPANS + 1):
|
||||
for _ in range(otel_providers._MAX_OPEN_REQUESTS + 1):
|
||||
contextvars.Context().run(other_request)
|
||||
with tracer.start_as_current_span("chat gpt-4"):
|
||||
pass
|
||||
|
|
@ -1487,13 +1487,13 @@ class TestTenantSampling:
|
|||
"chat gpt-4",
|
||||
}
|
||||
|
||||
def test_a_request_whose_spans_fall_out_of_the_open_span_bound_is_still_decided(self):
|
||||
"""A request still open while the worker holds more open spans than it tracks is
|
||||
decided as each of its spans ends, rather than held for want of a span it can no
|
||||
longer place."""
|
||||
def test_a_failure_two_levels_down_reaches_its_request_however_many_spans_are_open(self):
|
||||
"""A classifier call that fails while another request holds more spans open than
|
||||
the worker tracks requests marks its own request, which is then kept at the error
|
||||
rate as a whole."""
|
||||
dest_exporter: Final = InMemorySpanExporter()
|
||||
provider: Final = self._fan_out(dest_exporter)
|
||||
destinations: Final = (arize_destination(arize_success_sampling_rate="1.0"),)
|
||||
destinations: Final = (arize_destination(arize_success_sampling_rate="0.0", arize_error_sampling_rate="1.0"),)
|
||||
others: Final = contextvars.Context()
|
||||
|
||||
def open_other_request():
|
||||
|
|
@ -1503,7 +1503,7 @@ class TestTenantSampling:
|
|||
parent: Final = trace_api.set_span_in_context(root)
|
||||
return (
|
||||
root,
|
||||
*(tracer.start_span("chat flood", context=parent) for _ in range(otel_providers._MAX_OPEN_SPANS)),
|
||||
*(tracer.start_span("chat flood", context=parent) for _ in range(otel_providers._MAX_OPEN_REQUESTS)),
|
||||
)
|
||||
|
||||
def run():
|
||||
|
|
@ -1511,8 +1511,10 @@ class TestTenantSampling:
|
|||
tracer: Final = get_tracer(provider, "litellm")
|
||||
with tracer.start_as_current_span("POST /v1/chat/completions"):
|
||||
with tracer.start_as_current_span("route gpt-4"):
|
||||
with tracer.start_as_current_span("chat classifier"):
|
||||
return others.run(open_other_request)
|
||||
with tracer.start_as_current_span("chat classifier") as classifier:
|
||||
held_open: Final = others.run(open_other_request)
|
||||
classifier.set_status(Status(StatusCode.ERROR, "classifier timed out"))
|
||||
return held_open
|
||||
|
||||
held_open: Final = in_fresh_context(run)
|
||||
|
||||
|
|
@ -1524,6 +1526,37 @@ class TestTenantSampling:
|
|||
for span in held_open:
|
||||
span.end()
|
||||
|
||||
def test_the_fan_out_forgets_a_span_once_it_ends_and_a_request_once_it_is_pushed_out(self):
|
||||
"""The request lookup holds only spans still open, and only those of the requests
|
||||
the fan-out still counts, so neither ended spans nor a flood of open requests grow
|
||||
it past the open request bound."""
|
||||
fan_out: Final = TenantFanOutSpanProcessor(
|
||||
processor_factory=lambda _d: SimpleSpanProcessor(InMemorySpanExporter())
|
||||
)
|
||||
provider: Final = TracerProvider()
|
||||
provider.add_span_processor(fan_out)
|
||||
|
||||
def run():
|
||||
set_request_destinations((arize_destination(arize_success_sampling_rate="1.0"),))
|
||||
tracer: Final = get_tracer(provider, "litellm")
|
||||
with tracer.start_as_current_span("POST /v1/chat/completions"):
|
||||
with tracer.start_as_current_span("chat gpt-4"):
|
||||
pass
|
||||
assert (fan_out._roots, dict(fan_out._open)) == ({}, {})
|
||||
root: Final = tracer.start_span("POST /v1/chat/completions", context=trace_api.Context())
|
||||
tracer.start_span("route gpt-4", context=trace_api.set_span_in_context(root))
|
||||
return tuple(
|
||||
tracer.start_span(f"POST {index}", context=trace_api.Context())
|
||||
for index in range(otel_providers._MAX_OPEN_REQUESTS)
|
||||
)
|
||||
|
||||
held_open: Final = in_fresh_context(run)
|
||||
|
||||
assert len(fan_out._roots) == otel_providers._MAX_OPEN_REQUESTS
|
||||
for span in held_open:
|
||||
span.end()
|
||||
assert (fan_out._roots, dict(fan_out._open)) == ({}, {})
|
||||
|
||||
def test_a_span_starting_under_an_ended_span_below_the_root_follows_its_request(self):
|
||||
"""A post-call write started under the route span after its failed request ended
|
||||
is kept with the request at the error rate, rather than decided on its own at the
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue