From 9e8eb0381795a15340e7f613b9efdc0fbadf9245 Mon Sep 17 00:00:00 2001 From: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> Date: Mon, 19 Oct 2026 19:18:18 -0700 Subject: [PATCH] fix(otel): decide team sampling per request, not per trace Calls sharing a caller's trace id are separate requests, keyed by their root span. Each is decided by the trace hash at the rate for its own outcome, and only its own late spans follow its verdict, so no pod applies an earlier call's verdict to a later call of the trace --- .../integrations/otel/plumbing/providers.py | 125 +++++++++++------- .../test_otel_team_sampling_across_pods.py | 67 +++++++++- .../otel/test_otel_v2_destinations.py | 110 ++++++++++++--- 3 files changed, 235 insertions(+), 67 deletions(-) diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index 6f812192b25..86eb3935956 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -7,7 +7,7 @@ from collections import OrderedDict from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from hashlib import sha256 from types import MappingProxyType -from typing import TYPE_CHECKING, Final, Literal +from typing import TYPE_CHECKING, Final, Literal, TypeAlias from opentelemetry import _logs, baggage, metrics, trace from opentelemetry._logs import Logger, LoggerProvider, NoOpLoggerProvider @@ -486,15 +486,16 @@ def is_llm_call_span(span: ReadableSpan) -> bool: return GenAI.OPERATION_NAME in attributes and MCP.METHOD_NAME not in attributes -#: Request trees held back for a sampled destination until the last span of the trace +#: Request trees held back for a sampled destination until the last span of the request #: still open ends, the verdicts kept for the spans that start after that (a post-call -#: database write), and the spans in flight per trace. All bounded, so a span that never -#: ends, or a flood of requests, cannot hold spans for ever: the oldest tree is decided -#: on what it has. +#: database write), the spans in flight per request, and the request each started span +#: belongs to. All bounded, so a span that never ends, or a flood of requests, cannot +#: hold spans for ever: the oldest tree is decided on what it has. _MAX_PENDING_TREES: Final = 1024 _MAX_PENDING_SPANS_PER_TREE: Final = 512 _MAX_REMEMBERED_VERDICTS: Final = 4096 -_MAX_OPEN_TRACES: Final = 16384 +_MAX_OPEN_REQUESTS: Final = 16384 +_MAX_REMEMBERED_ROOTS: Final = 16384 class _PendingTree: @@ -512,14 +513,19 @@ def _is_sampled(destination: "OtelDestination") -> bool: return destination.success_sampling_rate is not None or destination.error_sampling_rate is not None -#: A verdict is one trace's keep or drop for one destination at its rates: the exporter +#: A request is its trace id and the id of its root, the span of it whose parent is +#: remote or missing. Calls sharing a caller's trace are separate requests, each decided +#: on its own outcome. +_RequestKey: TypeAlias = tuple[int, int] + +#: A verdict is one request's keep or drop for one destination at its rates: the exporter #: ``cache_key`` leaves the rates out, so two destinations sharing an exporter with #: different rates would otherwise share one verdict. -_VerdictKey = tuple[int, object, float | None, float | None] +_VerdictKey: TypeAlias = tuple[int, int, object, float | None, float | None] -def _verdict_key(trace_id: int, destination: "OtelDestination") -> _VerdictKey: - return (trace_id, destination.cache_key(), destination.success_sampling_rate, destination.error_sampling_rate) +def _verdict_key(request: _RequestKey, destination: "OtelDestination") -> _VerdictKey: + return (*request, destination.cache_key(), destination.success_sampling_rate, destination.error_sampling_rate) def _keeps_tree(destination: "OtelDestination", failed: bool, trace_id: int) -> bool: @@ -529,7 +535,7 @@ def _keeps_tree(destination: "OtelDestination", failed: bool, trace_id: int) -> nothing. The first eight bytes of a SHA-256 of the trace id are read against the rate, so - every process deciding a tree of the trace with the same outcome reaches the same + every process deciding a request of the trace with the same outcome reaches the same verdict.""" rate: Final = destination.error_sampling_rate if failed else destination.success_sampling_rate if rate is None: @@ -685,24 +691,29 @@ class TenantFanOutSpanProcessor(SpanProcessor): self._processors: OrderedDict[object, SpanProcessor] = OrderedDict() # mutable-ok: bounded LRU self._retired: OrderedDict[int, SpanProcessor] = OrderedDict() # mutable-ok: drains as exports finish self._exporting: dict[int, int] = {} # mutable-ok: per-processor in-flight export count - self._pending: OrderedDict[int, _PendingTree] = OrderedDict() # mutable-ok: bounded, by trace id - self._open: OrderedDict[int, int] = OrderedDict() # mutable-ok: bounded, spans in flight by trace id + self._pending: OrderedDict[_RequestKey, _PendingTree] = OrderedDict() # mutable-ok: bounded, by request + self._open: OrderedDict[_RequestKey, int] = OrderedDict() # mutable-ok: bounded, spans in flight by request + self._roots: OrderedDict[int, int] = OrderedDict() # mutable-ok: bounded LRU, root span id by span id self._verdicts: OrderedDict[_VerdictKey, bool] = OrderedDict() # mutable-ok: bounded LRU self._drain: Final = drain_pool if drain_pool is not None else _DrainPool(capacity=pending_drains) def on_start(self, span: SDKSpan, parent_context: Context | None = None) -> None: context: Final = span.context - trace_id: Final = context.trace_id if context is not None else 0 with self._lock: - self._open[trace_id] = self._open.get(trace_id, 0) + 1 - while len(self._open) > _MAX_OPEN_TRACES: + request: Final = self._request_of(span) + if context is not None: + self._roots[context.span_id] = request[1] + while len(self._roots) > _MAX_REMEMBERED_ROOTS: + self._roots.popitem(last=False) + self._open[request] = self._open.get(request, 0) + 1 + while len(self._open) > _MAX_OPEN_REQUESTS: self._open.popitem(last=False) def on_end(self, span: ReadableSpan) -> None: suppressed: Final = suppressed_backends() attributes: Final = span.attributes or _NO_ATTRIBUTES - context: Final = span.context - trace_id: Final = context.trace_id if context is not None else 0 + with self._lock: + request: Final = self._request_of(span) failed: Final = span.status.status_code is StatusCode.ERROR for destination in request_destinations(): if ( @@ -719,11 +730,27 @@ class TenantFanOutSpanProcessor(SpanProcessor): if not _is_sampled(destination): self._forward(span, destination) continue - for held_span, held_destination in self._route(trace_id, span, destination, failed): + for held_span, held_destination in self._route(request, span, destination, failed): self._forward(held_span, held_destination) - for held_span, held_destination in self._settle(trace_id, failed): + for held_span, held_destination in self._settle(request, failed): self._forward(held_span, held_destination) + def _request_of(self, span: ReadableSpan) -> _RequestKey: + """The request ``span`` belongs to, read under ``_lock``: the root recorded when it + started, else the span itself when its parent is remote or missing, else its + parent's root, or the parent itself once that record is gone, which is right for + a late child of the root.""" + context: Final = span.context + if context is None: + return (0, 0) + recorded: Final = self._roots.get(context.span_id) + if recorded is not None: + return (context.trace_id, recorded) + parent: Final = span.parent + if parent is None or parent.is_remote: + return (context.trace_id, context.span_id) + return (context.trace_id, self._roots.get(parent.span_id, parent.span_id)) + def _default_capture_for(self, destination: "OtelDestination") -> str: backend: Final = destination.callback_name return ( @@ -746,11 +773,11 @@ class TenantFanOutSpanProcessor(SpanProcessor): self._release(processor) def _route( - self, trace_id: int, span: ReadableSpan, destination: "OtelDestination", failed: bool + self, request: _RequestKey, span: ReadableSpan, destination: "OtelDestination", failed: bool ) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]": """The spans to forward now for ``span`` on a sampled ``destination``: the span itself when its tree's verdict is already known and keeps it, nothing while the - tree is held for the rest of its trace, or what the bounds made the fan-out decide + tree is held for the rest of its request, or what the bounds made the fan-out decide early (a tree that outgrew its cap is decided on what it has, and so is the oldest tree once too many are waiting, so a span that never ends holds nothing back for ever). A failed span marks its tree before any such early decision. Once shutdown @@ -761,62 +788,58 @@ class TenantFanOutSpanProcessor(SpanProcessor): another thread between the two would leave this span in a fresh tree. """ with self._lock: - verdict: Final = self._verdicts.get(_verdict_key(trace_id, destination)) + verdict: Final = self._verdicts.get(_verdict_key(request, destination)) if verdict is not None: return ((span, destination),) if verdict else () - tree: Final = self._pending.setdefault(trace_id, _PendingTree()) + tree: Final = self._pending.setdefault(request, _PendingTree()) tree.held.append((span, destination)) if failed: tree.failed = True if self._draining or len(tree.held) >= _MAX_PENDING_SPANS_PER_TREE: - return self._decide_locked(trace_id) + return self._decide_locked(request) if len(self._pending) > _MAX_PENDING_TREES: oldest: Final = next(iter(self._pending)) return self._decide_locked(oldest) return () - def _decide(self, trace_id: int) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]": + def _decide(self, request: _RequestKey) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]": with self._lock: - return self._decide_locked(trace_id) + return self._decide_locked(request) - def _settle(self, trace_id: int, failed: bool) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]": - """What the trace's held tree exports once the span that just ended was the last - of its trace still open. The model-call span ends in the post-call callback, - after the server span, so a tree is decided when its trace goes quiet rather than + def _settle(self, request: _RequestKey, failed: bool) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]": + """What the request's held tree exports once the span that just ended was the last + of its request still open. The model-call span ends in the post-call callback, + after the server span, so a tree is decided when its request goes quiet rather than when its root ends, and a model call that fails late still answers to the error - rate. A trace whose starts this fan-out never saw (attached mid-flight, or forgotten - by the bound) is decided as each of its spans ends. + rate. A request whose starts this fan-out never saw (attached mid-flight, or + forgotten by the bound) is decided as each of its spans ends. """ with self._lock: - tree: Final = self._pending.get(trace_id) + tree: Final = self._pending.get(request) if tree is not None and failed: tree.failed = True - still_open: Final = self._open.pop(trace_id, 1) - 1 + still_open: Final = self._open.pop(request, 1) - 1 if still_open > 0: - self._open[trace_id] = still_open + self._open[request] = still_open return () - return self._decide_locked(trace_id) + return self._decide_locked(request) - def _decide_locked(self, trace_id: int) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]": - """Decide once per destination for the held tree; the spans the verdicts keep. A - failed tree's verdicts are remembered for the spans of its trace that start later, - such as a post-call database write that would otherwise answer to the success rate. - A successful tree's are not, so a later failed call of the trace answers to the - error rate here as it would on a process that never saw the trace.""" - tree: Final = self._pending.pop(trace_id, None) + def _decide_locked(self, request: _RequestKey) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]": + """Decide once per destination for the held tree; the spans the verdicts keep, and + a verdict remembered for the spans of this request that start later.""" + tree: Final = self._pending.pop(request, None) if tree is None: return () verdicts: dict[_VerdictKey, bool] = {} # mutable-ok: one verdict per destination of this tree for _, destination in tree.held: - key = _verdict_key(trace_id, destination) + key = _verdict_key(request, destination) if key not in verdicts: - verdicts[key] = _keeps_tree(destination, tree.failed, trace_id) - if tree.failed: - self._verdicts.update(verdicts) + verdicts[key] = _keeps_tree(destination, tree.failed, request[0]) + self._verdicts[key] = verdicts[key] while len(self._verdicts) > _MAX_REMEMBERED_VERDICTS: self._verdicts.popitem(last=False) return tuple( - (span, destination) for span, destination in tree.held if verdicts[_verdict_key(trace_id, destination)] + (span, destination) for span, destination in tree.held if verdicts[_verdict_key(request, destination)] ) def _operator_already_writes( @@ -857,8 +880,8 @@ class TenantFanOutSpanProcessor(SpanProcessor): with self._lock: self._draining = True undecided: Final = tuple(self._pending) - for trace_id in undecided: - for held_span, held_destination in self._decide(trace_id): + for request in undecided: + for held_span, held_destination in self._decide(request): self._forward(held_span, held_destination) deadline: Final = time.monotonic() + self._drain_seconds with self._lock: diff --git a/tests/integration/observability/test_otel_team_sampling_across_pods.py b/tests/integration/observability/test_otel_team_sampling_across_pods.py index 61946486da6..5036598ae9d 100644 --- a/tests/integration/observability/test_otel_team_sampling_across_pods.py +++ b/tests/integration/observability/test_otel_team_sampling_across_pods.py @@ -30,7 +30,7 @@ from integration._support.client import ( until_asserted, ) from integration._support.database import scratch_database -from integration._support.otlp_sink import Span, SpanSinks, recorded_requests, recorded_spans +from integration._support.otlp_sink import Span, SpanSinks, configure_sink, recorded_requests, recorded_spans from integration._support.process import OwnedProxy, WorkerHealthcheck, owned_proxy_process from integration._support.wire import Reply, Request, Wire, wire_server from pydantic import JsonValue, TypeAdapter @@ -477,6 +477,47 @@ def test_a_failed_call_answers_to_the_error_rate_on_a_pod_that_dropped_its_trace assert any(not kept[0] for kept in verdicts.values()), "no successful call was dropped before its trace failed" +def test_a_successful_call_answers_to_the_success_rate_on_a_pod_that_kept_its_failed_call(pods: Pods) -> None: + """A pod that already kept a failed call of the trace drops a later successful call of it at a zero + success rate, as the pod that only served the successful call does.""" + sampled: Final = pods.team_key({**CREDENTIALS, "arize_success_sampling_rate": "0.0"}) + cursors: Final = _cursors(pods) + traces: Final = _trace_ids("succeeded-after-kept") + failed: Final = { + trace_id: pods.send(pods.first, _call(sampled, trace_id, content="sampling-fail")) for trace_id in traces + } + _await_logged(pods, cursors.operator, tuple(failed.values())) + by_trace: Final = { + trace_id: ( + failed[trace_id], + pods.send(pods.first, _call(sampled, trace_id)), + pods.send(pods.second, _call(sampled, trace_id)), + ) + for trace_id in traces + } + + _settled(pods, cursors, by_trace, pods.team_key(CREDENTIALS), holds=lambda kept: kept == (True, False, False)) + + +def test_concurrent_calls_of_one_trace_on_one_worker_each_answer_to_their_own_outcome(pods: Pods) -> None: + """A failed and a successful call of the trace in flight at once on the single worker of the first pod + are decided apart: the failed one is kept at an unset error rate, the successful one dropped at a zero + success rate.""" + sampled: Final = pods.team_key({**CREDENTIALS, "arize_success_sampling_rate": "0.0"}) + cursors: Final = _cursors(pods) + traces: Final = _trace_ids("concurrent-mixed") + plan: Final = tuple(product(traces, ("sampling-fail", "hello"))) + + def send(job: tuple[str, str]) -> Sent: + return pods.send(pods.first, _call(sampled, job[0], content=job[1])) + + with ThreadPoolExecutor(max_workers=8) as pool: + sent: Final = tuple(pool.map(send, plan)) + by_trace: Final = {trace_id: tuple(call for call in sent if call.trace_id == trace_id) for trace_id in traces} + + _settled(pods, cursors, by_trace, pods.team_key(CREDENTIALS), holds=lambda kept: kept == (True, False)) + + @pytest.mark.parametrize( ("variables", "expected"), [({"arize_success_sampling_rate": "1.0"}, TRACES), ({"arize_success_sampling_rate": "0.0"}, 0), ({}, TRACES)], @@ -577,6 +618,30 @@ def test_a_concurrent_burst_keeps_every_trace_whole(pods: Pods) -> None: assert 0 < _kept(verdicts) < len(traces) +def test_a_stalled_team_space_during_a_burst_keeps_serving_and_each_trace_whole(pods: Pods) -> None: + """The team's Arize endpoint stops answering while a burst of calls runs over both pods: every call is + still served, and once the endpoint answers again each trace arrives whole or not at all.""" + sampled: Final = pods.team_key({**CREDENTIALS, "arize_success_sampling_rate": "0.5"}) + cursors: Final = _cursors(pods) + traces: Final = _trace_ids("stalled") + plan: Final = tuple(product(traces, (pods.first, pods.second))) + + def send(job: tuple[str, OwnedProxy]) -> Sent: + return pods.send(job[1], _call(sampled, job[0])) + + configure_sink(pods.sinks.arize, paused=True) + try: + with ThreadPoolExecutor(max_workers=8) as pool: + sent: Final = tuple(pool.map(send, plan)) + finally: + configure_sink(pods.sinks.arize, paused=False) + by_trace: Final = {trace_id: tuple(call for call in sent if call.trace_id == trace_id) for trace_id in traces} + + verdicts: Final = _settled(pods, cursors, by_trace, pods.team_key(CREDENTIALS)) + + assert 0 < _kept(verdicts) < len(traces) + + def _write_config(directory: Path, sinks: SpanSinks, provider: Wire) -> Path: config: Final = object_value( JSON.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) diff --git a/tests/unit/integrations/otel/test_otel_v2_destinations.py b/tests/unit/integrations/otel/test_otel_v2_destinations.py index 98627693218..c14ab7023f7 100644 --- a/tests/unit/integrations/otel/test_otel_v2_destinations.py +++ b/tests/unit/integrations/otel/test_otel_v2_destinations.py @@ -1321,7 +1321,7 @@ class TestTenantSampling: def test_a_span_that_ends_after_the_root_follows_the_requests_verdict(self): """A post-call database write that starts after the whole request tree has ended - goes where the rest of the tree went.""" + goes where the rest of the tree went, without a second verdict.""" kept_exporter, dropped_exporter = InMemorySpanExporter(), InMemorySpanExporter() kept = self._fan_out(kept_exporter, trace_id=KEPT_AT_HALF) dropped = self._fan_out(dropped_exporter, trace_id=DROPPED_AT_HALF) @@ -1386,24 +1386,104 @@ class TestTenantSampling: sorted(self.NAMES) ] * 2 - def test_a_late_span_whose_verdict_was_forgotten_is_decided_on_its_own_not_stranded(self): - """Once enough other failed requests have been decided to evict a failed request's - verdict, a span of it that starts late (a post-call database write) opens and closes - its own count, so it is decided on its own as soon as it ends instead of waiting in a - tree nothing would ever close.""" + def test_a_successful_call_after_a_kept_failed_call_of_its_trace_answers_to_the_success_rate(self): + """A process that kept a failed call of the trace drops a later successful call of it + at a zero success rate, as a process that only served the successful call does.""" + served_both, served_success = InMemorySpanExporter(), InMemorySpanExporter() + destinations = (arize_destination(arize_success_sampling_rate="0.0", arize_error_sampling_rate="1.0"),) + both = self._fan_out(served_both, trace_id=KEPT_AT_HALF) + + self._run(both, destinations, failed=True) + self._run(both, destinations) + self._run(self._fan_out(served_success, trace_id=KEPT_AT_HALF), destinations) + + assert len(served_both.get_finished_spans()) == len(self.NAMES), "only the failed call" + assert all(s.name in self.NAMES for s in served_both.get_finished_spans()) + assert served_success.get_finished_spans() == () + + def test_a_late_child_that_fails_after_its_request_was_dropped_is_dropped_with_it(self): + """A post-call write that fails after its successful request was dropped goes where + the request went, rather than reaching the space on its own at the error rate.""" dest_exporter = InMemorySpanExporter() provider = self._fan_out(dest_exporter) - destinations = (arize_destination(arize_success_sampling_rate="1.0", arize_error_sampling_rate="1.0"),) + destinations = (arize_destination(arize_success_sampling_rate="0.0", arize_error_sampling_rate="1.0"),) + + def run(): + set_request_destinations(destinations) + tracer = get_tracer(provider, "litellm") + with tracer.start_as_current_span("POST /v1/chat/completions") as root: + root_context = trace_api.set_span_in_context(root) + with tracer.start_as_current_span("postgres INSERT LiteLLM_SpendLogs", context=root_context) as write: + write.set_status(Status(StatusCode.ERROR, "connection reset")) + + in_fresh_context(run) + + assert dest_exporter.get_finished_spans() == () + + def test_a_failure_deep_in_a_request_marks_the_whole_request(self): + """A database span that fails under the auth span puts the whole request, root + included, on the error rate.""" + dest_exporter = InMemorySpanExporter() + provider = self._fan_out(dest_exporter) + destinations = (arize_destination(arize_success_sampling_rate="0.0", arize_error_sampling_rate="1.0"),) + + def run(): + set_request_destinations(destinations) + tracer = get_tracer(provider, "litellm") + with tracer.start_as_current_span("POST /v1/chat/completions"): + with tracer.start_as_current_span("auth /v1/chat/completions"): + with tracer.start_as_current_span("postgres SELECT LiteLLM_VerificationToken") as lookup: + lookup.set_status(Status(StatusCode.ERROR, "connection reset")) + with tracer.start_as_current_span("chat gpt-4"): + pass + + in_fresh_context(run) + + assert {s.name for s in dest_exporter.get_finished_spans()} == { + *self.NAMES, + "postgres SELECT LiteLLM_VerificationToken", + } + + def test_concurrent_calls_of_one_trace_each_answer_to_their_own_outcome(self): + """Two calls of the caller's trace in flight at once on one process, one failing, + are decided as two processes serving one each would decide them.""" + dest_exporter = InMemorySpanExporter() + provider = self._fan_out(dest_exporter) + destinations = (arize_destination(arize_success_sampling_rate="0.0", arize_error_sampling_rate="1.0"),) + remote = trace_api.SpanContext( + trace_id=KEPT_AT_HALF, span_id=1, is_remote=True, trace_flags=trace_api.TraceFlags(0x01) + ) + + def run(): + set_request_destinations(destinations) + tracer = get_tracer(provider, "litellm") + caller = trace_api.set_span_in_context(trace_api.NonRecordingSpan(remote)) + with tracer.start_as_current_span("POST failed", context=caller) as failed: + with tracer.start_as_current_span("POST succeeded", context=caller): + pass + failed.set_status(Status(StatusCode.ERROR, "upstream 500")) + + in_fresh_context(run) + + assert [s.name for s in dest_exporter.get_finished_spans()] == ["POST failed"] + + def test_a_late_span_whose_verdict_was_forgotten_is_decided_on_its_own_not_stranded(self): + """Once enough other requests have been decided to evict a request's verdict, a span + of it that starts late (a post-call database write) opens and closes its own count, + so it is decided on its own as soon as it ends instead of waiting in a tree nothing + would ever close.""" + dest_exporter = InMemorySpanExporter() + provider = self._fan_out(dest_exporter) + destinations = (arize_destination(arize_success_sampling_rate="1.0"),) def run(): set_request_destinations(destinations) tracer = get_tracer(provider, "litellm") with tracer.start_as_current_span("POST /v1/chat/completions") as root: - root.set_status(Status(StatusCode.ERROR, "upstream 500")) root_context = trace_api.set_span_in_context(root) for index in range(otel_providers._MAX_REMEMBERED_VERDICTS): - with tracer.start_as_current_span(f"POST {index}", context=trace_api.Context()) as other: - other.set_status(Status(StatusCode.ERROR, "upstream 500")) + with tracer.start_as_current_span(f"POST {index}", context=trace_api.Context()): + pass with tracer.start_as_current_span("postgres INSERT LiteLLM_SpendLogs", context=root_context): pass @@ -1472,10 +1552,10 @@ class TestTenantSampling: assert [s.name for s in dest_exporter.get_finished_spans()] == ["chat 0"] - def test_a_flood_of_open_traces_forgets_the_oldest_and_decides_it_as_its_spans_end(self): - """The fan-out counts open spans for a bounded number of traces, a bound far above - what one process keeps in flight, since every trace of the provider is counted, not - only the sampled ones. A trace pushed out of that count is decided on what it has as + def test_a_flood_of_open_requests_forgets_the_oldest_and_decides_it_as_its_spans_end(self): + """The fan-out counts open spans for a bounded number of requests, a bound far above + what one process keeps in flight, since every request of the provider is counted, not + only the sampled ones. A request pushed out of that count is decided on what it has as its next span ends, root still open, rather than held until a bound gets to it.""" dest_exporter = InMemorySpanExporter() provider = self._fan_out(dest_exporter) @@ -1486,7 +1566,7 @@ class TestTenantSampling: tracer = get_tracer(provider, "litellm") root = tracer.start_span("POST /v1/chat/completions", context=trace_api.Context()) tracer.start_span("auth /v1/chat/completions", context=trace_api.set_span_in_context(root)).end() - for index in range(otel_providers._MAX_OPEN_TRACES): + for index in range(otel_providers._MAX_OPEN_REQUESTS): tracer.start_span(f"POST {index}", context=trace_api.Context()) assert dest_exporter.get_finished_spans() == (), "still held before the count is forgotten" tracer.start_span("chat gpt-4", context=trace_api.set_span_in_context(root)).end()