diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index 6f812192b25..86eb3935956 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -7,7 +7,7 @@ from collections import OrderedDict from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from hashlib import sha256 from types import MappingProxyType -from typing import TYPE_CHECKING, Final, Literal +from typing import TYPE_CHECKING, Final, Literal, TypeAlias from opentelemetry import _logs, baggage, metrics, trace from opentelemetry._logs import Logger, LoggerProvider, NoOpLoggerProvider @@ -486,15 +486,16 @@ def is_llm_call_span(span: ReadableSpan) -> bool: return GenAI.OPERATION_NAME in attributes and MCP.METHOD_NAME not in attributes -#: Request trees held back for a sampled destination until the last span of the trace +#: Request trees held back for a sampled destination until the last span of the request #: still open ends, the verdicts kept for the spans that start after that (a post-call -#: database write), and the spans in flight per trace. All bounded, so a span that never -#: ends, or a flood of requests, cannot hold spans for ever: the oldest tree is decided -#: on what it has. +#: database write), the spans in flight per request, and the request each started span +#: belongs to. All bounded, so a span that never ends, or a flood of requests, cannot +#: hold spans for ever: the oldest tree is decided on what it has. _MAX_PENDING_TREES: Final = 1024 _MAX_PENDING_SPANS_PER_TREE: Final = 512 _MAX_REMEMBERED_VERDICTS: Final = 4096 -_MAX_OPEN_TRACES: Final = 16384 +_MAX_OPEN_REQUESTS: Final = 16384 +_MAX_REMEMBERED_ROOTS: Final = 16384 class _PendingTree: @@ -512,14 +513,19 @@ def _is_sampled(destination: "OtelDestination") -> bool: return destination.success_sampling_rate is not None or destination.error_sampling_rate is not None -#: A verdict is one trace's keep or drop for one destination at its rates: the exporter +#: A request is its trace id and the id of its root, the span of it whose parent is +#: remote or missing. Calls sharing a caller's trace are separate requests, each decided +#: on its own outcome. +_RequestKey: TypeAlias = tuple[int, int] + +#: A verdict is one request's keep or drop for one destination at its rates: the exporter #: ``cache_key`` leaves the rates out, so two destinations sharing an exporter with #: different rates would otherwise share one verdict. -_VerdictKey = tuple[int, object, float | None, float | None] +_VerdictKey: TypeAlias = tuple[int, int, object, float | None, float | None] -def _verdict_key(trace_id: int, destination: "OtelDestination") -> _VerdictKey: - return (trace_id, destination.cache_key(), destination.success_sampling_rate, destination.error_sampling_rate) +def _verdict_key(request: _RequestKey, destination: "OtelDestination") -> _VerdictKey: + return (*request, destination.cache_key(), destination.success_sampling_rate, destination.error_sampling_rate) def _keeps_tree(destination: "OtelDestination", failed: bool, trace_id: int) -> bool: @@ -529,7 +535,7 @@ def _keeps_tree(destination: "OtelDestination", failed: bool, trace_id: int) -> nothing. The first eight bytes of a SHA-256 of the trace id are read against the rate, so - every process deciding a tree of the trace with the same outcome reaches the same + every process deciding a request of the trace with the same outcome reaches the same verdict.""" rate: Final = destination.error_sampling_rate if failed else destination.success_sampling_rate if rate is None: @@ -685,24 +691,29 @@ class TenantFanOutSpanProcessor(SpanProcessor): self._processors: OrderedDict[object, SpanProcessor] = OrderedDict() # mutable-ok: bounded LRU self._retired: OrderedDict[int, SpanProcessor] = OrderedDict() # mutable-ok: drains as exports finish self._exporting: dict[int, int] = {} # mutable-ok: per-processor in-flight export count - self._pending: OrderedDict[int, _PendingTree] = OrderedDict() # mutable-ok: bounded, by trace id - self._open: OrderedDict[int, int] = OrderedDict() # mutable-ok: bounded, spans in flight by trace id + self._pending: OrderedDict[_RequestKey, _PendingTree] = OrderedDict() # mutable-ok: bounded, by request + self._open: OrderedDict[_RequestKey, int] = OrderedDict() # mutable-ok: bounded, spans in flight by request + self._roots: OrderedDict[int, int] = OrderedDict() # mutable-ok: bounded LRU, root span id by span id self._verdicts: OrderedDict[_VerdictKey, bool] = OrderedDict() # mutable-ok: bounded LRU self._drain: Final = drain_pool if drain_pool is not None else _DrainPool(capacity=pending_drains) def on_start(self, span: SDKSpan, parent_context: Context | None = None) -> None: context: Final = span.context - trace_id: Final = context.trace_id if context is not None else 0 with self._lock: - self._open[trace_id] = self._open.get(trace_id, 0) + 1 - while len(self._open) > _MAX_OPEN_TRACES: + request: Final = self._request_of(span) + if context is not None: + self._roots[context.span_id] = request[1] + while len(self._roots) > _MAX_REMEMBERED_ROOTS: + self._roots.popitem(last=False) + self._open[request] = self._open.get(request, 0) + 1 + while len(self._open) > _MAX_OPEN_REQUESTS: self._open.popitem(last=False) def on_end(self, span: ReadableSpan) -> None: suppressed: Final = suppressed_backends() attributes: Final = span.attributes or _NO_ATTRIBUTES - context: Final = span.context - trace_id: Final = context.trace_id if context is not None else 0 + with self._lock: + request: Final = self._request_of(span) failed: Final = span.status.status_code is StatusCode.ERROR for destination in request_destinations(): if ( @@ -719,11 +730,27 @@ class TenantFanOutSpanProcessor(SpanProcessor): if not _is_sampled(destination): self._forward(span, destination) continue - for held_span, held_destination in self._route(trace_id, span, destination, failed): + for held_span, held_destination in self._route(request, span, destination, failed): self._forward(held_span, held_destination) - for held_span, held_destination in self._settle(trace_id, failed): + for held_span, held_destination in self._settle(request, failed): self._forward(held_span, held_destination) + def _request_of(self, span: ReadableSpan) -> _RequestKey: + """The request ``span`` belongs to, read under ``_lock``: the root recorded when it + started, else the span itself when its parent is remote or missing, else its + parent's root, or the parent itself once that record is gone, which is right for + a late child of the root.""" + context: Final = span.context + if context is None: + return (0, 0) + recorded: Final = self._roots.get(context.span_id) + if recorded is not None: + return (context.trace_id, recorded) + parent: Final = span.parent + if parent is None or parent.is_remote: + return (context.trace_id, context.span_id) + return (context.trace_id, self._roots.get(parent.span_id, parent.span_id)) + def _default_capture_for(self, destination: "OtelDestination") -> str: backend: Final = destination.callback_name return ( @@ -746,11 +773,11 @@ class TenantFanOutSpanProcessor(SpanProcessor): self._release(processor) def _route( - self, trace_id: int, span: ReadableSpan, destination: "OtelDestination", failed: bool + self, request: _RequestKey, span: ReadableSpan, destination: "OtelDestination", failed: bool ) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]": """The spans to forward now for ``span`` on a sampled ``destination``: the span itself when its tree's verdict is already known and keeps it, nothing while the - tree is held for the rest of its trace, or what the bounds made the fan-out decide + tree is held for the rest of its request, or what the bounds made the fan-out decide early (a tree that outgrew its cap is decided on what it has, and so is the oldest tree once too many are waiting, so a span that never ends holds nothing back for ever). A failed span marks its tree before any such early decision. Once shutdown @@ -761,62 +788,58 @@ class TenantFanOutSpanProcessor(SpanProcessor): another thread between the two would leave this span in a fresh tree. """ with self._lock: - verdict: Final = self._verdicts.get(_verdict_key(trace_id, destination)) + verdict: Final = self._verdicts.get(_verdict_key(request, destination)) if verdict is not None: return ((span, destination),) if verdict else () - tree: Final = self._pending.setdefault(trace_id, _PendingTree()) + tree: Final = self._pending.setdefault(request, _PendingTree()) tree.held.append((span, destination)) if failed: tree.failed = True if self._draining or len(tree.held) >= _MAX_PENDING_SPANS_PER_TREE: - return self._decide_locked(trace_id) + return self._decide_locked(request) if len(self._pending) > _MAX_PENDING_TREES: oldest: Final = next(iter(self._pending)) return self._decide_locked(oldest) return () - def _decide(self, trace_id: int) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]": + def _decide(self, request: _RequestKey) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]": with self._lock: - return self._decide_locked(trace_id) + return self._decide_locked(request) - def _settle(self, trace_id: int, failed: bool) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]": - """What the trace's held tree exports once the span that just ended was the last - of its trace still open. The model-call span ends in the post-call callback, - after the server span, so a tree is decided when its trace goes quiet rather than + def _settle(self, request: _RequestKey, failed: bool) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]": + """What the request's held tree exports once the span that just ended was the last + of its request still open. The model-call span ends in the post-call callback, + after the server span, so a tree is decided when its request goes quiet rather than when its root ends, and a model call that fails late still answers to the error - rate. A trace whose starts this fan-out never saw (attached mid-flight, or forgotten - by the bound) is decided as each of its spans ends. + rate. A request whose starts this fan-out never saw (attached mid-flight, or + forgotten by the bound) is decided as each of its spans ends. """ with self._lock: - tree: Final = self._pending.get(trace_id) + tree: Final = self._pending.get(request) if tree is not None and failed: tree.failed = True - still_open: Final = self._open.pop(trace_id, 1) - 1 + still_open: Final = self._open.pop(request, 1) - 1 if still_open > 0: - self._open[trace_id] = still_open + self._open[request] = still_open return () - return self._decide_locked(trace_id) + return self._decide_locked(request) - def _decide_locked(self, trace_id: int) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]": - """Decide once per destination for the held tree; the spans the verdicts keep. A - failed tree's verdicts are remembered for the spans of its trace that start later, - such as a post-call database write that would otherwise answer to the success rate. - A successful tree's are not, so a later failed call of the trace answers to the - error rate here as it would on a process that never saw the trace.""" - tree: Final = self._pending.pop(trace_id, None) + def _decide_locked(self, request: _RequestKey) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]": + """Decide once per destination for the held tree; the spans the verdicts keep, and + a verdict remembered for the spans of this request that start later.""" + tree: Final = self._pending.pop(request, None) if tree is None: return () verdicts: dict[_VerdictKey, bool] = {} # mutable-ok: one verdict per destination of this tree for _, destination in tree.held: - key = _verdict_key(trace_id, destination) + key = _verdict_key(request, destination) if key not in verdicts: - verdicts[key] = _keeps_tree(destination, tree.failed, trace_id) - if tree.failed: - self._verdicts.update(verdicts) + verdicts[key] = _keeps_tree(destination, tree.failed, request[0]) + self._verdicts[key] = verdicts[key] while len(self._verdicts) > _MAX_REMEMBERED_VERDICTS: self._verdicts.popitem(last=False) return tuple( - (span, destination) for span, destination in tree.held if verdicts[_verdict_key(trace_id, destination)] + (span, destination) for span, destination in tree.held if verdicts[_verdict_key(request, destination)] ) def _operator_already_writes( @@ -857,8 +880,8 @@ class TenantFanOutSpanProcessor(SpanProcessor): with self._lock: self._draining = True undecided: Final = tuple(self._pending) - for trace_id in undecided: - for held_span, held_destination in self._decide(trace_id): + for request in undecided: + for held_span, held_destination in self._decide(request): self._forward(held_span, held_destination) deadline: Final = time.monotonic() + self._drain_seconds with self._lock: diff --git a/tests/integration/observability/test_otel_team_sampling_across_pods.py b/tests/integration/observability/test_otel_team_sampling_across_pods.py index 61946486da6..5036598ae9d 100644 --- a/tests/integration/observability/test_otel_team_sampling_across_pods.py +++ b/tests/integration/observability/test_otel_team_sampling_across_pods.py @@ -30,7 +30,7 @@ from integration._support.client import ( until_asserted, ) from integration._support.database import scratch_database -from integration._support.otlp_sink import Span, SpanSinks, recorded_requests, recorded_spans +from integration._support.otlp_sink import Span, SpanSinks, configure_sink, recorded_requests, recorded_spans from integration._support.process import OwnedProxy, WorkerHealthcheck, owned_proxy_process from integration._support.wire import Reply, Request, Wire, wire_server from pydantic import JsonValue, TypeAdapter @@ -477,6 +477,47 @@ def test_a_failed_call_answers_to_the_error_rate_on_a_pod_that_dropped_its_trace assert any(not kept[0] for kept in verdicts.values()), "no successful call was dropped before its trace failed" +def test_a_successful_call_answers_to_the_success_rate_on_a_pod_that_kept_its_failed_call(pods: Pods) -> None: + """A pod that already kept a failed call of the trace drops a later successful call of it at a zero + success rate, as the pod that only served the successful call does.""" + sampled: Final = pods.team_key({**CREDENTIALS, "arize_success_sampling_rate": "0.0"}) + cursors: Final = _cursors(pods) + traces: Final = _trace_ids("succeeded-after-kept") + failed: Final = { + trace_id: pods.send(pods.first, _call(sampled, trace_id, content="sampling-fail")) for trace_id in traces + } + _await_logged(pods, cursors.operator, tuple(failed.values())) + by_trace: Final = { + trace_id: ( + failed[trace_id], + pods.send(pods.first, _call(sampled, trace_id)), + pods.send(pods.second, _call(sampled, trace_id)), + ) + for trace_id in traces + } + + _settled(pods, cursors, by_trace, pods.team_key(CREDENTIALS), holds=lambda kept: kept == (True, False, False)) + + +def test_concurrent_calls_of_one_trace_on_one_worker_each_answer_to_their_own_outcome(pods: Pods) -> None: + """A failed and a successful call of the trace in flight at once on the single worker of the first pod + are decided apart: the failed one is kept at an unset error rate, the successful one dropped at a zero + success rate.""" + sampled: Final = pods.team_key({**CREDENTIALS, "arize_success_sampling_rate": "0.0"}) + cursors: Final = _cursors(pods) + traces: Final = _trace_ids("concurrent-mixed") + plan: Final = tuple(product(traces, ("sampling-fail", "hello"))) + + def send(job: tuple[str, str]) -> Sent: + return pods.send(pods.first, _call(sampled, job[0], content=job[1])) + + with ThreadPoolExecutor(max_workers=8) as pool: + sent: Final = tuple(pool.map(send, plan)) + by_trace: Final = {trace_id: tuple(call for call in sent if call.trace_id == trace_id) for trace_id in traces} + + _settled(pods, cursors, by_trace, pods.team_key(CREDENTIALS), holds=lambda kept: kept == (True, False)) + + @pytest.mark.parametrize( ("variables", "expected"), [({"arize_success_sampling_rate": "1.0"}, TRACES), ({"arize_success_sampling_rate": "0.0"}, 0), ({}, TRACES)], @@ -577,6 +618,30 @@ def test_a_concurrent_burst_keeps_every_trace_whole(pods: Pods) -> None: assert 0 < _kept(verdicts) < len(traces) +def test_a_stalled_team_space_during_a_burst_keeps_serving_and_each_trace_whole(pods: Pods) -> None: + """The team's Arize endpoint stops answering while a burst of calls runs over both pods: every call is + still served, and once the endpoint answers again each trace arrives whole or not at all.""" + sampled: Final = pods.team_key({**CREDENTIALS, "arize_success_sampling_rate": "0.5"}) + cursors: Final = _cursors(pods) + traces: Final = _trace_ids("stalled") + plan: Final = tuple(product(traces, (pods.first, pods.second))) + + def send(job: tuple[str, OwnedProxy]) -> Sent: + return pods.send(job[1], _call(sampled, job[0])) + + configure_sink(pods.sinks.arize, paused=True) + try: + with ThreadPoolExecutor(max_workers=8) as pool: + sent: Final = tuple(pool.map(send, plan)) + finally: + configure_sink(pods.sinks.arize, paused=False) + by_trace: Final = {trace_id: tuple(call for call in sent if call.trace_id == trace_id) for trace_id in traces} + + verdicts: Final = _settled(pods, cursors, by_trace, pods.team_key(CREDENTIALS)) + + assert 0 < _kept(verdicts) < len(traces) + + def _write_config(directory: Path, sinks: SpanSinks, provider: Wire) -> Path: config: Final = object_value( JSON.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) diff --git a/tests/unit/integrations/otel/test_otel_v2_destinations.py b/tests/unit/integrations/otel/test_otel_v2_destinations.py index 98627693218..c14ab7023f7 100644 --- a/tests/unit/integrations/otel/test_otel_v2_destinations.py +++ b/tests/unit/integrations/otel/test_otel_v2_destinations.py @@ -1321,7 +1321,7 @@ class TestTenantSampling: def test_a_span_that_ends_after_the_root_follows_the_requests_verdict(self): """A post-call database write that starts after the whole request tree has ended - goes where the rest of the tree went.""" + goes where the rest of the tree went, without a second verdict.""" kept_exporter, dropped_exporter = InMemorySpanExporter(), InMemorySpanExporter() kept = self._fan_out(kept_exporter, trace_id=KEPT_AT_HALF) dropped = self._fan_out(dropped_exporter, trace_id=DROPPED_AT_HALF) @@ -1386,24 +1386,104 @@ class TestTenantSampling: sorted(self.NAMES) ] * 2 - def test_a_late_span_whose_verdict_was_forgotten_is_decided_on_its_own_not_stranded(self): - """Once enough other failed requests have been decided to evict a failed request's - verdict, a span of it that starts late (a post-call database write) opens and closes - its own count, so it is decided on its own as soon as it ends instead of waiting in a - tree nothing would ever close.""" + def test_a_successful_call_after_a_kept_failed_call_of_its_trace_answers_to_the_success_rate(self): + """A process that kept a failed call of the trace drops a later successful call of it + at a zero success rate, as a process that only served the successful call does.""" + served_both, served_success = InMemorySpanExporter(), InMemorySpanExporter() + destinations = (arize_destination(arize_success_sampling_rate="0.0", arize_error_sampling_rate="1.0"),) + both = self._fan_out(served_both, trace_id=KEPT_AT_HALF) + + self._run(both, destinations, failed=True) + self._run(both, destinations) + self._run(self._fan_out(served_success, trace_id=KEPT_AT_HALF), destinations) + + assert len(served_both.get_finished_spans()) == len(self.NAMES), "only the failed call" + assert all(s.name in self.NAMES for s in served_both.get_finished_spans()) + assert served_success.get_finished_spans() == () + + def test_a_late_child_that_fails_after_its_request_was_dropped_is_dropped_with_it(self): + """A post-call write that fails after its successful request was dropped goes where + the request went, rather than reaching the space on its own at the error rate.""" dest_exporter = InMemorySpanExporter() provider = self._fan_out(dest_exporter) - destinations = (arize_destination(arize_success_sampling_rate="1.0", arize_error_sampling_rate="1.0"),) + destinations = (arize_destination(arize_success_sampling_rate="0.0", arize_error_sampling_rate="1.0"),) + + def run(): + set_request_destinations(destinations) + tracer = get_tracer(provider, "litellm") + with tracer.start_as_current_span("POST /v1/chat/completions") as root: + root_context = trace_api.set_span_in_context(root) + with tracer.start_as_current_span("postgres INSERT LiteLLM_SpendLogs", context=root_context) as write: + write.set_status(Status(StatusCode.ERROR, "connection reset")) + + in_fresh_context(run) + + assert dest_exporter.get_finished_spans() == () + + def test_a_failure_deep_in_a_request_marks_the_whole_request(self): + """A database span that fails under the auth span puts the whole request, root + included, on the error rate.""" + dest_exporter = InMemorySpanExporter() + provider = self._fan_out(dest_exporter) + destinations = (arize_destination(arize_success_sampling_rate="0.0", arize_error_sampling_rate="1.0"),) + + def run(): + set_request_destinations(destinations) + tracer = get_tracer(provider, "litellm") + with tracer.start_as_current_span("POST /v1/chat/completions"): + with tracer.start_as_current_span("auth /v1/chat/completions"): + with tracer.start_as_current_span("postgres SELECT LiteLLM_VerificationToken") as lookup: + lookup.set_status(Status(StatusCode.ERROR, "connection reset")) + with tracer.start_as_current_span("chat gpt-4"): + pass + + in_fresh_context(run) + + assert {s.name for s in dest_exporter.get_finished_spans()} == { + *self.NAMES, + "postgres SELECT LiteLLM_VerificationToken", + } + + def test_concurrent_calls_of_one_trace_each_answer_to_their_own_outcome(self): + """Two calls of the caller's trace in flight at once on one process, one failing, + are decided as two processes serving one each would decide them.""" + dest_exporter = InMemorySpanExporter() + provider = self._fan_out(dest_exporter) + destinations = (arize_destination(arize_success_sampling_rate="0.0", arize_error_sampling_rate="1.0"),) + remote = trace_api.SpanContext( + trace_id=KEPT_AT_HALF, span_id=1, is_remote=True, trace_flags=trace_api.TraceFlags(0x01) + ) + + def run(): + set_request_destinations(destinations) + tracer = get_tracer(provider, "litellm") + caller = trace_api.set_span_in_context(trace_api.NonRecordingSpan(remote)) + with tracer.start_as_current_span("POST failed", context=caller) as failed: + with tracer.start_as_current_span("POST succeeded", context=caller): + pass + failed.set_status(Status(StatusCode.ERROR, "upstream 500")) + + in_fresh_context(run) + + assert [s.name for s in dest_exporter.get_finished_spans()] == ["POST failed"] + + def test_a_late_span_whose_verdict_was_forgotten_is_decided_on_its_own_not_stranded(self): + """Once enough other requests have been decided to evict a request's verdict, a span + of it that starts late (a post-call database write) opens and closes its own count, + so it is decided on its own as soon as it ends instead of waiting in a tree nothing + would ever close.""" + dest_exporter = InMemorySpanExporter() + provider = self._fan_out(dest_exporter) + destinations = (arize_destination(arize_success_sampling_rate="1.0"),) def run(): set_request_destinations(destinations) tracer = get_tracer(provider, "litellm") with tracer.start_as_current_span("POST /v1/chat/completions") as root: - root.set_status(Status(StatusCode.ERROR, "upstream 500")) root_context = trace_api.set_span_in_context(root) for index in range(otel_providers._MAX_REMEMBERED_VERDICTS): - with tracer.start_as_current_span(f"POST {index}", context=trace_api.Context()) as other: - other.set_status(Status(StatusCode.ERROR, "upstream 500")) + with tracer.start_as_current_span(f"POST {index}", context=trace_api.Context()): + pass with tracer.start_as_current_span("postgres INSERT LiteLLM_SpendLogs", context=root_context): pass @@ -1472,10 +1552,10 @@ class TestTenantSampling: assert [s.name for s in dest_exporter.get_finished_spans()] == ["chat 0"] - def test_a_flood_of_open_traces_forgets_the_oldest_and_decides_it_as_its_spans_end(self): - """The fan-out counts open spans for a bounded number of traces, a bound far above - what one process keeps in flight, since every trace of the provider is counted, not - only the sampled ones. A trace pushed out of that count is decided on what it has as + def test_a_flood_of_open_requests_forgets_the_oldest_and_decides_it_as_its_spans_end(self): + """The fan-out counts open spans for a bounded number of requests, a bound far above + what one process keeps in flight, since every request of the provider is counted, not + only the sampled ones. A request pushed out of that count is decided on what it has as its next span ends, root still open, rather than held until a bound gets to it.""" dest_exporter = InMemorySpanExporter() provider = self._fan_out(dest_exporter) @@ -1486,7 +1566,7 @@ class TestTenantSampling: tracer = get_tracer(provider, "litellm") root = tracer.start_span("POST /v1/chat/completions", context=trace_api.Context()) tracer.start_span("auth /v1/chat/completions", context=trace_api.set_span_in_context(root)).end() - for index in range(otel_providers._MAX_OPEN_TRACES): + for index in range(otel_providers._MAX_OPEN_REQUESTS): tracer.start_span(f"POST {index}", context=trace_api.Context()) assert dest_exporter.get_finished_spans() == (), "still held before the count is forgotten" tracer.start_span("chat gpt-4", context=trace_api.set_span_in_context(root)).end()