From 11d5d91d1786bd1be325e644fedd603d609dfeaa Mon Sep 17 00:00:00 2001 From: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 15:51:01 -0700 Subject: [PATCH] fix(otel): decide a request as its spans end once it falls out of the open span table A request whose span was pushed out of the table of open spans, because the worker held more than 16384 open at once, ended that span under another key and was held from every sampled destination until shutdown. Pushing a span out now drops its request's open count too, so the request is decided as each of its spans ends, the way one pushed out of the open request table already is --- .../integrations/otel/plumbing/providers.py | 22 +++---- .../test_otel_team_sampling_across_pods.py | 37 ++++++------ .../otel/test_otel_v2_destinations.py | 60 +++++++++++++++---- 3 files changed, 80 insertions(+), 39 deletions(-) diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index 55be035de4f..b0a017c6c9b 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -694,7 +694,7 @@ class TenantFanOutSpanProcessor(SpanProcessor): self._exporting: dict[int, int] = {} # mutable-ok: per-processor in-flight export count self._pending: OrderedDict[_RequestKey, _PendingTree] = OrderedDict() # mutable-ok: bounded, by request self._open: OrderedDict[_RequestKey, int] = OrderedDict() # mutable-ok: bounded, spans in flight by request - self._roots: OrderedDict[int, int] = OrderedDict() # mutable-ok: bounded, root span id by open span id + self._roots: OrderedDict[int, _RequestKey] = OrderedDict() # mutable-ok: bounded, request by open span id self._span_roots: weakref.WeakKeyDictionary[SDKSpan, int] = weakref.WeakKeyDictionary() self._verdicts: OrderedDict[_VerdictKey, bool] = OrderedDict() # mutable-ok: bounded LRU self._drain: Final = drain_pool if drain_pool is not None else _DrainPool(capacity=pending_drains) @@ -705,9 +705,9 @@ class TenantFanOutSpanProcessor(SpanProcessor): request: Final = self._request_of(span, trace.get_current_span(parent_context)) if context is not None: self._span_roots[span] = request[1] - self._roots[context.span_id] = request[1] + self._roots[context.span_id] = request while len(self._roots) > _MAX_OPEN_SPANS: - self._roots.popitem(last=False) + self._open.pop(self._roots.popitem(last=False)[1], None) self._open[request] = self._open.get(request, 0) + 1 while len(self._open) > _MAX_OPEN_REQUESTS: self._open.popitem(last=False) @@ -741,22 +741,22 @@ class TenantFanOutSpanProcessor(SpanProcessor): self._forward(held_span, held_destination) def _request_of(self, span: ReadableSpan, parent_span: Span | None = None) -> _RequestKey: - """The request ``span`` belongs to, read under ``_lock``: the root recorded while it - is open, else the span itself when its parent is remote or missing, else its - parent's root, recorded while the parent is open or on ``parent_span`` for a child - starting under a parent that has ended, else the parent itself.""" + """The request ``span`` belongs to, read under ``_lock``: the request recorded while + it is open, else the span itself when its parent is remote or missing, else its + parent's request, recorded while the parent is open or on ``parent_span`` for a + child starting under a parent that has ended, else the parent itself.""" context: Final = span.context if context is None: return (0, 0) recorded: Final = self._roots.get(context.span_id) if recorded is not None: - return (context.trace_id, recorded) + return recorded parent: Final = span.parent if parent is None or parent.is_remote: return (context.trace_id, context.span_id) - root: Final = self._roots.get(parent.span_id) - if root is not None: - return (context.trace_id, root) + parent_request: Final = self._roots.get(parent.span_id) + if parent_request is not None: + return parent_request if isinstance(parent_span, SDKSpan): return (context.trace_id, self._span_roots.get(parent_span, parent.span_id)) return (context.trace_id, parent.span_id) diff --git a/tests/integration/observability/test_otel_team_sampling_across_pods.py b/tests/integration/observability/test_otel_team_sampling_across_pods.py index 5036598ae9d..dc7b295febd 100644 --- a/tests/integration/observability/test_otel_team_sampling_across_pods.py +++ b/tests/integration/observability/test_otel_team_sampling_across_pods.py @@ -643,28 +643,31 @@ def test_a_stalled_team_space_during_a_burst_keeps_serving_and_each_trace_whole( def _write_config(directory: Path, sinks: SpanSinks, provider: Wire) -> Path: - config: Final = object_value( + shared: Final = object_value( JSON.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) ) - config["model_list"] = [ - *list_value(config.get("model_list") or []), - { - "model_name": MODEL, - "litellm_params": { - "model": "openai/gpt-4o-mini", - "api_key": "integration-provider-key", - "api_base": provider.url + "/v1", + config: Final = { + **shared, + "model_list": [ + *list_value(shared.get("model_list") or []), + { + "model_name": MODEL, + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "integration-provider-key", + "api_base": provider.url + "/v1", + }, }, + ], + "litellm_settings": { + **object_value(shared.get("litellm_settings") or {}), + "callbacks": ["langfuse_otel"], + "otel_tenant_destination_mode": "additive", + "provider_url_destination_allowed_hosts": [urlparse(sinks.operator).netloc, urlparse(sinks.arize).netloc], }, - ] - config["litellm_settings"] = { - **object_value(config.get("litellm_settings") or {}), - "callbacks": ["langfuse_otel"], - "otel_tenant_destination_mode": "additive", - "provider_url_destination_allowed_hosts": [urlparse(sinks.operator).netloc, urlparse(sinks.arize).netloc], + "router_settings": {**object_value(shared.get("router_settings") or {}), "num_retries": 0}, + "general_settings": {**object_value(shared.get("general_settings") or {}), "user_api_key_cache_ttl": 2}, } - config["router_settings"] = {**object_value(config.get("router_settings") or {}), "num_retries": 0} - config["general_settings"] = {**object_value(config.get("general_settings") or {}), "user_api_key_cache_ttl": 2} path: Final = directory / "team-sampling.yaml" path.write_text(yaml.safe_dump(config)) return path diff --git a/tests/unit/integrations/otel/test_otel_v2_destinations.py b/tests/unit/integrations/otel/test_otel_v2_destinations.py index 0090cfa05ef..38592b1add8 100644 --- a/tests/unit/integrations/otel/test_otel_v2_destinations.py +++ b/tests/unit/integrations/otel/test_otel_v2_destinations.py @@ -1447,10 +1447,10 @@ class TestTenantSampling: def test_a_span_two_levels_down_that_outlives_many_other_spans_still_closes_its_request(self): """A classifier call under the route span that stays open while the worker starts more spans than it tracks at once still counts against its own request, so the - request reaches the space when it ends rather than being held.""" - dest_exporter = InMemorySpanExporter() - provider = self._fan_out(dest_exporter) - destinations = (arize_destination(arize_success_sampling_rate="1.0"),) + request is decided once it ends, on its own outcome.""" + dest_exporter: Final = InMemorySpanExporter() + provider: Final = self._fan_out(dest_exporter) + destinations: Final = (arize_destination(arize_success_sampling_rate="0.0", arize_error_sampling_rate="1.0"),) def other_request(): set_request_destinations(()) @@ -1459,14 +1459,15 @@ class TestTenantSampling: def run(): set_request_destinations(destinations) - tracer = get_tracer(provider, "litellm") - with tracer.start_as_current_span("POST /v1/chat/completions"): + tracer: Final = get_tracer(provider, "litellm") + with tracer.start_as_current_span("POST /v1/chat/completions") as root: with tracer.start_as_current_span("route gpt-4"): with tracer.start_as_current_span("chat classifier"): for _ in range(otel_providers._MAX_OPEN_SPANS + 1): contextvars.Context().run(other_request) with tracer.start_as_current_span("chat gpt-4"): pass + root.set_status(Status(StatusCode.ERROR, "upstream 500")) in_fresh_context(run) @@ -1477,20 +1478,57 @@ class TestTenantSampling: "chat gpt-4", } + def test_a_request_whose_spans_fall_out_of_the_open_span_bound_is_still_decided(self): + """A request still open while the worker holds more open spans than it tracks is + decided as each of its spans ends, rather than held for want of a span it can no + longer place.""" + dest_exporter: Final = InMemorySpanExporter() + provider: Final = self._fan_out(dest_exporter) + destinations: Final = (arize_destination(arize_success_sampling_rate="1.0"),) + others: Final = contextvars.Context() + + def open_other_request(): + set_request_destinations(()) + tracer: Final = get_tracer(provider, "litellm") + root: Final = tracer.start_span("POST /v1/chat/completions") + parent: Final = trace_api.set_span_in_context(root) + return ( + root, + *(tracer.start_span("chat flood", context=parent) for _ in range(otel_providers._MAX_OPEN_SPANS)), + ) + + def run(): + set_request_destinations(destinations) + tracer: Final = get_tracer(provider, "litellm") + with tracer.start_as_current_span("POST /v1/chat/completions"): + with tracer.start_as_current_span("route gpt-4"): + with tracer.start_as_current_span("chat classifier"): + return others.run(open_other_request) + + held_open: Final = in_fresh_context(run) + + assert {s.name for s in dest_exporter.get_finished_spans()} == { + "POST /v1/chat/completions", + "route gpt-4", + "chat classifier", + } + for span in held_open: + span.end() + def test_a_span_starting_under_an_ended_span_below_the_root_follows_its_request(self): """A post-call write started under the route span after its failed request ended is kept with the request at the error rate, rather than decided on its own at the success rate.""" - dest_exporter = InMemorySpanExporter() - provider = self._fan_out(dest_exporter) - destinations = (arize_destination(arize_success_sampling_rate="0.0", arize_error_sampling_rate="1.0"),) + dest_exporter: Final = InMemorySpanExporter() + provider: Final = self._fan_out(dest_exporter) + destinations: Final = (arize_destination(arize_success_sampling_rate="0.0", arize_error_sampling_rate="1.0"),) def run(): set_request_destinations(destinations) - tracer = get_tracer(provider, "litellm") + tracer: Final = get_tracer(provider, "litellm") with tracer.start_as_current_span("POST /v1/chat/completions") as root: with tracer.start_as_current_span("route gpt-4") as route: - route_context = trace_api.set_span_in_context(route) + route_context: Final = trace_api.set_span_in_context(route) root.set_status(Status(StatusCode.ERROR, "upstream 500")) with tracer.start_as_current_span("postgres INSERT LiteLLM_SpendLogs", context=route_context): pass