fix(otel): decide a request as its spans end once it falls out of the open span table

A request whose span was pushed out of the table of open spans, because the worker
held more than 16384 open at once, ended that span under another key and was held
from every sampled destination until shutdown. Pushing a span out now drops its
request's open count too, so the request is decided as each of its spans ends, the
way one pushed out of the open request table already is
This commit is contained in:
yucheng-berri 2026-10-20 15:51:01 -07:00
parent f5bfb292e6
commit 11d5d91d17
3 changed files with 80 additions and 39 deletions

View file

@ -694,7 +694,7 @@ class TenantFanOutSpanProcessor(SpanProcessor):
self._exporting: dict[int, int] = {} # mutable-ok: per-processor in-flight export count
self._pending: OrderedDict[_RequestKey, _PendingTree] = OrderedDict() # mutable-ok: bounded, by request
self._open: OrderedDict[_RequestKey, int] = OrderedDict() # mutable-ok: bounded, spans in flight by request
self._roots: OrderedDict[int, int] = OrderedDict() # mutable-ok: bounded, root span id by open span id
self._roots: OrderedDict[int, _RequestKey] = OrderedDict() # mutable-ok: bounded, request by open span id
self._span_roots: weakref.WeakKeyDictionary[SDKSpan, int] = weakref.WeakKeyDictionary()
self._verdicts: OrderedDict[_VerdictKey, bool] = OrderedDict() # mutable-ok: bounded LRU
self._drain: Final = drain_pool if drain_pool is not None else _DrainPool(capacity=pending_drains)
@ -705,9 +705,9 @@ class TenantFanOutSpanProcessor(SpanProcessor):
request: Final = self._request_of(span, trace.get_current_span(parent_context))
if context is not None:
self._span_roots[span] = request[1]
self._roots[context.span_id] = request[1]
self._roots[context.span_id] = request
while len(self._roots) > _MAX_OPEN_SPANS:
self._roots.popitem(last=False)
self._open.pop(self._roots.popitem(last=False)[1], None)
self._open[request] = self._open.get(request, 0) + 1
while len(self._open) > _MAX_OPEN_REQUESTS:
self._open.popitem(last=False)
@ -741,22 +741,22 @@ class TenantFanOutSpanProcessor(SpanProcessor):
self._forward(held_span, held_destination)
def _request_of(self, span: ReadableSpan, parent_span: Span | None = None) -> _RequestKey:
"""The request ``span`` belongs to, read under ``_lock``: the root recorded while it
is open, else the span itself when its parent is remote or missing, else its
parent's root, recorded while the parent is open or on ``parent_span`` for a child
starting under a parent that has ended, else the parent itself."""
"""The request ``span`` belongs to, read under ``_lock``: the request recorded while
it is open, else the span itself when its parent is remote or missing, else its
parent's request, recorded while the parent is open or on ``parent_span`` for a
child starting under a parent that has ended, else the parent itself."""
context: Final = span.context
if context is None:
return (0, 0)
recorded: Final = self._roots.get(context.span_id)
if recorded is not None:
return (context.trace_id, recorded)
return recorded
parent: Final = span.parent
if parent is None or parent.is_remote:
return (context.trace_id, context.span_id)
root: Final = self._roots.get(parent.span_id)
if root is not None:
return (context.trace_id, root)
parent_request: Final = self._roots.get(parent.span_id)
if parent_request is not None:
return parent_request
if isinstance(parent_span, SDKSpan):
return (context.trace_id, self._span_roots.get(parent_span, parent.span_id))
return (context.trace_id, parent.span_id)

View file

@ -643,28 +643,31 @@ def test_a_stalled_team_space_during_a_burst_keeps_serving_and_each_trace_whole(
def _write_config(directory: Path, sinks: SpanSinks, provider: Wire) -> Path:
config: Final = object_value(
shared: Final = object_value(
JSON.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
)
config["model_list"] = [
*list_value(config.get("model_list") or []),
{
"model_name": MODEL,
"litellm_params": {
"model": "openai/gpt-4o-mini",
"api_key": "integration-provider-key",
"api_base": provider.url + "/v1",
config: Final = {
**shared,
"model_list": [
*list_value(shared.get("model_list") or []),
{
"model_name": MODEL,
"litellm_params": {
"model": "openai/gpt-4o-mini",
"api_key": "integration-provider-key",
"api_base": provider.url + "/v1",
},
},
],
"litellm_settings": {
**object_value(shared.get("litellm_settings") or {}),
"callbacks": ["langfuse_otel"],
"otel_tenant_destination_mode": "additive",
"provider_url_destination_allowed_hosts": [urlparse(sinks.operator).netloc, urlparse(sinks.arize).netloc],
},
]
config["litellm_settings"] = {
**object_value(config.get("litellm_settings") or {}),
"callbacks": ["langfuse_otel"],
"otel_tenant_destination_mode": "additive",
"provider_url_destination_allowed_hosts": [urlparse(sinks.operator).netloc, urlparse(sinks.arize).netloc],
"router_settings": {**object_value(shared.get("router_settings") or {}), "num_retries": 0},
"general_settings": {**object_value(shared.get("general_settings") or {}), "user_api_key_cache_ttl": 2},
}
config["router_settings"] = {**object_value(config.get("router_settings") or {}), "num_retries": 0}
config["general_settings"] = {**object_value(config.get("general_settings") or {}), "user_api_key_cache_ttl": 2}
path: Final = directory / "team-sampling.yaml"
path.write_text(yaml.safe_dump(config))
return path

View file

@ -1447,10 +1447,10 @@ class TestTenantSampling:
def test_a_span_two_levels_down_that_outlives_many_other_spans_still_closes_its_request(self):
"""A classifier call under the route span that stays open while the worker starts
more spans than it tracks at once still counts against its own request, so the
request reaches the space when it ends rather than being held."""
dest_exporter = InMemorySpanExporter()
provider = self._fan_out(dest_exporter)
destinations = (arize_destination(arize_success_sampling_rate="1.0"),)
request is decided once it ends, on its own outcome."""
dest_exporter: Final = InMemorySpanExporter()
provider: Final = self._fan_out(dest_exporter)
destinations: Final = (arize_destination(arize_success_sampling_rate="0.0", arize_error_sampling_rate="1.0"),)
def other_request():
set_request_destinations(())
@ -1459,14 +1459,15 @@ class TestTenantSampling:
def run():
set_request_destinations(destinations)
tracer = get_tracer(provider, "litellm")
with tracer.start_as_current_span("POST /v1/chat/completions"):
tracer: Final = get_tracer(provider, "litellm")
with tracer.start_as_current_span("POST /v1/chat/completions") as root:
with tracer.start_as_current_span("route gpt-4"):
with tracer.start_as_current_span("chat classifier"):
for _ in range(otel_providers._MAX_OPEN_SPANS + 1):
contextvars.Context().run(other_request)
with tracer.start_as_current_span("chat gpt-4"):
pass
root.set_status(Status(StatusCode.ERROR, "upstream 500"))
in_fresh_context(run)
@ -1477,20 +1478,57 @@ class TestTenantSampling:
"chat gpt-4",
}
def test_a_request_whose_spans_fall_out_of_the_open_span_bound_is_still_decided(self):
"""A request still open while the worker holds more open spans than it tracks is
decided as each of its spans ends, rather than held for want of a span it can no
longer place."""
dest_exporter: Final = InMemorySpanExporter()
provider: Final = self._fan_out(dest_exporter)
destinations: Final = (arize_destination(arize_success_sampling_rate="1.0"),)
others: Final = contextvars.Context()
def open_other_request():
set_request_destinations(())
tracer: Final = get_tracer(provider, "litellm")
root: Final = tracer.start_span("POST /v1/chat/completions")
parent: Final = trace_api.set_span_in_context(root)
return (
root,
*(tracer.start_span("chat flood", context=parent) for _ in range(otel_providers._MAX_OPEN_SPANS)),
)
def run():
set_request_destinations(destinations)
tracer: Final = get_tracer(provider, "litellm")
with tracer.start_as_current_span("POST /v1/chat/completions"):
with tracer.start_as_current_span("route gpt-4"):
with tracer.start_as_current_span("chat classifier"):
return others.run(open_other_request)
held_open: Final = in_fresh_context(run)
assert {s.name for s in dest_exporter.get_finished_spans()} == {
"POST /v1/chat/completions",
"route gpt-4",
"chat classifier",
}
for span in held_open:
span.end()
def test_a_span_starting_under_an_ended_span_below_the_root_follows_its_request(self):
"""A post-call write started under the route span after its failed request ended
is kept with the request at the error rate, rather than decided on its own at the
success rate."""
dest_exporter = InMemorySpanExporter()
provider = self._fan_out(dest_exporter)
destinations = (arize_destination(arize_success_sampling_rate="0.0", arize_error_sampling_rate="1.0"),)
dest_exporter: Final = InMemorySpanExporter()
provider: Final = self._fan_out(dest_exporter)
destinations: Final = (arize_destination(arize_success_sampling_rate="0.0", arize_error_sampling_rate="1.0"),)
def run():
set_request_destinations(destinations)
tracer = get_tracer(provider, "litellm")
tracer: Final = get_tracer(provider, "litellm")
with tracer.start_as_current_span("POST /v1/chat/completions") as root:
with tracer.start_as_current_span("route gpt-4") as route:
route_context = trace_api.set_span_in_context(route)
route_context: Final = trace_api.set_span_in_context(route)
root.set_status(Status(StatusCode.ERROR, "upstream 500"))
with tracer.start_as_current_span("postgres INSERT LiteLLM_SpendLogs", context=route_context):
pass