mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-21 10:15:47 +00:00
fix(otel): decide a request as its spans end once it falls out of the open span table
A request whose span was pushed out of the table of open spans, because the worker held more than 16384 open at once, ended that span under another key and was held from every sampled destination until shutdown. Pushing a span out now drops its request's open count too, so the request is decided as each of its spans ends, the way one pushed out of the open request table already is
This commit is contained in:
parent
f5bfb292e6
commit
11d5d91d17
3 changed files with 80 additions and 39 deletions
|
|
@ -694,7 +694,7 @@ class TenantFanOutSpanProcessor(SpanProcessor):
|
|||
self._exporting: dict[int, int] = {} # mutable-ok: per-processor in-flight export count
|
||||
self._pending: OrderedDict[_RequestKey, _PendingTree] = OrderedDict() # mutable-ok: bounded, by request
|
||||
self._open: OrderedDict[_RequestKey, int] = OrderedDict() # mutable-ok: bounded, spans in flight by request
|
||||
self._roots: OrderedDict[int, int] = OrderedDict() # mutable-ok: bounded, root span id by open span id
|
||||
self._roots: OrderedDict[int, _RequestKey] = OrderedDict() # mutable-ok: bounded, request by open span id
|
||||
self._span_roots: weakref.WeakKeyDictionary[SDKSpan, int] = weakref.WeakKeyDictionary()
|
||||
self._verdicts: OrderedDict[_VerdictKey, bool] = OrderedDict() # mutable-ok: bounded LRU
|
||||
self._drain: Final = drain_pool if drain_pool is not None else _DrainPool(capacity=pending_drains)
|
||||
|
|
@ -705,9 +705,9 @@ class TenantFanOutSpanProcessor(SpanProcessor):
|
|||
request: Final = self._request_of(span, trace.get_current_span(parent_context))
|
||||
if context is not None:
|
||||
self._span_roots[span] = request[1]
|
||||
self._roots[context.span_id] = request[1]
|
||||
self._roots[context.span_id] = request
|
||||
while len(self._roots) > _MAX_OPEN_SPANS:
|
||||
self._roots.popitem(last=False)
|
||||
self._open.pop(self._roots.popitem(last=False)[1], None)
|
||||
self._open[request] = self._open.get(request, 0) + 1
|
||||
while len(self._open) > _MAX_OPEN_REQUESTS:
|
||||
self._open.popitem(last=False)
|
||||
|
|
@ -741,22 +741,22 @@ class TenantFanOutSpanProcessor(SpanProcessor):
|
|||
self._forward(held_span, held_destination)
|
||||
|
||||
def _request_of(self, span: ReadableSpan, parent_span: Span | None = None) -> _RequestKey:
|
||||
"""The request ``span`` belongs to, read under ``_lock``: the root recorded while it
|
||||
is open, else the span itself when its parent is remote or missing, else its
|
||||
parent's root, recorded while the parent is open or on ``parent_span`` for a child
|
||||
starting under a parent that has ended, else the parent itself."""
|
||||
"""The request ``span`` belongs to, read under ``_lock``: the request recorded while
|
||||
it is open, else the span itself when its parent is remote or missing, else its
|
||||
parent's request, recorded while the parent is open or on ``parent_span`` for a
|
||||
child starting under a parent that has ended, else the parent itself."""
|
||||
context: Final = span.context
|
||||
if context is None:
|
||||
return (0, 0)
|
||||
recorded: Final = self._roots.get(context.span_id)
|
||||
if recorded is not None:
|
||||
return (context.trace_id, recorded)
|
||||
return recorded
|
||||
parent: Final = span.parent
|
||||
if parent is None or parent.is_remote:
|
||||
return (context.trace_id, context.span_id)
|
||||
root: Final = self._roots.get(parent.span_id)
|
||||
if root is not None:
|
||||
return (context.trace_id, root)
|
||||
parent_request: Final = self._roots.get(parent.span_id)
|
||||
if parent_request is not None:
|
||||
return parent_request
|
||||
if isinstance(parent_span, SDKSpan):
|
||||
return (context.trace_id, self._span_roots.get(parent_span, parent.span_id))
|
||||
return (context.trace_id, parent.span_id)
|
||||
|
|
|
|||
|
|
@ -643,28 +643,31 @@ def test_a_stalled_team_space_during_a_burst_keeps_serving_and_each_trace_whole(
|
|||
|
||||
|
||||
def _write_config(directory: Path, sinks: SpanSinks, provider: Wire) -> Path:
|
||||
config: Final = object_value(
|
||||
shared: Final = object_value(
|
||||
JSON.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
|
||||
)
|
||||
config["model_list"] = [
|
||||
*list_value(config.get("model_list") or []),
|
||||
{
|
||||
"model_name": MODEL,
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-4o-mini",
|
||||
"api_key": "integration-provider-key",
|
||||
"api_base": provider.url + "/v1",
|
||||
config: Final = {
|
||||
**shared,
|
||||
"model_list": [
|
||||
*list_value(shared.get("model_list") or []),
|
||||
{
|
||||
"model_name": MODEL,
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-4o-mini",
|
||||
"api_key": "integration-provider-key",
|
||||
"api_base": provider.url + "/v1",
|
||||
},
|
||||
},
|
||||
],
|
||||
"litellm_settings": {
|
||||
**object_value(shared.get("litellm_settings") or {}),
|
||||
"callbacks": ["langfuse_otel"],
|
||||
"otel_tenant_destination_mode": "additive",
|
||||
"provider_url_destination_allowed_hosts": [urlparse(sinks.operator).netloc, urlparse(sinks.arize).netloc],
|
||||
},
|
||||
]
|
||||
config["litellm_settings"] = {
|
||||
**object_value(config.get("litellm_settings") or {}),
|
||||
"callbacks": ["langfuse_otel"],
|
||||
"otel_tenant_destination_mode": "additive",
|
||||
"provider_url_destination_allowed_hosts": [urlparse(sinks.operator).netloc, urlparse(sinks.arize).netloc],
|
||||
"router_settings": {**object_value(shared.get("router_settings") or {}), "num_retries": 0},
|
||||
"general_settings": {**object_value(shared.get("general_settings") or {}), "user_api_key_cache_ttl": 2},
|
||||
}
|
||||
config["router_settings"] = {**object_value(config.get("router_settings") or {}), "num_retries": 0}
|
||||
config["general_settings"] = {**object_value(config.get("general_settings") or {}), "user_api_key_cache_ttl": 2}
|
||||
path: Final = directory / "team-sampling.yaml"
|
||||
path.write_text(yaml.safe_dump(config))
|
||||
return path
|
||||
|
|
|
|||
|
|
@ -1447,10 +1447,10 @@ class TestTenantSampling:
|
|||
def test_a_span_two_levels_down_that_outlives_many_other_spans_still_closes_its_request(self):
|
||||
"""A classifier call under the route span that stays open while the worker starts
|
||||
more spans than it tracks at once still counts against its own request, so the
|
||||
request reaches the space when it ends rather than being held."""
|
||||
dest_exporter = InMemorySpanExporter()
|
||||
provider = self._fan_out(dest_exporter)
|
||||
destinations = (arize_destination(arize_success_sampling_rate="1.0"),)
|
||||
request is decided once it ends, on its own outcome."""
|
||||
dest_exporter: Final = InMemorySpanExporter()
|
||||
provider: Final = self._fan_out(dest_exporter)
|
||||
destinations: Final = (arize_destination(arize_success_sampling_rate="0.0", arize_error_sampling_rate="1.0"),)
|
||||
|
||||
def other_request():
|
||||
set_request_destinations(())
|
||||
|
|
@ -1459,14 +1459,15 @@ class TestTenantSampling:
|
|||
|
||||
def run():
|
||||
set_request_destinations(destinations)
|
||||
tracer = get_tracer(provider, "litellm")
|
||||
with tracer.start_as_current_span("POST /v1/chat/completions"):
|
||||
tracer: Final = get_tracer(provider, "litellm")
|
||||
with tracer.start_as_current_span("POST /v1/chat/completions") as root:
|
||||
with tracer.start_as_current_span("route gpt-4"):
|
||||
with tracer.start_as_current_span("chat classifier"):
|
||||
for _ in range(otel_providers._MAX_OPEN_SPANS + 1):
|
||||
contextvars.Context().run(other_request)
|
||||
with tracer.start_as_current_span("chat gpt-4"):
|
||||
pass
|
||||
root.set_status(Status(StatusCode.ERROR, "upstream 500"))
|
||||
|
||||
in_fresh_context(run)
|
||||
|
||||
|
|
@ -1477,20 +1478,57 @@ class TestTenantSampling:
|
|||
"chat gpt-4",
|
||||
}
|
||||
|
||||
def test_a_request_whose_spans_fall_out_of_the_open_span_bound_is_still_decided(self):
|
||||
"""A request still open while the worker holds more open spans than it tracks is
|
||||
decided as each of its spans ends, rather than held for want of a span it can no
|
||||
longer place."""
|
||||
dest_exporter: Final = InMemorySpanExporter()
|
||||
provider: Final = self._fan_out(dest_exporter)
|
||||
destinations: Final = (arize_destination(arize_success_sampling_rate="1.0"),)
|
||||
others: Final = contextvars.Context()
|
||||
|
||||
def open_other_request():
|
||||
set_request_destinations(())
|
||||
tracer: Final = get_tracer(provider, "litellm")
|
||||
root: Final = tracer.start_span("POST /v1/chat/completions")
|
||||
parent: Final = trace_api.set_span_in_context(root)
|
||||
return (
|
||||
root,
|
||||
*(tracer.start_span("chat flood", context=parent) for _ in range(otel_providers._MAX_OPEN_SPANS)),
|
||||
)
|
||||
|
||||
def run():
|
||||
set_request_destinations(destinations)
|
||||
tracer: Final = get_tracer(provider, "litellm")
|
||||
with tracer.start_as_current_span("POST /v1/chat/completions"):
|
||||
with tracer.start_as_current_span("route gpt-4"):
|
||||
with tracer.start_as_current_span("chat classifier"):
|
||||
return others.run(open_other_request)
|
||||
|
||||
held_open: Final = in_fresh_context(run)
|
||||
|
||||
assert {s.name for s in dest_exporter.get_finished_spans()} == {
|
||||
"POST /v1/chat/completions",
|
||||
"route gpt-4",
|
||||
"chat classifier",
|
||||
}
|
||||
for span in held_open:
|
||||
span.end()
|
||||
|
||||
def test_a_span_starting_under_an_ended_span_below_the_root_follows_its_request(self):
|
||||
"""A post-call write started under the route span after its failed request ended
|
||||
is kept with the request at the error rate, rather than decided on its own at the
|
||||
success rate."""
|
||||
dest_exporter = InMemorySpanExporter()
|
||||
provider = self._fan_out(dest_exporter)
|
||||
destinations = (arize_destination(arize_success_sampling_rate="0.0", arize_error_sampling_rate="1.0"),)
|
||||
dest_exporter: Final = InMemorySpanExporter()
|
||||
provider: Final = self._fan_out(dest_exporter)
|
||||
destinations: Final = (arize_destination(arize_success_sampling_rate="0.0", arize_error_sampling_rate="1.0"),)
|
||||
|
||||
def run():
|
||||
set_request_destinations(destinations)
|
||||
tracer = get_tracer(provider, "litellm")
|
||||
tracer: Final = get_tracer(provider, "litellm")
|
||||
with tracer.start_as_current_span("POST /v1/chat/completions") as root:
|
||||
with tracer.start_as_current_span("route gpt-4") as route:
|
||||
route_context = trace_api.set_span_in_context(route)
|
||||
route_context: Final = trace_api.set_span_in_context(route)
|
||||
root.set_status(Status(StatusCode.ERROR, "upstream 500"))
|
||||
with tracer.start_as_current_span("postgres INSERT LiteLLM_SpendLogs", context=route_context):
|
||||
pass
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue