test(e2e): identify trace roots by kind and request path

This commit is contained in:
Yuneng Jiang 2026-11-02 22:24:00 -08:00
parent 99d875abf1
commit 2411917468
No known key found for this signature in database
4 changed files with 116 additions and 26 deletions

View file

@ -79,7 +79,7 @@ def _chain_reaches(span_id: str, root_id: str, trace: JaegerTrace) -> bool:
return False
def _assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) -> None:
def assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) -> None:
"""The enforced behavior: the destination holds exactly one call-id-tagged
trace for the call, rooted at the SERVER span, with auth/db children and
the gen-AI span all connected into that one tree - no dangling parent
@ -112,10 +112,9 @@ def _assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) -
roots = [span for span in trace.spans if not span.references]
assert len(roots) == 1, f"expected exactly one root span, got {[s.operation_name for s in roots]}; spans: {names}"
root = roots[0]
assert root.operation_name == f"POST {route}", (
f"the root must be the SERVER span 'POST {route}', got {root.operation_name!r}"
)
assert root.kind == "server", f"the root span must have kind=server, got {root.kind!r}"
request_route: Final = root.tag("http.route") or root.tag("url.path") or root.tag("http.target")
assert request_route == route, f"the root must identify route {route!r}, got {request_route!r}"
assert f"auth {route}" in names, f"auth phase span 'auth {route}' missing; spans: {names}"
assert any(name.startswith(DB_SPAN_PREFIX) for name in names), (
@ -133,12 +132,12 @@ def _assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) -
def _poll(otel_reader: OtelReader, *, call_id: str, route: str, genai_span: str) -> CallTraces:
return otel_reader.poll_traces_for_call(
call_id=call_id,
settled_names={f"POST {route}", f"auth {route}", genai_span},
settled_names={f"auth {route}", genai_span},
settled_prefixes={DB_SPAN_PREFIX},
)
def _assert_real_ttft(hits: tuple[JaegerTrace, ...], *, genai_span: str) -> None:
def assert_real_ttft(hits: tuple[JaegerTrace, ...], *, genai_span: str) -> None:
"""The enforced behavior: the gen-AI span for the attempt that served the
stream records a TTFT that is a real measurement - present, numeric,
positive, and strictly less than that span's own total duration. A TTFT of
@ -282,7 +281,7 @@ class TestOtelTraceCompleteness:
assert outcome.call_id is not None, "success response must carry x-litellm-call-id"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=f"chat {MODEL}")
_assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}")
assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}")
@pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["chat_completions"])
@meta(
@ -315,7 +314,7 @@ class TestOtelTraceCompleteness:
assert outcome.call_id is not None, "success response must carry x-litellm-call-id"
hits: Final = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=f"chat {MODEL}")
_assert_complete_trace(hits, route=route, genai_span=f"chat {MODEL}")
assert_complete_trace(hits, route=route, genai_span=f"chat {MODEL}")
@pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["messages"])
@meta(
@ -333,7 +332,7 @@ class TestOtelTraceCompleteness:
"""This test verifies that one successful non-streaming /v1/messages request
produces exactly one complete OTEL trace.
The trace must have a single root span named "POST /v1/messages". The
The trace must have a single root SERVER span identifying /v1/messages. The
authentication, database, and model-call spans must all belong to
the same trace and have valid parent relationships leading back to that root.
@ -353,7 +352,7 @@ class TestOtelTraceCompleteness:
assert outcome.call_id is not None, "success response must carry x-litellm-call-id"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=f"chat {MODEL}")
_assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}")
assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}")
@pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["responses"])
@meta(
@ -392,7 +391,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {CHEAP_OPENAI_MODEL}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
_assert_complete_trace(traces, route=route, genai_span=genai_span)
assert_complete_trace(traces, route=route, genai_span=genai_span)
@pytest.mark.covers("logging.otel.stream.exports_metric", exercised_on=["chat_completions"])
@meta(
@ -443,7 +442,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {MODEL}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
_assert_complete_trace(traces, route=route, genai_span=genai_span)
assert_complete_trace(traces, route=route, genai_span=genai_span)
served = one_served_genai_span(traces.hits[0], genai_span)
assert served.tag("litellm.request.streaming") is True, (
@ -500,7 +499,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {MODEL}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
_assert_complete_trace(traces, route=route, genai_span=genai_span)
assert_complete_trace(traces, route=route, genai_span=genai_span)
served = one_served_genai_span(traces.hits[0], genai_span)
assert served.tag("litellm.request.streaming") is True, (
@ -557,7 +556,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {CHEAP_OPENAI_MODEL}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
_assert_complete_trace(traces, route=route, genai_span=genai_span)
assert_complete_trace(traces, route=route, genai_span=genai_span)
one_served_genai_span(traces.hits[0], genai_span)
@ -614,7 +613,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {MODEL}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
_assert_real_ttft(traces.hits, genai_span=genai_span)
assert_real_ttft(traces.hits, genai_span=genai_span)
@pytest.mark.covers("logging.otel.stream.records_ttft", exercised_on=["messages"])
@meta(
@ -661,7 +660,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {MODEL}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
_assert_real_ttft(traces.hits, genai_span=genai_span)
assert_real_ttft(traces.hits, genai_span=genai_span)
@pytest.mark.covers("logging.otel.stream.records_ttft", exercised_on=["responses"])
@meta(
@ -708,7 +707,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {CHEAP_OPENAI_MODEL}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
_assert_real_ttft(traces.hits, genai_span=genai_span)
assert_real_ttft(traces.hits, genai_span=genai_span)
@pytest.mark.covers("logging.otel.failure.exports_metric", exercised_on=["chat_completions"])
@meta(
@ -763,7 +762,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {model_name}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
_assert_complete_trace(traces, route=route, genai_span=genai_span)
assert_complete_trace(traces, route=route, genai_span=genai_span)
root = next(span for span in traces.hits[0].spans if not span.references)
assert str(root.tag("http.status_code")) == "401", (
@ -825,7 +824,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {model_name}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
_assert_complete_trace(traces, route=route, genai_span=genai_span)
assert_complete_trace(traces, route=route, genai_span=genai_span)
root = next(span for span in traces.hits[0].spans if not span.references)
assert str(root.tag("http.status_code")) == "401", (

View file

@ -28,11 +28,10 @@ from dataclasses import dataclass
from typing import Final
import pytest
from pydantic import BaseModel, ConfigDict, Field
from e2e_config import OTEL_QUERY_URL, POLL_INTERVAL, POLL_TIMEOUT
from e2e_http import URL, NetworkError, NoBody, Result, Success, get
from e2e_metadata import step
from pydantic import BaseModel, ConfigDict, Field
#: OTEL resource service.name the proxy exports under (OTEL_SERVICE_NAME default).
JAEGER_SERVICE = "litellm"
@ -118,9 +117,15 @@ def _ticks() -> Iterator[None]:
time.sleep(POLL_INTERVAL)
def _settled(trace: JaegerTrace, names: set[str], prefixes: set[str]) -> bool:
present = set(trace.span_names())
return names.issubset(present) and all(any(name.startswith(prefix) for name in present) for prefix in prefixes)
def trace_settled(trace: JaegerTrace, names: set[str], prefixes: set[str]) -> bool:
root: Final = root_span(trace)
present: Final = frozenset(trace.span_names())
return (
root is not None
and root.kind == "server"
and names.issubset(present)
and all(any(name.startswith(prefix) for name in present) for prefix in prefixes)
)
def root_span(trace: JaegerTrace) -> JaegerSpan | None:
@ -172,7 +177,7 @@ class _Observation:
def settled(self, names: set[str], prefixes: set[str]) -> bool:
hits: Final = self.traces.hits
return self.unreachable is None and len(hits) == 1 and not self.missing and _settled(hits[0], names, prefixes)
return self.unreachable is None and len(hits) == 1 and not self.missing and trace_settled(hits[0], names, prefixes)
@dataclass(frozen=True, slots=True)

View file

@ -8,8 +8,21 @@ helper is worth pinning here.
from __future__ import annotations
from typing import Final
import pytest
from otel_client import TTFT_TAG, JaegerTrace, one_served_genai_span, served_genai_spans
from otel_client import (
TTFT_TAG,
CallTraces,
JaegerReference,
JaegerSpan,
JaegerTag,
JaegerTrace,
one_served_genai_span,
served_genai_spans,
trace_settled,
)
from test_otel_trace_e2e import assert_complete_trace, assert_real_ttft
GENAI_SPAN = "chat claude-haiku-4-5"
@ -79,3 +92,73 @@ def test_other_operations_are_not_counted() -> None:
trace = _trace(_span("chat gpt-5.5", ttft=0.3), _span(GENAI_SPAN, ttft=0.3))
assert len(served_genai_spans(trace, GENAI_SPAN)) == 1
def _request_trace(
*,
root_name: str = "POST",
root_kind: str = "server",
route: str = "/v1/messages",
genai_parent: str = "root",
ttft: float = 0.2,
) -> JaegerTrace:
parent: Final = JaegerReference(refType="CHILD_OF", traceID="request", spanID="root")
return JaegerTrace(
traceID="request",
spans=[
JaegerSpan(
spanID="root",
operationName=root_name,
tags=[JaegerTag(key="span.kind", value=root_kind), JaegerTag(key="http.target", value=route)],
),
JaegerSpan(spanID="auth", operationName="auth /v1/messages", references=[parent]),
JaegerSpan(spanID="db", operationName="postgres.select tokens", references=[parent]),
JaegerSpan(
spanID="genai",
operationName=GENAI_SPAN,
duration=1_000_000,
references=[JaegerReference(refType="CHILD_OF", traceID="request", spanID=genai_parent)],
tags=[JaegerTag(key="span.kind", value="client"), JaegerTag(key=TTFT_TAG, value=ttft)],
),
],
)
@pytest.mark.parametrize("root_name", ("POST", "POST /v1/messages"))
def test_connected_request_trace_is_independent_of_server_display_name(root_name: str) -> None:
trace: Final = _request_trace(root_name=root_name)
assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN)
assert_real_ttft((trace,), genai_span=GENAI_SPAN)
assert trace_settled(trace, {"auth /v1/messages", GENAI_SPAN}, {"postgres."})
def test_incomplete_trace_cannot_settle_before_the_server_span_arrives() -> None:
trace: Final = _request_trace()
incomplete: Final = trace.model_copy(update={"spans": trace.spans[1:]})
assert not trace_settled(incomplete, {"auth /v1/messages", GENAI_SPAN}, {"postgres."})
@pytest.mark.parametrize("route", ("/v1/responses", ""))
def test_request_trace_must_still_identify_the_requested_route(route: str) -> None:
trace: Final = _request_trace(root_name="POST /v1/messages", route=route)
with pytest.raises(AssertionError, match="route"):
assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN)
def test_request_trace_must_still_be_rooted_at_a_server_span() -> None:
trace: Final = _request_trace(root_name="POST /v1/messages", root_kind="client")
with pytest.raises(AssertionError, match="kind=server"):
assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN)
def test_request_trace_must_still_connect_the_model_call_to_the_request() -> None:
trace: Final = _request_trace(genai_parent="absent")
with pytest.raises(AssertionError, match="orphaned trace"):
assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN)
@pytest.mark.parametrize("ttft", (0.0, 1.0))
def test_server_name_tolerance_does_not_accept_invalid_first_token_timing(ttft: float) -> None:
trace: Final = _request_trace(ttft=ttft)
with pytest.raises(AssertionError, match="TTFT"):
assert_real_ttft((trace,), genai_span=GENAI_SPAN)

View file

@ -5,5 +5,8 @@
addopts = --strict-markers --strict-config -p no:cacheprovider
pythonpath = ../e2e ../e2e/batches ../e2e/guardrails ../e2e/logging ../e2e/perf
markers =
e2e: live assertion modules imported by harness regression tests
meta(subject): metadata on live assertion modules imported by harness regression tests
otel_tls: TLS coverage in live assertion modules imported by harness regression tests
covers(cell_id, *, exercised_on=()): coverage-registry cell(s) a test covers; exercised here only to prove the collector and the JUnit properties read it
cli_determinism: drives the real claude CLI for several seconds