From 2411917468ebd350dcb89ad9533cad81bb9dc276 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Mon, 2 Nov 2026 22:24:00 -0800 Subject: [PATCH] test(e2e): identify trace roots by kind and request path --- tests/e2e/logging/test_otel_trace_e2e.py | 37 ++++---- tests/e2e/otel_client.py | 17 ++-- .../logging/test_span_selection.py | 85 ++++++++++++++++++- tests/e2e_harness/pytest.ini | 3 + 4 files changed, 116 insertions(+), 26 deletions(-) diff --git a/tests/e2e/logging/test_otel_trace_e2e.py b/tests/e2e/logging/test_otel_trace_e2e.py index 81a6407f642..fb6e481dc33 100644 --- a/tests/e2e/logging/test_otel_trace_e2e.py +++ b/tests/e2e/logging/test_otel_trace_e2e.py @@ -79,7 +79,7 @@ def _chain_reaches(span_id: str, root_id: str, trace: JaegerTrace) -> bool: return False -def _assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) -> None: +def assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) -> None: """The enforced behavior: the destination holds exactly one call-id-tagged trace for the call, rooted at the SERVER span, with auth/db children and the gen-AI span all connected into that one tree - no dangling parent @@ -112,10 +112,9 @@ def _assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) - roots = [span for span in trace.spans if not span.references] assert len(roots) == 1, f"expected exactly one root span, got {[s.operation_name for s in roots]}; spans: {names}" root = roots[0] - assert root.operation_name == f"POST {route}", ( - f"the root must be the SERVER span 'POST {route}', got {root.operation_name!r}" - ) assert root.kind == "server", f"the root span must have kind=server, got {root.kind!r}" + request_route: Final = root.tag("http.route") or root.tag("url.path") or root.tag("http.target") + assert request_route == route, f"the root must identify route {route!r}, got {request_route!r}" assert f"auth {route}" in names, f"auth phase span 'auth {route}' missing; spans: {names}" assert any(name.startswith(DB_SPAN_PREFIX) for name in names), ( @@ -133,12 +132,12 @@ def _assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) - def _poll(otel_reader: OtelReader, *, call_id: str, route: str, genai_span: str) -> CallTraces: return otel_reader.poll_traces_for_call( call_id=call_id, - settled_names={f"POST {route}", f"auth {route}", genai_span}, + settled_names={f"auth {route}", genai_span}, settled_prefixes={DB_SPAN_PREFIX}, ) -def _assert_real_ttft(hits: tuple[JaegerTrace, ...], *, genai_span: str) -> None: +def assert_real_ttft(hits: tuple[JaegerTrace, ...], *, genai_span: str) -> None: """The enforced behavior: the gen-AI span for the attempt that served the stream records a TTFT that is a real measurement - present, numeric, positive, and strictly less than that span's own total duration. A TTFT of @@ -282,7 +281,7 @@ class TestOtelTraceCompleteness: assert outcome.call_id is not None, "success response must carry x-litellm-call-id" traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=f"chat {MODEL}") - _assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}") + assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}") @pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["chat_completions"]) @meta( @@ -315,7 +314,7 @@ class TestOtelTraceCompleteness: assert outcome.call_id is not None, "success response must carry x-litellm-call-id" hits: Final = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=f"chat {MODEL}") - _assert_complete_trace(hits, route=route, genai_span=f"chat {MODEL}") + assert_complete_trace(hits, route=route, genai_span=f"chat {MODEL}") @pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["messages"]) @meta( @@ -333,7 +332,7 @@ class TestOtelTraceCompleteness: """This test verifies that one successful non-streaming /v1/messages request produces exactly one complete OTEL trace. - The trace must have a single root span named "POST /v1/messages". The + The trace must have a single root SERVER span identifying /v1/messages. The authentication, database, and model-call spans must all belong to the same trace and have valid parent relationships leading back to that root. @@ -353,7 +352,7 @@ class TestOtelTraceCompleteness: assert outcome.call_id is not None, "success response must carry x-litellm-call-id" traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=f"chat {MODEL}") - _assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}") + assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}") @pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["responses"]) @meta( @@ -392,7 +391,7 @@ class TestOtelTraceCompleteness: genai_span = f"chat {CHEAP_OPENAI_MODEL}" traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span) - _assert_complete_trace(traces, route=route, genai_span=genai_span) + assert_complete_trace(traces, route=route, genai_span=genai_span) @pytest.mark.covers("logging.otel.stream.exports_metric", exercised_on=["chat_completions"]) @meta( @@ -443,7 +442,7 @@ class TestOtelTraceCompleteness: genai_span = f"chat {MODEL}" traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span) - _assert_complete_trace(traces, route=route, genai_span=genai_span) + assert_complete_trace(traces, route=route, genai_span=genai_span) served = one_served_genai_span(traces.hits[0], genai_span) assert served.tag("litellm.request.streaming") is True, ( @@ -500,7 +499,7 @@ class TestOtelTraceCompleteness: genai_span = f"chat {MODEL}" traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span) - _assert_complete_trace(traces, route=route, genai_span=genai_span) + assert_complete_trace(traces, route=route, genai_span=genai_span) served = one_served_genai_span(traces.hits[0], genai_span) assert served.tag("litellm.request.streaming") is True, ( @@ -557,7 +556,7 @@ class TestOtelTraceCompleteness: genai_span = f"chat {CHEAP_OPENAI_MODEL}" traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span) - _assert_complete_trace(traces, route=route, genai_span=genai_span) + assert_complete_trace(traces, route=route, genai_span=genai_span) one_served_genai_span(traces.hits[0], genai_span) @@ -614,7 +613,7 @@ class TestOtelTraceCompleteness: genai_span = f"chat {MODEL}" traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span) - _assert_real_ttft(traces.hits, genai_span=genai_span) + assert_real_ttft(traces.hits, genai_span=genai_span) @pytest.mark.covers("logging.otel.stream.records_ttft", exercised_on=["messages"]) @meta( @@ -661,7 +660,7 @@ class TestOtelTraceCompleteness: genai_span = f"chat {MODEL}" traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span) - _assert_real_ttft(traces.hits, genai_span=genai_span) + assert_real_ttft(traces.hits, genai_span=genai_span) @pytest.mark.covers("logging.otel.stream.records_ttft", exercised_on=["responses"]) @meta( @@ -708,7 +707,7 @@ class TestOtelTraceCompleteness: genai_span = f"chat {CHEAP_OPENAI_MODEL}" traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span) - _assert_real_ttft(traces.hits, genai_span=genai_span) + assert_real_ttft(traces.hits, genai_span=genai_span) @pytest.mark.covers("logging.otel.failure.exports_metric", exercised_on=["chat_completions"]) @meta( @@ -763,7 +762,7 @@ class TestOtelTraceCompleteness: genai_span = f"chat {model_name}" traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span) - _assert_complete_trace(traces, route=route, genai_span=genai_span) + assert_complete_trace(traces, route=route, genai_span=genai_span) root = next(span for span in traces.hits[0].spans if not span.references) assert str(root.tag("http.status_code")) == "401", ( @@ -825,7 +824,7 @@ class TestOtelTraceCompleteness: genai_span = f"chat {model_name}" traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span) - _assert_complete_trace(traces, route=route, genai_span=genai_span) + assert_complete_trace(traces, route=route, genai_span=genai_span) root = next(span for span in traces.hits[0].spans if not span.references) assert str(root.tag("http.status_code")) == "401", ( diff --git a/tests/e2e/otel_client.py b/tests/e2e/otel_client.py index 5f93c677d89..b42235484dc 100644 --- a/tests/e2e/otel_client.py +++ b/tests/e2e/otel_client.py @@ -28,11 +28,10 @@ from dataclasses import dataclass from typing import Final import pytest -from pydantic import BaseModel, ConfigDict, Field - from e2e_config import OTEL_QUERY_URL, POLL_INTERVAL, POLL_TIMEOUT from e2e_http import URL, NetworkError, NoBody, Result, Success, get from e2e_metadata import step +from pydantic import BaseModel, ConfigDict, Field #: OTEL resource service.name the proxy exports under (OTEL_SERVICE_NAME default). JAEGER_SERVICE = "litellm" @@ -118,9 +117,15 @@ def _ticks() -> Iterator[None]: time.sleep(POLL_INTERVAL) -def _settled(trace: JaegerTrace, names: set[str], prefixes: set[str]) -> bool: - present = set(trace.span_names()) - return names.issubset(present) and all(any(name.startswith(prefix) for name in present) for prefix in prefixes) +def trace_settled(trace: JaegerTrace, names: set[str], prefixes: set[str]) -> bool: + root: Final = root_span(trace) + present: Final = frozenset(trace.span_names()) + return ( + root is not None + and root.kind == "server" + and names.issubset(present) + and all(any(name.startswith(prefix) for name in present) for prefix in prefixes) + ) def root_span(trace: JaegerTrace) -> JaegerSpan | None: @@ -172,7 +177,7 @@ class _Observation: def settled(self, names: set[str], prefixes: set[str]) -> bool: hits: Final = self.traces.hits - return self.unreachable is None and len(hits) == 1 and not self.missing and _settled(hits[0], names, prefixes) + return self.unreachable is None and len(hits) == 1 and not self.missing and trace_settled(hits[0], names, prefixes) @dataclass(frozen=True, slots=True) diff --git a/tests/e2e_harness/logging/test_span_selection.py b/tests/e2e_harness/logging/test_span_selection.py index e7487562e43..76f70e4d8bb 100644 --- a/tests/e2e_harness/logging/test_span_selection.py +++ b/tests/e2e_harness/logging/test_span_selection.py @@ -8,8 +8,21 @@ helper is worth pinning here. from __future__ import annotations +from typing import Final + import pytest -from otel_client import TTFT_TAG, JaegerTrace, one_served_genai_span, served_genai_spans +from otel_client import ( + TTFT_TAG, + CallTraces, + JaegerReference, + JaegerSpan, + JaegerTag, + JaegerTrace, + one_served_genai_span, + served_genai_spans, + trace_settled, +) +from test_otel_trace_e2e import assert_complete_trace, assert_real_ttft GENAI_SPAN = "chat claude-haiku-4-5" @@ -79,3 +92,73 @@ def test_other_operations_are_not_counted() -> None: trace = _trace(_span("chat gpt-5.5", ttft=0.3), _span(GENAI_SPAN, ttft=0.3)) assert len(served_genai_spans(trace, GENAI_SPAN)) == 1 + + +def _request_trace( + *, + root_name: str = "POST", + root_kind: str = "server", + route: str = "/v1/messages", + genai_parent: str = "root", + ttft: float = 0.2, +) -> JaegerTrace: + parent: Final = JaegerReference(refType="CHILD_OF", traceID="request", spanID="root") + return JaegerTrace( + traceID="request", + spans=[ + JaegerSpan( + spanID="root", + operationName=root_name, + tags=[JaegerTag(key="span.kind", value=root_kind), JaegerTag(key="http.target", value=route)], + ), + JaegerSpan(spanID="auth", operationName="auth /v1/messages", references=[parent]), + JaegerSpan(spanID="db", operationName="postgres.select tokens", references=[parent]), + JaegerSpan( + spanID="genai", + operationName=GENAI_SPAN, + duration=1_000_000, + references=[JaegerReference(refType="CHILD_OF", traceID="request", spanID=genai_parent)], + tags=[JaegerTag(key="span.kind", value="client"), JaegerTag(key=TTFT_TAG, value=ttft)], + ), + ], + ) + + +@pytest.mark.parametrize("root_name", ("POST", "POST /v1/messages")) +def test_connected_request_trace_is_independent_of_server_display_name(root_name: str) -> None: + trace: Final = _request_trace(root_name=root_name) + assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN) + assert_real_ttft((trace,), genai_span=GENAI_SPAN) + assert trace_settled(trace, {"auth /v1/messages", GENAI_SPAN}, {"postgres."}) + + +def test_incomplete_trace_cannot_settle_before_the_server_span_arrives() -> None: + trace: Final = _request_trace() + incomplete: Final = trace.model_copy(update={"spans": trace.spans[1:]}) + assert not trace_settled(incomplete, {"auth /v1/messages", GENAI_SPAN}, {"postgres."}) + + +@pytest.mark.parametrize("route", ("/v1/responses", "")) +def test_request_trace_must_still_identify_the_requested_route(route: str) -> None: + trace: Final = _request_trace(root_name="POST /v1/messages", route=route) + with pytest.raises(AssertionError, match="route"): + assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN) + + +def test_request_trace_must_still_be_rooted_at_a_server_span() -> None: + trace: Final = _request_trace(root_name="POST /v1/messages", root_kind="client") + with pytest.raises(AssertionError, match="kind=server"): + assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN) + + +def test_request_trace_must_still_connect_the_model_call_to_the_request() -> None: + trace: Final = _request_trace(genai_parent="absent") + with pytest.raises(AssertionError, match="orphaned trace"): + assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN) + + +@pytest.mark.parametrize("ttft", (0.0, 1.0)) +def test_server_name_tolerance_does_not_accept_invalid_first_token_timing(ttft: float) -> None: + trace: Final = _request_trace(ttft=ttft) + with pytest.raises(AssertionError, match="TTFT"): + assert_real_ttft((trace,), genai_span=GENAI_SPAN) diff --git a/tests/e2e_harness/pytest.ini b/tests/e2e_harness/pytest.ini index a29166d31bd..9ccf8ba83a0 100644 --- a/tests/e2e_harness/pytest.ini +++ b/tests/e2e_harness/pytest.ini @@ -5,5 +5,8 @@ addopts = --strict-markers --strict-config -p no:cacheprovider pythonpath = ../e2e ../e2e/batches ../e2e/guardrails ../e2e/logging ../e2e/perf markers = + e2e: live assertion modules imported by harness regression tests + meta(subject): metadata on live assertion modules imported by harness regression tests + otel_tls: TLS coverage in live assertion modules imported by harness regression tests covers(cell_id, *, exercised_on=()): coverage-registry cell(s) a test covers; exercised here only to prove the collector and the JUnit properties read it cli_determinism: drives the real claude CLI for several seconds