mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
test(e2e): identify trace roots by kind and request path
This commit is contained in:
parent
99d875abf1
commit
2411917468
4 changed files with 116 additions and 26 deletions
|
|
@ -79,7 +79,7 @@ def _chain_reaches(span_id: str, root_id: str, trace: JaegerTrace) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
def _assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) -> None:
|
||||
def assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) -> None:
|
||||
"""The enforced behavior: the destination holds exactly one call-id-tagged
|
||||
trace for the call, rooted at the SERVER span, with auth/db children and
|
||||
the gen-AI span all connected into that one tree - no dangling parent
|
||||
|
|
@ -112,10 +112,9 @@ def _assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) -
|
|||
roots = [span for span in trace.spans if not span.references]
|
||||
assert len(roots) == 1, f"expected exactly one root span, got {[s.operation_name for s in roots]}; spans: {names}"
|
||||
root = roots[0]
|
||||
assert root.operation_name == f"POST {route}", (
|
||||
f"the root must be the SERVER span 'POST {route}', got {root.operation_name!r}"
|
||||
)
|
||||
assert root.kind == "server", f"the root span must have kind=server, got {root.kind!r}"
|
||||
request_route: Final = root.tag("http.route") or root.tag("url.path") or root.tag("http.target")
|
||||
assert request_route == route, f"the root must identify route {route!r}, got {request_route!r}"
|
||||
|
||||
assert f"auth {route}" in names, f"auth phase span 'auth {route}' missing; spans: {names}"
|
||||
assert any(name.startswith(DB_SPAN_PREFIX) for name in names), (
|
||||
|
|
@ -133,12 +132,12 @@ def _assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) -
|
|||
def _poll(otel_reader: OtelReader, *, call_id: str, route: str, genai_span: str) -> CallTraces:
|
||||
return otel_reader.poll_traces_for_call(
|
||||
call_id=call_id,
|
||||
settled_names={f"POST {route}", f"auth {route}", genai_span},
|
||||
settled_names={f"auth {route}", genai_span},
|
||||
settled_prefixes={DB_SPAN_PREFIX},
|
||||
)
|
||||
|
||||
|
||||
def _assert_real_ttft(hits: tuple[JaegerTrace, ...], *, genai_span: str) -> None:
|
||||
def assert_real_ttft(hits: tuple[JaegerTrace, ...], *, genai_span: str) -> None:
|
||||
"""The enforced behavior: the gen-AI span for the attempt that served the
|
||||
stream records a TTFT that is a real measurement - present, numeric,
|
||||
positive, and strictly less than that span's own total duration. A TTFT of
|
||||
|
|
@ -282,7 +281,7 @@ class TestOtelTraceCompleteness:
|
|||
assert outcome.call_id is not None, "success response must carry x-litellm-call-id"
|
||||
|
||||
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=f"chat {MODEL}")
|
||||
_assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}")
|
||||
assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}")
|
||||
|
||||
@pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["chat_completions"])
|
||||
@meta(
|
||||
|
|
@ -315,7 +314,7 @@ class TestOtelTraceCompleteness:
|
|||
assert outcome.call_id is not None, "success response must carry x-litellm-call-id"
|
||||
|
||||
hits: Final = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=f"chat {MODEL}")
|
||||
_assert_complete_trace(hits, route=route, genai_span=f"chat {MODEL}")
|
||||
assert_complete_trace(hits, route=route, genai_span=f"chat {MODEL}")
|
||||
|
||||
@pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["messages"])
|
||||
@meta(
|
||||
|
|
@ -333,7 +332,7 @@ class TestOtelTraceCompleteness:
|
|||
"""This test verifies that one successful non-streaming /v1/messages request
|
||||
produces exactly one complete OTEL trace.
|
||||
|
||||
The trace must have a single root span named "POST /v1/messages". The
|
||||
The trace must have a single root SERVER span identifying /v1/messages. The
|
||||
authentication, database, and model-call spans must all belong to
|
||||
the same trace and have valid parent relationships leading back to that root.
|
||||
|
||||
|
|
@ -353,7 +352,7 @@ class TestOtelTraceCompleteness:
|
|||
assert outcome.call_id is not None, "success response must carry x-litellm-call-id"
|
||||
|
||||
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=f"chat {MODEL}")
|
||||
_assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}")
|
||||
assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}")
|
||||
|
||||
@pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["responses"])
|
||||
@meta(
|
||||
|
|
@ -392,7 +391,7 @@ class TestOtelTraceCompleteness:
|
|||
|
||||
genai_span = f"chat {CHEAP_OPENAI_MODEL}"
|
||||
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
|
||||
_assert_complete_trace(traces, route=route, genai_span=genai_span)
|
||||
assert_complete_trace(traces, route=route, genai_span=genai_span)
|
||||
|
||||
@pytest.mark.covers("logging.otel.stream.exports_metric", exercised_on=["chat_completions"])
|
||||
@meta(
|
||||
|
|
@ -443,7 +442,7 @@ class TestOtelTraceCompleteness:
|
|||
|
||||
genai_span = f"chat {MODEL}"
|
||||
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
|
||||
_assert_complete_trace(traces, route=route, genai_span=genai_span)
|
||||
assert_complete_trace(traces, route=route, genai_span=genai_span)
|
||||
|
||||
served = one_served_genai_span(traces.hits[0], genai_span)
|
||||
assert served.tag("litellm.request.streaming") is True, (
|
||||
|
|
@ -500,7 +499,7 @@ class TestOtelTraceCompleteness:
|
|||
|
||||
genai_span = f"chat {MODEL}"
|
||||
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
|
||||
_assert_complete_trace(traces, route=route, genai_span=genai_span)
|
||||
assert_complete_trace(traces, route=route, genai_span=genai_span)
|
||||
|
||||
served = one_served_genai_span(traces.hits[0], genai_span)
|
||||
assert served.tag("litellm.request.streaming") is True, (
|
||||
|
|
@ -557,7 +556,7 @@ class TestOtelTraceCompleteness:
|
|||
|
||||
genai_span = f"chat {CHEAP_OPENAI_MODEL}"
|
||||
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
|
||||
_assert_complete_trace(traces, route=route, genai_span=genai_span)
|
||||
assert_complete_trace(traces, route=route, genai_span=genai_span)
|
||||
|
||||
one_served_genai_span(traces.hits[0], genai_span)
|
||||
|
||||
|
|
@ -614,7 +613,7 @@ class TestOtelTraceCompleteness:
|
|||
|
||||
genai_span = f"chat {MODEL}"
|
||||
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
|
||||
_assert_real_ttft(traces.hits, genai_span=genai_span)
|
||||
assert_real_ttft(traces.hits, genai_span=genai_span)
|
||||
|
||||
@pytest.mark.covers("logging.otel.stream.records_ttft", exercised_on=["messages"])
|
||||
@meta(
|
||||
|
|
@ -661,7 +660,7 @@ class TestOtelTraceCompleteness:
|
|||
|
||||
genai_span = f"chat {MODEL}"
|
||||
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
|
||||
_assert_real_ttft(traces.hits, genai_span=genai_span)
|
||||
assert_real_ttft(traces.hits, genai_span=genai_span)
|
||||
|
||||
@pytest.mark.covers("logging.otel.stream.records_ttft", exercised_on=["responses"])
|
||||
@meta(
|
||||
|
|
@ -708,7 +707,7 @@ class TestOtelTraceCompleteness:
|
|||
|
||||
genai_span = f"chat {CHEAP_OPENAI_MODEL}"
|
||||
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
|
||||
_assert_real_ttft(traces.hits, genai_span=genai_span)
|
||||
assert_real_ttft(traces.hits, genai_span=genai_span)
|
||||
|
||||
@pytest.mark.covers("logging.otel.failure.exports_metric", exercised_on=["chat_completions"])
|
||||
@meta(
|
||||
|
|
@ -763,7 +762,7 @@ class TestOtelTraceCompleteness:
|
|||
|
||||
genai_span = f"chat {model_name}"
|
||||
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
|
||||
_assert_complete_trace(traces, route=route, genai_span=genai_span)
|
||||
assert_complete_trace(traces, route=route, genai_span=genai_span)
|
||||
|
||||
root = next(span for span in traces.hits[0].spans if not span.references)
|
||||
assert str(root.tag("http.status_code")) == "401", (
|
||||
|
|
@ -825,7 +824,7 @@ class TestOtelTraceCompleteness:
|
|||
|
||||
genai_span = f"chat {model_name}"
|
||||
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
|
||||
_assert_complete_trace(traces, route=route, genai_span=genai_span)
|
||||
assert_complete_trace(traces, route=route, genai_span=genai_span)
|
||||
|
||||
root = next(span for span in traces.hits[0].spans if not span.references)
|
||||
assert str(root.tag("http.status_code")) == "401", (
|
||||
|
|
|
|||
|
|
@ -28,11 +28,10 @@ from dataclasses import dataclass
|
|||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from e2e_config import OTEL_QUERY_URL, POLL_INTERVAL, POLL_TIMEOUT
|
||||
from e2e_http import URL, NetworkError, NoBody, Result, Success, get
|
||||
from e2e_metadata import step
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
#: OTEL resource service.name the proxy exports under (OTEL_SERVICE_NAME default).
|
||||
JAEGER_SERVICE = "litellm"
|
||||
|
|
@ -118,9 +117,15 @@ def _ticks() -> Iterator[None]:
|
|||
time.sleep(POLL_INTERVAL)
|
||||
|
||||
|
||||
def _settled(trace: JaegerTrace, names: set[str], prefixes: set[str]) -> bool:
|
||||
present = set(trace.span_names())
|
||||
return names.issubset(present) and all(any(name.startswith(prefix) for name in present) for prefix in prefixes)
|
||||
def trace_settled(trace: JaegerTrace, names: set[str], prefixes: set[str]) -> bool:
|
||||
root: Final = root_span(trace)
|
||||
present: Final = frozenset(trace.span_names())
|
||||
return (
|
||||
root is not None
|
||||
and root.kind == "server"
|
||||
and names.issubset(present)
|
||||
and all(any(name.startswith(prefix) for name in present) for prefix in prefixes)
|
||||
)
|
||||
|
||||
|
||||
def root_span(trace: JaegerTrace) -> JaegerSpan | None:
|
||||
|
|
@ -172,7 +177,7 @@ class _Observation:
|
|||
|
||||
def settled(self, names: set[str], prefixes: set[str]) -> bool:
|
||||
hits: Final = self.traces.hits
|
||||
return self.unreachable is None and len(hits) == 1 and not self.missing and _settled(hits[0], names, prefixes)
|
||||
return self.unreachable is None and len(hits) == 1 and not self.missing and trace_settled(hits[0], names, prefixes)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
|
|||
|
|
@ -8,8 +8,21 @@ helper is worth pinning here.
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from otel_client import TTFT_TAG, JaegerTrace, one_served_genai_span, served_genai_spans
|
||||
from otel_client import (
|
||||
TTFT_TAG,
|
||||
CallTraces,
|
||||
JaegerReference,
|
||||
JaegerSpan,
|
||||
JaegerTag,
|
||||
JaegerTrace,
|
||||
one_served_genai_span,
|
||||
served_genai_spans,
|
||||
trace_settled,
|
||||
)
|
||||
from test_otel_trace_e2e import assert_complete_trace, assert_real_ttft
|
||||
|
||||
GENAI_SPAN = "chat claude-haiku-4-5"
|
||||
|
||||
|
|
@ -79,3 +92,73 @@ def test_other_operations_are_not_counted() -> None:
|
|||
trace = _trace(_span("chat gpt-5.5", ttft=0.3), _span(GENAI_SPAN, ttft=0.3))
|
||||
|
||||
assert len(served_genai_spans(trace, GENAI_SPAN)) == 1
|
||||
|
||||
|
||||
def _request_trace(
|
||||
*,
|
||||
root_name: str = "POST",
|
||||
root_kind: str = "server",
|
||||
route: str = "/v1/messages",
|
||||
genai_parent: str = "root",
|
||||
ttft: float = 0.2,
|
||||
) -> JaegerTrace:
|
||||
parent: Final = JaegerReference(refType="CHILD_OF", traceID="request", spanID="root")
|
||||
return JaegerTrace(
|
||||
traceID="request",
|
||||
spans=[
|
||||
JaegerSpan(
|
||||
spanID="root",
|
||||
operationName=root_name,
|
||||
tags=[JaegerTag(key="span.kind", value=root_kind), JaegerTag(key="http.target", value=route)],
|
||||
),
|
||||
JaegerSpan(spanID="auth", operationName="auth /v1/messages", references=[parent]),
|
||||
JaegerSpan(spanID="db", operationName="postgres.select tokens", references=[parent]),
|
||||
JaegerSpan(
|
||||
spanID="genai",
|
||||
operationName=GENAI_SPAN,
|
||||
duration=1_000_000,
|
||||
references=[JaegerReference(refType="CHILD_OF", traceID="request", spanID=genai_parent)],
|
||||
tags=[JaegerTag(key="span.kind", value="client"), JaegerTag(key=TTFT_TAG, value=ttft)],
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("root_name", ("POST", "POST /v1/messages"))
|
||||
def test_connected_request_trace_is_independent_of_server_display_name(root_name: str) -> None:
|
||||
trace: Final = _request_trace(root_name=root_name)
|
||||
assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN)
|
||||
assert_real_ttft((trace,), genai_span=GENAI_SPAN)
|
||||
assert trace_settled(trace, {"auth /v1/messages", GENAI_SPAN}, {"postgres."})
|
||||
|
||||
|
||||
def test_incomplete_trace_cannot_settle_before_the_server_span_arrives() -> None:
|
||||
trace: Final = _request_trace()
|
||||
incomplete: Final = trace.model_copy(update={"spans": trace.spans[1:]})
|
||||
assert not trace_settled(incomplete, {"auth /v1/messages", GENAI_SPAN}, {"postgres."})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("route", ("/v1/responses", ""))
|
||||
def test_request_trace_must_still_identify_the_requested_route(route: str) -> None:
|
||||
trace: Final = _request_trace(root_name="POST /v1/messages", route=route)
|
||||
with pytest.raises(AssertionError, match="route"):
|
||||
assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN)
|
||||
|
||||
|
||||
def test_request_trace_must_still_be_rooted_at_a_server_span() -> None:
|
||||
trace: Final = _request_trace(root_name="POST /v1/messages", root_kind="client")
|
||||
with pytest.raises(AssertionError, match="kind=server"):
|
||||
assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN)
|
||||
|
||||
|
||||
def test_request_trace_must_still_connect_the_model_call_to_the_request() -> None:
|
||||
trace: Final = _request_trace(genai_parent="absent")
|
||||
with pytest.raises(AssertionError, match="orphaned trace"):
|
||||
assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ttft", (0.0, 1.0))
|
||||
def test_server_name_tolerance_does_not_accept_invalid_first_token_timing(ttft: float) -> None:
|
||||
trace: Final = _request_trace(ttft=ttft)
|
||||
with pytest.raises(AssertionError, match="TTFT"):
|
||||
assert_real_ttft((trace,), genai_span=GENAI_SPAN)
|
||||
|
|
|
|||
|
|
@ -5,5 +5,8 @@
|
|||
addopts = --strict-markers --strict-config -p no:cacheprovider
|
||||
pythonpath = ../e2e ../e2e/batches ../e2e/guardrails ../e2e/logging ../e2e/perf
|
||||
markers =
|
||||
e2e: live assertion modules imported by harness regression tests
|
||||
meta(subject): metadata on live assertion modules imported by harness regression tests
|
||||
otel_tls: TLS coverage in live assertion modules imported by harness regression tests
|
||||
covers(cell_id, *, exercised_on=()): coverage-registry cell(s) a test covers; exercised here only to prove the collector and the JUnit properties read it
|
||||
cli_determinism: drives the real claude CLI for several seconds
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue