test(e2e): include fixture and tracing repairs for qualification

This commit is contained in:
Yuneng Jiang 2026-11-02 22:32:50 -08:00
commit 247efdd50d
No known key found for this signature in database
17 changed files with 222 additions and 66 deletions

View file

@ -38,7 +38,7 @@ from models import (
from proxy_client import ProxyClient
from pydantic import BaseModel, Field
GUARDRAIL_BACKEND: Final = "gemini/gemini-2.5-flash"
GUARDRAIL_BACKEND: Final = "gemini/gemini-3.8-flash"
GuardrailMode = Literal["pre_call", "post_call", "during_call", "logging_only"]
BlockedWordAction = Literal["BLOCK", "MASK"]

View file

@ -45,7 +45,7 @@ from passthrough_client import PassthroughClient
pytestmark = pytest.mark.e2e
COHERE_BACKEND = "cohere/command-r-08-2024"
GEMINI_BACKEND = "gemini/gemini-2.5-flash"
GEMINI_BACKEND = "gemini/gemini-3.8-flash"
VERTEX_BACKEND: Final = "vertex_ai/gemini-2.5-flash"
AZURE_OPENAI_BACKEND: Final = "azure/gpt-5.4-nano"
AZURE_OPENAI_API_VERSION: Final = "v1"
@ -243,7 +243,7 @@ _PERSON_SCHEMA: dict[str, object] = {
OPENAI_CHAT_MODEL: Final = "gpt-5.5"
ANTHROPIC_CHAT_MODEL: Final = "claude-haiku-4-5"
GEMINI_FLASH_MODEL: Final = "gemini-2.5-flash"
GEMINI_FLASH_MODEL: Final = "gemini-3.8-flash"
CHAT_MODELS: Final = (
pytest.param(

View file

@ -5,7 +5,7 @@ teardown) instead of relying on a statically configured model, so the check is
self-contained and never inherits pricing another suite or a stale config left on
the shared proxy. custom-priced-flash sets input/output rates deliberately far
above the canonical gemini price; the isolation sibling shares the same
gemini/gemini-2.5-flash backend but sets no override. Three behaviors are checked
gemini/gemini-3.8-flash backend but sets no override. Three behaviors are checked
independently:
- billing: a real call's logged cost breakdown charges input and output tokens at
@ -37,10 +37,8 @@ from models import (
pytestmark = pytest.mark.e2e
BACKEND_MODEL = "gemini/gemini-2.5-flash"
BACKEND_MODEL = "gemini/gemini-3.8-flash"
GEMINI_API_KEY = "os.environ/GEMINI_API_KEY"
# Deliberately ~100x above canonical gemini-2.5-flash (input 3e-7 / output 2.5e-6)
# so an override that is ignored or under-applied bills at the base rate and fails.
CUSTOM_INPUT_RATE = 5e-05
CUSTOM_OUTPUT_RATE = 1e-04
@ -78,7 +76,7 @@ def _provision(
input_cost_per_token: float | None,
output_cost_per_token: float | None,
) -> str:
"""Register a fresh gemini/gemini-2.5-flash deployment (deleted on teardown) and
"""Register a fresh gemini/gemini-3.8-flash deployment (deleted on teardown) and
return its model name. With the cost fields set the deployment carries a custom
pricing override; with them None it is a plain sibling on the same backend. The
marker keeps the name unique so concurrent runs on the shared proxy never
@ -251,7 +249,7 @@ class TestCustomPricing:
assert sibling_entry is not None, f"{sibling} absent from /model/info"
# custom-priced-flash overrides pricing; the sibling shares the same
# gemini/gemini-2.5-flash backend but sets no override, so it must keep its
# gemini/gemini-3.8-flash backend but sets no override, so it must keep its
# own price. Equal rates mean the override leaked into the shared cost map.
assert (
sibling_entry.model_info.input_cost_per_token

View file

@ -21,7 +21,7 @@ from pydantic import BaseModel
pytestmark = pytest.mark.e2e
UPSTREAM_MODEL = "gemini/gemini-2.5-flash"
UPSTREAM_MODEL = "gemini/gemini-3.8-flash"
class _GenerateContentPart(BaseModel):

View file

@ -24,7 +24,7 @@ from sdk_clients import SdkClients
pytestmark = pytest.mark.e2e
IMAGE_EDIT_BACKEND: Final = "openai/gpt-image-1"
IMAGE_EDIT_BACKEND: Final = "openai/gpt-image-2.5-sunburst"
_TEST_PNG = base64.b64decode(
"iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAS0lEQVR42u3PMQ0AAAwDoPo3"

View file

@ -5,7 +5,7 @@ and verifies the proxy still logged a costed SpendLogs row
(call_type="pass_through_endpoint"), correlated by the id the caller was served:
the x-litellm-call-id header on gemini, the `msg_...` message id on anthropic.
Covered: gemini ("gemini-2.5-flash") + anthropic ("claude-haiku-4-5"), streaming +
Covered: gemini ("gemini-3.8-flash") + anthropic ("claude-haiku-4-5"), streaming +
non-streaming, plus native tool calls. See LLM_TRANSLATION_COVERAGE_MATRIX.md.
A passthrough call returning non-2xx fails hard (never a skip); once it returns
@ -32,7 +32,7 @@ from passthrough_client import (
completed_responses_object,
)
GEMINI_MODEL: Final = "gemini-2.5-flash"
GEMINI_MODEL: Final = "gemini-3.8-flash"
ANTHROPIC_PASSTHROUGH_MODEL: Final = "claude-haiku-4-5"
EMBEDDING_MODEL = "text-embedding-3-small"
REALTIME_MODEL = "gpt-realtime-2"
@ -291,7 +291,7 @@ class TestPassthroughModelAllowlist:
result = client.anthropic_message(key, ANTHROPIC_PASSTHROUGH_MODEL, f"say hi {unique_marker()}")
assert result.status_code == 403, (
"a key restricted to gemini-2.5-flash must be denied a claude passthrough call, "
"a key restricted to gemini-3.8-flash must be denied a claude passthrough call, "
f"got {result.status_code}: {result.body[:300]}"
)

View file

@ -57,7 +57,7 @@ OPENAI_VISION_BACKEND: Final = "openai/gpt-4o"
ANTHROPIC_BACKEND: Final = "anthropic/claude-haiku-4-5"
BEDROCK_CONVERSE_BACKEND: Final = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
VERTEX_BACKEND: Final = "vertex_ai/gemini-2.5-flash"
GEMINI_BACKEND: Final = "gemini/gemini-2.5-flash"
GEMINI_BACKEND: Final = "gemini/gemini-3.8-flash"
OPENAI_RESPONSES_BACKEND: Final = "openai/gpt-5.5"
INSTRUCTIONS = "You are a helpful assistant"
CAT_IMAGE_URL = "https://upload.wikimedia.org/wikipedia/commons/3/3a/Cat03.jpg"

View file

@ -18,10 +18,11 @@ import json
import os
import time
from dataclasses import dataclass
from typing import Callable, Literal
from datetime import datetime, timedelta, timezone
from typing import Callable, Final, Literal
import pytest
from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError
from pydantic import AliasChoices, BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError, field_validator
from e2e_config import POLL_INTERVAL, POLL_TIMEOUT, settle_propagation
from e2e_metadata import step
@ -124,23 +125,39 @@ class CreateGuardrailResponse(BaseModel):
guardrail_name: str | None = None
_LANGFUSE_JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
class LangfuseObservation(BaseModel):
model_config = ConfigDict(extra="ignore", populate_by_name=True)
model_config = ConfigDict(extra="ignore", populate_by_name=True, frozen=True)
id: str
trace_id: str | None = Field(default=None, alias="traceId")
name: str | None = None
type: str | None = None
calculated_total_cost: float | None = Field(default=None, alias="calculatedTotalCost")
calculated_total_cost: float | None = Field(
default=None, validation_alias=AliasChoices("totalCost", "calculatedTotalCost")
)
level: str | None = None
input: object | None = None
output: object | None = None
metadata: object | None = None
input: JsonValue = None
output: JsonValue = None
metadata: JsonValue = None
usage: object | None = None
usage_details: object | None = Field(default=None, alias="usageDetails")
model: str | None = None
@field_validator("input", "output", "metadata", mode="before")
@classmethod
def decode_json_fields(cls, value: object) -> JsonValue:
if not isinstance(value, str):
return _LANGFUSE_JSON.validate_python(value)
try:
return _LANGFUSE_JSON.validate_json(value)
except ValidationError:
return value
class LangfuseObservationList(BaseModel):
model_config = ConfigDict(extra="ignore")
@ -171,12 +188,15 @@ def is_otel_v2_generation(obs: LangfuseObservation, *, key_alias: str) -> bool:
class LangfuseListParams(BaseModel):
model_config = ConfigDict(populate_by_name=True)
model_config = ConfigDict(populate_by_name=True, frozen=True)
limit: int = 100
trace_id: str | None = Field(default=None, alias="traceId")
name: str | None = None
from_start_time: str | None = Field(default=None, alias="fromStartTime")
from_start_time: str = Field(alias="fromStartTime")
to_start_time: str = Field(alias="toStartTime")
fields: str = "core,basic,io,metadata,model,usage"
expand_metadata: str = Field(default="attributes", alias="expandMetadata")
@dataclass(frozen=True, slots=True)
@ -580,14 +600,16 @@ class LoggingClient:
name: str | None = None,
from_start_time: str | None = None,
) -> list[LangfuseObservation]:
result = get(
URL(f"{creds.host}/api/public/observations"),
now: Final = datetime.now(timezone.utc)
result: Final = get(
URL(f"{creds.host}/api/public/v2/observations"),
headers=creds.auth_headers,
params=LangfuseListParams(
limit=100,
traceId=trace_id,
name=name,
fromStartTime=from_start_time,
fromStartTime=from_start_time or (now - timedelta(minutes=15)).isoformat(),
toStartTime=now.isoformat(),
),
response_type=LangfuseObservationList,
timeout=30.0,

View file

@ -79,7 +79,7 @@ def _chain_reaches(span_id: str, root_id: str, trace: JaegerTrace) -> bool:
return False
def _assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) -> None:
def assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) -> None:
"""The enforced behavior: the destination holds exactly one call-id-tagged
trace for the call, rooted at the SERVER span, with auth/db children and
the gen-AI span all connected into that one tree - no dangling parent
@ -112,10 +112,9 @@ def _assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) -
roots = [span for span in trace.spans if not span.references]
assert len(roots) == 1, f"expected exactly one root span, got {[s.operation_name for s in roots]}; spans: {names}"
root = roots[0]
assert root.operation_name == f"POST {route}", (
f"the root must be the SERVER span 'POST {route}', got {root.operation_name!r}"
)
assert root.kind == "server", f"the root span must have kind=server, got {root.kind!r}"
request_route: Final = root.tag("http.route") or root.tag("url.path") or root.tag("http.target")
assert request_route == route, f"the root must identify route {route!r}, got {request_route!r}"
assert f"auth {route}" in names, f"auth phase span 'auth {route}' missing; spans: {names}"
assert any(name.startswith(DB_SPAN_PREFIX) for name in names), (
@ -133,12 +132,12 @@ def _assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) -
def _poll(otel_reader: OtelReader, *, call_id: str, route: str, genai_span: str) -> CallTraces:
return otel_reader.poll_traces_for_call(
call_id=call_id,
settled_names={f"POST {route}", f"auth {route}", genai_span},
settled_names={f"auth {route}", genai_span},
settled_prefixes={DB_SPAN_PREFIX},
)
def _assert_real_ttft(hits: tuple[JaegerTrace, ...], *, genai_span: str) -> None:
def assert_real_ttft(hits: tuple[JaegerTrace, ...], *, genai_span: str) -> None:
"""The enforced behavior: the gen-AI span for the attempt that served the
stream records a TTFT that is a real measurement - present, numeric,
positive, and strictly less than that span's own total duration. A TTFT of
@ -282,7 +281,7 @@ class TestOtelTraceCompleteness:
assert outcome.call_id is not None, "success response must carry x-litellm-call-id"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=f"chat {MODEL}")
_assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}")
assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}")
@pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["chat_completions"])
@meta(
@ -315,7 +314,7 @@ class TestOtelTraceCompleteness:
assert outcome.call_id is not None, "success response must carry x-litellm-call-id"
hits: Final = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=f"chat {MODEL}")
_assert_complete_trace(hits, route=route, genai_span=f"chat {MODEL}")
assert_complete_trace(hits, route=route, genai_span=f"chat {MODEL}")
@pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["messages"])
@meta(
@ -333,7 +332,7 @@ class TestOtelTraceCompleteness:
"""This test verifies that one successful non-streaming /v1/messages request
produces exactly one complete OTEL trace.
The trace must have a single root span named "POST /v1/messages". The
The trace must have a single root SERVER span identifying /v1/messages. The
authentication, database, and model-call spans must all belong to
the same trace and have valid parent relationships leading back to that root.
@ -353,7 +352,7 @@ class TestOtelTraceCompleteness:
assert outcome.call_id is not None, "success response must carry x-litellm-call-id"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=f"chat {MODEL}")
_assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}")
assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}")
@pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["responses"])
@meta(
@ -392,7 +391,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {CHEAP_OPENAI_MODEL}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
_assert_complete_trace(traces, route=route, genai_span=genai_span)
assert_complete_trace(traces, route=route, genai_span=genai_span)
@pytest.mark.covers("logging.otel.stream.exports_metric", exercised_on=["chat_completions"])
@meta(
@ -443,7 +442,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {MODEL}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
_assert_complete_trace(traces, route=route, genai_span=genai_span)
assert_complete_trace(traces, route=route, genai_span=genai_span)
served = one_served_genai_span(traces.hits[0], genai_span)
assert served.tag("litellm.request.streaming") is True, (
@ -500,7 +499,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {MODEL}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
_assert_complete_trace(traces, route=route, genai_span=genai_span)
assert_complete_trace(traces, route=route, genai_span=genai_span)
served = one_served_genai_span(traces.hits[0], genai_span)
assert served.tag("litellm.request.streaming") is True, (
@ -557,7 +556,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {CHEAP_OPENAI_MODEL}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
_assert_complete_trace(traces, route=route, genai_span=genai_span)
assert_complete_trace(traces, route=route, genai_span=genai_span)
one_served_genai_span(traces.hits[0], genai_span)
@ -614,7 +613,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {MODEL}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
_assert_real_ttft(traces.hits, genai_span=genai_span)
assert_real_ttft(traces.hits, genai_span=genai_span)
@pytest.mark.covers("logging.otel.stream.records_ttft", exercised_on=["messages"])
@meta(
@ -661,7 +660,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {MODEL}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
_assert_real_ttft(traces.hits, genai_span=genai_span)
assert_real_ttft(traces.hits, genai_span=genai_span)
@pytest.mark.covers("logging.otel.stream.records_ttft", exercised_on=["responses"])
@meta(
@ -708,7 +707,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {CHEAP_OPENAI_MODEL}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
_assert_real_ttft(traces.hits, genai_span=genai_span)
assert_real_ttft(traces.hits, genai_span=genai_span)
@pytest.mark.covers("logging.otel.failure.exports_metric", exercised_on=["chat_completions"])
@meta(
@ -763,7 +762,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {model_name}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
_assert_complete_trace(traces, route=route, genai_span=genai_span)
assert_complete_trace(traces, route=route, genai_span=genai_span)
root = next(span for span in traces.hits[0].spans if not span.references)
assert str(root.tag("http.status_code")) == "401", (
@ -825,7 +824,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {model_name}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
_assert_complete_trace(traces, route=route, genai_span=genai_span)
assert_complete_trace(traces, route=route, genai_span=genai_span)
root = next(span for span in traces.hits[0].spans if not span.references)
assert str(root.tag("http.status_code")) == "401", (

View file

@ -418,7 +418,7 @@ class TestBudgetListV1:
@pytest.mark.covers("mgmt.budget.list_v1.happy_path")
@meta(Subject(domain=Domain.SPEND_BUDGETS, route=Route.BUDGET_MANAGEMENT))
def test_refuses_a_sort_field_and_a_parameter_it_does_not_support(self, client: ManagementClient) -> None:
assert _list_status(client, BudgetPageParams(sort="budget_duration")) == 400
assert _list_status(client, BudgetPageParams(sort="not_a_sort_field")) == 400
assert _list_status(client, BudgetPageParams(not_a_parameter="b-1")) == 400
@pytest.mark.covers("mgmt.budget.list_v1.admin_only")

View file

@ -28,11 +28,10 @@ from dataclasses import dataclass
from typing import Final
import pytest
from pydantic import BaseModel, ConfigDict, Field
from e2e_config import OTEL_QUERY_URL, POLL_INTERVAL, POLL_TIMEOUT
from e2e_http import URL, NetworkError, NoBody, Result, Success, get
from e2e_metadata import step
from pydantic import BaseModel, ConfigDict, Field
#: OTEL resource service.name the proxy exports under (OTEL_SERVICE_NAME default).
JAEGER_SERVICE = "litellm"
@ -118,9 +117,15 @@ def _ticks() -> Iterator[None]:
time.sleep(POLL_INTERVAL)
def _settled(trace: JaegerTrace, names: set[str], prefixes: set[str]) -> bool:
present = set(trace.span_names())
return names.issubset(present) and all(any(name.startswith(prefix) for name in present) for prefix in prefixes)
def trace_settled(trace: JaegerTrace, names: set[str], prefixes: set[str]) -> bool:
root: Final = root_span(trace)
present: Final = frozenset(trace.span_names())
return (
root is not None
and root.kind == "server"
and names.issubset(present)
and all(any(name.startswith(prefix) for name in present) for prefix in prefixes)
)
def root_span(trace: JaegerTrace) -> JaegerSpan | None:
@ -172,7 +177,7 @@ class _Observation:
def settled(self, names: set[str], prefixes: set[str]) -> bool:
hits: Final = self.traces.hits
return self.unreachable is None and len(hits) == 1 and not self.missing and _settled(hits[0], names, prefixes)
return self.unreachable is None and len(hits) == 1 and not self.missing and trace_settled(hits[0], names, prefixes)
@dataclass(frozen=True, slots=True)

View file

@ -55,7 +55,7 @@ from spend_e2e_client import (
pytestmark = pytest.mark.e2e
CHAT_MODEL: Final = "gemini-2.5-flash"
CHAT_MODEL: Final = "gemini-3.8-flash"
MESSAGES_MODEL: Final = "claude-haiku-4-5"
RESPONSES_MODEL: Final = "openai-responses-codex"
EMBED_MODEL: Final = "openai-text-embedding-3-small"

View file

@ -3,7 +3,7 @@
Run against a proxy started with the gateway config. Coverage rationale:
SPEND_TRACKING_COVERAGE_MATRIX.md.
Model names are literals from that config: chat tests hit "gemini-2.5-flash",
Model names are literals from that config: chat tests hit "gemini-3.8-flash",
embedding tests hit "openai-text-embedding-3-small".
Every test: fresh scoped key (isolation) -> real provider call -> unwrap (hard
@ -37,7 +37,7 @@ from spend_reconciliation import BACKEND as TRAFFIC_BACKEND
pytestmark = pytest.mark.e2e
GEMINI_MODEL = "gemini-2.5-flash"
GEMINI_MODEL = "gemini-3.8-flash"
CLAUDE_MODEL = "claude-haiku-4-5"
CODEX_MODEL = "openai-responses-codex"
EMBEDDING_MODEL = "openai-text-embedding-3-small"
@ -780,7 +780,7 @@ def test_spend_calculate_returns_nonzero_cost(client: SpendClient) -> None:
GEMINI_MODEL, "estimate the cost of this request"
)
assert cost > 0, (
"/spend/calculate returned 0 for gemini-2.5-flash; "
"/spend/calculate returned 0 for gemini-3.8-flash; "
"cost map may be missing this model"
)

View file

@ -57,11 +57,9 @@ CONTENT_POLICY_PROMPT = (
COOLDOWN_SECONDS = 30.0
REPLICA_PROPAGATION_SECONDS = 15.0
# The smallest-context chat model OpenAI still serves (16385 tokens). A prompt
# past that limit comes back as a real `context_length_exceeded` 400, which is
# what litellm maps to ContextWindowExceededError.
SMALL_CONTEXT_MODEL = "openai/gpt-3.5-turbo"
SMALL_CONTEXT_LIMIT_TOKENS = 16385
SMALL_CONTEXT_MODEL: Final = "openai/gpt-5.4-nano"
# model_prices_and_context_window.json, gpt-5.4-nano, verified 2026-11-02
SMALL_CONTEXT_LIMIT_TOKENS: Final = 272000
def oversized_prompt(marker: str) -> str:
@ -107,7 +105,7 @@ def create_timeout_deployment(proxy: ProxyClient, name: str) -> str:
@step(f"Add a deployment named {{name}} that calls the small-context model {SMALL_CONTEXT_MODEL}")
def create_small_context_deployment(proxy: ProxyClient, name: str) -> str:
"""Register a deployment on the smallest-context model OpenAI still serves, so an
"""Register a deployment with a smaller context window than the backup, so an
oversized prompt earns a real context-window refusal from the provider."""
return proxy.create_model(name, LiteLLMParamsBody(model=SMALL_CONTEXT_MODEL, api_key=REAL_KEY))
@ -263,8 +261,7 @@ def spend_only_request_of(proxy: ProxyClient, spent_key: str) -> None:
@step(f"Add a deployment named {{name}} that calls {SMALL_CONTEXT_MODEL} and takes all of its group's traffic")
def create_always_picked_small_context_deployment(proxy: ProxyClient, name: str) -> str:
"""The always-picked half of a retry pair on the smallest-context model OpenAI
still serves: it holds all of the model group's shuffle weight, so an oversized
"""The always-picked half of a retry pair on the smaller-context model: it holds all of the model group's shuffle weight, so an oversized
prompt opens on it and earns a real context-window refusal, which never benches
a deployment, so only the retry itself can steer the request off it."""
return proxy.register_model(

View file

@ -0,0 +1,49 @@
import json
from typing import Final
import pytest
from pydantic import JsonValue
from logging_client import LangfuseObservation, is_otel_v2_generation, observation_spend
@pytest.mark.parametrize("encoded", [False, True])
@pytest.mark.parametrize("cost", [0.0, 0.000023])
def test_observation_preserves_v2_content_usage_and_cost(encoded: bool, cost: float) -> None:
content: Final[JsonValue] = [{"role": "user", "content": "unique-request-marker"}]
output: Final[JsonValue] = [{"role": "assistant", "content": "response"}]
metadata: Final[JsonValue] = {
"attributes": {
"langfuse.observation.type": "generation",
"litellm.metadata.user_api_key_alias": "test-key-alias",
}
}
usage: Final[JsonValue] = {"input": 17, "output": 3}
observation: Final = LangfuseObservation.model_validate(
{
"id": "observation",
"input": json.dumps(content) if encoded else content,
"output": json.dumps(output) if encoded else output,
"metadata": json.dumps(metadata) if encoded else metadata,
"usageDetails": usage,
"totalCost": cost,
}
)
assert observation.input == content
assert observation.output == output
assert observation.usage_details == usage
assert observation_spend(observation) == cost
assert is_otel_v2_generation(observation, key_alias="test-key-alias")
assert not is_otel_v2_generation(observation, key_alias="another-key")
def test_observation_preserves_plain_text_and_legacy_cost() -> None:
cost: Final = 0.000031
observation: Final = LangfuseObservation.model_validate(
{"id": "observation", "input": "plain prompt", "output": "plain reply", "calculatedTotalCost": cost}
)
assert observation.input == "plain prompt"
assert observation.output == "plain reply"
assert observation_spend(observation) == cost

View file

@ -8,8 +8,21 @@ helper is worth pinning here.
from __future__ import annotations
from typing import Final
import pytest
from otel_client import TTFT_TAG, JaegerTrace, one_served_genai_span, served_genai_spans
from otel_client import (
TTFT_TAG,
CallTraces,
JaegerReference,
JaegerSpan,
JaegerTag,
JaegerTrace,
one_served_genai_span,
served_genai_spans,
trace_settled,
)
from test_otel_trace_e2e import assert_complete_trace, assert_real_ttft
GENAI_SPAN = "chat claude-haiku-4-5"
@ -79,3 +92,73 @@ def test_other_operations_are_not_counted() -> None:
trace = _trace(_span("chat gpt-5.5", ttft=0.3), _span(GENAI_SPAN, ttft=0.3))
assert len(served_genai_spans(trace, GENAI_SPAN)) == 1
def _request_trace(
*,
root_name: str = "POST",
root_kind: str = "server",
route: str = "/v1/messages",
genai_parent: str = "root",
ttft: float = 0.2,
) -> JaegerTrace:
parent: Final = JaegerReference(refType="CHILD_OF", traceID="request", spanID="root")
return JaegerTrace(
traceID="request",
spans=[
JaegerSpan(
spanID="root",
operationName=root_name,
tags=[JaegerTag(key="span.kind", value=root_kind), JaegerTag(key="http.target", value=route)],
),
JaegerSpan(spanID="auth", operationName="auth /v1/messages", references=[parent]),
JaegerSpan(spanID="db", operationName="postgres.select tokens", references=[parent]),
JaegerSpan(
spanID="genai",
operationName=GENAI_SPAN,
duration=1_000_000,
references=[JaegerReference(refType="CHILD_OF", traceID="request", spanID=genai_parent)],
tags=[JaegerTag(key="span.kind", value="client"), JaegerTag(key=TTFT_TAG, value=ttft)],
),
],
)
@pytest.mark.parametrize("root_name", ("POST", "POST /v1/messages"))
def test_connected_request_trace_is_independent_of_server_display_name(root_name: str) -> None:
trace: Final = _request_trace(root_name=root_name)
assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN)
assert_real_ttft((trace,), genai_span=GENAI_SPAN)
assert trace_settled(trace, {"auth /v1/messages", GENAI_SPAN}, {"postgres."})
def test_incomplete_trace_cannot_settle_before_the_server_span_arrives() -> None:
trace: Final = _request_trace()
incomplete: Final = trace.model_copy(update={"spans": trace.spans[1:]})
assert not trace_settled(incomplete, {"auth /v1/messages", GENAI_SPAN}, {"postgres."})
@pytest.mark.parametrize("route", ("/v1/responses", ""))
def test_request_trace_must_still_identify_the_requested_route(route: str) -> None:
trace: Final = _request_trace(root_name="POST /v1/messages", route=route)
with pytest.raises(AssertionError, match="route"):
assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN)
def test_request_trace_must_still_be_rooted_at_a_server_span() -> None:
trace: Final = _request_trace(root_name="POST /v1/messages", root_kind="client")
with pytest.raises(AssertionError, match="kind=server"):
assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN)
def test_request_trace_must_still_connect_the_model_call_to_the_request() -> None:
trace: Final = _request_trace(genai_parent="absent")
with pytest.raises(AssertionError, match="orphaned trace"):
assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN)
@pytest.mark.parametrize("ttft", (0.0, 1.0))
def test_server_name_tolerance_does_not_accept_invalid_first_token_timing(ttft: float) -> None:
trace: Final = _request_trace(ttft=ttft)
with pytest.raises(AssertionError, match="TTFT"):
assert_real_ttft((trace,), genai_span=GENAI_SPAN)

View file

@ -5,5 +5,8 @@
addopts = --strict-markers --strict-config -p no:cacheprovider
pythonpath = ../e2e ../e2e/batches ../e2e/guardrails ../e2e/logging ../e2e/perf
markers =
e2e: live assertion modules imported by harness regression tests
meta(subject): metadata on live assertion modules imported by harness regression tests
otel_tls: TLS coverage in live assertion modules imported by harness regression tests
covers(cell_id, *, exercised_on=()): coverage-registry cell(s) a test covers; exercised here only to prove the collector and the JUnit properties read it
cli_determinism: drives the real claude CLI for several seconds