diff --git a/tests/code_coverage_tests/test_provider_cache.py b/tests/code_coverage_tests/test_provider_cache.py
index fc39f2c1e6c..81ad7913559 100644
--- a/tests/code_coverage_tests/test_provider_cache.py
+++ b/tests/code_coverage_tests/test_provider_cache.py
@@ -377,7 +377,11 @@ def test_stream_completion_controls_publication(store: RedisResponseStore, provi
assert len(provider.hits) == (2 if truncated else 1)
-def test_store_outage_preserves_provider_success(provider: Provider) -> None:
+@pytest.mark.parametrize("cluster_mode", ("0", "1"))
+def test_store_outage_preserves_provider_success(
+ provider: Provider, monkeypatch: pytest.MonkeyPatch, cluster_mode: str,
+) -> None:
+ monkeypatch.setenv("E2E_PROVIDER_CACHE_REDIS_CLUSTER", cluster_mode)
with socket.socket() as probe:
probe.bind(("127.0.0.1", 0))
port: Final = probe.getsockname()[1]
@@ -401,6 +405,20 @@ def test_old_lease_cannot_overwrite_new_owner(store: RedisResponseStore) -> None
assert isinstance(hit, CacheHit) and hit.payload == b"new"
+def test_separate_namespaces_keep_their_own_response(store: RedisResponseStore) -> None:
+ other: Final = replace(store, namespace=store.namespace + "-other")
+ first_lease: Final = store.lookup("shared-key")
+ assert isinstance(first_lease, CaptureLease)
+ assert store.publish("shared-key", first_lease, b"first namespace")
+ other_lease: Final = other.lookup("shared-key")
+ assert isinstance(other_lease, CaptureLease)
+ assert other.publish("shared-key", other_lease, b"other namespace")
+ first_hit: Final = store.lookup("shared-key")
+ other_hit: Final = other.lookup("shared-key")
+ assert isinstance(first_hit, CacheHit) and first_hit.payload == b"first namespace"
+ assert isinstance(other_hit, CacheHit) and other_hit.payload == b"other namespace"
+
+
def test_identity_preserves_values_and_never_contains_credentials() -> None:
variants: Final = (b'{}', b'{"a":null}', b'{"a":false}', b'{"a":0}', b'{"a":0.0}', b'{"a":"0"}', b' { }', None, b'')
url: Final = "https://example.invalid/v1/chat/completions"
diff --git a/tests/e2e/AGENTS.md b/tests/e2e/AGENTS.md
index 62ebfd12fe5..d38dafedf75 100644
--- a/tests/e2e/AGENTS.md
+++ b/tests/e2e/AGENTS.md
@@ -35,7 +35,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family
- `llm_translation/` - LLM endpoint and provider-translation behavior: passthrough, custom pricing, OCR, and the non-chat inference endpoints (`/v1/responses`, `/v1/messages`, `/embeddings`, `/v1/rerank`, `/v1/audio/speech`, `/v1/images/generations`), each against a deployment the test creates via `/model/new` and deletes on teardown
- `access_control/` - the gateway's authorization and error-shape contract: per-key model allow-lists, route-group permissions (`allowed_routes`), and unknown-model validation
- `embeddings/` - the `/embeddings` endpoint across providers
-- `batches/` - the `/batches` endpoint (placeholder until the first test lands)
+- `batches/` - the `/batches` endpoint. GovCloud batch coverage is explicitly skipped until its infrastructure is provisioned. Other missing credentials and provider errors remain failures
- `realtime/` - realtime websocket sessions, including the pipecat audio path
- `quota_management/` - quota enforcement and accounting, one subfolder per behavior: `ratelimit/` (rpm/tpm blocks, window reset, pacing headers on live traffic), `budgets/` (budget definition, enforcement, and reset windows: key, team, tag, soft, multi-window), and `spend_tracking/` (spend logging and cost attribution on `/spend/*`)
- `management/` - key/team/user/organization management routes: create/update/delete persistence via the info routes, team membership, and llm-only-key route denials (API surface, not Playwright)
@@ -326,3 +326,7 @@ other...
- do not use xfail markers, tests should be written in a form that the end user expects it to pass
- a cell addresses the stack through its front door (`PROXY_BASE_URL`) the way a customer does, never a gateway pod by address, and proves a cross-replica property with N independent calls through that door, naming the miss odds (2^-N at two pods) in its docstring. `PROXY_REPLICA_URLS` is for read-backs that are per pod by nature (polling until a management write has converged on every replica, `/metrics`, RSS), never for steering the scenario a cell asserts on at a chosen pod
+
+## Provider response cache
+
+The provider response cache can share the stage Valkey instance through a `rediss://` URL. Set `E2E_PROVIDER_CACHE_REDIS_CLUSTER=1` for a cluster-aware client, including ElastiCache Serverless. Keep the dedicated HMAC key and namespace, and restrict test cleanup to keys in the test's own namespace
diff --git a/tests/e2e/batches/COVERAGE.md b/tests/e2e/batches/COVERAGE.md
index 1ede9c89b97..ac6f9cd08e4 100644
--- a/tests/e2e/batches/COVERAGE.md
+++ b/tests/e2e/batches/COVERAGE.md
@@ -11,8 +11,9 @@ cost write-back via a cross-run marker baton (design below).
Only supported cells are tested. The capability table in `capabilities.py` holds one
row per supported (provider, scenario) pair, so there are no skipped cells in the
-parametrized run. The batches suite never skips: missing provider creds or upstream
-failures are hard test failures (see `tests/e2e/AGENTS.md`).
+parametrized run. Missing provider creds or upstream failures are hard test failures
+(see `tests/e2e/AGENTS.md`). GovCloud is explicitly skipped until its infrastructure
+is provisioned and does not count as qualified coverage.
| Provider | create | retrieve | cancel | list | content download | file backing |
|-----------|--------|----------|--------|------|------------------|--------------|
diff --git a/tests/e2e/batches/test_batches_e2e.py b/tests/e2e/batches/test_batches_e2e.py
index 54bcb80ed24..94751121fe6 100644
--- a/tests/e2e/batches/test_batches_e2e.py
+++ b/tests/e2e/batches/test_batches_e2e.py
@@ -1210,6 +1210,7 @@ def _govcloud_params() -> LiteLLMParamsBody:
)
+@pytest.mark.skip(reason="GovCloud E2E infrastructure is not provisioned")
class TestBedrockBatchGovCloud:
"""Bedrock batch lifecycle in the AWS GovCloud partition (us-gov-west-1).
diff --git a/tests/e2e/guardrails/guardrails_client.py b/tests/e2e/guardrails/guardrails_client.py
index 08124a03313..a06d86798f8 100644
--- a/tests/e2e/guardrails/guardrails_client.py
+++ b/tests/e2e/guardrails/guardrails_client.py
@@ -38,7 +38,7 @@ from models import (
from proxy_client import ProxyClient
from pydantic import BaseModel, Field
-GUARDRAIL_BACKEND: Final = "gemini/gemini-2.5-flash"
+GUARDRAIL_BACKEND: Final = "gemini/gemini-3.8-flash"
GuardrailMode = Literal["pre_call", "post_call", "during_call", "logging_only"]
PiiEntity = Literal["EMAIL_ADDRESS", "PHONE_NUMBER", "PERSON", "CREDIT_CARD", "US_SSN"]
diff --git a/tests/e2e/guardrails/test_openai_moderation_guardrail_e2e.py b/tests/e2e/guardrails/test_openai_moderation_guardrail_e2e.py
index b9593feb2ed..0eb1bf79daa 100644
--- a/tests/e2e/guardrails/test_openai_moderation_guardrail_e2e.py
+++ b/tests/e2e/guardrails/test_openai_moderation_guardrail_e2e.py
@@ -128,7 +128,7 @@ class TestOpenAIModerationGuardrail:
)
allowed = unwrap(
- client.messages(scoped_key, model, BENIGN_PROMPT, guardrails=[name], max_tokens=64)
+ client.messages(scoped_key, model, BENIGN_PROMPT, guardrails=[name], max_tokens=512)
)
assert allowed.content or allowed.choices, (
"the same moderation guardrail must let a benign /v1/messages prompt through, but "
diff --git a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py
index 1e0a8cbb412..4967f18818a 100644
--- a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py
+++ b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py
@@ -45,7 +45,7 @@ from passthrough_client import PassthroughClient
pytestmark = pytest.mark.e2e
COHERE_BACKEND = "cohere/command-r-08-2024"
-GEMINI_BACKEND = "gemini/gemini-2.5-flash"
+GEMINI_BACKEND = "gemini/gemini-3.8-flash"
VERTEX_BACKEND: Final = "vertex_ai/gemini-2.5-flash"
AZURE_OPENAI_BACKEND: Final = "azure/gpt-5.4-nano"
AZURE_OPENAI_API_VERSION: Final = "v1"
@@ -243,7 +243,7 @@ _PERSON_SCHEMA: dict[str, object] = {
OPENAI_CHAT_MODEL: Final = "gpt-5.5"
ANTHROPIC_CHAT_MODEL: Final = "claude-haiku-4-5"
-GEMINI_FLASH_MODEL: Final = "gemini-2.5-flash"
+GEMINI_FLASH_MODEL: Final = "gemini-3.8-flash"
CHAT_MODELS: Final = (
pytest.param(
diff --git a/tests/e2e/llm_translation/test_custom_pricing_e2e.py b/tests/e2e/llm_translation/test_custom_pricing_e2e.py
index 6d8cd24648e..63b8b60867a 100644
--- a/tests/e2e/llm_translation/test_custom_pricing_e2e.py
+++ b/tests/e2e/llm_translation/test_custom_pricing_e2e.py
@@ -5,7 +5,7 @@ teardown) instead of relying on a statically configured model, so the check is
self-contained and never inherits pricing another suite or a stale config left on
the shared proxy. custom-priced-flash sets input/output rates deliberately far
above the canonical gemini price; the isolation sibling shares the same
-gemini/gemini-2.5-flash backend but sets no override. Three behaviors are checked
+gemini/gemini-3.8-flash backend but sets no override. Three behaviors are checked
independently:
- billing: a real call's logged cost breakdown charges input and output tokens at
@@ -37,10 +37,8 @@ from models import (
pytestmark = pytest.mark.e2e
-BACKEND_MODEL = "gemini/gemini-2.5-flash"
+BACKEND_MODEL = "gemini/gemini-3.8-flash"
GEMINI_API_KEY = "os.environ/GEMINI_API_KEY"
-# Deliberately ~100x above canonical gemini-2.5-flash (input 3e-7 / output 2.5e-6)
-# so an override that is ignored or under-applied bills at the base rate and fails.
CUSTOM_INPUT_RATE = 5e-05
CUSTOM_OUTPUT_RATE = 1e-04
@@ -78,7 +76,7 @@ def _provision(
input_cost_per_token: float | None,
output_cost_per_token: float | None,
) -> str:
- """Register a fresh gemini/gemini-2.5-flash deployment (deleted on teardown) and
+ """Register a fresh gemini/gemini-3.8-flash deployment (deleted on teardown) and
return its model name. With the cost fields set the deployment carries a custom
pricing override; with them None it is a plain sibling on the same backend. The
marker keeps the name unique so concurrent runs on the shared proxy never
@@ -251,7 +249,7 @@ class TestCustomPricing:
assert sibling_entry is not None, f"{sibling} absent from /model/info"
# custom-priced-flash overrides pricing; the sibling shares the same
- # gemini/gemini-2.5-flash backend but sets no override, so it must keep its
+ # gemini/gemini-3.8-flash backend but sets no override, so it must keep its
# own price. Equal rates mean the override leaked into the shared cost map.
assert (
sibling_entry.model_info.input_cost_per_token
diff --git a/tests/e2e/llm_translation/test_google_native_e2e.py b/tests/e2e/llm_translation/test_google_native_e2e.py
index b18683bd4ad..65a341dbc2a 100644
--- a/tests/e2e/llm_translation/test_google_native_e2e.py
+++ b/tests/e2e/llm_translation/test_google_native_e2e.py
@@ -21,7 +21,7 @@ from pydantic import BaseModel
pytestmark = pytest.mark.e2e
-UPSTREAM_MODEL = "gemini/gemini-2.5-flash"
+UPSTREAM_MODEL = "gemini/gemini-3.8-flash"
class _GenerateContentPart(BaseModel):
diff --git a/tests/e2e/llm_translation/test_image_edits_e2e.py b/tests/e2e/llm_translation/test_image_edits_e2e.py
index 66b0ebfc538..917662b9e52 100644
--- a/tests/e2e/llm_translation/test_image_edits_e2e.py
+++ b/tests/e2e/llm_translation/test_image_edits_e2e.py
@@ -24,7 +24,7 @@ from sdk_clients import SdkClients
pytestmark = pytest.mark.e2e
-IMAGE_EDIT_BACKEND: Final = "openai/gpt-image-1"
+IMAGE_EDIT_BACKEND: Final = "openai/gpt-image-2.5-sunburst"
_TEST_PNG = base64.b64decode(
"iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAS0lEQVR42u3PMQ0AAAwDoPo3"
diff --git a/tests/e2e/llm_translation/test_messages_mid_conversation_system_e2e.py b/tests/e2e/llm_translation/test_messages_mid_conversation_system_e2e.py
index 84e0674d959..59b1224a479 100644
--- a/tests/e2e/llm_translation/test_messages_mid_conversation_system_e2e.py
+++ b/tests/e2e/llm_translation/test_messages_mid_conversation_system_e2e.py
@@ -41,7 +41,7 @@ from proxy_client import ProxyClient
from pydantic import BaseModel
from sdk_clients import NO_PROXY_CACHE, SdkClients
-pytestmark = pytest.mark.e2e
+pytestmark = [pytest.mark.e2e, pytest.mark.provider_live]
FLAGGED_INVOKE_MODEL = "bedrock/invoke/us.anthropic.claude-sonnet-5"
UNFLAGGED_INVOKE_MODEL = "bedrock/invoke/us.anthropic.claude-haiku-4-5-20251001-v1:0"
diff --git a/tests/e2e/llm_translation/test_passthrough_e2e.py b/tests/e2e/llm_translation/test_passthrough_e2e.py
index 52d1c10d829..3eaeef6bddd 100644
--- a/tests/e2e/llm_translation/test_passthrough_e2e.py
+++ b/tests/e2e/llm_translation/test_passthrough_e2e.py
@@ -5,7 +5,7 @@ and verifies the proxy still logged a costed SpendLogs row
(call_type="pass_through_endpoint"), correlated by the id the caller was served:
the x-litellm-call-id header on gemini, the `msg_...` message id on anthropic.
-Covered: gemini ("gemini-2.5-flash") + anthropic ("claude-haiku-4-5"), streaming +
+Covered: gemini ("gemini-3.8-flash") + anthropic ("claude-haiku-4-5"), streaming +
non-streaming, plus native tool calls. See LLM_TRANSLATION_COVERAGE_MATRIX.md.
A passthrough call returning non-2xx fails hard (never a skip); once it returns
@@ -32,7 +32,7 @@ from passthrough_client import (
completed_responses_object,
)
-GEMINI_MODEL: Final = "gemini-2.5-flash"
+GEMINI_MODEL: Final = "gemini-3.8-flash"
ANTHROPIC_PASSTHROUGH_MODEL: Final = "claude-haiku-4-5"
EMBEDDING_MODEL = "text-embedding-3-small"
REALTIME_MODEL = "gpt-realtime-2"
@@ -291,7 +291,7 @@ class TestPassthroughModelAllowlist:
result = client.anthropic_message(key, ANTHROPIC_PASSTHROUGH_MODEL, f"say hi {unique_marker()}")
assert result.status_code == 403, (
- "a key restricted to gemini-2.5-flash must be denied a claude passthrough call, "
+ "a key restricted to gemini-3.8-flash must be denied a claude passthrough call, "
f"got {result.status_code}: {result.body[:300]}"
)
diff --git a/tests/e2e/llm_translation/test_responses_e2e.py b/tests/e2e/llm_translation/test_responses_e2e.py
index ee4699703b2..af97a633842 100644
--- a/tests/e2e/llm_translation/test_responses_e2e.py
+++ b/tests/e2e/llm_translation/test_responses_e2e.py
@@ -57,7 +57,7 @@ OPENAI_VISION_BACKEND: Final = "openai/gpt-4o"
ANTHROPIC_BACKEND: Final = "anthropic/claude-haiku-4-5"
BEDROCK_CONVERSE_BACKEND: Final = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
VERTEX_BACKEND: Final = "vertex_ai/gemini-2.5-flash"
-GEMINI_BACKEND: Final = "gemini/gemini-2.5-flash"
+GEMINI_BACKEND: Final = "gemini/gemini-3.8-flash"
OPENAI_RESPONSES_BACKEND: Final = "openai/gpt-5.5"
INSTRUCTIONS = "You are a helpful assistant"
CAT_IMAGE_URL = "https://upload.wikimedia.org/wikipedia/commons/3/3a/Cat03.jpg"
diff --git a/tests/e2e/logging/logging_client.py b/tests/e2e/logging/logging_client.py
index c4d09800c79..2c7b6af190a 100644
--- a/tests/e2e/logging/logging_client.py
+++ b/tests/e2e/logging/logging_client.py
@@ -18,10 +18,11 @@ import json
import os
import time
from dataclasses import dataclass
-from typing import Callable, Literal
+from datetime import datetime, timedelta, timezone
+from typing import Callable, Final, Literal
import pytest
-from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError
+from pydantic import AliasChoices, BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError, field_validator
from e2e_config import POLL_INTERVAL, POLL_TIMEOUT, settle_propagation
from e2e_metadata import step
@@ -124,23 +125,39 @@ class CreateGuardrailResponse(BaseModel):
guardrail_name: str | None = None
+_LANGFUSE_JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
+
+
class LangfuseObservation(BaseModel):
- model_config = ConfigDict(extra="ignore", populate_by_name=True)
+ model_config = ConfigDict(extra="ignore", populate_by_name=True, frozen=True)
id: str
trace_id: str | None = Field(default=None, alias="traceId")
name: str | None = None
type: str | None = None
- calculated_total_cost: float | None = Field(default=None, alias="calculatedTotalCost")
+ calculated_total_cost: float | None = Field(
+ default=None, validation_alias=AliasChoices("totalCost", "calculatedTotalCost")
+ )
level: str | None = None
- input: object | None = None
- output: object | None = None
- metadata: object | None = None
+ input: JsonValue = None
+ output: JsonValue = None
+ metadata: JsonValue = None
usage: object | None = None
usage_details: object | None = Field(default=None, alias="usageDetails")
model: str | None = None
+ @field_validator("input", "output", "metadata", mode="before")
+ @classmethod
+ def decode_json_fields(cls, value: object) -> JsonValue:
+ if not isinstance(value, str):
+ return _LANGFUSE_JSON.validate_python(value)
+ try:
+ return _LANGFUSE_JSON.validate_json(value)
+ except ValidationError:
+ return value
+
+
class LangfuseObservationList(BaseModel):
model_config = ConfigDict(extra="ignore")
@@ -171,12 +188,15 @@ def is_otel_v2_generation(obs: LangfuseObservation, *, key_alias: str) -> bool:
class LangfuseListParams(BaseModel):
- model_config = ConfigDict(populate_by_name=True)
+ model_config = ConfigDict(populate_by_name=True, frozen=True)
limit: int = 100
trace_id: str | None = Field(default=None, alias="traceId")
name: str | None = None
- from_start_time: str | None = Field(default=None, alias="fromStartTime")
+ from_start_time: str = Field(alias="fromStartTime")
+ to_start_time: str = Field(alias="toStartTime")
+ fields: str = "core,basic,io,metadata,model,usage"
+ expand_metadata: str = Field(default="attributes", alias="expandMetadata")
@dataclass(frozen=True, slots=True)
@@ -619,14 +639,16 @@ class LoggingClient:
name: str | None = None,
from_start_time: str | None = None,
) -> list[LangfuseObservation]:
- result = get(
- URL(f"{creds.host}/api/public/observations"),
+ now: Final = datetime.now(timezone.utc)
+ result: Final = get(
+ URL(f"{creds.host}/api/public/v2/observations"),
headers=creds.auth_headers,
params=LangfuseListParams(
limit=100,
traceId=trace_id,
name=name,
- fromStartTime=from_start_time,
+ fromStartTime=from_start_time or (now - timedelta(minutes=15)).isoformat(),
+ toStartTime=now.isoformat(),
),
response_type=LangfuseObservationList,
timeout=30.0,
diff --git a/tests/e2e/logging/test_langsmith_batch_serialization_e2e.py b/tests/e2e/logging/test_langsmith_batch_serialization_e2e.py
index e2dadf9e696..37f363772ae 100644
--- a/tests/e2e/logging/test_langsmith_batch_serialization_e2e.py
+++ b/tests/e2e/logging/test_langsmith_batch_serialization_e2e.py
@@ -32,6 +32,7 @@ pytestmark = pytest.mark.e2e
class LangsmithHeaders(Headers):
x_api_key: str = Field(serialization_alias="x-api-key")
+ x_tenant_id: str | None = Field(default=None, serialization_alias="x-tenant-id")
class LangsmithRunExtra(BaseModel):
@@ -50,6 +51,7 @@ class LangsmithCreds:
api_key: str
base_url: str
project: str
+ tenant_id: str | None
def load_langsmith_creds() -> LangsmithCreds:
@@ -62,6 +64,7 @@ def load_langsmith_creds() -> LangsmithCreds:
api_key=api_key,
base_url=(os.getenv("LANGSMITH_BASE_URL") or "https://api.smith.langchain.com").rstrip("/"),
project=os.getenv("LANGSMITH_PROJECT") or "litellm-e2e",
+ tenant_id=os.getenv("LANGSMITH_TENANT_ID"),
)
@@ -69,7 +72,7 @@ def _fetch_run(creds: LangsmithCreds, run_id: uuid.UUID) -> LangsmithRun | None:
result = get_external(
f"{creds.base_url}/runs/{run_id}",
response_type=LangsmithRun,
- headers=LangsmithHeaders(x_api_key=creds.api_key),
+ headers=LangsmithHeaders(x_api_key=creds.api_key, x_tenant_id=creds.tenant_id),
)
match result:
case Success(data=run):
@@ -102,7 +105,10 @@ class TestLangsmithBatchSerialization:
async def test_non_json_native_metadata_reaches_langsmith(self) -> None:
creds: Final = load_langsmith_creds()
logger: Final = LangsmithLogger(
- langsmith_api_key=creds.api_key, langsmith_project=creds.project, langsmith_base_url=creds.base_url
+ langsmith_api_key=creds.api_key,
+ langsmith_project=creds.project,
+ langsmith_base_url=creds.base_url,
+ langsmith_tenant_id=creds.tenant_id,
)
assert not logger.is_mock_mode, "LangsmithLogger initialised in mock mode; this e2e needs the real API"
marker: Final = unique_marker()
diff --git a/tests/e2e/logging/test_otel_trace_e2e.py b/tests/e2e/logging/test_otel_trace_e2e.py
index a5be868bc00..007fa1b5811 100644
--- a/tests/e2e/logging/test_otel_trace_e2e.py
+++ b/tests/e2e/logging/test_otel_trace_e2e.py
@@ -79,7 +79,7 @@ def _chain_reaches(span_id: str, root_id: str, trace: JaegerTrace) -> bool:
return False
-def _assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) -> None:
+def assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) -> None:
"""The enforced behavior: the destination holds exactly one call-id-tagged
trace for the call, rooted at the SERVER span, with auth/db children and
the gen-AI span all connected into that one tree - no dangling parent
@@ -112,10 +112,9 @@ def _assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) -
roots = [span for span in trace.spans if not span.references]
assert len(roots) == 1, f"expected exactly one root span, got {[s.operation_name for s in roots]}; spans: {names}"
root = roots[0]
- assert root.operation_name == f"POST {route}", (
- f"the root must be the SERVER span 'POST {route}', got {root.operation_name!r}"
- )
assert root.kind == "server", f"the root span must have kind=server, got {root.kind!r}"
+ request_route: Final = root.tag("http.route") or root.tag("url.path") or root.tag("http.target")
+ assert request_route == route, f"the root must identify route {route!r}, got {request_route!r}"
assert f"auth {route}" in names, f"auth phase span 'auth {route}' missing; spans: {names}"
assert any(name.startswith(DB_SPAN_PREFIX) for name in names), (
@@ -133,12 +132,12 @@ def _assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) -
def _poll(otel_reader: OtelReader, *, call_id: str, route: str, genai_span: str) -> CallTraces:
return otel_reader.poll_traces_for_call(
call_id=call_id,
- settled_names={f"POST {route}", f"auth {route}", genai_span},
+ settled_names={f"auth {route}", genai_span},
settled_prefixes={DB_SPAN_PREFIX},
)
-def _assert_real_ttft(hits: tuple[JaegerTrace, ...], *, genai_span: str) -> None:
+def assert_real_ttft(hits: tuple[JaegerTrace, ...], *, genai_span: str) -> None:
"""The enforced behavior: the gen-AI span for the attempt that served the
stream records a TTFT that is a real measurement - present, numeric,
positive, and strictly less than that span's own total duration. A TTFT of
@@ -282,7 +281,7 @@ class TestOtelTraceCompleteness:
assert outcome.call_id is not None, "success response must carry x-litellm-call-id"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=f"chat {MODEL}")
- _assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}")
+ assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}")
@pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["chat_completions"])
@meta(
@@ -315,7 +314,7 @@ class TestOtelTraceCompleteness:
assert outcome.call_id is not None, "success response must carry x-litellm-call-id"
hits: Final = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=f"chat {MODEL}")
- _assert_complete_trace(hits, route=route, genai_span=f"chat {MODEL}")
+ assert_complete_trace(hits, route=route, genai_span=f"chat {MODEL}")
@pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["messages"])
@meta(
@@ -333,7 +332,7 @@ class TestOtelTraceCompleteness:
"""This test verifies that one successful non-streaming /v1/messages request
produces exactly one complete OTEL trace.
- The trace must have a single root span named "POST /v1/messages". The
+ The trace must have a single root SERVER span identifying /v1/messages. The
authentication, database, and model-call spans must all belong to
the same trace and have valid parent relationships leading back to that root.
@@ -353,7 +352,7 @@ class TestOtelTraceCompleteness:
assert outcome.call_id is not None, "success response must carry x-litellm-call-id"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=f"chat {MODEL}")
- _assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}")
+ assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}")
@pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["responses"])
@meta(
@@ -392,7 +391,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {CHEAP_OPENAI_MODEL}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
- _assert_complete_trace(traces, route=route, genai_span=genai_span)
+ assert_complete_trace(traces, route=route, genai_span=genai_span)
@pytest.mark.covers("logging.otel.stream.exports_metric", exercised_on=["chat_completions"])
@meta(
@@ -443,7 +442,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {MODEL}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
- _assert_complete_trace(traces, route=route, genai_span=genai_span)
+ assert_complete_trace(traces, route=route, genai_span=genai_span)
served = one_served_genai_span(traces.hits[0], genai_span)
assert served.tag("litellm.request.streaming") is True, (
@@ -500,7 +499,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {MODEL}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
- _assert_complete_trace(traces, route=route, genai_span=genai_span)
+ assert_complete_trace(traces, route=route, genai_span=genai_span)
served = one_served_genai_span(traces.hits[0], genai_span)
assert served.tag("litellm.request.streaming") is True, (
@@ -557,7 +556,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {CHEAP_OPENAI_MODEL}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
- _assert_complete_trace(traces, route=route, genai_span=genai_span)
+ assert_complete_trace(traces, route=route, genai_span=genai_span)
one_served_genai_span(traces.hits[0], genai_span)
@@ -614,7 +613,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {MODEL}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
- _assert_real_ttft(traces.hits, genai_span=genai_span)
+ assert_real_ttft(traces.hits, genai_span=genai_span)
@pytest.mark.covers("logging.otel.stream.records_ttft", exercised_on=["messages"])
@meta(
@@ -661,7 +660,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {MODEL}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
- _assert_real_ttft(traces.hits, genai_span=genai_span)
+ assert_real_ttft(traces.hits, genai_span=genai_span)
@pytest.mark.covers("logging.otel.stream.records_ttft", exercised_on=["responses"])
@meta(
@@ -708,7 +707,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {CHEAP_OPENAI_MODEL}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
- _assert_real_ttft(traces.hits, genai_span=genai_span)
+ assert_real_ttft(traces.hits, genai_span=genai_span)
@pytest.mark.covers("logging.otel.failure.exports_metric", exercised_on=["chat_completions"])
@meta(
@@ -763,7 +762,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {model_name}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
- _assert_complete_trace(traces, route=route, genai_span=genai_span)
+ assert_complete_trace(traces, route=route, genai_span=genai_span)
root = next(span for span in traces.hits[0].spans if not span.references)
assert str(root.tag("http.status_code")) == "401", (
@@ -825,7 +824,7 @@ class TestOtelTraceCompleteness:
genai_span = f"chat {model_name}"
traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
- _assert_complete_trace(traces, route=route, genai_span=genai_span)
+ assert_complete_trace(traces, route=route, genai_span=genai_span)
root = next(span for span in traces.hits[0].spans if not span.references)
assert str(root.tag("http.status_code")) == "401", (
diff --git a/tests/e2e/otel_client.py b/tests/e2e/otel_client.py
index 5f93c677d89..b42235484dc 100644
--- a/tests/e2e/otel_client.py
+++ b/tests/e2e/otel_client.py
@@ -28,11 +28,10 @@ from dataclasses import dataclass
from typing import Final
import pytest
-from pydantic import BaseModel, ConfigDict, Field
-
from e2e_config import OTEL_QUERY_URL, POLL_INTERVAL, POLL_TIMEOUT
from e2e_http import URL, NetworkError, NoBody, Result, Success, get
from e2e_metadata import step
+from pydantic import BaseModel, ConfigDict, Field
#: OTEL resource service.name the proxy exports under (OTEL_SERVICE_NAME default).
JAEGER_SERVICE = "litellm"
@@ -118,9 +117,15 @@ def _ticks() -> Iterator[None]:
time.sleep(POLL_INTERVAL)
-def _settled(trace: JaegerTrace, names: set[str], prefixes: set[str]) -> bool:
- present = set(trace.span_names())
- return names.issubset(present) and all(any(name.startswith(prefix) for name in present) for prefix in prefixes)
+def trace_settled(trace: JaegerTrace, names: set[str], prefixes: set[str]) -> bool:
+ root: Final = root_span(trace)
+ present: Final = frozenset(trace.span_names())
+ return (
+ root is not None
+ and root.kind == "server"
+ and names.issubset(present)
+ and all(any(name.startswith(prefix) for name in present) for prefix in prefixes)
+ )
def root_span(trace: JaegerTrace) -> JaegerSpan | None:
@@ -172,7 +177,7 @@ class _Observation:
def settled(self, names: set[str], prefixes: set[str]) -> bool:
hits: Final = self.traces.hits
- return self.unreachable is None and len(hits) == 1 and not self.missing and _settled(hits[0], names, prefixes)
+ return self.unreachable is None and len(hits) == 1 and not self.missing and trace_settled(hits[0], names, prefixes)
@dataclass(frozen=True, slots=True)
diff --git a/tests/e2e/provider_cache_redis.py b/tests/e2e/provider_cache_redis.py
index 2c7419cfc0f..c1a3eada4a9 100644
--- a/tests/e2e/provider_cache_redis.py
+++ b/tests/e2e/provider_cache_redis.py
@@ -14,8 +14,8 @@ from typing import Final, Protocol, cast
from provider_cache import LIFETIME_SECONDS, CacheBusy, CacheEdge, CacheHit, CacheLookup, CacheUnavailable, CaptureLease
from pydantic import TypeAdapter, ValidationError
-from redis import Redis
-from redis.exceptions import RedisError
+from redis import Redis, RedisCluster
+from redis.exceptions import RedisClusterException, RedisError
REDIS_ARRAY: Final[TypeAdapter[list[bytes]]] = TypeAdapter(list[bytes])
@@ -83,7 +83,7 @@ class RedisResponseStore:
started: Final = time.monotonic()
try:
result: Final = self.client.eval(LOOKUP, 2, *self.keys(key), token, self.lifetime_ms, self.lease_ms)
- except (RedisError, OSError):
+ except (RedisError, RedisClusterException, OSError):
return CacheUnavailable()
try:
parts: Final = tuple(REDIS_ARRAY.validate_python(result, strict=True))
@@ -102,28 +102,46 @@ class RedisResponseStore:
result: Final = self.client.eval(
PUBLISH, 2, *self.keys(key), lease.token, lease.captured_at_ms, lease.expires_at_ms, payload, self.lifetime_ms,
)
- except (RedisError, OSError):
+ except (RedisError, RedisClusterException, OSError):
return False
return result == 1
def release(self, key: str, lease: CaptureLease) -> bool:
try:
result: Final = self.client.eval(RELEASE, 1, self.keys(key)[1], lease.token)
- except (RedisError, OSError):
+ except (RedisError, RedisClusterException, OSError):
return False
return result == 1
def discard(self, key: str, payload: bytes) -> bool:
try:
result: Final = self.client.eval(DISCARD, 1, self.keys(key)[0], payload)
- except (RedisError, OSError):
+ except (RedisError, RedisClusterException, OSError):
return False
return result == 1
+@functools.lru_cache(maxsize=16)
+def _redis_commands(url: str, cluster: bool) -> RedisCommands:
+ if cluster:
+ return cast(
+ RedisCommands, RedisCluster(url=url, socket_timeout=0.25, socket_connect_timeout=0.25, decode_responses=False)
+ )
+ return cast(RedisCommands, Redis.from_url(url, socket_timeout=0.25, socket_connect_timeout=0.25, decode_responses=False))
+
+
+@dataclass(frozen=True, slots=True)
+class LazyRedisCommands:
+ url: str
+ cluster: bool
+
+ def eval(self, script: str, numkeys: int, *args: str | bytes | int) -> object:
+ return _redis_commands(self.url, self.cluster).eval(script, numkeys, *args)
+
+
def redis_store(url: str, namespace: str) -> RedisResponseStore:
- client: Final = Redis.from_url(url, socket_timeout=0.25, socket_connect_timeout=0.25, decode_responses=False)
- return RedisResponseStore(cast(RedisCommands, client), namespace)
+ client: Final = LazyRedisCommands(url, os.environ.get("E2E_PROVIDER_CACHE_REDIS_CLUSTER") == "1")
+ return RedisResponseStore(client, namespace)
def write_metrics(cache: CacheEdge) -> None:
diff --git a/tests/e2e/quota_management/spend_tracking/test_key_attribution_e2e.py b/tests/e2e/quota_management/spend_tracking/test_key_attribution_e2e.py
index e0c19ea1b6b..1d0878d073a 100644
--- a/tests/e2e/quota_management/spend_tracking/test_key_attribution_e2e.py
+++ b/tests/e2e/quota_management/spend_tracking/test_key_attribution_e2e.py
@@ -55,7 +55,7 @@ from spend_e2e_client import (
pytestmark = pytest.mark.e2e
-CHAT_MODEL: Final = "gemini-2.5-flash"
+CHAT_MODEL: Final = "gemini-3.8-flash"
MESSAGES_MODEL: Final = "claude-haiku-4-5"
RESPONSES_MODEL: Final = "openai-responses-codex"
EMBED_MODEL: Final = "openai-text-embedding-3-small"
diff --git a/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py b/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py
index c7ef81f826a..d452525ec91 100644
--- a/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py
+++ b/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py
@@ -3,7 +3,7 @@
Run against a proxy started with the gateway config. Coverage rationale:
SPEND_TRACKING_COVERAGE_MATRIX.md.
-Model names are literals from that config: chat tests hit "gemini-2.5-flash",
+Model names are literals from that config: chat tests hit "gemini-3.8-flash",
embedding tests hit "openai-text-embedding-3-small".
Every test: fresh scoped key (isolation) -> real provider call -> unwrap (hard
@@ -37,7 +37,7 @@ from spend_reconciliation import BACKEND as TRAFFIC_BACKEND
pytestmark = pytest.mark.e2e
-GEMINI_MODEL = "gemini-2.5-flash"
+GEMINI_MODEL = "gemini-3.8-flash"
CLAUDE_MODEL = "claude-haiku-4-5"
CODEX_MODEL = "openai-responses-codex"
EMBEDDING_MODEL = "openai-text-embedding-3-small"
@@ -780,7 +780,7 @@ def test_spend_calculate_returns_nonzero_cost(client: SpendClient) -> None:
GEMINI_MODEL, "estimate the cost of this request"
)
assert cost > 0, (
- "/spend/calculate returned 0 for gemini-2.5-flash; "
+ "/spend/calculate returned 0 for gemini-3.8-flash; "
"cost map may be missing this model"
)
diff --git a/tests/e2e/quota_management/spend_tracking/test_websearch_interception_session_e2e.py b/tests/e2e/quota_management/spend_tracking/test_websearch_interception_session_e2e.py
index ab7c2370365..84a598a2bbb 100644
--- a/tests/e2e/quota_management/spend_tracking/test_websearch_interception_session_e2e.py
+++ b/tests/e2e/quota_management/spend_tracking/test_websearch_interception_session_e2e.py
@@ -30,7 +30,7 @@ from models import (
from proxy_client import ProxyClient
from pydantic import BaseModel, ValidationError
-pytestmark = pytest.mark.e2e
+pytestmark = [pytest.mark.e2e, pytest.mark.provider_live]
BEDROCK_INVOKE_BACKEND: Final = "bedrock/invoke/us.anthropic.claude-haiku-4-5-20251001-v1:0"
SEARCH_CALL_TYPE: Final = "asearch"
diff --git a/tests/e2e/router/reliability_support.py b/tests/e2e/router/reliability_support.py
index 03e585273c0..193a94cb581 100644
--- a/tests/e2e/router/reliability_support.py
+++ b/tests/e2e/router/reliability_support.py
@@ -57,11 +57,9 @@ CONTENT_POLICY_PROMPT = (
COOLDOWN_SECONDS = 30.0
REPLICA_PROPAGATION_SECONDS = 15.0
-# The smallest-context chat model OpenAI still serves (16385 tokens). A prompt
-# past that limit comes back as a real `context_length_exceeded` 400, which is
-# what litellm maps to ContextWindowExceededError.
-SMALL_CONTEXT_MODEL = "openai/gpt-3.5-turbo"
-SMALL_CONTEXT_LIMIT_TOKENS = 16385
+SMALL_CONTEXT_MODEL: Final = "openai/gpt-5.4-nano"
+# model_prices_and_context_window.json, gpt-5.4-nano, verified 2026-11-02
+SMALL_CONTEXT_LIMIT_TOKENS: Final = 272000
def oversized_prompt(marker: str) -> str:
@@ -107,7 +105,7 @@ def create_timeout_deployment(proxy: ProxyClient, name: str) -> str:
@step(f"Add a deployment named {{name}} that calls the small-context model {SMALL_CONTEXT_MODEL}")
def create_small_context_deployment(proxy: ProxyClient, name: str) -> str:
- """Register a deployment on the smallest-context model OpenAI still serves, so an
+ """Register a deployment with a smaller context window than the backup, so an
oversized prompt earns a real context-window refusal from the provider."""
return proxy.create_model(name, LiteLLMParamsBody(model=SMALL_CONTEXT_MODEL, api_key=REAL_KEY))
@@ -263,8 +261,7 @@ def spend_only_request_of(proxy: ProxyClient, spent_key: str) -> None:
@step(f"Add a deployment named {{name}} that calls {SMALL_CONTEXT_MODEL} and takes all of its group's traffic")
def create_always_picked_small_context_deployment(proxy: ProxyClient, name: str) -> str:
- """The always-picked half of a retry pair on the smallest-context model OpenAI
- still serves: it holds all of the model group's shuffle weight, so an oversized
+ """The always-picked half of a retry pair on the smaller-context model: it holds all of the model group's shuffle weight, so an oversized
prompt opens on it and earns a real context-window refusal, which never benches
a deployment, so only the retry itself can steer the request off it."""
return proxy.register_model(
diff --git a/tests/e2e/ui/helpers/navigation.ts b/tests/e2e/ui/helpers/navigation.ts
index 78288fab743..c47dc9c7694 100644
--- a/tests/e2e/ui/helpers/navigation.ts
+++ b/tests/e2e/ui/helpers/navigation.ts
@@ -37,7 +37,7 @@ export async function navigateToPage(page: PlaywrightPage, pageEnum: Page): Prom
// assert against the default page.
for (let attempt = 0; attempt < 3; attempt++) {
await page.goto(`/ui?page=${pageEnum}`);
- await page.waitForLoadState("networkidle");
+ await expect(page.getByRole("complementary")).toBeVisible();
const url = new URL(page.url());
const onLegacyRoot = url.pathname.replace(/\/+$/, "").endsWith("/ui");
if (!onLegacyRoot || url.searchParams.get("page") === pageEnum) {
diff --git a/tests/e2e/ui/tests/modelsPage/addModel.spec.ts b/tests/e2e/ui/tests/modelsPage/addModel.spec.ts
index a99fb937b83..b5e876b36d2 100644
--- a/tests/e2e/ui/tests/modelsPage/addModel.spec.ts
+++ b/tests/e2e/ui/tests/modelsPage/addModel.spec.ts
@@ -391,7 +391,7 @@ test.describe("Add Model", () => {
// Navigate to Deployed Models tab
await page.getByRole("tab", { name: "Deployed Models" }).click();
- await page.waitForLoadState("networkidle");
+ await expect(page.getByPlaceholder("Search model names")).toBeVisible();
// Search for the model we just added
await page.getByPlaceholder("Search model names").fill("claude-haiku-4-5");
@@ -470,7 +470,7 @@ test.describe("Add Model", () => {
// The Models table renders team-scoped models with the team id in the row.
await page.getByRole("tab", { name: "Deployed Models" }).click();
- await page.waitForLoadState("networkidle");
+ await expect(page.getByPlaceholder("Search model names")).toBeVisible();
await page.getByPlaceholder("Search model names").fill("cohere");
@@ -515,7 +515,7 @@ test.describe("Add Model", () => {
// Navigate to Deployed Models tab
await page.getByRole("tab", { name: "Deployed Models" }).click();
- await page.waitForLoadState("networkidle");
+ await expect(page.getByPlaceholder("Search model names")).toBeVisible();
// Search for the wildcard model
await page.getByPlaceholder("Search model names").fill("cohere");
diff --git a/tests/e2e_harness/logging/test_logging_client.py b/tests/e2e_harness/logging/test_logging_client.py
new file mode 100644
index 00000000000..c396a6c68e5
--- /dev/null
+++ b/tests/e2e_harness/logging/test_logging_client.py
@@ -0,0 +1,49 @@
+import json
+from typing import Final
+
+import pytest
+from pydantic import JsonValue
+
+from logging_client import LangfuseObservation, is_otel_v2_generation, observation_spend
+
+
+@pytest.mark.parametrize("encoded", [False, True])
+@pytest.mark.parametrize("cost", [0.0, 0.000023])
+def test_observation_preserves_v2_content_usage_and_cost(encoded: bool, cost: float) -> None:
+ content: Final[JsonValue] = [{"role": "user", "content": "unique-request-marker"}]
+ output: Final[JsonValue] = [{"role": "assistant", "content": "response"}]
+ metadata: Final[JsonValue] = {
+ "attributes": {
+ "langfuse.observation.type": "generation",
+ "litellm.metadata.user_api_key_alias": "test-key-alias",
+ }
+ }
+ usage: Final[JsonValue] = {"input": 17, "output": 3}
+ observation: Final = LangfuseObservation.model_validate(
+ {
+ "id": "observation",
+ "input": json.dumps(content) if encoded else content,
+ "output": json.dumps(output) if encoded else output,
+ "metadata": json.dumps(metadata) if encoded else metadata,
+ "usageDetails": usage,
+ "totalCost": cost,
+ }
+ )
+
+ assert observation.input == content
+ assert observation.output == output
+ assert observation.usage_details == usage
+ assert observation_spend(observation) == cost
+ assert is_otel_v2_generation(observation, key_alias="test-key-alias")
+ assert not is_otel_v2_generation(observation, key_alias="another-key")
+
+
+def test_observation_preserves_plain_text_and_legacy_cost() -> None:
+ cost: Final = 0.000031
+ observation: Final = LangfuseObservation.model_validate(
+ {"id": "observation", "input": "plain prompt", "output": "plain reply", "calculatedTotalCost": cost}
+ )
+
+ assert observation.input == "plain prompt"
+ assert observation.output == "plain reply"
+ assert observation_spend(observation) == cost
diff --git a/tests/e2e_harness/logging/test_span_selection.py b/tests/e2e_harness/logging/test_span_selection.py
index e7487562e43..76f70e4d8bb 100644
--- a/tests/e2e_harness/logging/test_span_selection.py
+++ b/tests/e2e_harness/logging/test_span_selection.py
@@ -8,8 +8,21 @@ helper is worth pinning here.
from __future__ import annotations
+from typing import Final
+
import pytest
-from otel_client import TTFT_TAG, JaegerTrace, one_served_genai_span, served_genai_spans
+from otel_client import (
+ TTFT_TAG,
+ CallTraces,
+ JaegerReference,
+ JaegerSpan,
+ JaegerTag,
+ JaegerTrace,
+ one_served_genai_span,
+ served_genai_spans,
+ trace_settled,
+)
+from test_otel_trace_e2e import assert_complete_trace, assert_real_ttft
GENAI_SPAN = "chat claude-haiku-4-5"
@@ -79,3 +92,73 @@ def test_other_operations_are_not_counted() -> None:
trace = _trace(_span("chat gpt-5.5", ttft=0.3), _span(GENAI_SPAN, ttft=0.3))
assert len(served_genai_spans(trace, GENAI_SPAN)) == 1
+
+
+def _request_trace(
+ *,
+ root_name: str = "POST",
+ root_kind: str = "server",
+ route: str = "/v1/messages",
+ genai_parent: str = "root",
+ ttft: float = 0.2,
+) -> JaegerTrace:
+ parent: Final = JaegerReference(refType="CHILD_OF", traceID="request", spanID="root")
+ return JaegerTrace(
+ traceID="request",
+ spans=[
+ JaegerSpan(
+ spanID="root",
+ operationName=root_name,
+ tags=[JaegerTag(key="span.kind", value=root_kind), JaegerTag(key="http.target", value=route)],
+ ),
+ JaegerSpan(spanID="auth", operationName="auth /v1/messages", references=[parent]),
+ JaegerSpan(spanID="db", operationName="postgres.select tokens", references=[parent]),
+ JaegerSpan(
+ spanID="genai",
+ operationName=GENAI_SPAN,
+ duration=1_000_000,
+ references=[JaegerReference(refType="CHILD_OF", traceID="request", spanID=genai_parent)],
+ tags=[JaegerTag(key="span.kind", value="client"), JaegerTag(key=TTFT_TAG, value=ttft)],
+ ),
+ ],
+ )
+
+
+@pytest.mark.parametrize("root_name", ("POST", "POST /v1/messages"))
+def test_connected_request_trace_is_independent_of_server_display_name(root_name: str) -> None:
+ trace: Final = _request_trace(root_name=root_name)
+ assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN)
+ assert_real_ttft((trace,), genai_span=GENAI_SPAN)
+ assert trace_settled(trace, {"auth /v1/messages", GENAI_SPAN}, {"postgres."})
+
+
+def test_incomplete_trace_cannot_settle_before_the_server_span_arrives() -> None:
+ trace: Final = _request_trace()
+ incomplete: Final = trace.model_copy(update={"spans": trace.spans[1:]})
+ assert not trace_settled(incomplete, {"auth /v1/messages", GENAI_SPAN}, {"postgres."})
+
+
+@pytest.mark.parametrize("route", ("/v1/responses", ""))
+def test_request_trace_must_still_identify_the_requested_route(route: str) -> None:
+ trace: Final = _request_trace(root_name="POST /v1/messages", route=route)
+ with pytest.raises(AssertionError, match="route"):
+ assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN)
+
+
+def test_request_trace_must_still_be_rooted_at_a_server_span() -> None:
+ trace: Final = _request_trace(root_name="POST /v1/messages", root_kind="client")
+ with pytest.raises(AssertionError, match="kind=server"):
+ assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN)
+
+
+def test_request_trace_must_still_connect_the_model_call_to_the_request() -> None:
+ trace: Final = _request_trace(genai_parent="absent")
+ with pytest.raises(AssertionError, match="orphaned trace"):
+ assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN)
+
+
+@pytest.mark.parametrize("ttft", (0.0, 1.0))
+def test_server_name_tolerance_does_not_accept_invalid_first_token_timing(ttft: float) -> None:
+ trace: Final = _request_trace(ttft=ttft)
+ with pytest.raises(AssertionError, match="TTFT"):
+ assert_real_ttft((trace,), genai_span=GENAI_SPAN)
diff --git a/tests/e2e_harness/pytest.ini b/tests/e2e_harness/pytest.ini
index a29166d31bd..9ccf8ba83a0 100644
--- a/tests/e2e_harness/pytest.ini
+++ b/tests/e2e_harness/pytest.ini
@@ -5,5 +5,8 @@
addopts = --strict-markers --strict-config -p no:cacheprovider
pythonpath = ../e2e ../e2e/batches ../e2e/guardrails ../e2e/logging ../e2e/perf
markers =
+ e2e: live assertion modules imported by harness regression tests
+ meta(subject): metadata on live assertion modules imported by harness regression tests
+ otel_tls: TLS coverage in live assertion modules imported by harness regression tests
covers(cell_id, *, exercised_on=()): coverage-registry cell(s) a test covers; exercised here only to prove the collector and the JUnit properties read it
cli_determinism: drives the real claude CLI for several seconds