Merge pull request #49944 from BerriAI/litellm_e2e_qualification_rc108

test(e2e): backport qualification repairs to rc/1.108.0

tests/code_coverage_tests/test_provider_cache.py

@@ -377,7 +377,11 @@ def test_stream_completion_controls_publication(store: RedisResponseStore, provi
     assert len(provider.hits) == (2 if truncated else 1)
 
 
-def test_store_outage_preserves_provider_success(provider: Provider) -> None:
+@pytest.mark.parametrize("cluster_mode", ("0", "1"))
+def test_store_outage_preserves_provider_success(
+    provider: Provider, monkeypatch: pytest.MonkeyPatch, cluster_mode: str,
+) -> None:
+    monkeypatch.setenv("E2E_PROVIDER_CACHE_REDIS_CLUSTER", cluster_mode)
     with socket.socket() as probe:
         probe.bind(("127.0.0.1", 0))
         port: Final = probe.getsockname()[1]
@@ -401,6 +405,20 @@ def test_old_lease_cannot_overwrite_new_owner(store: RedisResponseStore) -> None
     assert isinstance(hit, CacheHit) and hit.payload == b"new"
 
 
+def test_separate_namespaces_keep_their_own_response(store: RedisResponseStore) -> None:
+    other: Final = replace(store, namespace=store.namespace + "-other")
+    first_lease: Final = store.lookup("shared-key")
+    assert isinstance(first_lease, CaptureLease)
+    assert store.publish("shared-key", first_lease, b"first namespace")
+    other_lease: Final = other.lookup("shared-key")
+    assert isinstance(other_lease, CaptureLease)
+    assert other.publish("shared-key", other_lease, b"other namespace")
+    first_hit: Final = store.lookup("shared-key")
+    other_hit: Final = other.lookup("shared-key")
+    assert isinstance(first_hit, CacheHit) and first_hit.payload == b"first namespace"
+    assert isinstance(other_hit, CacheHit) and other_hit.payload == b"other namespace"
+
+
 def test_identity_preserves_values_and_never_contains_credentials() -> None:
     variants: Final = (b'{}', b'{"a":null}', b'{"a":false}', b'{"a":0}', b'{"a":0.0}', b'{"a":"0"}', b' { }', None, b'')
     url: Final = "https://example.invalid/v1/chat/completions"

tests/e2e/AGENTS.md

@@ -35,7 +35,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family
 - `llm_translation/` - LLM endpoint and provider-translation behavior: passthrough, custom pricing, OCR, and the non-chat inference endpoints (`/v1/responses`, `/v1/messages`, `/embeddings`, `/v1/rerank`, `/v1/audio/speech`, `/v1/images/generations`), each against a deployment the test creates via `/model/new` and deletes on teardown
 - `access_control/` - the gateway's authorization and error-shape contract: per-key model allow-lists, route-group permissions (`allowed_routes`), and unknown-model validation
 - `embeddings/` - the `/embeddings` endpoint across providers
-- `batches/` - the `/batches` endpoint (placeholder until the first test lands)
+- `batches/` - the `/batches` endpoint. GovCloud batch coverage is explicitly skipped until its infrastructure is provisioned. Other missing credentials and provider errors remain failures
 - `realtime/` - realtime websocket sessions, including the pipecat audio path
 - `quota_management/` - quota enforcement and accounting, one subfolder per behavior: `ratelimit/` (rpm/tpm blocks, window reset, pacing headers on live traffic), `budgets/` (budget definition, enforcement, and reset windows: key, team, tag, soft, multi-window), and `spend_tracking/` (spend logging and cost attribution on `/spend/*`)
 - `management/` - key/team/user/organization management routes: create/update/delete persistence via the info routes, team membership, and llm-only-key route denials (API surface, not Playwright)
@@ -326,3 +326,7 @@ other.<area>.<case>.<assertion>
 - do not use xfail markers, tests should be written in a form that the end user expects it to pass
 
 - a cell addresses the stack through its front door (`PROXY_BASE_URL`) the way a customer does, never a gateway pod by address, and proves a cross-replica property with N independent calls through that door, naming the miss odds (2^-N at two pods) in its docstring. `PROXY_REPLICA_URLS` is for read-backs that are per pod by nature (polling until a management write has converged on every replica, `/metrics`, RSS), never for steering the scenario a cell asserts on at a chosen pod
+
+## Provider response cache
+
+The provider response cache can share the stage Valkey instance through a `rediss://` URL. Set `E2E_PROVIDER_CACHE_REDIS_CLUSTER=1` for a cluster-aware client, including ElastiCache Serverless. Keep the dedicated HMAC key and namespace, and restrict test cleanup to keys in the test's own namespace

tests/e2e/batches/COVERAGE.md

@@ -11,8 +11,9 @@ cost write-back via a cross-run marker baton (design below).
 
 Only supported cells are tested. The capability table in `capabilities.py` holds one
 row per supported (provider, scenario) pair, so there are no skipped cells in the
-parametrized run. The batches suite never skips: missing provider creds or upstream
-failures are hard test failures (see `tests/e2e/AGENTS.md`).
+parametrized run. Missing provider creds or upstream failures are hard test failures
+(see `tests/e2e/AGENTS.md`). GovCloud is explicitly skipped until its infrastructure
+is provisioned and does not count as qualified coverage.
 
 | Provider  | create | retrieve | cancel | list | content download | file backing |
 |-----------|--------|----------|--------|------|------------------|--------------|

tests/e2e/batches/test_batches_e2e.py

@@ -1210,6 +1210,7 @@ def _govcloud_params() -> LiteLLMParamsBody:
     )
 
 
+@pytest.mark.skip(reason="GovCloud E2E infrastructure is not provisioned")
 class TestBedrockBatchGovCloud:
     """Bedrock batch lifecycle in the AWS GovCloud partition (us-gov-west-1).
 

tests/e2e/guardrails/guardrails_client.py

@@ -38,7 +38,7 @@ from models import (
 from proxy_client import ProxyClient
 from pydantic import BaseModel, Field
 
-GUARDRAIL_BACKEND: Final = "gemini/gemini-2.5-flash"
+GUARDRAIL_BACKEND: Final = "gemini/gemini-3.8-flash"
 
 GuardrailMode = Literal["pre_call", "post_call", "during_call", "logging_only"]
 PiiEntity = Literal["EMAIL_ADDRESS", "PHONE_NUMBER", "PERSON", "CREDIT_CARD", "US_SSN"]

tests/e2e/guardrails/test_openai_moderation_guardrail_e2e.py

@@ -128,7 +128,7 @@ class TestOpenAIModerationGuardrail:
                 )
 
         allowed = unwrap(
-            client.messages(scoped_key, model, BENIGN_PROMPT, guardrails=[name], max_tokens=64)
+            client.messages(scoped_key, model, BENIGN_PROMPT, guardrails=[name], max_tokens=512)
         )
         assert allowed.content or allowed.choices, (
             "the same moderation guardrail must let a benign /v1/messages prompt through, but "

tests/e2e/llm_translation/test_chat_completions_regression_e2e.py

@@ -45,7 +45,7 @@ from passthrough_client import PassthroughClient
 pytestmark = pytest.mark.e2e
 
 COHERE_BACKEND = "cohere/command-r-08-2024"
-GEMINI_BACKEND = "gemini/gemini-2.5-flash"
+GEMINI_BACKEND = "gemini/gemini-3.8-flash"
 VERTEX_BACKEND: Final = "vertex_ai/gemini-2.5-flash"
 AZURE_OPENAI_BACKEND: Final = "azure/gpt-5.4-nano"
 AZURE_OPENAI_API_VERSION: Final = "v1"
@@ -243,7 +243,7 @@ _PERSON_SCHEMA: dict[str, object] = {
 
 OPENAI_CHAT_MODEL: Final = "gpt-5.5"
 ANTHROPIC_CHAT_MODEL: Final = "claude-haiku-4-5"
-GEMINI_FLASH_MODEL: Final = "gemini-2.5-flash"
+GEMINI_FLASH_MODEL: Final = "gemini-3.8-flash"
 
 CHAT_MODELS: Final = (
     pytest.param(

tests/e2e/llm_translation/test_custom_pricing_e2e.py

@@ -5,7 +5,7 @@ teardown) instead of relying on a statically configured model, so the check is
 self-contained and never inherits pricing another suite or a stale config left on
 the shared proxy. custom-priced-flash sets input/output rates deliberately far
 above the canonical gemini price; the isolation sibling shares the same
-gemini/gemini-2.5-flash backend but sets no override. Three behaviors are checked
+gemini/gemini-3.8-flash backend but sets no override. Three behaviors are checked
 independently:
 
 - billing: a real call's logged cost breakdown charges input and output tokens at
@@ -37,10 +37,8 @@ from models import (
 
 pytestmark = pytest.mark.e2e
 
-BACKEND_MODEL = "gemini/gemini-2.5-flash"
+BACKEND_MODEL = "gemini/gemini-3.8-flash"
 GEMINI_API_KEY = "os.environ/GEMINI_API_KEY"
-# Deliberately ~100x above canonical gemini-2.5-flash (input 3e-7 / output 2.5e-6)
-# so an override that is ignored or under-applied bills at the base rate and fails.
 CUSTOM_INPUT_RATE = 5e-05
 CUSTOM_OUTPUT_RATE = 1e-04
 
@@ -78,7 +76,7 @@ def _provision(
     input_cost_per_token: float | None,
     output_cost_per_token: float | None,
 ) -> str:
-    """Register a fresh gemini/gemini-2.5-flash deployment (deleted on teardown) and
+    """Register a fresh gemini/gemini-3.8-flash deployment (deleted on teardown) and
     return its model name. With the cost fields set the deployment carries a custom
     pricing override; with them None it is a plain sibling on the same backend. The
     marker keeps the name unique so concurrent runs on the shared proxy never
@@ -251,7 +249,7 @@ class TestCustomPricing:
         assert sibling_entry is not None, f"{sibling} absent from /model/info"
 
         # custom-priced-flash overrides pricing; the sibling shares the same
-        # gemini/gemini-2.5-flash backend but sets no override, so it must keep its
+        # gemini/gemini-3.8-flash backend but sets no override, so it must keep its
         # own price. Equal rates mean the override leaked into the shared cost map.
         assert (
             sibling_entry.model_info.input_cost_per_token

tests/e2e/llm_translation/test_google_native_e2e.py

@@ -21,7 +21,7 @@ from pydantic import BaseModel
 
 pytestmark = pytest.mark.e2e
 
-UPSTREAM_MODEL = "gemini/gemini-2.5-flash"
+UPSTREAM_MODEL = "gemini/gemini-3.8-flash"
 
 
 class _GenerateContentPart(BaseModel):

tests/e2e/llm_translation/test_image_edits_e2e.py

@@ -24,7 +24,7 @@ from sdk_clients import SdkClients
 
 pytestmark = pytest.mark.e2e
 
-IMAGE_EDIT_BACKEND: Final = "openai/gpt-image-1"
+IMAGE_EDIT_BACKEND: Final = "openai/gpt-image-2.5-sunburst"
 
 _TEST_PNG = base64.b64decode(
     "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAS0lEQVR42u3PMQ0AAAwDoPo3"

tests/e2e/llm_translation/test_messages_mid_conversation_system_e2e.py

@@ -41,7 +41,7 @@ from proxy_client import ProxyClient
 from pydantic import BaseModel
 from sdk_clients import NO_PROXY_CACHE, SdkClients
 
-pytestmark = pytest.mark.e2e
+pytestmark = [pytest.mark.e2e, pytest.mark.provider_live]
 
 FLAGGED_INVOKE_MODEL = "bedrock/invoke/us.anthropic.claude-sonnet-5"
 UNFLAGGED_INVOKE_MODEL = "bedrock/invoke/us.anthropic.claude-haiku-4-5-20251001-v1:0"

tests/e2e/llm_translation/test_passthrough_e2e.py

@@ -5,7 +5,7 @@ and verifies the proxy still logged a costed SpendLogs row
 (call_type="pass_through_endpoint"), correlated by the id the caller was served:
 the x-litellm-call-id header on gemini, the `msg_...` message id on anthropic.
 
-Covered: gemini ("gemini-2.5-flash") + anthropic ("claude-haiku-4-5"), streaming +
+Covered: gemini ("gemini-3.8-flash") + anthropic ("claude-haiku-4-5"), streaming +
 non-streaming, plus native tool calls. See LLM_TRANSLATION_COVERAGE_MATRIX.md.
 
 A passthrough call returning non-2xx fails hard (never a skip); once it returns
@@ -32,7 +32,7 @@ from passthrough_client import (
     completed_responses_object,
 )
 
-GEMINI_MODEL: Final = "gemini-2.5-flash"
+GEMINI_MODEL: Final = "gemini-3.8-flash"
 ANTHROPIC_PASSTHROUGH_MODEL: Final = "claude-haiku-4-5"
 EMBEDDING_MODEL = "text-embedding-3-small"
 REALTIME_MODEL = "gpt-realtime-2"
@@ -291,7 +291,7 @@ class TestPassthroughModelAllowlist:
 
         result = client.anthropic_message(key, ANTHROPIC_PASSTHROUGH_MODEL, f"say hi {unique_marker()}")
         assert result.status_code == 403, (
-            "a key restricted to gemini-2.5-flash must be denied a claude passthrough call, "
+            "a key restricted to gemini-3.8-flash must be denied a claude passthrough call, "
             f"got {result.status_code}: {result.body[:300]}"
         )
 

tests/e2e/llm_translation/test_responses_e2e.py

@@ -57,7 +57,7 @@ OPENAI_VISION_BACKEND: Final = "openai/gpt-4o"
 ANTHROPIC_BACKEND: Final = "anthropic/claude-haiku-4-5"
 BEDROCK_CONVERSE_BACKEND: Final = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
 VERTEX_BACKEND: Final = "vertex_ai/gemini-2.5-flash"
-GEMINI_BACKEND: Final = "gemini/gemini-2.5-flash"
+GEMINI_BACKEND: Final = "gemini/gemini-3.8-flash"
 OPENAI_RESPONSES_BACKEND: Final = "openai/gpt-5.5"
 INSTRUCTIONS = "You are a helpful assistant"
 CAT_IMAGE_URL = "https://upload.wikimedia.org/wikipedia/commons/3/3a/Cat03.jpg"

tests/e2e/logging/logging_client.py

@@ -18,10 +18,11 @@ import json
 import os
 import time
 from dataclasses import dataclass
-from typing import Callable, Literal
+from datetime import datetime, timedelta, timezone
+from typing import Callable, Final, Literal
 
 import pytest
-from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError
+from pydantic import AliasChoices, BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError, field_validator
 
 from e2e_config import POLL_INTERVAL, POLL_TIMEOUT, settle_propagation
 from e2e_metadata import step
@@ -124,23 +125,39 @@ class CreateGuardrailResponse(BaseModel):
     guardrail_name: str | None = None
 
 
+_LANGFUSE_JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
+
+
 class LangfuseObservation(BaseModel):
-    model_config = ConfigDict(extra="ignore", populate_by_name=True)
+    model_config = ConfigDict(extra="ignore", populate_by_name=True, frozen=True)
 
     id: str
     trace_id: str | None = Field(default=None, alias="traceId")
     name: str | None = None
     type: str | None = None
-    calculated_total_cost: float | None = Field(default=None, alias="calculatedTotalCost")
+    calculated_total_cost: float | None = Field(
+        default=None, validation_alias=AliasChoices("totalCost", "calculatedTotalCost")
+    )
     level: str | None = None
-    input: object | None = None
-    output: object | None = None
-    metadata: object | None = None
+    input: JsonValue = None
+    output: JsonValue = None
+    metadata: JsonValue = None
     usage: object | None = None
     usage_details: object | None = Field(default=None, alias="usageDetails")
     model: str | None = None
 
 
+    @field_validator("input", "output", "metadata", mode="before")
+    @classmethod
+    def decode_json_fields(cls, value: object) -> JsonValue:
+        if not isinstance(value, str):
+            return _LANGFUSE_JSON.validate_python(value)
+        try:
+            return _LANGFUSE_JSON.validate_json(value)
+        except ValidationError:
+            return value
+
+
 class LangfuseObservationList(BaseModel):
     model_config = ConfigDict(extra="ignore")
 
@@ -171,12 +188,15 @@ def is_otel_v2_generation(obs: LangfuseObservation, *, key_alias: str) -> bool:
 
 
 class LangfuseListParams(BaseModel):
-    model_config = ConfigDict(populate_by_name=True)
+    model_config = ConfigDict(populate_by_name=True, frozen=True)
 
     limit: int = 100
     trace_id: str | None = Field(default=None, alias="traceId")
     name: str | None = None
-    from_start_time: str | None = Field(default=None, alias="fromStartTime")
+    from_start_time: str = Field(alias="fromStartTime")
+    to_start_time: str = Field(alias="toStartTime")
+    fields: str = "core,basic,io,metadata,model,usage"
+    expand_metadata: str = Field(default="attributes", alias="expandMetadata")
 
 
 @dataclass(frozen=True, slots=True)
@@ -619,14 +639,16 @@ class LoggingClient:
         name: str | None = None,
         from_start_time: str | None = None,
     ) -> list[LangfuseObservation]:
-        result = get(
-            URL(f"{creds.host}/api/public/observations"),
+        now: Final = datetime.now(timezone.utc)
+        result: Final = get(
+            URL(f"{creds.host}/api/public/v2/observations"),
             headers=creds.auth_headers,
             params=LangfuseListParams(
                 limit=100,
                 traceId=trace_id,
                 name=name,
-                fromStartTime=from_start_time,
+                fromStartTime=from_start_time or (now - timedelta(minutes=15)).isoformat(),
+                toStartTime=now.isoformat(),
             ),
             response_type=LangfuseObservationList,
             timeout=30.0,

tests/e2e/logging/test_langsmith_batch_serialization_e2e.py

@@ -32,6 +32,7 @@ pytestmark = pytest.mark.e2e
 
 class LangsmithHeaders(Headers):
     x_api_key: str = Field(serialization_alias="x-api-key")
+    x_tenant_id: str | None = Field(default=None, serialization_alias="x-tenant-id")
 
 
 class LangsmithRunExtra(BaseModel):
@@ -50,6 +51,7 @@ class LangsmithCreds:
     api_key: str
     base_url: str
     project: str
+    tenant_id: str | None
 
 
 def load_langsmith_creds() -> LangsmithCreds:
@@ -62,6 +64,7 @@ def load_langsmith_creds() -> LangsmithCreds:
         api_key=api_key,
         base_url=(os.getenv("LANGSMITH_BASE_URL") or "https://api.smith.langchain.com").rstrip("/"),
         project=os.getenv("LANGSMITH_PROJECT") or "litellm-e2e",
+        tenant_id=os.getenv("LANGSMITH_TENANT_ID"),
     )
 
 
@@ -69,7 +72,7 @@ def _fetch_run(creds: LangsmithCreds, run_id: uuid.UUID) -> LangsmithRun | None:
     result = get_external(
         f"{creds.base_url}/runs/{run_id}",
         response_type=LangsmithRun,
-        headers=LangsmithHeaders(x_api_key=creds.api_key),
+        headers=LangsmithHeaders(x_api_key=creds.api_key, x_tenant_id=creds.tenant_id),
     )
     match result:
         case Success(data=run):
@@ -102,7 +105,10 @@ class TestLangsmithBatchSerialization:
     async def test_non_json_native_metadata_reaches_langsmith(self) -> None:
         creds: Final = load_langsmith_creds()
         logger: Final = LangsmithLogger(
-            langsmith_api_key=creds.api_key, langsmith_project=creds.project, langsmith_base_url=creds.base_url
+            langsmith_api_key=creds.api_key,
+            langsmith_project=creds.project,
+            langsmith_base_url=creds.base_url,
+            langsmith_tenant_id=creds.tenant_id,
         )
         assert not logger.is_mock_mode, "LangsmithLogger initialised in mock mode; this e2e needs the real API"
         marker: Final = unique_marker()

tests/e2e/logging/test_otel_trace_e2e.py

@@ -79,7 +79,7 @@ def _chain_reaches(span_id: str, root_id: str, trace: JaegerTrace) -> bool:
     return False
 
 
-def _assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) -> None:
+def assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) -> None:
     """The enforced behavior: the destination holds exactly one call-id-tagged
     trace for the call, rooted at the SERVER span, with auth/db children and
     the gen-AI span all connected into that one tree - no dangling parent
@@ -112,10 +112,9 @@ def _assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) -
     roots = [span for span in trace.spans if not span.references]
     assert len(roots) == 1, f"expected exactly one root span, got {[s.operation_name for s in roots]}; spans: {names}"
     root = roots[0]
-    assert root.operation_name == f"POST {route}", (
-        f"the root must be the SERVER span 'POST {route}', got {root.operation_name!r}"
-    )
     assert root.kind == "server", f"the root span must have kind=server, got {root.kind!r}"
+    request_route: Final = root.tag("http.route") or root.tag("url.path") or root.tag("http.target")
+    assert request_route == route, f"the root must identify route {route!r}, got {request_route!r}"
 
     assert f"auth {route}" in names, f"auth phase span 'auth {route}' missing; spans: {names}"
     assert any(name.startswith(DB_SPAN_PREFIX) for name in names), (
@@ -133,12 +132,12 @@ def _assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) -
 def _poll(otel_reader: OtelReader, *, call_id: str, route: str, genai_span: str) -> CallTraces:
     return otel_reader.poll_traces_for_call(
         call_id=call_id,
-        settled_names={f"POST {route}", f"auth {route}", genai_span},
+        settled_names={f"auth {route}", genai_span},
         settled_prefixes={DB_SPAN_PREFIX},
     )
 
 
-def _assert_real_ttft(hits: tuple[JaegerTrace, ...], *, genai_span: str) -> None:
+def assert_real_ttft(hits: tuple[JaegerTrace, ...], *, genai_span: str) -> None:
     """The enforced behavior: the gen-AI span for the attempt that served the
     stream records a TTFT that is a real measurement - present, numeric,
     positive, and strictly less than that span's own total duration. A TTFT of
@@ -282,7 +281,7 @@ class TestOtelTraceCompleteness:
         assert outcome.call_id is not None, "success response must carry x-litellm-call-id"
 
         traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=f"chat {MODEL}")
-        _assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}")
+        assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}")
 
     @pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["chat_completions"])
     @meta(
@@ -315,7 +314,7 @@ class TestOtelTraceCompleteness:
         assert outcome.call_id is not None, "success response must carry x-litellm-call-id"
 
         hits: Final = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=f"chat {MODEL}")
-        _assert_complete_trace(hits, route=route, genai_span=f"chat {MODEL}")
+        assert_complete_trace(hits, route=route, genai_span=f"chat {MODEL}")
 
     @pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["messages"])
     @meta(
@@ -333,7 +332,7 @@ class TestOtelTraceCompleteness:
         """This test verifies that one successful non-streaming /v1/messages request
         produces exactly one complete OTEL trace.
 
-        The trace must have a single root span named "POST /v1/messages". The
+        The trace must have a single root SERVER span identifying /v1/messages. The
         authentication, database, and model-call spans must all belong to
         the same trace and have valid parent relationships leading back to that root.
 
@@ -353,7 +352,7 @@ class TestOtelTraceCompleteness:
         assert outcome.call_id is not None, "success response must carry x-litellm-call-id"
 
         traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=f"chat {MODEL}")
-        _assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}")
+        assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}")
 
     @pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["responses"])
     @meta(
@@ -392,7 +391,7 @@ class TestOtelTraceCompleteness:
 
         genai_span = f"chat {CHEAP_OPENAI_MODEL}"
         traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
-        _assert_complete_trace(traces, route=route, genai_span=genai_span)
+        assert_complete_trace(traces, route=route, genai_span=genai_span)
 
     @pytest.mark.covers("logging.otel.stream.exports_metric", exercised_on=["chat_completions"])
     @meta(
@@ -443,7 +442,7 @@ class TestOtelTraceCompleteness:
 
         genai_span = f"chat {MODEL}"
         traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
-        _assert_complete_trace(traces, route=route, genai_span=genai_span)
+        assert_complete_trace(traces, route=route, genai_span=genai_span)
 
         served = one_served_genai_span(traces.hits[0], genai_span)
         assert served.tag("litellm.request.streaming") is True, (
@@ -500,7 +499,7 @@ class TestOtelTraceCompleteness:
 
         genai_span = f"chat {MODEL}"
         traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
-        _assert_complete_trace(traces, route=route, genai_span=genai_span)
+        assert_complete_trace(traces, route=route, genai_span=genai_span)
 
         served = one_served_genai_span(traces.hits[0], genai_span)
         assert served.tag("litellm.request.streaming") is True, (
@@ -557,7 +556,7 @@ class TestOtelTraceCompleteness:
 
         genai_span = f"chat {CHEAP_OPENAI_MODEL}"
         traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
-        _assert_complete_trace(traces, route=route, genai_span=genai_span)
+        assert_complete_trace(traces, route=route, genai_span=genai_span)
 
         one_served_genai_span(traces.hits[0], genai_span)
 
@@ -614,7 +613,7 @@ class TestOtelTraceCompleteness:
 
         genai_span = f"chat {MODEL}"
         traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
-        _assert_real_ttft(traces.hits, genai_span=genai_span)
+        assert_real_ttft(traces.hits, genai_span=genai_span)
 
     @pytest.mark.covers("logging.otel.stream.records_ttft", exercised_on=["messages"])
     @meta(
@@ -661,7 +660,7 @@ class TestOtelTraceCompleteness:
 
         genai_span = f"chat {MODEL}"
         traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
-        _assert_real_ttft(traces.hits, genai_span=genai_span)
+        assert_real_ttft(traces.hits, genai_span=genai_span)
 
     @pytest.mark.covers("logging.otel.stream.records_ttft", exercised_on=["responses"])
     @meta(
@@ -708,7 +707,7 @@ class TestOtelTraceCompleteness:
 
         genai_span = f"chat {CHEAP_OPENAI_MODEL}"
         traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
-        _assert_real_ttft(traces.hits, genai_span=genai_span)
+        assert_real_ttft(traces.hits, genai_span=genai_span)
 
     @pytest.mark.covers("logging.otel.failure.exports_metric", exercised_on=["chat_completions"])
     @meta(
@@ -763,7 +762,7 @@ class TestOtelTraceCompleteness:
 
         genai_span = f"chat {model_name}"
         traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
-        _assert_complete_trace(traces, route=route, genai_span=genai_span)
+        assert_complete_trace(traces, route=route, genai_span=genai_span)
 
         root = next(span for span in traces.hits[0].spans if not span.references)
         assert str(root.tag("http.status_code")) == "401", (
@@ -825,7 +824,7 @@ class TestOtelTraceCompleteness:
 
         genai_span = f"chat {model_name}"
         traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span)
-        _assert_complete_trace(traces, route=route, genai_span=genai_span)
+        assert_complete_trace(traces, route=route, genai_span=genai_span)
 
         root = next(span for span in traces.hits[0].spans if not span.references)
         assert str(root.tag("http.status_code")) == "401", (

tests/e2e/otel_client.py

@@ -28,11 +28,10 @@ from dataclasses import dataclass
 from typing import Final
 
 import pytest
-from pydantic import BaseModel, ConfigDict, Field
-
 from e2e_config import OTEL_QUERY_URL, POLL_INTERVAL, POLL_TIMEOUT
 from e2e_http import URL, NetworkError, NoBody, Result, Success, get
 from e2e_metadata import step
+from pydantic import BaseModel, ConfigDict, Field
 
 #: OTEL resource service.name the proxy exports under (OTEL_SERVICE_NAME default).
 JAEGER_SERVICE = "litellm"
@@ -118,9 +117,15 @@ def _ticks() -> Iterator[None]:
         time.sleep(POLL_INTERVAL)
 
 
-def _settled(trace: JaegerTrace, names: set[str], prefixes: set[str]) -> bool:
-    present = set(trace.span_names())
-    return names.issubset(present) and all(any(name.startswith(prefix) for name in present) for prefix in prefixes)
+def trace_settled(trace: JaegerTrace, names: set[str], prefixes: set[str]) -> bool:
+    root: Final = root_span(trace)
+    present: Final = frozenset(trace.span_names())
+    return (
+        root is not None
+        and root.kind == "server"
+        and names.issubset(present)
+        and all(any(name.startswith(prefix) for name in present) for prefix in prefixes)
+    )
 
 
 def root_span(trace: JaegerTrace) -> JaegerSpan | None:
@@ -172,7 +177,7 @@ class _Observation:
 
     def settled(self, names: set[str], prefixes: set[str]) -> bool:
         hits: Final = self.traces.hits
-        return self.unreachable is None and len(hits) == 1 and not self.missing and _settled(hits[0], names, prefixes)
+        return self.unreachable is None and len(hits) == 1 and not self.missing and trace_settled(hits[0], names, prefixes)
 
 
 @dataclass(frozen=True, slots=True)

tests/e2e/provider_cache_redis.py

@@ -14,8 +14,8 @@ from typing import Final, Protocol, cast
 
 from provider_cache import LIFETIME_SECONDS, CacheBusy, CacheEdge, CacheHit, CacheLookup, CacheUnavailable, CaptureLease
 from pydantic import TypeAdapter, ValidationError
-from redis import Redis
-from redis.exceptions import RedisError
+from redis import Redis, RedisCluster
+from redis.exceptions import RedisClusterException, RedisError
 
 REDIS_ARRAY: Final[TypeAdapter[list[bytes]]] = TypeAdapter(list[bytes])
 
@@ -83,7 +83,7 @@ class RedisResponseStore:
         started: Final = time.monotonic()
         try:
             result: Final = self.client.eval(LOOKUP, 2, *self.keys(key), token, self.lifetime_ms, self.lease_ms)
-        except (RedisError, OSError):
+        except (RedisError, RedisClusterException, OSError):
             return CacheUnavailable()
         try:
             parts: Final = tuple(REDIS_ARRAY.validate_python(result, strict=True))
@@ -102,28 +102,46 @@ class RedisResponseStore:
             result: Final = self.client.eval(
                 PUBLISH, 2, *self.keys(key), lease.token, lease.captured_at_ms, lease.expires_at_ms, payload, self.lifetime_ms,
             )
-        except (RedisError, OSError):
+        except (RedisError, RedisClusterException, OSError):
             return False
         return result == 1
 
     def release(self, key: str, lease: CaptureLease) -> bool:
         try:
             result: Final = self.client.eval(RELEASE, 1, self.keys(key)[1], lease.token)
-        except (RedisError, OSError):
+        except (RedisError, RedisClusterException, OSError):
             return False
         return result == 1
 
     def discard(self, key: str, payload: bytes) -> bool:
         try:
             result: Final = self.client.eval(DISCARD, 1, self.keys(key)[0], payload)
-        except (RedisError, OSError):
+        except (RedisError, RedisClusterException, OSError):
             return False
         return result == 1
 
 
+@functools.lru_cache(maxsize=16)
+def _redis_commands(url: str, cluster: bool) -> RedisCommands:
+    if cluster:
+        return cast(
+            RedisCommands, RedisCluster(url=url, socket_timeout=0.25, socket_connect_timeout=0.25, decode_responses=False)
+        )
+    return cast(RedisCommands, Redis.from_url(url, socket_timeout=0.25, socket_connect_timeout=0.25, decode_responses=False))
+
+
+@dataclass(frozen=True, slots=True)
+class LazyRedisCommands:
+    url: str
+    cluster: bool
+
+    def eval(self, script: str, numkeys: int, *args: str | bytes | int) -> object:
+        return _redis_commands(self.url, self.cluster).eval(script, numkeys, *args)
+
+
 def redis_store(url: str, namespace: str) -> RedisResponseStore:
-    client: Final = Redis.from_url(url, socket_timeout=0.25, socket_connect_timeout=0.25, decode_responses=False)
-    return RedisResponseStore(cast(RedisCommands, client), namespace)
+    client: Final = LazyRedisCommands(url, os.environ.get("E2E_PROVIDER_CACHE_REDIS_CLUSTER") == "1")
+    return RedisResponseStore(client, namespace)
 
 
 def write_metrics(cache: CacheEdge) -> None:

tests/e2e/quota_management/spend_tracking/test_key_attribution_e2e.py

@@ -55,7 +55,7 @@ from spend_e2e_client import (
 
 pytestmark = pytest.mark.e2e
 
-CHAT_MODEL: Final = "gemini-2.5-flash"
+CHAT_MODEL: Final = "gemini-3.8-flash"
 MESSAGES_MODEL: Final = "claude-haiku-4-5"
 RESPONSES_MODEL: Final = "openai-responses-codex"
 EMBED_MODEL: Final = "openai-text-embedding-3-small"

tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py

@@ -3,7 +3,7 @@
 Run against a proxy started with the gateway config. Coverage rationale:
 SPEND_TRACKING_COVERAGE_MATRIX.md.
 
-Model names are literals from that config: chat tests hit "gemini-2.5-flash",
+Model names are literals from that config: chat tests hit "gemini-3.8-flash",
 embedding tests hit "openai-text-embedding-3-small".
 
 Every test: fresh scoped key (isolation) -> real provider call -> unwrap (hard
@@ -37,7 +37,7 @@ from spend_reconciliation import BACKEND as TRAFFIC_BACKEND
 
 pytestmark = pytest.mark.e2e
 
-GEMINI_MODEL = "gemini-2.5-flash"
+GEMINI_MODEL = "gemini-3.8-flash"
 CLAUDE_MODEL = "claude-haiku-4-5"
 CODEX_MODEL = "openai-responses-codex"
 EMBEDDING_MODEL = "openai-text-embedding-3-small"
@@ -780,7 +780,7 @@ def test_spend_calculate_returns_nonzero_cost(client: SpendClient) -> None:
         GEMINI_MODEL, "estimate the cost of this request"
     )
     assert cost > 0, (
-        "/spend/calculate returned 0 for gemini-2.5-flash; "
+        "/spend/calculate returned 0 for gemini-3.8-flash; "
         "cost map may be missing this model"
     )
 

tests/e2e/quota_management/spend_tracking/test_websearch_interception_session_e2e.py

@@ -30,7 +30,7 @@ from models import (
 from proxy_client import ProxyClient
 from pydantic import BaseModel, ValidationError
 
-pytestmark = pytest.mark.e2e
+pytestmark = [pytest.mark.e2e, pytest.mark.provider_live]
 
 BEDROCK_INVOKE_BACKEND: Final = "bedrock/invoke/us.anthropic.claude-haiku-4-5-20251001-v1:0"
 SEARCH_CALL_TYPE: Final = "asearch"

tests/e2e/router/reliability_support.py

@@ -57,11 +57,9 @@ CONTENT_POLICY_PROMPT = (
 COOLDOWN_SECONDS = 30.0
 REPLICA_PROPAGATION_SECONDS = 15.0
 
-# The smallest-context chat model OpenAI still serves (16385 tokens). A prompt
-# past that limit comes back as a real `context_length_exceeded` 400, which is
-# what litellm maps to ContextWindowExceededError.
-SMALL_CONTEXT_MODEL = "openai/gpt-3.5-turbo"
-SMALL_CONTEXT_LIMIT_TOKENS = 16385
+SMALL_CONTEXT_MODEL: Final = "openai/gpt-5.4-nano"
+# model_prices_and_context_window.json, gpt-5.4-nano, verified 2026-11-02
+SMALL_CONTEXT_LIMIT_TOKENS: Final = 272000
 
 
 def oversized_prompt(marker: str) -> str:
@@ -107,7 +105,7 @@ def create_timeout_deployment(proxy: ProxyClient, name: str) -> str:
 
 @step(f"Add a deployment named {{name}} that calls the small-context model {SMALL_CONTEXT_MODEL}")
 def create_small_context_deployment(proxy: ProxyClient, name: str) -> str:
-    """Register a deployment on the smallest-context model OpenAI still serves, so an
+    """Register a deployment with a smaller context window than the backup, so an
     oversized prompt earns a real context-window refusal from the provider."""
     return proxy.create_model(name, LiteLLMParamsBody(model=SMALL_CONTEXT_MODEL, api_key=REAL_KEY))
 
@@ -263,8 +261,7 @@ def spend_only_request_of(proxy: ProxyClient, spent_key: str) -> None:
 
 @step(f"Add a deployment named {{name}} that calls {SMALL_CONTEXT_MODEL} and takes all of its group's traffic")
 def create_always_picked_small_context_deployment(proxy: ProxyClient, name: str) -> str:
-    """The always-picked half of a retry pair on the smallest-context model OpenAI
-    still serves: it holds all of the model group's shuffle weight, so an oversized
+    """The always-picked half of a retry pair on the smaller-context model: it holds all of the model group's shuffle weight, so an oversized
     prompt opens on it and earns a real context-window refusal, which never benches
     a deployment, so only the retry itself can steer the request off it."""
     return proxy.register_model(

tests/e2e/ui/helpers/navigation.ts

@@ -37,7 +37,7 @@ export async function navigateToPage(page: PlaywrightPage, pageEnum: Page): Prom
   // assert against the default page.
   for (let attempt = 0; attempt < 3; attempt++) {
     await page.goto(`/ui?page=${pageEnum}`);
-    await page.waitForLoadState("networkidle");
+    await expect(page.getByRole("complementary")).toBeVisible();
     const url = new URL(page.url());
     const onLegacyRoot = url.pathname.replace(/\/+$/, "").endsWith("/ui");
     if (!onLegacyRoot || url.searchParams.get("page") === pageEnum) {

tests/e2e/ui/tests/modelsPage/addModel.spec.ts

@@ -391,7 +391,7 @@ test.describe("Add Model", () => {
 
     // Navigate to Deployed Models tab
     await page.getByRole("tab", { name: "Deployed Models" }).click();
-    await page.waitForLoadState("networkidle");
+    await expect(page.getByPlaceholder("Search model names")).toBeVisible();
 
     // Search for the model we just added
     await page.getByPlaceholder("Search model names").fill("claude-haiku-4-5");
@@ -470,7 +470,7 @@ test.describe("Add Model", () => {
 
       // The Models table renders team-scoped models with the team id in the row.
       await page.getByRole("tab", { name: "Deployed Models" }).click();
-      await page.waitForLoadState("networkidle");
+      await expect(page.getByPlaceholder("Search model names")).toBeVisible();
 
       await page.getByPlaceholder("Search model names").fill("cohere");
 
@@ -515,7 +515,7 @@ test.describe("Add Model", () => {
 
     // Navigate to Deployed Models tab
     await page.getByRole("tab", { name: "Deployed Models" }).click();
-    await page.waitForLoadState("networkidle");
+    await expect(page.getByPlaceholder("Search model names")).toBeVisible();
 
     // Search for the wildcard model
     await page.getByPlaceholder("Search model names").fill("cohere");

tests/e2e_harness/logging/test_logging_client.py

@@ -0,0 +1,49 @@
+import json
+from typing import Final
+
+import pytest
+from pydantic import JsonValue
+
+from logging_client import LangfuseObservation, is_otel_v2_generation, observation_spend
+
+
+@pytest.mark.parametrize("encoded", [False, True])
+@pytest.mark.parametrize("cost", [0.0, 0.000023])
+def test_observation_preserves_v2_content_usage_and_cost(encoded: bool, cost: float) -> None:
+    content: Final[JsonValue] = [{"role": "user", "content": "unique-request-marker"}]
+    output: Final[JsonValue] = [{"role": "assistant", "content": "response"}]
+    metadata: Final[JsonValue] = {
+        "attributes": {
+            "langfuse.observation.type": "generation",
+            "litellm.metadata.user_api_key_alias": "test-key-alias",
+        }
+    }
+    usage: Final[JsonValue] = {"input": 17, "output": 3}
+    observation: Final = LangfuseObservation.model_validate(
+        {
+            "id": "observation",
+            "input": json.dumps(content) if encoded else content,
+            "output": json.dumps(output) if encoded else output,
+            "metadata": json.dumps(metadata) if encoded else metadata,
+            "usageDetails": usage,
+            "totalCost": cost,
+        }
+    )
+
+    assert observation.input == content
+    assert observation.output == output
+    assert observation.usage_details == usage
+    assert observation_spend(observation) == cost
+    assert is_otel_v2_generation(observation, key_alias="test-key-alias")
+    assert not is_otel_v2_generation(observation, key_alias="another-key")
+
+
+def test_observation_preserves_plain_text_and_legacy_cost() -> None:
+    cost: Final = 0.000031
+    observation: Final = LangfuseObservation.model_validate(
+        {"id": "observation", "input": "plain prompt", "output": "plain reply", "calculatedTotalCost": cost}
+    )
+
+    assert observation.input == "plain prompt"
+    assert observation.output == "plain reply"
+    assert observation_spend(observation) == cost

tests/e2e_harness/logging/test_span_selection.py

@@ -8,8 +8,21 @@ helper is worth pinning here.
 
 from __future__ import annotations
 
+from typing import Final
+
 import pytest
-from otel_client import TTFT_TAG, JaegerTrace, one_served_genai_span, served_genai_spans
+from otel_client import (
+    TTFT_TAG,
+    CallTraces,
+    JaegerReference,
+    JaegerSpan,
+    JaegerTag,
+    JaegerTrace,
+    one_served_genai_span,
+    served_genai_spans,
+    trace_settled,
+)
+from test_otel_trace_e2e import assert_complete_trace, assert_real_ttft
 
 GENAI_SPAN = "chat claude-haiku-4-5"
 
@@ -79,3 +92,73 @@ def test_other_operations_are_not_counted() -> None:
     trace = _trace(_span("chat gpt-5.5", ttft=0.3), _span(GENAI_SPAN, ttft=0.3))
 
     assert len(served_genai_spans(trace, GENAI_SPAN)) == 1
+
+
+def _request_trace(
+    *,
+    root_name: str = "POST",
+    root_kind: str = "server",
+    route: str = "/v1/messages",
+    genai_parent: str = "root",
+    ttft: float = 0.2,
+) -> JaegerTrace:
+    parent: Final = JaegerReference(refType="CHILD_OF", traceID="request", spanID="root")
+    return JaegerTrace(
+        traceID="request",
+        spans=[
+            JaegerSpan(
+                spanID="root",
+                operationName=root_name,
+                tags=[JaegerTag(key="span.kind", value=root_kind), JaegerTag(key="http.target", value=route)],
+            ),
+            JaegerSpan(spanID="auth", operationName="auth /v1/messages", references=[parent]),
+            JaegerSpan(spanID="db", operationName="postgres.select tokens", references=[parent]),
+            JaegerSpan(
+                spanID="genai",
+                operationName=GENAI_SPAN,
+                duration=1_000_000,
+                references=[JaegerReference(refType="CHILD_OF", traceID="request", spanID=genai_parent)],
+                tags=[JaegerTag(key="span.kind", value="client"), JaegerTag(key=TTFT_TAG, value=ttft)],
+            ),
+        ],
+    )
+
+
+@pytest.mark.parametrize("root_name", ("POST", "POST /v1/messages"))
+def test_connected_request_trace_is_independent_of_server_display_name(root_name: str) -> None:
+    trace: Final = _request_trace(root_name=root_name)
+    assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN)
+    assert_real_ttft((trace,), genai_span=GENAI_SPAN)
+    assert trace_settled(trace, {"auth /v1/messages", GENAI_SPAN}, {"postgres."})
+
+
+def test_incomplete_trace_cannot_settle_before_the_server_span_arrives() -> None:
+    trace: Final = _request_trace()
+    incomplete: Final = trace.model_copy(update={"spans": trace.spans[1:]})
+    assert not trace_settled(incomplete, {"auth /v1/messages", GENAI_SPAN}, {"postgres."})
+
+
+@pytest.mark.parametrize("route", ("/v1/responses", ""))
+def test_request_trace_must_still_identify_the_requested_route(route: str) -> None:
+    trace: Final = _request_trace(root_name="POST /v1/messages", route=route)
+    with pytest.raises(AssertionError, match="route"):
+        assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN)
+
+
+def test_request_trace_must_still_be_rooted_at_a_server_span() -> None:
+    trace: Final = _request_trace(root_name="POST /v1/messages", root_kind="client")
+    with pytest.raises(AssertionError, match="kind=server"):
+        assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN)
+
+
+def test_request_trace_must_still_connect_the_model_call_to_the_request() -> None:
+    trace: Final = _request_trace(genai_parent="absent")
+    with pytest.raises(AssertionError, match="orphaned trace"):
+        assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN)
+
+
+@pytest.mark.parametrize("ttft", (0.0, 1.0))
+def test_server_name_tolerance_does_not_accept_invalid_first_token_timing(ttft: float) -> None:
+    trace: Final = _request_trace(ttft=ttft)
+    with pytest.raises(AssertionError, match="TTFT"):
+        assert_real_ttft((trace,), genai_span=GENAI_SPAN)

tests/e2e_harness/pytest.ini

@@ -5,5 +5,8 @@
 addopts = --strict-markers --strict-config -p no:cacheprovider
 pythonpath = ../e2e ../e2e/batches ../e2e/guardrails ../e2e/logging ../e2e/perf
 markers =
+    e2e: live assertion modules imported by harness regression tests
+    meta(subject): metadata on live assertion modules imported by harness regression tests
+    otel_tls: TLS coverage in live assertion modules imported by harness regression tests
     covers(cell_id, *, exercised_on=()): coverage-registry cell(s) a test covers; exercised here only to prove the collector and the JUnit properties read it
     cli_determinism: drives the real claude CLI for several seconds