diff --git a/tests/e2e/guardrails/guardrails_client.py b/tests/e2e/guardrails/guardrails_client.py index cf517c7a4fe..9db95822734 100644 --- a/tests/e2e/guardrails/guardrails_client.py +++ b/tests/e2e/guardrails/guardrails_client.py @@ -38,7 +38,7 @@ from models import ( from proxy_client import ProxyClient from pydantic import BaseModel, Field -GUARDRAIL_BACKEND: Final = "gemini/gemini-2.5-flash" +GUARDRAIL_BACKEND: Final = "gemini/gemini-3.8-flash" GuardrailMode = Literal["pre_call", "post_call", "during_call", "logging_only"] BlockedWordAction = Literal["BLOCK", "MASK"] diff --git a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py index 1e0a8cbb412..4967f18818a 100644 --- a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py +++ b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py @@ -45,7 +45,7 @@ from passthrough_client import PassthroughClient pytestmark = pytest.mark.e2e COHERE_BACKEND = "cohere/command-r-08-2024" -GEMINI_BACKEND = "gemini/gemini-2.5-flash" +GEMINI_BACKEND = "gemini/gemini-3.8-flash" VERTEX_BACKEND: Final = "vertex_ai/gemini-2.5-flash" AZURE_OPENAI_BACKEND: Final = "azure/gpt-5.4-nano" AZURE_OPENAI_API_VERSION: Final = "v1" @@ -243,7 +243,7 @@ _PERSON_SCHEMA: dict[str, object] = { OPENAI_CHAT_MODEL: Final = "gpt-5.5" ANTHROPIC_CHAT_MODEL: Final = "claude-haiku-4-5" -GEMINI_FLASH_MODEL: Final = "gemini-2.5-flash" +GEMINI_FLASH_MODEL: Final = "gemini-3.8-flash" CHAT_MODELS: Final = ( pytest.param( diff --git a/tests/e2e/llm_translation/test_custom_pricing_e2e.py b/tests/e2e/llm_translation/test_custom_pricing_e2e.py index 6d8cd24648e..63b8b60867a 100644 --- a/tests/e2e/llm_translation/test_custom_pricing_e2e.py +++ b/tests/e2e/llm_translation/test_custom_pricing_e2e.py @@ -5,7 +5,7 @@ teardown) instead of relying on a statically configured model, so the check is self-contained and never inherits pricing another suite or a stale config left on the shared proxy. custom-priced-flash sets input/output rates deliberately far above the canonical gemini price; the isolation sibling shares the same -gemini/gemini-2.5-flash backend but sets no override. Three behaviors are checked +gemini/gemini-3.8-flash backend but sets no override. Three behaviors are checked independently: - billing: a real call's logged cost breakdown charges input and output tokens at @@ -37,10 +37,8 @@ from models import ( pytestmark = pytest.mark.e2e -BACKEND_MODEL = "gemini/gemini-2.5-flash" +BACKEND_MODEL = "gemini/gemini-3.8-flash" GEMINI_API_KEY = "os.environ/GEMINI_API_KEY" -# Deliberately ~100x above canonical gemini-2.5-flash (input 3e-7 / output 2.5e-6) -# so an override that is ignored or under-applied bills at the base rate and fails. CUSTOM_INPUT_RATE = 5e-05 CUSTOM_OUTPUT_RATE = 1e-04 @@ -78,7 +76,7 @@ def _provision( input_cost_per_token: float | None, output_cost_per_token: float | None, ) -> str: - """Register a fresh gemini/gemini-2.5-flash deployment (deleted on teardown) and + """Register a fresh gemini/gemini-3.8-flash deployment (deleted on teardown) and return its model name. With the cost fields set the deployment carries a custom pricing override; with them None it is a plain sibling on the same backend. The marker keeps the name unique so concurrent runs on the shared proxy never @@ -251,7 +249,7 @@ class TestCustomPricing: assert sibling_entry is not None, f"{sibling} absent from /model/info" # custom-priced-flash overrides pricing; the sibling shares the same - # gemini/gemini-2.5-flash backend but sets no override, so it must keep its + # gemini/gemini-3.8-flash backend but sets no override, so it must keep its # own price. Equal rates mean the override leaked into the shared cost map. assert ( sibling_entry.model_info.input_cost_per_token diff --git a/tests/e2e/llm_translation/test_google_native_e2e.py b/tests/e2e/llm_translation/test_google_native_e2e.py index b18683bd4ad..65a341dbc2a 100644 --- a/tests/e2e/llm_translation/test_google_native_e2e.py +++ b/tests/e2e/llm_translation/test_google_native_e2e.py @@ -21,7 +21,7 @@ from pydantic import BaseModel pytestmark = pytest.mark.e2e -UPSTREAM_MODEL = "gemini/gemini-2.5-flash" +UPSTREAM_MODEL = "gemini/gemini-3.8-flash" class _GenerateContentPart(BaseModel): diff --git a/tests/e2e/llm_translation/test_image_edits_e2e.py b/tests/e2e/llm_translation/test_image_edits_e2e.py index 66b0ebfc538..917662b9e52 100644 --- a/tests/e2e/llm_translation/test_image_edits_e2e.py +++ b/tests/e2e/llm_translation/test_image_edits_e2e.py @@ -24,7 +24,7 @@ from sdk_clients import SdkClients pytestmark = pytest.mark.e2e -IMAGE_EDIT_BACKEND: Final = "openai/gpt-image-1" +IMAGE_EDIT_BACKEND: Final = "openai/gpt-image-2.5-sunburst" _TEST_PNG = base64.b64decode( "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAS0lEQVR42u3PMQ0AAAwDoPo3" diff --git a/tests/e2e/llm_translation/test_passthrough_e2e.py b/tests/e2e/llm_translation/test_passthrough_e2e.py index 52d1c10d829..3eaeef6bddd 100644 --- a/tests/e2e/llm_translation/test_passthrough_e2e.py +++ b/tests/e2e/llm_translation/test_passthrough_e2e.py @@ -5,7 +5,7 @@ and verifies the proxy still logged a costed SpendLogs row (call_type="pass_through_endpoint"), correlated by the id the caller was served: the x-litellm-call-id header on gemini, the `msg_...` message id on anthropic. -Covered: gemini ("gemini-2.5-flash") + anthropic ("claude-haiku-4-5"), streaming + +Covered: gemini ("gemini-3.8-flash") + anthropic ("claude-haiku-4-5"), streaming + non-streaming, plus native tool calls. See LLM_TRANSLATION_COVERAGE_MATRIX.md. A passthrough call returning non-2xx fails hard (never a skip); once it returns @@ -32,7 +32,7 @@ from passthrough_client import ( completed_responses_object, ) -GEMINI_MODEL: Final = "gemini-2.5-flash" +GEMINI_MODEL: Final = "gemini-3.8-flash" ANTHROPIC_PASSTHROUGH_MODEL: Final = "claude-haiku-4-5" EMBEDDING_MODEL = "text-embedding-3-small" REALTIME_MODEL = "gpt-realtime-2" @@ -291,7 +291,7 @@ class TestPassthroughModelAllowlist: result = client.anthropic_message(key, ANTHROPIC_PASSTHROUGH_MODEL, f"say hi {unique_marker()}") assert result.status_code == 403, ( - "a key restricted to gemini-2.5-flash must be denied a claude passthrough call, " + "a key restricted to gemini-3.8-flash must be denied a claude passthrough call, " f"got {result.status_code}: {result.body[:300]}" ) diff --git a/tests/e2e/llm_translation/test_responses_e2e.py b/tests/e2e/llm_translation/test_responses_e2e.py index a0f185c4077..6b299fad18c 100644 --- a/tests/e2e/llm_translation/test_responses_e2e.py +++ b/tests/e2e/llm_translation/test_responses_e2e.py @@ -57,7 +57,7 @@ OPENAI_VISION_BACKEND: Final = "openai/gpt-4o" ANTHROPIC_BACKEND: Final = "anthropic/claude-haiku-4-5" BEDROCK_CONVERSE_BACKEND: Final = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" VERTEX_BACKEND: Final = "vertex_ai/gemini-2.5-flash" -GEMINI_BACKEND: Final = "gemini/gemini-2.5-flash" +GEMINI_BACKEND: Final = "gemini/gemini-3.8-flash" OPENAI_RESPONSES_BACKEND: Final = "openai/gpt-5.5" INSTRUCTIONS = "You are a helpful assistant" CAT_IMAGE_URL = "https://upload.wikimedia.org/wikipedia/commons/3/3a/Cat03.jpg" diff --git a/tests/e2e/logging/logging_client.py b/tests/e2e/logging/logging_client.py index 4dd681f59e8..3903d1a8363 100644 --- a/tests/e2e/logging/logging_client.py +++ b/tests/e2e/logging/logging_client.py @@ -18,10 +18,11 @@ import json import os import time from dataclasses import dataclass -from typing import Callable, Literal +from datetime import datetime, timedelta, timezone +from typing import Callable, Final, Literal import pytest -from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError +from pydantic import AliasChoices, BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError, field_validator from e2e_config import POLL_INTERVAL, POLL_TIMEOUT, settle_propagation from e2e_metadata import step @@ -124,23 +125,39 @@ class CreateGuardrailResponse(BaseModel): guardrail_name: str | None = None +_LANGFUSE_JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) + + class LangfuseObservation(BaseModel): - model_config = ConfigDict(extra="ignore", populate_by_name=True) + model_config = ConfigDict(extra="ignore", populate_by_name=True, frozen=True) id: str trace_id: str | None = Field(default=None, alias="traceId") name: str | None = None type: str | None = None - calculated_total_cost: float | None = Field(default=None, alias="calculatedTotalCost") + calculated_total_cost: float | None = Field( + default=None, validation_alias=AliasChoices("totalCost", "calculatedTotalCost") + ) level: str | None = None - input: object | None = None - output: object | None = None - metadata: object | None = None + input: JsonValue = None + output: JsonValue = None + metadata: JsonValue = None usage: object | None = None usage_details: object | None = Field(default=None, alias="usageDetails") model: str | None = None + @field_validator("input", "output", "metadata", mode="before") + @classmethod + def decode_json_fields(cls, value: object) -> JsonValue: + if not isinstance(value, str): + return _LANGFUSE_JSON.validate_python(value) + try: + return _LANGFUSE_JSON.validate_json(value) + except ValidationError: + return value + + class LangfuseObservationList(BaseModel): model_config = ConfigDict(extra="ignore") @@ -171,12 +188,15 @@ def is_otel_v2_generation(obs: LangfuseObservation, *, key_alias: str) -> bool: class LangfuseListParams(BaseModel): - model_config = ConfigDict(populate_by_name=True) + model_config = ConfigDict(populate_by_name=True, frozen=True) limit: int = 100 trace_id: str | None = Field(default=None, alias="traceId") name: str | None = None - from_start_time: str | None = Field(default=None, alias="fromStartTime") + from_start_time: str = Field(alias="fromStartTime") + to_start_time: str = Field(alias="toStartTime") + fields: str = "core,basic,io,metadata,model,usage" + expand_metadata: str = Field(default="attributes", alias="expandMetadata") @dataclass(frozen=True, slots=True) @@ -580,14 +600,16 @@ class LoggingClient: name: str | None = None, from_start_time: str | None = None, ) -> list[LangfuseObservation]: - result = get( - URL(f"{creds.host}/api/public/observations"), + now: Final = datetime.now(timezone.utc) + result: Final = get( + URL(f"{creds.host}/api/public/v2/observations"), headers=creds.auth_headers, params=LangfuseListParams( limit=100, traceId=trace_id, name=name, - fromStartTime=from_start_time, + fromStartTime=from_start_time or (now - timedelta(minutes=15)).isoformat(), + toStartTime=now.isoformat(), ), response_type=LangfuseObservationList, timeout=30.0, diff --git a/tests/e2e/logging/test_otel_trace_e2e.py b/tests/e2e/logging/test_otel_trace_e2e.py index 81a6407f642..fb6e481dc33 100644 --- a/tests/e2e/logging/test_otel_trace_e2e.py +++ b/tests/e2e/logging/test_otel_trace_e2e.py @@ -79,7 +79,7 @@ def _chain_reaches(span_id: str, root_id: str, trace: JaegerTrace) -> bool: return False -def _assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) -> None: +def assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) -> None: """The enforced behavior: the destination holds exactly one call-id-tagged trace for the call, rooted at the SERVER span, with auth/db children and the gen-AI span all connected into that one tree - no dangling parent @@ -112,10 +112,9 @@ def _assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) - roots = [span for span in trace.spans if not span.references] assert len(roots) == 1, f"expected exactly one root span, got {[s.operation_name for s in roots]}; spans: {names}" root = roots[0] - assert root.operation_name == f"POST {route}", ( - f"the root must be the SERVER span 'POST {route}', got {root.operation_name!r}" - ) assert root.kind == "server", f"the root span must have kind=server, got {root.kind!r}" + request_route: Final = root.tag("http.route") or root.tag("url.path") or root.tag("http.target") + assert request_route == route, f"the root must identify route {route!r}, got {request_route!r}" assert f"auth {route}" in names, f"auth phase span 'auth {route}' missing; spans: {names}" assert any(name.startswith(DB_SPAN_PREFIX) for name in names), ( @@ -133,12 +132,12 @@ def _assert_complete_trace(traces: CallTraces, *, route: str, genai_span: str) - def _poll(otel_reader: OtelReader, *, call_id: str, route: str, genai_span: str) -> CallTraces: return otel_reader.poll_traces_for_call( call_id=call_id, - settled_names={f"POST {route}", f"auth {route}", genai_span}, + settled_names={f"auth {route}", genai_span}, settled_prefixes={DB_SPAN_PREFIX}, ) -def _assert_real_ttft(hits: tuple[JaegerTrace, ...], *, genai_span: str) -> None: +def assert_real_ttft(hits: tuple[JaegerTrace, ...], *, genai_span: str) -> None: """The enforced behavior: the gen-AI span for the attempt that served the stream records a TTFT that is a real measurement - present, numeric, positive, and strictly less than that span's own total duration. A TTFT of @@ -282,7 +281,7 @@ class TestOtelTraceCompleteness: assert outcome.call_id is not None, "success response must carry x-litellm-call-id" traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=f"chat {MODEL}") - _assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}") + assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}") @pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["chat_completions"]) @meta( @@ -315,7 +314,7 @@ class TestOtelTraceCompleteness: assert outcome.call_id is not None, "success response must carry x-litellm-call-id" hits: Final = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=f"chat {MODEL}") - _assert_complete_trace(hits, route=route, genai_span=f"chat {MODEL}") + assert_complete_trace(hits, route=route, genai_span=f"chat {MODEL}") @pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["messages"]) @meta( @@ -333,7 +332,7 @@ class TestOtelTraceCompleteness: """This test verifies that one successful non-streaming /v1/messages request produces exactly one complete OTEL trace. - The trace must have a single root span named "POST /v1/messages". The + The trace must have a single root SERVER span identifying /v1/messages. The authentication, database, and model-call spans must all belong to the same trace and have valid parent relationships leading back to that root. @@ -353,7 +352,7 @@ class TestOtelTraceCompleteness: assert outcome.call_id is not None, "success response must carry x-litellm-call-id" traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=f"chat {MODEL}") - _assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}") + assert_complete_trace(traces, route=route, genai_span=f"chat {MODEL}") @pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["responses"]) @meta( @@ -392,7 +391,7 @@ class TestOtelTraceCompleteness: genai_span = f"chat {CHEAP_OPENAI_MODEL}" traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span) - _assert_complete_trace(traces, route=route, genai_span=genai_span) + assert_complete_trace(traces, route=route, genai_span=genai_span) @pytest.mark.covers("logging.otel.stream.exports_metric", exercised_on=["chat_completions"]) @meta( @@ -443,7 +442,7 @@ class TestOtelTraceCompleteness: genai_span = f"chat {MODEL}" traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span) - _assert_complete_trace(traces, route=route, genai_span=genai_span) + assert_complete_trace(traces, route=route, genai_span=genai_span) served = one_served_genai_span(traces.hits[0], genai_span) assert served.tag("litellm.request.streaming") is True, ( @@ -500,7 +499,7 @@ class TestOtelTraceCompleteness: genai_span = f"chat {MODEL}" traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span) - _assert_complete_trace(traces, route=route, genai_span=genai_span) + assert_complete_trace(traces, route=route, genai_span=genai_span) served = one_served_genai_span(traces.hits[0], genai_span) assert served.tag("litellm.request.streaming") is True, ( @@ -557,7 +556,7 @@ class TestOtelTraceCompleteness: genai_span = f"chat {CHEAP_OPENAI_MODEL}" traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span) - _assert_complete_trace(traces, route=route, genai_span=genai_span) + assert_complete_trace(traces, route=route, genai_span=genai_span) one_served_genai_span(traces.hits[0], genai_span) @@ -614,7 +613,7 @@ class TestOtelTraceCompleteness: genai_span = f"chat {MODEL}" traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span) - _assert_real_ttft(traces.hits, genai_span=genai_span) + assert_real_ttft(traces.hits, genai_span=genai_span) @pytest.mark.covers("logging.otel.stream.records_ttft", exercised_on=["messages"]) @meta( @@ -661,7 +660,7 @@ class TestOtelTraceCompleteness: genai_span = f"chat {MODEL}" traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span) - _assert_real_ttft(traces.hits, genai_span=genai_span) + assert_real_ttft(traces.hits, genai_span=genai_span) @pytest.mark.covers("logging.otel.stream.records_ttft", exercised_on=["responses"]) @meta( @@ -708,7 +707,7 @@ class TestOtelTraceCompleteness: genai_span = f"chat {CHEAP_OPENAI_MODEL}" traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span) - _assert_real_ttft(traces.hits, genai_span=genai_span) + assert_real_ttft(traces.hits, genai_span=genai_span) @pytest.mark.covers("logging.otel.failure.exports_metric", exercised_on=["chat_completions"]) @meta( @@ -763,7 +762,7 @@ class TestOtelTraceCompleteness: genai_span = f"chat {model_name}" traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span) - _assert_complete_trace(traces, route=route, genai_span=genai_span) + assert_complete_trace(traces, route=route, genai_span=genai_span) root = next(span for span in traces.hits[0].spans if not span.references) assert str(root.tag("http.status_code")) == "401", ( @@ -825,7 +824,7 @@ class TestOtelTraceCompleteness: genai_span = f"chat {model_name}" traces = _poll(otel_reader, call_id=outcome.call_id, route=route, genai_span=genai_span) - _assert_complete_trace(traces, route=route, genai_span=genai_span) + assert_complete_trace(traces, route=route, genai_span=genai_span) root = next(span for span in traces.hits[0].spans if not span.references) assert str(root.tag("http.status_code")) == "401", ( diff --git a/tests/e2e/management/test_budget_customer_user_org_e2e.py b/tests/e2e/management/test_budget_customer_user_org_e2e.py index ae4e6db4e15..ea4afee78ca 100644 --- a/tests/e2e/management/test_budget_customer_user_org_e2e.py +++ b/tests/e2e/management/test_budget_customer_user_org_e2e.py @@ -418,7 +418,7 @@ class TestBudgetListV1: @pytest.mark.covers("mgmt.budget.list_v1.happy_path") @meta(Subject(domain=Domain.SPEND_BUDGETS, route=Route.BUDGET_MANAGEMENT)) def test_refuses_a_sort_field_and_a_parameter_it_does_not_support(self, client: ManagementClient) -> None: - assert _list_status(client, BudgetPageParams(sort="budget_duration")) == 400 + assert _list_status(client, BudgetPageParams(sort="not_a_sort_field")) == 400 assert _list_status(client, BudgetPageParams(not_a_parameter="b-1")) == 400 @pytest.mark.covers("mgmt.budget.list_v1.admin_only") diff --git a/tests/e2e/otel_client.py b/tests/e2e/otel_client.py index 5f93c677d89..b42235484dc 100644 --- a/tests/e2e/otel_client.py +++ b/tests/e2e/otel_client.py @@ -28,11 +28,10 @@ from dataclasses import dataclass from typing import Final import pytest -from pydantic import BaseModel, ConfigDict, Field - from e2e_config import OTEL_QUERY_URL, POLL_INTERVAL, POLL_TIMEOUT from e2e_http import URL, NetworkError, NoBody, Result, Success, get from e2e_metadata import step +from pydantic import BaseModel, ConfigDict, Field #: OTEL resource service.name the proxy exports under (OTEL_SERVICE_NAME default). JAEGER_SERVICE = "litellm" @@ -118,9 +117,15 @@ def _ticks() -> Iterator[None]: time.sleep(POLL_INTERVAL) -def _settled(trace: JaegerTrace, names: set[str], prefixes: set[str]) -> bool: - present = set(trace.span_names()) - return names.issubset(present) and all(any(name.startswith(prefix) for name in present) for prefix in prefixes) +def trace_settled(trace: JaegerTrace, names: set[str], prefixes: set[str]) -> bool: + root: Final = root_span(trace) + present: Final = frozenset(trace.span_names()) + return ( + root is not None + and root.kind == "server" + and names.issubset(present) + and all(any(name.startswith(prefix) for name in present) for prefix in prefixes) + ) def root_span(trace: JaegerTrace) -> JaegerSpan | None: @@ -172,7 +177,7 @@ class _Observation: def settled(self, names: set[str], prefixes: set[str]) -> bool: hits: Final = self.traces.hits - return self.unreachable is None and len(hits) == 1 and not self.missing and _settled(hits[0], names, prefixes) + return self.unreachable is None and len(hits) == 1 and not self.missing and trace_settled(hits[0], names, prefixes) @dataclass(frozen=True, slots=True) diff --git a/tests/e2e/quota_management/spend_tracking/test_key_attribution_e2e.py b/tests/e2e/quota_management/spend_tracking/test_key_attribution_e2e.py index e0c19ea1b6b..1d0878d073a 100644 --- a/tests/e2e/quota_management/spend_tracking/test_key_attribution_e2e.py +++ b/tests/e2e/quota_management/spend_tracking/test_key_attribution_e2e.py @@ -55,7 +55,7 @@ from spend_e2e_client import ( pytestmark = pytest.mark.e2e -CHAT_MODEL: Final = "gemini-2.5-flash" +CHAT_MODEL: Final = "gemini-3.8-flash" MESSAGES_MODEL: Final = "claude-haiku-4-5" RESPONSES_MODEL: Final = "openai-responses-codex" EMBED_MODEL: Final = "openai-text-embedding-3-small" diff --git a/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py b/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py index c7ef81f826a..d452525ec91 100644 --- a/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py +++ b/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py @@ -3,7 +3,7 @@ Run against a proxy started with the gateway config. Coverage rationale: SPEND_TRACKING_COVERAGE_MATRIX.md. -Model names are literals from that config: chat tests hit "gemini-2.5-flash", +Model names are literals from that config: chat tests hit "gemini-3.8-flash", embedding tests hit "openai-text-embedding-3-small". Every test: fresh scoped key (isolation) -> real provider call -> unwrap (hard @@ -37,7 +37,7 @@ from spend_reconciliation import BACKEND as TRAFFIC_BACKEND pytestmark = pytest.mark.e2e -GEMINI_MODEL = "gemini-2.5-flash" +GEMINI_MODEL = "gemini-3.8-flash" CLAUDE_MODEL = "claude-haiku-4-5" CODEX_MODEL = "openai-responses-codex" EMBEDDING_MODEL = "openai-text-embedding-3-small" @@ -780,7 +780,7 @@ def test_spend_calculate_returns_nonzero_cost(client: SpendClient) -> None: GEMINI_MODEL, "estimate the cost of this request" ) assert cost > 0, ( - "/spend/calculate returned 0 for gemini-2.5-flash; " + "/spend/calculate returned 0 for gemini-3.8-flash; " "cost map may be missing this model" ) diff --git a/tests/e2e/router/reliability_support.py b/tests/e2e/router/reliability_support.py index 03e585273c0..193a94cb581 100644 --- a/tests/e2e/router/reliability_support.py +++ b/tests/e2e/router/reliability_support.py @@ -57,11 +57,9 @@ CONTENT_POLICY_PROMPT = ( COOLDOWN_SECONDS = 30.0 REPLICA_PROPAGATION_SECONDS = 15.0 -# The smallest-context chat model OpenAI still serves (16385 tokens). A prompt -# past that limit comes back as a real `context_length_exceeded` 400, which is -# what litellm maps to ContextWindowExceededError. -SMALL_CONTEXT_MODEL = "openai/gpt-3.5-turbo" -SMALL_CONTEXT_LIMIT_TOKENS = 16385 +SMALL_CONTEXT_MODEL: Final = "openai/gpt-5.4-nano" +# model_prices_and_context_window.json, gpt-5.4-nano, verified 2026-11-02 +SMALL_CONTEXT_LIMIT_TOKENS: Final = 272000 def oversized_prompt(marker: str) -> str: @@ -107,7 +105,7 @@ def create_timeout_deployment(proxy: ProxyClient, name: str) -> str: @step(f"Add a deployment named {{name}} that calls the small-context model {SMALL_CONTEXT_MODEL}") def create_small_context_deployment(proxy: ProxyClient, name: str) -> str: - """Register a deployment on the smallest-context model OpenAI still serves, so an + """Register a deployment with a smaller context window than the backup, so an oversized prompt earns a real context-window refusal from the provider.""" return proxy.create_model(name, LiteLLMParamsBody(model=SMALL_CONTEXT_MODEL, api_key=REAL_KEY)) @@ -263,8 +261,7 @@ def spend_only_request_of(proxy: ProxyClient, spent_key: str) -> None: @step(f"Add a deployment named {{name}} that calls {SMALL_CONTEXT_MODEL} and takes all of its group's traffic") def create_always_picked_small_context_deployment(proxy: ProxyClient, name: str) -> str: - """The always-picked half of a retry pair on the smallest-context model OpenAI - still serves: it holds all of the model group's shuffle weight, so an oversized + """The always-picked half of a retry pair on the smaller-context model: it holds all of the model group's shuffle weight, so an oversized prompt opens on it and earns a real context-window refusal, which never benches a deployment, so only the retry itself can steer the request off it.""" return proxy.register_model( diff --git a/tests/e2e_harness/logging/test_logging_client.py b/tests/e2e_harness/logging/test_logging_client.py new file mode 100644 index 00000000000..c396a6c68e5 --- /dev/null +++ b/tests/e2e_harness/logging/test_logging_client.py @@ -0,0 +1,49 @@ +import json +from typing import Final + +import pytest +from pydantic import JsonValue + +from logging_client import LangfuseObservation, is_otel_v2_generation, observation_spend + + +@pytest.mark.parametrize("encoded", [False, True]) +@pytest.mark.parametrize("cost", [0.0, 0.000023]) +def test_observation_preserves_v2_content_usage_and_cost(encoded: bool, cost: float) -> None: + content: Final[JsonValue] = [{"role": "user", "content": "unique-request-marker"}] + output: Final[JsonValue] = [{"role": "assistant", "content": "response"}] + metadata: Final[JsonValue] = { + "attributes": { + "langfuse.observation.type": "generation", + "litellm.metadata.user_api_key_alias": "test-key-alias", + } + } + usage: Final[JsonValue] = {"input": 17, "output": 3} + observation: Final = LangfuseObservation.model_validate( + { + "id": "observation", + "input": json.dumps(content) if encoded else content, + "output": json.dumps(output) if encoded else output, + "metadata": json.dumps(metadata) if encoded else metadata, + "usageDetails": usage, + "totalCost": cost, + } + ) + + assert observation.input == content + assert observation.output == output + assert observation.usage_details == usage + assert observation_spend(observation) == cost + assert is_otel_v2_generation(observation, key_alias="test-key-alias") + assert not is_otel_v2_generation(observation, key_alias="another-key") + + +def test_observation_preserves_plain_text_and_legacy_cost() -> None: + cost: Final = 0.000031 + observation: Final = LangfuseObservation.model_validate( + {"id": "observation", "input": "plain prompt", "output": "plain reply", "calculatedTotalCost": cost} + ) + + assert observation.input == "plain prompt" + assert observation.output == "plain reply" + assert observation_spend(observation) == cost diff --git a/tests/e2e_harness/logging/test_span_selection.py b/tests/e2e_harness/logging/test_span_selection.py index e7487562e43..76f70e4d8bb 100644 --- a/tests/e2e_harness/logging/test_span_selection.py +++ b/tests/e2e_harness/logging/test_span_selection.py @@ -8,8 +8,21 @@ helper is worth pinning here. from __future__ import annotations +from typing import Final + import pytest -from otel_client import TTFT_TAG, JaegerTrace, one_served_genai_span, served_genai_spans +from otel_client import ( + TTFT_TAG, + CallTraces, + JaegerReference, + JaegerSpan, + JaegerTag, + JaegerTrace, + one_served_genai_span, + served_genai_spans, + trace_settled, +) +from test_otel_trace_e2e import assert_complete_trace, assert_real_ttft GENAI_SPAN = "chat claude-haiku-4-5" @@ -79,3 +92,73 @@ def test_other_operations_are_not_counted() -> None: trace = _trace(_span("chat gpt-5.5", ttft=0.3), _span(GENAI_SPAN, ttft=0.3)) assert len(served_genai_spans(trace, GENAI_SPAN)) == 1 + + +def _request_trace( + *, + root_name: str = "POST", + root_kind: str = "server", + route: str = "/v1/messages", + genai_parent: str = "root", + ttft: float = 0.2, +) -> JaegerTrace: + parent: Final = JaegerReference(refType="CHILD_OF", traceID="request", spanID="root") + return JaegerTrace( + traceID="request", + spans=[ + JaegerSpan( + spanID="root", + operationName=root_name, + tags=[JaegerTag(key="span.kind", value=root_kind), JaegerTag(key="http.target", value=route)], + ), + JaegerSpan(spanID="auth", operationName="auth /v1/messages", references=[parent]), + JaegerSpan(spanID="db", operationName="postgres.select tokens", references=[parent]), + JaegerSpan( + spanID="genai", + operationName=GENAI_SPAN, + duration=1_000_000, + references=[JaegerReference(refType="CHILD_OF", traceID="request", spanID=genai_parent)], + tags=[JaegerTag(key="span.kind", value="client"), JaegerTag(key=TTFT_TAG, value=ttft)], + ), + ], + ) + + +@pytest.mark.parametrize("root_name", ("POST", "POST /v1/messages")) +def test_connected_request_trace_is_independent_of_server_display_name(root_name: str) -> None: + trace: Final = _request_trace(root_name=root_name) + assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN) + assert_real_ttft((trace,), genai_span=GENAI_SPAN) + assert trace_settled(trace, {"auth /v1/messages", GENAI_SPAN}, {"postgres."}) + + +def test_incomplete_trace_cannot_settle_before_the_server_span_arrives() -> None: + trace: Final = _request_trace() + incomplete: Final = trace.model_copy(update={"spans": trace.spans[1:]}) + assert not trace_settled(incomplete, {"auth /v1/messages", GENAI_SPAN}, {"postgres."}) + + +@pytest.mark.parametrize("route", ("/v1/responses", "")) +def test_request_trace_must_still_identify_the_requested_route(route: str) -> None: + trace: Final = _request_trace(root_name="POST /v1/messages", route=route) + with pytest.raises(AssertionError, match="route"): + assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN) + + +def test_request_trace_must_still_be_rooted_at_a_server_span() -> None: + trace: Final = _request_trace(root_name="POST /v1/messages", root_kind="client") + with pytest.raises(AssertionError, match="kind=server"): + assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN) + + +def test_request_trace_must_still_connect_the_model_call_to_the_request() -> None: + trace: Final = _request_trace(genai_parent="absent") + with pytest.raises(AssertionError, match="orphaned trace"): + assert_complete_trace(CallTraces((trace,), ()), route="/v1/messages", genai_span=GENAI_SPAN) + + +@pytest.mark.parametrize("ttft", (0.0, 1.0)) +def test_server_name_tolerance_does_not_accept_invalid_first_token_timing(ttft: float) -> None: + trace: Final = _request_trace(ttft=ttft) + with pytest.raises(AssertionError, match="TTFT"): + assert_real_ttft((trace,), genai_span=GENAI_SPAN) diff --git a/tests/e2e_harness/pytest.ini b/tests/e2e_harness/pytest.ini index a29166d31bd..9ccf8ba83a0 100644 --- a/tests/e2e_harness/pytest.ini +++ b/tests/e2e_harness/pytest.ini @@ -5,5 +5,8 @@ addopts = --strict-markers --strict-config -p no:cacheprovider pythonpath = ../e2e ../e2e/batches ../e2e/guardrails ../e2e/logging ../e2e/perf markers = + e2e: live assertion modules imported by harness regression tests + meta(subject): metadata on live assertion modules imported by harness regression tests + otel_tls: TLS coverage in live assertion modules imported by harness regression tests covers(cell_id, *, exercised_on=()): coverage-registry cell(s) a test covers; exercised here only to prove the collector and the JUnit properties read it cli_determinism: drives the real claude CLI for several seconds