test(e2e): refresh retired provider fixtures

This commit is contained in:
Yuneng Jiang 2026-11-02 16:57:17 -08:00
parent 7472f43421
commit 9d64fa4201
No known key found for this signature in database
10 changed files with 22 additions and 27 deletions

View file

@ -38,7 +38,7 @@ from models import (
from proxy_client import ProxyClient
from pydantic import BaseModel, Field
GUARDRAIL_BACKEND: Final = "gemini/gemini-2.5-flash"
GUARDRAIL_BACKEND: Final = "gemini/gemini-3.8-flash"
GuardrailMode = Literal["pre_call", "post_call", "during_call", "logging_only"]
BlockedWordAction = Literal["BLOCK", "MASK"]

View file

@ -45,7 +45,7 @@ from passthrough_client import PassthroughClient
pytestmark = pytest.mark.e2e
COHERE_BACKEND = "cohere/command-r-08-2024"
GEMINI_BACKEND = "gemini/gemini-2.5-flash"
GEMINI_BACKEND = "gemini/gemini-3.8-flash"
VERTEX_BACKEND: Final = "vertex_ai/gemini-2.5-flash"
AZURE_OPENAI_BACKEND: Final = "azure/gpt-5.4-nano"
AZURE_OPENAI_API_VERSION: Final = "v1"
@ -243,7 +243,7 @@ _PERSON_SCHEMA: dict[str, object] = {
OPENAI_CHAT_MODEL: Final = "gpt-5.5"
ANTHROPIC_CHAT_MODEL: Final = "claude-haiku-4-5"
GEMINI_FLASH_MODEL: Final = "gemini-2.5-flash"
GEMINI_FLASH_MODEL: Final = "gemini-3.8-flash"
CHAT_MODELS: Final = (
pytest.param(

View file

@ -5,7 +5,7 @@ teardown) instead of relying on a statically configured model, so the check is
self-contained and never inherits pricing another suite or a stale config left on
the shared proxy. custom-priced-flash sets input/output rates deliberately far
above the canonical gemini price; the isolation sibling shares the same
gemini/gemini-2.5-flash backend but sets no override. Three behaviors are checked
gemini/gemini-3.8-flash backend but sets no override. Three behaviors are checked
independently:
- billing: a real call's logged cost breakdown charges input and output tokens at
@ -37,10 +37,8 @@ from models import (
pytestmark = pytest.mark.e2e
BACKEND_MODEL = "gemini/gemini-2.5-flash"
BACKEND_MODEL = "gemini/gemini-3.8-flash"
GEMINI_API_KEY = "os.environ/GEMINI_API_KEY"
# Deliberately ~100x above canonical gemini-2.5-flash (input 3e-7 / output 2.5e-6)
# so an override that is ignored or under-applied bills at the base rate and fails.
CUSTOM_INPUT_RATE = 5e-05
CUSTOM_OUTPUT_RATE = 1e-04
@ -78,7 +76,7 @@ def _provision(
input_cost_per_token: float | None,
output_cost_per_token: float | None,
) -> str:
"""Register a fresh gemini/gemini-2.5-flash deployment (deleted on teardown) and
"""Register a fresh gemini/gemini-3.8-flash deployment (deleted on teardown) and
return its model name. With the cost fields set the deployment carries a custom
pricing override; with them None it is a plain sibling on the same backend. The
marker keeps the name unique so concurrent runs on the shared proxy never
@ -251,7 +249,7 @@ class TestCustomPricing:
assert sibling_entry is not None, f"{sibling} absent from /model/info"
# custom-priced-flash overrides pricing; the sibling shares the same
# gemini/gemini-2.5-flash backend but sets no override, so it must keep its
# gemini/gemini-3.8-flash backend but sets no override, so it must keep its
# own price. Equal rates mean the override leaked into the shared cost map.
assert (
sibling_entry.model_info.input_cost_per_token

View file

@ -21,7 +21,7 @@ from pydantic import BaseModel
pytestmark = pytest.mark.e2e
UPSTREAM_MODEL = "gemini/gemini-2.5-flash"
UPSTREAM_MODEL = "gemini/gemini-3.8-flash"
class _GenerateContentPart(BaseModel):

View file

@ -24,7 +24,7 @@ from sdk_clients import SdkClients
pytestmark = pytest.mark.e2e
IMAGE_EDIT_BACKEND: Final = "openai/gpt-image-1"
IMAGE_EDIT_BACKEND: Final = "openai/gpt-image-2.5-sunburst"
_TEST_PNG = base64.b64decode(
"iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAS0lEQVR42u3PMQ0AAAwDoPo3"

View file

@ -5,7 +5,7 @@ and verifies the proxy still logged a costed SpendLogs row
(call_type="pass_through_endpoint"), correlated by the id the caller was served:
the x-litellm-call-id header on gemini, the `msg_...` message id on anthropic.
Covered: gemini ("gemini-2.5-flash") + anthropic ("claude-haiku-4-5"), streaming +
Covered: gemini ("gemini-3.8-flash") + anthropic ("claude-haiku-4-5"), streaming +
non-streaming, plus native tool calls. See LLM_TRANSLATION_COVERAGE_MATRIX.md.
A passthrough call returning non-2xx fails hard (never a skip); once it returns
@ -32,7 +32,7 @@ from passthrough_client import (
completed_responses_object,
)
GEMINI_MODEL: Final = "gemini-2.5-flash"
GEMINI_MODEL: Final = "gemini-3.8-flash"
ANTHROPIC_PASSTHROUGH_MODEL: Final = "claude-haiku-4-5"
EMBEDDING_MODEL = "text-embedding-3-small"
REALTIME_MODEL = "gpt-realtime-2"
@ -291,7 +291,7 @@ class TestPassthroughModelAllowlist:
result = client.anthropic_message(key, ANTHROPIC_PASSTHROUGH_MODEL, f"say hi {unique_marker()}")
assert result.status_code == 403, (
"a key restricted to gemini-2.5-flash must be denied a claude passthrough call, "
"a key restricted to gemini-3.8-flash must be denied a claude passthrough call, "
f"got {result.status_code}: {result.body[:300]}"
)

View file

@ -57,7 +57,7 @@ OPENAI_VISION_BACKEND: Final = "openai/gpt-4o"
ANTHROPIC_BACKEND: Final = "anthropic/claude-haiku-4-5"
BEDROCK_CONVERSE_BACKEND: Final = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
VERTEX_BACKEND: Final = "vertex_ai/gemini-2.5-flash"
GEMINI_BACKEND: Final = "gemini/gemini-2.5-flash"
GEMINI_BACKEND: Final = "gemini/gemini-3.8-flash"
OPENAI_RESPONSES_BACKEND: Final = "openai/gpt-5.5"
INSTRUCTIONS = "You are a helpful assistant"
CAT_IMAGE_URL = "https://upload.wikimedia.org/wikipedia/commons/3/3a/Cat03.jpg"

View file

@ -55,7 +55,7 @@ from spend_e2e_client import (
pytestmark = pytest.mark.e2e
CHAT_MODEL: Final = "gemini-2.5-flash"
CHAT_MODEL: Final = "gemini-3.8-flash"
MESSAGES_MODEL: Final = "claude-haiku-4-5"
RESPONSES_MODEL: Final = "openai-responses-codex"
EMBED_MODEL: Final = "openai-text-embedding-3-small"

View file

@ -3,7 +3,7 @@
Run against a proxy started with the gateway config. Coverage rationale:
SPEND_TRACKING_COVERAGE_MATRIX.md.
Model names are literals from that config: chat tests hit "gemini-2.5-flash",
Model names are literals from that config: chat tests hit "gemini-3.8-flash",
embedding tests hit "openai-text-embedding-3-small".
Every test: fresh scoped key (isolation) -> real provider call -> unwrap (hard
@ -37,7 +37,7 @@ from spend_reconciliation import BACKEND as TRAFFIC_BACKEND
pytestmark = pytest.mark.e2e
GEMINI_MODEL = "gemini-2.5-flash"
GEMINI_MODEL = "gemini-3.8-flash"
CLAUDE_MODEL = "claude-haiku-4-5"
CODEX_MODEL = "openai-responses-codex"
EMBEDDING_MODEL = "openai-text-embedding-3-small"
@ -780,7 +780,7 @@ def test_spend_calculate_returns_nonzero_cost(client: SpendClient) -> None:
GEMINI_MODEL, "estimate the cost of this request"
)
assert cost > 0, (
"/spend/calculate returned 0 for gemini-2.5-flash; "
"/spend/calculate returned 0 for gemini-3.8-flash; "
"cost map may be missing this model"
)

View file

@ -57,11 +57,9 @@ CONTENT_POLICY_PROMPT = (
COOLDOWN_SECONDS = 30.0
REPLICA_PROPAGATION_SECONDS = 15.0
# The smallest-context chat model OpenAI still serves (16385 tokens). A prompt
# past that limit comes back as a real `context_length_exceeded` 400, which is
# what litellm maps to ContextWindowExceededError.
SMALL_CONTEXT_MODEL = "openai/gpt-3.5-turbo"
SMALL_CONTEXT_LIMIT_TOKENS = 16385
SMALL_CONTEXT_MODEL: Final = "openai/gpt-5.4-nano"
# model_prices_and_context_window.json, gpt-5.4-nano, verified 2026-11-02
SMALL_CONTEXT_LIMIT_TOKENS: Final = 272000
def oversized_prompt(marker: str) -> str:
@ -107,7 +105,7 @@ def create_timeout_deployment(proxy: ProxyClient, name: str) -> str:
@step(f"Add a deployment named {{name}} that calls the small-context model {SMALL_CONTEXT_MODEL}")
def create_small_context_deployment(proxy: ProxyClient, name: str) -> str:
"""Register a deployment on the smallest-context model OpenAI still serves, so an
"""Register a deployment with a smaller context window than the backup, so an
oversized prompt earns a real context-window refusal from the provider."""
return proxy.create_model(name, LiteLLMParamsBody(model=SMALL_CONTEXT_MODEL, api_key=REAL_KEY))
@ -263,8 +261,7 @@ def spend_only_request_of(proxy: ProxyClient, spent_key: str) -> None:
@step(f"Add a deployment named {{name}} that calls {SMALL_CONTEXT_MODEL} and takes all of its group's traffic")
def create_always_picked_small_context_deployment(proxy: ProxyClient, name: str) -> str:
"""The always-picked half of a retry pair on the smallest-context model OpenAI
still serves: it holds all of the model group's shuffle weight, so an oversized
"""The always-picked half of a retry pair on the smaller-context model: it holds all of the model group's shuffle weight, so an oversized
prompt opens on it and earns a real context-window refusal, which never benches
a deployment, so only the retry itself can steer the request off it."""
return proxy.register_model(