scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-02 07:13:02 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 48

test(e2e): move the Gemini Flash alias off the retired gemini-2.5-flash

Browse source 
Google AI Studio now answers 404 for gemini-2.5-flash on new projects, so the
e2e aliases move to gemini-3.8-flash. The Vertex alias and every Vertex
deployment, batch, and passthrough URL use the global location, because
gemini-3.8-flash is only served there

3.8 Flash thinks by default, so the Vertex chat tests send reasoning_effort
none with more max_tokens, and the Presidio pre_call tests get the same 512
token room as post_call. The custom pricing override rates are scaled to stay
about 100x above the new base price
This commit is contained in:
mateo-berri 2026-10-30 13:52:08 -07:00
parent 67954b5c3a
commit 9c5d6516f3
34 changed files with 80 additions and 77 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
tests/e2e/AGENTS.md
View file

@ -75,7 +75,7 @@ class TestPromptCompression:
def test_prompt_compression_accumulate_spend(self, key_id, user_id):
for _ in range(10):
response = self.resources.proxy.post("gemini-2.5-flash", key_id, user_id)
response = self.resources.proxy.post("gemini-3.8-flash", key_id, user_id)
compressed_value = ...
assert response.cost == compressed_value # the cost was actually reduced
```

4
tests/e2e/CONTRIBUTING.md
View file

@ -9,7 +9,7 @@ When contributing to this directory, please first discuss the change you wish to
## Setup
The suites run against a live proxy, so bring one up first by running the litellm proxy locally. Point it at a config that prewires the example models the suites use (`gpt-5.5`, `claude-haiku-4-5`, `gemini-2.5-flash`, `openai-text-embedding-3-small`) with keys from your `.env`, and enables prompt storage, a redis cache, the fast budget rescheduler the quota suites rely on, and `router_settings.optional_pre_call_checks: ["prompt_caching"]`, which the router suite's prompt-cache affinity test reads back from `GET /router/settings` and fails without. If your test needs another model, a pricing override, or a guardrail declared up front, add it to that config and read it back in the test rather than hardcoding values
The suites run against a live proxy, so bring one up first by running the litellm proxy locally. Point it at a config that prewires the example models the suites use (`gpt-5.5`, `claude-haiku-4-5`, `gemini-3.8-flash`, `openai-text-embedding-3-small`) with keys from your `.env`, and enables prompt storage, a redis cache, the fast budget rescheduler the quota suites rely on, and `router_settings.optional_pre_call_checks: ["prompt_caching"]`, which the router suite's prompt-cache affinity test reads back from `GET /router/settings` and fails without. If your test needs another model, a pricing override, or a guardrail declared up front, add it to that config and read it back in the test rather than hardcoding values
## Running the tests locally
@ -206,7 +206,7 @@ class TestPromptCompression:
def test_prompt_compression_accumulate_spend(self, key_id, user_id):
for _ in range(10):
response = self.resources.proxy.post("gemini-2.5-flash", key_id, user_id)
response = self.resources.proxy.post("gemini-3.8-flash", key_id, user_id)
compressed_value = ...
assert response.cost == compressed_value # the cost was actually reduced
```

2
tests/e2e/access_control/test_access_control_e2e.py
View file

@ -30,7 +30,7 @@ from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
pytestmark = pytest.mark.e2e
ALLOWED_MODEL = "gemini-2.5-flash"
ALLOWED_MODEL = "gemini-3.8-flash"
DISALLOWED_MODEL = "gpt-5.5"
VIRTUAL_KEY_BACKEND = "anthropic/claude-haiku-4-5-20251001"
EMBEDDING_MODEL = "openai-text-embedding-3-small"

2
tests/e2e/access_control/test_project_all_team_models_e2e.py
View file

@ -28,7 +28,7 @@ from models import ChatResponse, TeamInfoResponse
pytestmark = pytest.mark.e2e
TEAM_MODEL = "gemini-2.5-flash"
TEAM_MODEL = "gemini-3.8-flash"
OUTSIDE_MODEL = "gpt-5.5"

4
tests/e2e/batches/capabilities.py
View file

@ -23,7 +23,7 @@ def batch_model_name(base: str) -> str:
OPENAI_BATCH_BACKEND: Final = "gpt-4o-mini"
AZURE_BATCH_BACKEND: Final = "gpt-5.4-mini-batch"
VERTEX_BATCH_BACKEND: Final = "gemini-2.5-flash"
VERTEX_BATCH_BACKEND: Final = "gemini-3.8-flash"
BEDROCK_BATCH_BACKEND: Final = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
@ -81,7 +81,7 @@ class Provider:
return LiteLLMParamsBody(
model=f"vertex_ai/{VERTEX_BATCH_BACKEND}",
vertex_project="os.environ/VERTEXAI_PROJECT",
vertex_location="us-central1",
vertex_location="global",
vertex_credentials="os.environ/VERTEXAI_CREDENTIALS",
gcs_bucket_name="os.environ/GCS_BUCKET_NAME",
bucket_name="os.environ/GCS_BUCKET_NAME",

2
tests/e2e/batches/test_batches_e2e.py
View file

@ -1286,7 +1286,7 @@ class TestBedrockBatchGovCloud:
assert fetched.id == batch.id
GEMINI_FILES_RAW_MODEL = "gemini-2.5-flash"
GEMINI_FILES_RAW_MODEL = "gemini-3.8-flash"
class TestGeminiFiles:

10
tests/e2e/gateway/stage_mirror_ci_config.yml
View file

@ -55,15 +55,15 @@ model_list:
litellm_params:
model: anthropic/claude-haiku-4-5
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: gemini-2.5-flash-vertex
- model_name: gemini-3.8-flash-vertex
litellm_params:
model: vertex_ai/gemini-2.5-flash
model: vertex_ai/gemini-3.8-flash
vertex_project: os.environ/VERTEXAI_PROJECT
vertex_location: us-central1
vertex_location: global
vertex_credentials: os.environ/VERTEXAI_CREDENTIALS
- model_name: gemini-2.5-flash
- model_name: gemini-3.8-flash
litellm_params:
model: gemini/gemini-2.5-flash
model: gemini/gemini-3.8-flash
api_key: os.environ/GEMINI_API_KEY
- model_name: e2e-alias-rl-target
litellm_params:

2
tests/e2e/guardrails/guardrails_client.py
View file

@ -38,7 +38,7 @@ from models import (
from proxy_client import ProxyClient
from pydantic import BaseModel, Field
GUARDRAIL_BACKEND: Final = "gemini/gemini-2.5-flash"
GUARDRAIL_BACKEND: Final = "gemini/gemini-3.8-flash"
GuardrailMode = Literal["pre_call", "post_call", "during_call", "logging_only"]
PiiEntity = Literal["EMAIL_ADDRESS", "PHONE_NUMBER", "PERSON", "CREDIT_CARD", "US_SSN"]

2
tests/e2e/guardrails/test_bedrock_guardrail_e2e.py
View file

@ -48,7 +48,7 @@ def _without_assessments(value: JsonValue) -> JsonValue:
return value
MODEL = "gemini-2.5-flash"
MODEL = "gemini-3.8-flash"
# Matches the word/topic policy the guardrail this suite points at actually denies.
# Content filters are not assumed: the guardrail resource carries no contentPolicy,
# so a hate/violence prompt returns action=NONE and never blocks.

2
tests/e2e/guardrails/test_guardrail_dispatch_e2e.py
View file

@ -16,7 +16,7 @@ from guardrails_client import GuardrailsClient
pytestmark = pytest.mark.e2e
MODEL = "gemini-2.5-flash"
MODEL = "gemini-3.8-flash"
@pytest.mark.skip(

2
tests/e2e/guardrails/test_key_guardrail_image_edit_e2e.py
View file

@ -13,7 +13,7 @@ from models import LiteLLMParamsBody
pytestmark = pytest.mark.e2e
CHAT_MODEL: Final = "gemini-2.5-flash"
CHAT_MODEL: Final = "gemini-3.8-flash"
IMAGE_BACKEND: Final = "openai/gpt-image-2.5-flare"
SOURCE_PNG: Final = base64.b64decode(
"iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAS0lEQVR42u3PMQ0AAAwDoPo3"

2
tests/e2e/guardrails/test_key_guardrail_video_e2e.py
View file

@ -10,7 +10,7 @@ from models import LiteLLMParamsBody
pytestmark = pytest.mark.e2e
CHAT_MODEL = "gemini-2.5-flash"
CHAT_MODEL = "gemini-3.8-flash"
VIDEO_BACKEND = "vertex_ai/veo-3.1-fast-generate-001"

2
tests/e2e/guardrails/test_openai_moderation_category_matrix_e2e.py
View file

@ -19,7 +19,7 @@ from models import AnthropicMessagesResponse, ChatResponse
pytestmark = pytest.mark.e2e
GEMINI_BACKEND: Final = "gemini/gemini-2.5-flash"
GEMINI_BACKEND: Final = "gemini/gemini-3.8-flash"
ANTHROPIC_BACKEND: Final = "anthropic/claude-haiku-4-5"
OPENAI_BACKEND: Final = "openai/gpt-4o-mini"

6
tests/e2e/guardrails/test_presidio_masking_e2e.py
View file

@ -53,7 +53,7 @@ from models import (
pytestmark = pytest.mark.e2e
MODEL = "gemini-2.5-flash"
MODEL = "gemini-3.8-flash"
# A guardrail created via POST /guardrails reaches the worker that served the
# create immediately, but every other worker only picks it up on its next
@ -268,7 +268,7 @@ class TestPresidioPreCallMasking:
prompt = _pii_prompt(unique_marker(), email)
_assert_eventually_masked(
lambda: client.chat(scoped_key, MODEL, prompt, guardrails=[name], max_tokens=128),
lambda: client.chat(scoped_key, MODEL, prompt, guardrails=[name], max_tokens=512),
_first_content,
email=email,
)
@ -296,7 +296,7 @@ class TestPresidioPreCallMasking:
prompt = _pii_prompt(unique_marker(), email)
_assert_eventually_masked(
lambda: client.messages(scoped_key, MODEL, prompt, guardrails=[name], max_tokens=128),
lambda: client.messages(scoped_key, MODEL, prompt, guardrails=[name], max_tokens=512),
_messages_text,
email=email,
)

2
tests/e2e/guardrails/test_streaming_guardrail_e2e.py
View file

@ -32,7 +32,7 @@ from lifecycle import ResourceManager
pytestmark = pytest.mark.e2e
MODEL = "gemini-2.5-flash"
MODEL = "gemini-3.8-flash"
class TestBedrockDuringCallStreaming:

2
tests/e2e/guardrails/test_team_disable_global_guardrail_e2e.py
View file

@ -20,7 +20,7 @@ from lifecycle import ResourceManager
pytestmark = pytest.mark.e2e
MODEL = "gemini-2.5-flash"
MODEL = "gemini-3.8-flash"
# A guardrail created via POST /guardrails is registered in-process immediately
# on the worker that served the create call, but the proxy runs multiple

2
tests/e2e/guardrails/test_tool_permission_guardrail_e2e.py
View file

@ -38,7 +38,7 @@ from models import ChatResponse, ChatTool, ChatToolFunction
pytestmark = pytest.mark.e2e
MODEL = "gemini-2.5-flash"
MODEL = "gemini-3.8-flash"
#: The one tool the guardrail permits, and one it does not. Both are declared by
#: the caller in the request body; the guardrail reads them there.

6
tests/e2e/llm_translation/test_cache_control.py
View file

@ -8,7 +8,7 @@ Each case asserts the feature actually happened, not just a 200. Coverage matrix
cache-read usage tokens > 0. service_tier is out of scope for Bedrock; AWS
Bedrock does not expose an OpenAI-style request service tier, so that cell is
intentionally not covered here.
- Vertex (gemini-2.5-flash): explicit context caching via ``cache_control``
- Vertex (gemini-3.8-flash): explicit context caching via ``cache_control``
with a 5-minute ttl. litellm builds the Vertex cache before the generate
call, so a never-seen prefix must come back cached on its very first call
(Gemini's implicit caching cannot hit a cold prefix). That call bills the
@ -80,7 +80,7 @@ import os
pytestmark = [pytest.mark.e2e, pytest.mark.provider_live]
BEDROCK_MODEL = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
VERTEX_MODEL = "vertex_ai/gemini-2.5-flash"
VERTEX_MODEL = "vertex_ai/gemini-3.8-flash"
ANTHROPIC_MODEL = "anthropic/claude-haiku-4-5-20251001"
OPENAI_MODEL = "openai/gpt-5.6"
VERTEX_CACHE_TTL: Final = "300s"
@ -380,7 +380,7 @@ class TestCacheControl:
LiteLLMParamsBody(
model=VERTEX_MODEL,
vertex_project=os.environ.get("VERTEXAI_PROJECT"),
vertex_location="us-central1",
vertex_location="global",
vertex_credentials=os.environ.get("VERTEXAI_CREDENTIALS"),
),
)

16
tests/e2e/llm_translation/test_chat_completions_regression_e2e.py
View file

@ -45,8 +45,8 @@ from passthrough_client import PassthroughClient
pytestmark = pytest.mark.e2e
COHERE_BACKEND = "cohere/command-r-08-2024"
GEMINI_BACKEND = "gemini/gemini-2.5-flash"
VERTEX_BACKEND: Final = "vertex_ai/gemini-2.5-flash"
GEMINI_BACKEND = "gemini/gemini-3.8-flash"
VERTEX_BACKEND: Final = "vertex_ai/gemini-3.8-flash"
AZURE_OPENAI_BACKEND: Final = "azure/gpt-5.4-nano"
AZURE_OPENAI_API_VERSION: Final = "v1"
AZURE_FOUNDRY_BACKEND: Final = "azure_ai/claude-haiku-4-5"
@ -243,7 +243,7 @@ _PERSON_SCHEMA: dict[str, object] = {
OPENAI_CHAT_MODEL: Final = "gpt-5.5"
ANTHROPIC_CHAT_MODEL: Final = "claude-haiku-4-5"
GEMINI_FLASH_MODEL: Final = "gemini-2.5-flash"
GEMINI_FLASH_MODEL: Final = "gemini-3.8-flash"
CHAT_MODELS: Final = (
pytest.param(
@ -449,7 +449,7 @@ class TestVertexChatCompletions:
LiteLLMParamsBody(
model=VERTEX_BACKEND,
vertex_project="os.environ/VERTEXAI_PROJECT",
vertex_location="us-central1",
vertex_location="global",
),
)
resources.defer(lambda: client.proxy.delete_model(model_id))
@ -485,7 +485,8 @@ class TestVertexChatCompletions:
content=f"Reply with the single word pong. {unique_marker()}",
)
],
max_tokens=32,
max_tokens=128,
reasoning_effort="none",
),
)
)
@ -554,7 +555,7 @@ class TestVertexChatCompletions:
response = unwrap(
client.proxy.chat(
key, ChatBody(model=model, messages=_vision_messages(), max_tokens=32, reasoning_effort="none")
key, ChatBody(model=model, messages=_vision_messages(), max_tokens=256, reasoning_effort="none")
)
)
_assert_describes_cat(response)
@ -588,7 +589,8 @@ class TestVertexChatCompletions:
content=f"Count from 1 to 5, one number per line. {unique_marker()}",
)
],
max_tokens=64,
max_tokens=128,
reasoning_effort="none",
stream=True,
),
)

14
tests/e2e/llm_translation/test_custom_pricing_e2e.py
View file

@ -5,7 +5,7 @@ teardown) instead of relying on a statically configured model, so the check is
self-contained and never inherits pricing another suite or a stale config left on
the shared proxy. custom-priced-flash sets input/output rates deliberately far
above the canonical gemini price; the isolation sibling shares the same
gemini/gemini-2.5-flash backend but sets no override. Three behaviors are checked
gemini/gemini-3.8-flash backend but sets no override. Three behaviors are checked
independently:
- billing: a real call's logged cost breakdown charges input and output tokens at
@ -37,12 +37,12 @@ from models import (
pytestmark = pytest.mark.e2e
BACKEND_MODEL = "gemini/gemini-2.5-flash"
BACKEND_MODEL = "gemini/gemini-3.8-flash"
GEMINI_API_KEY = "os.environ/GEMINI_API_KEY"
# Deliberately ~100x above canonical gemini-2.5-flash (input 3e-7 / output 2.5e-6)
# Deliberately ~100x above canonical gemini-3.8-flash (input 7.5e-7 / output 3.75e-6)
# so an override that is ignored or under-applied bills at the base rate and fails.
CUSTOM_INPUT_RATE = 5e-05
CUSTOM_OUTPUT_RATE = 1e-04
CUSTOM_INPUT_RATE = 7.5e-05
CUSTOM_OUTPUT_RATE = 3.75e-04
class _CostBreakdown(BaseModel):
@ -78,7 +78,7 @@ def _provision(
input_cost_per_token: float | None,
output_cost_per_token: float | None,
) -> str:
"""Register a fresh gemini/gemini-2.5-flash deployment (deleted on teardown) and
"""Register a fresh gemini/gemini-3.8-flash deployment (deleted on teardown) and
return its model name. With the cost fields set the deployment carries a custom
pricing override; with them None it is a plain sibling on the same backend. The
marker keeps the name unique so concurrent runs on the shared proxy never
@ -251,7 +251,7 @@ class TestCustomPricing:
assert sibling_entry is not None, f"{sibling} absent from /model/info"
# custom-priced-flash overrides pricing; the sibling shares the same
# gemini/gemini-2.5-flash backend but sets no override, so it must keep its
# gemini/gemini-3.8-flash backend but sets no override, so it must keep its
# own price. Equal rates mean the override leaked into the shared cost map.
assert (
sibling_entry.model_info.input_cost_per_token

2
tests/e2e/llm_translation/test_google_native_e2e.py
View file

@ -21,7 +21,7 @@ from pydantic import BaseModel
pytestmark = pytest.mark.e2e
UPSTREAM_MODEL = "gemini/gemini-2.5-flash"
UPSTREAM_MODEL = "gemini/gemini-3.8-flash"
class _GenerateContentPart(BaseModel):

6
tests/e2e/llm_translation/test_passthrough_e2e.py
View file

@ -5,7 +5,7 @@ and verifies the proxy still logged a costed SpendLogs row
(call_type="pass_through_endpoint"), correlated by the id the caller was served:
the x-litellm-call-id header on gemini, the `msg_...` message id on anthropic.
Covered: gemini ("gemini-2.5-flash") + anthropic ("claude-haiku-4-5"), streaming +
Covered: gemini ("gemini-3.8-flash") + anthropic ("claude-haiku-4-5"), streaming +
non-streaming, plus native tool calls. See LLM_TRANSLATION_COVERAGE_MATRIX.md.
A passthrough call returning non-2xx fails hard (never a skip); once it returns
@ -32,7 +32,7 @@ from passthrough_client import (
completed_responses_object,
)
GEMINI_MODEL: Final = "gemini-2.5-flash"
GEMINI_MODEL: Final = "gemini-3.8-flash"
ANTHROPIC_PASSTHROUGH_MODEL: Final = "claude-haiku-4-5"
EMBEDDING_MODEL = "text-embedding-3-small"
REALTIME_MODEL = "gpt-realtime-2"
@ -291,7 +291,7 @@ class TestPassthroughModelAllowlist:
result = client.anthropic_message(key, ANTHROPIC_PASSTHROUGH_MODEL, f"say hi {unique_marker()}")
assert result.status_code == 403, (
"a key restricted to gemini-2.5-flash must be denied a claude passthrough call, "
"a key restricted to gemini-3.8-flash must be denied a claude passthrough call, "
f"got {result.status_code}: {result.body[:300]}"
)

6
tests/e2e/llm_translation/test_responses_e2e.py
View file

@ -56,8 +56,8 @@ OPENAI_MINI_BACKEND: Final = "openai/gpt-4o-mini"
OPENAI_VISION_BACKEND: Final = "openai/gpt-4o"
ANTHROPIC_BACKEND: Final = "anthropic/claude-haiku-4-5"
BEDROCK_CONVERSE_BACKEND: Final = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
VERTEX_BACKEND: Final = "vertex_ai/gemini-2.5-flash"
GEMINI_BACKEND: Final = "gemini/gemini-2.5-flash"
VERTEX_BACKEND: Final = "vertex_ai/gemini-3.8-flash"
GEMINI_BACKEND: Final = "gemini/gemini-3.8-flash"
OPENAI_RESPONSES_BACKEND: Final = "openai/gpt-5.5"
INSTRUCTIONS = "You are a helpful assistant"
CAT_IMAGE_URL = "https://upload.wikimedia.org/wikipedia/commons/3/3a/Cat03.jpg"
@ -128,7 +128,7 @@ def _vertex_params() -> LiteLLMParamsBody:
return LiteLLMParamsBody(
model=VERTEX_BACKEND,
vertex_project="os.environ/VERTEXAI_PROJECT",
vertex_location="us-central1",
vertex_location="global",
)

4
tests/e2e/llm_translation/test_token_counter_gemini_contents_e2e.py
View file

@ -20,8 +20,8 @@ from pydantic import BaseModel
pytestmark = pytest.mark.e2e
GEMINI_STUDIO_DEPLOYMENT: Final = "gemini-2.5-flash"
GEMINI_VERTEX_DEPLOYMENT: Final = "gemini-2.5-flash-vertex"
GEMINI_STUDIO_DEPLOYMENT: Final = "gemini-3.8-flash"
GEMINI_VERTEX_DEPLOYMENT: Final = "gemini-3.8-flash-vertex"
GEMINI_DEPLOYMENTS = (
pytest.param(
GEMINI_STUDIO_DEPLOYMENT,

5
tests/e2e/llm_translation/test_vertex_passthrough_e2e.py
View file

@ -24,6 +24,7 @@ model, spend > 0), correlated by the x-litellm-call-id header.
import os
import time
from typing import Final
import pytest
from pydantic import BaseModel
@ -37,10 +38,10 @@ from passthrough_client import PassthroughClient
pytestmark = pytest.mark.e2e
VERTEX_MODEL = "gemini-2.5-flash"
VERTEX_MODEL = "gemini-3.8-flash"
# The added deployment's region and the passthrough URL's region are the same constant,
# so they always agree; the proxy registers passthrough credentials per project+region.
VERTEX_LOCATION = os.environ.get("VERTEXAI_LOCATION", "us-central1")
VERTEX_LOCATION: Final = "global"
@pytest.fixture(scope="session")

2
tests/e2e/logging/test_prometheus_cardinality_e2e.py
View file

@ -30,7 +30,7 @@ from logging_client import LoggingClient
pytestmark = pytest.mark.e2e
DRIVER_MODEL = "gemini-2.5-flash"
DRIVER_MODEL = "gemini-3.8-flash"
REQUESTS_METRIC = "litellm_requests_metric_total"
ALIAS_LABEL = "api_key_alias"
DISTINCT_KEYS = 3

2
tests/e2e/logging/test_prometheus_queue_time_e2e.py
View file

@ -12,7 +12,7 @@ from logging_client import LoggingClient
pytestmark = pytest.mark.e2e
DRIVER_MODEL = "gemini-2.5-flash"
DRIVER_MODEL = "gemini-3.8-flash"
QUEUE_TIME_METRIC = "litellm_request_queue_time_seconds"
ALIAS_LABEL = "api_key_alias"

8
tests/e2e/management/test_key_management_e2e.py
View file

@ -242,7 +242,7 @@ class TestKeyManagementRoutes:
client,
resources,
KeyGenerateBody(
models=["gpt-5.5", "gemini-2.5-flash"],
models=["gpt-5.5", "gemini-3.8-flash"],
key_alias=alias,
tpm_limit=131313,
rpm_limit=141414,
@ -251,8 +251,8 @@ class TestKeyManagementRoutes:
info = client.proxy.key_info(key)
assert info.key_alias == alias, f"/key/info reports key_alias {info.key_alias!r}, configured {alias!r}"
assert info.models == ["gpt-5.5", "gemini-2.5-flash"], (
f"/key/info reports models {info.models}, configured ['gpt-5.5', 'gemini-2.5-flash']"
assert info.models == ["gpt-5.5", "gemini-3.8-flash"], (
f"/key/info reports models {info.models}, configured ['gpt-5.5', 'gemini-3.8-flash']"
)
assert info.tpm_limit == 131313, f"/key/info reports tpm_limit {info.tpm_limit}, configured 131313"
assert info.rpm_limit == 141414, f"/key/info reports rpm_limit {info.rpm_limit}, configured 141414"
@ -410,7 +410,7 @@ class TestKeyManagementRoutes:
outcome = client.proxy.transport.send(
"/key/update",
headers=client.proxy.transport.bearer(nonadmin),
json=KeyUpdateBody(key=target, models=["gemini-2.5-flash"]),
json=KeyUpdateBody(key=target, models=["gemini-3.8-flash"]),
)
assert outcome.status_code in (401, 403), (
f"non-admin key POSTing /key/update must be denied 401/403, got {outcome.status_code}: {outcome.body[:300]}"

8
tests/e2e/management/test_management_e2e.py
View file

@ -48,7 +48,7 @@ from proxy_client import Converged, await_converged
pytestmark = pytest.mark.e2e
GEMINI_MODEL: Final = "gemini-2.5-flash"
GEMINI_MODEL: Final = "gemini-3.8-flash"
OPENAI_MODEL: Final = "gpt-5.5"
REGENERATE_GRACE_PERIOD = "15s"
REGENERATE_GRACE_SECONDS = 15.0
@ -155,7 +155,7 @@ class TestKeyRoutes:
info = client.proxy.key_info(key)
assert info.key_alias == alias, f"/key/info reports key_alias {info.key_alias!r}, configured {alias!r}"
assert info.models == [GEMINI_MODEL], (
f"/key/info reports models {info.models}, configured ['gemini-2.5-flash']"
f"/key/info reports models {info.models}, configured ['gemini-3.8-flash']"
)
assert info.tpm_limit == 424242, (
f"/key/info reports tpm_limit {info.tpm_limit}, configured 424242"
@ -501,7 +501,7 @@ class TestTeamRoutes:
info = client.team_info(team_id)
assert info.team_alias == alias, f"/team/info reports team_alias {info.team_alias!r}, configured {alias!r}"
assert info.models == [GEMINI_MODEL], (
f"/team/info reports models {info.models}, configured ['gemini-2.5-flash']"
f"/team/info reports models {info.models}, configured ['gemini-3.8-flash']"
)
key = _generate_key(client, resources, KeyGenerateBody(team_id=team_id))
@ -775,7 +775,7 @@ class TestOrganizationRoutes:
f"/organization/info reports alias {info.organization_alias!r}, configured {alias!r}"
)
assert info.models == [GEMINI_MODEL], (
f"/organization/info reports models {info.models}, configured ['gemini-2.5-flash']"
f"/organization/info reports models {info.models}, configured ['gemini-3.8-flash']"
)
@pytest.mark.covers("mgmt.organization.update.persists")

16
tests/e2e/management/test_team_management_e2e.py
View file

@ -248,15 +248,15 @@ class TestTeamManagementRoutes:
self, client: ManagementClient, resources: ResourceManager
) -> None:
alias = f"e2e-team-info-{unique_marker()}"
team_id = _create_team(client, resources, alias, ["gemini-2.5-flash"])
team_id = _create_team(client, resources, alias, ["gemini-3.8-flash"])
info = _read_team(client, team_id)
assert info.team_id == team_id, f"/team/info echoed team_id {info.team_id!r}, requested {team_id!r}"
assert info.team_info.team_alias == alias, (
f"/team/info reports team_alias {info.team_info.team_alias!r}, configured {alias!r}"
)
assert info.team_info.models == ["gemini-2.5-flash"], (
f"/team/info reports models {info.team_info.models}, configured ['gemini-2.5-flash']"
assert info.team_info.models == ["gemini-3.8-flash"], (
f"/team/info reports models {info.team_info.models}, configured ['gemini-3.8-flash']"
)
@pytest.mark.covers("mgmt.team.block.persists")
@ -264,7 +264,7 @@ class TestTeamManagementRoutes:
def test_block_then_unblock_persists_to_team_info(
self, client: ManagementClient, resources: ResourceManager
) -> None:
team_id = _create_team(client, resources, f"e2e-team-block-{unique_marker()}", ["gemini-2.5-flash"])
team_id = _create_team(client, resources, f"e2e-team-block-{unique_marker()}", ["gemini-3.8-flash"])
assert not _read_team(client, team_id).team_info.blocked, "/team/info reports the team blocked before /team/block"
_set_blocked(client, team_id, blocked=True)
@ -287,7 +287,7 @@ class TestTeamManagementRoutes:
self, client: ManagementClient, resources: ResourceManager
) -> None:
user_id = _create_user(client, resources, f"e2e-team-mu-{unique_marker()}@example.com")
team_id = _create_team(client, resources, f"e2e-team-mu-{unique_marker()}", ["gemini-2.5-flash"])
team_id = _create_team(client, resources, f"e2e-team-mu-{unique_marker()}", ["gemini-3.8-flash"])
client.add_team_member(team_id, user_id)
assert _member_role(_read_team(client, team_id), user_id) == "user", (
f"member {user_id} should start as role 'user' after /team/member_add"
@ -312,7 +312,7 @@ class TestTeamManagementRoutes:
self, client: ManagementClient, resources: ResourceManager
) -> None:
user_id = _create_user(client, resources, f"e2e-team-md-{unique_marker()}@example.com")
team_id = _create_team(client, resources, f"e2e-team-md-{unique_marker()}", ["gemini-2.5-flash"])
team_id = _create_team(client, resources, f"e2e-team-md-{unique_marker()}", ["gemini-3.8-flash"])
client.add_team_member(team_id, user_id)
assert _member_role(_read_team(client, team_id), user_id) == "user", (
f"/team/info does not list {user_id} as a member after /team/member_add"
@ -378,7 +378,7 @@ class TestTeamManagementRoutes:
user + team, plus a second user id the member could try to add."""
member_id = _create_user(client, resources, f"e2e-team-fb-{unique_marker()}@example.com")
other_id = _create_user(client, resources, f"e2e-team-fb-{unique_marker()}@example.com")
team_id = _create_team(client, resources, f"e2e-team-fb-{unique_marker()}", ["gemini-2.5-flash"])
team_id = _create_team(client, resources, f"e2e-team-fb-{unique_marker()}", ["gemini-3.8-flash"])
client.add_team_member(team_id, member_id)
member_key = _generate_key(client, resources, KeyGenerateBody(user_id=member_id, team_id=team_id))
return member_id, other_id, member_key, team_id
@ -534,7 +534,7 @@ class TestTeamAdminWithTpmLimitEnabled:
pytest.param(TeamSettingsChange(rpm_limit=10), id="rpm_limit"),
pytest.param(TeamSettingsChange(max_budget=0.5), id="max_budget"),
pytest.param(TeamSettingsChange(team_alias="renamed-by-team-admin"), id="team_alias"),
pytest.param(TeamSettingsChange(models=["gemini-2.5-flash"]), id="models"),
pytest.param(TeamSettingsChange(models=["gemini-3.8-flash"]), id="models"),
pytest.param(TeamSettingsChange(budget_duration="1d"), id="budget_duration"),
pytest.param(TeamSettingsChange(metadata=TeamCustomMetadata(cost_center="team-admin")), id="metadata"),
],

2
tests/e2e/quota_management/budgets/test_model_max_budget_e2e.py
View file

@ -20,7 +20,7 @@ from models import ModelBudgetEntry
pytestmark = pytest.mark.e2e
CAPPED_MODEL = "claude-haiku-4-5"
FREE_MODEL = "gemini-2.5-flash"
FREE_MODEL = "gemini-3.8-flash"
def _call(client: BudgetClient, key: str, model: str):

2
tests/e2e/quota_management/spend_tracking/conftest.py
View file

@ -32,7 +32,7 @@ def _driver_params(provider_model: str, env_var: str) -> LiteLLMParamsBody:
DRIVER_MODELS: tuple[tuple[str, str, str], ...] = (
("gemini-2.5-flash", "gemini/gemini-2.5-flash", "GEMINI_API_KEY"),
("gemini-3.8-flash", "gemini/gemini-3.8-flash", "GEMINI_API_KEY"),
("claude-haiku-4-5", "anthropic/claude-haiku-4-5", "ANTHROPIC_API_KEY"),
("openai-text-embedding-3-small", "openai/text-embedding-3-small", "OPENAI_API_KEY"),
("openai-responses-codex", "openai/gpt-5.3-codex", "OPENAI_API_KEY"),

2
tests/e2e/quota_management/spend_tracking/test_key_attribution_e2e.py
View file

@ -55,7 +55,7 @@ from spend_e2e_client import (
pytestmark = pytest.mark.e2e
CHAT_MODEL: Final = "gemini-2.5-flash"
CHAT_MODEL: Final = "gemini-3.8-flash"
MESSAGES_MODEL: Final = "claude-haiku-4-5"
RESPONSES_MODEL: Final = "openai-responses-codex"
EMBED_MODEL: Final = "openai-text-embedding-3-small"

6
tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py
View file

@ -3,7 +3,7 @@
Run against a proxy started with the gateway config. Coverage rationale:
SPEND_TRACKING_COVERAGE_MATRIX.md.
Model names are literals from that config: chat tests hit "gemini-2.5-flash",
Model names are literals from that config: chat tests hit "gemini-3.8-flash",
embedding tests hit "openai-text-embedding-3-small".
Every test: fresh scoped key (isolation) -> real provider call -> unwrap (hard
@ -37,7 +37,7 @@ from spend_reconciliation import BACKEND as TRAFFIC_BACKEND
pytestmark = pytest.mark.e2e
GEMINI_MODEL = "gemini-2.5-flash"
GEMINI_MODEL = "gemini-3.8-flash"
CLAUDE_MODEL = "claude-haiku-4-5"
CODEX_MODEL = "openai-responses-codex"
EMBEDDING_MODEL = "openai-text-embedding-3-small"
@ -780,7 +780,7 @@ def test_spend_calculate_returns_nonzero_cost(client: SpendClient) -> None:
GEMINI_MODEL, "estimate the cost of this request"
)
assert cost > 0, (
"/spend/calculate returned 0 for gemini-2.5-flash; "
"/spend/calculate returned 0 for gemini-3.8-flash; "
"cost map may be missing this model"
)