diff --git a/tests/e2e/AGENTS.md b/tests/e2e/AGENTS.md index cf1e41a59a6..dee7e88443f 100644 --- a/tests/e2e/AGENTS.md +++ b/tests/e2e/AGENTS.md @@ -75,7 +75,7 @@ class TestPromptCompression: def test_prompt_compression_accumulate_spend(self, key_id, user_id): for _ in range(10): - response = self.resources.proxy.post("gemini-2.5-flash", key_id, user_id) + response = self.resources.proxy.post("gemini-3.8-flash", key_id, user_id) compressed_value = ... assert response.cost == compressed_value # the cost was actually reduced ``` diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index b4379425854..53613313def 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -9,7 +9,7 @@ When contributing to this directory, please first discuss the change you wish to ## Setup -The suites run against a live proxy, so bring one up first by running the litellm proxy locally. Point it at a config that prewires the example models the suites use (`gpt-5.5`, `claude-haiku-4-5`, `gemini-2.5-flash`, `openai-text-embedding-3-small`) with keys from your `.env`, and enables prompt storage, a redis cache, the fast budget rescheduler the quota suites rely on, and `router_settings.optional_pre_call_checks: ["prompt_caching"]`, which the router suite's prompt-cache affinity test reads back from `GET /router/settings` and fails without. If your test needs another model, a pricing override, or a guardrail declared up front, add it to that config and read it back in the test rather than hardcoding values +The suites run against a live proxy, so bring one up first by running the litellm proxy locally. Point it at a config that prewires the example models the suites use (`gpt-5.5`, `claude-haiku-4-5`, `gemini-3.8-flash`, `openai-text-embedding-3-small`) with keys from your `.env`, and enables prompt storage, a redis cache, the fast budget rescheduler the quota suites rely on, and `router_settings.optional_pre_call_checks: ["prompt_caching"]`, which the router suite's prompt-cache affinity test reads back from `GET /router/settings` and fails without. If your test needs another model, a pricing override, or a guardrail declared up front, add it to that config and read it back in the test rather than hardcoding values ## Running the tests locally @@ -206,7 +206,7 @@ class TestPromptCompression: def test_prompt_compression_accumulate_spend(self, key_id, user_id): for _ in range(10): - response = self.resources.proxy.post("gemini-2.5-flash", key_id, user_id) + response = self.resources.proxy.post("gemini-3.8-flash", key_id, user_id) compressed_value = ... assert response.cost == compressed_value # the cost was actually reduced ``` diff --git a/tests/e2e/access_control/test_access_control_e2e.py b/tests/e2e/access_control/test_access_control_e2e.py index 419cae78df2..b1965748197 100644 --- a/tests/e2e/access_control/test_access_control_e2e.py +++ b/tests/e2e/access_control/test_access_control_e2e.py @@ -30,7 +30,7 @@ from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta pytestmark = pytest.mark.e2e -ALLOWED_MODEL = "gemini-2.5-flash" +ALLOWED_MODEL = "gemini-3.8-flash" DISALLOWED_MODEL = "gpt-5.5" VIRTUAL_KEY_BACKEND = "anthropic/claude-haiku-4-5-20251001" EMBEDDING_MODEL = "openai-text-embedding-3-small" diff --git a/tests/e2e/access_control/test_project_all_team_models_e2e.py b/tests/e2e/access_control/test_project_all_team_models_e2e.py index d70e3de554a..410578ece08 100644 --- a/tests/e2e/access_control/test_project_all_team_models_e2e.py +++ b/tests/e2e/access_control/test_project_all_team_models_e2e.py @@ -28,7 +28,7 @@ from models import ChatResponse, TeamInfoResponse pytestmark = pytest.mark.e2e -TEAM_MODEL = "gemini-2.5-flash" +TEAM_MODEL = "gemini-3.8-flash" OUTSIDE_MODEL = "gpt-5.5" diff --git a/tests/e2e/batches/capabilities.py b/tests/e2e/batches/capabilities.py index c03b481f060..90d80bc1bb6 100644 --- a/tests/e2e/batches/capabilities.py +++ b/tests/e2e/batches/capabilities.py @@ -23,7 +23,7 @@ def batch_model_name(base: str) -> str: OPENAI_BATCH_BACKEND: Final = "gpt-4o-mini" AZURE_BATCH_BACKEND: Final = "gpt-5.4-mini-batch" -VERTEX_BATCH_BACKEND: Final = "gemini-2.5-flash" +VERTEX_BATCH_BACKEND: Final = "gemini-3.8-flash" BEDROCK_BATCH_BACKEND: Final = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" @@ -81,7 +81,7 @@ class Provider: return LiteLLMParamsBody( model=f"vertex_ai/{VERTEX_BATCH_BACKEND}", vertex_project="os.environ/VERTEXAI_PROJECT", - vertex_location="us-central1", + vertex_location="global", vertex_credentials="os.environ/VERTEXAI_CREDENTIALS", gcs_bucket_name="os.environ/GCS_BUCKET_NAME", bucket_name="os.environ/GCS_BUCKET_NAME", diff --git a/tests/e2e/batches/test_batches_e2e.py b/tests/e2e/batches/test_batches_e2e.py index b7a52da5902..8d76e5d9693 100644 --- a/tests/e2e/batches/test_batches_e2e.py +++ b/tests/e2e/batches/test_batches_e2e.py @@ -1286,7 +1286,7 @@ class TestBedrockBatchGovCloud: assert fetched.id == batch.id -GEMINI_FILES_RAW_MODEL = "gemini-2.5-flash" +GEMINI_FILES_RAW_MODEL = "gemini-3.8-flash" class TestGeminiFiles: diff --git a/tests/e2e/gateway/stage_mirror_ci_config.yml b/tests/e2e/gateway/stage_mirror_ci_config.yml index 02a131f60ae..983214e8e45 100644 --- a/tests/e2e/gateway/stage_mirror_ci_config.yml +++ b/tests/e2e/gateway/stage_mirror_ci_config.yml @@ -55,15 +55,15 @@ model_list: litellm_params: model: anthropic/claude-haiku-4-5 api_key: os.environ/ANTHROPIC_API_KEY - - model_name: gemini-2.5-flash-vertex + - model_name: gemini-3.8-flash-vertex litellm_params: - model: vertex_ai/gemini-2.5-flash + model: vertex_ai/gemini-3.8-flash vertex_project: os.environ/VERTEXAI_PROJECT - vertex_location: us-central1 + vertex_location: global vertex_credentials: os.environ/VERTEXAI_CREDENTIALS - - model_name: gemini-2.5-flash + - model_name: gemini-3.8-flash litellm_params: - model: gemini/gemini-2.5-flash + model: gemini/gemini-3.8-flash api_key: os.environ/GEMINI_API_KEY - model_name: e2e-alias-rl-target litellm_params: diff --git a/tests/e2e/guardrails/guardrails_client.py b/tests/e2e/guardrails/guardrails_client.py index 08124a03313..a06d86798f8 100644 --- a/tests/e2e/guardrails/guardrails_client.py +++ b/tests/e2e/guardrails/guardrails_client.py @@ -38,7 +38,7 @@ from models import ( from proxy_client import ProxyClient from pydantic import BaseModel, Field -GUARDRAIL_BACKEND: Final = "gemini/gemini-2.5-flash" +GUARDRAIL_BACKEND: Final = "gemini/gemini-3.8-flash" GuardrailMode = Literal["pre_call", "post_call", "during_call", "logging_only"] PiiEntity = Literal["EMAIL_ADDRESS", "PHONE_NUMBER", "PERSON", "CREDIT_CARD", "US_SSN"] diff --git a/tests/e2e/guardrails/test_bedrock_guardrail_e2e.py b/tests/e2e/guardrails/test_bedrock_guardrail_e2e.py index 8f52eb56d5f..bd74a8a3b9d 100644 --- a/tests/e2e/guardrails/test_bedrock_guardrail_e2e.py +++ b/tests/e2e/guardrails/test_bedrock_guardrail_e2e.py @@ -48,7 +48,7 @@ def _without_assessments(value: JsonValue) -> JsonValue: return value -MODEL = "gemini-2.5-flash" +MODEL = "gemini-3.8-flash" # Matches the word/topic policy the guardrail this suite points at actually denies. # Content filters are not assumed: the guardrail resource carries no contentPolicy, # so a hate/violence prompt returns action=NONE and never blocks. diff --git a/tests/e2e/guardrails/test_guardrail_dispatch_e2e.py b/tests/e2e/guardrails/test_guardrail_dispatch_e2e.py index c8d295ee459..1ce41dd8ff7 100644 --- a/tests/e2e/guardrails/test_guardrail_dispatch_e2e.py +++ b/tests/e2e/guardrails/test_guardrail_dispatch_e2e.py @@ -16,7 +16,7 @@ from guardrails_client import GuardrailsClient pytestmark = pytest.mark.e2e -MODEL = "gemini-2.5-flash" +MODEL = "gemini-3.8-flash" @pytest.mark.skip( diff --git a/tests/e2e/guardrails/test_key_guardrail_image_edit_e2e.py b/tests/e2e/guardrails/test_key_guardrail_image_edit_e2e.py index 153d5fd0aa7..f26b0adeb54 100644 --- a/tests/e2e/guardrails/test_key_guardrail_image_edit_e2e.py +++ b/tests/e2e/guardrails/test_key_guardrail_image_edit_e2e.py @@ -13,7 +13,7 @@ from models import LiteLLMParamsBody pytestmark = pytest.mark.e2e -CHAT_MODEL: Final = "gemini-2.5-flash" +CHAT_MODEL: Final = "gemini-3.8-flash" IMAGE_BACKEND: Final = "openai/gpt-image-2.5-flare" SOURCE_PNG: Final = base64.b64decode( "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAS0lEQVR42u3PMQ0AAAwDoPo3" diff --git a/tests/e2e/guardrails/test_key_guardrail_video_e2e.py b/tests/e2e/guardrails/test_key_guardrail_video_e2e.py index 233cbfb8881..4a8affb5259 100644 --- a/tests/e2e/guardrails/test_key_guardrail_video_e2e.py +++ b/tests/e2e/guardrails/test_key_guardrail_video_e2e.py @@ -10,7 +10,7 @@ from models import LiteLLMParamsBody pytestmark = pytest.mark.e2e -CHAT_MODEL = "gemini-2.5-flash" +CHAT_MODEL = "gemini-3.8-flash" VIDEO_BACKEND = "vertex_ai/veo-3.1-fast-generate-001" diff --git a/tests/e2e/guardrails/test_openai_moderation_category_matrix_e2e.py b/tests/e2e/guardrails/test_openai_moderation_category_matrix_e2e.py index 10155cf96a5..df60e44135e 100644 --- a/tests/e2e/guardrails/test_openai_moderation_category_matrix_e2e.py +++ b/tests/e2e/guardrails/test_openai_moderation_category_matrix_e2e.py @@ -19,7 +19,7 @@ from models import AnthropicMessagesResponse, ChatResponse pytestmark = pytest.mark.e2e -GEMINI_BACKEND: Final = "gemini/gemini-2.5-flash" +GEMINI_BACKEND: Final = "gemini/gemini-3.8-flash" ANTHROPIC_BACKEND: Final = "anthropic/claude-haiku-4-5" OPENAI_BACKEND: Final = "openai/gpt-4o-mini" diff --git a/tests/e2e/guardrails/test_presidio_masking_e2e.py b/tests/e2e/guardrails/test_presidio_masking_e2e.py index 9fc2c7d3ea2..ddeff9bc6e2 100644 --- a/tests/e2e/guardrails/test_presidio_masking_e2e.py +++ b/tests/e2e/guardrails/test_presidio_masking_e2e.py @@ -53,7 +53,7 @@ from models import ( pytestmark = pytest.mark.e2e -MODEL = "gemini-2.5-flash" +MODEL = "gemini-3.8-flash" # A guardrail created via POST /guardrails reaches the worker that served the # create immediately, but every other worker only picks it up on its next @@ -268,7 +268,7 @@ class TestPresidioPreCallMasking: prompt = _pii_prompt(unique_marker(), email) _assert_eventually_masked( - lambda: client.chat(scoped_key, MODEL, prompt, guardrails=[name], max_tokens=128), + lambda: client.chat(scoped_key, MODEL, prompt, guardrails=[name], max_tokens=512), _first_content, email=email, ) @@ -296,7 +296,7 @@ class TestPresidioPreCallMasking: prompt = _pii_prompt(unique_marker(), email) _assert_eventually_masked( - lambda: client.messages(scoped_key, MODEL, prompt, guardrails=[name], max_tokens=128), + lambda: client.messages(scoped_key, MODEL, prompt, guardrails=[name], max_tokens=512), _messages_text, email=email, ) diff --git a/tests/e2e/guardrails/test_streaming_guardrail_e2e.py b/tests/e2e/guardrails/test_streaming_guardrail_e2e.py index 324dd4107a4..4e4251b9367 100644 --- a/tests/e2e/guardrails/test_streaming_guardrail_e2e.py +++ b/tests/e2e/guardrails/test_streaming_guardrail_e2e.py @@ -32,7 +32,7 @@ from lifecycle import ResourceManager pytestmark = pytest.mark.e2e -MODEL = "gemini-2.5-flash" +MODEL = "gemini-3.8-flash" class TestBedrockDuringCallStreaming: diff --git a/tests/e2e/guardrails/test_team_disable_global_guardrail_e2e.py b/tests/e2e/guardrails/test_team_disable_global_guardrail_e2e.py index 0cc6106abfd..68b1fa99d95 100644 --- a/tests/e2e/guardrails/test_team_disable_global_guardrail_e2e.py +++ b/tests/e2e/guardrails/test_team_disable_global_guardrail_e2e.py @@ -20,7 +20,7 @@ from lifecycle import ResourceManager pytestmark = pytest.mark.e2e -MODEL = "gemini-2.5-flash" +MODEL = "gemini-3.8-flash" # A guardrail created via POST /guardrails is registered in-process immediately # on the worker that served the create call, but the proxy runs multiple diff --git a/tests/e2e/guardrails/test_tool_permission_guardrail_e2e.py b/tests/e2e/guardrails/test_tool_permission_guardrail_e2e.py index d5a6a9d9866..588248c5aba 100644 --- a/tests/e2e/guardrails/test_tool_permission_guardrail_e2e.py +++ b/tests/e2e/guardrails/test_tool_permission_guardrail_e2e.py @@ -38,7 +38,7 @@ from models import ChatResponse, ChatTool, ChatToolFunction pytestmark = pytest.mark.e2e -MODEL = "gemini-2.5-flash" +MODEL = "gemini-3.8-flash" #: The one tool the guardrail permits, and one it does not. Both are declared by #: the caller in the request body; the guardrail reads them there. diff --git a/tests/e2e/llm_translation/test_cache_control.py b/tests/e2e/llm_translation/test_cache_control.py index 7fd51bd6e8c..8a7a315b70b 100644 --- a/tests/e2e/llm_translation/test_cache_control.py +++ b/tests/e2e/llm_translation/test_cache_control.py @@ -8,7 +8,7 @@ Each case asserts the feature actually happened, not just a 200. Coverage matrix cache-read usage tokens > 0. service_tier is out of scope for Bedrock; AWS Bedrock does not expose an OpenAI-style request service tier, so that cell is intentionally not covered here. -- Vertex (gemini-2.5-flash): explicit context caching via ``cache_control`` +- Vertex (gemini-3.8-flash): explicit context caching via ``cache_control`` with a 5-minute ttl. litellm builds the Vertex cache before the generate call, so a never-seen prefix must come back cached on its very first call (Gemini's implicit caching cannot hit a cold prefix). That call bills the @@ -80,7 +80,7 @@ import os pytestmark = [pytest.mark.e2e, pytest.mark.provider_live] BEDROCK_MODEL = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" -VERTEX_MODEL = "vertex_ai/gemini-2.5-flash" +VERTEX_MODEL = "vertex_ai/gemini-3.8-flash" ANTHROPIC_MODEL = "anthropic/claude-haiku-4-5-20251001" OPENAI_MODEL = "openai/gpt-5.6" VERTEX_CACHE_TTL: Final = "300s" @@ -380,7 +380,7 @@ class TestCacheControl: LiteLLMParamsBody( model=VERTEX_MODEL, vertex_project=os.environ.get("VERTEXAI_PROJECT"), - vertex_location="us-central1", + vertex_location="global", vertex_credentials=os.environ.get("VERTEXAI_CREDENTIALS"), ), ) diff --git a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py index 1e0a8cbb412..542871a461c 100644 --- a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py +++ b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py @@ -45,8 +45,8 @@ from passthrough_client import PassthroughClient pytestmark = pytest.mark.e2e COHERE_BACKEND = "cohere/command-r-08-2024" -GEMINI_BACKEND = "gemini/gemini-2.5-flash" -VERTEX_BACKEND: Final = "vertex_ai/gemini-2.5-flash" +GEMINI_BACKEND = "gemini/gemini-3.8-flash" +VERTEX_BACKEND: Final = "vertex_ai/gemini-3.8-flash" AZURE_OPENAI_BACKEND: Final = "azure/gpt-5.4-nano" AZURE_OPENAI_API_VERSION: Final = "v1" AZURE_FOUNDRY_BACKEND: Final = "azure_ai/claude-haiku-4-5" @@ -243,7 +243,7 @@ _PERSON_SCHEMA: dict[str, object] = { OPENAI_CHAT_MODEL: Final = "gpt-5.5" ANTHROPIC_CHAT_MODEL: Final = "claude-haiku-4-5" -GEMINI_FLASH_MODEL: Final = "gemini-2.5-flash" +GEMINI_FLASH_MODEL: Final = "gemini-3.8-flash" CHAT_MODELS: Final = ( pytest.param( @@ -449,7 +449,7 @@ class TestVertexChatCompletions: LiteLLMParamsBody( model=VERTEX_BACKEND, vertex_project="os.environ/VERTEXAI_PROJECT", - vertex_location="us-central1", + vertex_location="global", ), ) resources.defer(lambda: client.proxy.delete_model(model_id)) @@ -485,7 +485,8 @@ class TestVertexChatCompletions: content=f"Reply with the single word pong. {unique_marker()}", ) ], - max_tokens=32, + max_tokens=128, + reasoning_effort="none", ), ) ) @@ -554,7 +555,7 @@ class TestVertexChatCompletions: response = unwrap( client.proxy.chat( - key, ChatBody(model=model, messages=_vision_messages(), max_tokens=32, reasoning_effort="none") + key, ChatBody(model=model, messages=_vision_messages(), max_tokens=256, reasoning_effort="none") ) ) _assert_describes_cat(response) @@ -588,7 +589,8 @@ class TestVertexChatCompletions: content=f"Count from 1 to 5, one number per line. {unique_marker()}", ) ], - max_tokens=64, + max_tokens=128, + reasoning_effort="none", stream=True, ), ) diff --git a/tests/e2e/llm_translation/test_custom_pricing_e2e.py b/tests/e2e/llm_translation/test_custom_pricing_e2e.py index 6d8cd24648e..5d0cb73a46a 100644 --- a/tests/e2e/llm_translation/test_custom_pricing_e2e.py +++ b/tests/e2e/llm_translation/test_custom_pricing_e2e.py @@ -5,7 +5,7 @@ teardown) instead of relying on a statically configured model, so the check is self-contained and never inherits pricing another suite or a stale config left on the shared proxy. custom-priced-flash sets input/output rates deliberately far above the canonical gemini price; the isolation sibling shares the same -gemini/gemini-2.5-flash backend but sets no override. Three behaviors are checked +gemini/gemini-3.8-flash backend but sets no override. Three behaviors are checked independently: - billing: a real call's logged cost breakdown charges input and output tokens at @@ -37,12 +37,12 @@ from models import ( pytestmark = pytest.mark.e2e -BACKEND_MODEL = "gemini/gemini-2.5-flash" +BACKEND_MODEL = "gemini/gemini-3.8-flash" GEMINI_API_KEY = "os.environ/GEMINI_API_KEY" -# Deliberately ~100x above canonical gemini-2.5-flash (input 3e-7 / output 2.5e-6) +# Deliberately ~100x above canonical gemini-3.8-flash (input 7.5e-7 / output 3.75e-6) # so an override that is ignored or under-applied bills at the base rate and fails. -CUSTOM_INPUT_RATE = 5e-05 -CUSTOM_OUTPUT_RATE = 1e-04 +CUSTOM_INPUT_RATE = 7.5e-05 +CUSTOM_OUTPUT_RATE = 3.75e-04 class _CostBreakdown(BaseModel): @@ -78,7 +78,7 @@ def _provision( input_cost_per_token: float | None, output_cost_per_token: float | None, ) -> str: - """Register a fresh gemini/gemini-2.5-flash deployment (deleted on teardown) and + """Register a fresh gemini/gemini-3.8-flash deployment (deleted on teardown) and return its model name. With the cost fields set the deployment carries a custom pricing override; with them None it is a plain sibling on the same backend. The marker keeps the name unique so concurrent runs on the shared proxy never @@ -251,7 +251,7 @@ class TestCustomPricing: assert sibling_entry is not None, f"{sibling} absent from /model/info" # custom-priced-flash overrides pricing; the sibling shares the same - # gemini/gemini-2.5-flash backend but sets no override, so it must keep its + # gemini/gemini-3.8-flash backend but sets no override, so it must keep its # own price. Equal rates mean the override leaked into the shared cost map. assert ( sibling_entry.model_info.input_cost_per_token diff --git a/tests/e2e/llm_translation/test_google_native_e2e.py b/tests/e2e/llm_translation/test_google_native_e2e.py index b18683bd4ad..65a341dbc2a 100644 --- a/tests/e2e/llm_translation/test_google_native_e2e.py +++ b/tests/e2e/llm_translation/test_google_native_e2e.py @@ -21,7 +21,7 @@ from pydantic import BaseModel pytestmark = pytest.mark.e2e -UPSTREAM_MODEL = "gemini/gemini-2.5-flash" +UPSTREAM_MODEL = "gemini/gemini-3.8-flash" class _GenerateContentPart(BaseModel): diff --git a/tests/e2e/llm_translation/test_passthrough_e2e.py b/tests/e2e/llm_translation/test_passthrough_e2e.py index 52d1c10d829..3eaeef6bddd 100644 --- a/tests/e2e/llm_translation/test_passthrough_e2e.py +++ b/tests/e2e/llm_translation/test_passthrough_e2e.py @@ -5,7 +5,7 @@ and verifies the proxy still logged a costed SpendLogs row (call_type="pass_through_endpoint"), correlated by the id the caller was served: the x-litellm-call-id header on gemini, the `msg_...` message id on anthropic. -Covered: gemini ("gemini-2.5-flash") + anthropic ("claude-haiku-4-5"), streaming + +Covered: gemini ("gemini-3.8-flash") + anthropic ("claude-haiku-4-5"), streaming + non-streaming, plus native tool calls. See LLM_TRANSLATION_COVERAGE_MATRIX.md. A passthrough call returning non-2xx fails hard (never a skip); once it returns @@ -32,7 +32,7 @@ from passthrough_client import ( completed_responses_object, ) -GEMINI_MODEL: Final = "gemini-2.5-flash" +GEMINI_MODEL: Final = "gemini-3.8-flash" ANTHROPIC_PASSTHROUGH_MODEL: Final = "claude-haiku-4-5" EMBEDDING_MODEL = "text-embedding-3-small" REALTIME_MODEL = "gpt-realtime-2" @@ -291,7 +291,7 @@ class TestPassthroughModelAllowlist: result = client.anthropic_message(key, ANTHROPIC_PASSTHROUGH_MODEL, f"say hi {unique_marker()}") assert result.status_code == 403, ( - "a key restricted to gemini-2.5-flash must be denied a claude passthrough call, " + "a key restricted to gemini-3.8-flash must be denied a claude passthrough call, " f"got {result.status_code}: {result.body[:300]}" ) diff --git a/tests/e2e/llm_translation/test_responses_e2e.py b/tests/e2e/llm_translation/test_responses_e2e.py index a0f185c4077..c6cdf8e50de 100644 --- a/tests/e2e/llm_translation/test_responses_e2e.py +++ b/tests/e2e/llm_translation/test_responses_e2e.py @@ -56,8 +56,8 @@ OPENAI_MINI_BACKEND: Final = "openai/gpt-4o-mini" OPENAI_VISION_BACKEND: Final = "openai/gpt-4o" ANTHROPIC_BACKEND: Final = "anthropic/claude-haiku-4-5" BEDROCK_CONVERSE_BACKEND: Final = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" -VERTEX_BACKEND: Final = "vertex_ai/gemini-2.5-flash" -GEMINI_BACKEND: Final = "gemini/gemini-2.5-flash" +VERTEX_BACKEND: Final = "vertex_ai/gemini-3.8-flash" +GEMINI_BACKEND: Final = "gemini/gemini-3.8-flash" OPENAI_RESPONSES_BACKEND: Final = "openai/gpt-5.5" INSTRUCTIONS = "You are a helpful assistant" CAT_IMAGE_URL = "https://upload.wikimedia.org/wikipedia/commons/3/3a/Cat03.jpg" @@ -128,7 +128,7 @@ def _vertex_params() -> LiteLLMParamsBody: return LiteLLMParamsBody( model=VERTEX_BACKEND, vertex_project="os.environ/VERTEXAI_PROJECT", - vertex_location="us-central1", + vertex_location="global", ) diff --git a/tests/e2e/llm_translation/test_token_counter_gemini_contents_e2e.py b/tests/e2e/llm_translation/test_token_counter_gemini_contents_e2e.py index d50a7bae9d1..737c8bf2602 100644 --- a/tests/e2e/llm_translation/test_token_counter_gemini_contents_e2e.py +++ b/tests/e2e/llm_translation/test_token_counter_gemini_contents_e2e.py @@ -20,8 +20,8 @@ from pydantic import BaseModel pytestmark = pytest.mark.e2e -GEMINI_STUDIO_DEPLOYMENT: Final = "gemini-2.5-flash" -GEMINI_VERTEX_DEPLOYMENT: Final = "gemini-2.5-flash-vertex" +GEMINI_STUDIO_DEPLOYMENT: Final = "gemini-3.8-flash" +GEMINI_VERTEX_DEPLOYMENT: Final = "gemini-3.8-flash-vertex" GEMINI_DEPLOYMENTS = ( pytest.param( GEMINI_STUDIO_DEPLOYMENT, diff --git a/tests/e2e/llm_translation/test_vertex_passthrough_e2e.py b/tests/e2e/llm_translation/test_vertex_passthrough_e2e.py index 3daf78a31b2..3a70a858e9b 100644 --- a/tests/e2e/llm_translation/test_vertex_passthrough_e2e.py +++ b/tests/e2e/llm_translation/test_vertex_passthrough_e2e.py @@ -24,6 +24,7 @@ model, spend > 0), correlated by the x-litellm-call-id header. import os import time +from typing import Final import pytest from pydantic import BaseModel @@ -37,10 +38,10 @@ from passthrough_client import PassthroughClient pytestmark = pytest.mark.e2e -VERTEX_MODEL = "gemini-2.5-flash" +VERTEX_MODEL = "gemini-3.8-flash" # The added deployment's region and the passthrough URL's region are the same constant, # so they always agree; the proxy registers passthrough credentials per project+region. -VERTEX_LOCATION = os.environ.get("VERTEXAI_LOCATION", "us-central1") +VERTEX_LOCATION: Final = "global" @pytest.fixture(scope="session") diff --git a/tests/e2e/logging/test_prometheus_cardinality_e2e.py b/tests/e2e/logging/test_prometheus_cardinality_e2e.py index 597e1e814e3..d03268d9a0d 100644 --- a/tests/e2e/logging/test_prometheus_cardinality_e2e.py +++ b/tests/e2e/logging/test_prometheus_cardinality_e2e.py @@ -30,7 +30,7 @@ from logging_client import LoggingClient pytestmark = pytest.mark.e2e -DRIVER_MODEL = "gemini-2.5-flash" +DRIVER_MODEL = "gemini-3.8-flash" REQUESTS_METRIC = "litellm_requests_metric_total" ALIAS_LABEL = "api_key_alias" DISTINCT_KEYS = 3 diff --git a/tests/e2e/logging/test_prometheus_queue_time_e2e.py b/tests/e2e/logging/test_prometheus_queue_time_e2e.py index 9668322b362..f02269d2631 100644 --- a/tests/e2e/logging/test_prometheus_queue_time_e2e.py +++ b/tests/e2e/logging/test_prometheus_queue_time_e2e.py @@ -12,7 +12,7 @@ from logging_client import LoggingClient pytestmark = pytest.mark.e2e -DRIVER_MODEL = "gemini-2.5-flash" +DRIVER_MODEL = "gemini-3.8-flash" QUEUE_TIME_METRIC = "litellm_request_queue_time_seconds" ALIAS_LABEL = "api_key_alias" diff --git a/tests/e2e/management/test_key_management_e2e.py b/tests/e2e/management/test_key_management_e2e.py index 34f912fe604..5ce0d8e49a9 100644 --- a/tests/e2e/management/test_key_management_e2e.py +++ b/tests/e2e/management/test_key_management_e2e.py @@ -242,7 +242,7 @@ class TestKeyManagementRoutes: client, resources, KeyGenerateBody( - models=["gpt-5.5", "gemini-2.5-flash"], + models=["gpt-5.5", "gemini-3.8-flash"], key_alias=alias, tpm_limit=131313, rpm_limit=141414, @@ -251,8 +251,8 @@ class TestKeyManagementRoutes: info = client.proxy.key_info(key) assert info.key_alias == alias, f"/key/info reports key_alias {info.key_alias!r}, configured {alias!r}" - assert info.models == ["gpt-5.5", "gemini-2.5-flash"], ( - f"/key/info reports models {info.models}, configured ['gpt-5.5', 'gemini-2.5-flash']" + assert info.models == ["gpt-5.5", "gemini-3.8-flash"], ( + f"/key/info reports models {info.models}, configured ['gpt-5.5', 'gemini-3.8-flash']" ) assert info.tpm_limit == 131313, f"/key/info reports tpm_limit {info.tpm_limit}, configured 131313" assert info.rpm_limit == 141414, f"/key/info reports rpm_limit {info.rpm_limit}, configured 141414" @@ -410,7 +410,7 @@ class TestKeyManagementRoutes: outcome = client.proxy.transport.send( "/key/update", headers=client.proxy.transport.bearer(nonadmin), - json=KeyUpdateBody(key=target, models=["gemini-2.5-flash"]), + json=KeyUpdateBody(key=target, models=["gemini-3.8-flash"]), ) assert outcome.status_code in (401, 403), ( f"non-admin key POSTing /key/update must be denied 401/403, got {outcome.status_code}: {outcome.body[:300]}" diff --git a/tests/e2e/management/test_management_e2e.py b/tests/e2e/management/test_management_e2e.py index c167a1323cb..7f3ceaeca28 100644 --- a/tests/e2e/management/test_management_e2e.py +++ b/tests/e2e/management/test_management_e2e.py @@ -48,7 +48,7 @@ from proxy_client import Converged, await_converged pytestmark = pytest.mark.e2e -GEMINI_MODEL: Final = "gemini-2.5-flash" +GEMINI_MODEL: Final = "gemini-3.8-flash" OPENAI_MODEL: Final = "gpt-5.5" REGENERATE_GRACE_PERIOD = "15s" REGENERATE_GRACE_SECONDS = 15.0 @@ -155,7 +155,7 @@ class TestKeyRoutes: info = client.proxy.key_info(key) assert info.key_alias == alias, f"/key/info reports key_alias {info.key_alias!r}, configured {alias!r}" assert info.models == [GEMINI_MODEL], ( - f"/key/info reports models {info.models}, configured ['gemini-2.5-flash']" + f"/key/info reports models {info.models}, configured ['gemini-3.8-flash']" ) assert info.tpm_limit == 424242, ( f"/key/info reports tpm_limit {info.tpm_limit}, configured 424242" @@ -501,7 +501,7 @@ class TestTeamRoutes: info = client.team_info(team_id) assert info.team_alias == alias, f"/team/info reports team_alias {info.team_alias!r}, configured {alias!r}" assert info.models == [GEMINI_MODEL], ( - f"/team/info reports models {info.models}, configured ['gemini-2.5-flash']" + f"/team/info reports models {info.models}, configured ['gemini-3.8-flash']" ) key = _generate_key(client, resources, KeyGenerateBody(team_id=team_id)) @@ -775,7 +775,7 @@ class TestOrganizationRoutes: f"/organization/info reports alias {info.organization_alias!r}, configured {alias!r}" ) assert info.models == [GEMINI_MODEL], ( - f"/organization/info reports models {info.models}, configured ['gemini-2.5-flash']" + f"/organization/info reports models {info.models}, configured ['gemini-3.8-flash']" ) @pytest.mark.covers("mgmt.organization.update.persists") diff --git a/tests/e2e/management/test_team_management_e2e.py b/tests/e2e/management/test_team_management_e2e.py index 905e3cde206..e513ab1e91b 100644 --- a/tests/e2e/management/test_team_management_e2e.py +++ b/tests/e2e/management/test_team_management_e2e.py @@ -248,15 +248,15 @@ class TestTeamManagementRoutes: self, client: ManagementClient, resources: ResourceManager ) -> None: alias = f"e2e-team-info-{unique_marker()}" - team_id = _create_team(client, resources, alias, ["gemini-2.5-flash"]) + team_id = _create_team(client, resources, alias, ["gemini-3.8-flash"]) info = _read_team(client, team_id) assert info.team_id == team_id, f"/team/info echoed team_id {info.team_id!r}, requested {team_id!r}" assert info.team_info.team_alias == alias, ( f"/team/info reports team_alias {info.team_info.team_alias!r}, configured {alias!r}" ) - assert info.team_info.models == ["gemini-2.5-flash"], ( - f"/team/info reports models {info.team_info.models}, configured ['gemini-2.5-flash']" + assert info.team_info.models == ["gemini-3.8-flash"], ( + f"/team/info reports models {info.team_info.models}, configured ['gemini-3.8-flash']" ) @pytest.mark.covers("mgmt.team.block.persists") @@ -264,7 +264,7 @@ class TestTeamManagementRoutes: def test_block_then_unblock_persists_to_team_info( self, client: ManagementClient, resources: ResourceManager ) -> None: - team_id = _create_team(client, resources, f"e2e-team-block-{unique_marker()}", ["gemini-2.5-flash"]) + team_id = _create_team(client, resources, f"e2e-team-block-{unique_marker()}", ["gemini-3.8-flash"]) assert not _read_team(client, team_id).team_info.blocked, "/team/info reports the team blocked before /team/block" _set_blocked(client, team_id, blocked=True) @@ -287,7 +287,7 @@ class TestTeamManagementRoutes: self, client: ManagementClient, resources: ResourceManager ) -> None: user_id = _create_user(client, resources, f"e2e-team-mu-{unique_marker()}@example.com") - team_id = _create_team(client, resources, f"e2e-team-mu-{unique_marker()}", ["gemini-2.5-flash"]) + team_id = _create_team(client, resources, f"e2e-team-mu-{unique_marker()}", ["gemini-3.8-flash"]) client.add_team_member(team_id, user_id) assert _member_role(_read_team(client, team_id), user_id) == "user", ( f"member {user_id} should start as role 'user' after /team/member_add" @@ -312,7 +312,7 @@ class TestTeamManagementRoutes: self, client: ManagementClient, resources: ResourceManager ) -> None: user_id = _create_user(client, resources, f"e2e-team-md-{unique_marker()}@example.com") - team_id = _create_team(client, resources, f"e2e-team-md-{unique_marker()}", ["gemini-2.5-flash"]) + team_id = _create_team(client, resources, f"e2e-team-md-{unique_marker()}", ["gemini-3.8-flash"]) client.add_team_member(team_id, user_id) assert _member_role(_read_team(client, team_id), user_id) == "user", ( f"/team/info does not list {user_id} as a member after /team/member_add" @@ -378,7 +378,7 @@ class TestTeamManagementRoutes: user + team, plus a second user id the member could try to add.""" member_id = _create_user(client, resources, f"e2e-team-fb-{unique_marker()}@example.com") other_id = _create_user(client, resources, f"e2e-team-fb-{unique_marker()}@example.com") - team_id = _create_team(client, resources, f"e2e-team-fb-{unique_marker()}", ["gemini-2.5-flash"]) + team_id = _create_team(client, resources, f"e2e-team-fb-{unique_marker()}", ["gemini-3.8-flash"]) client.add_team_member(team_id, member_id) member_key = _generate_key(client, resources, KeyGenerateBody(user_id=member_id, team_id=team_id)) return member_id, other_id, member_key, team_id @@ -534,7 +534,7 @@ class TestTeamAdminWithTpmLimitEnabled: pytest.param(TeamSettingsChange(rpm_limit=10), id="rpm_limit"), pytest.param(TeamSettingsChange(max_budget=0.5), id="max_budget"), pytest.param(TeamSettingsChange(team_alias="renamed-by-team-admin"), id="team_alias"), - pytest.param(TeamSettingsChange(models=["gemini-2.5-flash"]), id="models"), + pytest.param(TeamSettingsChange(models=["gemini-3.8-flash"]), id="models"), pytest.param(TeamSettingsChange(budget_duration="1d"), id="budget_duration"), pytest.param(TeamSettingsChange(metadata=TeamCustomMetadata(cost_center="team-admin")), id="metadata"), ], diff --git a/tests/e2e/quota_management/budgets/test_model_max_budget_e2e.py b/tests/e2e/quota_management/budgets/test_model_max_budget_e2e.py index c69b0e232ff..77f779a6efc 100644 --- a/tests/e2e/quota_management/budgets/test_model_max_budget_e2e.py +++ b/tests/e2e/quota_management/budgets/test_model_max_budget_e2e.py @@ -20,7 +20,7 @@ from models import ModelBudgetEntry pytestmark = pytest.mark.e2e CAPPED_MODEL = "claude-haiku-4-5" -FREE_MODEL = "gemini-2.5-flash" +FREE_MODEL = "gemini-3.8-flash" def _call(client: BudgetClient, key: str, model: str): diff --git a/tests/e2e/quota_management/spend_tracking/conftest.py b/tests/e2e/quota_management/spend_tracking/conftest.py index 9c8ffd18144..75ba870137f 100644 --- a/tests/e2e/quota_management/spend_tracking/conftest.py +++ b/tests/e2e/quota_management/spend_tracking/conftest.py @@ -32,7 +32,7 @@ def _driver_params(provider_model: str, env_var: str) -> LiteLLMParamsBody: DRIVER_MODELS: tuple[tuple[str, str, str], ...] = ( - ("gemini-2.5-flash", "gemini/gemini-2.5-flash", "GEMINI_API_KEY"), + ("gemini-3.8-flash", "gemini/gemini-3.8-flash", "GEMINI_API_KEY"), ("claude-haiku-4-5", "anthropic/claude-haiku-4-5", "ANTHROPIC_API_KEY"), ("openai-text-embedding-3-small", "openai/text-embedding-3-small", "OPENAI_API_KEY"), ("openai-responses-codex", "openai/gpt-5.3-codex", "OPENAI_API_KEY"), diff --git a/tests/e2e/quota_management/spend_tracking/test_key_attribution_e2e.py b/tests/e2e/quota_management/spend_tracking/test_key_attribution_e2e.py index e0c19ea1b6b..1d0878d073a 100644 --- a/tests/e2e/quota_management/spend_tracking/test_key_attribution_e2e.py +++ b/tests/e2e/quota_management/spend_tracking/test_key_attribution_e2e.py @@ -55,7 +55,7 @@ from spend_e2e_client import ( pytestmark = pytest.mark.e2e -CHAT_MODEL: Final = "gemini-2.5-flash" +CHAT_MODEL: Final = "gemini-3.8-flash" MESSAGES_MODEL: Final = "claude-haiku-4-5" RESPONSES_MODEL: Final = "openai-responses-codex" EMBED_MODEL: Final = "openai-text-embedding-3-small" diff --git a/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py b/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py index c7ef81f826a..d452525ec91 100644 --- a/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py +++ b/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py @@ -3,7 +3,7 @@ Run against a proxy started with the gateway config. Coverage rationale: SPEND_TRACKING_COVERAGE_MATRIX.md. -Model names are literals from that config: chat tests hit "gemini-2.5-flash", +Model names are literals from that config: chat tests hit "gemini-3.8-flash", embedding tests hit "openai-text-embedding-3-small". Every test: fresh scoped key (isolation) -> real provider call -> unwrap (hard @@ -37,7 +37,7 @@ from spend_reconciliation import BACKEND as TRAFFIC_BACKEND pytestmark = pytest.mark.e2e -GEMINI_MODEL = "gemini-2.5-flash" +GEMINI_MODEL = "gemini-3.8-flash" CLAUDE_MODEL = "claude-haiku-4-5" CODEX_MODEL = "openai-responses-codex" EMBEDDING_MODEL = "openai-text-embedding-3-small" @@ -780,7 +780,7 @@ def test_spend_calculate_returns_nonzero_cost(client: SpendClient) -> None: GEMINI_MODEL, "estimate the cost of this request" ) assert cost > 0, ( - "/spend/calculate returned 0 for gemini-2.5-flash; " + "/spend/calculate returned 0 for gemini-3.8-flash; " "cost map may be missing this model" )