From e48f813d4dfb88d4cd38832bfc210762b751d1b3 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 13:07:37 -0700 Subject: [PATCH] test(google_genai): move unified_google_tests to gemini-3.5-flash-lite (#42520) gemini-2.5-flash-lite retires on 2026-10-20 per Google's Vertex model lifecycle table, which would turn the google_generate_content_endpoint_testing CircleCI job red on every PR from that day. gemini-3.5-flash-lite is the newest flash-lite on both AI Studio and Vertex, with a retirement of 2027-07-21 or later. Vertex serves gemini-3.5-flash-lite only on the global, us, and eu endpoints, never on a single region, so the direct SDK Vertex tests now pass vertex_location global like the suite's proxy config already does. The 429 backoff test derives its mocked generateContent URL from litellm's own Gemini URL builder, since litellm routes Gemini 3 and newer models to v1alpha instead of v1beta. Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- tests/unified_google_tests/conftest.py | 2 +- .../google_genai_proxy_test_config.yaml | 8 ++++---- .../unified_google_tests/test_google_ai_studio.py | 14 +++++++------- .../test_google_genai_proxy_test_config.py | 11 +++++------ .../unified_google_tests/test_vertex_ai_native.py | 5 +++-- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/tests/unified_google_tests/conftest.py b/tests/unified_google_tests/conftest.py index cd05c856faf..9eac5f49651 100644 --- a/tests/unified_google_tests/conftest.py +++ b/tests/unified_google_tests/conftest.py @@ -102,7 +102,7 @@ def google_genai_proxy_url() -> Iterator[str]: credentials_file = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", "") if not (credentials_file and os.path.isfile(credentials_file)): vertex_credentials_path = load_vertex_ai_credentials( - model="vertex_ai/gemini-2.5-flash-lite" + model="vertex_ai/gemini-3.5-flash-lite" ) if vertex_credentials_path: temp_credentials_path = vertex_credentials_path diff --git a/tests/unified_google_tests/google_genai_proxy_test_config.yaml b/tests/unified_google_tests/google_genai_proxy_test_config.yaml index 0a1779aa3ec..99f24e62916 100644 --- a/tests/unified_google_tests/google_genai_proxy_test_config.yaml +++ b/tests/unified_google_tests/google_genai_proxy_test_config.yaml @@ -1,12 +1,12 @@ model_list: - - model_name: gemini-2.5-flash-lite + - model_name: gemini-3.5-flash-lite litellm_params: - model: gemini/gemini-2.5-flash-lite + model: gemini/gemini-3.5-flash-lite api_key: os.environ/GEMINI_API_KEY - - model_name: vertex-gemini-2.5-flash-lite + - model_name: vertex-gemini-3.5-flash-lite litellm_params: - model: vertex_ai/gemini-2.5-flash-lite + model: vertex_ai/gemini-3.5-flash-lite vertex_location: global router_settings: diff --git a/tests/unified_google_tests/test_google_ai_studio.py b/tests/unified_google_tests/test_google_ai_studio.py index 6d4c3725080..2364a01cedb 100644 --- a/tests/unified_google_tests/test_google_ai_studio.py +++ b/tests/unified_google_tests/test_google_ai_studio.py @@ -13,12 +13,12 @@ class TestGoogleGenAIStudio(BaseGoogleGenAITest, BaseGoogleGenAIProxySDKTest): @property def model_config(self): return { - "model": "gemini/gemini-2.5-flash-lite", + "model": "gemini/gemini-3.5-flash-lite", } @property def proxy_model_name(self) -> str: - return "gemini-2.5-flash-lite" + return "gemini-3.5-flash-lite" @pytest.mark.asyncio @@ -94,7 +94,7 @@ async def test_mock_stream_generate_content_with_tools(): "\n--- Testing async agenerate_content_stream with function call parsing ---" ) response = await litellm.google_genai.agenerate_content_stream( - model="gemini/gemini-2.5-flash-lite", + model="gemini/gemini-3.5-flash-lite", contents=contents, tools=[ { @@ -343,7 +343,7 @@ async def test_validate_post_request_parameters(): # Make the API call response = await litellm.google_genai.agenerate_content_stream( - model="gemini/gemini-2.5-flash-lite", contents=contents, tools=tools + model="gemini/gemini-3.5-flash-lite", contents=contents, tools=tools ) # Consume the response to ensure the request is made @@ -387,11 +387,11 @@ async def test_validate_post_request_parameters(): # Validate model field assert "model" in request_data, "Expected 'model' field in request data" - # Model might be transformed, but should contain gemini-2.5-flash-lite + # Model might be transformed, but should contain gemini-3.5-flash-lite model_value = request_data["model"] assert ( - "gemini-2.5-flash-lite" in model_value - ), f"Expected model to contain 'gemini-2.5-flash-lite', got: {model_value}" + "gemini-3.5-flash-lite" in model_value + ), f"Expected model to contain 'gemini-3.5-flash-lite', got: {model_value}" print(f"✅ Model validation passed: {model_value}") # Validate contents field diff --git a/tests/unified_google_tests/test_google_genai_proxy_test_config.py b/tests/unified_google_tests/test_google_genai_proxy_test_config.py index 694ec336bac..272e589e94a 100644 --- a/tests/unified_google_tests/test_google_genai_proxy_test_config.py +++ b/tests/unified_google_tests/test_google_genai_proxy_test_config.py @@ -12,14 +12,13 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm import Router from litellm.constants import INITIAL_RETRY_DELAY, MAX_RETRY_DELAY -from litellm.llms.vertex_ai.common_utils import get_vertex_base_url +from litellm.llms.vertex_ai.common_utils import _get_gemini_url, get_vertex_base_url from litellm.llms.vertex_ai.vertex_llm_base import VertexBase CONFIG_PATH: Final = Path(__file__).parent / "google_genai_proxy_test_config.yaml" -GEMINI_DEPLOYMENT: Final = "gemini-2.5-flash-lite" -VERTEX_DEPLOYMENT: Final = "vertex-gemini-2.5-flash-lite" -GEMINI_HOST: Final = "generativelanguage.googleapis.com" -GEMINI_GENERATE_CONTENT_PATH: Final = "/v1beta/models/gemini-2.5-flash-lite:generateContent" +GEMINI_DEPLOYMENT: Final = "gemini-3.5-flash-lite" +VERTEX_DEPLOYMENT: Final = "vertex-gemini-3.5-flash-lite" +GEMINI_GENERATE_CONTENT_URL: Final = _get_gemini_url(mode="chat", model=GEMINI_DEPLOYMENT, stream=False)[0] VERTEX_GLOBAL_BASE_URL: Final = "https://aiplatform.googleapis.com" RESOURCE_EXHAUSTED: Final = { "error": {"code": 429, "message": "Resource exhausted. Please try again later.", "status": "RESOURCE_EXHAUSTED"} @@ -81,7 +80,7 @@ async def test_ci_proxy_config_rides_out_consecutive_429s_with_backoff( ) -> None: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) litellm.in_memory_llm_clients_cache.flush_cache() - route: Final = respx_mock.post(host=GEMINI_HOST, path=GEMINI_GENERATE_CONTENT_PATH).mock( + route: Final = respx_mock.post(GEMINI_GENERATE_CONTENT_URL).mock( side_effect=[httpx.Response(429, json=RESOURCE_EXHAUSTED)] * CONSECUTIVE_RATE_LIMITS + [httpx.Response(200, json=PONG)] ) diff --git a/tests/unified_google_tests/test_vertex_ai_native.py b/tests/unified_google_tests/test_vertex_ai_native.py index 640157bc33e..4bdc1a55283 100644 --- a/tests/unified_google_tests/test_vertex_ai_native.py +++ b/tests/unified_google_tests/test_vertex_ai_native.py @@ -8,9 +8,10 @@ class TestVertexAIGenerateContent(BaseGoogleGenAITest, BaseGoogleGenAIProxySDKTe @property def model_config(self): return { - "model": "vertex_ai/gemini-2.5-flash-lite", + "model": "vertex_ai/gemini-3.5-flash-lite", + "vertex_location": "global", } @property def proxy_model_name(self) -> str: - return "vertex-gemini-2.5-flash-lite" + return "vertex-gemini-3.5-flash-lite"