From cc812cdfc7ffa25f739b3d86e3bd96cb67750647 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 20 Aug 2026 17:03:35 -0700 Subject: [PATCH] test: point the live web search, groq and vertex image suites at models that still exist (#37733) * test: point the live web search, groq and vertex image suites at models that still exist Three CircleCI jobs on the staging-to-main promotion are red because the models their live suites call have been retired by the providers, not because anything in litellm changed. openai/gpt-4o-search-preview now answers "has been deprecated" (its dated id gpt-4o-search-preview-2025-03-11 carries deprecation_date 2026-07-23), so the two web search conformance tests and the web search cost tracking test move to gpt-5-search-api, the current search model. It keeps mode chat, supports_web_search and a search_context_cost_per_query map, so the cost assertion still resolves. groq/llama-3.1-8b-instant reached its deprecation_date of 2026-08-16 and Groq answers "does not exist or you do not have access to it". It follows groq/llama-3.3-70b-versatile to groq/openai/gpt-oss-120b, the same replacement PR #37422 already picked. The proxy config that job boots routes on a */* wildcard, so no config change is needed. vertex_ai/imagen-3.0-fast-generate-001 404s with "was not found or your project does not have access to it". Google retired the whole Imagen family across Vertex and the Gemini API, so there is no Imagen id left to point at. The class is removed rather than repointed: Vertex image generation is already covered live by TestVertexAIGeminiImageGeneration on vertex_ai/gemini-2.5-flash-image, and the Imagen request and response transformations keep their offline coverage in tests/test_litellm/llms/vertex_ai/image_generation/. Only live call sites move. Remaining references to the old ids sit in offline cost-map and transformation tests, where the string is a lookup key and no request leaves the process. * chore(lint): ratchet the TQ005 ceiling down to the count this branch reached Removing the retired TestVertexImageGeneration class cleared one TQ005 violation, so the gate demands the limit come down with it. make lint-budget-update only lowers a limit by the delta a branch cleared, and this ceiling already sat 2 above the base count, so the tool landed on 2834 while the gate wants the limit at or below the 2832 this branch reached. The remaining 2 are that stale headroom, which is exactly what the gate is asking to reclaim. --- test-quality-budget.json | 2 +- tests/image_gen_tests/test_image_generation.py | 14 -------------- tests/llm_translation/test_openai.py | 4 ++-- .../test_built_in_tools_cost_tracking.py | 2 +- tests/test_openai_endpoints.py | 2 +- 5 files changed, 5 insertions(+), 19 deletions(-) diff --git a/test-quality-budget.json b/test-quality-budget.json index 17baf64601b..fcb29c3191d 100644 --- a/test-quality-budget.json +++ b/test-quality-budget.json @@ -12,7 +12,7 @@ "limit": 770 }, "TQ005": { - "limit": 2835 + "limit": 2832 }, "TQ006": { "limit": 34 diff --git a/tests/image_gen_tests/test_image_generation.py b/tests/image_gen_tests/test_image_generation.py index 33dcdbb57a5..ad141a651e8 100644 --- a/tests/image_gen_tests/test_image_generation.py +++ b/tests/image_gen_tests/test_image_generation.py @@ -105,20 +105,6 @@ def load_vertex_ai_credentials(): os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = os.path.abspath(temp_file.name) -class TestVertexImageGeneration(BaseImageGenTest): - def get_base_image_generation_call_args(self) -> dict: - # comment this when running locally - load_vertex_ai_credentials() - - litellm.in_memory_llm_clients_cache = InMemoryCache() - return { - "model": "vertex_ai/imagen-3.0-fast-generate-001", - "vertex_ai_project": "litellm-ci-cd", - "vertex_ai_location": "us-central1", - "n": 1, - } - - class TestVertexAIGeminiImageGeneration(BaseImageGenTest): """Test Gemini image generation models (Nano Banana)""" diff --git a/tests/llm_translation/test_openai.py b/tests/llm_translation/test_openai.py index 1a14b00c7d7..61819dfc860 100644 --- a/tests/llm_translation/test_openai.py +++ b/tests/llm_translation/test_openai.py @@ -422,7 +422,7 @@ def test_openai_web_search(): """Makes a simple web search request and validates the response contains web search annotations and all expected fields are present""" litellm._turn_on_debug() response = litellm.completion( - model="openai/gpt-4o-search-preview", + model="openai/gpt-5-search-api", messages=[ { "role": "user", @@ -442,7 +442,7 @@ def test_openai_web_search_streaming(): # litellm._turn_on_debug() test_openai_web_search: Optional[ChatCompletionAnnotation] = None response = litellm.completion( - model="openai/gpt-4o-search-preview", + model="openai/gpt-5-search-api", messages=[ { "role": "user", diff --git a/tests/logging_callback_tests/test_built_in_tools_cost_tracking.py b/tests/logging_callback_tests/test_built_in_tools_cost_tracking.py index 335661d46d0..0e73ad834da 100644 --- a/tests/logging_callback_tests/test_built_in_tools_cost_tracking.py +++ b/tests/logging_callback_tests/test_built_in_tools_cost_tracking.py @@ -105,7 +105,7 @@ async def test_openai_web_search_logging_cost_tracking( from litellm._uuid import uuid request_kwargs = { - "model": "openai/gpt-4o-search-preview", + "model": "openai/gpt-5-search-api", "messages": [ { "role": "user", diff --git a/tests/test_openai_endpoints.py b/tests/test_openai_endpoints.py index 0d44064997e..ab43d1acb00 100644 --- a/tests/test_openai_endpoints.py +++ b/tests/test_openai_endpoints.py @@ -550,7 +550,7 @@ async def test_proxy_all_models(): async with aiohttp.ClientSession() as session: # call chat/completions with a model that the key was not created for + the model is not on the config.yaml await chat_completion( - session=session, key=LITELLM_MASTER_KEY, model="groq/llama-3.1-8b-instant" + session=session, key=LITELLM_MASTER_KEY, model="groq/openai/gpt-oss-120b" ) await chat_completion(