fix(ci): stabilize CI tests - conditional import, mock fixes, timing adjustments

Fix 1.1: Make ResponseApplyPatchToolCall import conditional with try/except
  for compatibility with openai==1.100.1 (CI environment)
Fix 1.2: Move Router creation inside mock context in vector store tests
  so mocks are applied before Router captures function references
Fix 1.3: Update test_model_group_info_e2e to check for 'anthropic/*'
  wildcard group instead of specific model names not in proxy config
Fix 2.1: Increase redis cache test sleep from 1s to 5s
Fix 2.2: Increase spend accuracy test sleep from 25s to 45s
Fix 2.3: Add 0.5s sleep between budget test calls
Fix 2.4: Increase vertex AI spend test sleep from 20s to 40s

Co-authored-by: yuneng-jiang <yuneng-jiang@users.noreply.github.com>

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -398,9 +398,12 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             ResponseOutputMessage,
             ResponseReasoningItem,
         )
-        from openai.types.responses.response_output_item import (
+        try:
-            ResponseApplyPatchToolCall,
+            from openai.types.responses.response_output_item import (
-        )
+                ResponseApplyPatchToolCall,
+            )
+        except ImportError:
+            ResponseApplyPatchToolCall = None  # type: ignore[assignment,misc]
 
         from litellm.types.utils import Choices, Message
 
@@ -457,7 +460,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                 accumulated_tool_calls.append(tool_call_dict)
                 tool_call_index += 1
 
-            elif isinstance(item, ResponseApplyPatchToolCall):
+            elif ResponseApplyPatchToolCall is not None and isinstance(item, ResponseApplyPatchToolCall):
                 from litellm.responses.litellm_completion_transformation.transformation import (
                     LiteLLMCompletionResponsesConfig,
                 )

tests/local_testing/test_custom_logger.py

@@ -531,7 +531,7 @@ def test_redis_cache_completion_stream():
             response_1_content += chunk.choices[0].delta.content or ""
         print(response_1_content)
 
-        time.sleep(1)  # sleep for 0.1 seconds allow set cache to occur
+        time.sleep(5)  # sleep for cache write to propagate
         response2 = completion(
             model="gpt-3.5-turbo",
             messages=messages,

tests/otel_tests/test_e2e_budgeting.py

@@ -14,6 +14,7 @@ async def make_calls_until_budget_exceeded(session, key: str, call_function, **k
         while call_count < MAX_CALLS:
             await call_function(session=session, key=key, **kwargs)
             call_count += 1
+            await asyncio.sleep(0.5)  # allow spend tracking to catch up
         pytest.fail(f"Budget was not exceeded after {MAX_CALLS} calls")
     except Exception as e:
         print("vars: ", vars(e))

tests/pass_through_tests/test_vertex_ai.py

@@ -109,7 +109,7 @@ async def test_basic_vertex_ai_pass_through_with_spendlog():
 
     print("response", response)
 
-    await asyncio.sleep(20)
+    await asyncio.sleep(40)
     spend_after = await call_spend_logs_endpoint()
     print("spend_after", spend_after)
     assert (

tests/spend_tracking_tests/test_spend_accuracy_tests.py

@@ -156,8 +156,8 @@ async def test_basic_spend_accuracy():
             response = await chat_completion(session, key)
             print("response: ", response)
 
-        # wait 25 seconds for spend to be updated
+        # wait for spend to be updated (batch writes can take a while)
-        await asyncio.sleep(25)
+        await asyncio.sleep(45)
 
         # Get spend information for each entity
         key_info = await get_spend_info(session, "key", key)
@@ -235,7 +235,7 @@ async def test_long_term_spend_accuracy_with_bursts():
             print(f"Burst 1 - Request {i+1}/{BURST_1_REQUESTS} completed")
 
         # Wait for spend to be updated
-        await asyncio.sleep(15)
+        await asyncio.sleep(30)
 
         # Check intermediate spend
         intermediate_key_info = await get_spend_info(session, "key", key)
@@ -248,7 +248,7 @@ async def test_long_term_spend_accuracy_with_bursts():
             print(f"Burst 2 - Request {i+1}/{BURST_2_REQUESTS} completed")
 
         # Wait for spend to be updated
-        await asyncio.sleep(15)
+        await asyncio.sleep(30)
 
         # Get final spend information for each entity
         key_info = await get_spend_info(session, "key", key)

tests/test_models.py

@@ -489,23 +489,20 @@ async def test_model_group_info_e2e():
         models = await get_models(session=session, key="sk-1234")
         print(models)
 
-        expected_models = [
-            "anthropic/claude-3-5-haiku-20241022",
-            "anthropic/claude-3-opus-20240229",
-        ]
-
         model_group_info = await get_model_group_info(session=session, key="sk-1234")
         print(model_group_info)
 
-        has_anthropic_claude_3_5_haiku = False
+        # Check that the endpoint returns data and contains the wildcard
-        has_anthropic_claude_3_opus = False
+        # anthropic model group from the proxy config
+        has_anthropic_wildcard = False
         for model in model_group_info["data"]:
-            if model["model_group"] == "anthropic/claude-3-5-haiku-20241022":
+            if model["model_group"] == "anthropic/*":
-                has_anthropic_claude_3_5_haiku = True
+                has_anthropic_wildcard = True
-            if model["model_group"] == "anthropic/claude-3-opus-20240229":
-                has_anthropic_claude_3_opus = True
 
-        assert has_anthropic_claude_3_5_haiku and has_anthropic_claude_3_opus
+        assert has_anthropic_wildcard, (
+            f"Expected 'anthropic/*' in model groups, got: "
+            f"{[m['model_group'] for m in model_group_info['data']]}"
+        )
 
 
 @pytest.mark.asyncio

tests/test_new_vector_store_endpoints.py

@@ -18,8 +18,6 @@ from litellm.proxy._types import UserAPIKeyAuth
 @pytest.mark.asyncio
 async def test_vector_store_retrieve_basic():
     """Test basic vector store retrieve functionality."""
-    router = litellm.Router(model_list=[])
-    
     mock_response = {
         "id": "vs_test123",
         "object": "vector_store",
@@ -40,6 +38,7 @@ async def test_vector_store_retrieve_basic():
         "litellm.vector_stores.main.aretrieve",
         new=AsyncMock(return_value=mock_response),
     ) as mock_retrieve:
+        router = litellm.Router(model_list=[])
         result = await router.avector_store_retrieve(
             vector_store_id="vs_test123",
             custom_llm_provider="openai",
@@ -54,8 +53,6 @@ async def test_vector_store_retrieve_basic():
 @pytest.mark.asyncio
 async def test_vector_store_list_basic():
     """Test basic vector store list functionality."""
-    router = litellm.Router(model_list=[])
-    
     mock_response = {
         "object": "list",
         "data": [
@@ -81,6 +78,7 @@ async def test_vector_store_list_basic():
         "litellm.vector_stores.main.alist",
         new=AsyncMock(return_value=mock_response),
     ) as mock_list:
+        router = litellm.Router(model_list=[])
         result = await router.avector_store_list(
             limit=20,
             order="desc",
@@ -96,8 +94,6 @@ async def test_vector_store_list_basic():
 @pytest.mark.asyncio
 async def test_vector_store_update_basic():
     """Test basic vector store update functionality."""
-    router = litellm.Router(model_list=[])
-    
     mock_response = {
         "id": "vs_test123",
         "object": "vector_store",
@@ -111,6 +107,7 @@ async def test_vector_store_update_basic():
         "litellm.vector_stores.main.aupdate",
         new=AsyncMock(return_value=mock_response),
     ) as mock_update:
+        router = litellm.Router(model_list=[])
         result = await router.avector_store_update(
             vector_store_id="vs_test123",
             name="Updated Name",
@@ -127,8 +124,6 @@ async def test_vector_store_update_basic():
 @pytest.mark.asyncio
 async def test_vector_store_delete_basic():
     """Test basic vector store delete functionality."""
-    router = litellm.Router(model_list=[])
-    
     mock_response = {
         "id": "vs_test123",
         "object": "vector_store.deleted",
@@ -139,6 +134,7 @@ async def test_vector_store_delete_basic():
         "litellm.vector_stores.main.adelete",
         new=AsyncMock(return_value=mock_response),
     ) as mock_delete:
+        router = litellm.Router(model_list=[])
         result = await router.avector_store_delete(
             vector_store_id="vs_test123",
             custom_llm_provider="openai",
@@ -153,8 +149,6 @@ async def test_vector_store_delete_basic():
 @pytest.mark.asyncio
 async def test_async_vector_store_retrieve():
     """Test async vector store retrieve."""
-    router = litellm.Router(model_list=[])
-    
     mock_response = {
         "id": "vs_async123",
         "object": "vector_store",
@@ -165,6 +159,7 @@ async def test_async_vector_store_retrieve():
         "litellm.vector_stores.main.aretrieve",
         new=AsyncMock(return_value=mock_response),
     ) as mock_aretrieve:
+        router = litellm.Router(model_list=[])
         result = await router.avector_store_retrieve(
             vector_store_id="vs_async123",
             custom_llm_provider="openai",
@@ -177,8 +172,6 @@ async def test_async_vector_store_retrieve():
 @pytest.mark.asyncio
 async def test_async_vector_store_list():
     """Test async vector store list."""
-    router = litellm.Router(model_list=[])
-    
     mock_response = {
         "object": "list",
         "data": [{"id": "vs_1"}, {"id": "vs_2"}],
@@ -188,6 +181,7 @@ async def test_async_vector_store_list():
         "litellm.vector_stores.main.alist",
         new=AsyncMock(return_value=mock_response),
     ) as mock_alist:
+        router = litellm.Router(model_list=[])
         result = await router.avector_store_list(
             limit=10,
             custom_llm_provider="openai",
@@ -200,8 +194,6 @@ async def test_async_vector_store_list():
 @pytest.mark.asyncio
 async def test_async_vector_store_update():
     """Test async vector store update."""
-    router = litellm.Router(model_list=[])
-    
     mock_response = {
         "id": "vs_async123",
         "name": "Updated Async Name",
@@ -211,6 +203,7 @@ async def test_async_vector_store_update():
         "litellm.vector_stores.main.aupdate",
         new=AsyncMock(return_value=mock_response),
     ) as mock_aupdate:
+        router = litellm.Router(model_list=[])
         result = await router.avector_store_update(
             vector_store_id="vs_async123",
             name="Updated Async Name",
@@ -224,8 +217,6 @@ async def test_async_vector_store_update():
 @pytest.mark.asyncio
 async def test_async_vector_store_delete():
     """Test async vector store delete."""
-    router = litellm.Router(model_list=[])
-    
     mock_response = {
         "id": "vs_async123",
         "deleted": True,
@@ -235,6 +226,7 @@ async def test_async_vector_store_delete():
         "litellm.vector_stores.main.adelete",
         new=AsyncMock(return_value=mock_response),
     ) as mock_adelete:
+        router = litellm.Router(model_list=[])
         result = await router.avector_store_delete(
             vector_store_id="vs_async123",
             custom_llm_provider="openai",
@@ -247,8 +239,6 @@ async def test_async_vector_store_delete():
 @pytest.mark.asyncio
 async def test_vector_store_list_with_pagination():
     """Test vector store list with pagination parameters."""
-    router = litellm.Router(model_list=[])
-    
     mock_response = {
         "object": "list",
         "data": [{"id": f"vs_{i}"} for i in range(5)],
@@ -261,6 +251,7 @@ async def test_vector_store_list_with_pagination():
         "litellm.vector_stores.main.list",
         return_value=mock_response,
     ) as mock_list:
+        router = litellm.Router(model_list=[])
         result = router.vector_store_list(
             limit=5,
             after="vs_previous",
@@ -281,8 +272,6 @@ async def test_vector_store_list_with_pagination():
 @pytest.mark.asyncio
 async def test_vector_store_update_with_expires_after():
     """Test vector store update with expiration policy."""
-    router = litellm.Router(model_list=[])
-    
     expires_after = {
         "anchor": "last_active_at",
         "days": 7,
@@ -298,6 +287,7 @@ async def test_vector_store_update_with_expires_after():
         "litellm.vector_stores.main.update",
         return_value=mock_response,
     ) as mock_update:
+        router = litellm.Router(model_list=[])
         result = router.vector_store_update(
             vector_store_id="vs_test123",
             expires_after=expires_after,