test: deflake two tests whose shared-state leaks failed once and passed on CI rerun

The Redis semantic cache tests wrapped the first import of litellm.caching.redis_semantic_cache in patch.dict("sys.modules", ...), which snapshots and restores all of sys.modules on exit. Every module first imported inside the block, including litellm.proxy.proxy_server, was dropped from sys.modules while staying cached as an attribute on the litellm.proxy package. The next test that patched litellm.proxy.proxy_server.<attr> hit the stale attribute while production code re-imported a fresh module, so the patch never reached it. Replace the whole-dict patch with MonkeyPatch.setitem on the two redisvl keys only

The LangSmith init test globally patched asyncio.get_running_loop while constructing the logger. Any orphaned AsyncHTTPHandler finalized by the cyclic GC during that window also called loop.create_task on the mock, tripping assert_called_once. Run the test under a real event loop and assert on the real task instead of patching asyncio

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

test-quality-budget.json

@@ -3,7 +3,7 @@
     "limit": 733
   },
   "TQ002": {
-    "limit": 737
+    "limit": 736
   },
   "TQ003": {
     "limit": 62

tests/test_litellm/caching/test_redis_semantic_cache.py

@@ -1,3 +1,5 @@
+from collections.abc import Iterator
+from contextlib import contextmanager
 from importlib import import_module
 import sys
 from unittest.mock import AsyncMock, MagicMock, patch
@@ -5,18 +7,19 @@ from unittest.mock import AsyncMock, MagicMock, patch
 import pytest
 
 
+@contextmanager
+def _fake_redisvl_modules(semantic_cache_mock: MagicMock, custom_vectorizer_mock: MagicMock) -> Iterator[None]:
+    with pytest.MonkeyPatch.context() as mp:
+        mp.setitem(sys.modules, "redisvl.extensions.llmcache", MagicMock(SemanticCache=semantic_cache_mock))
+        mp.setitem(sys.modules, "redisvl.utils.vectorize", MagicMock(CustomTextVectorizer=custom_vectorizer_mock))
+        yield
+
 
 # Tests for RedisSemanticCache
 def test_redis_semantic_cache_initialization(monkeypatch):
     # Mock the redisvl import
     semantic_cache_mock = MagicMock()
-    with patch.dict(
-        "sys.modules",
-        {
-            "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock),
-            "redisvl.utils.vectorize": MagicMock(CustomTextVectorizer=MagicMock()),
-        },
-    ):
+    with _fake_redisvl_modules(semantic_cache_mock, MagicMock()):
         from litellm.caching.redis_semantic_cache import RedisSemanticCache
 
         # Set environment variables
@@ -44,15 +47,7 @@ def test_redis_semantic_cache_get_cache(monkeypatch):
     semantic_cache_mock = MagicMock()
     custom_vectorizer_mock = MagicMock()
 
-    with patch.dict(
-        "sys.modules",
-        {
-            "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock),
-            "redisvl.utils.vectorize": MagicMock(
-                CustomTextVectorizer=custom_vectorizer_mock
-            ),
-        },
-    ):
+    with _fake_redisvl_modules(semantic_cache_mock, custom_vectorizer_mock):
         from litellm.caching.redis_semantic_cache import RedisSemanticCache
 
         # Set environment variables
@@ -110,15 +105,7 @@ def test_redis_semantic_cache_rejects_unscoped_cache_hit(monkeypatch):
     semantic_cache_mock = MagicMock()
     custom_vectorizer_mock = MagicMock()
 
-    with patch.dict(
-        "sys.modules",
-        {
-            "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock),
-            "redisvl.utils.vectorize": MagicMock(
-                CustomTextVectorizer=custom_vectorizer_mock
-            ),
-        },
-    ):
+    with _fake_redisvl_modules(semantic_cache_mock, custom_vectorizer_mock):
         from litellm.caching.redis_semantic_cache import RedisSemanticCache
 
         monkeypatch.setenv("REDIS_HOST", "localhost")
@@ -162,15 +149,7 @@ def test_redis_semantic_cache_set_cache_stores_cache_key_filter(monkeypatch):
     semantic_cache_mock = MagicMock()
     custom_vectorizer_mock = MagicMock()
 
-    with patch.dict(
-        "sys.modules",
-        {
-            "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock),
-            "redisvl.utils.vectorize": MagicMock(
-                CustomTextVectorizer=custom_vectorizer_mock
-            ),
-        },
-    ):
+    with _fake_redisvl_modules(semantic_cache_mock, custom_vectorizer_mock):
         from litellm.caching.redis_semantic_cache import RedisSemanticCache
 
         monkeypatch.setenv("REDIS_HOST", "localhost")
@@ -210,15 +189,7 @@ def test_redis_semantic_cache_uses_isolated_index_for_old_schema(monkeypatch):
     )
     custom_vectorizer_mock = MagicMock()
 
-    with patch.dict(
-        "sys.modules",
-        {
-            "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock),
-            "redisvl.utils.vectorize": MagicMock(
-                CustomTextVectorizer=custom_vectorizer_mock
-            ),
-        },
-    ):
+    with _fake_redisvl_modules(semantic_cache_mock, custom_vectorizer_mock):
         from litellm.caching.redis_semantic_cache import RedisSemanticCache
 
         monkeypatch.setenv("REDIS_HOST", "localhost")
@@ -252,15 +223,7 @@ def test_redis_semantic_cache_overwrites_stale_isolated_index(monkeypatch):
     )
     custom_vectorizer_mock = MagicMock()
 
-    with patch.dict(
-        "sys.modules",
-        {
-            "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock),
-            "redisvl.utils.vectorize": MagicMock(
-                CustomTextVectorizer=custom_vectorizer_mock
-            ),
-        },
-    ):
+    with _fake_redisvl_modules(semantic_cache_mock, custom_vectorizer_mock):
         from litellm.caching.redis_semantic_cache import RedisSemanticCache
 
         monkeypatch.setenv("REDIS_HOST", "localhost")
@@ -292,15 +255,7 @@ def test_redis_semantic_cache_reraises_unexpected_isolated_index_error(monkeypat
     )
     custom_vectorizer_mock = MagicMock()
 
-    with patch.dict(
-        "sys.modules",
-        {
-            "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock),
-            "redisvl.utils.vectorize": MagicMock(
-                CustomTextVectorizer=custom_vectorizer_mock
-            ),
-        },
-    ):
+    with _fake_redisvl_modules(semantic_cache_mock, custom_vectorizer_mock):
         from litellm.caching.redis_semantic_cache import RedisSemanticCache
 
         monkeypatch.setenv("REDIS_HOST", "localhost")
@@ -369,15 +324,15 @@ def test_redis_semantic_cache_builds_filter_expression(monkeypatch):
         def __eq__(self, value):
             return (self.field_name, value)
 
-    with patch.dict("sys.modules", {"redisvl.query.filter": MagicMock(Tag=FakeTag)}):
-        from litellm.caching.redis_semantic_cache import RedisSemanticCache
+    monkeypatch.setitem(sys.modules, "redisvl.query.filter", MagicMock(Tag=FakeTag))
+    from litellm.caching.redis_semantic_cache import RedisSemanticCache
 
-        redis_semantic_cache = RedisSemanticCache.__new__(RedisSemanticCache)
+    redis_semantic_cache = RedisSemanticCache.__new__(RedisSemanticCache)
 
-        assert redis_semantic_cache._get_cache_key_filter_expression("test_key") == (
-            RedisSemanticCache.CACHE_KEY_FIELD_NAME,
-            "test_key",
-        )
+    assert redis_semantic_cache._get_cache_key_filter_expression("test_key") == (
+        RedisSemanticCache.CACHE_KEY_FIELD_NAME,
+        "test_key",
+    )
 
 
 @pytest.mark.asyncio
@@ -386,15 +341,7 @@ async def test_redis_semantic_cache_async_get_cache(monkeypatch):
     semantic_cache_mock = MagicMock()
     custom_vectorizer_mock = MagicMock()
 
-    with patch.dict(
-        "sys.modules",
-        {
-            "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock),
-            "redisvl.utils.vectorize": MagicMock(
-                CustomTextVectorizer=custom_vectorizer_mock
-            ),
-        },
-    ):
+    with _fake_redisvl_modules(semantic_cache_mock, custom_vectorizer_mock):
         from litellm.caching.redis_semantic_cache import RedisSemanticCache
 
         # Set environment variables
@@ -449,15 +396,7 @@ async def test_redis_semantic_cache_async_get_cache_rejects_unscoped_hit(monkeyp
     semantic_cache_mock = MagicMock()
     custom_vectorizer_mock = MagicMock()
 
-    with patch.dict(
-        "sys.modules",
-        {
-            "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock),
-            "redisvl.utils.vectorize": MagicMock(
-                CustomTextVectorizer=custom_vectorizer_mock
-            ),
-        },
-    ):
+    with _fake_redisvl_modules(semantic_cache_mock, custom_vectorizer_mock):
         from litellm.caching.redis_semantic_cache import RedisSemanticCache
 
         monkeypatch.setenv("REDIS_HOST", "localhost")
@@ -499,15 +438,7 @@ async def test_redis_semantic_cache_async_set_cache_stores_cache_key_filter(
     semantic_cache_mock = MagicMock()
     custom_vectorizer_mock = MagicMock()
 
-    with patch.dict(
-        "sys.modules",
-        {
-            "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock),
-            "redisvl.utils.vectorize": MagicMock(
-                CustomTextVectorizer=custom_vectorizer_mock
-            ),
-        },
-    ):
+    with _fake_redisvl_modules(semantic_cache_mock, custom_vectorizer_mock):
         from litellm.caching.redis_semantic_cache import RedisSemanticCache
 
         monkeypatch.setenv("REDIS_HOST", "localhost")
@@ -1255,15 +1186,7 @@ def test_redis_init_defers_redisvl_construction(monkeypatch):
     semantic_cache_mock = MagicMock()
     custom_vectorizer_mock = MagicMock()
 
-    with patch.dict(
-        "sys.modules",
-        {
-            "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock),
-            "redisvl.utils.vectorize": MagicMock(
-                CustomTextVectorizer=custom_vectorizer_mock
-            ),
-        },
-    ):
+    with _fake_redisvl_modules(semantic_cache_mock, custom_vectorizer_mock):
         from litellm.caching.redis_semantic_cache import RedisSemanticCache
 
         monkeypatch.setenv("REDIS_HOST", "localhost")
@@ -1291,15 +1214,7 @@ def test_redis_failed_llmcache_build_is_not_memoized(monkeypatch):
     )
     custom_vectorizer_mock = MagicMock()
 
-    with patch.dict(
-        "sys.modules",
-        {
-            "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock),
-            "redisvl.utils.vectorize": MagicMock(
-                CustomTextVectorizer=custom_vectorizer_mock
-            ),
-        },
-    ):
+    with _fake_redisvl_modules(semantic_cache_mock, custom_vectorizer_mock):
         from litellm.caching.redis_semantic_cache import RedisSemanticCache
 
         monkeypatch.setenv("REDIS_HOST", "localhost")

tests/test_litellm/integrations/test_langsmith_init.py

@@ -1,3 +1,4 @@
+import asyncio
 import os
 from unittest.mock import MagicMock, patch
 
@@ -154,24 +155,20 @@ class TestLangsmithLoggerInit:
         assert logger._start_periodic_flush_task() is None
         mock_get_running_loop.assert_called_once()
 
-    @patch("asyncio.get_running_loop")
-    def test_langsmith_init_starts_periodic_flush_with_running_loop(
-        self, mock_get_running_loop
-    ):
+    @pytest.mark.asyncio
+    async def test_langsmith_init_starts_periodic_flush_with_running_loop(self):
         """Test that init schedules periodic flush when a running loop exists."""
-        mock_loop = MagicMock()
-        mock_task = MagicMock()
-        mock_loop.create_task.return_value = mock_task
-        mock_get_running_loop.return_value = mock_loop
-
         logger = LangsmithLogger(
             langsmith_api_key="test-key", langsmith_project="test-project"
         )
 
-        assert logger._flush_task == mock_task
-        mock_loop.create_task.assert_called_once()
-        scheduled_coro = mock_loop.create_task.call_args.args[0]
-        scheduled_coro.close()
+        flush_task = logger._flush_task
+        assert isinstance(flush_task, asyncio.Task)
+        assert not flush_task.done()
+        assert flush_task.get_coro().__qualname__ == "CustomBatchLogger.periodic_flush"
+        flush_task.cancel()
+        with pytest.raises(asyncio.CancelledError):
+            await flush_task
 
     @pytest.mark.asyncio
     async def test_async_log_success_event_lazily_starts_periodic_flush(self):