From bd5f066c67795ee3113f2fde9e70e3f95f3a3685 Mon Sep 17 00:00:00 2001 From: mateo Date: Sat, 5 Sep 2026 10:27:01 +0000 Subject: [PATCH] test: deflake two tests whose shared-state leaks failed once and passed on CI rerun The Redis semantic cache tests wrapped the first import of litellm.caching.redis_semantic_cache in patch.dict("sys.modules", ...), which snapshots and restores all of sys.modules on exit. Every module first imported inside the block, including litellm.proxy.proxy_server, was dropped from sys.modules while staying cached as an attribute on the litellm.proxy package. The next test that patched litellm.proxy.proxy_server. hit the stale attribute while production code re-imported a fresh module, so the patch never reached it. Replace the whole-dict patch with MonkeyPatch.setitem on the two redisvl keys only The LangSmith init test globally patched asyncio.get_running_loop while constructing the logger. Any orphaned AsyncHTTPHandler finalized by the cyclic GC during that window also called loop.create_task on the mock, tripping assert_called_once. Run the test under a real event loop and assert on the real task instead of patching asyncio Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- test-quality-budget.json | 2 +- .../caching/test_redis_semantic_cache.py | 141 ++++-------------- .../integrations/test_langsmith_init.py | 23 ++- 3 files changed, 39 insertions(+), 127 deletions(-) diff --git a/test-quality-budget.json b/test-quality-budget.json index 3c12371f02f..f382f2479a9 100644 --- a/test-quality-budget.json +++ b/test-quality-budget.json @@ -3,7 +3,7 @@ "limit": 733 }, "TQ002": { - "limit": 737 + "limit": 736 }, "TQ003": { "limit": 62 diff --git a/tests/test_litellm/caching/test_redis_semantic_cache.py b/tests/test_litellm/caching/test_redis_semantic_cache.py index df990c43530..9884e9d9bc0 100644 --- a/tests/test_litellm/caching/test_redis_semantic_cache.py +++ b/tests/test_litellm/caching/test_redis_semantic_cache.py @@ -1,3 +1,5 @@ +from collections.abc import Iterator +from contextlib import contextmanager from importlib import import_module import sys from unittest.mock import AsyncMock, MagicMock, patch @@ -5,18 +7,19 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest +@contextmanager +def _fake_redisvl_modules(semantic_cache_mock: MagicMock, custom_vectorizer_mock: MagicMock) -> Iterator[None]: + with pytest.MonkeyPatch.context() as mp: + mp.setitem(sys.modules, "redisvl.extensions.llmcache", MagicMock(SemanticCache=semantic_cache_mock)) + mp.setitem(sys.modules, "redisvl.utils.vectorize", MagicMock(CustomTextVectorizer=custom_vectorizer_mock)) + yield + # Tests for RedisSemanticCache def test_redis_semantic_cache_initialization(monkeypatch): # Mock the redisvl import semantic_cache_mock = MagicMock() - with patch.dict( - "sys.modules", - { - "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock), - "redisvl.utils.vectorize": MagicMock(CustomTextVectorizer=MagicMock()), - }, - ): + with _fake_redisvl_modules(semantic_cache_mock, MagicMock()): from litellm.caching.redis_semantic_cache import RedisSemanticCache # Set environment variables @@ -44,15 +47,7 @@ def test_redis_semantic_cache_get_cache(monkeypatch): semantic_cache_mock = MagicMock() custom_vectorizer_mock = MagicMock() - with patch.dict( - "sys.modules", - { - "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock), - "redisvl.utils.vectorize": MagicMock( - CustomTextVectorizer=custom_vectorizer_mock - ), - }, - ): + with _fake_redisvl_modules(semantic_cache_mock, custom_vectorizer_mock): from litellm.caching.redis_semantic_cache import RedisSemanticCache # Set environment variables @@ -110,15 +105,7 @@ def test_redis_semantic_cache_rejects_unscoped_cache_hit(monkeypatch): semantic_cache_mock = MagicMock() custom_vectorizer_mock = MagicMock() - with patch.dict( - "sys.modules", - { - "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock), - "redisvl.utils.vectorize": MagicMock( - CustomTextVectorizer=custom_vectorizer_mock - ), - }, - ): + with _fake_redisvl_modules(semantic_cache_mock, custom_vectorizer_mock): from litellm.caching.redis_semantic_cache import RedisSemanticCache monkeypatch.setenv("REDIS_HOST", "localhost") @@ -162,15 +149,7 @@ def test_redis_semantic_cache_set_cache_stores_cache_key_filter(monkeypatch): semantic_cache_mock = MagicMock() custom_vectorizer_mock = MagicMock() - with patch.dict( - "sys.modules", - { - "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock), - "redisvl.utils.vectorize": MagicMock( - CustomTextVectorizer=custom_vectorizer_mock - ), - }, - ): + with _fake_redisvl_modules(semantic_cache_mock, custom_vectorizer_mock): from litellm.caching.redis_semantic_cache import RedisSemanticCache monkeypatch.setenv("REDIS_HOST", "localhost") @@ -210,15 +189,7 @@ def test_redis_semantic_cache_uses_isolated_index_for_old_schema(monkeypatch): ) custom_vectorizer_mock = MagicMock() - with patch.dict( - "sys.modules", - { - "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock), - "redisvl.utils.vectorize": MagicMock( - CustomTextVectorizer=custom_vectorizer_mock - ), - }, - ): + with _fake_redisvl_modules(semantic_cache_mock, custom_vectorizer_mock): from litellm.caching.redis_semantic_cache import RedisSemanticCache monkeypatch.setenv("REDIS_HOST", "localhost") @@ -252,15 +223,7 @@ def test_redis_semantic_cache_overwrites_stale_isolated_index(monkeypatch): ) custom_vectorizer_mock = MagicMock() - with patch.dict( - "sys.modules", - { - "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock), - "redisvl.utils.vectorize": MagicMock( - CustomTextVectorizer=custom_vectorizer_mock - ), - }, - ): + with _fake_redisvl_modules(semantic_cache_mock, custom_vectorizer_mock): from litellm.caching.redis_semantic_cache import RedisSemanticCache monkeypatch.setenv("REDIS_HOST", "localhost") @@ -292,15 +255,7 @@ def test_redis_semantic_cache_reraises_unexpected_isolated_index_error(monkeypat ) custom_vectorizer_mock = MagicMock() - with patch.dict( - "sys.modules", - { - "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock), - "redisvl.utils.vectorize": MagicMock( - CustomTextVectorizer=custom_vectorizer_mock - ), - }, - ): + with _fake_redisvl_modules(semantic_cache_mock, custom_vectorizer_mock): from litellm.caching.redis_semantic_cache import RedisSemanticCache monkeypatch.setenv("REDIS_HOST", "localhost") @@ -369,15 +324,15 @@ def test_redis_semantic_cache_builds_filter_expression(monkeypatch): def __eq__(self, value): return (self.field_name, value) - with patch.dict("sys.modules", {"redisvl.query.filter": MagicMock(Tag=FakeTag)}): - from litellm.caching.redis_semantic_cache import RedisSemanticCache + monkeypatch.setitem(sys.modules, "redisvl.query.filter", MagicMock(Tag=FakeTag)) + from litellm.caching.redis_semantic_cache import RedisSemanticCache - redis_semantic_cache = RedisSemanticCache.__new__(RedisSemanticCache) + redis_semantic_cache = RedisSemanticCache.__new__(RedisSemanticCache) - assert redis_semantic_cache._get_cache_key_filter_expression("test_key") == ( - RedisSemanticCache.CACHE_KEY_FIELD_NAME, - "test_key", - ) + assert redis_semantic_cache._get_cache_key_filter_expression("test_key") == ( + RedisSemanticCache.CACHE_KEY_FIELD_NAME, + "test_key", + ) @pytest.mark.asyncio @@ -386,15 +341,7 @@ async def test_redis_semantic_cache_async_get_cache(monkeypatch): semantic_cache_mock = MagicMock() custom_vectorizer_mock = MagicMock() - with patch.dict( - "sys.modules", - { - "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock), - "redisvl.utils.vectorize": MagicMock( - CustomTextVectorizer=custom_vectorizer_mock - ), - }, - ): + with _fake_redisvl_modules(semantic_cache_mock, custom_vectorizer_mock): from litellm.caching.redis_semantic_cache import RedisSemanticCache # Set environment variables @@ -449,15 +396,7 @@ async def test_redis_semantic_cache_async_get_cache_rejects_unscoped_hit(monkeyp semantic_cache_mock = MagicMock() custom_vectorizer_mock = MagicMock() - with patch.dict( - "sys.modules", - { - "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock), - "redisvl.utils.vectorize": MagicMock( - CustomTextVectorizer=custom_vectorizer_mock - ), - }, - ): + with _fake_redisvl_modules(semantic_cache_mock, custom_vectorizer_mock): from litellm.caching.redis_semantic_cache import RedisSemanticCache monkeypatch.setenv("REDIS_HOST", "localhost") @@ -499,15 +438,7 @@ async def test_redis_semantic_cache_async_set_cache_stores_cache_key_filter( semantic_cache_mock = MagicMock() custom_vectorizer_mock = MagicMock() - with patch.dict( - "sys.modules", - { - "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock), - "redisvl.utils.vectorize": MagicMock( - CustomTextVectorizer=custom_vectorizer_mock - ), - }, - ): + with _fake_redisvl_modules(semantic_cache_mock, custom_vectorizer_mock): from litellm.caching.redis_semantic_cache import RedisSemanticCache monkeypatch.setenv("REDIS_HOST", "localhost") @@ -1255,15 +1186,7 @@ def test_redis_init_defers_redisvl_construction(monkeypatch): semantic_cache_mock = MagicMock() custom_vectorizer_mock = MagicMock() - with patch.dict( - "sys.modules", - { - "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock), - "redisvl.utils.vectorize": MagicMock( - CustomTextVectorizer=custom_vectorizer_mock - ), - }, - ): + with _fake_redisvl_modules(semantic_cache_mock, custom_vectorizer_mock): from litellm.caching.redis_semantic_cache import RedisSemanticCache monkeypatch.setenv("REDIS_HOST", "localhost") @@ -1291,15 +1214,7 @@ def test_redis_failed_llmcache_build_is_not_memoized(monkeypatch): ) custom_vectorizer_mock = MagicMock() - with patch.dict( - "sys.modules", - { - "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock), - "redisvl.utils.vectorize": MagicMock( - CustomTextVectorizer=custom_vectorizer_mock - ), - }, - ): + with _fake_redisvl_modules(semantic_cache_mock, custom_vectorizer_mock): from litellm.caching.redis_semantic_cache import RedisSemanticCache monkeypatch.setenv("REDIS_HOST", "localhost") diff --git a/tests/test_litellm/integrations/test_langsmith_init.py b/tests/test_litellm/integrations/test_langsmith_init.py index 025aa86466c..34efc08ac3e 100644 --- a/tests/test_litellm/integrations/test_langsmith_init.py +++ b/tests/test_litellm/integrations/test_langsmith_init.py @@ -1,3 +1,4 @@ +import asyncio import os from unittest.mock import MagicMock, patch @@ -154,24 +155,20 @@ class TestLangsmithLoggerInit: assert logger._start_periodic_flush_task() is None mock_get_running_loop.assert_called_once() - @patch("asyncio.get_running_loop") - def test_langsmith_init_starts_periodic_flush_with_running_loop( - self, mock_get_running_loop - ): + @pytest.mark.asyncio + async def test_langsmith_init_starts_periodic_flush_with_running_loop(self): """Test that init schedules periodic flush when a running loop exists.""" - mock_loop = MagicMock() - mock_task = MagicMock() - mock_loop.create_task.return_value = mock_task - mock_get_running_loop.return_value = mock_loop - logger = LangsmithLogger( langsmith_api_key="test-key", langsmith_project="test-project" ) - assert logger._flush_task == mock_task - mock_loop.create_task.assert_called_once() - scheduled_coro = mock_loop.create_task.call_args.args[0] - scheduled_coro.close() + flush_task = logger._flush_task + assert isinstance(flush_task, asyncio.Task) + assert not flush_task.done() + assert flush_task.get_coro().__qualname__ == "CustomBatchLogger.periodic_flush" + flush_task.cancel() + with pytest.raises(asyncio.CancelledError): + await flush_task @pytest.mark.asyncio async def test_async_log_success_event_lazily_starts_periodic_flush(self):