From bd5f066c67795ee3113f2fde9e70e3f95f3a3685 Mon Sep 17 00:00:00 2001 From: mateo Date: Sat, 5 Sep 2026 10:27:01 +0000 Subject: [PATCH 1/6] test: deflake two tests whose shared-state leaks failed once and passed on CI rerun The Redis semantic cache tests wrapped the first import of litellm.caching.redis_semantic_cache in patch.dict("sys.modules", ...), which snapshots and restores all of sys.modules on exit. Every module first imported inside the block, including litellm.proxy.proxy_server, was dropped from sys.modules while staying cached as an attribute on the litellm.proxy package. The next test that patched litellm.proxy.proxy_server. hit the stale attribute while production code re-imported a fresh module, so the patch never reached it. Replace the whole-dict patch with MonkeyPatch.setitem on the two redisvl keys only The LangSmith init test globally patched asyncio.get_running_loop while constructing the logger. Any orphaned AsyncHTTPHandler finalized by the cyclic GC during that window also called loop.create_task on the mock, tripping assert_called_once. Run the test under a real event loop and assert on the real task instead of patching asyncio Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- test-quality-budget.json | 2 +- .../caching/test_redis_semantic_cache.py | 141 ++++-------------- .../integrations/test_langsmith_init.py | 23 ++- 3 files changed, 39 insertions(+), 127 deletions(-) diff --git a/test-quality-budget.json b/test-quality-budget.json index 3c12371f02f..f382f2479a9 100644 --- a/test-quality-budget.json +++ b/test-quality-budget.json @@ -3,7 +3,7 @@ "limit": 733 }, "TQ002": { - "limit": 737 + "limit": 736 }, "TQ003": { "limit": 62 diff --git a/tests/test_litellm/caching/test_redis_semantic_cache.py b/tests/test_litellm/caching/test_redis_semantic_cache.py index df990c43530..9884e9d9bc0 100644 --- a/tests/test_litellm/caching/test_redis_semantic_cache.py +++ b/tests/test_litellm/caching/test_redis_semantic_cache.py @@ -1,3 +1,5 @@ +from collections.abc import Iterator +from contextlib import contextmanager from importlib import import_module import sys from unittest.mock import AsyncMock, MagicMock, patch @@ -5,18 +7,19 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest +@contextmanager +def _fake_redisvl_modules(semantic_cache_mock: MagicMock, custom_vectorizer_mock: MagicMock) -> Iterator[None]: + with pytest.MonkeyPatch.context() as mp: + mp.setitem(sys.modules, "redisvl.extensions.llmcache", MagicMock(SemanticCache=semantic_cache_mock)) + mp.setitem(sys.modules, "redisvl.utils.vectorize", MagicMock(CustomTextVectorizer=custom_vectorizer_mock)) + yield + # Tests for RedisSemanticCache def test_redis_semantic_cache_initialization(monkeypatch): # Mock the redisvl import semantic_cache_mock = MagicMock() - with patch.dict( - "sys.modules", - { - "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock), - "redisvl.utils.vectorize": MagicMock(CustomTextVectorizer=MagicMock()), - }, - ): + with _fake_redisvl_modules(semantic_cache_mock, MagicMock()): from litellm.caching.redis_semantic_cache import RedisSemanticCache # Set environment variables @@ -44,15 +47,7 @@ def test_redis_semantic_cache_get_cache(monkeypatch): semantic_cache_mock = MagicMock() custom_vectorizer_mock = MagicMock() - with patch.dict( - "sys.modules", - { - "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock), - "redisvl.utils.vectorize": MagicMock( - CustomTextVectorizer=custom_vectorizer_mock - ), - }, - ): + with _fake_redisvl_modules(semantic_cache_mock, custom_vectorizer_mock): from litellm.caching.redis_semantic_cache import RedisSemanticCache # Set environment variables @@ -110,15 +105,7 @@ def test_redis_semantic_cache_rejects_unscoped_cache_hit(monkeypatch): semantic_cache_mock = MagicMock() custom_vectorizer_mock = MagicMock() - with patch.dict( - "sys.modules", - { - "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock), - "redisvl.utils.vectorize": MagicMock( - CustomTextVectorizer=custom_vectorizer_mock - ), - }, - ): + with _fake_redisvl_modules(semantic_cache_mock, custom_vectorizer_mock): from litellm.caching.redis_semantic_cache import RedisSemanticCache monkeypatch.setenv("REDIS_HOST", "localhost") @@ -162,15 +149,7 @@ def test_redis_semantic_cache_set_cache_stores_cache_key_filter(monkeypatch): semantic_cache_mock = MagicMock() custom_vectorizer_mock = MagicMock() - with patch.dict( - "sys.modules", - { - "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock), - "redisvl.utils.vectorize": MagicMock( - CustomTextVectorizer=custom_vectorizer_mock - ), - }, - ): + with _fake_redisvl_modules(semantic_cache_mock, custom_vectorizer_mock): from litellm.caching.redis_semantic_cache import RedisSemanticCache monkeypatch.setenv("REDIS_HOST", "localhost") @@ -210,15 +189,7 @@ def test_redis_semantic_cache_uses_isolated_index_for_old_schema(monkeypatch): ) custom_vectorizer_mock = MagicMock() - with patch.dict( - "sys.modules", - { - "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock), - "redisvl.utils.vectorize": MagicMock( - CustomTextVectorizer=custom_vectorizer_mock - ), - }, - ): + with _fake_redisvl_modules(semantic_cache_mock, custom_vectorizer_mock): from litellm.caching.redis_semantic_cache import RedisSemanticCache monkeypatch.setenv("REDIS_HOST", "localhost") @@ -252,15 +223,7 @@ def test_redis_semantic_cache_overwrites_stale_isolated_index(monkeypatch): ) custom_vectorizer_mock = MagicMock() - with patch.dict( - "sys.modules", - { - "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock), - "redisvl.utils.vectorize": MagicMock( - CustomTextVectorizer=custom_vectorizer_mock - ), - }, - ): + with _fake_redisvl_modules(semantic_cache_mock, custom_vectorizer_mock): from litellm.caching.redis_semantic_cache import RedisSemanticCache monkeypatch.setenv("REDIS_HOST", "localhost") @@ -292,15 +255,7 @@ def test_redis_semantic_cache_reraises_unexpected_isolated_index_error(monkeypat ) custom_vectorizer_mock = MagicMock() - with patch.dict( - "sys.modules", - { - "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock), - "redisvl.utils.vectorize": MagicMock( - CustomTextVectorizer=custom_vectorizer_mock - ), - }, - ): + with _fake_redisvl_modules(semantic_cache_mock, custom_vectorizer_mock): from litellm.caching.redis_semantic_cache import RedisSemanticCache monkeypatch.setenv("REDIS_HOST", "localhost") @@ -369,15 +324,15 @@ def test_redis_semantic_cache_builds_filter_expression(monkeypatch): def __eq__(self, value): return (self.field_name, value) - with patch.dict("sys.modules", {"redisvl.query.filter": MagicMock(Tag=FakeTag)}): - from litellm.caching.redis_semantic_cache import RedisSemanticCache + monkeypatch.setitem(sys.modules, "redisvl.query.filter", MagicMock(Tag=FakeTag)) + from litellm.caching.redis_semantic_cache import RedisSemanticCache - redis_semantic_cache = RedisSemanticCache.__new__(RedisSemanticCache) + redis_semantic_cache = RedisSemanticCache.__new__(RedisSemanticCache) - assert redis_semantic_cache._get_cache_key_filter_expression("test_key") == ( - RedisSemanticCache.CACHE_KEY_FIELD_NAME, - "test_key", - ) + assert redis_semantic_cache._get_cache_key_filter_expression("test_key") == ( + RedisSemanticCache.CACHE_KEY_FIELD_NAME, + "test_key", + ) @pytest.mark.asyncio @@ -386,15 +341,7 @@ async def test_redis_semantic_cache_async_get_cache(monkeypatch): semantic_cache_mock = MagicMock() custom_vectorizer_mock = MagicMock() - with patch.dict( - "sys.modules", - { - "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock), - "redisvl.utils.vectorize": MagicMock( - CustomTextVectorizer=custom_vectorizer_mock - ), - }, - ): + with _fake_redisvl_modules(semantic_cache_mock, custom_vectorizer_mock): from litellm.caching.redis_semantic_cache import RedisSemanticCache # Set environment variables @@ -449,15 +396,7 @@ async def test_redis_semantic_cache_async_get_cache_rejects_unscoped_hit(monkeyp semantic_cache_mock = MagicMock() custom_vectorizer_mock = MagicMock() - with patch.dict( - "sys.modules", - { - "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock), - "redisvl.utils.vectorize": MagicMock( - CustomTextVectorizer=custom_vectorizer_mock - ), - }, - ): + with _fake_redisvl_modules(semantic_cache_mock, custom_vectorizer_mock): from litellm.caching.redis_semantic_cache import RedisSemanticCache monkeypatch.setenv("REDIS_HOST", "localhost") @@ -499,15 +438,7 @@ async def test_redis_semantic_cache_async_set_cache_stores_cache_key_filter( semantic_cache_mock = MagicMock() custom_vectorizer_mock = MagicMock() - with patch.dict( - "sys.modules", - { - "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock), - "redisvl.utils.vectorize": MagicMock( - CustomTextVectorizer=custom_vectorizer_mock - ), - }, - ): + with _fake_redisvl_modules(semantic_cache_mock, custom_vectorizer_mock): from litellm.caching.redis_semantic_cache import RedisSemanticCache monkeypatch.setenv("REDIS_HOST", "localhost") @@ -1255,15 +1186,7 @@ def test_redis_init_defers_redisvl_construction(monkeypatch): semantic_cache_mock = MagicMock() custom_vectorizer_mock = MagicMock() - with patch.dict( - "sys.modules", - { - "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock), - "redisvl.utils.vectorize": MagicMock( - CustomTextVectorizer=custom_vectorizer_mock - ), - }, - ): + with _fake_redisvl_modules(semantic_cache_mock, custom_vectorizer_mock): from litellm.caching.redis_semantic_cache import RedisSemanticCache monkeypatch.setenv("REDIS_HOST", "localhost") @@ -1291,15 +1214,7 @@ def test_redis_failed_llmcache_build_is_not_memoized(monkeypatch): ) custom_vectorizer_mock = MagicMock() - with patch.dict( - "sys.modules", - { - "redisvl.extensions.llmcache": MagicMock(SemanticCache=semantic_cache_mock), - "redisvl.utils.vectorize": MagicMock( - CustomTextVectorizer=custom_vectorizer_mock - ), - }, - ): + with _fake_redisvl_modules(semantic_cache_mock, custom_vectorizer_mock): from litellm.caching.redis_semantic_cache import RedisSemanticCache monkeypatch.setenv("REDIS_HOST", "localhost") diff --git a/tests/test_litellm/integrations/test_langsmith_init.py b/tests/test_litellm/integrations/test_langsmith_init.py index 025aa86466c..34efc08ac3e 100644 --- a/tests/test_litellm/integrations/test_langsmith_init.py +++ b/tests/test_litellm/integrations/test_langsmith_init.py @@ -1,3 +1,4 @@ +import asyncio import os from unittest.mock import MagicMock, patch @@ -154,24 +155,20 @@ class TestLangsmithLoggerInit: assert logger._start_periodic_flush_task() is None mock_get_running_loop.assert_called_once() - @patch("asyncio.get_running_loop") - def test_langsmith_init_starts_periodic_flush_with_running_loop( - self, mock_get_running_loop - ): + @pytest.mark.asyncio + async def test_langsmith_init_starts_periodic_flush_with_running_loop(self): """Test that init schedules periodic flush when a running loop exists.""" - mock_loop = MagicMock() - mock_task = MagicMock() - mock_loop.create_task.return_value = mock_task - mock_get_running_loop.return_value = mock_loop - logger = LangsmithLogger( langsmith_api_key="test-key", langsmith_project="test-project" ) - assert logger._flush_task == mock_task - mock_loop.create_task.assert_called_once() - scheduled_coro = mock_loop.create_task.call_args.args[0] - scheduled_coro.close() + flush_task = logger._flush_task + assert isinstance(flush_task, asyncio.Task) + assert not flush_task.done() + assert flush_task.get_coro().__qualname__ == "CustomBatchLogger.periodic_flush" + flush_task.cancel() + with pytest.raises(asyncio.CancelledError): + await flush_task @pytest.mark.asyncio async def test_async_log_success_event_lazily_starts_periodic_flush(self): From 8a7dc64ab4d2082f5dd3def35a13eb71294d650c Mon Sep 17 00:00:00 2001 From: mateo Date: Sat, 5 Sep 2026 10:36:15 +0000 Subject: [PATCH 2/6] test: assert LangSmith periodic flush by observing a batch send Replace the coroutine __qualname__ check with a functional check: queue one event, run with a short flush interval, and wait for async_send_batch to be awaited by the task init scheduled Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/test_litellm/integrations/test_langsmith_init.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_litellm/integrations/test_langsmith_init.py b/tests/test_litellm/integrations/test_langsmith_init.py index 34efc08ac3e..0bc9e279fbf 100644 --- a/tests/test_litellm/integrations/test_langsmith_init.py +++ b/tests/test_litellm/integrations/test_langsmith_init.py @@ -1,6 +1,6 @@ import asyncio import os -from unittest.mock import MagicMock, patch +from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -159,13 +159,15 @@ class TestLangsmithLoggerInit: async def test_langsmith_init_starts_periodic_flush_with_running_loop(self): """Test that init schedules periodic flush when a running loop exists.""" logger = LangsmithLogger( - langsmith_api_key="test-key", langsmith_project="test-project" + langsmith_api_key="test-key", langsmith_project="test-project", flush_interval=0.01 ) + batch_sent = asyncio.Event() + logger.async_send_batch = AsyncMock(side_effect=batch_sent.set) + logger.log_queue.append({"id": "run-id"}) flush_task = logger._flush_task assert isinstance(flush_task, asyncio.Task) - assert not flush_task.done() - assert flush_task.get_coro().__qualname__ == "CustomBatchLogger.periodic_flush" + await asyncio.wait_for(batch_sent.wait(), timeout=5) flush_task.cancel() with pytest.raises(asyncio.CancelledError): await flush_task From ae2f1ae512c1ce5b5f029d8c13d8034e2a2d1525 Mon Sep 17 00:00:00 2001 From: mateo Date: Mon, 7 Sep 2026 09:19:02 +0000 Subject: [PATCH 3/6] chore: drop the budget ratchet from the PR branch, the scheduled automation owns it Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- test-quality-budget.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-quality-budget.json b/test-quality-budget.json index f382f2479a9..3c12371f02f 100644 --- a/test-quality-budget.json +++ b/test-quality-budget.json @@ -3,7 +3,7 @@ "limit": 733 }, "TQ002": { - "limit": 736 + "limit": 737 }, "TQ003": { "limit": 62 From 61e664088c9ad57ec99e4433c380ec00a7b8aa95 Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 8 Sep 2026 09:51:09 +0000 Subject: [PATCH 4/6] test: make the explicit stagger offset assertion independent of the wall clock Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../common_utils/test_scheduled_job_stagger.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/tests/test_litellm/proxy/common_utils/test_scheduled_job_stagger.py b/tests/test_litellm/proxy/common_utils/test_scheduled_job_stagger.py index ca4d62737b6..10c61a5dd87 100644 --- a/tests/test_litellm/proxy/common_utils/test_scheduled_job_stagger.py +++ b/tests/test_litellm/proxy/common_utils/test_scheduled_job_stagger.py @@ -64,6 +64,10 @@ def _stagger(scheduler: AsyncIOScheduler, identity: str = "pod-a:1", **overrides return apply_scheduled_job_stagger(scheduler=scheduler, settings=_settings(**overrides), identity=identity) +def _trigger_of(scheduler: AsyncIOScheduler, job_id: str): + return next(job.trigger for job in scheduler.get_jobs() if job.id == job_id) + + def _fire_times(trigger, start: datetime, steps: int) -> tuple[datetime, ...]: """The fire times APScheduler would produce, each computed from the one before it""" return tuple( @@ -133,22 +137,24 @@ def test_default_cron_is_staggered_and_keeps_its_offset_on_every_later_fire(): applied = _stagger(scheduler) assert applied[PTU_ROLLUP_JOB_ID] > 0 - trigger = next(job.trigger for job in scheduler.get_jobs() if job.id == PTU_ROLLUP_JOB_ID) - fires = _fire_times(trigger, datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc), 3) + start = datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc) + fires = _fire_times(_trigger_of(scheduler, PTU_ROLLUP_JOB_ID), start, 3) expected = timedelta(minutes=15) + timedelta(seconds=applied[PTU_ROLLUP_JOB_ID]) assert [fire - fire.replace(hour=0, minute=0, second=0, microsecond=0) for fire in fires] == [expected] * 3 -async def test_explicit_offset_overrides_the_derived_one_and_zero_pins_a_job(): +def test_explicit_offset_overrides_the_derived_one_and_zero_pins_a_job(): scheduler = _with_jobs(_scheduler()) applied = _stagger(scheduler, offsets={"periodic_reload_job": 0, PTU_ROLLUP_JOB_ID: 7}) - unstaggered = _next_run_times(_with_jobs(_scheduler())) - staggered = _next_run_times(scheduler) assert applied["periodic_reload_job"] == 0 assert applied[PTU_ROLLUP_JOB_ID] == 7 - assert staggered[PTU_ROLLUP_JOB_ID] - unstaggered[PTU_ROLLUP_JOB_ID] == timedelta(seconds=7) + + start = datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc) + staggered = _trigger_of(scheduler, PTU_ROLLUP_JOB_ID) + unstaggered = _trigger_of(_with_jobs(_scheduler()), PTU_ROLLUP_JOB_ID) + assert _fire_times(staggered, start, 1)[0] - _fire_times(unstaggered, start, 1)[0] == timedelta(seconds=7) async def test_disabling_the_stagger_leaves_every_schedule_untouched(): From fdd6f6021642b912c14513d8af91d1c9d65674bd Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 9 Sep 2026 10:01:22 +0000 Subject: [PATCH 5/6] test: count a zombie grandchild as killed in the fake prisma cli Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/test_litellm/proxy/db/conftest.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_litellm/proxy/db/conftest.py b/tests/test_litellm/proxy/db/conftest.py index d3226b0ec50..bcb7794a20d 100644 --- a/tests/test_litellm/proxy/db/conftest.py +++ b/tests/test_litellm/proxy/db/conftest.py @@ -35,6 +35,14 @@ DB_ENV_KEYS = ( _db_env_snapshot_key = pytest.StashKey[dict[str, Optional[str]]]() +def _is_zombie(pid: int) -> bool: + try: + stat: Final = Path(f"/proc/{pid}/stat").read_text() + except OSError: + return False + return stat.rpartition(")")[2].split()[0] == "Z" + + def _db_env_snapshot() -> dict[str, Optional[str]]: return {key: os.environ.get(key) for key in DB_ENV_KEYS} @@ -136,6 +144,8 @@ class FakePrismaCli: os.kill(pid, 0) except ProcessLookupError: return True + if _is_zombie(pid): + return True time.sleep(0.05) return False From da4052dcd77459108894434716509129b79b1d69 Mon Sep 17 00:00:00 2001 From: mateo Date: Fri, 11 Sep 2026 10:18:54 +0000 Subject: [PATCH 6/6] test: give the fake pooler a readiness budget that survives a loaded CI worker Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/test_litellm/proxy/db/test_pgbouncer.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_litellm/proxy/db/test_pgbouncer.py b/tests/test_litellm/proxy/db/test_pgbouncer.py index 7b5a0bf10f9..83b9be8044e 100644 --- a/tests/test_litellm/proxy/db/test_pgbouncer.py +++ b/tests/test_litellm/proxy/db/test_pgbouncer.py @@ -411,7 +411,7 @@ class TestPgBouncerProcess: port=port, socket_path=unix_socket_path(tmp_path, port), restart_delay_seconds=0.1, - ready_timeout_seconds=0.3, + ready_timeout_seconds=3.0, ) assert pooler.start() is None first_pid: Final = pooler.pid @@ -421,7 +421,10 @@ class TestPgBouncerProcess: with caplog.at_level(logging.ERROR, logger=verbose_proxy_logger.name): os.kill(first_pid, signal.SIGKILL) assert _wait_until(lambda: _listening(wrong_port)) - assert _wait_until(lambda: any("did not start listening" in record.message for record in caplog.records)) + assert _wait_until( + lambda: any("did not start listening" in record.message for record in caplog.records), + timeout_seconds=10.0, + ) port_file.write_text(str(port)) assert _wait_until(lambda: _listening(port)) assert _wait_until(lambda: not _listening(wrong_port))