From a529ddc8a5e71a6f4d6d9d3881572e5ab9ecf224 Mon Sep 17 00:00:00 2001 From: ryan Date: Sun, 20 Sep 2026 09:22:39 +0000 Subject: [PATCH] fix(proxy): keep invalidating the remaining spend caches when one delete fails Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/db/db_spend_update_writer.py | 26 +++++++------- .../proxy/db/test_db_spend_update_writer.py | 34 +++++++++++++++++-- 2 files changed, 46 insertions(+), 14 deletions(-) diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 4582d0c9cb7..b7b3afe8246 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -2130,13 +2130,13 @@ class DBSpendUpdateWriter: user_api_key_cache: Final = proxy_logging_obj.call_details.get("user_api_key_cache") if user_api_key_cache is None: return - try: - for project_id in project_ids: + for project_id in project_ids: + try: await user_api_key_cache.async_delete_cache(key=project_cache_key(project_id)) - except Exception as e: # noqa: BLE001 # a stale cache entry must not requeue spend that already committed - verbose_proxy_logger.warning( - "Spend tracking - failed to invalidate %s cache after spend commit: %s", "project", e - ) + except Exception as e: # noqa: BLE001 # a stale cache entry must not requeue spend that already committed + verbose_proxy_logger.warning( + "Spend tracking - failed to invalidate %s cache after spend commit: %s", f"project:{project_id}", e + ) @staticmethod async def _invalidate_team_membership_caches( @@ -2147,17 +2147,19 @@ class DBSpendUpdateWriter: user_api_key_cache: Final = proxy_logging_obj.call_details.get("user_api_key_cache") if user_api_key_cache is None: return - try: - for user_id, team_id in memberships: + for user_id, team_id in memberships: + try: cache_key = f"team_membership:{user_id}:{team_id}" await user_api_key_cache.async_delete_cache(key=cache_key) verbose_proxy_logger.debug( "Invalidated team membership cache for user_id=%s, team_id=%s", user_id, team_id ) - except Exception as e: # noqa: BLE001 # a stale cache entry must not requeue spend that already committed - verbose_proxy_logger.warning( - "Spend tracking - failed to invalidate %s cache after spend commit: %s", "team membership", e - ) + except Exception as e: # noqa: BLE001 # a stale cache entry must not requeue spend that already committed + verbose_proxy_logger.warning( + "Spend tracking - failed to invalidate %s cache after spend commit: %s", + cache_key, + e, + ) @staticmethod async def _update_entity_spend_in_db( diff --git a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py index 8f8085a55ae..c546d230137 100644 --- a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py +++ b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py @@ -3311,7 +3311,7 @@ async def test_failed_org_spend_commit_from_redis_restores_only_the_org_rows(): mock_redis_update_buffer.restore_transactions_to_redis.assert_awaited_once() restored = mock_redis_update_buffer.restore_transactions_to_redis.call_args.kwargs["db_spend_update_transactions"] assert restored["org_list_transactions"] == {"org-1": 0.5} - assert not restored["team_list_transactions"] + assert restored["team_list_transactions"] is None db_writer.pod_lock_manager.release_lock.assert_awaited_once() @@ -3358,7 +3358,7 @@ async def test_failed_team_spend_commit_from_redis_still_commits_org_spend(): mock_redis_update_buffer.restore_transactions_to_redis.assert_awaited_once() restored = mock_redis_update_buffer.restore_transactions_to_redis.call_args.kwargs["db_spend_update_transactions"] assert restored["team_list_transactions"] == {"team-1": 0.5} - assert not restored["org_list_transactions"] + assert restored["org_list_transactions"] is None db_writer.pod_lock_manager.release_lock.assert_awaited_once() @@ -3404,6 +3404,36 @@ async def test_cache_invalidation_failure_after_commit_does_not_requeue_the_rows cache.async_delete_cache.assert_awaited_once_with(key=cache_key) +@pytest.mark.asyncio +async def test_cache_invalidation_failure_for_one_project_does_not_skip_remaining_projects(): + db_writer = DBSpendUpdateWriter() + mock_batcher = MagicMock() + mock_transaction = _good_tx(mock_batcher) + mock_prisma = _WindowSpendFakePrisma(_WindowSpendFakeDB()) + mock_prisma.db.tx = MagicMock(return_value=mock_transaction) + cache = MagicMock() + cache.async_delete_cache = AsyncMock(side_effect=[RuntimeError("redis down"), None]) + proxy_logging_obj = MagicMock() + proxy_logging_obj.call_details = {"user_api_key_cache": cache} + proxy_logging_obj.failure_handler = AsyncMock() + + result = await db_writer._commit_spend_updates_to_db_per_table( + prisma_client=mock_prisma, + n_retry_times=0, + proxy_logging_obj=proxy_logging_obj, + db_spend_update_transactions=_empty_spend_transactions( + project_list_transactions={"p1": 0.5, "p2": 0.5} + ), + ) + + assert result is None + assert cache.async_delete_cache.await_count == 2 + assert cache.async_delete_cache.await_args_list == [ + call(key=project_cache_key("p1")), + call(key=project_cache_key("p2")), + ] + + @pytest.mark.asyncio async def test_failed_window_spend_commit_from_redis_is_restored_to_redis(): """The Redis drain is destructive, so a failed window commit has to push