fix(proxy): keep invalidating the remaining spend caches when one delete fails
Some checks failed
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
ryan 2026-09-20 09:22:39 +00:00
parent 7c644e8495
commit a529ddc8a5
2 changed files with 46 additions and 14 deletions

View file

@ -2130,13 +2130,13 @@ class DBSpendUpdateWriter:
user_api_key_cache: Final = proxy_logging_obj.call_details.get("user_api_key_cache")
if user_api_key_cache is None:
return
try:
for project_id in project_ids:
for project_id in project_ids:
try:
await user_api_key_cache.async_delete_cache(key=project_cache_key(project_id))
except Exception as e: # noqa: BLE001 # a stale cache entry must not requeue spend that already committed
verbose_proxy_logger.warning(
"Spend tracking - failed to invalidate %s cache after spend commit: %s", "project", e
)
except Exception as e: # noqa: BLE001 # a stale cache entry must not requeue spend that already committed
verbose_proxy_logger.warning(
"Spend tracking - failed to invalidate %s cache after spend commit: %s", f"project:{project_id}", e
)
@staticmethod
async def _invalidate_team_membership_caches(
@ -2147,17 +2147,19 @@ class DBSpendUpdateWriter:
user_api_key_cache: Final = proxy_logging_obj.call_details.get("user_api_key_cache")
if user_api_key_cache is None:
return
try:
for user_id, team_id in memberships:
for user_id, team_id in memberships:
try:
cache_key = f"team_membership:{user_id}:{team_id}"
await user_api_key_cache.async_delete_cache(key=cache_key)
verbose_proxy_logger.debug(
"Invalidated team membership cache for user_id=%s, team_id=%s", user_id, team_id
)
except Exception as e: # noqa: BLE001 # a stale cache entry must not requeue spend that already committed
verbose_proxy_logger.warning(
"Spend tracking - failed to invalidate %s cache after spend commit: %s", "team membership", e
)
except Exception as e: # noqa: BLE001 # a stale cache entry must not requeue spend that already committed
verbose_proxy_logger.warning(
"Spend tracking - failed to invalidate %s cache after spend commit: %s",
cache_key,
e,
)
@staticmethod
async def _update_entity_spend_in_db(

View file

@ -3311,7 +3311,7 @@ async def test_failed_org_spend_commit_from_redis_restores_only_the_org_rows():
mock_redis_update_buffer.restore_transactions_to_redis.assert_awaited_once()
restored = mock_redis_update_buffer.restore_transactions_to_redis.call_args.kwargs["db_spend_update_transactions"]
assert restored["org_list_transactions"] == {"org-1": 0.5}
assert not restored["team_list_transactions"]
assert restored["team_list_transactions"] is None
db_writer.pod_lock_manager.release_lock.assert_awaited_once()
@ -3358,7 +3358,7 @@ async def test_failed_team_spend_commit_from_redis_still_commits_org_spend():
mock_redis_update_buffer.restore_transactions_to_redis.assert_awaited_once()
restored = mock_redis_update_buffer.restore_transactions_to_redis.call_args.kwargs["db_spend_update_transactions"]
assert restored["team_list_transactions"] == {"team-1": 0.5}
assert not restored["org_list_transactions"]
assert restored["org_list_transactions"] is None
db_writer.pod_lock_manager.release_lock.assert_awaited_once()
@ -3404,6 +3404,36 @@ async def test_cache_invalidation_failure_after_commit_does_not_requeue_the_rows
cache.async_delete_cache.assert_awaited_once_with(key=cache_key)
@pytest.mark.asyncio
async def test_cache_invalidation_failure_for_one_project_does_not_skip_remaining_projects():
db_writer = DBSpendUpdateWriter()
mock_batcher = MagicMock()
mock_transaction = _good_tx(mock_batcher)
mock_prisma = _WindowSpendFakePrisma(_WindowSpendFakeDB())
mock_prisma.db.tx = MagicMock(return_value=mock_transaction)
cache = MagicMock()
cache.async_delete_cache = AsyncMock(side_effect=[RuntimeError("redis down"), None])
proxy_logging_obj = MagicMock()
proxy_logging_obj.call_details = {"user_api_key_cache": cache}
proxy_logging_obj.failure_handler = AsyncMock()
result = await db_writer._commit_spend_updates_to_db_per_table(
prisma_client=mock_prisma,
n_retry_times=0,
proxy_logging_obj=proxy_logging_obj,
db_spend_update_transactions=_empty_spend_transactions(
project_list_transactions={"p1": 0.5, "p2": 0.5}
),
)
assert result is None
assert cache.async_delete_cache.await_count == 2
assert cache.async_delete_cache.await_args_list == [
call(key=project_cache_key("p1")),
call(key=project_cache_key("p2")),
]
@pytest.mark.asyncio
async def test_failed_window_spend_commit_from_redis_is_restored_to_redis():
"""The Redis drain is destructive, so a failed window commit has to push