mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(proxy): keep a failed cache invalidation from requeueing committed spend
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
ddc527ddf2
commit
48f86834c3
2 changed files with 72 additions and 13 deletions
|
|
@ -1898,17 +1898,10 @@ class DBSpendUpdateWriter:
|
|||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
|
||||
# Invalidate cache for updated team memberships
|
||||
# This ensures budget checks read fresh spend data from the database
|
||||
if team_memberships_to_invalidate and proxy_logging_obj is not None:
|
||||
user_api_key_cache: Final = proxy_logging_obj.call_details.get("user_api_key_cache")
|
||||
if user_api_key_cache is not None:
|
||||
for user_id, team_id in team_memberships_to_invalidate:
|
||||
cache_key = f"team_membership:{user_id}:{team_id}"
|
||||
await user_api_key_cache.async_delete_cache(key=cache_key)
|
||||
verbose_proxy_logger.debug(
|
||||
"Invalidated team membership cache for user_id=%s, team_id=%s", user_id, team_id
|
||||
)
|
||||
await DBSpendUpdateWriter._invalidate_team_membership_caches(
|
||||
memberships=team_memberships_to_invalidate,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
|
||||
### UPDATE ORG TABLE ###
|
||||
org_list_transactions: Final = db_spend_update_transactions["org_list_transactions"]
|
||||
|
|
@ -2019,8 +2012,34 @@ class DBSpendUpdateWriter:
|
|||
user_api_key_cache: Final = proxy_logging_obj.call_details.get("user_api_key_cache")
|
||||
if user_api_key_cache is None:
|
||||
return
|
||||
for project_id in project_ids:
|
||||
await user_api_key_cache.async_delete_cache(key=project_cache_key(project_id))
|
||||
try:
|
||||
for project_id in project_ids:
|
||||
await user_api_key_cache.async_delete_cache(key=project_cache_key(project_id))
|
||||
except Exception as e: # noqa: BLE001 # a stale cache entry must not requeue spend that already committed
|
||||
verbose_proxy_logger.warning(
|
||||
"Spend tracking - failed to invalidate %s cache after spend commit: %s", "project", e
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
async def _invalidate_team_membership_caches(
|
||||
memberships: Sequence[tuple[str, str]], proxy_logging_obj: ProxyLogging | None
|
||||
) -> None:
|
||||
if not memberships or proxy_logging_obj is None:
|
||||
return
|
||||
user_api_key_cache: Final = proxy_logging_obj.call_details.get("user_api_key_cache")
|
||||
if user_api_key_cache is None:
|
||||
return
|
||||
try:
|
||||
for user_id, team_id in memberships:
|
||||
cache_key = f"team_membership:{user_id}:{team_id}"
|
||||
await user_api_key_cache.async_delete_cache(key=cache_key)
|
||||
verbose_proxy_logger.debug(
|
||||
"Invalidated team membership cache for user_id=%s, team_id=%s", user_id, team_id
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 # a stale cache entry must not requeue spend that already committed
|
||||
verbose_proxy_logger.warning(
|
||||
"Spend tracking - failed to invalidate %s cache after spend commit: %s", "team membership", e
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
async def _update_entity_spend_in_db(
|
||||
|
|
|
|||
|
|
@ -3320,6 +3320,46 @@ async def test_failed_team_spend_commit_from_redis_still_commits_org_spend():
|
|||
db_writer.pod_lock_manager.release_lock.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cache_invalidation_failure_after_team_member_commit_does_not_requeue_the_rows():
|
||||
"""A cache outage after a team-member commit must not retry the already committed spend."""
|
||||
db_writer = DBSpendUpdateWriter()
|
||||
await db_writer.spend_update_queue.add_update(
|
||||
{
|
||||
"entity_type": Litellm_EntityType.TEAM_MEMBER,
|
||||
"entity_id": "team_id::team-1::user_id::user-1",
|
||||
"response_cost": 0.5,
|
||||
}
|
||||
)
|
||||
await db_writer.spend_update_queue.add_update(
|
||||
{"entity_type": Litellm_EntityType.ORGANIZATION, "entity_id": "org-1", "response_cost": 0.5}
|
||||
)
|
||||
mock_batcher = MagicMock()
|
||||
mock_transaction = _good_tx(mock_batcher)
|
||||
mock_prisma = _WindowSpendFakePrisma(_WindowSpendFakeDB())
|
||||
mock_prisma.db.tx = MagicMock(return_value=mock_transaction)
|
||||
cache = MagicMock()
|
||||
cache.async_delete_cache = AsyncMock(side_effect=RuntimeError("redis down"))
|
||||
proxy_logging_obj = MagicMock()
|
||||
proxy_logging_obj.call_details = {"user_api_key_cache": cache}
|
||||
proxy_logging_obj.failure_handler = AsyncMock()
|
||||
db_writer._flush_tool_discovery_queue = AsyncMock()
|
||||
|
||||
await db_writer._commit_spend_updates_to_db_without_redis_buffer(
|
||||
prisma_client=mock_prisma,
|
||||
n_retry_times=0,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
|
||||
assert mock_transaction.execute_raw.await_count == 2
|
||||
mock_batcher.litellm_organizationtable.update_many.assert_called_once_with(
|
||||
where={"organization_id": "org-1"},
|
||||
data={"spend": {"increment": 0.5}},
|
||||
)
|
||||
cache.async_delete_cache.assert_awaited_once_with(key="team_membership:user-1:team-1")
|
||||
assert db_writer.spend_update_queue.update_queue.empty()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failed_window_spend_commit_from_redis_is_restored_to_redis():
|
||||
"""The Redis drain is destructive, so a failed window commit has to push
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue