mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-19 09:53:33 +00:00
refactor(proxy): share db_write_failure_is_requeue_safe from exception_handler so the spend writer stays untouched
This commit is contained in:
parent
f3109d5596
commit
41e25d0f1d
6 changed files with 30 additions and 26 deletions
|
|
@ -37,7 +37,6 @@ from litellm.constants import (
|
|||
from litellm.litellm_core_utils.litellm_logging import coerce_model_access_groups
|
||||
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
|
||||
from litellm.proxy._types import (
|
||||
DB_CONNECTION_ERROR_TYPES,
|
||||
DB_RETRY_SAFE_ERROR_TYPES,
|
||||
BaseDailySpendTransaction,
|
||||
DailyAgentSpendTransaction,
|
||||
|
|
@ -74,7 +73,7 @@ from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import (
|
|||
WindowSpendTransaction,
|
||||
WindowSpendUpdateQueue,
|
||||
)
|
||||
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
|
||||
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler, db_write_failure_is_requeue_safe
|
||||
from litellm.proxy.db.model_usage_rollup import build_model_usage_transaction
|
||||
from litellm.proxy.db.request_error_tracking import request_error_accumulator
|
||||
from litellm.proxy.db.rollup_lock_timeout import apply_rollup_lock_timeout
|
||||
|
|
@ -249,16 +248,6 @@ class _DailySpendCommit(Protocol[_DailySpendTransactionT]):
|
|||
) -> None: ...
|
||||
|
||||
|
||||
_DATA_REJECTED_SQLSTATE_CLASSES: Final = frozenset({"22", "23"})
|
||||
|
||||
|
||||
def spend_commit_failure_is_requeue_safe(e: Exception) -> bool:
|
||||
if isinstance(e, DB_CONNECTION_ERROR_TYPES):
|
||||
return isinstance(e, DB_RETRY_SAFE_ERROR_TYPES)
|
||||
sqlstate: Final = PrismaDBExceptionHandler.postgres_sqlstate(e)
|
||||
return sqlstate is None or sqlstate[:2] not in _DATA_REJECTED_SQLSTATE_CLASSES
|
||||
|
||||
|
||||
_SpendTableName = Literal[
|
||||
"user_list_transactions",
|
||||
"end_user_list_transactions",
|
||||
|
|
@ -296,7 +285,7 @@ def _spend_tables_left_to_send(
|
|||
_SPEND_TABLE_COMMIT_ORDER[len(committed)] if len(committed) < len(_SPEND_TABLE_COMMIT_ORDER) else None
|
||||
)
|
||||
dropped: Final[frozenset[_SpendTableName]] = (
|
||||
frozenset() if in_flight is None or spend_commit_failure_is_requeue_safe(failure) else frozenset({in_flight})
|
||||
frozenset() if in_flight is None or db_write_failure_is_requeue_safe(failure) else frozenset({in_flight})
|
||||
)
|
||||
if dropped and in_flight is not None:
|
||||
spend_log_error(
|
||||
|
|
@ -1770,7 +1759,7 @@ class DBSpendUpdateWriter:
|
|||
window_spend_transactions=window_spend_update_transactions,
|
||||
)
|
||||
except Exception as e:
|
||||
if not spend_commit_failure_is_requeue_safe(e):
|
||||
if not db_write_failure_is_requeue_safe(e):
|
||||
uncommitted.pop("window_spend_update_transactions", None)
|
||||
spend_log_error(
|
||||
"Spend tracking - dropped %d budget window increments: the failed statement may have "
|
||||
|
|
@ -1936,7 +1925,7 @@ class DBSpendUpdateWriter:
|
|||
window_spend_transactions=window_spend_update_transactions,
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 # the increments go back on the queue; the rest of the flush must run
|
||||
if spend_commit_failure_is_requeue_safe(e):
|
||||
if db_write_failure_is_requeue_safe(e):
|
||||
spend_log_error(
|
||||
"Spend tracking - failed to commit budget window spend updates. "
|
||||
"Re-queued %d window increments for retry on next tick. Error: %s",
|
||||
|
|
@ -2096,7 +2085,6 @@ class DBSpendUpdateWriter:
|
|||
"""Retry a failed spend-update transaction on connection errors, deadlocks or a rollup
|
||||
``lock_timeout`` (55P03), else re-raise. All three roll the transaction back before any
|
||||
increment applied, so re-sending the same batch cannot double-count."""
|
||||
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
|
||||
from litellm.proxy.utils import raise_failed_update_spend_exception
|
||||
|
||||
is_retryable = (
|
||||
|
|
@ -2605,7 +2593,7 @@ class DBSpendUpdateWriter:
|
|||
_mark_daily_spend_commit_started()
|
||||
_mark_daily_spend_commit_finished()
|
||||
except Exception as batch_error:
|
||||
if spend_commit_failure_is_requeue_safe(batch_error):
|
||||
if db_write_failure_is_requeue_safe(batch_error):
|
||||
spend_log_error(
|
||||
"Daily %s spend batch upsert failed. Table: %s, Rows: %d, Error: %s",
|
||||
entity_type,
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ from pydantic import TypeAdapter, ValidationError
|
|||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.proxy._types import (
|
||||
DB_CONNECTION_ERROR_TYPES,
|
||||
DB_RETRY_SAFE_ERROR_TYPES,
|
||||
ProxyErrorTypes,
|
||||
ProxyException,
|
||||
)
|
||||
|
|
@ -637,3 +638,14 @@ async def call_with_db_reconnect_retry(
|
|||
# propagate — repeated reconnect-loops are the watchdog's job, not
|
||||
# this helper's.
|
||||
return await coro_factory()
|
||||
|
||||
|
||||
_DATA_REJECTED_SQLSTATE_CLASSES: Final = frozenset({"22", "23"})
|
||||
|
||||
|
||||
def db_write_failure_is_requeue_safe(e: Exception) -> bool:
|
||||
"""Whether a failed write provably never applied, so its rows can be sent again without double counting."""
|
||||
if isinstance(e, DB_CONNECTION_ERROR_TYPES):
|
||||
return isinstance(e, DB_RETRY_SAFE_ERROR_TYPES)
|
||||
sqlstate: Final = PrismaDBExceptionHandler.postgres_sqlstate(e)
|
||||
return sqlstate is None or sqlstate[:2] not in _DATA_REJECTED_SQLSTATE_CLASSES
|
||||
|
|
|
|||
|
|
@ -17,14 +17,13 @@ from litellm._logging import verbose_proxy_logger
|
|||
from litellm.constants import TOOL_DISCOVERY_COMMIT_JOB_NAME
|
||||
from litellm.proxy._types import ToolDiscoveryQueueItem
|
||||
from litellm.proxy.db.db_span import db_span
|
||||
from litellm.proxy.db.db_spend_update_writer import spend_commit_failure_is_requeue_safe
|
||||
from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManager
|
||||
from litellm.proxy.db.db_transaction_queue.redis_update_buffer import RedisUpdateBuffer
|
||||
from litellm.proxy.db.db_transaction_queue.tool_discovery_queue import (
|
||||
ToolDiscoveryQueue,
|
||||
aggregate_tool_discoveries,
|
||||
)
|
||||
from litellm.proxy.db.exception_handler import call_with_db_reconnect_retry
|
||||
from litellm.proxy.db.exception_handler import call_with_db_reconnect_retry, db_write_failure_is_requeue_safe
|
||||
from litellm.proxy.db.tool_table_bulk_upsert import build_tool_bulk_upsert
|
||||
from litellm.repositories.object_permission_repository import ObjectPermissionRepository
|
||||
from litellm.repositories.prisma_protocols import TableActions
|
||||
|
|
@ -172,7 +171,7 @@ async def _commit_tool_discoveries_from_redis(
|
|||
try:
|
||||
await batch_upsert_tools(prisma_client=prisma_client, items=items)
|
||||
except Exception as e: # noqa: BLE001 # the rows go back to redis or are dropped, the scheduler job must not fail
|
||||
if spend_commit_failure_is_requeue_safe(e):
|
||||
if db_write_failure_is_requeue_safe(e):
|
||||
await redis_update_buffer.restore_tool_discoveries_to_redis(items)
|
||||
verbose_proxy_logger.error(
|
||||
"Tool discovery - %d tool rows went back to Redis for the next tick. Error: %s", len(items), str(e)
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
Unit tests for ToolDiscoveryQueue.
|
||||
"""
|
||||
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.proxy.db.db_transaction_queue.tool_discovery_queue import (
|
||||
|
|
|
|||
|
|
@ -19,9 +19,7 @@ from redis.exceptions import DataError
|
|||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm._service_logger import ServiceTypes
|
||||
from litellm.constants import (
|
||||
SPEND_ROLLUP_LOCK_TIMEOUT_MS,
|
||||
)
|
||||
from litellm.constants import SPEND_ROLLUP_LOCK_TIMEOUT_MS
|
||||
from litellm.proxy._types import DailyTagSpendTransaction, Litellm_EntityType, SpendUpdateQueueItem
|
||||
from litellm.proxy.db.db_spend_update_writer import (
|
||||
_TEAM_ADVISORY_LOCK_SQL,
|
||||
|
|
|
|||
|
|
@ -69,7 +69,9 @@ def _make_prisma(
|
|||
prisma.db.litellm_tooltable = MagicMock()
|
||||
prisma.db.litellm_tooltable.upsert = AsyncMock(return_value=upsert_return)
|
||||
prisma.db.execute_raw = AsyncMock(return_value=1)
|
||||
prisma.db.litellm_tooltable.find_many = AsyncMock(return_value=find_many_rows if find_many_rows is not None else [])
|
||||
prisma.db.litellm_tooltable.find_many = AsyncMock(
|
||||
return_value=find_many_rows if find_many_rows is not None else []
|
||||
)
|
||||
prisma.db.litellm_tooltable.find_unique = AsyncMock(return_value=find_unique_row)
|
||||
prisma.db.litellm_verificationtoken = MagicMock()
|
||||
prisma.db.litellm_verificationtoken.find_many = AsyncMock(return_value=list(key_rows))
|
||||
|
|
@ -225,7 +227,9 @@ async def test_get_tool_found():
|
|||
result = await get_tool(prisma, "my_tool")
|
||||
assert result is not None
|
||||
assert result.tool_name == "my_tool"
|
||||
prisma.db.litellm_tooltable.find_unique.assert_awaited_once_with(where={"tool_name": "my_tool"})
|
||||
prisma.db.litellm_tooltable.find_unique.assert_awaited_once_with(
|
||||
where={"tool_name": "my_tool"}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -262,7 +266,9 @@ async def test_update_tool_policy_calls_upsert_then_get_tool():
|
|||
updated_by="admin",
|
||||
)
|
||||
prisma = _make_prisma(find_unique_row=row)
|
||||
result = await update_tool_policy(prisma, "my_tool", updated_by="admin", input_policy="blocked")
|
||||
result = await update_tool_policy(
|
||||
prisma, "my_tool", updated_by="admin", input_policy="blocked"
|
||||
)
|
||||
assert result is not None
|
||||
assert result.input_policy == "blocked"
|
||||
prisma.db.litellm_tooltable.upsert.assert_awaited_once()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue