From 41e25d0f1df7fbd73bef39aea18a3389253ff2f8 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 24 Oct 2026 19:02:00 +0000 Subject: [PATCH] refactor(proxy): share db_write_failure_is_requeue_safe from exception_handler so the spend writer stays untouched --- litellm/proxy/db/db_spend_update_writer.py | 22 +++++-------------- litellm/proxy/db/exception_handler.py | 12 ++++++++++ litellm/proxy/db/tool_registry_writer.py | 5 ++--- .../test_tool_discovery_queue.py | 1 + .../proxy/db/test_db_spend_update_writer.py | 4 +--- .../proxy/db/test_tool_registry_writer.py | 12 +++++++--- 6 files changed, 30 insertions(+), 26 deletions(-) diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index e2f8863e131..8baeab6b11b 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -37,7 +37,6 @@ from litellm.constants import ( from litellm.litellm_core_utils.litellm_logging import coerce_model_access_groups from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.proxy._types import ( - DB_CONNECTION_ERROR_TYPES, DB_RETRY_SAFE_ERROR_TYPES, BaseDailySpendTransaction, DailyAgentSpendTransaction, @@ -74,7 +73,7 @@ from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import ( WindowSpendTransaction, WindowSpendUpdateQueue, ) -from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler +from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler, db_write_failure_is_requeue_safe from litellm.proxy.db.model_usage_rollup import build_model_usage_transaction from litellm.proxy.db.request_error_tracking import request_error_accumulator from litellm.proxy.db.rollup_lock_timeout import apply_rollup_lock_timeout @@ -249,16 +248,6 @@ class _DailySpendCommit(Protocol[_DailySpendTransactionT]): ) -> None: ... -_DATA_REJECTED_SQLSTATE_CLASSES: Final = frozenset({"22", "23"}) - - -def spend_commit_failure_is_requeue_safe(e: Exception) -> bool: - if isinstance(e, DB_CONNECTION_ERROR_TYPES): - return isinstance(e, DB_RETRY_SAFE_ERROR_TYPES) - sqlstate: Final = PrismaDBExceptionHandler.postgres_sqlstate(e) - return sqlstate is None or sqlstate[:2] not in _DATA_REJECTED_SQLSTATE_CLASSES - - _SpendTableName = Literal[ "user_list_transactions", "end_user_list_transactions", @@ -296,7 +285,7 @@ def _spend_tables_left_to_send( _SPEND_TABLE_COMMIT_ORDER[len(committed)] if len(committed) < len(_SPEND_TABLE_COMMIT_ORDER) else None ) dropped: Final[frozenset[_SpendTableName]] = ( - frozenset() if in_flight is None or spend_commit_failure_is_requeue_safe(failure) else frozenset({in_flight}) + frozenset() if in_flight is None or db_write_failure_is_requeue_safe(failure) else frozenset({in_flight}) ) if dropped and in_flight is not None: spend_log_error( @@ -1770,7 +1759,7 @@ class DBSpendUpdateWriter: window_spend_transactions=window_spend_update_transactions, ) except Exception as e: - if not spend_commit_failure_is_requeue_safe(e): + if not db_write_failure_is_requeue_safe(e): uncommitted.pop("window_spend_update_transactions", None) spend_log_error( "Spend tracking - dropped %d budget window increments: the failed statement may have " @@ -1936,7 +1925,7 @@ class DBSpendUpdateWriter: window_spend_transactions=window_spend_update_transactions, ) except Exception as e: # noqa: BLE001 # the increments go back on the queue; the rest of the flush must run - if spend_commit_failure_is_requeue_safe(e): + if db_write_failure_is_requeue_safe(e): spend_log_error( "Spend tracking - failed to commit budget window spend updates. " "Re-queued %d window increments for retry on next tick. Error: %s", @@ -2096,7 +2085,6 @@ class DBSpendUpdateWriter: """Retry a failed spend-update transaction on connection errors, deadlocks or a rollup ``lock_timeout`` (55P03), else re-raise. All three roll the transaction back before any increment applied, so re-sending the same batch cannot double-count.""" - from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler from litellm.proxy.utils import raise_failed_update_spend_exception is_retryable = ( @@ -2605,7 +2593,7 @@ class DBSpendUpdateWriter: _mark_daily_spend_commit_started() _mark_daily_spend_commit_finished() except Exception as batch_error: - if spend_commit_failure_is_requeue_safe(batch_error): + if db_write_failure_is_requeue_safe(batch_error): spend_log_error( "Daily %s spend batch upsert failed. Table: %s, Rows: %d, Error: %s", entity_type, diff --git a/litellm/proxy/db/exception_handler.py b/litellm/proxy/db/exception_handler.py index 379e0012ea8..5dfcaa4b2f8 100644 --- a/litellm/proxy/db/exception_handler.py +++ b/litellm/proxy/db/exception_handler.py @@ -8,6 +8,7 @@ from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_proxy_logger from litellm.proxy._types import ( DB_CONNECTION_ERROR_TYPES, + DB_RETRY_SAFE_ERROR_TYPES, ProxyErrorTypes, ProxyException, ) @@ -637,3 +638,14 @@ async def call_with_db_reconnect_retry( # propagate — repeated reconnect-loops are the watchdog's job, not # this helper's. return await coro_factory() + + +_DATA_REJECTED_SQLSTATE_CLASSES: Final = frozenset({"22", "23"}) + + +def db_write_failure_is_requeue_safe(e: Exception) -> bool: + """Whether a failed write provably never applied, so its rows can be sent again without double counting.""" + if isinstance(e, DB_CONNECTION_ERROR_TYPES): + return isinstance(e, DB_RETRY_SAFE_ERROR_TYPES) + sqlstate: Final = PrismaDBExceptionHandler.postgres_sqlstate(e) + return sqlstate is None or sqlstate[:2] not in _DATA_REJECTED_SQLSTATE_CLASSES diff --git a/litellm/proxy/db/tool_registry_writer.py b/litellm/proxy/db/tool_registry_writer.py index 42f9e812269..f7ec88945a1 100644 --- a/litellm/proxy/db/tool_registry_writer.py +++ b/litellm/proxy/db/tool_registry_writer.py @@ -17,14 +17,13 @@ from litellm._logging import verbose_proxy_logger from litellm.constants import TOOL_DISCOVERY_COMMIT_JOB_NAME from litellm.proxy._types import ToolDiscoveryQueueItem from litellm.proxy.db.db_span import db_span -from litellm.proxy.db.db_spend_update_writer import spend_commit_failure_is_requeue_safe from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManager from litellm.proxy.db.db_transaction_queue.redis_update_buffer import RedisUpdateBuffer from litellm.proxy.db.db_transaction_queue.tool_discovery_queue import ( ToolDiscoveryQueue, aggregate_tool_discoveries, ) -from litellm.proxy.db.exception_handler import call_with_db_reconnect_retry +from litellm.proxy.db.exception_handler import call_with_db_reconnect_retry, db_write_failure_is_requeue_safe from litellm.proxy.db.tool_table_bulk_upsert import build_tool_bulk_upsert from litellm.repositories.object_permission_repository import ObjectPermissionRepository from litellm.repositories.prisma_protocols import TableActions @@ -172,7 +171,7 @@ async def _commit_tool_discoveries_from_redis( try: await batch_upsert_tools(prisma_client=prisma_client, items=items) except Exception as e: # noqa: BLE001 # the rows go back to redis or are dropped, the scheduler job must not fail - if spend_commit_failure_is_requeue_safe(e): + if db_write_failure_is_requeue_safe(e): await redis_update_buffer.restore_tool_discoveries_to_redis(items) verbose_proxy_logger.error( "Tool discovery - %d tool rows went back to Redis for the next tick. Error: %s", len(items), str(e) diff --git a/tests/unit/proxy/db/db_transaction_queue/test_tool_discovery_queue.py b/tests/unit/proxy/db/db_transaction_queue/test_tool_discovery_queue.py index 25786df069a..0a015255f2e 100644 --- a/tests/unit/proxy/db/db_transaction_queue/test_tool_discovery_queue.py +++ b/tests/unit/proxy/db/db_transaction_queue/test_tool_discovery_queue.py @@ -2,6 +2,7 @@ Unit tests for ToolDiscoveryQueue. """ + import pytest from litellm.proxy.db.db_transaction_queue.tool_discovery_queue import ( diff --git a/tests/unit/proxy/db/test_db_spend_update_writer.py b/tests/unit/proxy/db/test_db_spend_update_writer.py index 68621ed01fc..9194e3aa24b 100644 --- a/tests/unit/proxy/db/test_db_spend_update_writer.py +++ b/tests/unit/proxy/db/test_db_spend_update_writer.py @@ -19,9 +19,7 @@ from redis.exceptions import DataError import litellm from litellm._logging import verbose_proxy_logger from litellm._service_logger import ServiceTypes -from litellm.constants import ( - SPEND_ROLLUP_LOCK_TIMEOUT_MS, -) +from litellm.constants import SPEND_ROLLUP_LOCK_TIMEOUT_MS from litellm.proxy._types import DailyTagSpendTransaction, Litellm_EntityType, SpendUpdateQueueItem from litellm.proxy.db.db_spend_update_writer import ( _TEAM_ADVISORY_LOCK_SQL, diff --git a/tests/unit/proxy/db/test_tool_registry_writer.py b/tests/unit/proxy/db/test_tool_registry_writer.py index 9c65498e74f..76898b0eeab 100644 --- a/tests/unit/proxy/db/test_tool_registry_writer.py +++ b/tests/unit/proxy/db/test_tool_registry_writer.py @@ -69,7 +69,9 @@ def _make_prisma( prisma.db.litellm_tooltable = MagicMock() prisma.db.litellm_tooltable.upsert = AsyncMock(return_value=upsert_return) prisma.db.execute_raw = AsyncMock(return_value=1) - prisma.db.litellm_tooltable.find_many = AsyncMock(return_value=find_many_rows if find_many_rows is not None else []) + prisma.db.litellm_tooltable.find_many = AsyncMock( + return_value=find_many_rows if find_many_rows is not None else [] + ) prisma.db.litellm_tooltable.find_unique = AsyncMock(return_value=find_unique_row) prisma.db.litellm_verificationtoken = MagicMock() prisma.db.litellm_verificationtoken.find_many = AsyncMock(return_value=list(key_rows)) @@ -225,7 +227,9 @@ async def test_get_tool_found(): result = await get_tool(prisma, "my_tool") assert result is not None assert result.tool_name == "my_tool" - prisma.db.litellm_tooltable.find_unique.assert_awaited_once_with(where={"tool_name": "my_tool"}) + prisma.db.litellm_tooltable.find_unique.assert_awaited_once_with( + where={"tool_name": "my_tool"} + ) @pytest.mark.asyncio @@ -262,7 +266,9 @@ async def test_update_tool_policy_calls_upsert_then_get_tool(): updated_by="admin", ) prisma = _make_prisma(find_unique_row=row) - result = await update_tool_policy(prisma, "my_tool", updated_by="admin", input_policy="blocked") + result = await update_tool_policy( + prisma, "my_tool", updated_by="admin", input_policy="blocked" + ) assert result is not None assert result.input_policy == "blocked" prisma.db.litellm_tooltable.upsert.assert_awaited_once()