From 6b3e30e6601ac4fb285e97525061ccdd93aaac16 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 28 Aug 2026 01:56:15 -0700 Subject: [PATCH] fix(proxy-extras): bound the db push retries in the reachable branch The retry loop already raises on the final attempt, so the raise that followed the loop could never run. Drop it and cover the exhaustion path with a test that pins the attempt count and keeps the prisma error in the message, which is the only thing that tells an operator why the boot stopped. --- .../litellm_proxy_extras/utils.py | 3 -- .../test_setup_database_fail_fast.py | 35 +++++++++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/litellm-proxy-extras/litellm_proxy_extras/utils.py b/litellm-proxy-extras/litellm_proxy_extras/utils.py index a31016e0f17..f751c4087eb 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/utils.py +++ b/litellm-proxy-extras/litellm_proxy_extras/utils.py @@ -732,9 +732,6 @@ class ProxyExtrasDBManager: stderr, ) time.sleep(random.randrange(5, 15)) - raise RuntimeError( - f"prisma db push failed after {_PRISMA_ATTEMPTS} attempts." - ) finally: os.chdir(original_dir) diff --git a/tests/litellm-proxy-extras/test_setup_database_fail_fast.py b/tests/litellm-proxy-extras/test_setup_database_fail_fast.py index 964355492a1..ed82037590a 100644 --- a/tests/litellm-proxy-extras/test_setup_database_fail_fast.py +++ b/tests/litellm-proxy-extras/test_setup_database_fail_fast.py @@ -10,6 +10,7 @@ from unittest.mock import patch import pytest from litellm_proxy_extras.utils import ( + _PRISMA_ATTEMPTS, ProxyExtrasDBManager, _max_migration_timestamp, _migration_timestamp, @@ -425,6 +426,40 @@ def test_v2_db_push_retries_transient_failures(monkeypatch, tmp_path): assert pushes["n"] == 2 +def test_v2_db_push_retries_are_bounded_and_report_the_prisma_error( + monkeypatch, tmp_path +): + """v2: a database that never comes back stops after _PRISMA_ATTEMPTS and + surfaces the prisma error, rather than retrying the boot forever.""" + _prepare_v2_resolver(monkeypatch, tmp_path) + + pushes = {"n": 0} + + def _run(*args, **kwargs): + cmd = list(args[0] if args else kwargs.get("args", [])) + if cmd[-3:] != ["db", "push", "--accept-data-loss"]: + return _DeployApplied() + pushes["n"] += 1 + raise subprocess.CalledProcessError( + returncode=1, + cmd=cmd, + stderr="Error: P1001: Can't reach database server at `db`:`5432`", + output="", + ) + + monkeypatch.setattr( + ProxyExtrasDBManager, "spend_logs_is_partitioned", lambda: False + ) + with patch("subprocess.run", side_effect=_run): + with pytest.raises(RuntimeError) as exc: + ProxyExtrasDBManager.setup_database( + use_migrate=False, use_v2_resolver=True + ) + + assert pushes["n"] == _PRISMA_ATTEMPTS + assert "P1001" in str(exc.value) + + def test_v2_unclassified_failure_is_not_treated_as_transient(monkeypatch, tmp_path): """v2: an unrecognised deploy failure still raises on the first attempt.""" _prepare_v2_resolver(monkeypatch, tmp_path)