diff --git a/litellm-proxy-extras/litellm_proxy_extras/utils.py b/litellm-proxy-extras/litellm_proxy_extras/utils.py index a31016e0f17..f751c4087eb 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/utils.py +++ b/litellm-proxy-extras/litellm_proxy_extras/utils.py @@ -732,9 +732,6 @@ class ProxyExtrasDBManager: stderr, ) time.sleep(random.randrange(5, 15)) - raise RuntimeError( - f"prisma db push failed after {_PRISMA_ATTEMPTS} attempts." - ) finally: os.chdir(original_dir) diff --git a/tests/litellm-proxy-extras/test_setup_database_fail_fast.py b/tests/litellm-proxy-extras/test_setup_database_fail_fast.py index 964355492a1..ed82037590a 100644 --- a/tests/litellm-proxy-extras/test_setup_database_fail_fast.py +++ b/tests/litellm-proxy-extras/test_setup_database_fail_fast.py @@ -10,6 +10,7 @@ from unittest.mock import patch import pytest from litellm_proxy_extras.utils import ( + _PRISMA_ATTEMPTS, ProxyExtrasDBManager, _max_migration_timestamp, _migration_timestamp, @@ -425,6 +426,40 @@ def test_v2_db_push_retries_transient_failures(monkeypatch, tmp_path): assert pushes["n"] == 2 +def test_v2_db_push_retries_are_bounded_and_report_the_prisma_error( + monkeypatch, tmp_path +): + """v2: a database that never comes back stops after _PRISMA_ATTEMPTS and + surfaces the prisma error, rather than retrying the boot forever.""" + _prepare_v2_resolver(monkeypatch, tmp_path) + + pushes = {"n": 0} + + def _run(*args, **kwargs): + cmd = list(args[0] if args else kwargs.get("args", [])) + if cmd[-3:] != ["db", "push", "--accept-data-loss"]: + return _DeployApplied() + pushes["n"] += 1 + raise subprocess.CalledProcessError( + returncode=1, + cmd=cmd, + stderr="Error: P1001: Can't reach database server at `db`:`5432`", + output="", + ) + + monkeypatch.setattr( + ProxyExtrasDBManager, "spend_logs_is_partitioned", lambda: False + ) + with patch("subprocess.run", side_effect=_run): + with pytest.raises(RuntimeError) as exc: + ProxyExtrasDBManager.setup_database( + use_migrate=False, use_v2_resolver=True + ) + + assert pushes["n"] == _PRISMA_ATTEMPTS + assert "P1001" in str(exc.value) + + def test_v2_unclassified_failure_is_not_treated_as_transient(monkeypatch, tmp_path): """v2: an unrecognised deploy failure still raises on the first attempt.""" _prepare_v2_resolver(monkeypatch, tmp_path)