mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-08 22:21:35 +00:00
5dfe32c889
The proxy CLI's --use_prisma_db_push never gets here; PrismaManager keeps its own db push loop and only delegates when use_migrate is true. The caller this covers is the migrations Job with USE_PRISMA_DB_PUSH=true.
571 lines
20 KiB
Python
571 lines
20 KiB
Python
"""Regression tests for ProxyExtrasDBManager's v2 migration resolver.
|
|
|
|
v2 is the proxy CLI default; v1 stays reachable via the `use_v2_resolver`
|
|
kwarg, which still defaults to False for direct callers.
|
|
"""
|
|
|
|
import subprocess
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
|
|
from litellm_proxy_extras.utils import (
|
|
_PRISMA_ATTEMPTS,
|
|
ProxyExtrasDBManager,
|
|
_max_migration_timestamp,
|
|
_migration_timestamp,
|
|
)
|
|
|
|
|
|
def _fake_migrate_deploy_failure(returncode: int, stderr: str):
|
|
def _run(*args, **kwargs):
|
|
raise subprocess.CalledProcessError(
|
|
returncode=returncode,
|
|
cmd=args[0],
|
|
stderr=stderr,
|
|
output="",
|
|
)
|
|
|
|
return _run
|
|
|
|
|
|
def test_v2_p3018_permission_error_raises_runtime_error(monkeypatch, tmp_path):
|
|
"""v2: a permission failure during migrate deploy raises RuntimeError."""
|
|
monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x")
|
|
monkeypatch.setattr(
|
|
ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None
|
|
)
|
|
monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
|
|
(tmp_path / "schema.prisma").write_text("// stub")
|
|
|
|
stderr = (
|
|
"Error: P3018\nMigration name: 20250326162113_baseline\n"
|
|
"Database error code: 42501\npermission denied for schema public"
|
|
)
|
|
with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)):
|
|
with pytest.raises(RuntimeError, match="permission"):
|
|
ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
|
|
|
|
|
|
def test_v2_non_idempotent_p3009_raises_runtime_error(monkeypatch, tmp_path):
|
|
"""v2: a non-idempotent migration failure raises (no silent recovery)."""
|
|
monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x")
|
|
monkeypatch.setattr(
|
|
ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None
|
|
)
|
|
monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
|
|
(tmp_path / "schema.prisma").write_text("// stub")
|
|
|
|
stderr = (
|
|
"Error: P3009\nMigration `20260101000000_genuinely_broken` failed\n"
|
|
'Reason: syntax error at or near "BRKN" LINE 42'
|
|
)
|
|
with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)):
|
|
with pytest.raises(RuntimeError, match="cannot be auto-recovered"):
|
|
ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
|
|
|
|
|
|
def test_strip_prisma_query_params_removes_connection_limit():
|
|
"""DATABASE_URLs with Prisma-specific params should be parseable by psycopg."""
|
|
url = "postgresql://u:p@h:5432/db?connection_limit=100&pool_timeout=60&sslmode=require"
|
|
stripped = ProxyExtrasDBManager._strip_prisma_query_params(url)
|
|
assert "connection_limit" not in stripped
|
|
assert "pool_timeout" not in stripped
|
|
assert "sslmode=require" in stripped
|
|
|
|
|
|
def test_strip_prisma_query_params_passthrough_no_query():
|
|
"""URLs without query strings are returned unchanged."""
|
|
url = "postgresql://u:p@h:5432/db"
|
|
assert ProxyExtrasDBManager._strip_prisma_query_params(url) == url
|
|
|
|
|
|
def test_migration_timestamp_extracts_leading_digits():
|
|
assert _migration_timestamp("20260101000000_add_foo") == 20260101000000
|
|
assert _migration_timestamp("20250326162113_baseline") == 20250326162113
|
|
|
|
|
|
def test_migration_timestamp_returns_zero_on_malformed():
|
|
assert _migration_timestamp("0_init") == 0
|
|
assert _migration_timestamp("not_a_migration") == 0
|
|
|
|
|
|
def test_max_migration_timestamp():
|
|
names = {"20250326000000_a", "20260415000000_b", "20251115000000_c"}
|
|
assert _max_migration_timestamp(names) == 20260415000000
|
|
|
|
|
|
def test_max_migration_timestamp_empty_set():
|
|
assert _max_migration_timestamp(set()) == 0
|
|
|
|
|
|
def test_v1_default_still_calls_resolve_all_migrations(monkeypatch, tmp_path):
|
|
"""v1 (default) continues to call _resolve_all_migrations on the happy path.
|
|
|
|
This is the existing buggy behavior — we're not fixing it in v1, only
|
|
offering v2 as opt-in. This test pins the default so that a future
|
|
inadvertent default flip is caught.
|
|
"""
|
|
monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
|
|
(tmp_path / "schema.prisma").write_text("// stub")
|
|
|
|
# Stub `prisma migrate deploy` to claim success with pending migrations
|
|
# applied, which is the code path that triggers the legacy post-migration
|
|
# sanity check (a call to _resolve_all_migrations).
|
|
class FakeResult:
|
|
stdout = "Applied migration.\n"
|
|
stderr = ""
|
|
|
|
def fake_run(cmd, *args, **kwargs):
|
|
return FakeResult()
|
|
|
|
resolve_called = {"n": 0}
|
|
|
|
def fake_resolve(*args, **kwargs):
|
|
resolve_called["n"] += 1
|
|
|
|
monkeypatch.setattr("subprocess.run", fake_run)
|
|
monkeypatch.setattr(ProxyExtrasDBManager, "_resolve_all_migrations", fake_resolve)
|
|
|
|
ok = ProxyExtrasDBManager.setup_database(use_migrate=True) # v2 flag NOT set
|
|
assert ok is True
|
|
assert resolve_called["n"] == 1, "v1 default should still invoke the legacy path"
|
|
|
|
|
|
def test_v2_db_push_wraps_subprocess_error_as_runtime_error(monkeypatch, tmp_path):
|
|
"""v2: a failing `prisma db push` must raise RuntimeError, not leak
|
|
CalledProcessError past proxy_cli.py's `except RuntimeError`."""
|
|
monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
|
|
(tmp_path / "schema.prisma").write_text("// stub")
|
|
|
|
stderr = "db push error"
|
|
with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)):
|
|
with pytest.raises(RuntimeError, match="prisma db push failed"):
|
|
ProxyExtrasDBManager.setup_database(use_migrate=False, use_v2_resolver=True)
|
|
|
|
|
|
def test_v2_warn_ahead_of_head_swallows_db_errors(monkeypatch, tmp_path):
|
|
"""_warn_if_db_ahead_of_head must never raise — it's informational.
|
|
|
|
Non-connection DB errors (e.g. InsufficientPrivilege from a user
|
|
without SELECT on _prisma_migrations) must be caught, not propagated.
|
|
"""
|
|
import psycopg
|
|
|
|
monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x")
|
|
monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
|
|
(tmp_path / "schema.prisma").write_text("// stub")
|
|
|
|
class _FakeConn:
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *a):
|
|
return False
|
|
|
|
def execute(self, *a, **kw):
|
|
# Simulate an InsufficientPrivilege (subclass of DatabaseError).
|
|
raise psycopg.errors.InsufficientPrivilege("permission denied")
|
|
|
|
connects = {"n": 0}
|
|
|
|
def _fake_connect(*a, **kw):
|
|
connects["n"] += 1
|
|
return _FakeConn()
|
|
|
|
monkeypatch.setattr("psycopg.connect", _fake_connect)
|
|
|
|
assert ProxyExtrasDBManager._warn_if_db_ahead_of_head(str(tmp_path)) is None
|
|
assert connects["n"] == 1, "the failing query must actually have been reached"
|
|
|
|
|
|
def test_v2_resolve_specific_migration_failure_raises_runtime_error(
|
|
monkeypatch, tmp_path
|
|
):
|
|
"""If marking a migration as applied fails inside P3009 idempotent
|
|
recovery, the subprocess error must be re-raised as RuntimeError so
|
|
proxy_cli.py catches it cleanly (instead of leaking CalledProcessError)."""
|
|
monkeypatch.setattr(
|
|
ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None
|
|
)
|
|
monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
|
|
(tmp_path / "schema.prisma").write_text("// stub")
|
|
monkeypatch.setattr(
|
|
ProxyExtrasDBManager, "_roll_back_migration", lambda *a, **kw: None
|
|
)
|
|
|
|
# First call: migrate deploy -> P3009 idempotent error.
|
|
# Recovery path tries _resolve_specific_migration; that also raises.
|
|
def _failing_resolve(*a, **kw):
|
|
raise subprocess.CalledProcessError(
|
|
returncode=1,
|
|
cmd="prisma migrate resolve --applied",
|
|
stderr="resolve failed",
|
|
output="",
|
|
)
|
|
|
|
monkeypatch.setattr(
|
|
ProxyExtrasDBManager, "_resolve_specific_migration", _failing_resolve
|
|
)
|
|
|
|
stderr = (
|
|
"Error: P3009\nMigration `20260101000000_some_migration` failed\n"
|
|
"relation already exists"
|
|
)
|
|
with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)):
|
|
with pytest.raises(
|
|
RuntimeError, match=r"Failed to mark migration .* as applied"
|
|
):
|
|
ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
|
|
|
|
|
|
def test_v2_does_not_call_resolve_all_migrations(monkeypatch, tmp_path):
|
|
"""v2 must never call _resolve_all_migrations — that's the bug it fixes."""
|
|
monkeypatch.setattr(
|
|
ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None
|
|
)
|
|
monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
|
|
(tmp_path / "schema.prisma").write_text("// stub")
|
|
|
|
class FakeResult:
|
|
stdout = "Applied migration.\n"
|
|
stderr = ""
|
|
|
|
monkeypatch.setattr("subprocess.run", lambda *a, **kw: FakeResult())
|
|
|
|
resolve_called = {"n": 0}
|
|
monkeypatch.setattr(
|
|
ProxyExtrasDBManager,
|
|
"_resolve_all_migrations",
|
|
lambda *a, **kw: resolve_called.__setitem__("n", resolve_called["n"] + 1),
|
|
)
|
|
|
|
ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
|
|
assert ok is True
|
|
assert resolve_called["n"] == 0, "v2 must not invoke the diff-and-force recovery"
|
|
|
|
|
|
_DEADLOCK_STDERR = (
|
|
"Error: ERROR: deadlock detected\n"
|
|
"DETAIL: Process 277 waits for ExclusiveLock on advisory lock "
|
|
"[17556,0,72707369,1]; blocked by process 278.\n"
|
|
"Process 278 waits for ShareLock on virtual transaction 3/1041; "
|
|
"blocked by process 277."
|
|
)
|
|
|
|
|
|
class _DeployApplied:
|
|
stdout = "All migrations have been successfully applied."
|
|
stderr = ""
|
|
returncode = 0
|
|
|
|
|
|
def _deploy_only(deploy_side_effect):
|
|
"""subprocess.run stand-in that only intercepts `prisma migrate deploy`.
|
|
|
|
Scoped by argv so the Prisma toolchain check cannot consume the mock first.
|
|
"""
|
|
deploys = {"n": 0}
|
|
|
|
def _run(*args, **kwargs):
|
|
cmd = args[0] if args else kwargs.get("args", [])
|
|
if list(cmd)[-2:] == ["migrate", "deploy"]:
|
|
deploys["n"] += 1
|
|
return deploy_side_effect(deploys["n"], cmd)
|
|
return _DeployApplied()
|
|
|
|
return _run, deploys
|
|
|
|
|
|
def _prepare_v2_resolver(monkeypatch, tmp_path):
|
|
monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x")
|
|
monkeypatch.setattr(
|
|
ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None
|
|
)
|
|
monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
|
|
(tmp_path / "schema.prisma").write_text("// stub")
|
|
monkeypatch.setattr("time.sleep", lambda *_a, **_k: None)
|
|
|
|
|
|
def test_v2_retries_transient_advisory_lock_deadlock(monkeypatch, tmp_path):
|
|
"""v2: replicas racing `migrate deploy` deadlock on Prisma's advisory
|
|
lock, which is transient and must be retried rather than kill the boot."""
|
|
_prepare_v2_resolver(monkeypatch, tmp_path)
|
|
|
|
def _side_effect(n, cmd):
|
|
if n == 1:
|
|
raise subprocess.CalledProcessError(
|
|
returncode=1, cmd=cmd, stderr=_DEADLOCK_STDERR, output=""
|
|
)
|
|
return _DeployApplied()
|
|
|
|
run, deploys = _deploy_only(_side_effect)
|
|
with patch("subprocess.run", side_effect=run):
|
|
ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
|
|
|
|
assert ok is True
|
|
assert deploys["n"] == 2, "the deadlocked deploy must be retried, not raised"
|
|
|
|
|
|
def test_v2_persistent_advisory_lock_deadlock_eventually_raises(monkeypatch, tmp_path):
|
|
"""v2: the deadlock retry is bounded, so a deadlock that never clears
|
|
still raises instead of looping or reporting success."""
|
|
_prepare_v2_resolver(monkeypatch, tmp_path)
|
|
|
|
def _side_effect(n, cmd):
|
|
raise subprocess.CalledProcessError(
|
|
returncode=1, cmd=cmd, stderr=_DEADLOCK_STDERR, output=""
|
|
)
|
|
|
|
run, deploys = _deploy_only(_side_effect)
|
|
with patch("subprocess.run", side_effect=run):
|
|
with pytest.raises(RuntimeError, match="after 4 attempts"):
|
|
ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
|
|
|
|
assert deploys["n"] == 4
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"stderr",
|
|
[
|
|
"Error: P1001: Can't reach database server at `db`:`5432`",
|
|
"Error: P1002: The database server was reached but timed out.",
|
|
],
|
|
)
|
|
def test_v2_retries_transient_database_connectivity_errors(monkeypatch, tmp_path, stderr):
|
|
"""v2: a database not accepting connections yet is retried, not fatal."""
|
|
_prepare_v2_resolver(monkeypatch, tmp_path)
|
|
|
|
def _side_effect(n, cmd):
|
|
if n == 1:
|
|
raise subprocess.CalledProcessError(
|
|
returncode=1, cmd=cmd, stderr=stderr, output=""
|
|
)
|
|
return _DeployApplied()
|
|
|
|
run, deploys = _deploy_only(_side_effect)
|
|
with patch("subprocess.run", side_effect=run):
|
|
ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
|
|
|
|
assert ok is True
|
|
assert deploys["n"] == 2, "an unreachable database must be retried, not raised"
|
|
|
|
|
|
def test_v2_unreachable_database_still_fails_after_the_retries(monkeypatch, tmp_path):
|
|
"""v2: a genuinely unreachable database still raises once the attempts
|
|
are spent, rather than passing as a successful migration."""
|
|
_prepare_v2_resolver(monkeypatch, tmp_path)
|
|
|
|
def _side_effect(n, cmd):
|
|
raise subprocess.CalledProcessError(
|
|
returncode=1,
|
|
cmd=cmd,
|
|
stderr="Error: P1001: Can't reach database server at `db`:`5432`",
|
|
output="",
|
|
)
|
|
|
|
run, deploys = _deploy_only(_side_effect)
|
|
with patch("subprocess.run", side_effect=run):
|
|
with pytest.raises(RuntimeError, match="after 4 attempts"):
|
|
ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
|
|
|
|
assert deploys["n"] == 4
|
|
|
|
|
|
def test_v2_exhausted_retries_report_the_prisma_error(monkeypatch, tmp_path, caplog):
|
|
"""v2: retrying must not swallow Prisma's stderr, which is captured and is
|
|
the only place the cause appears for an operator or a boot-log grep."""
|
|
_prepare_v2_resolver(monkeypatch, tmp_path)
|
|
stderr = "Error: P1001: Can't reach database server at `wrong`:`5432`"
|
|
|
|
def _side_effect(n, cmd):
|
|
raise subprocess.CalledProcessError(
|
|
returncode=1, cmd=cmd, stderr=stderr, output=""
|
|
)
|
|
|
|
run, _ = _deploy_only(_side_effect)
|
|
with caplog.at_level("INFO", logger="litellm_proxy_extras"):
|
|
with patch("subprocess.run", side_effect=run):
|
|
with pytest.raises(RuntimeError) as exc_info:
|
|
ProxyExtrasDBManager.setup_database(
|
|
use_migrate=True, use_v2_resolver=True
|
|
)
|
|
|
|
assert "P1001" in str(exc_info.value)
|
|
assert "P1001" in caplog.text
|
|
|
|
|
|
def test_v2_db_push_retries_transient_failures(monkeypatch, tmp_path):
|
|
"""v2: `prisma db push` retries a transient failure like v1 did.
|
|
|
|
Reached from the migrations Job (USE_PRISMA_DB_PUSH=true), not from the
|
|
proxy CLI, whose --use_prisma_db_push has its own loop in prisma_client.
|
|
"""
|
|
_prepare_v2_resolver(monkeypatch, tmp_path)
|
|
|
|
pushes = {"n": 0}
|
|
|
|
def _run(*args, **kwargs):
|
|
cmd = list(args[0] if args else kwargs.get("args", []))
|
|
if cmd[-3:] != ["db", "push", "--accept-data-loss"]:
|
|
return _DeployApplied()
|
|
pushes["n"] += 1
|
|
if pushes["n"] == 1:
|
|
raise subprocess.CalledProcessError(
|
|
returncode=1,
|
|
cmd=cmd,
|
|
stderr="Error: P1001: Can't reach database server at `db`:`5432`",
|
|
output="",
|
|
)
|
|
return _DeployApplied()
|
|
|
|
monkeypatch.setattr(
|
|
ProxyExtrasDBManager, "spend_logs_is_partitioned", lambda: False
|
|
)
|
|
with patch("subprocess.run", side_effect=_run):
|
|
ok = ProxyExtrasDBManager.setup_database(use_migrate=False, use_v2_resolver=True)
|
|
|
|
assert ok is True
|
|
assert pushes["n"] == 2
|
|
|
|
|
|
def test_v2_db_push_retries_are_bounded_and_report_the_prisma_error(
|
|
monkeypatch, tmp_path
|
|
):
|
|
"""v2: a database that never comes back stops after _PRISMA_ATTEMPTS and
|
|
surfaces the prisma error, rather than retrying the boot forever."""
|
|
_prepare_v2_resolver(monkeypatch, tmp_path)
|
|
|
|
pushes = {"n": 0}
|
|
|
|
def _run(*args, **kwargs):
|
|
cmd = list(args[0] if args else kwargs.get("args", []))
|
|
if cmd[-3:] != ["db", "push", "--accept-data-loss"]:
|
|
return _DeployApplied()
|
|
pushes["n"] += 1
|
|
raise subprocess.CalledProcessError(
|
|
returncode=1,
|
|
cmd=cmd,
|
|
stderr="Error: P1001: Can't reach database server at `db`:`5432`",
|
|
output="",
|
|
)
|
|
|
|
monkeypatch.setattr(
|
|
ProxyExtrasDBManager, "spend_logs_is_partitioned", lambda: False
|
|
)
|
|
with patch("subprocess.run", side_effect=_run):
|
|
with pytest.raises(RuntimeError) as exc:
|
|
ProxyExtrasDBManager.setup_database(
|
|
use_migrate=False, use_v2_resolver=True
|
|
)
|
|
|
|
assert pushes["n"] == _PRISMA_ATTEMPTS
|
|
assert "P1001" in str(exc.value)
|
|
|
|
|
|
def _db_push_only(push_side_effect):
|
|
"""subprocess.run stand-in that only intercepts `prisma db push`."""
|
|
pushes = {"n": 0}
|
|
|
|
def _run(*args, **kwargs):
|
|
cmd = list(args[0] if args else kwargs.get("args", []))
|
|
if cmd[-3:] != ["db", "push", "--accept-data-loss"]:
|
|
return _DeployApplied()
|
|
pushes["n"] += 1
|
|
return push_side_effect(pushes["n"], cmd)
|
|
|
|
return _run, pushes
|
|
|
|
|
|
def _timed_out_for_real():
|
|
"""Capture what subprocess.run really puts on a TimeoutExpired.
|
|
|
|
Under text=True it still leaves stderr as bytes, unlike CalledProcessError,
|
|
so hardcoding a str here would test a shape production never sees. Derived
|
|
at import, before any test patches subprocess.run.
|
|
"""
|
|
try:
|
|
subprocess.run(
|
|
["sh", "-c", "echo 'Error: P1001 unreachable' >&2; sleep 5"],
|
|
timeout=0.2,
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
except subprocess.TimeoutExpired as e:
|
|
return e
|
|
raise AssertionError("the helper command was supposed to time out")
|
|
|
|
|
|
_TIMEOUT_TEMPLATE = _timed_out_for_real()
|
|
|
|
|
|
def _real_timeout_expired(cmd):
|
|
return subprocess.TimeoutExpired(
|
|
cmd=cmd,
|
|
timeout=_TIMEOUT_TEMPLATE.timeout,
|
|
output=_TIMEOUT_TEMPLATE.stdout,
|
|
stderr=_TIMEOUT_TEMPLATE.stderr,
|
|
)
|
|
|
|
|
|
def test_v2_db_push_retries_a_timeout(monkeypatch, tmp_path):
|
|
"""v2: a `prisma db push` that times out is retried, not turned into a
|
|
TypeError by classifying its bytes stderr as if it were text."""
|
|
_prepare_v2_resolver(monkeypatch, tmp_path)
|
|
|
|
def _side_effect(n, cmd):
|
|
if n == 1:
|
|
raise _real_timeout_expired(cmd)
|
|
return _DeployApplied()
|
|
|
|
monkeypatch.setattr(
|
|
ProxyExtrasDBManager, "spend_logs_is_partitioned", lambda: False
|
|
)
|
|
run, pushes = _db_push_only(_side_effect)
|
|
with patch("subprocess.run", side_effect=run):
|
|
ok = ProxyExtrasDBManager.setup_database(use_migrate=False, use_v2_resolver=True)
|
|
|
|
assert ok is True
|
|
assert pushes["n"] == 2
|
|
|
|
|
|
def test_v2_db_push_timeouts_are_bounded(monkeypatch, tmp_path):
|
|
"""v2: a `prisma db push` that never stops timing out gives up as a
|
|
RuntimeError, which is the only exception proxy_cli.py exits cleanly on."""
|
|
_prepare_v2_resolver(monkeypatch, tmp_path)
|
|
|
|
def _side_effect(n, cmd):
|
|
raise _real_timeout_expired(cmd)
|
|
|
|
monkeypatch.setattr(
|
|
ProxyExtrasDBManager, "spend_logs_is_partitioned", lambda: False
|
|
)
|
|
run, pushes = _db_push_only(_side_effect)
|
|
with patch("subprocess.run", side_effect=run):
|
|
with pytest.raises(RuntimeError, match=r"prisma db push failed after \d+"):
|
|
ProxyExtrasDBManager.setup_database(use_migrate=False, use_v2_resolver=True)
|
|
|
|
assert pushes["n"] == _PRISMA_ATTEMPTS
|
|
|
|
|
|
def test_v2_unclassified_failure_is_not_treated_as_transient(monkeypatch, tmp_path):
|
|
"""v2: an unrecognised deploy failure still raises on the first attempt."""
|
|
_prepare_v2_resolver(monkeypatch, tmp_path)
|
|
|
|
def _side_effect(n, cmd):
|
|
raise subprocess.CalledProcessError(
|
|
returncode=1,
|
|
cmd=cmd,
|
|
stderr="Error: relation \"LiteLLM_SpendLogs\" does not exist",
|
|
output="",
|
|
)
|
|
|
|
run, deploys = _deploy_only(_side_effect)
|
|
with patch("subprocess.run", side_effect=run):
|
|
with pytest.raises(RuntimeError, match="cannot be auto-recovered"):
|
|
ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
|
|
|
|
assert deploys["n"] == 1
|
|
|
|
|