mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge 46f88198ce into 1595adfb2f
This commit is contained in:
commit
62b8cbb485
2 changed files with 113 additions and 24 deletions
|
|
@ -140,6 +140,7 @@ class _MigrateAttemptBudget:
|
|||
|
||||
_SPEND_LOGS_ALTER_RE = re.compile(r'^ALTER\s+TABLE\s+"LiteLLM_SpendLogs"\s', re.IGNORECASE)
|
||||
_SPEND_LOGS_ARTIFACT_DROP_RE = re.compile(r'^DROP\s+TABLE\s+"LiteLLM_SpendLogs_[^"]*"', re.IGNORECASE)
|
||||
_MISSING_RELATION: Final = re.compile(r'(?<!of )relation "[^"]+" does not exist')
|
||||
_SPEND_LOGS_PK_CLAUSE_RE = re.compile(
|
||||
r'^(?:DROP\s+CONSTRAINT\s+"[^"]*_pkey"'
|
||||
r'|ADD\s+(?:CONSTRAINT\s+"[^"]*"\s+)?PRIMARY\s+KEY\s*\([^)]*\))$',
|
||||
|
|
@ -536,12 +537,21 @@ class ProxyExtrasDBManager:
|
|||
env=_get_prisma_env(),
|
||||
)
|
||||
except subprocess.CalledProcessError as e:
|
||||
diff_sql_path.unlink(missing_ok=True)
|
||||
logger.warning(f"Failed to generate migration diff: {e.stderr}")
|
||||
except subprocess.TimeoutExpired:
|
||||
diff_sql_path.unlink(missing_ok=True)
|
||||
logger.warning("Migration diff generation timed out.")
|
||||
|
||||
# check if the migration was created
|
||||
if not diff_sql_path.exists():
|
||||
if not mark_all_applied:
|
||||
logger.warning(
|
||||
"Migration diff was not created (prisma migrate diff failed, likely a pooler URL). "
|
||||
"The ledger already records every migration file, so skipping the drift check "
|
||||
"instead of replaying them"
|
||||
)
|
||||
return
|
||||
logger.warning(
|
||||
"Migration diff was not created (prisma migrate diff failed — "
|
||||
"likely a pooler URL). Falling back to direct SQL execution of "
|
||||
|
|
@ -550,27 +560,12 @@ class ProxyExtrasDBManager:
|
|||
# Fall back: run each migration SQL file directly via prisma db execute.
|
||||
# This works with pooler URLs (no schema introspection needed) and is
|
||||
# safe to re-run because migrations use IF NOT EXISTS / IF EXISTS guards.
|
||||
migration_files = sorted(Path(migrations_dir).glob("*/migration.sql"))
|
||||
for mig_file in migration_files:
|
||||
try:
|
||||
prisma_toolchain.run_prisma(
|
||||
[
|
||||
_get_prisma_command(),
|
||||
"db",
|
||||
"execute",
|
||||
"--file",
|
||||
str(mig_file),
|
||||
"--schema",
|
||||
schema_path,
|
||||
],
|
||||
timeout=prisma_command_timeout(),
|
||||
env=_get_prisma_env(),
|
||||
)
|
||||
logger.info(f"Applied migration: {mig_file.parent.name}")
|
||||
except subprocess.CalledProcessError as e:
|
||||
logger.warning(f"Failed to apply migration {mig_file.parent.name}: {e.stderr}")
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warning(f"Migration {mig_file.parent.name} timed out.")
|
||||
results: Final = tuple(
|
||||
ProxyExtrasDBManager._execute_migration_file(mig_file, schema_path)
|
||||
for mig_file in sorted(Path(migrations_dir).glob("migrations/*/migration.sql"))
|
||||
)
|
||||
if mark_all_applied and all(results):
|
||||
ProxyExtrasDBManager._mark_migrations_applied(migrations_dir)
|
||||
return
|
||||
logger.info(f"Migration diff created at {diff_sql_path}")
|
||||
|
||||
|
|
@ -619,6 +614,34 @@ class ProxyExtrasDBManager:
|
|||
return
|
||||
ProxyExtrasDBManager._mark_migrations_applied(migrations_dir)
|
||||
|
||||
@staticmethod
|
||||
def _execute_migration_file(mig_file: Path, schema_path: str) -> bool:
|
||||
try:
|
||||
prisma_toolchain.run_prisma(
|
||||
[
|
||||
_get_prisma_command(),
|
||||
"db",
|
||||
"execute",
|
||||
"--file",
|
||||
str(mig_file),
|
||||
"--schema",
|
||||
schema_path,
|
||||
],
|
||||
timeout=prisma_command_timeout(),
|
||||
env=_get_prisma_env(),
|
||||
)
|
||||
logger.info(f"Applied migration: {mig_file.parent.name}")
|
||||
return True
|
||||
except subprocess.CalledProcessError as e:
|
||||
stderr: Final = str(e.stderr or "") # pyright: ignore[reportAny] # CalledProcessError.stderr is Any
|
||||
logger.warning(f"Failed to apply migration {mig_file.parent.name}: {stderr}")
|
||||
if _MISSING_RELATION.search(stderr):
|
||||
return False
|
||||
return ProxyExtrasDBManager._is_idempotent_error(stderr)
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warning(f"Migration {mig_file.parent.name} timed out.")
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _mark_migrations_applied(migrations_dir: str) -> None:
|
||||
migration_names = ProxyExtrasDBManager._get_migration_names(migrations_dir)
|
||||
|
|
|
|||
|
|
@ -662,7 +662,16 @@ class _FakeCompleted:
|
|||
|
||||
|
||||
class TestResolveAllMigrationsLedger:
|
||||
def _run(self, monkeypatch, tmp_path, partitioned, execute_fails):
|
||||
def _run(
|
||||
self,
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
partitioned,
|
||||
execute_fails,
|
||||
diff_fails: bool = False,
|
||||
execute_stderr: str = "boom",
|
||||
mark_all_applied: bool = True,
|
||||
):
|
||||
import subprocess as subprocess_module
|
||||
|
||||
import litellm_proxy_extras.utils as utils_module
|
||||
|
|
@ -679,21 +688,31 @@ class TestResolveAllMigrationsLedger:
|
|||
)
|
||||
calls = []
|
||||
|
||||
for migration_name in (
|
||||
"20250326162113_baseline",
|
||||
"20260520120000_add_mcp_env_vars",
|
||||
):
|
||||
migration_dir = tmp_path / "migrations" / migration_name
|
||||
migration_dir.mkdir(parents=True)
|
||||
(migration_dir / "migration.sql").write_text("SELECT 1;")
|
||||
|
||||
def fake_run(cmd, **kwargs):
|
||||
calls.append(cmd)
|
||||
if "diff" in cmd:
|
||||
if diff_fails:
|
||||
raise subprocess_module.CalledProcessError(1, cmd, stderr="P1001")
|
||||
kwargs["stdout"].write(_PARTITIONED_DRIFT_SQL)
|
||||
return _FakeCompleted()
|
||||
if "execute" in cmd:
|
||||
executed_sql = open(cmd[cmd.index("--file") + 1]).read()
|
||||
calls.append(("executed_sql", executed_sql))
|
||||
if execute_fails:
|
||||
raise subprocess_module.CalledProcessError(1, cmd, stderr="boom")
|
||||
raise subprocess_module.CalledProcessError(1, cmd, stderr=execute_stderr)
|
||||
return _FakeCompleted()
|
||||
return _FakeCompleted()
|
||||
|
||||
monkeypatch.setattr(utils_module.prisma_toolchain, "run_prisma", fake_run)
|
||||
ProxyExtrasDBManager._resolve_all_migrations(str(tmp_path), "schema.prisma")
|
||||
ProxyExtrasDBManager._resolve_all_migrations(str(tmp_path), "schema.prisma", mark_all_applied=mark_all_applied)
|
||||
return calls
|
||||
|
||||
def _resolved(self, calls):
|
||||
|
|
@ -723,6 +742,53 @@ class TestResolveAllMigrationsLedger:
|
|||
calls = self._run(monkeypatch, tmp_path, partitioned=False, execute_fails=False)
|
||||
assert self._executed_sql(calls) == _PARTITIONED_DRIFT_SQL
|
||||
|
||||
def test_failed_diff_runs_the_migration_files_directly_and_marks_them_applied(self, monkeypatch, tmp_path):
|
||||
calls = self._run(monkeypatch, tmp_path, partitioned=False, execute_fails=False, diff_fails=True)
|
||||
executed_files = [c[c.index("--file") + 1] for c in calls if isinstance(c, list) and "execute" in c]
|
||||
assert executed_files == [
|
||||
str(tmp_path / "migrations" / "20250326162113_baseline" / "migration.sql"),
|
||||
str(tmp_path / "migrations" / "20260520120000_add_mcp_env_vars" / "migration.sql"),
|
||||
]
|
||||
assert len(self._resolved(calls)) == 1
|
||||
|
||||
def test_failed_diff_with_already_exists_errors_still_marks_migrations_applied(self, monkeypatch, tmp_path):
|
||||
calls = self._run(
|
||||
monkeypatch, tmp_path, partitioned=False, execute_fails=True, diff_fails=True,
|
||||
execute_stderr='ERROR: relation "LiteLLM_MCPUserEnvVars" already exists',
|
||||
)
|
||||
assert len(self._resolved(calls)) == 1
|
||||
|
||||
def test_failed_diff_with_a_real_execute_failure_marks_nothing_applied(self, monkeypatch, tmp_path):
|
||||
calls = self._run(
|
||||
monkeypatch, tmp_path, partitioned=False, execute_fails=True, diff_fails=True,
|
||||
execute_stderr="ERROR: permission denied for schema public",
|
||||
)
|
||||
assert self._resolved(calls) == []
|
||||
|
||||
def test_failed_diff_with_a_missing_table_marks_nothing_applied(self, monkeypatch, tmp_path):
|
||||
calls = self._run(
|
||||
monkeypatch, tmp_path, partitioned=False, execute_fails=True, diff_fails=True,
|
||||
execute_stderr='ERROR: relation "LiteLLM_TeamTable" does not exist',
|
||||
)
|
||||
assert self._resolved(calls) == []
|
||||
|
||||
def test_failed_diff_with_an_already_dropped_constraint_still_marks_migrations_applied(self, monkeypatch, tmp_path):
|
||||
calls = self._run(
|
||||
monkeypatch, tmp_path, partitioned=False, execute_fails=True, diff_fails=True,
|
||||
execute_stderr=(
|
||||
'ERROR: constraint "LiteLLM_TeamMembership_budget_id_fkey" '
|
||||
'of relation "LiteLLM_TeamMembership" does not exist'
|
||||
),
|
||||
)
|
||||
assert len(self._resolved(calls)) == 1
|
||||
|
||||
def test_failed_diff_during_the_post_deploy_sanity_check_replays_nothing(self, monkeypatch, tmp_path):
|
||||
calls = self._run(
|
||||
monkeypatch, tmp_path, partitioned=False, execute_fails=False, diff_fails=True, mark_all_applied=False
|
||||
)
|
||||
assert [c for c in calls if isinstance(c, list) and "execute" in c] == []
|
||||
assert self._resolved(calls) == []
|
||||
|
||||
|
||||
class TestPartitionedSpendLogsPushGuard:
|
||||
def _forbid_subprocess(self, monkeypatch):
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue