diff --git a/litellm-proxy-extras/litellm_proxy_extras/utils.py b/litellm-proxy-extras/litellm_proxy_extras/utils.py index 8b19576d86e..b2d66525138 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/utils.py +++ b/litellm-proxy-extras/litellm_proxy_extras/utils.py @@ -140,6 +140,7 @@ class _MigrateAttemptBudget: _SPEND_LOGS_ALTER_RE = re.compile(r'^ALTER\s+TABLE\s+"LiteLLM_SpendLogs"\s', re.IGNORECASE) _SPEND_LOGS_ARTIFACT_DROP_RE = re.compile(r'^DROP\s+TABLE\s+"LiteLLM_SpendLogs_[^"]*"', re.IGNORECASE) +_MISSING_RELATION: Final = re.compile(r'(? bool: + try: + prisma_toolchain.run_prisma( + [ + _get_prisma_command(), + "db", + "execute", + "--file", + str(mig_file), + "--schema", + schema_path, + ], + timeout=prisma_command_timeout(), + env=_get_prisma_env(), + ) + logger.info(f"Applied migration: {mig_file.parent.name}") + return True + except subprocess.CalledProcessError as e: + stderr: Final = str(e.stderr or "") # pyright: ignore[reportAny] # CalledProcessError.stderr is Any + logger.warning(f"Failed to apply migration {mig_file.parent.name}: {stderr}") + if _MISSING_RELATION.search(stderr): + return False + return ProxyExtrasDBManager._is_idempotent_error(stderr) + except subprocess.TimeoutExpired: + logger.warning(f"Migration {mig_file.parent.name} timed out.") + return False + @staticmethod def _mark_migrations_applied(migrations_dir: str) -> None: migration_names = ProxyExtrasDBManager._get_migration_names(migrations_dir) diff --git a/tests/unit/litellm_proxy_extras/test_litellm_proxy_extras_utils.py b/tests/unit/litellm_proxy_extras/test_litellm_proxy_extras_utils.py index 02bdf3429ea..f3d215f81db 100644 --- a/tests/unit/litellm_proxy_extras/test_litellm_proxy_extras_utils.py +++ b/tests/unit/litellm_proxy_extras/test_litellm_proxy_extras_utils.py @@ -662,7 +662,16 @@ class _FakeCompleted: class TestResolveAllMigrationsLedger: - def _run(self, monkeypatch, tmp_path, partitioned, execute_fails): + def _run( + self, + monkeypatch, + tmp_path, + partitioned, + execute_fails, + diff_fails: bool = False, + execute_stderr: str = "boom", + mark_all_applied: bool = True, + ): import subprocess as subprocess_module import litellm_proxy_extras.utils as utils_module @@ -679,21 +688,31 @@ class TestResolveAllMigrationsLedger: ) calls = [] + for migration_name in ( + "20250326162113_baseline", + "20260520120000_add_mcp_env_vars", + ): + migration_dir = tmp_path / "migrations" / migration_name + migration_dir.mkdir(parents=True) + (migration_dir / "migration.sql").write_text("SELECT 1;") + def fake_run(cmd, **kwargs): calls.append(cmd) if "diff" in cmd: + if diff_fails: + raise subprocess_module.CalledProcessError(1, cmd, stderr="P1001") kwargs["stdout"].write(_PARTITIONED_DRIFT_SQL) return _FakeCompleted() if "execute" in cmd: executed_sql = open(cmd[cmd.index("--file") + 1]).read() calls.append(("executed_sql", executed_sql)) if execute_fails: - raise subprocess_module.CalledProcessError(1, cmd, stderr="boom") + raise subprocess_module.CalledProcessError(1, cmd, stderr=execute_stderr) return _FakeCompleted() return _FakeCompleted() monkeypatch.setattr(utils_module.prisma_toolchain, "run_prisma", fake_run) - ProxyExtrasDBManager._resolve_all_migrations(str(tmp_path), "schema.prisma") + ProxyExtrasDBManager._resolve_all_migrations(str(tmp_path), "schema.prisma", mark_all_applied=mark_all_applied) return calls def _resolved(self, calls): @@ -723,6 +742,53 @@ class TestResolveAllMigrationsLedger: calls = self._run(monkeypatch, tmp_path, partitioned=False, execute_fails=False) assert self._executed_sql(calls) == _PARTITIONED_DRIFT_SQL + def test_failed_diff_runs_the_migration_files_directly_and_marks_them_applied(self, monkeypatch, tmp_path): + calls = self._run(monkeypatch, tmp_path, partitioned=False, execute_fails=False, diff_fails=True) + executed_files = [c[c.index("--file") + 1] for c in calls if isinstance(c, list) and "execute" in c] + assert executed_files == [ + str(tmp_path / "migrations" / "20250326162113_baseline" / "migration.sql"), + str(tmp_path / "migrations" / "20260520120000_add_mcp_env_vars" / "migration.sql"), + ] + assert len(self._resolved(calls)) == 1 + + def test_failed_diff_with_already_exists_errors_still_marks_migrations_applied(self, monkeypatch, tmp_path): + calls = self._run( + monkeypatch, tmp_path, partitioned=False, execute_fails=True, diff_fails=True, + execute_stderr='ERROR: relation "LiteLLM_MCPUserEnvVars" already exists', + ) + assert len(self._resolved(calls)) == 1 + + def test_failed_diff_with_a_real_execute_failure_marks_nothing_applied(self, monkeypatch, tmp_path): + calls = self._run( + monkeypatch, tmp_path, partitioned=False, execute_fails=True, diff_fails=True, + execute_stderr="ERROR: permission denied for schema public", + ) + assert self._resolved(calls) == [] + + def test_failed_diff_with_a_missing_table_marks_nothing_applied(self, monkeypatch, tmp_path): + calls = self._run( + monkeypatch, tmp_path, partitioned=False, execute_fails=True, diff_fails=True, + execute_stderr='ERROR: relation "LiteLLM_TeamTable" does not exist', + ) + assert self._resolved(calls) == [] + + def test_failed_diff_with_an_already_dropped_constraint_still_marks_migrations_applied(self, monkeypatch, tmp_path): + calls = self._run( + monkeypatch, tmp_path, partitioned=False, execute_fails=True, diff_fails=True, + execute_stderr=( + 'ERROR: constraint "LiteLLM_TeamMembership_budget_id_fkey" ' + 'of relation "LiteLLM_TeamMembership" does not exist' + ), + ) + assert len(self._resolved(calls)) == 1 + + def test_failed_diff_during_the_post_deploy_sanity_check_replays_nothing(self, monkeypatch, tmp_path): + calls = self._run( + monkeypatch, tmp_path, partitioned=False, execute_fails=False, diff_fails=True, mark_all_applied=False + ) + assert [c for c in calls if isinstance(c, list) and "execute" in c] == [] + assert self._resolved(calls) == [] + class TestPartitionedSpendLogsPushGuard: def _forbid_subprocess(self, monkeypatch):