From c8020168594935e9ee293372c24fbcd32fc0fb1f Mon Sep 17 00:00:00 2001 From: jesus Date: Tue, 22 Sep 2026 18:57:27 +0000 Subject: [PATCH 1/5] fix(proxy): stop the legacy resolver marking migrations applied after a failed diff Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../litellm_proxy_extras/utils.py | 2 ++ .../test_litellm_proxy_extras_utils.py | 21 ++++++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/litellm-proxy-extras/litellm_proxy_extras/utils.py b/litellm-proxy-extras/litellm_proxy_extras/utils.py index 8a83c786e02..8e8732b8732 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/utils.py +++ b/litellm-proxy-extras/litellm_proxy_extras/utils.py @@ -473,8 +473,10 @@ class ProxyExtrasDBManager: env=_get_prisma_env(), ) except subprocess.CalledProcessError as e: + diff_sql_path.unlink(missing_ok=True) logger.warning(f"Failed to generate migration diff: {e.stderr}") except subprocess.TimeoutExpired: + diff_sql_path.unlink(missing_ok=True) logger.warning("Migration diff generation timed out.") # check if the migration was created diff --git a/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py b/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py index bb329264a11..dd03e3cc0dd 100644 --- a/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py +++ b/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py @@ -538,7 +538,7 @@ class _FakeCompleted: class TestResolveAllMigrationsLedger: - def _run(self, monkeypatch, tmp_path, partitioned, execute_fails): + def _run(self, monkeypatch, tmp_path, partitioned, execute_fails, diff_fails=False): import subprocess as subprocess_module import litellm_proxy_extras.utils as utils_module @@ -555,9 +555,19 @@ class TestResolveAllMigrationsLedger: ) calls = [] + for migration_name in ( + "20250326162113_baseline", + "20260520120000_add_mcp_env_vars", + ): + migration_dir = tmp_path / migration_name + migration_dir.mkdir(parents=True) + (migration_dir / "migration.sql").write_text("SELECT 1;") + def fake_run(cmd, **kwargs): calls.append(cmd) if "diff" in cmd: + if diff_fails: + raise subprocess_module.CalledProcessError(1, cmd, stderr="P1001") kwargs["stdout"].write(_PARTITIONED_DRIFT_SQL) return _FakeCompleted() if "execute" in cmd: @@ -599,6 +609,15 @@ class TestResolveAllMigrationsLedger: calls = self._run(monkeypatch, tmp_path, partitioned=False, execute_fails=False) assert self._executed_sql(calls) == _PARTITIONED_DRIFT_SQL + def test_failed_diff_runs_the_migration_files_directly_and_marks_nothing_applied(self, monkeypatch, tmp_path): + calls = self._run(monkeypatch, tmp_path, partitioned=False, execute_fails=False, diff_fails=True) + executed_files = [c[c.index("--file") + 1] for c in calls if isinstance(c, list) and "execute" in c] + assert executed_files == [ + str(tmp_path / "20250326162113_baseline" / "migration.sql"), + str(tmp_path / "20260520120000_add_mcp_env_vars" / "migration.sql"), + ] + assert self._resolved(calls) == [] + class TestPartitionedSpendLogsPushGuard: def _forbid_subprocess(self, monkeypatch): From 2ab2718c21dfe4db9584d31d2fcc2ecaca6dc868 Mon Sep 17 00:00:00 2001 From: jesus Date: Tue, 22 Sep 2026 19:06:05 +0000 Subject: [PATCH 2/5] fix(proxy): make the legacy resolver fallback find and record migration files Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../litellm_proxy_extras/utils.py | 8 ++++- .../test_litellm_proxy_extras_utils.py | 34 +++++++++++++++---- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/litellm-proxy-extras/litellm_proxy_extras/utils.py b/litellm-proxy-extras/litellm_proxy_extras/utils.py index 8e8732b8732..2c3c906990b 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/utils.py +++ b/litellm-proxy-extras/litellm_proxy_extras/utils.py @@ -489,7 +489,8 @@ class ProxyExtrasDBManager: # Fall back: run each migration SQL file directly via prisma db execute. # This works with pooler URLs (no schema introspection needed) and is # safe to re-run because migrations use IF NOT EXISTS / IF EXISTS guards. - migration_files = sorted(Path(migrations_dir).glob("*/migration.sql")) + migration_files = sorted(Path(migrations_dir).glob("migrations/*/migration.sql")) + fallback_ok = True for mig_file in migration_files: try: prisma_toolchain.run_prisma( @@ -510,8 +511,13 @@ class ProxyExtrasDBManager: logger.warning( f"Failed to apply migration {mig_file.parent.name}: {e.stderr}" ) + if not ProxyExtrasDBManager._is_idempotent_error(str(e.stderr or "")): # pyright: ignore[reportAny] # CalledProcessError.stderr is Any + fallback_ok = False except subprocess.TimeoutExpired: logger.warning(f"Migration {mig_file.parent.name} timed out.") + fallback_ok = False + if mark_all_applied and fallback_ok: + ProxyExtrasDBManager._mark_migrations_applied(migrations_dir) return logger.info(f"Migration diff created at {diff_sql_path}") diff --git a/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py b/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py index dd03e3cc0dd..b115047b4b2 100644 --- a/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py +++ b/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py @@ -538,7 +538,15 @@ class _FakeCompleted: class TestResolveAllMigrationsLedger: - def _run(self, monkeypatch, tmp_path, partitioned, execute_fails, diff_fails=False): + def _run( + self, + monkeypatch, + tmp_path, + partitioned, + execute_fails, + diff_fails=False, + execute_stderr="boom", + ): import subprocess as subprocess_module import litellm_proxy_extras.utils as utils_module @@ -559,7 +567,7 @@ class TestResolveAllMigrationsLedger: "20250326162113_baseline", "20260520120000_add_mcp_env_vars", ): - migration_dir = tmp_path / migration_name + migration_dir = tmp_path / "migrations" / migration_name migration_dir.mkdir(parents=True) (migration_dir / "migration.sql").write_text("SELECT 1;") @@ -574,7 +582,7 @@ class TestResolveAllMigrationsLedger: executed_sql = open(cmd[cmd.index("--file") + 1]).read() calls.append(("executed_sql", executed_sql)) if execute_fails: - raise subprocess_module.CalledProcessError(1, cmd, stderr="boom") + raise subprocess_module.CalledProcessError(1, cmd, stderr=execute_stderr) return _FakeCompleted() return _FakeCompleted() @@ -609,13 +617,27 @@ class TestResolveAllMigrationsLedger: calls = self._run(monkeypatch, tmp_path, partitioned=False, execute_fails=False) assert self._executed_sql(calls) == _PARTITIONED_DRIFT_SQL - def test_failed_diff_runs_the_migration_files_directly_and_marks_nothing_applied(self, monkeypatch, tmp_path): + def test_failed_diff_runs_the_migration_files_directly_and_marks_them_applied(self, monkeypatch, tmp_path): calls = self._run(monkeypatch, tmp_path, partitioned=False, execute_fails=False, diff_fails=True) executed_files = [c[c.index("--file") + 1] for c in calls if isinstance(c, list) and "execute" in c] assert executed_files == [ - str(tmp_path / "20250326162113_baseline" / "migration.sql"), - str(tmp_path / "20260520120000_add_mcp_env_vars" / "migration.sql"), + str(tmp_path / "migrations" / "20250326162113_baseline" / "migration.sql"), + str(tmp_path / "migrations" / "20260520120000_add_mcp_env_vars" / "migration.sql"), ] + assert len(self._resolved(calls)) == 1 + + def test_failed_diff_with_already_exists_errors_still_marks_migrations_applied(self, monkeypatch, tmp_path): + calls = self._run( + monkeypatch, tmp_path, partitioned=False, execute_fails=True, diff_fails=True, + execute_stderr='ERROR: relation "LiteLLM_MCPUserEnvVars" already exists', + ) + assert len(self._resolved(calls)) == 1 + + def test_failed_diff_with_a_real_execute_failure_marks_nothing_applied(self, monkeypatch, tmp_path): + calls = self._run( + monkeypatch, tmp_path, partitioned=False, execute_fails=True, diff_fails=True, + execute_stderr="ERROR: permission denied for schema public", + ) assert self._resolved(calls) == [] From 4f69f0dacc65d35d2fca649b0c9124bddae03765 Mon Sep 17 00:00:00 2001 From: jesus Date: Tue, 22 Sep 2026 19:49:52 +0000 Subject: [PATCH 3/5] refactor(proxy): build the legacy fallback result without rebinding Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../litellm_proxy_extras/utils.py | 59 ++++++++++--------- 1 file changed, 31 insertions(+), 28 deletions(-) diff --git a/litellm-proxy-extras/litellm_proxy_extras/utils.py b/litellm-proxy-extras/litellm_proxy_extras/utils.py index 2c3c906990b..74dd4fefcd0 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/utils.py +++ b/litellm-proxy-extras/litellm_proxy_extras/utils.py @@ -489,34 +489,11 @@ class ProxyExtrasDBManager: # Fall back: run each migration SQL file directly via prisma db execute. # This works with pooler URLs (no schema introspection needed) and is # safe to re-run because migrations use IF NOT EXISTS / IF EXISTS guards. - migration_files = sorted(Path(migrations_dir).glob("migrations/*/migration.sql")) - fallback_ok = True - for mig_file in migration_files: - try: - prisma_toolchain.run_prisma( - [ - _get_prisma_command(), - "db", - "execute", - "--file", - str(mig_file), - "--schema", - schema_path, - ], - timeout=prisma_command_timeout(), - env=_get_prisma_env(), - ) - logger.info(f"Applied migration: {mig_file.parent.name}") - except subprocess.CalledProcessError as e: - logger.warning( - f"Failed to apply migration {mig_file.parent.name}: {e.stderr}" - ) - if not ProxyExtrasDBManager._is_idempotent_error(str(e.stderr or "")): # pyright: ignore[reportAny] # CalledProcessError.stderr is Any - fallback_ok = False - except subprocess.TimeoutExpired: - logger.warning(f"Migration {mig_file.parent.name} timed out.") - fallback_ok = False - if mark_all_applied and fallback_ok: + results: Final = tuple( + ProxyExtrasDBManager._execute_migration_file(mig_file, schema_path) + for mig_file in sorted(Path(migrations_dir).glob("migrations/*/migration.sql")) + ) + if mark_all_applied and all(results): ProxyExtrasDBManager._mark_migrations_applied(migrations_dir) return logger.info(f"Migration diff created at {diff_sql_path}") @@ -573,6 +550,32 @@ class ProxyExtrasDBManager: return ProxyExtrasDBManager._mark_migrations_applied(migrations_dir) + @staticmethod + def _execute_migration_file(mig_file: Path, schema_path: str) -> bool: + try: + prisma_toolchain.run_prisma( + [ + _get_prisma_command(), + "db", + "execute", + "--file", + str(mig_file), + "--schema", + schema_path, + ], + timeout=prisma_command_timeout(), + env=_get_prisma_env(), + ) + logger.info(f"Applied migration: {mig_file.parent.name}") + return True + except subprocess.CalledProcessError as e: + stderr: Final = str(e.stderr or "") # pyright: ignore[reportAny] # CalledProcessError.stderr is Any + logger.warning(f"Failed to apply migration {mig_file.parent.name}: {stderr}") + return ProxyExtrasDBManager._is_idempotent_error(stderr) + except subprocess.TimeoutExpired: + logger.warning(f"Migration {mig_file.parent.name} timed out.") + return False + @staticmethod def _mark_migrations_applied(migrations_dir: str) -> None: migration_names = ProxyExtrasDBManager._get_migration_names(migrations_dir) From a8c761727990419337a75756ca9b8a99501bcc55 Mon Sep 17 00:00:00 2001 From: jesus Date: Tue, 22 Sep 2026 20:45:31 +0000 Subject: [PATCH 4/5] test(proxy): annotate the new resolver test helper parameters Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py b/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py index b115047b4b2..9b031c06738 100644 --- a/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py +++ b/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py @@ -544,8 +544,8 @@ class TestResolveAllMigrationsLedger: tmp_path, partitioned, execute_fails, - diff_fails=False, - execute_stderr="boom", + diff_fails: bool = False, + execute_stderr: str = "boom", ): import subprocess as subprocess_module From 46f88198ce9a549e6a18e80b72ed96a287999c74 Mon Sep 17 00:00:00 2001 From: jesus Date: Wed, 28 Oct 2026 00:59:18 +0000 Subject: [PATCH 5/5] fix(proxy): skip the migration replay when the ledger already records the files A failed migrate diff during the post-deploy sanity check (mark_all_applied=False) replayed every migration file onto an up-to-date database, which recreated the renamed LiteLLM_Engine tables and the dropped (prompt_id, version) unique index. The replay now only runs on the legacy no-ledger path, and a missing relation in the replay counts as a real failure instead of an idempotent one --- .../litellm_proxy_extras/utils.py | 10 +++++++ .../test_litellm_proxy_extras_utils.py | 27 ++++++++++++++++++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/litellm-proxy-extras/litellm_proxy_extras/utils.py b/litellm-proxy-extras/litellm_proxy_extras/utils.py index 3f825d2b026..aacbed6b8e1 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/utils.py +++ b/litellm-proxy-extras/litellm_proxy_extras/utils.py @@ -146,6 +146,7 @@ class _MigrateAttemptBudget: _SPEND_LOGS_ALTER_RE = re.compile(r'^ALTER\s+TABLE\s+"LiteLLM_SpendLogs"\s', re.IGNORECASE) _SPEND_LOGS_ARTIFACT_DROP_RE = re.compile(r'^DROP\s+TABLE\s+"LiteLLM_SpendLogs_[^"]*"', re.IGNORECASE) +_MISSING_RELATION: Final = re.compile(r'(?