diff --git a/litellm-proxy-extras/litellm_proxy_extras/utils.py b/litellm-proxy-extras/litellm_proxy_extras/utils.py index 8a83c786e02..7c65adebf23 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/utils.py +++ b/litellm-proxy-extras/litellm_proxy_extras/utils.py @@ -1018,19 +1018,26 @@ class ProxyExtrasDBManager: if "P3009" in stderr: migration_name = ProxyExtrasDBManager._v2_failed_migration_name(stderr) - if migration_name: - ledger_logs = ProxyExtrasDBManager._failed_migration_logs(migration_name) - if ledger_logs and _MIGRATION_DEADLOCK_MARKER in ledger_logs: - logger.info( - "Migration %s failed in a concurrent migrate deploy " - "deadlock race, rolling its ledger row back and retrying", - migration_name, - ) - ProxyExtrasDBManager._v2_roll_back_migration_best_effort(migration_name) - return budget.spend() + ledger_logs = ( + ProxyExtrasDBManager._failed_migration_logs(migration_name) if migration_name else None + ) + if ledger_logs and _MIGRATION_DEADLOCK_MARKER in ledger_logs: + logger.info( + "Migration %s failed in a concurrent migrate deploy " + "deadlock race, rolling its ledger row back and retrying", + migration_name, + ) + ProxyExtrasDBManager._v2_roll_back_migration_best_effort(migration_name) + return budget.spend() + ledger_detail = ( + f"Failed migration {migration_name} logs from _prisma_migrations:\n{ledger_logs}\n\n" + if ledger_logs + else "" + ) raise RuntimeError( "Migration completion could not be verified. LiteLLM startup has stopped.\n\n" f"Prisma migration history (migration name and start time):\n{stderr}\n\n" + f"{ledger_detail}" "A migration has a start record but no successful completion record. " "LiteLLM cannot determine whether its SQL committed from this record alone. " "Startup stopped to avoid repeating or skipping database changes.\n\n" diff --git a/litellm-proxy-extras/tests/test_setup_database_fail_fast.py b/litellm-proxy-extras/tests/test_setup_database_fail_fast.py index 832075f6fbe..e36c7a9b597 100644 --- a/litellm-proxy-extras/tests/test_setup_database_fail_fast.py +++ b/litellm-proxy-extras/tests/test_setup_database_fail_fast.py @@ -315,6 +315,27 @@ def test_v2_p3009_empty_ledger_logs_do_not_prove_completion(monkeypatch, tmp_pat ) +def test_v2_p3009_ledger_logs_surface_in_error(monkeypatch, tmp_path): + """v2: Prisma's P3009 stderr never carries the SQL error, only the ledger row does. + The failed row's logs are included so the operator sees why the migration failed.""" + _stub_v2_env(monkeypatch, tmp_path, ledger_logs="ERROR: the pgaudit extension is not enabled") + + stderr = ( + "Error: P3009\n" + "migrate found failed migrations in the target database\n" + "The `20250327180120_add_api_requests_to_daily_user_table` migration " + "started at 2026-09-01 18:46:13 UTC failed" + ) + with patch( + "litellm_proxy_extras.prisma_toolchain.run_prisma", side_effect=_fake_migrate_deploy_failure(1, stderr) + ) as run: + with pytest.raises(RuntimeError, match="the pgaudit extension is not enabled"): + ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) + assert tuple(call.args[0][1:] for call in run.call_args_list if "migrate" in call.args[0]) == ( + ["migrate", "deploy"], + ) + + def test_v2_p3009_unreadable_ledger_still_raises(monkeypatch, tmp_path): """v2: an unreadable ledger cannot establish that P3009 was a deadlock.""" _stub_v2_env(monkeypatch, tmp_path, ledger_logs=None)