fix: P3009 try/except, CREATE TABLE IF NOT EXISTS, restore fail-fast default

Four fixes:

1. P3009 idempotent handler now has the same try/except around
   _roll_back_migration and _resolve_specific_migration as the P3018
   handler. Previously a rollback or resolve failure in the P3009 path
   would propagate and leave the migration unresolved.

2. Added IF NOT EXISTS to all 57 bare CREATE TABLE statements across
   34 migration files. Added test_create_table_uses_if_not_exists to
   catch this pattern.

3. Reverted the backwards-incompatible default behavior change: the
   proxy now fails fast on migration failure (original behavior).
   Added --skip_db_migration_check / SKIP_DB_MIGRATION_CHECK to
   opt into warn-and-continue instead.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Krrish Dholakia 2026-03-14 16:31:02 -07:00
parent b44099c1e8
commit 8e1ed91d23
37 changed files with 101 additions and 76 deletions

View file

@ -1,5 +1,5 @@
-- CreateTable
CREATE TABLE "LiteLLM_BudgetTable" (
CREATE TABLE IF NOT EXISTS "LiteLLM_BudgetTable" (
"budget_id" TEXT NOT NULL,
"max_budget" DOUBLE PRECISION,
"soft_budget" DOUBLE PRECISION,
@ -18,7 +18,7 @@ CREATE TABLE "LiteLLM_BudgetTable" (
);
-- CreateTable
CREATE TABLE "LiteLLM_CredentialsTable" (
CREATE TABLE IF NOT EXISTS "LiteLLM_CredentialsTable" (
"credential_id" TEXT NOT NULL,
"credential_name" TEXT NOT NULL,
"credential_values" JSONB NOT NULL,
@ -32,7 +32,7 @@ CREATE TABLE "LiteLLM_CredentialsTable" (
);
-- CreateTable
CREATE TABLE "LiteLLM_ProxyModelTable" (
CREATE TABLE IF NOT EXISTS "LiteLLM_ProxyModelTable" (
"model_id" TEXT NOT NULL,
"model_name" TEXT NOT NULL,
"litellm_params" JSONB NOT NULL,
@ -46,7 +46,7 @@ CREATE TABLE "LiteLLM_ProxyModelTable" (
);
-- CreateTable
CREATE TABLE "LiteLLM_OrganizationTable" (
CREATE TABLE IF NOT EXISTS "LiteLLM_OrganizationTable" (
"organization_id" TEXT NOT NULL,
"organization_alias" TEXT NOT NULL,
"budget_id" TEXT NOT NULL,
@ -63,7 +63,7 @@ CREATE TABLE "LiteLLM_OrganizationTable" (
);
-- CreateTable
CREATE TABLE "LiteLLM_ModelTable" (
CREATE TABLE IF NOT EXISTS "LiteLLM_ModelTable" (
"id" SERIAL NOT NULL,
"aliases" JSONB,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
@ -75,7 +75,7 @@ CREATE TABLE "LiteLLM_ModelTable" (
);
-- CreateTable
CREATE TABLE "LiteLLM_TeamTable" (
CREATE TABLE IF NOT EXISTS "LiteLLM_TeamTable" (
"team_id" TEXT NOT NULL,
"team_alias" TEXT,
"organization_id" TEXT,
@ -102,7 +102,7 @@ CREATE TABLE "LiteLLM_TeamTable" (
);
-- CreateTable
CREATE TABLE "LiteLLM_UserTable" (
CREATE TABLE IF NOT EXISTS "LiteLLM_UserTable" (
"user_id" TEXT NOT NULL,
"user_alias" TEXT,
"team_id" TEXT,
@ -131,7 +131,7 @@ CREATE TABLE "LiteLLM_UserTable" (
);
-- CreateTable
CREATE TABLE "LiteLLM_VerificationToken" (
CREATE TABLE IF NOT EXISTS "LiteLLM_VerificationToken" (
"token" TEXT NOT NULL,
"key_name" TEXT,
"key_alias" TEXT,
@ -166,7 +166,7 @@ CREATE TABLE "LiteLLM_VerificationToken" (
);
-- CreateTable
CREATE TABLE "LiteLLM_EndUserTable" (
CREATE TABLE IF NOT EXISTS "LiteLLM_EndUserTable" (
"user_id" TEXT NOT NULL,
"alias" TEXT,
"spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0,
@ -179,7 +179,7 @@ CREATE TABLE "LiteLLM_EndUserTable" (
);
-- CreateTable
CREATE TABLE "LiteLLM_Config" (
CREATE TABLE IF NOT EXISTS "LiteLLM_Config" (
"param_name" TEXT NOT NULL,
"param_value" JSONB,
@ -187,7 +187,7 @@ CREATE TABLE "LiteLLM_Config" (
);
-- CreateTable
CREATE TABLE "LiteLLM_SpendLogs" (
CREATE TABLE IF NOT EXISTS "LiteLLM_SpendLogs" (
"request_id" TEXT NOT NULL,
"call_type" TEXT NOT NULL,
"api_key" TEXT NOT NULL DEFAULT '',
@ -218,7 +218,7 @@ CREATE TABLE "LiteLLM_SpendLogs" (
);
-- CreateTable
CREATE TABLE "LiteLLM_ErrorLogs" (
CREATE TABLE IF NOT EXISTS "LiteLLM_ErrorLogs" (
"request_id" TEXT NOT NULL,
"startTime" TIMESTAMP(3) NOT NULL,
"endTime" TIMESTAMP(3) NOT NULL,
@ -235,7 +235,7 @@ CREATE TABLE "LiteLLM_ErrorLogs" (
);
-- CreateTable
CREATE TABLE "LiteLLM_UserNotifications" (
CREATE TABLE IF NOT EXISTS "LiteLLM_UserNotifications" (
"request_id" TEXT NOT NULL,
"user_id" TEXT NOT NULL,
"models" TEXT[],
@ -246,7 +246,7 @@ CREATE TABLE "LiteLLM_UserNotifications" (
);
-- CreateTable
CREATE TABLE "LiteLLM_TeamMembership" (
CREATE TABLE IF NOT EXISTS "LiteLLM_TeamMembership" (
"user_id" TEXT NOT NULL,
"team_id" TEXT NOT NULL,
"spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0,
@ -256,7 +256,7 @@ CREATE TABLE "LiteLLM_TeamMembership" (
);
-- CreateTable
CREATE TABLE "LiteLLM_OrganizationMembership" (
CREATE TABLE IF NOT EXISTS "LiteLLM_OrganizationMembership" (
"user_id" TEXT NOT NULL,
"organization_id" TEXT NOT NULL,
"user_role" TEXT,
@ -269,7 +269,7 @@ CREATE TABLE "LiteLLM_OrganizationMembership" (
);
-- CreateTable
CREATE TABLE "LiteLLM_InvitationLink" (
CREATE TABLE IF NOT EXISTS "LiteLLM_InvitationLink" (
"id" TEXT NOT NULL,
"user_id" TEXT NOT NULL,
"is_accepted" BOOLEAN NOT NULL DEFAULT false,
@ -284,7 +284,7 @@ CREATE TABLE "LiteLLM_InvitationLink" (
);
-- CreateTable
CREATE TABLE "LiteLLM_AuditLog" (
CREATE TABLE IF NOT EXISTS "LiteLLM_AuditLog" (
"id" TEXT NOT NULL,
"updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"changed_by" TEXT NOT NULL DEFAULT '',

View file

@ -1,5 +1,5 @@
-- CreateTable
CREATE TABLE "LiteLLM_DailyUserSpend" (
CREATE TABLE IF NOT EXISTS "LiteLLM_DailyUserSpend" (
"id" TEXT NOT NULL,
"user_id" TEXT NOT NULL,
"date" TEXT NOT NULL,

View file

@ -2,7 +2,7 @@
CREATE TYPE "JobStatus" AS ENUM ('ACTIVE', 'INACTIVE');
-- CreateTable
CREATE TABLE "LiteLLM_CronJob" (
CREATE TABLE IF NOT EXISTS "LiteLLM_CronJob" (
"cronjob_id" TEXT NOT NULL,
"pod_id" TEXT NOT NULL,
"status" "JobStatus" NOT NULL DEFAULT 'INACTIVE',

View file

@ -1,5 +1,5 @@
-- CreateTable
CREATE TABLE "LiteLLM_ManagedFileTable" (
CREATE TABLE IF NOT EXISTS "LiteLLM_ManagedFileTable" (
"id" TEXT NOT NULL,
"unified_file_id" TEXT NOT NULL,
"file_object" JSONB NOT NULL,

View file

@ -1,5 +1,5 @@
-- CreateTable
CREATE TABLE "LiteLLM_DailyTeamSpend" (
CREATE TABLE IF NOT EXISTS "LiteLLM_DailyTeamSpend" (
"id" TEXT NOT NULL,
"team_id" TEXT NOT NULL,
"date" TEXT NOT NULL,

View file

@ -3,7 +3,7 @@ ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "cache_creation_in
ADD COLUMN IF NOT EXISTS "cache_read_input_tokens" INTEGER NOT NULL DEFAULT 0;
-- CreateTable
CREATE TABLE "LiteLLM_DailyTagSpend" (
CREATE TABLE IF NOT EXISTS "LiteLLM_DailyTagSpend" (
"id" TEXT NOT NULL,
"tag" TEXT NOT NULL,
"date" TEXT NOT NULL,

View file

@ -1,5 +1,5 @@
-- CreateTable
CREATE TABLE "LiteLLM_ManagedVectorStoresTable" (
CREATE TABLE IF NOT EXISTS "LiteLLM_ManagedVectorStoresTable" (
"vector_store_id" TEXT NOT NULL,
"custom_llm_provider" TEXT NOT NULL,
"vector_store_name" TEXT,

View file

@ -1,5 +1,5 @@
-- CreateTable
CREATE TABLE "LiteLLM_MCPServerTable" (
CREATE TABLE IF NOT EXISTS "LiteLLM_MCPServerTable" (
"server_id" TEXT NOT NULL,
"alias" TEXT,
"description" TEXT,

View file

@ -11,7 +11,7 @@ ALTER TABLE "LiteLLM_UserTable" ADD COLUMN IF NOT EXISTS "object_permission_id"
ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN IF NOT EXISTS "object_permission_id" TEXT;
-- CreateTable
CREATE TABLE "LiteLLM_ObjectPermissionTable" (
CREATE TABLE IF NOT EXISTS "LiteLLM_ObjectPermissionTable" (
"object_permission_id" TEXT NOT NULL,
"mcp_servers" TEXT[] DEFAULT ARRAY[]::TEXT[],

View file

@ -1,5 +1,5 @@
-- CreateTable
CREATE TABLE "LiteLLM_GuardrailsTable" (
CREATE TABLE IF NOT EXISTS "LiteLLM_GuardrailsTable" (
"guardrail_id" TEXT NOT NULL,
"guardrail_name" TEXT NOT NULL,
"litellm_params" JSONB NOT NULL,

View file

@ -4,7 +4,7 @@ ADD COLUMN IF NOT EXISTS "flat_model_file_ids" TEXT[] DEFAULT ARRAY[]::TEXT[],
ADD COLUMN IF NOT EXISTS "updated_by" TEXT;
-- CreateTable
CREATE TABLE "LiteLLM_ManagedObjectTable" (
CREATE TABLE IF NOT EXISTS "LiteLLM_ManagedObjectTable" (
"id" TEXT NOT NULL,
"unified_object_id" TEXT NOT NULL,
"model_object_id" TEXT NOT NULL,

View file

@ -1,5 +1,5 @@
-- CreateTable
CREATE TABLE "LiteLLM_HealthCheckTable" (
CREATE TABLE IF NOT EXISTS "LiteLLM_HealthCheckTable" (
"health_check_id" TEXT NOT NULL,
"model_name" TEXT NOT NULL,
"model_id" TEXT,

View file

@ -1,5 +1,5 @@
-- CreateTable
CREATE TABLE "LiteLLM_PromptTable" (
CREATE TABLE IF NOT EXISTS "LiteLLM_PromptTable" (
"id" TEXT NOT NULL,
"prompt_id" TEXT NOT NULL,
"litellm_params" JSONB NOT NULL,

View file

@ -1,5 +1,5 @@
-- CreateTable
CREATE TABLE "LiteLLM_TagTable" (
CREATE TABLE IF NOT EXISTS "LiteLLM_TagTable" (
"tag_name" TEXT NOT NULL,
"description" TEXT,
"models" TEXT[],

View file

@ -1,5 +1,5 @@
-- CreateTable
CREATE TABLE "LiteLLM_SearchToolsTable" (
CREATE TABLE IF NOT EXISTS "LiteLLM_SearchToolsTable" (
"search_tool_id" TEXT NOT NULL,
"search_tool_name" TEXT NOT NULL,
"litellm_params" JSONB NOT NULL,

View file

@ -1,5 +1,5 @@
-- CreateTable
CREATE TABLE "LiteLLM_SSOConfig" (
CREATE TABLE IF NOT EXISTS "LiteLLM_SSOConfig" (
"id" TEXT NOT NULL DEFAULT 'sso_config',
"sso_settings" JSONB NOT NULL,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
@ -9,7 +9,7 @@ CREATE TABLE "LiteLLM_SSOConfig" (
);
-- CreateTable
CREATE TABLE "LiteLLM_CacheConfig" (
CREATE TABLE IF NOT EXISTS "LiteLLM_CacheConfig" (
"id" TEXT NOT NULL DEFAULT 'cache_config',
"cache_settings" JSONB NOT NULL,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,

View file

@ -1,5 +1,5 @@
-- CreateTable
CREATE TABLE "LiteLLM_ManagedVectorStoreIndexTable" (
CREATE TABLE IF NOT EXISTS "LiteLLM_ManagedVectorStoreIndexTable" (
"id" TEXT NOT NULL,
"index_name" TEXT NOT NULL,
"litellm_params" JSONB NOT NULL,

View file

@ -1,5 +1,5 @@
-- CreateTable
CREATE TABLE "LiteLLM_ProjectTable" (
CREATE TABLE IF NOT EXISTS "LiteLLM_ProjectTable" (
"project_id" TEXT NOT NULL,
"project_alias" TEXT,
"team_id" TEXT,

View file

@ -1,5 +1,5 @@
-- CreateTable
CREATE TABLE "LiteLLM_DailyOrganizationSpend" (
CREATE TABLE IF NOT EXISTS "LiteLLM_DailyOrganizationSpend" (
"id" TEXT NOT NULL,
"organization_id" TEXT,
"date" TEXT NOT NULL,

View file

@ -1,5 +1,5 @@
-- CreateTable
CREATE TABLE "LiteLLM_AgentsTable" (
CREATE TABLE IF NOT EXISTS "LiteLLM_AgentsTable" (
"agent_id" TEXT NOT NULL,
"agent_name" TEXT NOT NULL,
"litellm_params" JSONB,

View file

@ -1,5 +1,5 @@
-- CreateTable
CREATE TABLE "LiteLLM_DailyEndUserSpend" (
CREATE TABLE IF NOT EXISTS "LiteLLM_DailyEndUserSpend" (
"id" TEXT NOT NULL,
"end_user_id" TEXT,
"date" TEXT NOT NULL,

View file

@ -1,5 +1,5 @@
-- CreateTable
CREATE TABLE "LiteLLM_UISettings" (
CREATE TABLE IF NOT EXISTS "LiteLLM_UISettings" (
"id" TEXT NOT NULL DEFAULT 'ui_settings',
"ui_settings" JSONB NOT NULL,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,

View file

@ -2,7 +2,7 @@
ALTER TABLE "LiteLLM_SpendLogs" ADD COLUMN IF NOT EXISTS "agent_id" TEXT;
-- CreateTable
CREATE TABLE "LiteLLM_DailyAgentSpend" (
CREATE TABLE IF NOT EXISTS "LiteLLM_DailyAgentSpend" (
"id" TEXT NOT NULL,
"agent_id" TEXT,
"date" TEXT NOT NULL,

View file

@ -1,5 +1,5 @@
-- CreateTable
CREATE TABLE "LiteLLM_DeletedTeamTable" (
CREATE TABLE IF NOT EXISTS "LiteLLM_DeletedTeamTable" (
"id" TEXT NOT NULL,
"team_id" TEXT NOT NULL,
"team_alias" TEXT,
@ -33,7 +33,7 @@ CREATE TABLE "LiteLLM_DeletedTeamTable" (
);
-- CreateTable
CREATE TABLE "LiteLLM_DeletedVerificationToken" (
CREATE TABLE IF NOT EXISTS "LiteLLM_DeletedVerificationToken" (
"id" TEXT NOT NULL,
"token" TEXT NOT NULL,
"key_name" TEXT,

View file

@ -1,5 +1,5 @@
-- CreateTable
CREATE TABLE "LiteLLM_SkillsTable" (
CREATE TABLE IF NOT EXISTS "LiteLLM_SkillsTable" (
"skill_id" TEXT NOT NULL,
"display_title" TEXT,
"description" TEXT,

View file

@ -14,7 +14,7 @@ ALTER TABLE "LiteLLM_UserTable" ADD COLUMN IF NOT EXISTS "policies" TEXT[] DEFAU
ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN IF NOT EXISTS "policies" TEXT[] DEFAULT ARRAY[]::TEXT[];
-- CreateTable
CREATE TABLE "LiteLLM_PolicyTable" (
CREATE TABLE IF NOT EXISTS "LiteLLM_PolicyTable" (
"policy_id" TEXT NOT NULL,
"policy_name" TEXT NOT NULL,
"inherit" TEXT,
@ -31,7 +31,7 @@ CREATE TABLE "LiteLLM_PolicyTable" (
);
-- CreateTable
CREATE TABLE "LiteLLM_PolicyAttachmentTable" (
CREATE TABLE IF NOT EXISTS "LiteLLM_PolicyAttachmentTable" (
"attachment_id" TEXT NOT NULL,
"policy_name" TEXT NOT NULL,
"scope" TEXT,

View file

@ -1,5 +1,5 @@
-- CreateTable
CREATE TABLE "LiteLLM_DeprecatedVerificationToken" (
CREATE TABLE IF NOT EXISTS "LiteLLM_DeprecatedVerificationToken" (
"id" TEXT NOT NULL,
"token" TEXT NOT NULL,
"active_token_id" TEXT NOT NULL,

View file

@ -11,7 +11,7 @@ ALTER TABLE "LiteLLM_TeamTable" ADD COLUMN IF NOT EXISTS "access_group_ids" TEXT
ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN IF NOT EXISTS "access_group_ids" TEXT[] DEFAULT ARRAY[]::TEXT[];
-- CreateTable
CREATE TABLE "LiteLLM_AccessGroupTable" (
CREATE TABLE IF NOT EXISTS "LiteLLM_AccessGroupTable" (
"access_group_id" TEXT NOT NULL,
"access_group_name" TEXT NOT NULL,
"description" TEXT,

View file

@ -1,5 +1,5 @@
-- CreateTable
CREATE TABLE "LiteLLM_ManagedVectorStoreTable" (
CREATE TABLE IF NOT EXISTS "LiteLLM_ManagedVectorStoreTable" (
"id" TEXT NOT NULL,
"unified_resource_id" TEXT NOT NULL,
"resource_object" JSONB,

View file

@ -1,5 +1,5 @@
-- CreateTable
CREATE TABLE "LiteLLM_DailyGuardrailMetrics" (
CREATE TABLE IF NOT EXISTS "LiteLLM_DailyGuardrailMetrics" (
"guardrail_id" TEXT NOT NULL,
"date" TEXT NOT NULL,
"requests_evaluated" BIGINT NOT NULL DEFAULT 0,
@ -15,7 +15,7 @@ CREATE TABLE "LiteLLM_DailyGuardrailMetrics" (
);
-- CreateTable
CREATE TABLE "LiteLLM_DailyPolicyMetrics" (
CREATE TABLE IF NOT EXISTS "LiteLLM_DailyPolicyMetrics" (
"policy_id" TEXT NOT NULL,
"date" TEXT NOT NULL,
"requests_evaluated" BIGINT NOT NULL DEFAULT 0,
@ -31,7 +31,7 @@ CREATE TABLE "LiteLLM_DailyPolicyMetrics" (
);
-- CreateTable
CREATE TABLE "LiteLLM_SpendLogGuardrailIndex" (
CREATE TABLE IF NOT EXISTS "LiteLLM_SpendLogGuardrailIndex" (
"request_id" TEXT NOT NULL,
"guardrail_id" TEXT NOT NULL,
"policy_id" TEXT,

View file

@ -8,7 +8,7 @@ ALTER TABLE "LiteLLM_MCPServerTable" DROP COLUMN IF EXISTS "spec_path";
ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN IF NOT EXISTS "agent_id" TEXT;
-- CreateTable
CREATE TABLE "LiteLLM_ToolTable" (
CREATE TABLE IF NOT EXISTS "LiteLLM_ToolTable" (
"tool_id" TEXT NOT NULL,
"tool_name" TEXT NOT NULL,
"origin" TEXT,

View file

@ -1,5 +1,5 @@
-- CreateTable
CREATE TABLE "LiteLLM_SpendLogToolIndex" (
CREATE TABLE IF NOT EXISTS "LiteLLM_SpendLogToolIndex" (
"request_id" TEXT NOT NULL,
"tool_name" TEXT NOT NULL,
"start_time" TIMESTAMP(3) NOT NULL,

View file

@ -1,5 +1,5 @@
-- CreateTable
CREATE TABLE "LiteLLM_ClaudeCodePluginTable" (
CREATE TABLE IF NOT EXISTS "LiteLLM_ClaudeCodePluginTable" (
"id" TEXT NOT NULL,
"name" TEXT NOT NULL,
"version" TEXT,

View file

@ -6,7 +6,7 @@ ADD COLUMN IF NOT EXISTS "tool_name_to_description" JSONB DEFAULT '{}',
ADD COLUMN IF NOT EXISTS "tool_name_to_display_name" JSONB DEFAULT '{}';
-- CreateTable
CREATE TABLE "LiteLLM_MCPUserCredentials" (
CREATE TABLE IF NOT EXISTS "LiteLLM_MCPUserCredentials" (
"id" TEXT NOT NULL,
"user_id" TEXT NOT NULL,
"server_id" TEXT NOT NULL,
@ -18,7 +18,7 @@ CREATE TABLE "LiteLLM_MCPUserCredentials" (
);
-- CreateTable
CREATE TABLE "LiteLLM_JWTKeyMapping" (
CREATE TABLE IF NOT EXISTS "LiteLLM_JWTKeyMapping" (
"id" TEXT NOT NULL,
"jwt_claim_name" TEXT NOT NULL,
"jwt_claim_value" TEXT NOT NULL,
@ -34,7 +34,7 @@ CREATE TABLE "LiteLLM_JWTKeyMapping" (
);
-- CreateTable
CREATE TABLE "LiteLLM_ConfigOverrides" (
CREATE TABLE IF NOT EXISTS "LiteLLM_ConfigOverrides" (
"config_type" TEXT NOT NULL,
"config_value" JSONB NOT NULL,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,

View file

@ -415,15 +415,26 @@ class ProxyExtrasDBManager:
logger.info(
f"Migration {failed_migration} failed due to idempotent error (e.g., column already exists), resolving as applied"
)
ProxyExtrasDBManager._roll_back_migration(
failed_migration
)
ProxyExtrasDBManager._resolve_specific_migration(
failed_migration
)
logger.info(
f"✅ Migration {failed_migration} resolved, retrying to apply remaining migrations"
)
try:
ProxyExtrasDBManager._roll_back_migration(
failed_migration
)
except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as rollback_err:
logger.warning(
f"Failed to roll back migration {failed_migration}: {rollback_err}. "
f"It may already be in a rolled-back state."
)
try:
ProxyExtrasDBManager._resolve_specific_migration(
failed_migration
)
logger.info(
f"✅ Migration {failed_migration} resolved, retrying to apply remaining migrations"
)
except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as resolve_err:
logger.warning(
f"Failed to resolve migration {failed_migration}: {resolve_err}"
)
else:
logger.info(
f"Found failed migration: {failed_migration}, marking as rolled back"

View file

@ -557,11 +557,11 @@ class ProxyInitializationHelpers:
envvar="MAX_REQUESTS_BEFORE_RESTART",
)
@click.option(
"--require_db_migration",
"--skip_db_migration_check",
is_flag=True,
default=False,
help="Exit with error if database migration fails. By default the proxy will warn and continue.",
envvar="REQUIRE_DB_MIGRATION",
help="Warn and continue instead of exiting when database migration fails.",
envvar="SKIP_DB_MIGRATION_CHECK",
)
def run_server( # noqa: PLR0915
host,
@ -602,7 +602,7 @@ def run_server( # noqa: PLR0915
skip_server_startup,
keepalive_timeout,
max_requests_before_restart,
require_db_migration: bool,
skip_db_migration_check: bool,
):
args = locals()
if local:
@ -865,17 +865,17 @@ def run_server( # noqa: PLR0915
if not PrismaManager.setup_database(
use_migrate=not use_prisma_db_push
):
if require_db_migration:
if skip_db_migration_check:
print( # noqa
"\033[1;33mLiteLLM Proxy: Database migration failed but continuing startup. "
"Pass --skip_db_migration_check to allow this.\033[0m"
)
else:
print( # noqa
"\033[1;31mLiteLLM Proxy: Database setup failed after multiple retries. "
"The proxy cannot start safely. Please check your database connection and migration status.\033[0m"
)
sys.exit(1)
else:
print( # noqa
"\033[1;33mLiteLLM Proxy: Database migration failed but continuing startup. "
"Use --require_db_migration to exit on migration failure.\033[0m"
)
else:
print( # noqa
f"Unable to connect to DB. DATABASE_URL found in environment, but prisma package not found." # noqa

View file

@ -180,6 +180,20 @@ class TestMigrationSQLIdempotency:
)
return migrations
def test_create_table_uses_if_not_exists(self, all_migrations):
"""CREATE TABLE statements must use IF NOT EXISTS"""
violations = []
for migration_name, sql in all_migrations:
for line_num, line in enumerate(sql.splitlines(), 1):
if re.search(r"CREATE\s+TABLE\s+", line, re.IGNORECASE) and not re.search(
r"CREATE\s+TABLE\s+IF\s+NOT\s+EXISTS", line, re.IGNORECASE
):
violations.append(f" {migration_name}:{line_num}: {line.strip()}")
assert not violations, (
"CREATE TABLE without IF NOT EXISTS found in migrations:\n"
+ "\n".join(violations)
)
def test_add_column_uses_if_not_exists(self, all_migrations):
"""ADD COLUMN statements must use IF NOT EXISTS"""
violations = []