From 8e1ed91d2324310c16a437cad87ba48df7b540d3 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 14 Mar 2026 16:31:02 -0700 Subject: [PATCH] fix: P3009 try/except, CREATE TABLE IF NOT EXISTS, restore fail-fast default Four fixes: 1. P3009 idempotent handler now has the same try/except around _roll_back_migration and _resolve_specific_migration as the P3018 handler. Previously a rollback or resolve failure in the P3009 path would propagate and leave the migration unresolved. 2. Added IF NOT EXISTS to all 57 bare CREATE TABLE statements across 34 migration files. Added test_create_table_uses_if_not_exists to catch this pattern. 3. Reverted the backwards-incompatible default behavior change: the proxy now fails fast on migration failure (original behavior). Added --skip_db_migration_check / SKIP_DB_MIGRATION_CHECK to opt into warn-and-continue instead. Co-Authored-By: Claude Opus 4.6 --- .../20250326162113_baseline/migration.sql | 34 +++++++++---------- .../migration.sql | 2 +- .../migration.sql | 2 +- .../migration.sql | 2 +- .../migration.sql | 2 +- .../migration.sql | 2 +- .../migration.sql | 2 +- .../migration.sql | 2 +- .../migration.sql | 2 +- .../migration.sql | 2 +- .../migration.sql | 2 +- .../migration.sql | 2 +- .../20250802162330_prompt_table/migration.sql | 2 +- .../migration.sql | 2 +- .../migration.sql | 2 +- .../migration.sql | 4 +-- .../migration.sql | 2 +- .../migration.sql | 2 +- .../migration.sql | 2 +- .../20251114182247_agents_table/migration.sql | 2 +- .../migration.sql | 2 +- .../migration.sql | 2 +- .../migration.sql | 2 +- .../migration.sql | 4 +-- .../migration.sql | 2 +- .../migration.sql | 4 +-- .../migration.sql | 2 +- .../migration.sql | 2 +- .../migration.sql | 2 +- .../migration.sql | 6 ++-- .../migration.sql | 2 +- .../migration.sql | 2 +- .../migration.sql | 2 +- .../20260306233848_schema_sync/migration.sql | 6 ++-- .../litellm_proxy_extras/utils.py | 29 +++++++++++----- litellm/proxy/proxy_cli.py | 20 +++++------ .../test_litellm_proxy_extras_utils.py | 14 ++++++++ 37 files changed, 101 insertions(+), 76 deletions(-) diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20250326162113_baseline/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20250326162113_baseline/migration.sql index 0b0cde1e35a..7b33d58899e 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20250326162113_baseline/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20250326162113_baseline/migration.sql @@ -1,5 +1,5 @@ -- CreateTable -CREATE TABLE "LiteLLM_BudgetTable" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_BudgetTable" ( "budget_id" TEXT NOT NULL, "max_budget" DOUBLE PRECISION, "soft_budget" DOUBLE PRECISION, @@ -18,7 +18,7 @@ CREATE TABLE "LiteLLM_BudgetTable" ( ); -- CreateTable -CREATE TABLE "LiteLLM_CredentialsTable" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_CredentialsTable" ( "credential_id" TEXT NOT NULL, "credential_name" TEXT NOT NULL, "credential_values" JSONB NOT NULL, @@ -32,7 +32,7 @@ CREATE TABLE "LiteLLM_CredentialsTable" ( ); -- CreateTable -CREATE TABLE "LiteLLM_ProxyModelTable" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_ProxyModelTable" ( "model_id" TEXT NOT NULL, "model_name" TEXT NOT NULL, "litellm_params" JSONB NOT NULL, @@ -46,7 +46,7 @@ CREATE TABLE "LiteLLM_ProxyModelTable" ( ); -- CreateTable -CREATE TABLE "LiteLLM_OrganizationTable" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_OrganizationTable" ( "organization_id" TEXT NOT NULL, "organization_alias" TEXT NOT NULL, "budget_id" TEXT NOT NULL, @@ -63,7 +63,7 @@ CREATE TABLE "LiteLLM_OrganizationTable" ( ); -- CreateTable -CREATE TABLE "LiteLLM_ModelTable" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_ModelTable" ( "id" SERIAL NOT NULL, "aliases" JSONB, "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, @@ -75,7 +75,7 @@ CREATE TABLE "LiteLLM_ModelTable" ( ); -- CreateTable -CREATE TABLE "LiteLLM_TeamTable" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_TeamTable" ( "team_id" TEXT NOT NULL, "team_alias" TEXT, "organization_id" TEXT, @@ -102,7 +102,7 @@ CREATE TABLE "LiteLLM_TeamTable" ( ); -- CreateTable -CREATE TABLE "LiteLLM_UserTable" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_UserTable" ( "user_id" TEXT NOT NULL, "user_alias" TEXT, "team_id" TEXT, @@ -131,7 +131,7 @@ CREATE TABLE "LiteLLM_UserTable" ( ); -- CreateTable -CREATE TABLE "LiteLLM_VerificationToken" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_VerificationToken" ( "token" TEXT NOT NULL, "key_name" TEXT, "key_alias" TEXT, @@ -166,7 +166,7 @@ CREATE TABLE "LiteLLM_VerificationToken" ( ); -- CreateTable -CREATE TABLE "LiteLLM_EndUserTable" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_EndUserTable" ( "user_id" TEXT NOT NULL, "alias" TEXT, "spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0, @@ -179,7 +179,7 @@ CREATE TABLE "LiteLLM_EndUserTable" ( ); -- CreateTable -CREATE TABLE "LiteLLM_Config" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_Config" ( "param_name" TEXT NOT NULL, "param_value" JSONB, @@ -187,7 +187,7 @@ CREATE TABLE "LiteLLM_Config" ( ); -- CreateTable -CREATE TABLE "LiteLLM_SpendLogs" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_SpendLogs" ( "request_id" TEXT NOT NULL, "call_type" TEXT NOT NULL, "api_key" TEXT NOT NULL DEFAULT '', @@ -218,7 +218,7 @@ CREATE TABLE "LiteLLM_SpendLogs" ( ); -- CreateTable -CREATE TABLE "LiteLLM_ErrorLogs" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_ErrorLogs" ( "request_id" TEXT NOT NULL, "startTime" TIMESTAMP(3) NOT NULL, "endTime" TIMESTAMP(3) NOT NULL, @@ -235,7 +235,7 @@ CREATE TABLE "LiteLLM_ErrorLogs" ( ); -- CreateTable -CREATE TABLE "LiteLLM_UserNotifications" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_UserNotifications" ( "request_id" TEXT NOT NULL, "user_id" TEXT NOT NULL, "models" TEXT[], @@ -246,7 +246,7 @@ CREATE TABLE "LiteLLM_UserNotifications" ( ); -- CreateTable -CREATE TABLE "LiteLLM_TeamMembership" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_TeamMembership" ( "user_id" TEXT NOT NULL, "team_id" TEXT NOT NULL, "spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0, @@ -256,7 +256,7 @@ CREATE TABLE "LiteLLM_TeamMembership" ( ); -- CreateTable -CREATE TABLE "LiteLLM_OrganizationMembership" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_OrganizationMembership" ( "user_id" TEXT NOT NULL, "organization_id" TEXT NOT NULL, "user_role" TEXT, @@ -269,7 +269,7 @@ CREATE TABLE "LiteLLM_OrganizationMembership" ( ); -- CreateTable -CREATE TABLE "LiteLLM_InvitationLink" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_InvitationLink" ( "id" TEXT NOT NULL, "user_id" TEXT NOT NULL, "is_accepted" BOOLEAN NOT NULL DEFAULT false, @@ -284,7 +284,7 @@ CREATE TABLE "LiteLLM_InvitationLink" ( ); -- CreateTable -CREATE TABLE "LiteLLM_AuditLog" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_AuditLog" ( "id" TEXT NOT NULL, "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, "changed_by" TEXT NOT NULL DEFAULT '', diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20250326171002_add_daily_user_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20250326171002_add_daily_user_table/migration.sql index 4c23b49728e..52f20ee0f28 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20250326171002_add_daily_user_table/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20250326171002_add_daily_user_table/migration.sql @@ -1,5 +1,5 @@ -- CreateTable -CREATE TABLE "LiteLLM_DailyUserSpend" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_DailyUserSpend" ( "id" TEXT NOT NULL, "user_id" TEXT NOT NULL, "date" TEXT NOT NULL, diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20250329084805_new_cron_job_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20250329084805_new_cron_job_table/migration.sql index e7ea2e9015a..ba3000f7512 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20250329084805_new_cron_job_table/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20250329084805_new_cron_job_table/migration.sql @@ -2,7 +2,7 @@ CREATE TYPE "JobStatus" AS ENUM ('ACTIVE', 'INACTIVE'); -- CreateTable -CREATE TABLE "LiteLLM_CronJob" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_CronJob" ( "cronjob_id" TEXT NOT NULL, "pod_id" TEXT NOT NULL, "status" "JobStatus" NOT NULL DEFAULT 'INACTIVE', diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20250411215431_add_managed_file_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20250411215431_add_managed_file_table/migration.sql index 56f7aad0956..602cb5ada37 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20250411215431_add_managed_file_table/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20250411215431_add_managed_file_table/migration.sql @@ -1,5 +1,5 @@ -- CreateTable -CREATE TABLE "LiteLLM_ManagedFileTable" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_ManagedFileTable" ( "id" TEXT NOT NULL, "unified_file_id" TEXT NOT NULL, "file_object" JSONB NOT NULL, diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20250415191926_add_daily_team_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20250415191926_add_daily_team_table/migration.sql index c23d59ad6ee..c8ef4eff7d3 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20250415191926_add_daily_team_table/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20250415191926_add_daily_team_table/migration.sql @@ -1,5 +1,5 @@ -- CreateTable -CREATE TABLE "LiteLLM_DailyTeamSpend" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_DailyTeamSpend" ( "id" TEXT NOT NULL, "team_id" TEXT NOT NULL, "date" TEXT NOT NULL, diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20250416115320_add_tag_table_to_db/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20250416115320_add_tag_table_to_db/migration.sql index 7ba9bb437ba..0a528348144 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20250416115320_add_tag_table_to_db/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20250416115320_add_tag_table_to_db/migration.sql @@ -3,7 +3,7 @@ ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "cache_creation_in ADD COLUMN IF NOT EXISTS "cache_read_input_tokens" INTEGER NOT NULL DEFAULT 0; -- CreateTable -CREATE TABLE "LiteLLM_DailyTagSpend" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_DailyTagSpend" ( "id" TEXT NOT NULL, "tag" TEXT NOT NULL, "date" TEXT NOT NULL, diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20250430193429_add_managed_vector_stores/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20250430193429_add_managed_vector_stores/migration.sql index 39e7f2f3b20..ae73e40cd59 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20250430193429_add_managed_vector_stores/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20250430193429_add_managed_vector_stores/migration.sql @@ -1,5 +1,5 @@ -- CreateTable -CREATE TABLE "LiteLLM_ManagedVectorStoresTable" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_ManagedVectorStoresTable" ( "vector_store_id" TEXT NOT NULL, "custom_llm_provider" TEXT NOT NULL, "vector_store_name" TEXT, diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20250507161526_add_mcp_table_to_db/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20250507161526_add_mcp_table_to_db/migration.sql index 6b8adc6e7e8..95f26d5cb55 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20250507161526_add_mcp_table_to_db/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20250507161526_add_mcp_table_to_db/migration.sql @@ -1,5 +1,5 @@ -- CreateTable -CREATE TABLE "LiteLLM_MCPServerTable" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_MCPServerTable" ( "server_id" TEXT NOT NULL, "alias" TEXT, "description" TEXT, diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20250507184818_add_mcp_key_team_permission_mgmt/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20250507184818_add_mcp_key_team_permission_mgmt/migration.sql index 6c3dd2974ef..9937857875e 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20250507184818_add_mcp_key_team_permission_mgmt/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20250507184818_add_mcp_key_team_permission_mgmt/migration.sql @@ -11,7 +11,7 @@ ALTER TABLE "LiteLLM_UserTable" ADD COLUMN IF NOT EXISTS "object_permission_id" ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN IF NOT EXISTS "object_permission_id" TEXT; -- CreateTable -CREATE TABLE "LiteLLM_ObjectPermissionTable" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_ObjectPermissionTable" ( "object_permission_id" TEXT NOT NULL, "mcp_servers" TEXT[] DEFAULT ARRAY[]::TEXT[], diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20250514142245_add_guardrails_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20250514142245_add_guardrails_table/migration.sql index 5efe1e21e78..41d8c188947 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20250514142245_add_guardrails_table/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20250514142245_add_guardrails_table/migration.sql @@ -1,5 +1,5 @@ -- CreateTable -CREATE TABLE "LiteLLM_GuardrailsTable" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_GuardrailsTable" ( "guardrail_id" TEXT NOT NULL, "guardrail_name" TEXT NOT NULL, "litellm_params" JSONB NOT NULL, diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20250522223020_managed_object_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20250522223020_managed_object_table/migration.sql index 4f9f57b8eeb..564811655f3 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20250522223020_managed_object_table/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20250522223020_managed_object_table/migration.sql @@ -4,7 +4,7 @@ ADD COLUMN IF NOT EXISTS "flat_model_file_ids" TEXT[] DEFAULT ARRAY[]::TEXT[], ADD COLUMN IF NOT EXISTS "updated_by" TEXT; -- CreateTable -CREATE TABLE "LiteLLM_ManagedObjectTable" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_ManagedObjectTable" ( "id" TEXT NOT NULL, "unified_object_id" TEXT NOT NULL, "model_object_id" TEXT NOT NULL, diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20250618225828_add_health_check_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20250618225828_add_health_check_table/migration.sql index 43d78608089..2b4dd1d17d3 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20250618225828_add_health_check_table/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20250618225828_add_health_check_table/migration.sql @@ -1,5 +1,5 @@ -- CreateTable -CREATE TABLE "LiteLLM_HealthCheckTable" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_HealthCheckTable" ( "health_check_id" TEXT NOT NULL, "model_name" TEXT NOT NULL, "model_id" TEXT, diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20250802162330_prompt_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20250802162330_prompt_table/migration.sql index 45c5d2679a4..81b3574499b 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20250802162330_prompt_table/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20250802162330_prompt_table/migration.sql @@ -1,5 +1,5 @@ -- CreateTable -CREATE TABLE "LiteLLM_PromptTable" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_PromptTable" ( "id" TEXT NOT NULL, "prompt_id" TEXT NOT NULL, "litellm_params" JSONB NOT NULL, diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20251011084309_add_tag_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20251011084309_add_tag_table/migration.sql index 4321cd5440c..c1005a75a4a 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20251011084309_add_tag_table/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20251011084309_add_tag_table/migration.sql @@ -1,5 +1,5 @@ -- CreateTable -CREATE TABLE "LiteLLM_TagTable" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_TagTable" ( "tag_name" TEXT NOT NULL, "description" TEXT, "models" TEXT[], diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20251023141814_add_search_tool_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20251023141814_add_search_tool_table/migration.sql index d050c82764d..3cd2a4c4899 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20251023141814_add_search_tool_table/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20251023141814_add_search_tool_table/migration.sql @@ -1,5 +1,5 @@ -- CreateTable -CREATE TABLE "LiteLLM_SearchToolsTable" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_SearchToolsTable" ( "search_tool_id" TEXT NOT NULL, "search_tool_name" TEXT NOT NULL, "litellm_params" JSONB NOT NULL, diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20251031181430_add_cache_config_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20251031181430_add_cache_config_table/migration.sql index 705a6fd4d9b..bb475b03d25 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20251031181430_add_cache_config_table/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20251031181430_add_cache_config_table/migration.sql @@ -1,5 +1,5 @@ -- CreateTable -CREATE TABLE "LiteLLM_SSOConfig" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_SSOConfig" ( "id" TEXT NOT NULL DEFAULT 'sso_config', "sso_settings" JSONB NOT NULL, "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, @@ -9,7 +9,7 @@ CREATE TABLE "LiteLLM_SSOConfig" ( ); -- CreateTable -CREATE TABLE "LiteLLM_CacheConfig" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_CacheConfig" ( "id" TEXT NOT NULL DEFAULT 'cache_config', "cache_settings" JSONB NOT NULL, "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20251101131415_add_managed_vector_store_index_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20251101131415_add_managed_vector_store_index_table/migration.sql index 0b2bcddea70..0997e569171 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20251101131415_add_managed_vector_store_index_table/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20251101131415_add_managed_vector_store_index_table/migration.sql @@ -1,5 +1,5 @@ -- CreateTable -CREATE TABLE "LiteLLM_ManagedVectorStoreIndexTable" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_ManagedVectorStoreIndexTable" ( "id" TEXT NOT NULL, "index_name" TEXT NOT NULL, "litellm_params" JSONB NOT NULL, diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20251113000000_add_project_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20251113000000_add_project_table/migration.sql index b6a66b9444a..9769699b33f 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20251113000000_add_project_table/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20251113000000_add_project_table/migration.sql @@ -1,5 +1,5 @@ -- CreateTable -CREATE TABLE "LiteLLM_ProjectTable" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_ProjectTable" ( "project_id" TEXT NOT NULL, "project_alias" TEXT, "team_id" TEXT, diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20251114180624_Add_org_usage_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20251114180624_Add_org_usage_table/migration.sql index e3e5ef3312c..387b40461a8 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20251114180624_Add_org_usage_table/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20251114180624_Add_org_usage_table/migration.sql @@ -1,5 +1,5 @@ -- CreateTable -CREATE TABLE "LiteLLM_DailyOrganizationSpend" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_DailyOrganizationSpend" ( "id" TEXT NOT NULL, "organization_id" TEXT, "date" TEXT NOT NULL, diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20251114182247_agents_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20251114182247_agents_table/migration.sql index 3267d6fc607..35595c7eedf 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20251114182247_agents_table/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20251114182247_agents_table/migration.sql @@ -1,5 +1,5 @@ -- CreateTable -CREATE TABLE "LiteLLM_AgentsTable" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_AgentsTable" ( "agent_id" TEXT NOT NULL, "agent_name" TEXT NOT NULL, "litellm_params" JSONB, diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20251204124859_add_end_user_spend_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20251204124859_add_end_user_spend_table/migration.sql index 9e3b144336f..3544768cc16 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20251204124859_add_end_user_spend_table/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20251204124859_add_end_user_spend_table/migration.sql @@ -1,5 +1,5 @@ -- CreateTable -CREATE TABLE "LiteLLM_DailyEndUserSpend" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_DailyEndUserSpend" ( "id" TEXT NOT NULL, "end_user_id" TEXT, "date" TEXT NOT NULL, diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20251209112246_add_ui_settings_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20251209112246_add_ui_settings_table/migration.sql index 1719ce646d4..01e2c9fa761 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20251209112246_add_ui_settings_table/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20251209112246_add_ui_settings_table/migration.sql @@ -1,5 +1,5 @@ -- CreateTable -CREATE TABLE "LiteLLM_UISettings" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_UISettings" ( "id" TEXT NOT NULL DEFAULT 'ui_settings', "ui_settings" JSONB NOT NULL, "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20251210205007_add_daily_agent_spend_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20251210205007_add_daily_agent_spend_table/migration.sql index cf25a1ced83..4cc894f9b22 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20251210205007_add_daily_agent_spend_table/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20251210205007_add_daily_agent_spend_table/migration.sql @@ -2,7 +2,7 @@ ALTER TABLE "LiteLLM_SpendLogs" ADD COLUMN IF NOT EXISTS "agent_id" TEXT; -- CreateTable -CREATE TABLE "LiteLLM_DailyAgentSpend" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_DailyAgentSpend" ( "id" TEXT NOT NULL, "agent_id" TEXT, "date" TEXT NOT NULL, diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20251219110931_add_deleted_keys_and_deleted_teams_tables/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20251219110931_add_deleted_keys_and_deleted_teams_tables/migration.sql index 5aae430546b..8acff490bb7 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20251219110931_add_deleted_keys_and_deleted_teams_tables/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20251219110931_add_deleted_keys_and_deleted_teams_tables/migration.sql @@ -1,5 +1,5 @@ -- CreateTable -CREATE TABLE "LiteLLM_DeletedTeamTable" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_DeletedTeamTable" ( "id" TEXT NOT NULL, "team_id" TEXT NOT NULL, "team_alias" TEXT, @@ -33,7 +33,7 @@ CREATE TABLE "LiteLLM_DeletedTeamTable" ( ); -- CreateTable -CREATE TABLE "LiteLLM_DeletedVerificationToken" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_DeletedVerificationToken" ( "id" TEXT NOT NULL, "token" TEXT NOT NULL, "key_name" TEXT, diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20251220144550_schema_update/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20251220144550_schema_update/migration.sql index b40defec309..c79ce17b9ee 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20251220144550_schema_update/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20251220144550_schema_update/migration.sql @@ -1,5 +1,5 @@ -- CreateTable -CREATE TABLE "LiteLLM_SkillsTable" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_SkillsTable" ( "skill_id" TEXT NOT NULL, "display_title" TEXT, "description" TEXT, diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260123131407_add_policy_tables_and_policies_field/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260123131407_add_policy_tables_and_policies_field/migration.sql index 7627c61c924..8b6ac7a7f3e 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260123131407_add_policy_tables_and_policies_field/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260123131407_add_policy_tables_and_policies_field/migration.sql @@ -14,7 +14,7 @@ ALTER TABLE "LiteLLM_UserTable" ADD COLUMN IF NOT EXISTS "policies" TEXT[] DEFAU ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN IF NOT EXISTS "policies" TEXT[] DEFAULT ARRAY[]::TEXT[]; -- CreateTable -CREATE TABLE "LiteLLM_PolicyTable" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_PolicyTable" ( "policy_id" TEXT NOT NULL, "policy_name" TEXT NOT NULL, "inherit" TEXT, @@ -31,7 +31,7 @@ CREATE TABLE "LiteLLM_PolicyTable" ( ); -- CreateTable -CREATE TABLE "LiteLLM_PolicyAttachmentTable" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_PolicyAttachmentTable" ( "attachment_id" TEXT NOT NULL, "policy_name" TEXT NOT NULL, "scope" TEXT, diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260203120000_add_deprecated_verification_token_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260203120000_add_deprecated_verification_token_table/migration.sql index a15f8ea47e1..58b2d4048c2 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260203120000_add_deprecated_verification_token_table/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260203120000_add_deprecated_verification_token_table/migration.sql @@ -1,5 +1,5 @@ -- CreateTable -CREATE TABLE "LiteLLM_DeprecatedVerificationToken" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_DeprecatedVerificationToken" ( "id" TEXT NOT NULL, "token" TEXT NOT NULL, "active_token_id" TEXT NOT NULL, diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260212143306_add_access_group_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260212143306_add_access_group_table/migration.sql index 0ad7ea9395e..b3ae05e202b 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260212143306_add_access_group_table/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260212143306_add_access_group_table/migration.sql @@ -11,7 +11,7 @@ ALTER TABLE "LiteLLM_TeamTable" ADD COLUMN IF NOT EXISTS "access_group_ids" TEXT ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN IF NOT EXISTS "access_group_ids" TEXT[] DEFAULT ARRAY[]::TEXT[]; -- CreateTable -CREATE TABLE "LiteLLM_AccessGroupTable" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_AccessGroupTable" ( "access_group_id" TEXT NOT NULL, "access_group_name" TEXT NOT NULL, "description" TEXT, diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260213105436_add_managed_vector_store_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260213105436_add_managed_vector_store_table/migration.sql index cf9f83fb51e..eb3dd90602d 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260213105436_add_managed_vector_store_table/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260213105436_add_managed_vector_store_table/migration.sql @@ -1,5 +1,5 @@ -- CreateTable -CREATE TABLE "LiteLLM_ManagedVectorStoreTable" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_ManagedVectorStoreTable" ( "id" TEXT NOT NULL, "unified_resource_id" TEXT NOT NULL, "resource_object" JSONB, diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260219181415_baseline_diff/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260219181415_baseline_diff/migration.sql index 7825ff4560f..13be156db8e 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260219181415_baseline_diff/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260219181415_baseline_diff/migration.sql @@ -1,5 +1,5 @@ -- CreateTable -CREATE TABLE "LiteLLM_DailyGuardrailMetrics" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_DailyGuardrailMetrics" ( "guardrail_id" TEXT NOT NULL, "date" TEXT NOT NULL, "requests_evaluated" BIGINT NOT NULL DEFAULT 0, @@ -15,7 +15,7 @@ CREATE TABLE "LiteLLM_DailyGuardrailMetrics" ( ); -- CreateTable -CREATE TABLE "LiteLLM_DailyPolicyMetrics" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_DailyPolicyMetrics" ( "policy_id" TEXT NOT NULL, "date" TEXT NOT NULL, "requests_evaluated" BIGINT NOT NULL DEFAULT 0, @@ -31,7 +31,7 @@ CREATE TABLE "LiteLLM_DailyPolicyMetrics" ( ); -- CreateTable -CREATE TABLE "LiteLLM_SpendLogGuardrailIndex" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_SpendLogGuardrailIndex" ( "request_id" TEXT NOT NULL, "guardrail_id" TEXT NOT NULL, "policy_id" TEXT, diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260224203854_add_agent_object_permissions_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260224203854_add_agent_object_permissions_table/migration.sql index d781fd23743..ecfe47a44d3 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260224203854_add_agent_object_permissions_table/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260224203854_add_agent_object_permissions_table/migration.sql @@ -8,7 +8,7 @@ ALTER TABLE "LiteLLM_MCPServerTable" DROP COLUMN IF EXISTS "spec_path"; ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN IF NOT EXISTS "agent_id" TEXT; -- CreateTable -CREATE TABLE "LiteLLM_ToolTable" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_ToolTable" ( "tool_id" TEXT NOT NULL, "tool_name" TEXT NOT NULL, "origin" TEXT, diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260226120000_add_spend_log_tool_index/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260226120000_add_spend_log_tool_index/migration.sql index c82d50410ae..d9c234696c8 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260226120000_add_spend_log_tool_index/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260226120000_add_spend_log_tool_index/migration.sql @@ -1,5 +1,5 @@ -- CreateTable -CREATE TABLE "LiteLLM_SpendLogToolIndex" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_SpendLogToolIndex" ( "request_id" TEXT NOT NULL, "tool_name" TEXT NOT NULL, "start_time" TIMESTAMP(3) NOT NULL, diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260228000000_add_claude_code_plugin_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260228000000_add_claude_code_plugin_table/migration.sql index 1e7cc7fbc2e..7fa354d9d80 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260228000000_add_claude_code_plugin_table/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260228000000_add_claude_code_plugin_table/migration.sql @@ -1,5 +1,5 @@ -- CreateTable -CREATE TABLE "LiteLLM_ClaudeCodePluginTable" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_ClaudeCodePluginTable" ( "id" TEXT NOT NULL, "name" TEXT NOT NULL, "version" TEXT, diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260306233848_schema_sync/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260306233848_schema_sync/migration.sql index 6c8161d8e4f..4f3c0b7485b 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260306233848_schema_sync/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260306233848_schema_sync/migration.sql @@ -6,7 +6,7 @@ ADD COLUMN IF NOT EXISTS "tool_name_to_description" JSONB DEFAULT '{}', ADD COLUMN IF NOT EXISTS "tool_name_to_display_name" JSONB DEFAULT '{}'; -- CreateTable -CREATE TABLE "LiteLLM_MCPUserCredentials" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_MCPUserCredentials" ( "id" TEXT NOT NULL, "user_id" TEXT NOT NULL, "server_id" TEXT NOT NULL, @@ -18,7 +18,7 @@ CREATE TABLE "LiteLLM_MCPUserCredentials" ( ); -- CreateTable -CREATE TABLE "LiteLLM_JWTKeyMapping" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_JWTKeyMapping" ( "id" TEXT NOT NULL, "jwt_claim_name" TEXT NOT NULL, "jwt_claim_value" TEXT NOT NULL, @@ -34,7 +34,7 @@ CREATE TABLE "LiteLLM_JWTKeyMapping" ( ); -- CreateTable -CREATE TABLE "LiteLLM_ConfigOverrides" ( +CREATE TABLE IF NOT EXISTS "LiteLLM_ConfigOverrides" ( "config_type" TEXT NOT NULL, "config_value" JSONB NOT NULL, "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, diff --git a/litellm-proxy-extras/litellm_proxy_extras/utils.py b/litellm-proxy-extras/litellm_proxy_extras/utils.py index 5b4feed2564..7eff0c00f75 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/utils.py +++ b/litellm-proxy-extras/litellm_proxy_extras/utils.py @@ -415,15 +415,26 @@ class ProxyExtrasDBManager: logger.info( f"Migration {failed_migration} failed due to idempotent error (e.g., column already exists), resolving as applied" ) - ProxyExtrasDBManager._roll_back_migration( - failed_migration - ) - ProxyExtrasDBManager._resolve_specific_migration( - failed_migration - ) - logger.info( - f"✅ Migration {failed_migration} resolved, retrying to apply remaining migrations" - ) + try: + ProxyExtrasDBManager._roll_back_migration( + failed_migration + ) + except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as rollback_err: + logger.warning( + f"Failed to roll back migration {failed_migration}: {rollback_err}. " + f"It may already be in a rolled-back state." + ) + try: + ProxyExtrasDBManager._resolve_specific_migration( + failed_migration + ) + logger.info( + f"✅ Migration {failed_migration} resolved, retrying to apply remaining migrations" + ) + except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as resolve_err: + logger.warning( + f"Failed to resolve migration {failed_migration}: {resolve_err}" + ) else: logger.info( f"Found failed migration: {failed_migration}, marking as rolled back" diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 44a7c011df3..0f6cc592ff5 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -557,11 +557,11 @@ class ProxyInitializationHelpers: envvar="MAX_REQUESTS_BEFORE_RESTART", ) @click.option( - "--require_db_migration", + "--skip_db_migration_check", is_flag=True, default=False, - help="Exit with error if database migration fails. By default the proxy will warn and continue.", - envvar="REQUIRE_DB_MIGRATION", + help="Warn and continue instead of exiting when database migration fails.", + envvar="SKIP_DB_MIGRATION_CHECK", ) def run_server( # noqa: PLR0915 host, @@ -602,7 +602,7 @@ def run_server( # noqa: PLR0915 skip_server_startup, keepalive_timeout, max_requests_before_restart, - require_db_migration: bool, + skip_db_migration_check: bool, ): args = locals() if local: @@ -865,17 +865,17 @@ def run_server( # noqa: PLR0915 if not PrismaManager.setup_database( use_migrate=not use_prisma_db_push ): - if require_db_migration: + if skip_db_migration_check: + print( # noqa + "\033[1;33mLiteLLM Proxy: Database migration failed but continuing startup. " + "Pass --skip_db_migration_check to allow this.\033[0m" + ) + else: print( # noqa "\033[1;31mLiteLLM Proxy: Database setup failed after multiple retries. " "The proxy cannot start safely. Please check your database connection and migration status.\033[0m" ) sys.exit(1) - else: - print( # noqa - "\033[1;33mLiteLLM Proxy: Database migration failed but continuing startup. " - "Use --require_db_migration to exit on migration failure.\033[0m" - ) else: print( # noqa f"Unable to connect to DB. DATABASE_URL found in environment, but prisma package not found." # noqa diff --git a/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py b/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py index 05f4b8a98e1..c1e3fd5072f 100644 --- a/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py +++ b/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py @@ -180,6 +180,20 @@ class TestMigrationSQLIdempotency: ) return migrations + def test_create_table_uses_if_not_exists(self, all_migrations): + """CREATE TABLE statements must use IF NOT EXISTS""" + violations = [] + for migration_name, sql in all_migrations: + for line_num, line in enumerate(sql.splitlines(), 1): + if re.search(r"CREATE\s+TABLE\s+", line, re.IGNORECASE) and not re.search( + r"CREATE\s+TABLE\s+IF\s+NOT\s+EXISTS", line, re.IGNORECASE + ): + violations.append(f" {migration_name}:{line_num}: {line.strip()}") + assert not violations, ( + "CREATE TABLE without IF NOT EXISTS found in migrations:\n" + + "\n".join(violations) + ) + def test_add_column_uses_if_not_exists(self, all_migrations): """ADD COLUMN statements must use IF NOT EXISTS""" violations = []