scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-03 02:22:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 81

fix(proxy): always exit when database setup fails at boot (#44141)

Browse source 
Remove the ENFORCE_PRISMA_MIGRATION_CHECK opt-in. When PrismaManager.setup_database returns False (database unreachable, connection retries exhausted, or prisma migrate deploy failing after retries) the proxy now always prints the red failure message and exits 1 instead of serving requests against a database whose schema may be behind the code. The --enforce_prisma_migration_check flag stays as a hidden no-op that prints a one-line deprecation warning so existing container args keep parsing; the env var is no longer read anywhere. The standalone migration entrypoint always runs run_server(("--skip_server_startup",)), and the integration launchers drop the flag.

Co-authored-by: yassin <yassin@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot] 2026-10-01 21:51:06 -07:00 • committed by GitHub
parent 5ccb1a143b
commit e6639d15b5
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 126 additions and 65 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.circleci/scripts/run_integration.sh
View file

@ -173,7 +173,7 @@ start_proxy() {
AWS_EC2_METADATA_DISABLED=true DO_NOT_TRACK=1 COVERAGE_FILE="$coverage_data" \
"${proxy_command[@]}" --config tests/integration/proxy_config.yaml \
--host 127.0.0.1 --port "$port" --num_workers 1 --telemetry False \
--use_prisma_db_push --enforce_prisma_migration_check \
--use_prisma_db_push \
> "$results/$log_name" 2>&1 &
launched_pid=$!
}

17
litellm/proxy/prisma_migration.py
View file

@ -1,9 +1,9 @@
"""Standalone entrypoint for applying database migrations and generating the Prisma client.
Migration failures fail the entrypoint by default; set ENFORCE_PRISMA_MIGRATION_CHECK=false
for log-only behavior. A failed 'prisma generate' is always log-only: every shipped image
bakes the client at build time, and refreshing it writes into site-packages, which an
arbitrary non-root uid or a read-only root filesystem cannot do.
A failed migration fails the entrypoint, the same way it fails proxy startup. A failed
'prisma generate' is log-only: every shipped image bakes the client at build time, and
refreshing it writes into site-packages, which an arbitrary non-root uid or a read-only
root filesystem cannot do.
"""
import os
@ -18,17 +18,10 @@ from litellm_proxy_extras.prisma_toolchain import resolve_prisma_argv
from litellm._logging import verbose_proxy_logger
from litellm.proxy.proxy_cli import run_server
from litellm.secret_managers.main import str_to_bool
def main() -> int:
enforce_prisma_migration_check: Final = str_to_bool(os.getenv("ENFORCE_PRISMA_MIGRATION_CHECK")) is not False
run_server_args: Final = (
("--skip_server_startup", "--enforce_prisma_migration_check")
if enforce_prisma_migration_check
else ("--skip_server_startup",)
)
run_server(run_server_args, standalone_mode=False)
run_server(("--skip_server_startup",), standalone_mode=False)
verbose_proxy_logger.info("Running 'prisma generate'...")
result: Final = subprocess.run(resolve_prisma_argv(("prisma", "generate")), capture_output=True, text=True)

29
litellm/proxy/proxy_cli.py
View file

@ -966,8 +966,11 @@ class ProxyInitializationHelpers:
"--enforce_prisma_migration_check",
is_flag=True,
default=False,
help="Exit with error if database migration fails on startup.",
envvar="ENFORCE_PRISMA_MIGRATION_CHECK",
hidden=True,
help=(
"Deprecated and ignored: the proxy always exits when database setup fails at "
"startup. It is still accepted so existing commands keep working."
),
)
@click.option(
"--use_v2_migration_resolver",
@ -1098,6 +1101,12 @@ def run_server(
if validate_config is True:
ProxyInitializationHelpers._run_config_validation(config)
return
if enforce_prisma_migration_check:
print(
"\033[1;33mLiteLLM Proxy: --enforce_prisma_migration_check is "
"deprecated and has no effect, because the proxy always exits "
"when database setup fails at startup. You can safely remove it.\033[0m"
)
if model and "ollama" in model and api_base is None:
ProxyInitializationHelpers._run_ollama_serve()
if health is True:
@ -1431,17 +1440,11 @@ def run_server(
)
sys.exit(2)
if not setup_ok:
if enforce_prisma_migration_check:
print(
"\033[1;31mLiteLLM Proxy: Database setup failed after multiple retries. "
"The proxy cannot start safely. Please check your database connection and migration status.\033[0m"
)
sys.exit(1)
else:
print(
"\033[1;33mLiteLLM Proxy: Database migration failed but continuing startup. "
"Set --enforce_prisma_migration_check or ENFORCE_PRISMA_MIGRATION_CHECK=true to exit on failure.\033[0m"
)
print(
"\033[1;31mLiteLLM Proxy: Database setup failed after multiple retries. "
"The proxy cannot start safely. Please check your database connection and migration status.\033[0m"
)
sys.exit(1)
else:
print(
"Unable to connect to DB. DATABASE_URL found in environment, but the prisma CLI is neither on "

1
tests/integration/_support/process.py
View file

@ -209,7 +209,6 @@ def owned_proxy_process(
"--num_workers",
str(workers),
*database_setup,
"--enforce_prisma_migration_check",
)
launch: Final = _launch_until_bound(command, root, environment, output, _PORT_ATTEMPTS)
process: Final = launch.process

1
tests/integration/database/test_request_log_indexes_at_boot.py
View file

@ -116,7 +116,6 @@ def migration_cli(database_url: str, gateway: Gateway, resolver: Resolver) -> su
"tests/integration/proxy_config.yaml",
*resolver.proxy_flags,
"--skip_server_startup",
"--enforce_prisma_migration_check",
],
capture_output=True,
text=True,

2
tests/unit/proxy/db/test_prisma_client.py
View file

@ -448,7 +448,7 @@ def test_db_push_without_the_prisma_runner_fails_the_migration_instead_of_crashi
):
"""
An ImportError out of setup_database escapes the caller's RuntimeError handler and
kills boot, bypassing the operator's enforce_prisma_migration_check choice.
kills boot with a traceback instead of the failed-setup message and exit code.
"""
monkeypatch.setitem(sys.modules, "litellm_proxy_extras.prisma_toolchain", None)

32
tests/unit/proxy/test_prisma_migration.py
View file

@ -9,41 +9,23 @@ from litellm.proxy import prisma_migration
class TestPrismaMigration:
@pytest.mark.parametrize("env", [{}, {"ENFORCE_PRISMA_MIGRATION_CHECK": "false"}], ids=("unset", "legacy-opt-out"))
@patch("litellm.proxy.prisma_migration.subprocess.run")
@patch("litellm.proxy.prisma_migration.run_server")
def test_main_enforces_migration_check_by_default(
self, mock_run_server: MagicMock, mock_subprocess_run: MagicMock
def test_main_runs_the_migration_job_with_no_opt_out(
self, mock_run_server: MagicMock, mock_subprocess_run: MagicMock, env: dict[str, str]
) -> None:
mock_subprocess_run.return_value = MagicMock(returncode=0, stdout="", stderr="")
with patch.dict(os.environ, {}, clear=True):
assert prisma_migration.main() == 0
mock_run_server.assert_called_once_with(
("--skip_server_startup", "--enforce_prisma_migration_check"),
standalone_mode=False,
)
@patch("litellm.proxy.prisma_migration.subprocess.run")
@patch("litellm.proxy.prisma_migration.run_server")
def test_main_disables_migration_check_when_explicitly_false(
self, mock_run_server: MagicMock, mock_subprocess_run: MagicMock
) -> None:
mock_subprocess_run.return_value = MagicMock(returncode=0, stdout="", stderr="")
with patch.dict(os.environ, {"ENFORCE_PRISMA_MIGRATION_CHECK": "false"}, clear=True):
with patch.dict(os.environ, env, clear=True):
assert prisma_migration.main() == 0
mock_run_server.assert_called_once_with(("--skip_server_startup",), standalone_mode=False)
@pytest.mark.parametrize("env", [{}, {"ENFORCE_PRISMA_MIGRATION_CHECK": "false"}])
@patch("litellm.proxy.prisma_migration.subprocess.run")
@patch("litellm.proxy.prisma_migration.run_server")
def test_main_exits_zero_when_only_prisma_generate_fails(
self,
mock_run_server: MagicMock,
mock_subprocess_run: MagicMock,
env: dict[str, str],
self, mock_run_server: MagicMock, mock_subprocess_run: MagicMock
) -> None:
mock_subprocess_run.return_value = MagicMock(
returncode=1,
@ -51,7 +33,7 @@ class TestPrismaMigration:
stderr="PermissionError: [Errno 13] Permission denied: '/app/.venv/lib/python3.13/site-packages/prisma/schema.prisma'",
)
with patch.dict(os.environ, env, clear=True):
with patch.dict(os.environ, {}, clear=True):
assert prisma_migration.main() == 0
@patch("litellm.proxy.prisma_migration.subprocess.run")
@ -61,7 +43,7 @@ class TestPrismaMigration:
) -> None:
mock_run_server.side_effect = SystemExit(1)
with patch.dict(os.environ, {}, clear=True):
with patch.dict(os.environ, {"ENFORCE_PRISMA_MIGRATION_CHECK": "false"}, clear=True):
with pytest.raises(SystemExit, match="1"):
prisma_migration.main()

107
tests/unit/proxy/test_proxy_cli.py
View file

@ -2280,7 +2280,7 @@ class TestRunServerDbSetup:
mock_atexit_register,
mock_subprocess_run,
):
"""Test that proxy exits with code 1 when PrismaManager.setup_database returns False and --enforce_prisma_migration_check is set"""
"""Test that proxy exits with code 1 when PrismaManager.setup_database returns False, with no opt-in flag"""
from litellm.proxy.proxy_cli import run_server
mock_subprocess_run.return_value = MagicMock(returncode=0)
@ -2321,14 +2321,7 @@ class TestRunServerDbSetup:
}
with pytest.raises(SystemExit) as exc_info:
run_server.main(
[
"--local",
"--skip_server_startup",
"--enforce_prisma_migration_check",
],
standalone_mode=False,
)
run_server.main(["--local", "--skip_server_startup"], standalone_mode=False)
assert exc_info.value.code == 1
mock_setup_database.assert_called_once_with(use_migrate=True, use_v2_resolver=True)
@ -2445,6 +2438,98 @@ class TestRunServerDbSetup:
mock_setup_database.assert_called_once_with(use_migrate=True, use_v2_resolver=True)
assert "--use_v2_migration_resolver is deprecated" not in capsys.readouterr().out
@pytest.mark.parametrize(
("arguments", "environment", "warned"),
(
(("--local", "--skip_server_startup", "--enforce_prisma_migration_check"), {}, True),
(("--local", "--skip_server_startup"), {"ENFORCE_PRISMA_MIGRATION_CHECK": "true"}, False),
(("--local", "--skip_server_startup"), {"ENFORCE_PRISMA_MIGRATION_CHECK": "false"}, False),
),
ids=("cli-flag", "env-true", "env-false"),
)
@patch("subprocess.run")
@patch("atexit.register")
@patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database", return_value=True)
@patch("litellm.proxy.db.prisma_client.PrismaManager.build_request_log_indexes", return_value=True)
@patch("litellm.proxy.db.check_migration.check_prisma_schema_diff")
@patch("litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=True)
def test_the_retired_enforce_prisma_migration_check_opt_in_still_parses_and_changes_nothing(
self,
mock_should_update_schema,
mock_check_schema_diff,
mock_build_indexes,
mock_setup_database,
mock_atexit_register,
mock_subprocess_run,
arguments,
environment,
warned,
capsys,
):
"""Deployments still pass the flag or set the env var; the flag is accepted with a
deprecation line and the env var is ignored, and a successful setup boots either way."""
from litellm.proxy.proxy_cli import run_server
mock_subprocess_run.return_value = MagicMock(returncode=0)
mock_proxy_module = MagicMock(
app=MagicMock(),
ProxyConfig=MagicMock(),
KeyManagementSettings=MagicMock(),
save_worker_config=MagicMock(),
)
clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")}
clean_env["DATABASE_URL"] = "postgresql://test:test@localhost:5432/test"
with (
patch.dict(os.environ, {**clean_env, **environment}, clear=True),
patch.dict(
"sys.modules",
{"proxy_server": mock_proxy_module, "litellm.proxy.proxy_server": mock_proxy_module},
),
):
run_server.main(list(arguments), standalone_mode=False)
mock_setup_database.assert_called_once_with(use_migrate=True, use_v2_resolver=True)
assert ("--enforce_prisma_migration_check is deprecated and has no effect" in capsys.readouterr().out) is warned
@patch("subprocess.run")
@patch("atexit.register")
@patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database", return_value=True)
def test_the_retired_enforce_prisma_migration_check_opt_in_warns_without_a_database(
self,
mock_setup_database,
mock_atexit_register,
mock_subprocess_run,
capsys,
):
"""The deprecation line does not depend on reaching database setup: a deployment that
passes the flag with no DATABASE_URL still learns the flag is dead."""
from litellm.proxy.proxy_cli import run_server
mock_subprocess_run.return_value = MagicMock(returncode=0)
mock_proxy_module = MagicMock(
app=MagicMock(),
ProxyConfig=MagicMock(),
KeyManagementSettings=MagicMock(),
save_worker_config=MagicMock(),
)
clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")}
with (
patch.dict(os.environ, clean_env, clear=True),
patch.dict(
"sys.modules",
{"proxy_server": mock_proxy_module, "litellm.proxy.proxy_server": mock_proxy_module},
),
):
run_server.main(
["--local", "--skip_server_startup", "--enforce_prisma_migration_check"],
standalone_mode=False,
)
mock_setup_database.assert_not_called()
assert "--enforce_prisma_migration_check is deprecated and has no effect" in capsys.readouterr().out
@pytest.mark.parametrize(
"use_legacy_flag, env_value, expected",
[
@ -2573,7 +2658,7 @@ class TestRunServerDbSetup:
"""`--skip_server_startup` is the migration job: it waits for the index build after the
migrations and exits 1 when one could not be built. A serving proxy that ran the
migrations starts the build in the background and serves whatever the build does; one
whose migrations failed exits 1 under `--enforce_prisma_migration_check` and starts no build."""
whose migrations failed exits 1 and starts no build."""
from litellm.proxy.proxy_cli import run_server
mock_setup_database.return_value = migrated
@ -2600,7 +2685,7 @@ class TestRunServerDbSetup:
outcome as exc_info,
):
mock_get_args.return_value = {"app": "litellm.proxy.proxy_server:app", "host": "localhost", "port": 8000}
run_server.main([*arguments, "--enforce_prisma_migration_check"], standalone_mode=False)
run_server.main(list(arguments), standalone_mode=False)
assert (exc_info is not None and exc_info.value.code == 1) is exits
mock_setup_database.assert_called_once_with(use_migrate=True, use_v2_resolver=True)