From fac518dbf7dd0449b6cbd8f941c822f695f11233 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sun, 20 Sep 2026 00:08:06 -0700 Subject: [PATCH 1/4] feat(proxy): default to the v2 migration resolver The migrations Job entrypoint (migrations/run.py) has defaulted to v2 with USE_V2_MIGRATION_RESOLVER=false as the opt-out, and the Helm chart documents that knob. Proxy startup still defaulted to v1, so the two paths disagreed about which resolver a deployment runs. Proxy startup now resolves the same way: v2 unless USE_V2_MIGRATION_RESOLVER is false or --use_legacy_migration_resolver is passed. - --use_v2_migration_resolver stays accepted as a no-op that warns, so existing commands and Helm values do not fail on an unknown option. - The dedicated Postgres smoke-test job is repointed at the legacy resolver so v1 keeps real-DB proxy-boot coverage, and the two jobs that deselected it by name are updated to match the rename. #39178 reverted an earlier flip because two replicas sharing a database deadlocked (40P01 / P3018) with neither answering /health/liveliness. That contention is what #40932 coordinates, which is why this builds on it. --- .circleci/config.yml | 8 +- litellm/proxy/proxy_cli.py | 50 ++++++++-- .../test_basic_python_version.py | 10 +- tests/test_litellm/proxy/test_proxy_cli.py | 92 +++++++++++++++++-- 4 files changed, 135 insertions(+), 25 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2dcedbfac4a..6d5b9a2258e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1508,7 +1508,7 @@ jobs: - run: name: Run tests command: | - uv run --no-sync python -m pytest -vv tests/local_testing/test_basic_python_version.py -k "not v2_resolver" + uv run --no-sync python -m pytest -vv tests/local_testing/test_basic_python_version.py -k "not legacy_resolver" installing_litellm_on_python_3_13: docker: @@ -1532,7 +1532,7 @@ jobs: - run: name: Run tests command: | - uv run --no-sync python -m pytest -v tests/local_testing/test_basic_python_version.py -k "not v2_resolver" + uv run --no-sync python -m pytest -v tests/local_testing/test_basic_python_version.py -k "not legacy_resolver" installing_litellm_on_python_v2_migration_resolver: docker: @@ -1561,10 +1561,10 @@ jobs: url: tcp://localhost:5432 timeout: "60" - run: - name: Run v2 migration resolver proxy smoke test + name: Run legacy migration resolver proxy smoke test command: | uv run --no-sync python -m pytest -vv \ - tests/local_testing/test_basic_python_version.py::test_litellm_proxy_server_config_no_general_settings_v2_resolver + tests/local_testing/test_basic_python_version.py::test_litellm_proxy_server_config_no_general_settings_legacy_resolver helm_chart_testing: machine: diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 464d1141f8d..14d0331c0ff 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -181,6 +181,14 @@ def append_query_params(url: str | None, params: dict) -> str: return modified_url +def resolve_v2_migration_resolver(*, use_legacy_flag: bool) -> bool: + from litellm_proxy_extras.utils import str_to_bool + + if use_legacy_flag: + return False + return bool(str_to_bool(os.getenv("USE_V2_MIGRATION_RESOLVER", "true"))) + + class ProxyInitializationHelpers: @staticmethod def _echo_litellm_version(): @@ -932,12 +940,24 @@ class ProxyInitializationHelpers: is_flag=True, default=False, help=( - "Opt into the v2 migration resolver. Avoids the diff-and-force recovery " - "path that can cause schema thrashing during rolling deploys where two " - "LiteLLM versions contend for the same DB. Default is the v1 resolver." + "Deprecated and ignored: the v2 migration resolver is now the default, " + "so this flag has no effect. It is still accepted so existing commands " + "keep working. Pass --use_legacy_migration_resolver, or set " + "USE_V2_MIGRATION_RESOLVER=false, to opt back into v1." ), envvar="USE_V2_MIGRATION_RESOLVER", ) +@click.option( + "--use_legacy_migration_resolver", + is_flag=True, + default=False, + help=( + "Fall back to the legacy v1 migration resolver. By default the proxy " + "uses the v2 resolver, which avoids the diff-and-force recovery path " + "that can cause schema thrashing during rolling deploys where two " + "LiteLLM versions contend for the same DB." + ), +) @click.option( "--reload", is_flag=True, @@ -1005,6 +1025,7 @@ def run_server( limit_concurrency: int | None, enforce_prisma_migration_check: bool, use_v2_migration_resolver: bool, + use_legacy_migration_resolver: bool, reload: bool, prometheus_metrics_port: int | None, ): @@ -1346,17 +1367,28 @@ def run_server( if should_update_prisma_schema(general_settings.get("disable_prisma_schema_update")) is False: check_prisma_schema_diff(db_url=None) else: - if not use_v2_migration_resolver: + use_v2_resolver: Final = resolve_v2_migration_resolver( + use_legacy_flag=use_legacy_migration_resolver + ) + if use_v2_migration_resolver and use_v2_resolver: print( - "\033[1;33mLiteLLM Proxy: Using default (v1) migration resolver. " - "If your deployment has seen schema thrashing during rolling " - "deploys, try --use_v2_migration_resolver (safer: avoids the " - "diff-and-force recovery that caused the thrash).\033[0m" + "\033[1;33mLiteLLM Proxy: --use_v2_migration_resolver is " + "deprecated and has no effect \u2014 the v2 migration resolver " + "is now the default. You can safely remove it. To opt back " + "into the legacy v1 resolver, pass " + "--use_legacy_migration_resolver.\033[0m" + ) + if not use_v2_resolver: + print( + "\033[1;33mLiteLLM Proxy: Using the legacy (v1) migration " + "resolver. It performs the diff-and-force recovery that can " + "cause schema thrashing during rolling deploys where two " + "LiteLLM versions contend for the same DB.\033[0m" ) try: setup_ok: Final = PrismaManager.setup_database( use_migrate=not use_prisma_db_push, - use_v2_resolver=use_v2_migration_resolver, + use_v2_resolver=use_v2_resolver, ) except RuntimeError as e: # Raised on unrecoverable migration errors: the v2 diff --git a/tests/local_testing/test_basic_python_version.py b/tests/local_testing/test_basic_python_version.py index fb06ed6b69d..0ce59332417 100644 --- a/tests/local_testing/test_basic_python_version.py +++ b/tests/local_testing/test_basic_python_version.py @@ -305,14 +305,14 @@ def _run_proxy_server_smoke_test(extra_proxy_args=None): def test_litellm_proxy_server_config_no_general_settings(): - """Exercises the default (v1) migration resolver.""" + """Exercises the default (v2) migration resolver.""" _run_proxy_server_smoke_test() -def test_litellm_proxy_server_config_no_general_settings_v2_resolver(): - """Exercises the opt-in v2 migration resolver. +def test_litellm_proxy_server_config_no_general_settings_legacy_resolver(): + """Exercises the opt-out legacy (v1) migration resolver. Runs in a separate CI job against a local Postgres to avoid collisions - with the v1 variant when they share a database. + with the default variant when they share a database. """ - _run_proxy_server_smoke_test(extra_proxy_args=["--use_v2_migration_resolver"]) + _run_proxy_server_smoke_test(extra_proxy_args=["--use_legacy_migration_resolver"]) diff --git a/tests/test_litellm/proxy/test_proxy_cli.py b/tests/test_litellm/proxy/test_proxy_cli.py index 8cbae859b5c..4b83044b36d 100644 --- a/tests/test_litellm/proxy/test_proxy_cli.py +++ b/tests/test_litellm/proxy/test_proxy_cli.py @@ -1995,7 +1995,7 @@ class TestRunServerDbSetup: # use_prisma_db_push should be False (default), so use_migrate should be True run_server.main(["--local", "--skip_server_startup"], standalone_mode=False) mock_setup_database.assert_called_with( - use_migrate=True, use_v2_resolver=False + use_migrate=True, use_v2_resolver=True ) # Reset mocks @@ -2010,7 +2010,7 @@ class TestRunServerDbSetup: standalone_mode=False, ) mock_setup_database.assert_called_with( - use_migrate=False, use_v2_resolver=False + use_migrate=False, use_v2_resolver=True ) @patch("atexit.register") @@ -2070,7 +2070,7 @@ class TestRunServerDbSetup: assert "prisma CLI is neither on PATH" not in capsys.readouterr().out mock_setup_database.assert_called_once_with( - use_migrate=True, use_v2_resolver=False + use_migrate=True, use_v2_resolver=True ) @patch("subprocess.run") @@ -2137,7 +2137,7 @@ class TestRunServerDbSetup: ) assert exc_info.value.code == 1 mock_setup_database.assert_called_once_with( - use_migrate=True, use_v2_resolver=False + use_migrate=True, use_v2_resolver=True ) @patch("subprocess.run") @@ -2204,11 +2204,11 @@ class TestRunServerDbSetup: mock_atexit_register, mock_subprocess_run, ): - """USE_V2_MIGRATION_RESOLVER must select the v2 resolver. + """USE_V2_MIGRATION_RESOLVER=true must select the v2 resolver. The Helm migrations Job runs `python litellm/proxy/prisma_migration.py`, - which calls run_server with a fixed argv, so a deployment has no way to - pass --use_v2_migration_resolver and an env var is the only route in. + which calls run_server with a fixed argv, so a deployment reaches the + resolver through the env var rather than a CLI flag. """ from litellm.proxy.proxy_cli import run_server @@ -2249,6 +2249,84 @@ class TestRunServerDbSetup: use_migrate=True, use_v2_resolver=True ) + @pytest.mark.parametrize( + "argv_extra, env_extra, expected_v2", + [ + ([], {}, True), + ([], {"USE_V2_MIGRATION_RESOLVER": "false"}, False), + (["--use_legacy_migration_resolver"], {}, False), + ( + ["--use_legacy_migration_resolver"], + {"USE_V2_MIGRATION_RESOLVER": "true"}, + False, + ), + (["--use_v2_migration_resolver"], {}, True), + ], + ids=[ + "default-is-v2", + "env-false-opts-out", + "legacy-flag-opts-out", + "legacy-flag-beats-env-true", + "deprecated-v2-flag-still-accepted", + ], + ) + @patch("subprocess.run") + @patch("atexit.register") + @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") # test-quality-ok: run_server always wires the DB; same isolation as the sibling CLI tests above + @patch("litellm.proxy.db.check_migration.check_prisma_schema_diff") # test-quality-ok: run_server always wires the DB; same isolation as the sibling CLI tests above + @patch("litellm.proxy.db.prisma_client.should_update_prisma_schema") # test-quality-ok: run_server always wires the DB; same isolation as the sibling CLI tests above + def test_migration_resolver_selection( + self, + mock_should_update_schema, + mock_check_schema_diff, + mock_setup_database, + mock_atexit_register, + mock_subprocess_run, + argv_extra, + env_extra, + expected_v2, + ): + from litellm.proxy.proxy_cli import run_server + + mock_subprocess_run.return_value = MagicMock(returncode=0) + mock_should_update_schema.return_value = True + mock_setup_database.return_value = True + + mock_proxy_module = MagicMock( + app=MagicMock(), + ProxyConfig=MagicMock(), + KeyManagementSettings=MagicMock(), + save_worker_config=MagicMock(), + ) + + clean_env = { + k: v + for k, v in os.environ.items() + if k + not in ("DATABASE_URL", "DIRECT_URL", "USE_V2_MIGRATION_RESOLVER") + } + clean_env["DATABASE_URL"] = "postgresql://test:test@localhost:5432/test" + clean_env.update(env_extra) + + with ( + patch.dict(os.environ, clean_env, clear=True), + patch.dict( + "sys.modules", + { + "proxy_server": mock_proxy_module, + "litellm.proxy.proxy_server": mock_proxy_module, + }, + ), + ): + run_server.main( + ["--local", "--skip_server_startup", *argv_extra], + standalone_mode=False, + ) + + mock_setup_database.assert_called_once_with( + use_migrate=True, use_v2_resolver=expected_v2 + ) + # --- Module-level helpers for worker startup hook tests --- From 556c7f6b68ec5e38bc13a3d9ad10b58a300ef7bc Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sun, 20 Sep 2026 00:50:05 -0700 Subject: [PATCH 2/4] ci: keep real-database coverage for both migration resolvers The Postgres-backed smoke job previously exercised one resolver. Running the default and the legacy variants in it covers v2 now that it is the default, without losing v1's coverage. --- .circleci/config.yml | 3 ++- tests/local_testing/test_basic_python_version.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6d5b9a2258e..ba107472b8d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1561,9 +1561,10 @@ jobs: url: tcp://localhost:5432 timeout: "60" - run: - name: Run legacy migration resolver proxy smoke test + name: Run both migration resolvers against Postgres command: | uv run --no-sync python -m pytest -vv \ + tests/local_testing/test_basic_python_version.py::test_litellm_proxy_server_config_no_general_settings \ tests/local_testing/test_basic_python_version.py::test_litellm_proxy_server_config_no_general_settings_legacy_resolver helm_chart_testing: diff --git a/tests/local_testing/test_basic_python_version.py b/tests/local_testing/test_basic_python_version.py index 0ce59332417..ef500fdff42 100644 --- a/tests/local_testing/test_basic_python_version.py +++ b/tests/local_testing/test_basic_python_version.py @@ -312,7 +312,7 @@ def test_litellm_proxy_server_config_no_general_settings(): def test_litellm_proxy_server_config_no_general_settings_legacy_resolver(): """Exercises the opt-out legacy (v1) migration resolver. - Runs in a separate CI job against a local Postgres to avoid collisions - with the default variant when they share a database. + Runs after the default variant in the CI job that provides a local + Postgres, so both resolvers get real-database proxy-boot coverage. """ _run_proxy_server_smoke_test(extra_proxy_args=["--use_legacy_migration_resolver"]) From 9aec964bace897dd4713ee5820da357b35d19eaa Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sun, 20 Sep 2026 01:02:19 -0700 Subject: [PATCH 3/4] Merge remote-tracking branch 'origin/main' into litellm_flip_v2_migration_resolver_default Drops the TQ008 suppressions the new test carried; main removed that rule. --- tests/test_litellm/proxy/test_proxy_cli.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_litellm/proxy/test_proxy_cli.py b/tests/test_litellm/proxy/test_proxy_cli.py index 4b83044b36d..d2835142194 100644 --- a/tests/test_litellm/proxy/test_proxy_cli.py +++ b/tests/test_litellm/proxy/test_proxy_cli.py @@ -2272,9 +2272,9 @@ class TestRunServerDbSetup: ) @patch("subprocess.run") @patch("atexit.register") - @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") # test-quality-ok: run_server always wires the DB; same isolation as the sibling CLI tests above - @patch("litellm.proxy.db.check_migration.check_prisma_schema_diff") # test-quality-ok: run_server always wires the DB; same isolation as the sibling CLI tests above - @patch("litellm.proxy.db.prisma_client.should_update_prisma_schema") # test-quality-ok: run_server always wires the DB; same isolation as the sibling CLI tests above + @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") + @patch("litellm.proxy.db.check_migration.check_prisma_schema_diff") + @patch("litellm.proxy.db.prisma_client.should_update_prisma_schema") def test_migration_resolver_selection( self, mock_should_update_schema, From 129c4a703b7224965b7d0c6ff402a4db2b85a635 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sun, 20 Sep 2026 01:21:01 -0700 Subject: [PATCH 4/4] fix(proxy): only warn about the deprecated flag when it came from the CLI USE_V2_MIGRATION_RESOLVER=true is a supported way to select v2, but click sets the same parameter from that env var, so the deprecation notice fired for environment-based config that is not deprecated. The notice now keys off click's parameter source. Also drops an em dash from the notice, and moves the resolver decision under mock-free tests by making it take the env value as an argument. --- litellm/proxy/proxy_cli.py | 25 ++++++--- tests/test_litellm/proxy/test_proxy_cli.py | 63 ++++++++++++++-------- 2 files changed, 58 insertions(+), 30 deletions(-) diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 14d0331c0ff..78885461724 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -13,6 +13,7 @@ from typing import TYPE_CHECKING, Any, Final import click import httpx +from click.core import ParameterSource from dotenv import load_dotenv from pydantic import BaseModel, ConfigDict @@ -181,12 +182,21 @@ def append_query_params(url: str | None, params: dict) -> str: return modified_url -def resolve_v2_migration_resolver(*, use_legacy_flag: bool) -> bool: +def resolve_v2_migration_resolver(*, use_legacy_flag: bool, env_value: str | None) -> bool: from litellm_proxy_extras.utils import str_to_bool if use_legacy_flag: return False - return bool(str_to_bool(os.getenv("USE_V2_MIGRATION_RESOLVER", "true"))) + if env_value is None: + return True + return bool(str_to_bool(env_value)) + + +def deprecated_v2_flag_passed_on_cli() -> bool: + ctx: Final = click.get_current_context(silent=True) + if ctx is None: + return False + return ctx.get_parameter_source("use_v2_migration_resolver") is ParameterSource.COMMANDLINE class ProxyInitializationHelpers: @@ -1368,14 +1378,15 @@ def run_server( check_prisma_schema_diff(db_url=None) else: use_v2_resolver: Final = resolve_v2_migration_resolver( - use_legacy_flag=use_legacy_migration_resolver + use_legacy_flag=use_legacy_migration_resolver, + env_value=os.getenv("USE_V2_MIGRATION_RESOLVER"), ) - if use_v2_migration_resolver and use_v2_resolver: + if deprecated_v2_flag_passed_on_cli() and use_v2_resolver: print( "\033[1;33mLiteLLM Proxy: --use_v2_migration_resolver is " - "deprecated and has no effect \u2014 the v2 migration resolver " - "is now the default. You can safely remove it. To opt back " - "into the legacy v1 resolver, pass " + "deprecated and has no effect, because the v2 migration " + "resolver is now the default. You can safely remove it. To " + "opt back into the legacy v1 resolver, pass " "--use_legacy_migration_resolver.\033[0m" ) if not use_v2_resolver: diff --git a/tests/test_litellm/proxy/test_proxy_cli.py b/tests/test_litellm/proxy/test_proxy_cli.py index d2835142194..a38470d1fdf 100644 --- a/tests/test_litellm/proxy/test_proxy_cli.py +++ b/tests/test_litellm/proxy/test_proxy_cli.py @@ -2203,6 +2203,7 @@ class TestRunServerDbSetup: mock_setup_database, mock_atexit_register, mock_subprocess_run, + capsys, ): """USE_V2_MIGRATION_RESOLVER=true must select the v2 resolver. @@ -2248,44 +2249,58 @@ class TestRunServerDbSetup: mock_setup_database.assert_called_once_with( use_migrate=True, use_v2_resolver=True ) + assert "--use_v2_migration_resolver is deprecated" not in capsys.readouterr().out @pytest.mark.parametrize( - "argv_extra, env_extra, expected_v2", + "use_legacy_flag, env_value, expected", [ - ([], {}, True), - ([], {"USE_V2_MIGRATION_RESOLVER": "false"}, False), - (["--use_legacy_migration_resolver"], {}, False), - ( - ["--use_legacy_migration_resolver"], - {"USE_V2_MIGRATION_RESOLVER": "true"}, - False, - ), - (["--use_v2_migration_resolver"], {}, True), + (False, None, True), + (False, "true", True), + (False, "false", False), + (True, None, False), + (True, "true", False), ], ids=[ - "default-is-v2", - "env-false-opts-out", - "legacy-flag-opts-out", + "unset-env-defaults-to-v2", + "env-true-selects-v2", + "env-false-selects-v1", + "legacy-flag-selects-v1", "legacy-flag-beats-env-true", - "deprecated-v2-flag-still-accepted", ], ) + def test_resolve_v2_migration_resolver(self, use_legacy_flag, env_value, expected): + from litellm.proxy.proxy_cli import resolve_v2_migration_resolver + + assert ( + resolve_v2_migration_resolver( + use_legacy_flag=use_legacy_flag, env_value=env_value + ) + is expected + ) + + def test_deprecated_v2_flag_not_reported_outside_a_cli_invocation(self): + from litellm.proxy.proxy_cli import deprecated_v2_flag_passed_on_cli + + assert deprecated_v2_flag_passed_on_cli() is False + @patch("subprocess.run") @patch("atexit.register") @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") @patch("litellm.proxy.db.check_migration.check_prisma_schema_diff") @patch("litellm.proxy.db.prisma_client.should_update_prisma_schema") - def test_migration_resolver_selection( + def test_legacy_resolver_flag_reaches_database_setup( self, mock_should_update_schema, mock_check_schema_diff, mock_setup_database, mock_atexit_register, mock_subprocess_run, - argv_extra, - env_extra, - expected_v2, ): + """--use_legacy_migration_resolver must reach the database setup call. + + The resolver decision itself is covered mock-free above; this is the + one wiring check that the flag is threaded through run_server. + """ from litellm.proxy.proxy_cli import run_server mock_subprocess_run.return_value = MagicMock(returncode=0) @@ -2302,11 +2317,9 @@ class TestRunServerDbSetup: clean_env = { k: v for k, v in os.environ.items() - if k - not in ("DATABASE_URL", "DIRECT_URL", "USE_V2_MIGRATION_RESOLVER") + if k not in ("DATABASE_URL", "DIRECT_URL", "USE_V2_MIGRATION_RESOLVER") } clean_env["DATABASE_URL"] = "postgresql://test:test@localhost:5432/test" - clean_env.update(env_extra) with ( patch.dict(os.environ, clean_env, clear=True), @@ -2319,12 +2332,16 @@ class TestRunServerDbSetup: ), ): run_server.main( - ["--local", "--skip_server_startup", *argv_extra], + [ + "--local", + "--skip_server_startup", + "--use_legacy_migration_resolver", + ], standalone_mode=False, ) mock_setup_database.assert_called_once_with( - use_migrate=True, use_v2_resolver=expected_v2 + use_migrate=True, use_v2_resolver=False )