From 4f6fd85ab1e36af0cb3f44b045c3767d0ebe89b3 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 28 Aug 2026 01:10:05 -0700 Subject: [PATCH 01/12] feat(proxy): default to the v2 migration resolver, keep v1 as an opt-out The v2 resolver skips the diff-and-force recovery that caused schema thrashing when two LiteLLM versions contend for one database during a rolling deploy. The standalone migration Job already defaulted to v2; this aligns the proxy-server path. v1 stays reachable two ways: --use_legacy_migration_resolver on the CLI, and USE_V2_MIGRATION_RESOLVER=false for containerised deploys, where prisma_migration.py calls run_server with a fixed argv and the env var is the only route in. --use_v2_migration_resolver still parses, so existing commands do not die on an unknown option. Because v2 fails fast where v1 retried every failed deploy, a database that is not accepting connections yet, or another instance holding the migration advisory lock, would now kill a boot that used to ride it out. Those two failures are retried, with Prisma's stderr logged each round, and still raise once the attempts are spent. Moves the resolver tests from litellm-proxy-extras/tests, which no CI job runs, into tests/litellm-proxy-extras, and repoints the dedicated Postgres CircleCI job at the legacy path so v1 keeps real-DB and proxy-boot coverage. --- .circleci/config.yml | 15 +- CLAUDE.md | 2 +- .../litellm_proxy_extras/utils.py | 81 ++++-- litellm-proxy-extras/tests/__init__.py | 0 litellm/proxy/proxy_cli.py | 22 +- .../test_setup_database_fail_fast.py | 238 +++++++++++++++++- .../test_basic_python_version.py | 14 +- tests/test_litellm/proxy/test_proxy_cli.py | 83 +++++- 8 files changed, 409 insertions(+), 46 deletions(-) delete mode 100644 litellm-proxy-extras/tests/__init__.py rename {litellm-proxy-extras/tests => tests/litellm-proxy-extras}/test_setup_database_fail_fast.py (50%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4615a6a5a7e..0fbbd5ec7f7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1483,7 +1483,7 @@ jobs: - run: name: Run tests command: | - uv run --no-sync python -m pytest -vv tests/local_testing/test_basic_python_version.py -k "not v2_resolver" + uv run --no-sync python -m pytest -vv tests/local_testing/test_basic_python_version.py -k "not legacy_resolver" installing_litellm_on_python_3_13: docker: @@ -1507,9 +1507,9 @@ jobs: - run: name: Run tests command: | - uv run --no-sync python -m pytest -v tests/local_testing/test_basic_python_version.py -k "not v2_resolver" + uv run --no-sync python -m pytest -v tests/local_testing/test_basic_python_version.py -k "not legacy_resolver" - installing_litellm_on_python_v2_migration_resolver: + installing_litellm_on_python_legacy_migration_resolver: docker: - *python312_image - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 @@ -1536,10 +1536,10 @@ jobs: url: tcp://localhost:5432 timeout: "60" - run: - name: Run v2 migration resolver proxy smoke test + name: Run legacy migration resolver proxy smoke test command: | uv run --no-sync python -m pytest -vv \ - tests/local_testing/test_basic_python_version.py::test_litellm_proxy_server_config_no_general_settings_v2_resolver + tests/local_testing/test_basic_python_version.py::test_litellm_proxy_server_config_no_general_settings_legacy_resolver helm_chart_testing: machine: @@ -2918,7 +2918,8 @@ jobs: command: | if grep -q "Error: P1001: Can't reach database server at" docker_output.log && \ (grep -q "Database setup failed after multiple retries" docker_output.log || \ - grep -q "ERROR: Application startup failed. Exiting." docker_output.log); then + grep -q "ERROR: Application startup failed. Exiting." docker_output.log || \ + grep -q "Database migration cannot proceed" docker_output.log); then echo "Expected error found. Test passed." else echo "Expected error not found. Test failed." @@ -3050,7 +3051,7 @@ workflows: filters: *main_branches - installing_litellm_on_python_3_13: filters: *main_branches - - installing_litellm_on_python_v2_migration_resolver: + - installing_litellm_on_python_legacy_migration_resolver: filters: *main_branches - helm_chart_testing: requires: diff --git a/CLAUDE.md b/CLAUDE.md index 930825aeb89..819f65059b7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -35,7 +35,7 @@ Same applies for filing bug reports and feature requests, with .github/ISSUE_TEM If you're resolving a linear ticket, in the "## Linear ticket" section of the PR, say "Resolves LIT-1234", replacing "LIT-1234" with the actual ticket id that you're resolving. If you don't have the ticket id, don't make one up or search for it. Just leave the section blank -Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`; the Admin UI dev server is `npm run dev` in `ui/litellm-dashboard`, served on port 3000) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into; I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, just tell me which URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, what fields to fill out, etc. along with the other commands to run in an ordered list, and I'll do it myself and post the screenshots after you make the PR +Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload 2>&1 | tee litellm.log`; the Admin UI dev server is `npm run dev` in `ui/litellm-dashboard`, served on port 3000) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into; I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, just tell me which URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, what fields to fill out, etc. along with the other commands to run in an ordered list, and I'll do it myself and post the screenshots after you make the PR If you ever write any human-facing text (pull requests, issues, commit messages, discussion posts, github comments, release notes, docs, etc.), always follow these guidelines to sound less AI-y: - don't use emojis diff --git a/litellm-proxy-extras/litellm_proxy_extras/utils.py b/litellm-proxy-extras/litellm_proxy_extras/utils.py index b27221c9beb..22da9b834ab 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/utils.py +++ b/litellm-proxy-extras/litellm_proxy_extras/utils.py @@ -7,7 +7,8 @@ import subprocess import tempfile import time from pathlib import Path -from typing import Optional +from types import MappingProxyType +from typing import Final, Optional from litellm_proxy_extras._logging import logger from litellm_proxy_extras.replica_identity import ( @@ -50,6 +51,35 @@ _SPEND_LOGS_PK_CLAUSE_RE = re.compile( re.IGNORECASE, ) +_MIGRATE_DEPLOY_ATTEMPTS: Final = 4 + +_TRANSIENT_DEPLOY_FAILURES: Final = MappingProxyType( + { + "deadlock detected": "a deadlock on the migration advisory lock (a concurrent migrate deploy)", + "P1001": "an unreachable database server", + "P1002": "a database server that timed out", + } +) + + +def _transient_deploy_failure(stderr: str) -> str | None: + """Describe why a failed `prisma migrate deploy` is worth retrying, or None. + + These are environment failures, not migration failures: the database is not + up yet, or another instance holds the migration lock. v1 retried every + failed deploy and absorbed them; failing fast on them instead would turn a + database that is ten seconds late into a dead proxy. + """ + return next( + ( + reason + for marker, reason in _TRANSIENT_DEPLOY_FAILURES.items() + if marker in stderr + ), + None, + ) + + PARTITIONED_SPEND_LOGS_PUSH_ERROR = ( "LiteLLM_SpendLogs is a partitioned table (see db_scripts/partition_spend_logs.sql), " "so its primary key must include the partition key (\"startTime\"). `prisma db push` " @@ -648,7 +678,7 @@ class ProxyExtrasDBManager: @staticmethod def _setup_database_v2(use_migrate: bool) -> bool: """ - v2 migration resolver (opt-in via --use_v2_migration_resolver). + v2 migration resolver (what the proxy CLI selects by default). Runs `prisma migrate deploy` and handles standard recovery paths (P3005 baseline, P3009/P3018 idempotent errors). Critically, it does @@ -692,7 +722,7 @@ class ProxyExtrasDBManager: original_dir = os.getcwd() os.chdir(migrations_dir) try: - for attempt in range(4): + for attempt in range(_MIGRATE_DEPLOY_ATTEMPTS): try: result = subprocess.run( [_get_prisma_command(), "migrate", "deploy"], @@ -807,16 +837,36 @@ class ProxyExtrasDBManager: f"Manual intervention required.\n\nPrisma error:\n{stderr}" ) from e - raise RuntimeError( - "Database migration failed and cannot be auto-recovered. " - f"Manual intervention required.\n\nPrisma error:\n{stderr}" - ) from e + transient = _transient_deploy_failure(stderr) + if transient is None: + raise RuntimeError( + "Database migration failed and cannot be auto-recovered. " + f"Manual intervention required.\n\nPrisma error:\n{stderr}" + ) from e + + if attempt == _MIGRATE_DEPLOY_ATTEMPTS - 1: + raise RuntimeError( + f"Database migration failed after " + f"{_MIGRATE_DEPLOY_ATTEMPTS} attempts on {transient}. " + "Check database connectivity and load." + f"\n\nPrisma error:\n{stderr}" + ) from e + + logger.info( + "prisma migrate deploy attempt %s failed on %s, retrying. " + "Prisma error:\n%s", + attempt + 1, + transient, + stderr, + ) + time.sleep(random.randrange(5, 15)) + continue raise RuntimeError( - "Database migration failed after 4 attempts (retry loop " - "exhausted by timeouts or repeated idempotent-recovery " - "continues). Check database connectivity, load, and " - "_prisma_migrations ledger state." + f"Database migration failed after {_MIGRATE_DEPLOY_ATTEMPTS} " + "attempts (retry loop exhausted by timeouts or repeated " + "idempotent-recovery continues). Check database connectivity, " + "load, and _prisma_migrations ledger state." ) finally: os.chdir(original_dir) @@ -864,10 +914,11 @@ class ProxyExtrasDBManager: Args: use_migrate: Whether to use prisma migrate instead of db push - use_v2_resolver: Opt into the v2 migration resolver (safer during + use_v2_resolver: Run the v2 migration resolver (safer during rolling deploys; does not run the diff-and-force recovery - that causes schema thrashing). Defaults to False for - backwards compatibility. + that causes schema thrashing). Defaults to False here so + direct callers keep the old behavior; the proxy CLI passes + True, so the proxy's runtime default is v2. Returns: bool: True if setup was successful, False otherwise @@ -885,7 +936,7 @@ class ProxyExtrasDBManager: @staticmethod def _run_migrations(use_migrate: bool, use_v2_resolver: bool) -> bool: if use_v2_resolver: - logger.info("Using v2 migration resolver (--use_v2_migration_resolver)") + logger.info("Using v2 migration resolver") return ProxyExtrasDBManager._setup_database_v2(use_migrate=use_migrate) schema_path = ProxyExtrasDBManager._get_prisma_dir() + "/schema.prisma" diff --git a/litellm-proxy-extras/tests/__init__.py b/litellm-proxy-extras/tests/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 8ac63ba25c9..c1d86344a1e 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -913,13 +913,14 @@ class ProxyInitializationHelpers: envvar="ENFORCE_PRISMA_MIGRATION_CHECK", ) @click.option( - "--use_v2_migration_resolver", - is_flag=True, - default=False, + "--use_v2_migration_resolver/--use_legacy_migration_resolver", + default=True, help=( - "Opt into the v2 migration resolver. Avoids the diff-and-force recovery " - "path that can cause schema thrashing during rolling deploys where two " - "LiteLLM versions contend for the same DB. Default is the v1 resolver." + "Which database migration resolver to run at startup. The default v2 " + "resolver avoids the diff-and-force recovery path that can cause schema " + "thrashing during rolling deploys where two LiteLLM versions contend for " + "the same DB. Pass --use_legacy_migration_resolver, or set " + "USE_V2_MIGRATION_RESOLVER=false, to fall back to v1." ), envvar="USE_V2_MIGRATION_RESOLVER", ) @@ -1310,10 +1311,11 @@ def run_server( else: if not use_v2_migration_resolver: print( - "\033[1;33mLiteLLM Proxy: Using default (v1) migration resolver. " - "If your deployment has seen schema thrashing during rolling " - "deploys, try --use_v2_migration_resolver (safer: avoids the " - "diff-and-force recovery that caused the thrash).\033[0m" + "\033[1;33mLiteLLM Proxy: Using the legacy (v1) migration resolver. " + "The default v2 resolver is safer: it avoids the diff-and-force " + "recovery that caused schema thrashing during rolling deploys. " + "Remove --use_legacy_migration_resolver / " + "USE_V2_MIGRATION_RESOLVER=false to switch back to it.\033[0m" ) try: setup_ok: Final = PrismaManager.setup_database( diff --git a/litellm-proxy-extras/tests/test_setup_database_fail_fast.py b/tests/litellm-proxy-extras/test_setup_database_fail_fast.py similarity index 50% rename from litellm-proxy-extras/tests/test_setup_database_fail_fast.py rename to tests/litellm-proxy-extras/test_setup_database_fail_fast.py index 8d66bf872de..f637e139e86 100644 --- a/litellm-proxy-extras/tests/test_setup_database_fail_fast.py +++ b/tests/litellm-proxy-extras/test_setup_database_fail_fast.py @@ -1,15 +1,26 @@ -"""Regression tests for ProxyExtrasDBManager v2 migration resolver. +"""Regression tests for ProxyExtrasDBManager's v2 migration resolver. -The v2 resolver is opt-in via `--use_v2_migration_resolver` / the -`use_v2_resolver=True` kwarg. These tests exercise the v2 path; the v1 -(default) behavior is unchanged from pre-fix. +v2 is what the proxy CLI selects by default; v1 stays reachable via +`--use_legacy_migration_resolver` or `USE_V2_MIGRATION_RESOLVER=false`. At the +library level the resolver is picked with the `use_v2_resolver` kwarg, which +still defaults to False so `migrations/run.py` and any direct caller keep their +own explicit choice. """ +import os import subprocess +import sys from unittest.mock import patch import pytest +sys.path.insert( + 0, + os.path.abspath( + os.path.join(os.path.dirname(__file__), "../../litellm-proxy-extras") + ), +) + from litellm_proxy_extras.utils import ( ProxyExtrasDBManager, _max_migration_timestamp, @@ -240,3 +251,222 @@ def test_v2_does_not_call_resolve_all_migrations(monkeypatch, tmp_path): ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) assert ok is True assert resolve_called["n"] == 0, "v2 must not invoke the diff-and-force recovery" + + +_DEADLOCK_STDERR = ( + "Error: ERROR: deadlock detected\n" + "DETAIL: Process 277 waits for ExclusiveLock on advisory lock " + "[17556,0,72707369,1]; blocked by process 278.\n" + "Process 278 waits for ShareLock on virtual transaction 3/1041; " + "blocked by process 277." +) + + +class _DeployApplied: + stdout = "All migrations have been successfully applied." + stderr = "" + returncode = 0 + + +def _deploy_only(deploy_side_effect): + """subprocess.run stand-in that only intercepts `prisma migrate deploy`. + + Everything else the resolver shells out to, the Prisma toolchain check + above all, succeeds untouched, so a mock meant for the deploy call cannot + be silently consumed by an earlier subprocess call. + """ + deploys = {"n": 0} + + def _run(*args, **kwargs): + cmd = args[0] if args else kwargs.get("args", []) + if list(cmd)[-2:] == ["migrate", "deploy"]: + deploys["n"] += 1 + return deploy_side_effect(deploys["n"], cmd) + return _DeployApplied() + + return _run, deploys + + +def _prepare_v2_resolver(monkeypatch, tmp_path): + monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x") + monkeypatch.setattr( + ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None + ) + monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path)) + (tmp_path / "schema.prisma").write_text("// stub") + monkeypatch.setattr("time.sleep", lambda *_a, **_k: None) + + +def test_v2_retries_transient_advisory_lock_deadlock(monkeypatch, tmp_path): + """A deadlock on Prisma's migration advisory lock is transient and retried. + + Several proxy replicas booting against one database race `migrate deploy`, + and Postgres aborts one side. v1 retried any failed deploy, so it rode this + out; v2 classifies unrecognised stderr as unrecoverable and raises, which + with v2 as the default would take a replica's whole boot down. + """ + _prepare_v2_resolver(monkeypatch, tmp_path) + + def _side_effect(n, cmd): + if n == 1: + raise subprocess.CalledProcessError( + returncode=1, cmd=cmd, stderr=_DEADLOCK_STDERR, output="" + ) + return _DeployApplied() + + run, deploys = _deploy_only(_side_effect) + with patch("subprocess.run", side_effect=run): + ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) + + assert ok is True + assert deploys["n"] == 2, "the deadlocked deploy must be retried, not raised" + + +def test_v2_persistent_advisory_lock_deadlock_eventually_raises(monkeypatch, tmp_path): + """The deadlock retry stays bounded: a deadlock that never clears still + raises rather than looping forever or reporting a successful migration.""" + _prepare_v2_resolver(monkeypatch, tmp_path) + + def _side_effect(n, cmd): + raise subprocess.CalledProcessError( + returncode=1, cmd=cmd, stderr=_DEADLOCK_STDERR, output="" + ) + + run, deploys = _deploy_only(_side_effect) + with patch("subprocess.run", side_effect=run): + with pytest.raises(RuntimeError, match="after 4 attempts"): + ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) + + assert deploys["n"] == 4 + + +@pytest.mark.parametrize( + "stderr", + [ + "Error: P1001: Can't reach database server at `db`:`5432`", + "Error: P1002: The database server was reached but timed out.", + ], +) +def test_v2_retries_transient_database_connectivity_errors(monkeypatch, tmp_path, stderr): + """A database that is not accepting connections yet is retried, not fatal. + + A proxy and its database starting together race routinely, and v1 rode that + out by retrying every failed deploy. v2 treats unrecognised stderr as + unrecoverable, so without this the default flip would turn a database that + is a few seconds late into a dead proxy instead of a slow boot. + """ + _prepare_v2_resolver(monkeypatch, tmp_path) + + def _side_effect(n, cmd): + if n == 1: + raise subprocess.CalledProcessError( + returncode=1, cmd=cmd, stderr=stderr, output="" + ) + return _DeployApplied() + + run, deploys = _deploy_only(_side_effect) + with patch("subprocess.run", side_effect=run): + ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) + + assert ok is True + assert deploys["n"] == 2, "an unreachable database must be retried, not raised" + + +def test_v2_unreachable_database_still_fails_after_the_retries(monkeypatch, tmp_path): + """Retrying connectivity errors must not turn a genuinely unreachable + database into a silent success: after the attempts are spent it still + raises, so the proxy exits instead of serving without its database.""" + _prepare_v2_resolver(monkeypatch, tmp_path) + + def _side_effect(n, cmd): + raise subprocess.CalledProcessError( + returncode=1, + cmd=cmd, + stderr="Error: P1001: Can't reach database server at `db`:`5432`", + output="", + ) + + run, deploys = _deploy_only(_side_effect) + with patch("subprocess.run", side_effect=run): + with pytest.raises(RuntimeError, match="after 4 attempts"): + ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) + + assert deploys["n"] == 4 + + +def test_v2_exhausted_retries_report_the_prisma_error(monkeypatch, tmp_path, caplog): + """Retrying must not swallow why the database was unreachable. + + Prisma's stderr is captured, so if the retry path neither logs it nor puts + it in the final error, an operator (and CI's bad-DATABASE_URL job, which + greps the boot log for the P1001 line) sees four silent retries and no + cause. + """ + _prepare_v2_resolver(monkeypatch, tmp_path) + stderr = "Error: P1001: Can't reach database server at `wrong`:`5432`" + + def _side_effect(n, cmd): + raise subprocess.CalledProcessError( + returncode=1, cmd=cmd, stderr=stderr, output="" + ) + + run, _ = _deploy_only(_side_effect) + with caplog.at_level("INFO", logger="litellm_proxy_extras"): + with patch("subprocess.run", side_effect=run): + with pytest.raises(RuntimeError) as exc_info: + ProxyExtrasDBManager.setup_database( + use_migrate=True, use_v2_resolver=True + ) + + assert "P1001" in str(exc_info.value) + assert "P1001" in caplog.text + + +def test_v2_migration_failure_is_not_treated_as_transient(monkeypatch, tmp_path): + """The transient classification must stay narrow: a genuinely broken + migration still fails fast on the first attempt rather than being retried + into the same error four times.""" + _prepare_v2_resolver(monkeypatch, tmp_path) + + stderr = ( + "Error: P3009\n" + "The `20260101000000_genuinely_broken` migration failed to apply.\n" + 'Reason: syntax error at or near "BRKN" LINE 42' + ) + + def _side_effect(n, cmd): + raise subprocess.CalledProcessError( + returncode=1, cmd=cmd, stderr=stderr, output="" + ) + + run, deploys = _deploy_only(_side_effect) + with patch("subprocess.run", side_effect=run): + with pytest.raises(RuntimeError, match="cannot be auto-recovered"): + ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) + + assert deploys["n"] == 1 + + +def test_v1_still_runs_the_diff_and_force_recovery(monkeypatch, tmp_path): + """v1 remains the pre-existing diff-and-force resolver, unchanged by the + default flip: it still calls _resolve_all_migrations after a deploy that + applied something. Operators opting back in must get exactly the old path. + """ + monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path)) + (tmp_path / "schema.prisma").write_text("// stub") + + class FakeResult: + stdout = "Applied migration.\n" + stderr = "" + + resolve_called = {"n": 0} + + def fake_resolve(*args, **kwargs): + resolve_called["n"] += 1 + + monkeypatch.setattr("subprocess.run", lambda *a, **kw: FakeResult()) + monkeypatch.setattr(ProxyExtrasDBManager, "_resolve_all_migrations", fake_resolve) + + ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=False) + assert ok is True + assert resolve_called["n"] == 1 diff --git a/tests/local_testing/test_basic_python_version.py b/tests/local_testing/test_basic_python_version.py index fb06ed6b69d..506c58d26b4 100644 --- a/tests/local_testing/test_basic_python_version.py +++ b/tests/local_testing/test_basic_python_version.py @@ -305,14 +305,16 @@ def _run_proxy_server_smoke_test(extra_proxy_args=None): def test_litellm_proxy_server_config_no_general_settings(): - """Exercises the default (v1) migration resolver.""" + """Exercises the default (v2) migration resolver.""" _run_proxy_server_smoke_test() -def test_litellm_proxy_server_config_no_general_settings_v2_resolver(): - """Exercises the opt-in v2 migration resolver. +def test_litellm_proxy_server_config_no_general_settings_legacy_resolver(): + """Exercises the legacy (v1) migration resolver against a real database. - Runs in a separate CI job against a local Postgres to avoid collisions - with the v1 variant when they share a database. + v2 is the default, so the no-arg test above already covers it. This one is + the only place the v1 opt-out gets real-DB migration plus proxy-boot + coverage, and it runs in a separate CI job against its own Postgres to + avoid collisions with the default variant. """ - _run_proxy_server_smoke_test(extra_proxy_args=["--use_v2_migration_resolver"]) + _run_proxy_server_smoke_test(extra_proxy_args=["--use_legacy_migration_resolver"]) diff --git a/tests/test_litellm/proxy/test_proxy_cli.py b/tests/test_litellm/proxy/test_proxy_cli.py index 6ea6f208bb5..ac1defe8d79 100644 --- a/tests/test_litellm/proxy/test_proxy_cli.py +++ b/tests/test_litellm/proxy/test_proxy_cli.py @@ -1787,7 +1787,7 @@ class TestRunServerDbSetup: # use_prisma_db_push should be False (default), so use_migrate should be True run_server.main(["--local", "--skip_server_startup"], standalone_mode=False) mock_setup_database.assert_called_with( - use_migrate=True, use_v2_resolver=False + use_migrate=True, use_v2_resolver=True ) # Reset mocks @@ -1802,7 +1802,7 @@ class TestRunServerDbSetup: standalone_mode=False, ) mock_setup_database.assert_called_with( - use_migrate=False, use_v2_resolver=False + use_migrate=False, use_v2_resolver=True ) @patch("subprocess.run") @@ -1869,7 +1869,7 @@ class TestRunServerDbSetup: ) assert exc_info.value.code == 1 mock_setup_database.assert_called_once_with( - use_migrate=True, use_v2_resolver=False + use_migrate=True, use_v2_resolver=True ) @patch("subprocess.run") @@ -1981,6 +1981,83 @@ class TestRunServerDbSetup: use_migrate=True, use_v2_resolver=True ) + @pytest.mark.parametrize( + "argv, env_value, expected_v2", + [ + ([], None, True), + (["--use_v2_migration_resolver"], None, True), + (["--use_legacy_migration_resolver"], None, False), + ([], "false", False), + ([], "true", True), + (["--use_v2_migration_resolver"], "false", True), + ], + ) + @patch("subprocess.run") + @patch("atexit.register") + @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") + @patch("litellm.proxy.db.check_migration.check_prisma_schema_diff") + @patch("litellm.proxy.db.prisma_client.should_update_prisma_schema") + def test_migration_resolver_default_and_opt_out( + self, + mock_should_update_schema, + mock_check_schema_diff, + mock_setup_database, + mock_atexit_register, + mock_subprocess_run, + argv, + env_value, + expected_v2, + ): + """The proxy defaults to the v2 resolver, and v1 stays reachable. + + Both opt-out routes matter: --use_legacy_migration_resolver for a CLI + boot, and USE_V2_MIGRATION_RESOLVER=false for containerised deploys, + where litellm/proxy/prisma_migration.py calls run_server with a fixed + argv and an env var is the only way in. The deprecated + --use_v2_migration_resolver must still parse so existing commands do + not die on an unknown option, and an explicit flag still beats the env. + """ + from litellm.proxy.proxy_cli import run_server + + mock_subprocess_run.return_value = MagicMock(returncode=0) + mock_should_update_schema.return_value = True + mock_setup_database.return_value = True + + mock_proxy_module = MagicMock( + app=MagicMock(), + ProxyConfig=MagicMock(), + KeyManagementSettings=MagicMock(), + save_worker_config=MagicMock(), + ) + + clean_env = { + k: v + for k, v in os.environ.items() + if k + not in ("DATABASE_URL", "DIRECT_URL", "USE_V2_MIGRATION_RESOLVER") + } + clean_env["DATABASE_URL"] = "postgresql://test:test@localhost:5432/test" + if env_value is not None: + clean_env["USE_V2_MIGRATION_RESOLVER"] = env_value + + with ( + patch.dict(os.environ, clean_env, clear=True), + patch.dict( + "sys.modules", + { + "proxy_server": mock_proxy_module, + "litellm.proxy.proxy_server": mock_proxy_module, + }, + ), + ): + run_server.main( + ["--local", "--skip_server_startup", *argv], standalone_mode=False + ) + + mock_setup_database.assert_called_once_with( + use_migrate=True, use_v2_resolver=expected_v2 + ) + # --- Module-level helpers for worker startup hook tests --- From 7b36bfb96720651c756c911ad5bcdb7a819e2006 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 28 Aug 2026 01:31:44 -0700 Subject: [PATCH 02/12] fix(proxy-extras): retry transient db push failures, drop a vacuous test `prisma db push` under v2 raised on the first failure while v1 retried it four times, so making v2 the default silently cost --use_prisma_db_push its retries. It now uses the same transient classification as migrate deploy. The classifier moves onto ProxyExtrasDBManager next to _is_permission_error and _is_idempotent_error, which do the same kind of stderr matching. Replaces a test that claimed to pin the transient classification but fed it a P3009 stderr, which an earlier branch catches, so it passed even when the classifier was mutated to treat everything as transient. The replacement uses an unclassified error and fails on that mutant. Drops a v1 test that duplicated test_v1_default_still_calls_resolve_all_migrations. --- .../litellm_proxy_extras/utils.py | 98 +++++++++------ .../test_setup_database_fail_fast.py | 113 ++++++++---------- tests/test_litellm/proxy/test_proxy_cli.py | 12 +- 3 files changed, 110 insertions(+), 113 deletions(-) diff --git a/litellm-proxy-extras/litellm_proxy_extras/utils.py b/litellm-proxy-extras/litellm_proxy_extras/utils.py index 22da9b834ab..a31016e0f17 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/utils.py +++ b/litellm-proxy-extras/litellm_proxy_extras/utils.py @@ -51,9 +51,9 @@ _SPEND_LOGS_PK_CLAUSE_RE = re.compile( re.IGNORECASE, ) -_MIGRATE_DEPLOY_ATTEMPTS: Final = 4 +_PRISMA_ATTEMPTS: Final = 4 -_TRANSIENT_DEPLOY_FAILURES: Final = MappingProxyType( +_TRANSIENT_PRISMA_FAILURES: Final = MappingProxyType( { "deadlock detected": "a deadlock on the migration advisory lock (a concurrent migrate deploy)", "P1001": "an unreachable database server", @@ -62,24 +62,6 @@ _TRANSIENT_DEPLOY_FAILURES: Final = MappingProxyType( ) -def _transient_deploy_failure(stderr: str) -> str | None: - """Describe why a failed `prisma migrate deploy` is worth retrying, or None. - - These are environment failures, not migration failures: the database is not - up yet, or another instance holds the migration lock. v1 retried every - failed deploy and absorbed them; failing fast on them instead would turn a - database that is ten seconds late into a dead proxy. - """ - return next( - ( - reason - for marker, reason in _TRANSIENT_DEPLOY_FAILURES.items() - if marker in stderr - ), - None, - ) - - PARTITIONED_SPEND_LOGS_PUSH_ERROR = ( "LiteLLM_SpendLogs is a partitioned table (see db_scripts/partition_spend_logs.sql), " "so its primary key must include the partition key (\"startTime\"). `prisma db push` " @@ -304,6 +286,23 @@ class ProxyExtrasDBManager: env=prisma_env, ) + @staticmethod + def _transient_prisma_failure(stderr: str) -> str | None: + """Why a failed prisma command is worth retrying, or None. + + v1 retried every failure, so it absorbed a database that was not up yet + or another instance holding the migration lock. v2 fails fast, which is + right for a broken migration and wrong for these. + """ + return next( + ( + reason + for marker, reason in _TRANSIENT_PRISMA_FAILURES.items() + if marker in stderr + ), + None, + ) + @staticmethod def _is_permission_error(error_message: str) -> bool: """ @@ -699,20 +698,43 @@ class ProxyExtrasDBManager: original_dir = os.getcwd() os.chdir(migrations_dir) try: - subprocess.run( - [_get_prisma_command(), "db", "push", "--accept-data-loss"], - timeout=prisma_command_timeout(), - check=True, - env=_get_prisma_env(), + for attempt in range(_PRISMA_ATTEMPTS): + try: + subprocess.run( + [_get_prisma_command(), "db", "push", "--accept-data-loss"], + timeout=prisma_command_timeout(), + check=True, + capture_output=True, + text=True, + env=_get_prisma_env(), + ) + return True + except ( + subprocess.CalledProcessError, + subprocess.TimeoutExpired, + ) as e: + stderr = e.stderr or "" + transient = ProxyExtrasDBManager._transient_prisma_failure( + stderr + ) + # Re-raise as RuntimeError so proxy_cli.py's + # `except RuntimeError` catches it and exits cleanly. + if transient is None or attempt == _PRISMA_ATTEMPTS - 1: + raise RuntimeError( + f"prisma db push failed.\n\nDetail: {e}" + f"\n\nPrisma error:\n{stderr}" + ) from e + logger.info( + "prisma db push attempt %s failed on %s, retrying. " + "Prisma error:\n%s", + attempt + 1, + transient, + stderr, + ) + time.sleep(random.randrange(5, 15)) + raise RuntimeError( + f"prisma db push failed after {_PRISMA_ATTEMPTS} attempts." ) - return True - except ( - subprocess.CalledProcessError, - subprocess.TimeoutExpired, - ) as e: - # Re-raise as RuntimeError so proxy_cli.py's - # `except RuntimeError` catches it and exits cleanly. - raise RuntimeError(f"prisma db push failed.\n\nDetail: {e}") from e finally: os.chdir(original_dir) @@ -722,7 +744,7 @@ class ProxyExtrasDBManager: original_dir = os.getcwd() os.chdir(migrations_dir) try: - for attempt in range(_MIGRATE_DEPLOY_ATTEMPTS): + for attempt in range(_PRISMA_ATTEMPTS): try: result = subprocess.run( [_get_prisma_command(), "migrate", "deploy"], @@ -837,17 +859,17 @@ class ProxyExtrasDBManager: f"Manual intervention required.\n\nPrisma error:\n{stderr}" ) from e - transient = _transient_deploy_failure(stderr) + transient = ProxyExtrasDBManager._transient_prisma_failure(stderr) if transient is None: raise RuntimeError( "Database migration failed and cannot be auto-recovered. " f"Manual intervention required.\n\nPrisma error:\n{stderr}" ) from e - if attempt == _MIGRATE_DEPLOY_ATTEMPTS - 1: + if attempt == _PRISMA_ATTEMPTS - 1: raise RuntimeError( f"Database migration failed after " - f"{_MIGRATE_DEPLOY_ATTEMPTS} attempts on {transient}. " + f"{_PRISMA_ATTEMPTS} attempts on {transient}. " "Check database connectivity and load." f"\n\nPrisma error:\n{stderr}" ) from e @@ -863,7 +885,7 @@ class ProxyExtrasDBManager: continue raise RuntimeError( - f"Database migration failed after {_MIGRATE_DEPLOY_ATTEMPTS} " + f"Database migration failed after {_PRISMA_ATTEMPTS} " "attempts (retry loop exhausted by timeouts or repeated " "idempotent-recovery continues). Check database connectivity, " "load, and _prisma_migrations ledger state." diff --git a/tests/litellm-proxy-extras/test_setup_database_fail_fast.py b/tests/litellm-proxy-extras/test_setup_database_fail_fast.py index f637e139e86..deb78dcdb30 100644 --- a/tests/litellm-proxy-extras/test_setup_database_fail_fast.py +++ b/tests/litellm-proxy-extras/test_setup_database_fail_fast.py @@ -1,10 +1,7 @@ """Regression tests for ProxyExtrasDBManager's v2 migration resolver. -v2 is what the proxy CLI selects by default; v1 stays reachable via -`--use_legacy_migration_resolver` or `USE_V2_MIGRATION_RESOLVER=false`. At the -library level the resolver is picked with the `use_v2_resolver` kwarg, which -still defaults to False so `migrations/run.py` and any direct caller keep their -own explicit choice. +v2 is the proxy CLI default; v1 stays reachable via the `use_v2_resolver` +kwarg, which still defaults to False for direct callers. """ import os @@ -271,9 +268,7 @@ class _DeployApplied: def _deploy_only(deploy_side_effect): """subprocess.run stand-in that only intercepts `prisma migrate deploy`. - Everything else the resolver shells out to, the Prisma toolchain check - above all, succeeds untouched, so a mock meant for the deploy call cannot - be silently consumed by an earlier subprocess call. + Scoped by argv so the Prisma toolchain check cannot consume the mock first. """ deploys = {"n": 0} @@ -298,13 +293,8 @@ def _prepare_v2_resolver(monkeypatch, tmp_path): def test_v2_retries_transient_advisory_lock_deadlock(monkeypatch, tmp_path): - """A deadlock on Prisma's migration advisory lock is transient and retried. - - Several proxy replicas booting against one database race `migrate deploy`, - and Postgres aborts one side. v1 retried any failed deploy, so it rode this - out; v2 classifies unrecognised stderr as unrecoverable and raises, which - with v2 as the default would take a replica's whole boot down. - """ + """v2: replicas racing `migrate deploy` deadlock on Prisma's advisory + lock, which is transient and must be retried rather than kill the boot.""" _prepare_v2_resolver(monkeypatch, tmp_path) def _side_effect(n, cmd): @@ -323,8 +313,8 @@ def test_v2_retries_transient_advisory_lock_deadlock(monkeypatch, tmp_path): def test_v2_persistent_advisory_lock_deadlock_eventually_raises(monkeypatch, tmp_path): - """The deadlock retry stays bounded: a deadlock that never clears still - raises rather than looping forever or reporting a successful migration.""" + """v2: the deadlock retry is bounded, so a deadlock that never clears + still raises instead of looping or reporting success.""" _prepare_v2_resolver(monkeypatch, tmp_path) def _side_effect(n, cmd): @@ -348,13 +338,7 @@ def test_v2_persistent_advisory_lock_deadlock_eventually_raises(monkeypatch, tmp ], ) def test_v2_retries_transient_database_connectivity_errors(monkeypatch, tmp_path, stderr): - """A database that is not accepting connections yet is retried, not fatal. - - A proxy and its database starting together race routinely, and v1 rode that - out by retrying every failed deploy. v2 treats unrecognised stderr as - unrecoverable, so without this the default flip would turn a database that - is a few seconds late into a dead proxy instead of a slow boot. - """ + """v2: a database not accepting connections yet is retried, not fatal.""" _prepare_v2_resolver(monkeypatch, tmp_path) def _side_effect(n, cmd): @@ -373,9 +357,8 @@ def test_v2_retries_transient_database_connectivity_errors(monkeypatch, tmp_path def test_v2_unreachable_database_still_fails_after_the_retries(monkeypatch, tmp_path): - """Retrying connectivity errors must not turn a genuinely unreachable - database into a silent success: after the attempts are spent it still - raises, so the proxy exits instead of serving without its database.""" + """v2: a genuinely unreachable database still raises once the attempts + are spent, rather than passing as a successful migration.""" _prepare_v2_resolver(monkeypatch, tmp_path) def _side_effect(n, cmd): @@ -395,13 +378,8 @@ def test_v2_unreachable_database_still_fails_after_the_retries(monkeypatch, tmp_ def test_v2_exhausted_retries_report_the_prisma_error(monkeypatch, tmp_path, caplog): - """Retrying must not swallow why the database was unreachable. - - Prisma's stderr is captured, so if the retry path neither logs it nor puts - it in the final error, an operator (and CI's bad-DATABASE_URL job, which - greps the boot log for the P1001 line) sees four silent retries and no - cause. - """ + """v2: retrying must not swallow Prisma's stderr, which is captured and is + the only place the cause appears for an operator or a boot-log grep.""" _prepare_v2_resolver(monkeypatch, tmp_path) stderr = "Error: P1001: Can't reach database server at `wrong`:`5432`" @@ -422,21 +400,47 @@ def test_v2_exhausted_retries_report_the_prisma_error(monkeypatch, tmp_path, cap assert "P1001" in caplog.text -def test_v2_migration_failure_is_not_treated_as_transient(monkeypatch, tmp_path): - """The transient classification must stay narrow: a genuinely broken - migration still fails fast on the first attempt rather than being retried - into the same error four times.""" +def test_v2_db_push_retries_transient_failures(monkeypatch, tmp_path): + """v2: `prisma db push` retries a transient failure like v1 did, so the + default flip does not cost --use_prisma_db_push its retries.""" _prepare_v2_resolver(monkeypatch, tmp_path) - stderr = ( - "Error: P3009\n" - "The `20260101000000_genuinely_broken` migration failed to apply.\n" - 'Reason: syntax error at or near "BRKN" LINE 42' + pushes = {"n": 0} + + def _run(*args, **kwargs): + cmd = list(args[0] if args else kwargs.get("args", [])) + if cmd[-3:] != ["db", "push", "--accept-data-loss"]: + return _DeployApplied() + pushes["n"] += 1 + if pushes["n"] == 1: + raise subprocess.CalledProcessError( + returncode=1, + cmd=cmd, + stderr="Error: P1001: Can't reach database server at `db`:`5432`", + output="", + ) + return _DeployApplied() + + monkeypatch.setattr( + ProxyExtrasDBManager, "spend_logs_is_partitioned", lambda: False ) + with patch("subprocess.run", side_effect=_run): + ok = ProxyExtrasDBManager.setup_database(use_migrate=False, use_v2_resolver=True) + + assert ok is True + assert pushes["n"] == 2 + + +def test_v2_unclassified_failure_is_not_treated_as_transient(monkeypatch, tmp_path): + """v2: an unrecognised deploy failure still raises on the first attempt.""" + _prepare_v2_resolver(monkeypatch, tmp_path) def _side_effect(n, cmd): raise subprocess.CalledProcessError( - returncode=1, cmd=cmd, stderr=stderr, output="" + returncode=1, + cmd=cmd, + stderr="Error: relation \"LiteLLM_SpendLogs\" does not exist", + output="", ) run, deploys = _deploy_only(_side_effect) @@ -447,26 +451,3 @@ def test_v2_migration_failure_is_not_treated_as_transient(monkeypatch, tmp_path) assert deploys["n"] == 1 -def test_v1_still_runs_the_diff_and_force_recovery(monkeypatch, tmp_path): - """v1 remains the pre-existing diff-and-force resolver, unchanged by the - default flip: it still calls _resolve_all_migrations after a deploy that - applied something. Operators opting back in must get exactly the old path. - """ - monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path)) - (tmp_path / "schema.prisma").write_text("// stub") - - class FakeResult: - stdout = "Applied migration.\n" - stderr = "" - - resolve_called = {"n": 0} - - def fake_resolve(*args, **kwargs): - resolve_called["n"] += 1 - - monkeypatch.setattr("subprocess.run", lambda *a, **kw: FakeResult()) - monkeypatch.setattr(ProxyExtrasDBManager, "_resolve_all_migrations", fake_resolve) - - ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=False) - assert ok is True - assert resolve_called["n"] == 1 diff --git a/tests/test_litellm/proxy/test_proxy_cli.py b/tests/test_litellm/proxy/test_proxy_cli.py index ac1defe8d79..dac36965a6c 100644 --- a/tests/test_litellm/proxy/test_proxy_cli.py +++ b/tests/test_litellm/proxy/test_proxy_cli.py @@ -2008,15 +2008,9 @@ class TestRunServerDbSetup: env_value, expected_v2, ): - """The proxy defaults to the v2 resolver, and v1 stays reachable. - - Both opt-out routes matter: --use_legacy_migration_resolver for a CLI - boot, and USE_V2_MIGRATION_RESOLVER=false for containerised deploys, - where litellm/proxy/prisma_migration.py calls run_server with a fixed - argv and an env var is the only way in. The deprecated - --use_v2_migration_resolver must still parse so existing commands do - not die on an unknown option, and an explicit flag still beats the env. - """ + """The proxy defaults to v2, and both v1 opt-out routes work: the + flag for a CLI boot, USE_V2_MIGRATION_RESOLVER=false for deploys that + cannot pass one. An explicit flag beats the env var.""" from litellm.proxy.proxy_cli import run_server mock_subprocess_run.return_value = MagicMock(returncode=0) From fbd1339993252ca3c134494ef431a22ce591c83f Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 28 Aug 2026 01:38:27 -0700 Subject: [PATCH 03/12] fix(tests): satisfy the tests-tree ruff config and correct a stale comment Moving the resolver tests under tests/ brings them under ruff-tests.toml, which the package-internal directory they came from was never linted by, so a pre-existing pytest.raises pattern now needs to be a raw string (RUF043). Also corrects the comment on proxy_cli's RuntimeError handler: both resolvers raise on permission failures, not just v2. --- litellm/proxy/proxy_cli.py | 8 ++++---- .../litellm-proxy-extras/test_setup_database_fail_fast.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index c1d86344a1e..86f6853a625 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -1323,10 +1323,10 @@ def run_server( use_v2_resolver=use_v2_migration_resolver, ) except RuntimeError as e: - # Raised on unrecoverable migration errors: the v2 - # resolver's non-idempotent failures and permission - # issues, and any `prisma db push` against a - # partitioned LiteLLM_SpendLogs. + # Raised on unrecoverable migration errors: permission + # failures from either resolver, the v2 resolver's + # non-idempotent failures, and any `prisma db push` + # against a partitioned LiteLLM_SpendLogs. print( f"\033[1;31mLiteLLM Proxy: Database migration cannot proceed. {e}\033[0m", file=sys.stderr, diff --git a/tests/litellm-proxy-extras/test_setup_database_fail_fast.py b/tests/litellm-proxy-extras/test_setup_database_fail_fast.py index deb78dcdb30..0b59c5f9430 100644 --- a/tests/litellm-proxy-extras/test_setup_database_fail_fast.py +++ b/tests/litellm-proxy-extras/test_setup_database_fail_fast.py @@ -219,7 +219,7 @@ def test_v2_resolve_specific_migration_failure_raises_runtime_error( ) with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)): with pytest.raises( - RuntimeError, match="Failed to mark migration .* as applied" + RuntimeError, match=r"Failed to mark migration .* as applied" ): ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True) From c3fc86869d45e2b5057dce53ec7bec0617f84149 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 28 Aug 2026 01:52:15 -0700 Subject: [PATCH 04/12] test: keep the resolver tests inside the test-quality ceilings The moved fail-fast test carried a sys.path.insert that the uv workspace makes unnecessary, and one pre-existing case asserted nothing beyond "did not raise", so it could not tell a swallowed error from a skipped query. Give it a liveness gate on the connect count instead. Fold the resolver default/opt-out matrix into the existing db-push flag test rather than standing up another patched test, so the flag pair, the env var, and their precedence are covered without new mock scaffolding. --- .../test_setup_database_fail_fast.py | 16 +-- tests/test_litellm/proxy/test_proxy_cli.py | 105 ++++++------------ 2 files changed, 38 insertions(+), 83 deletions(-) diff --git a/tests/litellm-proxy-extras/test_setup_database_fail_fast.py b/tests/litellm-proxy-extras/test_setup_database_fail_fast.py index 0b59c5f9430..964355492a1 100644 --- a/tests/litellm-proxy-extras/test_setup_database_fail_fast.py +++ b/tests/litellm-proxy-extras/test_setup_database_fail_fast.py @@ -4,20 +4,11 @@ v2 is the proxy CLI default; v1 stays reachable via the `use_v2_resolver` kwarg, which still defaults to False for direct callers. """ -import os import subprocess -import sys from unittest.mock import patch import pytest -sys.path.insert( - 0, - os.path.abspath( - os.path.join(os.path.dirname(__file__), "../../litellm-proxy-extras") - ), -) - from litellm_proxy_extras.utils import ( ProxyExtrasDBManager, _max_migration_timestamp, @@ -175,13 +166,16 @@ def test_v2_warn_ahead_of_head_swallows_db_errors(monkeypatch, tmp_path): # Simulate an InsufficientPrivilege (subclass of DatabaseError). raise psycopg.errors.InsufficientPrivilege("permission denied") + connects = {"n": 0} + def _fake_connect(*a, **kw): + connects["n"] += 1 return _FakeConn() monkeypatch.setattr("psycopg.connect", _fake_connect) - # Must not raise. - ProxyExtrasDBManager._warn_if_db_ahead_of_head(str(tmp_path)) + assert ProxyExtrasDBManager._warn_if_db_ahead_of_head(str(tmp_path)) is None + assert connects["n"] == 1, "the failing query must actually have been reached" def test_v2_resolve_specific_migration_failure_raises_runtime_error( diff --git a/tests/test_litellm/proxy/test_proxy_cli.py b/tests/test_litellm/proxy/test_proxy_cli.py index dac36965a6c..4f205ada609 100644 --- a/tests/test_litellm/proxy/test_proxy_cli.py +++ b/tests/test_litellm/proxy/test_proxy_cli.py @@ -1805,6 +1805,39 @@ class TestRunServerDbSetup: use_migrate=False, use_v2_resolver=True ) + # Test 3+: the resolver default and both routes back to v1. The flag + # covers a CLI boot; USE_V2_MIGRATION_RESOLVER covers deploys that + # cannot pass one, where prisma_migration.py fixes the argv. An + # explicit flag beats the env var. + for argv, env_value, expected_v2 in ( + ([], None, True), + (["--use_v2_migration_resolver"], None, True), + (["--use_legacy_migration_resolver"], None, False), + ([], "false", False), + ([], "true", True), + (["--use_v2_migration_resolver"], "false", True), + (["--use_legacy_migration_resolver"], "true", False), + ): + mock_setup_database.reset_mock() + mock_should_update_schema.reset_mock() + mock_should_update_schema.return_value = True + + resolver_env = ( + {"USE_V2_MIGRATION_RESOLVER": env_value} + if env_value is not None + else {} + ) + os.environ.pop("USE_V2_MIGRATION_RESOLVER", None) + with patch.dict(os.environ, resolver_env): + run_server.main( + ["--local", "--skip_server_startup", *argv], + standalone_mode=False, + ) + assert mock_setup_database.call_args.kwargs == { + "use_migrate": True, + "use_v2_resolver": expected_v2, + }, f"argv={argv} env={env_value}" + @patch("subprocess.run") @patch("atexit.register") @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") @@ -1981,78 +2014,6 @@ class TestRunServerDbSetup: use_migrate=True, use_v2_resolver=True ) - @pytest.mark.parametrize( - "argv, env_value, expected_v2", - [ - ([], None, True), - (["--use_v2_migration_resolver"], None, True), - (["--use_legacy_migration_resolver"], None, False), - ([], "false", False), - ([], "true", True), - (["--use_v2_migration_resolver"], "false", True), - ], - ) - @patch("subprocess.run") - @patch("atexit.register") - @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") - @patch("litellm.proxy.db.check_migration.check_prisma_schema_diff") - @patch("litellm.proxy.db.prisma_client.should_update_prisma_schema") - def test_migration_resolver_default_and_opt_out( - self, - mock_should_update_schema, - mock_check_schema_diff, - mock_setup_database, - mock_atexit_register, - mock_subprocess_run, - argv, - env_value, - expected_v2, - ): - """The proxy defaults to v2, and both v1 opt-out routes work: the - flag for a CLI boot, USE_V2_MIGRATION_RESOLVER=false for deploys that - cannot pass one. An explicit flag beats the env var.""" - from litellm.proxy.proxy_cli import run_server - - mock_subprocess_run.return_value = MagicMock(returncode=0) - mock_should_update_schema.return_value = True - mock_setup_database.return_value = True - - mock_proxy_module = MagicMock( - app=MagicMock(), - ProxyConfig=MagicMock(), - KeyManagementSettings=MagicMock(), - save_worker_config=MagicMock(), - ) - - clean_env = { - k: v - for k, v in os.environ.items() - if k - not in ("DATABASE_URL", "DIRECT_URL", "USE_V2_MIGRATION_RESOLVER") - } - clean_env["DATABASE_URL"] = "postgresql://test:test@localhost:5432/test" - if env_value is not None: - clean_env["USE_V2_MIGRATION_RESOLVER"] = env_value - - with ( - patch.dict(os.environ, clean_env, clear=True), - patch.dict( - "sys.modules", - { - "proxy_server": mock_proxy_module, - "litellm.proxy.proxy_server": mock_proxy_module, - }, - ), - ): - run_server.main( - ["--local", "--skip_server_startup", *argv], standalone_mode=False - ) - - mock_setup_database.assert_called_once_with( - use_migrate=True, use_v2_resolver=expected_v2 - ) - - # --- Module-level helpers for worker startup hook tests --- _dummy_hook_called = False From 6b3e30e6601ac4fb285e97525061ccdd93aaac16 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 28 Aug 2026 01:56:15 -0700 Subject: [PATCH 05/12] fix(proxy-extras): bound the db push retries in the reachable branch The retry loop already raises on the final attempt, so the raise that followed the loop could never run. Drop it and cover the exhaustion path with a test that pins the attempt count and keeps the prisma error in the message, which is the only thing that tells an operator why the boot stopped. --- .../litellm_proxy_extras/utils.py | 3 -- .../test_setup_database_fail_fast.py | 35 +++++++++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/litellm-proxy-extras/litellm_proxy_extras/utils.py b/litellm-proxy-extras/litellm_proxy_extras/utils.py index a31016e0f17..f751c4087eb 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/utils.py +++ b/litellm-proxy-extras/litellm_proxy_extras/utils.py @@ -732,9 +732,6 @@ class ProxyExtrasDBManager: stderr, ) time.sleep(random.randrange(5, 15)) - raise RuntimeError( - f"prisma db push failed after {_PRISMA_ATTEMPTS} attempts." - ) finally: os.chdir(original_dir) diff --git a/tests/litellm-proxy-extras/test_setup_database_fail_fast.py b/tests/litellm-proxy-extras/test_setup_database_fail_fast.py index 964355492a1..ed82037590a 100644 --- a/tests/litellm-proxy-extras/test_setup_database_fail_fast.py +++ b/tests/litellm-proxy-extras/test_setup_database_fail_fast.py @@ -10,6 +10,7 @@ from unittest.mock import patch import pytest from litellm_proxy_extras.utils import ( + _PRISMA_ATTEMPTS, ProxyExtrasDBManager, _max_migration_timestamp, _migration_timestamp, @@ -425,6 +426,40 @@ def test_v2_db_push_retries_transient_failures(monkeypatch, tmp_path): assert pushes["n"] == 2 +def test_v2_db_push_retries_are_bounded_and_report_the_prisma_error( + monkeypatch, tmp_path +): + """v2: a database that never comes back stops after _PRISMA_ATTEMPTS and + surfaces the prisma error, rather than retrying the boot forever.""" + _prepare_v2_resolver(monkeypatch, tmp_path) + + pushes = {"n": 0} + + def _run(*args, **kwargs): + cmd = list(args[0] if args else kwargs.get("args", [])) + if cmd[-3:] != ["db", "push", "--accept-data-loss"]: + return _DeployApplied() + pushes["n"] += 1 + raise subprocess.CalledProcessError( + returncode=1, + cmd=cmd, + stderr="Error: P1001: Can't reach database server at `db`:`5432`", + output="", + ) + + monkeypatch.setattr( + ProxyExtrasDBManager, "spend_logs_is_partitioned", lambda: False + ) + with patch("subprocess.run", side_effect=_run): + with pytest.raises(RuntimeError) as exc: + ProxyExtrasDBManager.setup_database( + use_migrate=False, use_v2_resolver=True + ) + + assert pushes["n"] == _PRISMA_ATTEMPTS + assert "P1001" in str(exc.value) + + def test_v2_unclassified_failure_is_not_treated_as_transient(monkeypatch, tmp_path): """v2: an unrecognised deploy failure still raises on the first attempt.""" _prepare_v2_resolver(monkeypatch, tmp_path) From 2495673d01ee26c73d54adb6b78e30ff87cf0bc2 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 28 Aug 2026 02:14:57 -0700 Subject: [PATCH 06/12] fix(proxy-extras): stop a db push timeout crashing the migration job subprocess.run leaves stderr as bytes on TimeoutExpired even under text=True, unlike CalledProcessError. Classifying both in one handler meant a real `prisma db push` timeout died on a TypeError, which proxy_cli.py's `except RuntimeError` does not catch, so the migrations Job container ended on an unhandled traceback instead of a clean exit. Give the timeout its own handler and retry it, matching what the migrate deploy loop beside it already does. That puts a fallthrough back into the loop, so the trailing raise removed in the previous commit is reachable again and comes back with it. Also drop a comment restating why the resolver cases exist and widen the db push test's docstring, which had stopped describing what it covers. --- .../litellm_proxy_extras/utils.py | 14 ++- .../test_setup_database_fail_fast.py | 86 +++++++++++++++++++ tests/test_litellm/proxy/test_proxy_cli.py | 7 +- 3 files changed, 98 insertions(+), 9 deletions(-) diff --git a/litellm-proxy-extras/litellm_proxy_extras/utils.py b/litellm-proxy-extras/litellm_proxy_extras/utils.py index f751c4087eb..22e30dfa897 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/utils.py +++ b/litellm-proxy-extras/litellm_proxy_extras/utils.py @@ -709,10 +709,13 @@ class ProxyExtrasDBManager: env=_get_prisma_env(), ) return True - except ( - subprocess.CalledProcessError, - subprocess.TimeoutExpired, - ) as e: + except subprocess.TimeoutExpired: + logger.info( + "prisma db push attempt %s timed out, retrying", + attempt + 1, + ) + time.sleep(random.randrange(5, 15)) + except subprocess.CalledProcessError as e: stderr = e.stderr or "" transient = ProxyExtrasDBManager._transient_prisma_failure( stderr @@ -732,6 +735,9 @@ class ProxyExtrasDBManager: stderr, ) time.sleep(random.randrange(5, 15)) + raise RuntimeError( + f"prisma db push failed after {_PRISMA_ATTEMPTS} attempts." + ) finally: os.chdir(original_dir) diff --git a/tests/litellm-proxy-extras/test_setup_database_fail_fast.py b/tests/litellm-proxy-extras/test_setup_database_fail_fast.py index ed82037590a..3996b91c2a6 100644 --- a/tests/litellm-proxy-extras/test_setup_database_fail_fast.py +++ b/tests/litellm-proxy-extras/test_setup_database_fail_fast.py @@ -460,6 +460,92 @@ def test_v2_db_push_retries_are_bounded_and_report_the_prisma_error( assert "P1001" in str(exc.value) +def _db_push_only(push_side_effect): + """subprocess.run stand-in that only intercepts `prisma db push`.""" + pushes = {"n": 0} + + def _run(*args, **kwargs): + cmd = list(args[0] if args else kwargs.get("args", [])) + if cmd[-3:] != ["db", "push", "--accept-data-loss"]: + return _DeployApplied() + pushes["n"] += 1 + return push_side_effect(pushes["n"], cmd) + + return _run, pushes + + +def _timed_out_for_real(): + """Capture what subprocess.run really puts on a TimeoutExpired. + + Under text=True it still leaves stderr as bytes, unlike CalledProcessError, + so hardcoding a str here would test a shape production never sees. Derived + at import, before any test patches subprocess.run. + """ + try: + subprocess.run( + ["sh", "-c", "echo 'Error: P1001 unreachable' >&2; sleep 5"], + timeout=0.2, + check=True, + capture_output=True, + text=True, + ) + except subprocess.TimeoutExpired as e: + return e + raise AssertionError("the helper command was supposed to time out") + + +_TIMEOUT_TEMPLATE = _timed_out_for_real() + + +def _real_timeout_expired(cmd): + return subprocess.TimeoutExpired( + cmd=cmd, + timeout=_TIMEOUT_TEMPLATE.timeout, + output=_TIMEOUT_TEMPLATE.stdout, + stderr=_TIMEOUT_TEMPLATE.stderr, + ) + + +def test_v2_db_push_retries_a_timeout(monkeypatch, tmp_path): + """v2: a `prisma db push` that times out is retried, not turned into a + TypeError by classifying its bytes stderr as if it were text.""" + _prepare_v2_resolver(monkeypatch, tmp_path) + + def _side_effect(n, cmd): + if n == 1: + raise _real_timeout_expired(cmd) + return _DeployApplied() + + monkeypatch.setattr( + ProxyExtrasDBManager, "spend_logs_is_partitioned", lambda: False + ) + run, pushes = _db_push_only(_side_effect) + with patch("subprocess.run", side_effect=run): + ok = ProxyExtrasDBManager.setup_database(use_migrate=False, use_v2_resolver=True) + + assert ok is True + assert pushes["n"] == 2 + + +def test_v2_db_push_timeouts_are_bounded(monkeypatch, tmp_path): + """v2: a `prisma db push` that never stops timing out gives up as a + RuntimeError, which is the only exception proxy_cli.py exits cleanly on.""" + _prepare_v2_resolver(monkeypatch, tmp_path) + + def _side_effect(n, cmd): + raise _real_timeout_expired(cmd) + + monkeypatch.setattr( + ProxyExtrasDBManager, "spend_logs_is_partitioned", lambda: False + ) + run, pushes = _db_push_only(_side_effect) + with patch("subprocess.run", side_effect=run): + with pytest.raises(RuntimeError, match=r"prisma db push failed after \d+"): + ProxyExtrasDBManager.setup_database(use_migrate=False, use_v2_resolver=True) + + assert pushes["n"] == _PRISMA_ATTEMPTS + + def test_v2_unclassified_failure_is_not_treated_as_transient(monkeypatch, tmp_path): """v2: an unrecognised deploy failure still raises on the first attempt.""" _prepare_v2_resolver(monkeypatch, tmp_path) diff --git a/tests/test_litellm/proxy/test_proxy_cli.py b/tests/test_litellm/proxy/test_proxy_cli.py index 4f205ada609..5e2dd358d75 100644 --- a/tests/test_litellm/proxy/test_proxy_cli.py +++ b/tests/test_litellm/proxy/test_proxy_cli.py @@ -1737,7 +1737,8 @@ class TestRunServerDbSetup: mock_atexit_register, mock_subprocess_run, ): - """Test that use_prisma_db_push flag correctly controls PrismaManager.setup_database use_migrate parameter""" + """Which resolver and which migration mode run_server hands setup_database, + across the db push flag, the v2/legacy flag pair and USE_V2_MIGRATION_RESOLVER.""" from litellm.proxy.proxy_cli import run_server # Mock subprocess.run to simulate prisma being available @@ -1805,10 +1806,6 @@ class TestRunServerDbSetup: use_migrate=False, use_v2_resolver=True ) - # Test 3+: the resolver default and both routes back to v1. The flag - # covers a CLI boot; USE_V2_MIGRATION_RESOLVER covers deploys that - # cannot pass one, where prisma_migration.py fixes the argv. An - # explicit flag beats the env var. for argv, env_value, expected_v2 in ( ([], None, True), (["--use_v2_migration_resolver"], None, True), From 5dfe32c889ca778b682bcab6e7772a4ebb14d47f Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 28 Aug 2026 02:16:20 -0700 Subject: [PATCH 07/12] docs(tests): name the entrypoint that actually reaches the db push branch The proxy CLI's --use_prisma_db_push never gets here; PrismaManager keeps its own db push loop and only delegates when use_migrate is true. The caller this covers is the migrations Job with USE_PRISMA_DB_PUSH=true. --- .../litellm-proxy-extras/test_setup_database_fail_fast.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/litellm-proxy-extras/test_setup_database_fail_fast.py b/tests/litellm-proxy-extras/test_setup_database_fail_fast.py index 3996b91c2a6..ef447315a8c 100644 --- a/tests/litellm-proxy-extras/test_setup_database_fail_fast.py +++ b/tests/litellm-proxy-extras/test_setup_database_fail_fast.py @@ -396,8 +396,11 @@ def test_v2_exhausted_retries_report_the_prisma_error(monkeypatch, tmp_path, cap def test_v2_db_push_retries_transient_failures(monkeypatch, tmp_path): - """v2: `prisma db push` retries a transient failure like v1 did, so the - default flip does not cost --use_prisma_db_push its retries.""" + """v2: `prisma db push` retries a transient failure like v1 did. + + Reached from the migrations Job (USE_PRISMA_DB_PUSH=true), not from the + proxy CLI, whose --use_prisma_db_push has its own loop in prisma_client. + """ _prepare_v2_resolver(monkeypatch, tmp_path) pushes = {"n": 0} From be5997f3664f7359204fe58f3111a93f24471aee Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Mon, 31 Aug 2026 14:48:16 -0700 Subject: [PATCH 08/12] feat: add Azure AI DeepSeek V4 Flash 0731 pricing --- model_prices_and_context_window.json | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 7071eaa0807..6b4a9818554 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -9959,6 +9959,22 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "azure_ai/deepseek-v4-flash-0731": { + "cache_read_input_token_cost": 2.8e-08, + "deprecation_date": "2026-12-03", + "input_cost_per_token": 1.9e-07, + "litellm_provider": "azure_ai", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5.1e-07, + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "azure_ai/embed-v-4-0": { "input_cost_per_token": 1.2e-07, "litellm_provider": "azure_ai", From 1a80b7ae252e017b937dfb9cf665cc04152ce4ec Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Mon, 31 Aug 2026 15:00:39 -0700 Subject: [PATCH 09/12] fix: sync Azure AI model backup registry --- .../model_prices_and_context_window_backup.json | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 7071eaa0807..6b4a9818554 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -9959,6 +9959,22 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "azure_ai/deepseek-v4-flash-0731": { + "cache_read_input_token_cost": 2.8e-08, + "deprecation_date": "2026-12-03", + "input_cost_per_token": 1.9e-07, + "litellm_provider": "azure_ai", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5.1e-07, + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/deepseek/", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "azure_ai/embed-v-4-0": { "input_cost_per_token": 1.2e-07, "litellm_provider": "azure_ai", From 2757399c99d45e3b9ae1143a9bd3d3a981bfdfae Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Tue, 1 Sep 2026 09:43:47 -0700 Subject: [PATCH 10/12] fix(ui): render the skill detail page with theme tokens The page painted every surface, border and text color inline with a fixed light palette (#202124, #5f6368, #dadce0, #f8f9fa, #fff), so in dark mode it drew dark text on hardcoded white cards. Move the whole component to the foreground/muted/border/card/info tokens, which already resolve for both themes. --- .../claude_code_plugins/skill_detail.tsx | 310 +++++------------- 1 file changed, 78 insertions(+), 232 deletions(-) diff --git a/ui/litellm-dashboard/src/components/claude_code_plugins/skill_detail.tsx b/ui/litellm-dashboard/src/components/claude_code_plugins/skill_detail.tsx index 25b35c34861..809272c2ac0 100644 --- a/ui/litellm-dashboard/src/components/claude_code_plugins/skill_detail.tsx +++ b/ui/litellm-dashboard/src/components/claude_code_plugins/skill_detail.tsx @@ -1,5 +1,6 @@ import React, { useState } from "react"; import { ArrowLeft, Check, Copy, Link2 } from "lucide-react"; +import { cn } from "@/lib/cva.config"; import { buildMarketplaceSettingsSnippet, formatInstallCommand } from "./helpers"; import { Plugin } from "./types"; @@ -50,48 +51,37 @@ const SkillDetail: React.FC = ({ skill, onBack }) => { ]; return ( -
+
{/* Back link */}
Skills
{/* Header */} -
-

{skill.name}

+
+

{skill.name}

{skill.description && ( -

{skill.description}

+

{skill.description}

)}
{/* Tab bar */} -
-
+
+
{tabs.map((tab) => (
setActiveTab(tab.key)} - style={{ - padding: "12px 20px", - fontSize: 14, - color: activeTab === tab.key ? "#1a73e8" : "#5f6368", - borderBottom: activeTab === tab.key ? "3px solid #1a73e8" : "3px solid transparent", - cursor: "pointer", - fontWeight: activeTab === tab.key ? 500 : 400, - marginBottom: -1, - }} + className={cn( + "-mb-px cursor-pointer border-b-[3px] px-5 py-3 text-sm", + activeTab === tab.key + ? "border-info font-medium text-info" + : "border-transparent font-normal text-muted-foreground", + )} > {tab.label}
@@ -101,27 +91,23 @@ const SkillDetail: React.FC = ({ skill, onBack }) => { {/* Overview tab */} {activeTab === "overview" && ( -
+
{/* Left column */} -
-

Skill Details

-

Metadata registered with this skill

- +
+

Skill Details

+

Metadata registered with this skill

+
- - - + + + {detailRows.map((row, i) => ( - - - + + + ))} @@ -129,38 +115,27 @@ const SkillDetail: React.FC = ({ skill, onBack }) => { {/* Right sidebar */} -
-
-
Status
+
+
+
Status
{skill.enabled ? "Public" : "Draft"}
{sourceUrl && ( -
-
Source
+
+
Source
{sourceUrl.replace("https://", "")} @@ -169,20 +144,13 @@ const SkillDetail: React.FC = ({ skill, onBack }) => { )} {skill.keywords && skill.keywords.length > 0 && ( -
-
Tags
-
+
+
Tags
+
{skill.keywords.map((kw) => ( {kw} @@ -192,10 +160,8 @@ const SkillDetail: React.FC = ({ skill, onBack }) => { )}
-
Skill ID
-
- {skill.id} -
+
Skill ID
+
{skill.id}
@@ -203,93 +169,43 @@ const SkillDetail: React.FC = ({ skill, onBack }) => { {/* How to Use tab */} {activeTab === "usage" && ( -
-

Using this skill

-

+

+

Using this skill

+

Once your proxy is set as a marketplace, enable this skill in Claude Code with one command:

{/* Install command */} -
-
- Run in Claude Code +
+
+ Run in Claude Code
- 
-              {installCommand}
-
+ 
{installCommand}
{/* Shown when the marketplace catalog is stale and the plugin isn't found yet */} -
-

+

+

If you see "Plugin {skill.name} not found in marketplace", update the catalog first:

- 
+            
               /plugin marketplace update litellm
-

+

Don't have the marketplace configured yet?{" "} - setActiveTab("setup")} style={{ color: "#1a73e8", cursor: "pointer" }}> + setActiveTab("setup")} className="cursor-pointer text-info"> See one-time setup →

@@ -298,126 +214,56 @@ const SkillDetail: React.FC = ({ skill, onBack }) => { {/* Setup tab (linked from usage) */} {activeTab === "setup" && ( -
-

- One-time marketplace setup -

+
+

One-time marketplace setup

{/* Option 1: single command — fastest path for most users */} -

+

Run this command in Claude Code to register the marketplace:

-
-
- Run in Claude Code +
+
+ Run in Claude Code
- 
+            
               {`/plugin marketplace add ${typeof window !== "undefined" ? window.location.origin : ""}/claude-code/marketplace.json`}
{/* Option 2: settings.json — for persistent config or managed deployments. extraKnownMarketplaces requires source to be a nested object, not a flat string. */} -

- Or add this to{" "} - - ~/.claude/settings.json - {" "} +

+ Or add this to ~/.claude/settings.json{" "} for a persistent configuration:

-
-
- ~/.claude/settings.json +
+
+ ~/.claude/settings.json
- 
-              {settingsSnippet}
-
+ 
{settingsSnippet}
)} From 8acdb9208756fb306680878d06f53bbd75c2f04a Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Tue, 1 Sep 2026 10:27:43 -0700 Subject: [PATCH 11/12] fix(ui): keep the skill detail copy buttons transparent bg-none only clears background-image, so the buttons fell back to the browser's default button background instead of the transparent one the inline style had. --- .../src/components/claude_code_plugins/skill_detail.tsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ui/litellm-dashboard/src/components/claude_code_plugins/skill_detail.tsx b/ui/litellm-dashboard/src/components/claude_code_plugins/skill_detail.tsx index 809272c2ac0..7cbbb08b623 100644 --- a/ui/litellm-dashboard/src/components/claude_code_plugins/skill_detail.tsx +++ b/ui/litellm-dashboard/src/components/claude_code_plugins/skill_detail.tsx @@ -182,7 +182,7 @@ const SkillDetail: React.FC = ({ skill, onBack }) => {
- Property - - {skill.name} -
Property{skill.name}
{row.property}{row.value}
{row.property}{row.value}