mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-27 01:22:18 +00:00
45d22dc5e1
The migration e2e harness only ever used one image: it seeded the database with the candidate build and then applied synthetic migrations on top. That proves the migration machinery (locking, crash recovery, legacy baselining, pooling) but never executes the real schema of release N against the real migrations of release N+1, which is the path operators actually run. Adds a baseline image alongside the candidate, so a test can seed with a published release and upgrade with the build under test. Suites: - test_upgrade.py: the candidate applies the pending release migrations, keys minted by the baseline release survive, and concurrent replicas upgrade a baseline database exactly once. - test_rolling_upgrade.py: a baseline replica keeps serving virtual-key auth while the candidate migrates underneath it, and both releases serve and resolve each other's keys during the overlap. This is the reported failure: a new column on LiteLLM_VerificationToken invalidates prepared plans on pods still running the old release, which the proxy reads whole-row, and auth starts failing until those pods leave service. - test_shaped_database.py: the upgrade completes and preserves rows on a populated spend log, rather than on the empty database every other migration test starts from. Every upgrade assertion is gated on the candidate having actually applied migrations the baseline had not, so a stale pin fails loudly instead of passing on an empty delta. CI adds two jobs to the migration_startup workflow. The baseline defaults to a committed release pin and is overridable per pipeline, matching how migration_candidate_image already works; only the upgrade jobs pull it. Verified against a real v1.101.0 -> v1.102.0 upgrade: 6 passed, with the baseline seeding 165 migrations and the candidate applying the 6 that landed between the two releases.
47 lines
2.1 KiB
Python
47 lines
2.1 KiB
Python
from contextlib import ExitStack
|
|
from typing import Final
|
|
|
|
import pytest
|
|
|
|
from .checks import start_replicas
|
|
from .containers import Containers, ready
|
|
from .database import Database
|
|
from .upgrade import assert_history_clean, assert_upgraded, confirm, migration_names, provision
|
|
|
|
pytestmark: Final = [pytest.mark.e2e, pytest.mark.migration_startup]
|
|
|
|
|
|
class TestReleaseUpgrade:
|
|
def test_candidate_applies_the_pending_release_migrations(
|
|
self, containers: Containers, baseline_database: Database
|
|
) -> None:
|
|
before: Final = migration_names(baseline_database)
|
|
with containers.start(baseline_database) as replica:
|
|
ready((replica,), baseline_database)
|
|
assert_upgraded(before, migration_names(baseline_database))
|
|
assert_history_clean(baseline_database)
|
|
|
|
def test_upgrade_preserves_keys_minted_by_the_baseline_release(
|
|
self, containers: Containers, baseline_image: str, baseline_database: Database
|
|
) -> None:
|
|
with containers.using(baseline_image).start(baseline_database) as old:
|
|
ready((old,), baseline_database)
|
|
key, alias = provision(old)
|
|
confirm(old, key, alias)
|
|
before: Final = migration_names(baseline_database)
|
|
with containers.start(baseline_database) as new:
|
|
ready((new,), baseline_database)
|
|
assert_upgraded(before, migration_names(baseline_database))
|
|
confirm(new, key, alias)
|
|
|
|
def test_concurrent_replicas_upgrade_a_baseline_database_once(
|
|
self, containers: Containers, baseline_database: Database
|
|
) -> None:
|
|
before: Final = migration_names(baseline_database)
|
|
with ExitStack() as stack:
|
|
ready(start_replicas(stack, containers, baseline_database), baseline_database)
|
|
assert_upgraded(before, migration_names(baseline_database))
|
|
assert_history_clean(baseline_database)
|
|
assert baseline_database.query("SELECT count(*) FROM _prisma_migrations WHERE applied_steps_count > 1") == (
|
|
(0,),
|
|
), "A migration was executed more than once across the upgrading replicas"
|