mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-27 01:22:18 +00:00
dc85812971
Three holes in the new suite, all of which let a test pass without proving what its name claims: - A migration recorded twice, once per replica, each with applied_steps_count = 1, slipped past both the step-count check and migration_names(), which collapses the history into a set. Reject duplicate migration_name rows outright. - auth_traffic only asserted the failures it had seen by the time keep_serving hit its target. A request failing after that, or on the other replica while the test waited on one stream, was recorded and never read. Assert the recorded failures once the thread has joined. - The rolling test warmed the baseline replica's virtual-key cache before the upgrade, and that cache holds for 60 seconds by default (UserAPIKeyCacheTTLEnum.in_memory_cache_ttl). The candidate migrates well inside that window, so the post-upgrade requests could be served from cache without ever repeating the whole-row token lookup that the stale prepared statement breaks. Drive the baseline replica with a key minted after the schema moved, which it has never seen and must resolve from the database. Re-ran against v1.101.0 -> v1.102.0: 6 passed.
57 lines
2.8 KiB
Python
57 lines
2.8 KiB
Python
from typing import Final
|
|
|
|
import pytest
|
|
|
|
from .containers import Containers, ready
|
|
from .database import Database
|
|
from .upgrade import (
|
|
CACHED_PLAN,
|
|
assert_history_clean,
|
|
assert_upgraded,
|
|
auth_traffic,
|
|
confirm,
|
|
keep_serving,
|
|
migration_names,
|
|
provision,
|
|
)
|
|
|
|
pytestmark: Final = [pytest.mark.e2e, pytest.mark.migration_startup]
|
|
|
|
|
|
class TestRollingUpgrade:
|
|
def test_baseline_replica_keeps_serving_while_the_candidate_migrates(
|
|
self, containers: Containers, baseline_image: str, baseline_database: Database
|
|
) -> None:
|
|
with containers.using(baseline_image).start(baseline_database) as old:
|
|
ready((old,), baseline_database)
|
|
key, _ = provision(old)
|
|
before: Final = migration_names(baseline_database)
|
|
with auth_traffic(old, key) as traffic:
|
|
keep_serving(traffic, "the baseline replica authenticating before the upgrade")
|
|
with containers.start(baseline_database) as new:
|
|
ready((new,), baseline_database)
|
|
assert_upgraded(before, migration_names(baseline_database))
|
|
keep_serving(traffic, "the baseline replica authenticating after the schema moved")
|
|
with auth_traffic(old, provision(new)[0]) as uncached:
|
|
keep_serving(uncached, "the baseline replica resolving a key minted after the schema moved")
|
|
assert_history_clean(baseline_database)
|
|
assert CACHED_PLAN not in old.logs(), "The baseline replica hit a stale prepared statement"
|
|
assert old.state().Running, "The baseline replica died during the upgrade"
|
|
|
|
def test_both_releases_serve_and_share_keys_during_the_overlap(
|
|
self, containers: Containers, baseline_image: str, baseline_database: Database
|
|
) -> None:
|
|
with containers.using(baseline_image).start(baseline_database) as old:
|
|
ready((old,), baseline_database)
|
|
old_key, old_alias = provision(old)
|
|
before: Final = migration_names(baseline_database)
|
|
with containers.start(baseline_database) as new:
|
|
ready((new,), baseline_database)
|
|
assert_upgraded(before, migration_names(baseline_database))
|
|
new_key, new_alias = provision(new)
|
|
with auth_traffic(old, old_key) as old_traffic, auth_traffic(new, new_key) as new_traffic:
|
|
keep_serving(old_traffic, "the baseline replica serving through the overlap")
|
|
keep_serving(new_traffic, "the candidate replica serving through the overlap")
|
|
confirm(old, new_key, new_alias)
|
|
confirm(new, old_key, old_alias)
|
|
assert CACHED_PLAN not in old.logs(), "The baseline replica hit a stale prepared statement"
|