mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-24 00:52:24 +00:00
45d22dc5e1
The migration e2e harness only ever used one image: it seeded the database with the candidate build and then applied synthetic migrations on top. That proves the migration machinery (locking, crash recovery, legacy baselining, pooling) but never executes the real schema of release N against the real migrations of release N+1, which is the path operators actually run. Adds a baseline image alongside the candidate, so a test can seed with a published release and upgrade with the build under test. Suites: - test_upgrade.py: the candidate applies the pending release migrations, keys minted by the baseline release survive, and concurrent replicas upgrade a baseline database exactly once. - test_rolling_upgrade.py: a baseline replica keeps serving virtual-key auth while the candidate migrates underneath it, and both releases serve and resolve each other's keys during the overlap. This is the reported failure: a new column on LiteLLM_VerificationToken invalidates prepared plans on pods still running the old release, which the proxy reads whole-row, and auth starts failing until those pods leave service. - test_shaped_database.py: the upgrade completes and preserves rows on a populated spend log, rather than on the empty database every other migration test starts from. Every upgrade assertion is gated on the candidate having actually applied migrations the baseline had not, so a stale pin fails loudly instead of passing on an empty delta. CI adds two jobs to the migration_startup workflow. The baseline defaults to a committed release pin and is overridable per pipeline, matching how migration_candidate_image already works; only the upgrade jobs pull it. Verified against a real v1.101.0 -> v1.102.0 upgrade: 6 passed, with the baseline seeding 165 migrations and the candidate applying the 6 that landed between the two releases.
116 lines
4 KiB
Python
116 lines
4 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Final
|
|
from xml.etree import ElementTree
|
|
|
|
SUITES: Final = {
|
|
"startup": (("test_startup.py",), 12),
|
|
"recovery": (("test_recovery.py",), 15),
|
|
"legacy": (("test_legacy.py", "test_pooling.py"), 11),
|
|
"upgrade": (("test_upgrade.py", "test_rolling_upgrade.py"), 5),
|
|
"shaped": (("test_shaped_database.py",), 1),
|
|
}
|
|
|
|
|
|
def successful_junit(path: Path, expected: int, exit_code: int) -> bool:
|
|
if exit_code != 0 or not path.is_file():
|
|
return False
|
|
try:
|
|
root: Final = ElementTree.parse(path).getroot()
|
|
except ElementTree.ParseError:
|
|
return False
|
|
cases: Final = tuple(root.iter("testcase"))
|
|
identities: Final = frozenset((case.get("classname"), case.get("name")) for case in cases)
|
|
return len(cases) == len(identities) == expected and all(
|
|
not any(case.find(tag) is not None for tag in ("failure", "error", "skipped")) for case in cases
|
|
)
|
|
|
|
|
|
def output(*command: str) -> str:
|
|
return subprocess.check_output(command, text=True, timeout=90).strip()
|
|
|
|
|
|
def record_image() -> None:
|
|
source: Final = output("git", "rev-parse", "HEAD")
|
|
image: Final = output("docker", "image", "inspect", "litellm-docker-database:ci", "--format", "{{.Id}}")
|
|
revision: Final = output(
|
|
"docker",
|
|
"image",
|
|
"inspect",
|
|
"litellm-docker-database:ci",
|
|
"--format",
|
|
'{{index .Config.Labels "org.opencontainers.image.revision"}}',
|
|
)
|
|
assert re.fullmatch(r"[0-9a-f]{40}", source), "Invalid source revision"
|
|
assert revision == source, "Candidate image revision differs from the tested source"
|
|
Path("migration-image.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"source_sha": source,
|
|
"image_id": image,
|
|
"candidate_image": os.environ.get("MIGRATION_CANDIDATE_IMAGE", ""),
|
|
}
|
|
)
|
|
)
|
|
|
|
|
|
def main() -> int:
|
|
suite: Final = os.environ["MIGRATION_TEST_SUITE"]
|
|
files, expected = SUITES[suite]
|
|
metadata: Final = json.loads(Path("migration-image.json").read_text())
|
|
assert metadata["source_sha"] == output("git", "rev-parse", "HEAD"), "Image and test source revisions differ"
|
|
assert metadata["image_id"] == output(
|
|
"docker", "image", "inspect", os.environ["LITELLM_MIGRATION_TEST_IMAGE"], "--format", "{{.Id}}"
|
|
), "Loaded image differs from the build output"
|
|
assert metadata["candidate_image"] == os.environ.get("MIGRATION_CANDIDATE_IMAGE", ""), "Wrong release candidate"
|
|
destination: Final = Path(os.environ["MIGRATION_TEST_OUTPUT"])
|
|
junit: Final = destination / "junit" / "results.xml"
|
|
junit.parent.mkdir(parents=True, exist_ok=True)
|
|
result: Final = subprocess.run(
|
|
(
|
|
sys.executable,
|
|
"-m",
|
|
"pytest",
|
|
*(f"tests/e2e/migrations/{name}" for name in files),
|
|
"-vv",
|
|
"--tb=short",
|
|
"--durations=10",
|
|
f"--junitxml={junit}",
|
|
"-o",
|
|
"addopts=",
|
|
"--reruns=0",
|
|
),
|
|
check=False,
|
|
timeout=1200,
|
|
)
|
|
passed: Final = successful_junit(junit, expected, result.returncode)
|
|
(destination / "verdict.json").write_text(
|
|
json.dumps(
|
|
{
|
|
**metadata,
|
|
"suite": suite,
|
|
"baseline_image": os.environ.get("LITELLM_MIGRATION_BASELINE_IMAGE", ""),
|
|
"expected_cases": expected,
|
|
"passed": passed,
|
|
"pytest_exit_code": result.returncode,
|
|
"test_revision": metadata["source_sha"],
|
|
"workflow_id": os.environ.get("CIRCLE_WORKFLOW_ID", ""),
|
|
"job_number": os.environ.get("CIRCLE_BUILD_NUM", ""),
|
|
},
|
|
indent=2,
|
|
)
|
|
)
|
|
return 0 if passed else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
if sys.argv[1:] == ["record-image"]:
|
|
record_image()
|
|
else:
|
|
raise SystemExit(main())
|