mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-26 06:03:19 +00:00
464a4dba9d
* test(migrations): assert lock-timeout retry and exhaustion instead of an unbounded wait Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(migrations): accumulate follower lock wait observations Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: mateo <mateo@berri.ai> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
116 lines
4 KiB
Python
116 lines
4 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Final
|
|
from xml.etree import ElementTree
|
|
|
|
SUITES: Final = {
|
|
"startup": (("test_startup.py",), 13),
|
|
"recovery": (("test_recovery.py",), 15),
|
|
"legacy": (("test_legacy.py", "test_pooling.py"), 11),
|
|
"upgrade": (("test_upgrade.py", "test_rolling_upgrade.py"), 5),
|
|
"shaped": (("test_shaped_database.py",), 1),
|
|
}
|
|
|
|
|
|
def successful_junit(path: Path, expected: int, exit_code: int) -> bool:
|
|
if exit_code != 0 or not path.is_file():
|
|
return False
|
|
try:
|
|
root: Final = ElementTree.parse(path).getroot()
|
|
except ElementTree.ParseError:
|
|
return False
|
|
cases: Final = tuple(root.iter("testcase"))
|
|
identities: Final = frozenset((case.get("classname"), case.get("name")) for case in cases)
|
|
return len(cases) == len(identities) == expected and all(
|
|
not any(case.find(tag) is not None for tag in ("failure", "error", "skipped")) for case in cases
|
|
)
|
|
|
|
|
|
def output(*command: str) -> str:
|
|
return subprocess.check_output(command, text=True, timeout=90).strip()
|
|
|
|
|
|
def record_image() -> None:
|
|
source: Final = output("git", "rev-parse", "HEAD")
|
|
image: Final = output("docker", "image", "inspect", "litellm-docker-database:ci", "--format", "{{.Id}}")
|
|
revision: Final = output(
|
|
"docker",
|
|
"image",
|
|
"inspect",
|
|
"litellm-docker-database:ci",
|
|
"--format",
|
|
'{{index .Config.Labels "org.opencontainers.image.revision"}}',
|
|
)
|
|
assert re.fullmatch(r"[0-9a-f]{40}", source), "Invalid source revision"
|
|
assert revision == source, "Candidate image revision differs from the tested source"
|
|
Path("migration-image.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"source_sha": source,
|
|
"image_id": image,
|
|
"candidate_image": os.environ.get("MIGRATION_CANDIDATE_IMAGE", ""),
|
|
}
|
|
)
|
|
)
|
|
|
|
|
|
def main() -> int:
|
|
suite: Final = os.environ["MIGRATION_TEST_SUITE"]
|
|
files, expected = SUITES[suite]
|
|
metadata: Final = json.loads(Path("migration-image.json").read_text())
|
|
assert metadata["source_sha"] == output("git", "rev-parse", "HEAD"), "Image and test source revisions differ"
|
|
assert metadata["image_id"] == output(
|
|
"docker", "image", "inspect", os.environ["LITELLM_MIGRATION_TEST_IMAGE"], "--format", "{{.Id}}"
|
|
), "Loaded image differs from the build output"
|
|
assert metadata["candidate_image"] == os.environ.get("MIGRATION_CANDIDATE_IMAGE", ""), "Wrong release candidate"
|
|
destination: Final = Path(os.environ["MIGRATION_TEST_OUTPUT"])
|
|
junit: Final = destination / "junit" / "results.xml"
|
|
junit.parent.mkdir(parents=True, exist_ok=True)
|
|
result: Final = subprocess.run(
|
|
(
|
|
sys.executable,
|
|
"-m",
|
|
"pytest",
|
|
*(f"tests/e2e/migrations/{name}" for name in files),
|
|
"-vv",
|
|
"--tb=short",
|
|
"--durations=10",
|
|
f"--junitxml={junit}",
|
|
"-o",
|
|
"addopts=",
|
|
"--reruns=0",
|
|
),
|
|
check=False,
|
|
timeout=1200,
|
|
)
|
|
passed: Final = successful_junit(junit, expected, result.returncode)
|
|
(destination / "verdict.json").write_text(
|
|
json.dumps(
|
|
{
|
|
**metadata,
|
|
"suite": suite,
|
|
"baseline_image": os.environ.get("LITELLM_MIGRATION_BASELINE_IMAGE", ""),
|
|
"expected_cases": expected,
|
|
"passed": passed,
|
|
"pytest_exit_code": result.returncode,
|
|
"test_revision": metadata["source_sha"],
|
|
"workflow_id": os.environ.get("CIRCLE_WORKFLOW_ID", ""),
|
|
"job_number": os.environ.get("CIRCLE_BUILD_NUM", ""),
|
|
},
|
|
indent=2,
|
|
)
|
|
)
|
|
return 0 if passed else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
if sys.argv[1:] == ["record-image"]:
|
|
record_image()
|
|
else:
|
|
raise SystemExit(main())
|