mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-15 23:31:29 +00:00
71 lines
2.6 KiB
Python
71 lines
2.6 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
from types import MappingProxyType
|
|
from typing import Final
|
|
|
|
GROUPS: Final = MappingProxyType(json.loads(Path(__file__).with_name("contracts.json").read_text())["groups"])
|
|
|
|
|
|
def main() -> int:
|
|
parser: Final = argparse.ArgumentParser()
|
|
parser.add_argument("group", choices=tuple(GROUPS))
|
|
parser.add_argument("--results", type=Path, default=Path("test-results/integration"))
|
|
parser.add_argument("--seed", type=int, default=int(os.environ.get("INTEGRATION_SEED", "4106601")))
|
|
options: Final = parser.parse_args()
|
|
root: Final = Path(__file__).resolve().parents[2]
|
|
selected: Final = tuple(
|
|
str(path.relative_to(root))
|
|
for folder in GROUPS[options.group]
|
|
for path in sorted((root / "tests/integration" / folder).glob("test_*.py"))
|
|
)
|
|
if not selected:
|
|
parser.error(f"No integration contracts selected for {options.group}")
|
|
output: Final = options.results.resolve()
|
|
output.mkdir(parents=True, exist_ok=True)
|
|
manifest: Final = json.loads((root / "tests/integration/contracts.json").read_text())["tests"]
|
|
expected: Final = sorted(node for node in manifest if node.split("::", 1)[0] in selected)
|
|
if not expected or set(selected) != {node.split("::", 1)[0] for node in expected}:
|
|
parser.error("Every selected file must have canonical manifest nodes")
|
|
environment: Final = {
|
|
**os.environ,
|
|
"PYTHONPATH": os.pathsep.join((str(root), str(root / "tests"), str(root / "tests/e2e"))),
|
|
"INTEGRATION_RESULTS_DIR": str(output),
|
|
"LITELLM_LOCAL_MODEL_COST_MAP": "True",
|
|
}
|
|
result: Final = subprocess.call(
|
|
[
|
|
sys.executable,
|
|
"-m",
|
|
"pytest",
|
|
*selected,
|
|
"-vv",
|
|
"--strict-markers",
|
|
"-p",
|
|
"no:pytest-retry",
|
|
"-p",
|
|
"no:rerunfailures",
|
|
"--timeout=90",
|
|
"--durations=15",
|
|
f"--hypothesis-seed={options.seed}",
|
|
f"--junitxml={output / 'junit.xml'}",
|
|
],
|
|
cwd=root,
|
|
env=environment,
|
|
)
|
|
if result != 0:
|
|
return result
|
|
evidence: Final = json.loads((output / "execution.json").read_text())
|
|
if not evidence["complete"] or sorted(evidence["passed"]) != expected or sorted(evidence["collected"]) != expected:
|
|
print("Executed integration nodes differ from the canonical manifest", file=sys.stderr)
|
|
return 1
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|