mirror of
https://github.com/BerriAI/litellm.git
synced 2026-08-28 05:25:59 +00:00
53c9d48bd2
Adds a scheduled GitHub Actions lane on top of the merged record/replay transport. A Saturday cron records the `replayable` e2e tests against the real providers and publishes the fixture bundle as a private `e2e-fixtures-bundle` artifact with a SHA-256 sidecar. Weekday crons pull that artifact by its pinned digest, verify the checksum before extracting, and replay it with provider credentials set to bogus values, so a run that ever reached a real provider fails instead of passing. An egress sentinel pins the provider hostnames to a local sink for the whole replay job and counts every connection that reaches them; the job asserts that count is zero, so hermeticity is proven by measurement. A red Saturday publishes no bundle, so the next weekday finds nothing fresh and fails loudly rather than replaying a week-old recording, and the transport's seven-day freshness gate hard-fails any bundle that has drifted too far. The lane also runs on demand from the Actions tab with a record/replay `mode` input. Tests join the lane with `@pytest.mark.replayable`. The streaming Anthropic test now counts to twenty so its recorded response banks several content deltas, matching the assertion that the stream arrives incrementally.
36 lines
1.2 KiB
Bash
Executable file
36 lines
1.2 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
if [[ $# -ne 2 ]]; then
|
|
echo "usage: $0 <bundle-dir> <out-tarball>" >&2
|
|
exit 2
|
|
fi
|
|
|
|
BUNDLE_DIR="$1"
|
|
OUT_TARBALL="$2"
|
|
|
|
MANIFEST="${BUNDLE_DIR}/manifest.json"
|
|
if [[ ! -f "${MANIFEST}" ]]; then
|
|
echo "no ${MANIFEST}: refusing to publish a bundle with no manifest (record produced nothing)" >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo "packing fixture bundle from ${BUNDLE_DIR}"
|
|
python3 -c "import json,sys; m=json.load(open(sys.argv[1])); print(' format_version', m['format_version'], 'recorded_at', m['recorded_at'], 'harness', m['harness_version'])" "${MANIFEST}"
|
|
|
|
TEST_DIRS=$(find "${BUNDLE_DIR}" -mindepth 1 -maxdepth 1 -type d | wc -l | tr -d ' ')
|
|
if [[ "${TEST_DIRS}" -eq 0 ]]; then
|
|
echo "bundle at ${BUNDLE_DIR} has a manifest but no recorded interactions; refusing to publish an empty bundle" >&2
|
|
exit 1
|
|
fi
|
|
echo " ${TEST_DIRS} recorded test director(ies)"
|
|
|
|
mkdir -p "$(dirname "${OUT_TARBALL}")"
|
|
tar czf "${OUT_TARBALL}" -C "${BUNDLE_DIR}" .
|
|
|
|
OUT_DIR="$(cd "$(dirname "${OUT_TARBALL}")" && pwd)"
|
|
OUT_BASE="$(basename "${OUT_TARBALL}")"
|
|
( cd "${OUT_DIR}" && sha256sum "${OUT_BASE}" > "${OUT_BASE}.sha256" )
|
|
|
|
echo "wrote ${OUT_TARBALL} ($(du -h "${OUT_TARBALL}" | cut -f1)) and ${OUT_BASE}.sha256"
|
|
cat "${OUT_DIR}/${OUT_BASE}.sha256"
|