mirror of
https://github.com/BerriAI/litellm.git
synced 2026-08-28 05:25:59 +00:00
53c9d48bd2
Adds a scheduled GitHub Actions lane on top of the merged record/replay transport. A Saturday cron records the `replayable` e2e tests against the real providers and publishes the fixture bundle as a private `e2e-fixtures-bundle` artifact with a SHA-256 sidecar. Weekday crons pull that artifact by its pinned digest, verify the checksum before extracting, and replay it with provider credentials set to bogus values, so a run that ever reached a real provider fails instead of passing. An egress sentinel pins the provider hostnames to a local sink for the whole replay job and counts every connection that reaches them; the job asserts that count is zero, so hermeticity is proven by measurement. A red Saturday publishes no bundle, so the next weekday finds nothing fresh and fails loudly rather than replaying a week-old recording, and the transport's seven-day freshness gate hard-fails any bundle that has drifted too far. The lane also runs on demand from the Actions tab with a record/replay `mode` input. Tests join the lane with `@pytest.mark.replayable`. The streaming Anthropic test now counts to twenty so its recorded response banks several content deltas, matching the assertion that the stream arrives incrementally.
55 lines
2.3 KiB
Bash
Executable file
55 lines
2.3 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
REPO="${1:-${GITHUB_REPOSITORY:?REPO required}}"
|
|
ARTIFACT_NAME="${2:-e2e-fixtures-bundle}"
|
|
BASE_BRANCH="${3:?base branch required}"
|
|
DEST_DIR="${4:?destination bundle dir required}"
|
|
|
|
: "${GH_TOKEN:?GH_TOKEN required to query and download artifacts}"
|
|
|
|
WORKDIR="$(mktemp -d)"
|
|
trap 'rm -rf "${WORKDIR}"' EXIT
|
|
|
|
echo "resolving newest non-expired '${ARTIFACT_NAME}' artifact on ${REPO}@${BASE_BRANCH}"
|
|
|
|
SELECTED="$(
|
|
gh api "repos/${REPO}/actions/artifacts" -X GET -f per_page=100 --paginate \
|
|
--jq ".artifacts[] | select(.name == \"${ARTIFACT_NAME}\" and .expired == false and .workflow_run.head_branch == \"${BASE_BRANCH}\") | {id, digest, created_at, run_id: .workflow_run.id, run_number: .workflow_run.run_number}" \
|
|
| jq -s 'sort_by(.created_at) | reverse | .[0] // empty'
|
|
)"
|
|
|
|
if [[ -z "${SELECTED}" ]]; then
|
|
echo "no usable '${ARTIFACT_NAME}' artifact on ${BASE_BRANCH}: the last record run produced none (a red Saturday), so there is nothing fresh to replay; failing loudly instead of replaying a stale bundle" >&2
|
|
exit 1
|
|
fi
|
|
|
|
RUN_ID="$(echo "${SELECTED}" | jq -r '.run_id')"
|
|
RUN_NUMBER="$(echo "${SELECTED}" | jq -r '.run_number')"
|
|
ARTIFACT_ID="$(echo "${SELECTED}" | jq -r '.id')"
|
|
GH_DIGEST="$(echo "${SELECTED}" | jq -r '.digest // "unknown"')"
|
|
CREATED_AT="$(echo "${SELECTED}" | jq -r '.created_at')"
|
|
|
|
echo "pinned bundle: run #${RUN_NUMBER} (run_id=${RUN_ID}, artifact_id=${ARTIFACT_ID}), recorded ${CREATED_AT}, github digest ${GH_DIGEST}"
|
|
|
|
gh run download "${RUN_ID}" --repo "${REPO}" -n "${ARTIFACT_NAME}" -D "${WORKDIR}"
|
|
|
|
TARBALL="$(find "${WORKDIR}" -name '*.tar.gz' -type f | head -n 1)"
|
|
if [[ -z "${TARBALL}" ]]; then
|
|
echo "downloaded artifact contained no tarball" >&2
|
|
exit 1
|
|
fi
|
|
SIDECAR="${TARBALL}.sha256"
|
|
if [[ ! -f "${SIDECAR}" ]]; then
|
|
echo "downloaded artifact has no ${SIDECAR}: cannot verify the bundle digest" >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo "verifying bundle against its recorded sha256 digest"
|
|
( cd "$(dirname "${TARBALL}")" && sha256sum -c "$(basename "${SIDECAR}")" )
|
|
|
|
mkdir -p "${DEST_DIR}"
|
|
tar xzf "${TARBALL}" -C "${DEST_DIR}"
|
|
|
|
echo "extracted bundle into ${DEST_DIR}"
|
|
python3 -c "import json,sys; m=json.load(open(sys.argv[1])); print(' recorded_at', m['recorded_at'], 'harness', m['harness_version'], 'format_version', m['format_version'])" "${DEST_DIR}/manifest.json"
|