mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-10 22:41:41 +00:00
08bea8d0dd
The mateo-berri PAT now arrives via systemd LoadCredential as a file instead of the EnvironmentFile, so pytest, the proxy, and the model-driven claude CLI never inherit it and a same-UID /proc read cannot lift it. run_daily.sh reads the credential when present, still accepts an exported GITHUB_TOKEN for manual runs, and dies up front when publishing is enabled with neither. Full CLI sandboxing is tracked in LIT-5420
672 lines
29 KiB
Bash
Executable file
672 lines
29 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Daily Claude Code compatibility-matrix populator.
|
|
#
|
|
# Runs from the GCP VM `litellm-compatibility-matrix-populator` via the
|
|
# systemd timer in this directory. The flow is:
|
|
#
|
|
# 1. Resolve the latest LiteLLM final release tag from the GitHub
|
|
# Releases API.
|
|
# 2. Update a long-lived worktree at $WORKTREE to that tag and `uv sync` it.
|
|
# 3. Boot the proxy as a background subprocess on $PROXY_PORT (default
|
|
# 4100; a separate port from the human-tended :4000 proxy).
|
|
# 4. Run `pytest tests/e2e/claude_code/` against the proxy. Test
|
|
# failures become `fail` cells in the JSON, not script errors.
|
|
# 5. Hand the per-test results artifact + manifest to a small Python
|
|
# CLI (`build_matrix.py`) that wraps the existing
|
|
# `matrix_builder.build_from_paths` to produce the published
|
|
# compatibility-matrix.json.
|
|
# 6. `gh repo clone` litellm-docs, write the JSON to a deterministic
|
|
# branch (`compat-matrix/<litellm>-<claude>-<UTC-date>`), commit,
|
|
# push the branch straight to BerriAI/litellm-docs (mateo-berri has
|
|
# write access), `gh pr create`, then — *only if no cell regressed
|
|
# green→red versus the currently-published matrix* — enable squash
|
|
# auto-merge so the PR merges itself once required checks pass. A
|
|
# green→red regression leaves auto-merge off for human review; an
|
|
# already-red cell (red→red) does not block.
|
|
# 7. Sweep stale compat-matrix PRs: once today's PR exists, close any
|
|
# other open `compat-matrix/*` PR (and delete its bot-owned branch)
|
|
# so at most ONE compat-matrix PR is ever open — the newest. A
|
|
# gate-withheld PR that nobody triages is superseded by the next
|
|
# day's run rather than accumulating in the queue.
|
|
#
|
|
# Same-day reruns land on the same branch so they update the existing PR
|
|
# rather than spawning a new one. If the JSON is byte-identical to the
|
|
# docs branch, we skip the push entirely.
|
|
#
|
|
# Required commands on $PATH: git, uv, gh, jq, curl, claude, npm.
|
|
# Required state: a litellm checkout at $LITELLM_REPO (this file lives in
|
|
# it), $WORKTREE is created on first run, gh is already authenticated.
|
|
#
|
|
# Override any default by setting the matching env var; see the systemd
|
|
# unit for the production wiring.
|
|
|
|
set -Eeuo pipefail
|
|
|
|
LITELLM_REPO="${LITELLM_REPO:-${HOME}/litellm/litellm}"
|
|
WORKTREE="${LITELLM_WORKTREE:-${HOME}/litellm-cron-worktree}"
|
|
PROXY_PORT="${PROXY_PORT:-4100}"
|
|
PROXY_API_KEY="${PROXY_API_KEY:-sk-cron-matrix}"
|
|
DOCS_REPO="${DOCS_REPO:-BerriAI/litellm-docs}"
|
|
DOCS_BRANCH="${DOCS_BRANCH:-main}"
|
|
DOCS_TARGET_PATH="${DOCS_TARGET_PATH:-src/data/compatibility-matrix.json}"
|
|
SKIP_PUBLISH="${SKIP_PUBLISH:-0}"
|
|
PYTEST_K="${PYTEST_K:-}"
|
|
# The e2e suite uses PEP 695 `type` aliases, so the venv needs Python
|
|
# >= 3.12 (also what repo CI runs) even when the VM's system python is
|
|
# older. uv fetches a managed CPython of this version on first use --
|
|
# checksum-verified against the manifest baked into the pinned uv
|
|
# binary -- and installs it under ${WORKTREE}/.uv-python (see
|
|
# UV_PYTHON_INSTALL_DIR below) so it lives inside the one tree the
|
|
# systemd sandbox lets us write to.
|
|
CRON_PYTHON_VERSION="${CRON_PYTHON_VERSION:-3.12}"
|
|
# Merge method for auto-merge. BerriAI/litellm-docs only allows squash
|
|
# merges (merge-commit and rebase are disabled at the repo level), so
|
|
# `squash` is the only valid value here unless that changes upstream.
|
|
AUTO_MERGE_METHOD="${AUTO_MERGE_METHOD:-squash}"
|
|
|
|
POPULATOR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
WORKDIR="$(mktemp -d -t litellm-compat-matrix.XXXXXX)"
|
|
PROXY_PID_FILE="${WORKDIR}/proxy.pid"
|
|
|
|
# Cleanup is intentionally aggressive: it can run on normal exit, on a
|
|
# signal received by the script, or after a partial failure where the
|
|
# proxy is up but ${PROXY_PID_FILE} is stale. We try four things in
|
|
# order and stop as soon as the proxy port is free:
|
|
#
|
|
# 1. SIGTERM the pid recorded in proxy.pid.
|
|
# 2. SIGKILL anything from `pgrep -f "litellm.*--port ${PROXY_PORT}"`
|
|
# that survived. This catches the common case where the recorded
|
|
# pid was the sh wrapper, not the long-lived python child.
|
|
# 3. ss -K on the port (kernel kills sockets but not processes;
|
|
# mostly useful for catching lingering CLOSE_WAITs).
|
|
# 4. wipe ${WORKDIR}.
|
|
cleanup() {
|
|
local rc=$?
|
|
set +e
|
|
local proxy_pid
|
|
if [[ -f "${PROXY_PID_FILE}" ]]; then
|
|
proxy_pid="$(cat "${PROXY_PID_FILE}")"
|
|
if [[ -n "${proxy_pid}" ]]; then
|
|
kill -TERM "-${proxy_pid}" 2>/dev/null || kill -TERM "${proxy_pid}" 2>/dev/null || true
|
|
for _ in 1 2 3 4 5; do
|
|
kill -0 "${proxy_pid}" 2>/dev/null || break
|
|
sleep 1
|
|
done
|
|
fi
|
|
fi
|
|
# Belt-and-braces: any python or uv talking to ${PROXY_PORT} that
|
|
# survived the SIGTERM gets SIGKILL'd by name.
|
|
pgrep -f "litellm.*--port[ =]?${PROXY_PORT}([^0-9]|$)" 2>/dev/null \
|
|
| xargs -r kill -KILL 2>/dev/null || true
|
|
pgrep -f "${WORKTREE}/.uv-bin/uv.*run litellm" 2>/dev/null \
|
|
| xargs -r kill -KILL 2>/dev/null || true
|
|
rm -rf "${WORKDIR}"
|
|
exit "${rc}"
|
|
}
|
|
trap cleanup EXIT INT TERM
|
|
|
|
log() { printf '==> %s\n' "$*" >&2; }
|
|
die() { printf 'ERROR: %s\n' "$*" >&2; exit 1; }
|
|
|
|
for cmd in git uv gh jq curl claude; do
|
|
command -v "${cmd}" >/dev/null 2>&1 || die "missing required command: ${cmd}"
|
|
done
|
|
|
|
# Publishing pushes the branch straight to BerriAI/litellm-docs and opens
|
|
# the PR as mateo-berri, who has write access on the docs repo. Under
|
|
# systemd the PAT arrives as a file via LoadCredential=, NOT via the
|
|
# EnvironmentFile: several suite cells let the model-driven claude CLI
|
|
# read arbitrary files as this user, and /proc/<pid>/environ of the
|
|
# script, pytest, and the proxy would hand an env-borne token to any
|
|
# same-UID reader. Kept as an unexported shell variable and passed per
|
|
# invocation (GH_TOKEN=... / curl header / push URL), it never enters a
|
|
# child's environment. Manual runs may export GITHUB_TOKEN instead.
|
|
# Require it up front -- failing 30 minutes into a run is a waste of CI
|
|
# quota.
|
|
if [[ -z "${GITHUB_TOKEN:-}" && -n "${CREDENTIALS_DIRECTORY:-}" && -f "${CREDENTIALS_DIRECTORY}/github-token" ]]; then
|
|
GITHUB_TOKEN="$(<"${CREDENTIALS_DIRECTORY}/github-token")"
|
|
log "publish token source: systemd credential store"
|
|
elif [[ -n "${GITHUB_TOKEN:-}" ]]; then
|
|
log "publish token source: process environment"
|
|
fi
|
|
if [[ "${SKIP_PUBLISH}" != "1" ]]; then
|
|
[[ -n "${GITHUB_TOKEN:-}" ]] \
|
|
|| die "publish token required: /etc/litellm-compat-matrix-github-token via LoadCredential under systemd, or an exported GITHUB_TOKEN for manual runs (or set SKIP_PUBLISH=1)"
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 1. Resolve versions
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# Newest PEP 440 *final* release on BerriAI/litellm. LiteLLM moved off
|
|
# the legacy `vX.Y.Z-stable` tag convention to PEP 440: a final/stable
|
|
# release is now a bare `vX.Y.Z` tag, while pre-releases carry a
|
|
# `-rc.N` / `-dev.N` segment (and the old `…-stable` / `…-stable.patch.N`
|
|
# tags are legacy and frozen at v1.83.x). We therefore select the newest
|
|
# tag with no pre-release segment -- matching `^v[0-9]+\.[0-9]+\.[0-9]+$`
|
|
# -- and skip drafts. The numeric version_key sort handles 1.10 > 1.9.
|
|
#
|
|
# Paginate through the releases endpoint instead of grabbing only page 1
|
|
# (default page_size=30). LiteLLM ships multiple pre-releases per day, so
|
|
# it's common to need to walk past 30+ entries before hitting the most
|
|
# recent final release. We cap at 5 pages (500 releases) which is
|
|
# conservatively beyond the worst observed gap.
|
|
GH_AUTH_HEADER=()
|
|
if [[ -n "${GITHUB_TOKEN:-}" ]]; then
|
|
GH_AUTH_HEADER=(-H "Authorization: Bearer ${GITHUB_TOKEN}")
|
|
fi
|
|
RELEASES_JSON="${WORKDIR}/releases.json"
|
|
echo "[]" >"${RELEASES_JSON}"
|
|
for page in 1 2 3 4 5; do
|
|
PAGE_JSON="${WORKDIR}/releases.page${page}.json"
|
|
curl -fsS \
|
|
-H 'Accept: application/vnd.github+json' \
|
|
-H 'User-Agent: litellm-compat-matrix' \
|
|
"${GH_AUTH_HEADER[@]}" \
|
|
"https://api.github.com/repos/BerriAI/litellm/releases?per_page=100&page=${page}" \
|
|
>"${PAGE_JSON}"
|
|
jq -s '.[0] + .[1]' "${RELEASES_JSON}" "${PAGE_JSON}" >"${RELEASES_JSON}.merged"
|
|
mv "${RELEASES_JSON}.merged" "${RELEASES_JSON}"
|
|
# Stop early once we've seen at least one final release tag — no point
|
|
# paging further for a daily script that only needs the newest.
|
|
if jq -e '[.[] | select((.draft // false) == false) | .tag_name // "" | select(test("^v[0-9]+\\.[0-9]+\\.[0-9]+$"))] | length > 0' "${PAGE_JSON}" >/dev/null; then
|
|
break
|
|
fi
|
|
# No more pages? GitHub returns an empty array past the last page.
|
|
if [[ "$(jq 'length' "${PAGE_JSON}")" == "0" ]]; then
|
|
break
|
|
fi
|
|
done
|
|
LITELLM_VERSION="$(
|
|
jq -r '
|
|
[ .[]
|
|
| select((.draft // false) == false)
|
|
| .tag_name // empty
|
|
| select(test("^v[0-9]+\\.[0-9]+\\.[0-9]+$"))
|
|
]
|
|
| sort_by(
|
|
capture("^v(?<a>[0-9]+)\\.(?<b>[0-9]+)\\.(?<c>[0-9]+)$")
|
|
| [(.a|tonumber), (.b|tonumber), (.c|tonumber)]
|
|
)
|
|
| last // empty
|
|
' "${RELEASES_JSON}"
|
|
)"
|
|
[[ -n "${LITELLM_VERSION}" ]] || die "could not resolve latest PEP 440 final release (vX.Y.Z) in 5 pages of releases"
|
|
log "resolved litellm: ${LITELLM_VERSION}"
|
|
|
|
CLAUDE_CODE_VERSION="$(claude --version 2>/dev/null | awk '{print $1}')"
|
|
[[ -n "${CLAUDE_CODE_VERSION}" ]] || die "could not read 'claude --version'"
|
|
log "local claude code: ${CLAUDE_CODE_VERSION}"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 2. Update the worktree to that tag
|
|
# ---------------------------------------------------------------------------
|
|
|
|
if [[ ! -d "${WORKTREE}/.git" ]]; then
|
|
log "first run: cloning litellm into ${WORKTREE}"
|
|
mkdir -p "$(dirname "${WORKTREE}")"
|
|
git clone https://github.com/BerriAI/litellm.git "${WORKTREE}"
|
|
fi
|
|
|
|
log "updating worktree to ${LITELLM_VERSION}"
|
|
git -C "${WORKTREE}" fetch --tags --force
|
|
git -C "${WORKTREE}" reset --hard
|
|
# Keep the venv, the .uv-bin cache, and the .uv-python managed
|
|
# interpreter around — uv sync will reconcile the venv on every run,
|
|
# and we don't want to re-download the pinned uv binary or the managed
|
|
# CPython each time. Drop everything else (including any prior
|
|
# tests/e2e/ shim) so each run starts clean before the shim below
|
|
# rewrites it from the dev checkout.
|
|
git -C "${WORKTREE}" clean -fdx -e .venv -e .uv-bin -e .uv-python
|
|
git -C "${WORKTREE}" checkout --force "${LITELLM_VERSION}"
|
|
|
|
# Always rebuild tests/e2e/ in the worktree from the dev checkout,
|
|
# regardless of what the resolved ${LITELLM_VERSION} tag ships. Two
|
|
# reasons:
|
|
#
|
|
# * The matrix populator's job is to exercise *today's* tests against
|
|
# the latest stable proxy. The dev checkout carries the most recent
|
|
# test fixes that haven't yet rolled into a stable release, and we
|
|
# want every cron run to pick those up the moment they land on
|
|
# ${LITELLM_REPO}, not whenever the next stable release happens.
|
|
# * The tag's own tests/e2e/ ships the full EKS e2e harness, whose
|
|
# top-level conftest.py imports modules (e2e_db, lifecycle,
|
|
# otel_client, ...) that the stable venv does not install. Copying
|
|
# the whole tree would make pytest collection blow up on those
|
|
# imports.
|
|
#
|
|
# So the shim is a fresh `rm -rf` of tests/e2e/ followed by copying ONLY
|
|
# the claude_code suite plus the shared transport helpers it imports.
|
|
# pytest puts tests/e2e/ itself on sys.path (it has no __init__.py, while
|
|
# claude_code/ does), which is what resolves both the `claude_code.*`
|
|
# and the bare `proxy_client` / `e2e_http` imports inside the suite.
|
|
E2E_HELPER_FILES=(proxy_client.py e2e_http.py models.py e2e_config.py transport.py)
|
|
if [[ ! -d "${LITELLM_REPO}/tests/e2e/claude_code" ]]; then
|
|
die "no shim source at ${LITELLM_REPO}/tests/e2e/claude_code"
|
|
fi
|
|
for helper in "${E2E_HELPER_FILES[@]}"; do
|
|
[[ -f "${LITELLM_REPO}/tests/e2e/${helper}" ]] \
|
|
|| die "missing shim helper: ${LITELLM_REPO}/tests/e2e/${helper}"
|
|
done
|
|
log "shimming tests/e2e/claude_code/ + helpers from ${LITELLM_REPO} (always-overwrite)"
|
|
rm -rf "${WORKTREE}/tests/e2e"
|
|
mkdir -p "${WORKTREE}/tests/e2e"
|
|
cp -r "${LITELLM_REPO}/tests/e2e/claude_code" "${WORKTREE}/tests/e2e/"
|
|
for helper in "${E2E_HELPER_FILES[@]}"; do
|
|
cp "${LITELLM_REPO}/tests/e2e/${helper}" "${WORKTREE}/tests/e2e/"
|
|
done
|
|
|
|
# litellm pins an exact uv version in pyproject.toml's [tool.uv]
|
|
# `required-version` field, so a system uv that's newer or older
|
|
# refuses to sync. We pin our own local copy at the version the
|
|
# checked-out tag asks for, cached under .uv-bin/ inside the worktree
|
|
# so subsequent runs skip the download.
|
|
PINNED_UV_VERSION="$(
|
|
awk -F'"' '
|
|
/^required-version[[:space:]]*=/ {
|
|
# Field 2 is the value between the quotes, e.g. ">=0.10.9" or
|
|
# "0.10.9". Strip any leading specifier prefix so we end up with
|
|
# the bare version string, which is what /releases/download/<v>/
|
|
# expects.
|
|
v = $2
|
|
sub(/^[[:space:]=<>!~]+/, "", v)
|
|
if (v != "") { print v; exit }
|
|
}
|
|
' "${WORKTREE}/pyproject.toml"
|
|
)"
|
|
if [[ -z "${PINNED_UV_VERSION}" ]]; then
|
|
log "no uv version pin in pyproject.toml; using system uv"
|
|
WORKTREE_UV="$(command -v uv)"
|
|
else
|
|
WORKTREE_UV="${WORKTREE}/.uv-bin/uv-${PINNED_UV_VERSION}"
|
|
if [[ ! -x "${WORKTREE_UV}" ]]; then
|
|
log "downloading uv ${PINNED_UV_VERSION} for the worktree"
|
|
mkdir -p "${WORKTREE}/.uv-bin"
|
|
UV_TARBALL_NAME="uv-x86_64-unknown-linux-gnu.tar.gz"
|
|
UV_DOWNLOAD_URL="https://github.com/astral-sh/uv/releases/download/${PINNED_UV_VERSION}/${UV_TARBALL_NAME}"
|
|
UV_TMPDIR="$(mktemp -d -t uv-download.XXXXXX)"
|
|
# Download the tarball and Astral's official .sha256 sidecar to disk
|
|
# and verify the digest before extracting/executing anything. This
|
|
# closes the supply-chain trust gap of piping a remote binary
|
|
# straight into `tar -xzO ... > file ; chmod +x` (see CLAUDE.md
|
|
# "CI Supply-Chain Safety").
|
|
curl -fsSL --output "${UV_TMPDIR}/${UV_TARBALL_NAME}" "${UV_DOWNLOAD_URL}"
|
|
curl -fsSL --output "${UV_TMPDIR}/${UV_TARBALL_NAME}.sha256" "${UV_DOWNLOAD_URL}.sha256"
|
|
(cd "${UV_TMPDIR}" && sha256sum -c "${UV_TARBALL_NAME}.sha256") \
|
|
|| { rm -rf "${UV_TMPDIR}"; die "uv ${PINNED_UV_VERSION} sha256 mismatch — refusing to install"; }
|
|
tar -xzf "${UV_TMPDIR}/${UV_TARBALL_NAME}" -C "${UV_TMPDIR}" "uv-x86_64-unknown-linux-gnu/uv"
|
|
mv "${UV_TMPDIR}/uv-x86_64-unknown-linux-gnu/uv" "${WORKTREE_UV}.tmp"
|
|
chmod +x "${WORKTREE_UV}.tmp"
|
|
mv "${WORKTREE_UV}.tmp" "${WORKTREE_UV}"
|
|
rm -rf "${UV_TMPDIR}"
|
|
fi
|
|
fi
|
|
# `--extra proxy` pulls fastapi/uvicorn/etc. so `uv run litellm` can
|
|
# actually serve. `--group proxy-dev` brings in pytest and the rest of
|
|
# what tests/e2e/claude_code/ needs. `--python` pins the venv to
|
|
# ${CRON_PYTHON_VERSION}; the first run after a version bump recreates
|
|
# the venv from scratch (a one-time cold sync).
|
|
export UV_PYTHON_INSTALL_DIR="${WORKTREE}/.uv-python"
|
|
log "uv sync --frozen --group proxy-dev --extra proxy --python ${CRON_PYTHON_VERSION} (uv ${PINNED_UV_VERSION:-system})"
|
|
(cd "${WORKTREE}" && "${WORKTREE_UV}" sync --frozen --group proxy-dev --extra proxy --python "${CRON_PYTHON_VERSION}")
|
|
|
|
PROXY_CONFIG="${WORKTREE}/tests/e2e/claude_code/test_config.yaml"
|
|
[[ -f "${PROXY_CONFIG}" ]] || die "proxy config not found at ${PROXY_CONFIG} (shim incomplete?)"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 3. Boot the proxy
|
|
# ---------------------------------------------------------------------------
|
|
|
|
log "starting proxy on 127.0.0.1:${PROXY_PORT}"
|
|
# Bind the proxy to loopback only. The populator proxy is talked to
|
|
# exclusively by the pytest run on the same host (the health check and
|
|
# the test env set `LITELLM_PROXY_URL=http://127.0.0.1:...`),
|
|
# so there's no reason to expose it on the VM's external interfaces.
|
|
# Without `--host`, `litellm` defaults to 0.0.0.0, which combined with
|
|
# the predictable default `LITELLM_MASTER_KEY=sk-cron-matrix` would
|
|
# allow anything that can reach :${PROXY_PORT} on the VM to authenticate
|
|
# and burn upstream provider credentials.
|
|
#
|
|
# `setsid` puts the proxy in its own session+pgroup so cleanup() can
|
|
# SIGTERM the whole tree by passing the pgid as a negative pid. We
|
|
# write that pid to a file so cleanup() doesn't need to remember a
|
|
# variable that might be stale by the time the trap fires.
|
|
setsid env LITELLM_MASTER_KEY="${PROXY_API_KEY}" bash -c '
|
|
echo "$$" > "$0"
|
|
cd "$1"
|
|
exec "$2" run litellm --config "$3" --host 127.0.0.1 --port "$4"
|
|
' "${PROXY_PID_FILE}" "${WORKTREE}" "${WORKTREE_UV}" "${PROXY_CONFIG}" "${PROXY_PORT}" \
|
|
>"${WORKDIR}/proxy.log" 2>&1 &
|
|
disown
|
|
|
|
HEALTH_URL="http://127.0.0.1:${PROXY_PORT}/health/liveliness"
|
|
for _ in $(seq 1 45); do
|
|
if curl -fsS "${HEALTH_URL}" >/dev/null 2>&1; then
|
|
break
|
|
fi
|
|
sleep 2
|
|
done
|
|
curl -fsS "${HEALTH_URL}" >/dev/null \
|
|
|| { tail -50 "${WORKDIR}/proxy.log" >&2; die "proxy did not become healthy"; }
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 4. Run pytest
|
|
# ---------------------------------------------------------------------------
|
|
|
|
RESULTS_JSON="${WORKDIR}/compat-results.json"
|
|
# The `_*_unit_tests` ignore is defensive: those harness-only trees are
|
|
# markerless (they run without a proxy) and don't feed matrix cells, so
|
|
# the cron skips them if/when they land in the suite.
|
|
PYTEST_ARGS=(
|
|
tests/e2e/claude_code/
|
|
"--ignore-glob=*_unit_tests*"
|
|
)
|
|
if [[ -n "${PYTEST_K}" ]]; then
|
|
log "PYTEST_K set; narrowing to: ${PYTEST_K}"
|
|
PYTEST_ARGS+=(-k "${PYTEST_K}")
|
|
fi
|
|
|
|
log "running pytest"
|
|
set +e
|
|
(
|
|
cd "${WORKTREE}" \
|
|
&& LITELLM_PROXY_URL="http://127.0.0.1:${PROXY_PORT}" \
|
|
LITELLM_MASTER_KEY="${PROXY_API_KEY}" \
|
|
COMPAT_RESULTS_PATH="${RESULTS_JSON}" \
|
|
"${WORKTREE_UV}" run pytest "${PYTEST_ARGS[@]}"
|
|
)
|
|
PYTEST_EXIT=$?
|
|
set -e
|
|
log "pytest exit code: ${PYTEST_EXIT} (failures become 'fail' cells, not script errors)"
|
|
# 0=green, 1=test failures (fail cells); >=2 = interrupted/internal/usage/no
|
|
# tests, i.e. a partial run whose missing cells would publish as not_tested.
|
|
[[ ${PYTEST_EXIT} -le 1 ]] \
|
|
|| die "pytest exited abnormally (${PYTEST_EXIT}); refusing to publish a partial matrix"
|
|
[[ -f "${RESULTS_JSON}" ]] || die "pytest did not produce ${RESULTS_JSON}"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 5. Build the matrix JSON
|
|
# ---------------------------------------------------------------------------
|
|
|
|
MATRIX_JSON="${WORKDIR}/compatibility-matrix.json"
|
|
log "building ${MATRIX_JSON}"
|
|
(
|
|
cd "${WORKTREE}" \
|
|
&& "${WORKTREE_UV}" run python "${POPULATOR_DIR}/build_matrix.py" \
|
|
--manifest "${WORKTREE}/tests/e2e/claude_code/manifest.yaml" \
|
|
--results "${RESULTS_JSON}" \
|
|
--output "${MATRIX_JSON}" \
|
|
--litellm-version "${LITELLM_VERSION}" \
|
|
--claude-code-version "${CLAUDE_CODE_VERSION}"
|
|
)
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 6. Open a docs-repo PR
|
|
# ---------------------------------------------------------------------------
|
|
|
|
if [[ "${SKIP_PUBLISH}" == "1" ]]; then
|
|
cp "${MATRIX_JSON}" "${LITELLM_REPO}/compatibility-matrix.json"
|
|
log "SKIP_PUBLISH=1; matrix written to ${LITELLM_REPO}/compatibility-matrix.json"
|
|
exit 0
|
|
fi
|
|
|
|
DATE_UTC="$(date -u +%Y-%m-%d)"
|
|
BRANCH_NAME="compat-matrix/${LITELLM_VERSION}-${CLAUDE_CODE_VERSION}-${DATE_UTC}"
|
|
DOCS_CLONE="${WORKDIR}/litellm-docs"
|
|
|
|
log "cloning ${DOCS_REPO}@${DOCS_BRANCH}"
|
|
gh repo clone "${DOCS_REPO}" "${DOCS_CLONE}" -- --depth 1 --branch "${DOCS_BRANCH}"
|
|
|
|
cd "${DOCS_CLONE}"
|
|
git config user.email "litellm-bot@berri.ai"
|
|
git config user.name "litellm-compat-matrix-bot"
|
|
git checkout -b "${BRANCH_NAME}"
|
|
|
|
# Snapshot the currently-published matrix *before* we overwrite it, so the
|
|
# auto-merge gate below can diff old→new cell statuses. On the first-ever
|
|
# publish the file won't exist yet; we leave ${PUBLISHED_MATRIX} pointing
|
|
# at a path that doesn't exist and let check_regressions.py treat that as
|
|
# "no baseline → no regressions".
|
|
PUBLISHED_MATRIX="${WORKDIR}/published-matrix.json"
|
|
if [[ -f "${DOCS_TARGET_PATH}" ]]; then
|
|
cp "${DOCS_TARGET_PATH}" "${PUBLISHED_MATRIX}"
|
|
fi
|
|
|
|
mkdir -p "$(dirname "${DOCS_TARGET_PATH}")"
|
|
cp "${MATRIX_JSON}" "${DOCS_TARGET_PATH}"
|
|
git add "${DOCS_TARGET_PATH}"
|
|
|
|
if git diff --cached --quiet; then
|
|
log "matrix JSON unchanged from ${DOCS_BRANCH}; skipping PR"
|
|
exit 0
|
|
fi
|
|
|
|
# --- Auto-merge regression gate --------------------------------------------
|
|
# Only auto-merge when the new matrix is improvement-or-equal: every cell
|
|
# transition is red→green, green→green, or red→red. If any cell flips
|
|
# green→red (a `pass` that became `fail`), we still open/refresh the PR but
|
|
# leave auto-merge OFF so a human reviews the regression before it lands on
|
|
# the public docs table. A pre-existing red cell (e.g. Anthropic out of API
|
|
# credits) is red→red and does NOT block, so the daily PR keeps flowing.
|
|
log "checking for green->red regressions vs the published matrix"
|
|
set +e
|
|
REGRESSION_REPORT="$(
|
|
cd "${WORKTREE}" \
|
|
&& "${WORKTREE_UV}" run python "${POPULATOR_DIR}/check_regressions.py" \
|
|
--old "${PUBLISHED_MATRIX}" \
|
|
--new "${MATRIX_JSON}"
|
|
)"
|
|
REGRESSION_EXIT=$?
|
|
set -e
|
|
printf '%s\n' "${REGRESSION_REPORT}" | sed 's/^/ /' >&2
|
|
# Exit 0 = clean. Exit 3 = green→red regression(s) found. Any other code
|
|
# means the checker itself errored; fail *closed* (withhold auto-merge) so a
|
|
# bug in the gate can never silently auto-merge a regression.
|
|
if [[ ${REGRESSION_EXIT} -eq 0 ]]; then
|
|
ALLOW_AUTOMERGE=1
|
|
elif [[ ${REGRESSION_EXIT} -eq 3 ]]; then
|
|
ALLOW_AUTOMERGE=0
|
|
log "WARN: green->red regression(s) detected; auto-merge will be left OFF for review"
|
|
else
|
|
ALLOW_AUTOMERGE=0
|
|
log "WARN: regression check errored (exit ${REGRESSION_EXIT}); withholding auto-merge to be safe"
|
|
fi
|
|
|
|
GENERATED_AT="$(jq -r '.generated_at' "${MATRIX_JSON}")"
|
|
COMMIT_MSG="$(cat <<EOF
|
|
Update Claude Code compatibility matrix
|
|
|
|
litellm_version: ${LITELLM_VERSION}
|
|
claude_code_version: ${CLAUDE_CODE_VERSION}
|
|
generated_at: ${GENERATED_AT}
|
|
EOF
|
|
)"
|
|
git commit -m "${COMMIT_MSG}"
|
|
|
|
# Push the branch straight to BerriAI/litellm-docs. mateo-berri has write
|
|
# access on the docs repo, so there's no fork hop: the PR is a same-repo
|
|
# branch PR. The temp remote carries the token in its URL, so we add it,
|
|
# push, then immediately remove it so the token never lingers in
|
|
# ${DOCS_CLONE}/.git/config. (${DOCS_CLONE} is also rm -rf'd by the
|
|
# cleanup trap on exit.)
|
|
#
|
|
# Plain --force (not --force-with-lease) is acceptable here: the
|
|
# compat-matrix/* branch is bot-owned, only this script ever writes to
|
|
# it, and runs are serialized by the systemd timer. --force-with-lease
|
|
# would require a fetch to populate the remote-tracking ref before each
|
|
# push and adds no safety in this single-writer setup.
|
|
PUBLISH_PUSH_URL="https://x-access-token:${GITHUB_TOKEN}@github.com/${DOCS_REPO}.git"
|
|
git remote remove publish 2>/dev/null || true
|
|
git remote add publish "${PUBLISH_PUSH_URL}"
|
|
git push --force --set-upstream publish "${BRANCH_NAME}"
|
|
git remote remove publish
|
|
unset PUBLISH_PUSH_URL
|
|
|
|
# Per-feature status table for the PR body. Reviewers triage from this.
|
|
PR_FEATURE_TABLE="$(jq -r '
|
|
.features[] as $f
|
|
| "- **\($f.name)**: " +
|
|
([ .providers[] as $p
|
|
| "\($p)=\($f.providers[$p].status // "not_tested")"
|
|
] | join(", "))
|
|
' "${MATRIX_JSON}")"
|
|
|
|
# When the gate withheld auto-merge, call it out at the top of the PR body
|
|
# (with the offending cells) so a reviewer knows this PR needs a human and
|
|
# why. On the clean path this section is empty. Note `$(...)` strips the
|
|
# trailing newline, so the body below puts explicit blank lines *around*
|
|
# the placeholder rather than relying on the heredoc's own spacing.
|
|
if [[ "${ALLOW_AUTOMERGE}" != "1" ]]; then
|
|
PR_REGRESSION_SECTION="$(cat <<EOF
|
|
> [!WARNING]
|
|
> **Auto-merge disabled:** one or more cells regressed green→red versus the
|
|
> currently-published matrix. Review the diff before merging.
|
|
|
|
\`\`\`
|
|
${REGRESSION_REPORT}
|
|
\`\`\`
|
|
EOF
|
|
)"
|
|
else
|
|
PR_REGRESSION_SECTION=""
|
|
fi
|
|
|
|
PR_TITLE="chore(compat-matrix): refresh for ${LITELLM_VERSION} + claude-code ${CLAUDE_CODE_VERSION}"
|
|
PR_BODY="$(cat <<EOF
|
|
Automated daily refresh of the Claude Code compatibility matrix.
|
|
|
|
${PR_REGRESSION_SECTION}
|
|
|
|
| Field | Value |
|
|
| --- | --- |
|
|
| litellm_version | \`${LITELLM_VERSION}\` |
|
|
| claude_code_version | \`${CLAUDE_CODE_VERSION}\` |
|
|
| generated_at | \`${GENERATED_AT}\` |
|
|
|
|
## Per-feature results
|
|
|
|
${PR_FEATURE_TABLE}
|
|
|
|
---
|
|
|
|
Generated by \`tests/e2e/claude_code/cron_vm/run_daily.sh\`. Close without merging if the diff looks wrong; the next cron run will reopen with fresh results.
|
|
EOF
|
|
)"
|
|
|
|
log "opening PR from ${BRANCH_NAME} -> ${DOCS_REPO}:${DOCS_BRANCH} (as mateo-berri)"
|
|
# GH_TOKEN is mateo-berri's write-scoped token, the same identity used
|
|
# for release-listing above. The branch lives on ${DOCS_REPO} itself, so
|
|
# --head is a bare branch name (a same-repo PR), not `OWNER:BRANCH`.
|
|
set +e
|
|
PR_OUT="$(
|
|
GH_TOKEN="${GITHUB_TOKEN}" gh pr create \
|
|
--repo "${DOCS_REPO}" \
|
|
--base "${DOCS_BRANCH}" \
|
|
--head "${BRANCH_NAME}" \
|
|
--title "${PR_TITLE}" \
|
|
--body "${PR_BODY}" 2>&1
|
|
)"
|
|
PR_EXIT=$?
|
|
set -e
|
|
echo "${PR_OUT}"
|
|
|
|
if [[ ${PR_EXIT} -ne 0 ]]; then
|
|
if grep -q "a pull request for branch.*already exists" <<<"${PR_OUT}"; then
|
|
log "PR already exists for ${BRANCH_NAME}; updated branch in place"
|
|
else
|
|
die "gh pr create failed (exit ${PR_EXIT})"
|
|
fi
|
|
fi
|
|
|
|
# Enable auto-merge so the PR merges itself once the docs repo's required
|
|
# checks pass -- we no longer gate these bot PRs on a second human
|
|
# approval. mateo-berri authors and merges them directly. The repo only
|
|
# permits squash merges and has auto-merge enabled at the repo level
|
|
# (${AUTO_MERGE_METHOD} defaults to squash accordingly).
|
|
#
|
|
# This only fires when the regression gate above is satisfied
|
|
# (${ALLOW_AUTOMERGE}==1): a green→red regression — or a gate error —
|
|
# leaves auto-merge OFF so a human triages the PR.
|
|
#
|
|
# `gh pr merge --auto` is idempotent: re-enabling auto-merge on a PR that
|
|
# already has it set is a no-op, so same-day reruns stay clean. It's
|
|
# non-fatal: if auto-merge can't be enabled (e.g. the PR is already in a
|
|
# clean/mergeable state with nothing left to wait on, or branch
|
|
# protection isn't configured), the matrix JSON has still landed on the
|
|
# PR and the worst case is a manual merge click.
|
|
if [[ "${ALLOW_AUTOMERGE}" == "1" ]]; then
|
|
log "enabling ${AUTO_MERGE_METHOD} auto-merge on ${BRANCH_NAME}"
|
|
set +e
|
|
GH_TOKEN="${GITHUB_TOKEN}" gh pr merge \
|
|
"${BRANCH_NAME}" \
|
|
--repo "${DOCS_REPO}" \
|
|
--auto \
|
|
"--${AUTO_MERGE_METHOD}" 2>&1 | sed 's/^/ /'
|
|
AUTOMERGE_EXIT=${PIPESTATUS[0]}
|
|
set -e
|
|
if [[ ${AUTOMERGE_EXIT} -ne 0 ]]; then
|
|
log "WARN: gh pr merge --auto exited ${AUTOMERGE_EXIT} (non-fatal)"
|
|
fi
|
|
else
|
|
# Regression (or gate error): make sure auto-merge is OFF. A same-day
|
|
# rerun may have enabled it on an earlier, clean pass, so explicitly
|
|
# disable rather than just skipping. The disable call itself is allowed
|
|
# to error (`--disable-auto` fails harmlessly when auto-merge was never
|
|
# enabled), but the read-back below is authoritative: a regressed matrix
|
|
# must never be left armed to merge, so a still-armed PR is fatal.
|
|
log "leaving ${BRANCH_NAME} for manual review; disabling any prior auto-merge"
|
|
set +e
|
|
GH_TOKEN="${GITHUB_TOKEN}" gh pr merge \
|
|
"${BRANCH_NAME}" \
|
|
--repo "${DOCS_REPO}" \
|
|
--disable-auto 2>&1 | sed 's/^/ /'
|
|
set -e
|
|
AUTOMERGE_ARMED="$(
|
|
GH_TOKEN="${GITHUB_TOKEN}" gh pr view \
|
|
"${BRANCH_NAME}" \
|
|
--repo "${DOCS_REPO}" \
|
|
--json autoMergeRequest \
|
|
--jq '.autoMergeRequest.enabledAt // empty'
|
|
)" || die "could not read back the auto-merge state on ${BRANCH_NAME}"
|
|
[[ -z "${AUTOMERGE_ARMED}" ]] \
|
|
|| die "auto-merge still armed on ${BRANCH_NAME} (enabled ${AUTOMERGE_ARMED}) after --disable-auto"
|
|
fi
|
|
|
|
# --- Stale-PR sweep ----------------------------------------------------------
|
|
# Keep at most ONE compat-matrix PR open: today's. Any other open
|
|
# `compat-matrix/*` PR is a leftover from a day whose regression gate
|
|
# withheld auto-merge and nobody triaged it; the PR we just opened or
|
|
# refreshed above carries strictly fresher results, so the old one is
|
|
# pure queue noise. Closing is non-destructive — the PR record and its
|
|
# regression report stay browsable; only the bot-owned branch is
|
|
# deleted. This runs only after today's PR exists (a `die` above skips
|
|
# it), so a failed publish can never close the queue down to zero.
|
|
#
|
|
# Non-fatal: a sweep failure (rate limit, transient API error) leaves
|
|
# stale PRs for the next run to retry; it must not fail the pipeline.
|
|
log "sweeping stale compat-matrix PRs (keeping ${BRANCH_NAME})"
|
|
set +e
|
|
STALE_PRS="$(
|
|
GH_TOKEN="${GITHUB_TOKEN}" gh pr list \
|
|
--repo "${DOCS_REPO}" \
|
|
--state open \
|
|
--limit 100 \
|
|
--json number,headRefName \
|
|
--jq '.[] | select(.headRefName | startswith("compat-matrix/")) | "\(.number)\t\(.headRefName)"'
|
|
)"
|
|
while IFS=$'\t' read -r stale_pr stale_head; do
|
|
[[ -z "${stale_pr}" ]] && continue
|
|
[[ "${stale_head}" == "${BRANCH_NAME}" ]] && continue
|
|
GH_TOKEN="${GITHUB_TOKEN}" gh pr close "${stale_pr}" \
|
|
--repo "${DOCS_REPO}" \
|
|
--delete-branch \
|
|
--comment "Superseded by the newer daily compat-matrix PR from \`${BRANCH_NAME}\`; the populator keeps only the most recent compat-matrix PR open." 2>&1 | sed 's/^/ /'
|
|
if [[ ${PIPESTATUS[0]} -eq 0 ]]; then
|
|
log "closed stale compat-matrix PR #${stale_pr} (${stale_head})"
|
|
else
|
|
log "WARN: could not close stale compat-matrix PR #${stale_pr} (non-fatal)"
|
|
fi
|
|
done <<<"${STALE_PRS}"
|
|
set -e
|
|
|
|
log "done"
|