mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-03 07:23:33 +00:00
281bf4e48b
* ci(circleci): clone the public repo over HTTPS so main jobs stop failing at checkout The project's SSH checkout key stopped authenticating on Oct 27, so every job's built-in checkout step failed with Permission denied (publickey). The repo is public, so each job now fetches its pipeline's commit anonymously over HTTPS, the way the migration jobs already did * fix(circleci): let the PR path filter read history after the shallow clone and test the clone step itself * fix(circleci): clone full history without file contents, retry the fetch, and record the integration schedule The depth-1 clone left the PR path filter without a merge base, and the unshallow that fixed it pulled about 1.4 GB per filtered job. A blobless fetch gives every job the history the built-in checkout step gave, so the path filter change goes back to main's version. The fetch now gets three tries, the tests run the step under the Windows job's bare bash too, and .circleci/AGENTS.md records the integration schedule's cron * fix(circleci): stop on the fetch error once the three tries run out, and name the GitHub App fallback as the cause * fix(circleci): retry the checkout's file download with the fetch, since a dropped download failed the job outside the retry * fix(circleci): fetch the routing parity base with its file contents, since the blobless clone downloaded each file separately --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
1780 lines
73 KiB
YAML
1780 lines
73 KiB
YAML
version: 2.1
|
|
parameters:
|
|
run_migration_tests:
|
|
type: boolean
|
|
default: false
|
|
migration_candidate_image:
|
|
type: string
|
|
default: ""
|
|
migration_baseline_image:
|
|
type: string
|
|
default: "ghcr.io/berriai/litellm-database:v1.102.0"
|
|
migration_source_sha:
|
|
type: string
|
|
default: ""
|
|
routing_parity_base:
|
|
type: string
|
|
default: ""
|
|
orbs:
|
|
codecov: codecov/codecov@4.0.1
|
|
node: circleci/node@5.1.0 # Add this line to declare the node orb
|
|
win: circleci/windows@5.0 # Add Windows orb
|
|
|
|
commands:
|
|
checkout_over_https:
|
|
parameters:
|
|
revision:
|
|
type: string
|
|
default: ""
|
|
shell:
|
|
type: string
|
|
default: /bin/bash -eo pipefail
|
|
steps:
|
|
- run:
|
|
name: Check out the public repository over HTTPS
|
|
shell: << parameters.shell >>
|
|
environment:
|
|
REQUESTED_REVISION: << parameters.revision >>
|
|
command: |
|
|
revision="${REQUESTED_REVISION:-$CIRCLE_SHA1}"
|
|
[[ "$revision" =~ ^[0-9a-f]{40}$ ]] || exit 1
|
|
git init
|
|
git remote add origin https://github.com/BerriAI/litellm.git
|
|
for attempt in 1 2 3; do
|
|
git fetch --filter=blob:none origin "$revision" && git checkout --detach FETCH_HEAD && break
|
|
[[ $attempt -lt 3 ]] || exit 1
|
|
sleep 5
|
|
done
|
|
skip_if_unrelated_changes:
|
|
parameters:
|
|
category:
|
|
type: enum
|
|
enum: ["backend", "client"]
|
|
default: "backend"
|
|
steps:
|
|
- run:
|
|
name: "Skip job when no << parameters.category >>-relevant files changed"
|
|
command: bash .circleci/scripts/path_filter.sh << parameters.category >>
|
|
setup_google_dns:
|
|
steps:
|
|
- run:
|
|
name: "Configure Google DNS"
|
|
command: |
|
|
# Backup original resolv.conf
|
|
sudo cp /etc/resolv.conf /etc/resolv.conf.backup
|
|
# Add both local and Google DNS servers
|
|
echo "nameserver 127.0.0.11" | sudo tee /etc/resolv.conf
|
|
echo "nameserver 8.8.8.8" | sudo tee -a /etc/resolv.conf
|
|
echo "nameserver 8.8.4.4" | sudo tee -a /etc/resolv.conf
|
|
wait_for_service:
|
|
description: "Poll a TCP or HTTP endpoint until it responds (replaces dockerize -wait)"
|
|
parameters:
|
|
url:
|
|
type: string
|
|
timeout:
|
|
type: string
|
|
default: "60"
|
|
steps:
|
|
- run:
|
|
name: "Wait for << parameters.url >>"
|
|
command: |
|
|
TIMEOUT=<< parameters.timeout >>
|
|
URL="<< parameters.url >>"
|
|
ELAPSED=0
|
|
echo "Waiting up to ${TIMEOUT}s for ${URL} ..."
|
|
if echo "$URL" | grep -q '^tcp://'; then
|
|
HOST=$(echo "$URL" | sed 's|tcp://||' | cut -d: -f1)
|
|
PORT=$(echo "$URL" | sed 's|tcp://||' | cut -d: -f2)
|
|
while ! bash -c "echo > /dev/tcp/$HOST/$PORT" 2>/dev/null; do
|
|
sleep 2; ELAPSED=$((ELAPSED+2))
|
|
if [ "$ELAPSED" -ge "$TIMEOUT" ]; then echo "Timed out"; exit 1; fi
|
|
done
|
|
else
|
|
while ! curl -sf --max-time 5 "$URL" > /dev/null 2>&1; do
|
|
sleep 2; ELAPSED=$((ELAPSED+2))
|
|
if [ "$ELAPSED" -ge "$TIMEOUT" ]; then echo "Timed out"; exit 1; fi
|
|
done
|
|
fi
|
|
echo "Service ready after ${ELAPSED}s"
|
|
install_helm:
|
|
steps:
|
|
- run:
|
|
name: Install Helm v3.17.3
|
|
command: |
|
|
curl -sSLf -o /tmp/helm.tar.gz \
|
|
https://get.helm.sh/helm-v3.17.3-linux-amd64.tar.gz
|
|
echo "ee88b3c851ae6466a3de507f7be73fe94d54cbf2987cbaa3d1a3832ea331f2cd /tmp/helm.tar.gz" | sha256sum -c -
|
|
sudo tar -C /usr/local/bin --strip-components=1 -xzf /tmp/helm.tar.gz linux-amd64/helm
|
|
rm -f /tmp/helm.tar.gz
|
|
install_kind:
|
|
steps:
|
|
- run:
|
|
name: Install Kind v0.20.0
|
|
command: |
|
|
curl -sSLf -o /tmp/kind \
|
|
https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64
|
|
echo "513a7213d6d3332dd9ef27c24dab35e5ef10a04fa27274fe1c14d8a246493ded /tmp/kind" | sha256sum -c -
|
|
chmod +x /tmp/kind
|
|
sudo mv /tmp/kind /usr/local/bin/kind
|
|
install_uv:
|
|
description: "Install pinned uv (0.10.9) with checksum verification. Adds ~/.local/bin to PATH."
|
|
steps:
|
|
- run:
|
|
name: Install uv (pinned 0.10.9)
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
install_node:
|
|
description: "Install the Node.js version pinned in ui/litellm-dashboard/.nvmrc (24.19.0, which bundles npm 11.17.0) with checksum verification, and prepend it to PATH. Run this on any executor whose image does not already ship that version, or `npm ci` in ui/litellm-dashboard fails EBADENGINE against the engines floor. Installs into /opt/node rather than over /usr/local on purpose: cimg/python:*-browsers ships its own node there, and unpacking the tarball on top of it leaves npm 11.17 files merged with the image's npm 11.9 tree, which reports the new version and then exits 1 on `npm ci` with no error text at all. Requires checkout, which the .nvmrc drift check reads."
|
|
steps:
|
|
- run:
|
|
name: Install Node.js 24.19.0
|
|
command: |
|
|
NODE_VERSION="24.19.0"
|
|
NODE_TARBALL="node-v${NODE_VERSION}-linux-x64.tar.xz"
|
|
NODE_EXPECTED_SHA="14b342e71204f811bde6153be8e04b62aef63c236fef92b55f9c83154b409647"
|
|
NVMRC_VERSION="$(tr -d '[:space:]' < ui/litellm-dashboard/.nvmrc)"
|
|
if [ "$NVMRC_VERSION" != "$NODE_VERSION" ]; then
|
|
echo "install_node: ui/litellm-dashboard/.nvmrc pins ${NVMRC_VERSION} but this command pins ${NODE_VERSION}; update NODE_VERSION and NODE_EXPECTED_SHA together" >&2
|
|
exit 1
|
|
fi
|
|
curl -sSLf -o "/tmp/${NODE_TARBALL}" "https://nodejs.org/dist/v${NODE_VERSION}/${NODE_TARBALL}"
|
|
echo "${NODE_EXPECTED_SHA} /tmp/${NODE_TARBALL}" | sha256sum -c -
|
|
sudo mkdir -p /opt/node
|
|
sudo tar -xJf "/tmp/${NODE_TARBALL}" -C /opt/node --strip-components=1
|
|
rm -f "/tmp/${NODE_TARBALL}"
|
|
echo 'export PATH="/opt/node/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="/opt/node/bin:$PATH"
|
|
node --version
|
|
npm --version
|
|
install_rust:
|
|
description: "Install pinned rustup (1.28.2) and Rust toolchain (1.98.0) with checksum verification. Adds ~/.cargo/bin to PATH. Run this before any `uv sync` or `uv build` of the workspace: the root package builds litellm-rust through maturin, and on an image without cargo maturin fetches an unpinned rustup and a floating toolchain by itself. Also restores the dev-profile cargo cache that save_cargo_target writes on main, minus the workspace crates' fingerprints so those always rebuild from the checked-out source."
|
|
steps:
|
|
- run:
|
|
name: Install Rust (rustup 1.28.2, toolchain 1.98.0)
|
|
command: |
|
|
case "$(uname -m)" in
|
|
x86_64)
|
|
RUSTUP_TRIPLE=x86_64-unknown-linux-gnu
|
|
RUSTUP_SHA256=20a06e644b0d9bd2fbdbfd52d42540bdde820ea7df86e92e533c073da0cdd43c
|
|
;;
|
|
aarch64)
|
|
RUSTUP_TRIPLE=aarch64-unknown-linux-gnu
|
|
RUSTUP_SHA256=e3853c5a252fca15252d07cb23a1bdd9377a8c6f3efa01531109281ae47f841c
|
|
;;
|
|
*)
|
|
echo "install_rust: unsupported architecture $(uname -m)" >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
curl -sSLf --retry 3 --retry-all-errors --retry-max-time 120 --connect-timeout 15 --max-time 60 -o /tmp/rustup-init \
|
|
"https://static.rust-lang.org/rustup/archive/1.28.2/${RUSTUP_TRIPLE}/rustup-init"
|
|
echo "${RUSTUP_SHA256} /tmp/rustup-init" | sha256sum -c -
|
|
chmod +x /tmp/rustup-init
|
|
/tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain 1.98.0
|
|
rm -f /tmp/rustup-init
|
|
echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> "$BASH_ENV"
|
|
echo 'export CARGO_INCREMENTAL=0' >> "$BASH_ENV"
|
|
export PATH="$HOME/.cargo/bin:$PATH"
|
|
rustc --version
|
|
cargo --version
|
|
{ rustc -vV; cc --version; cat /etc/os-release; } > /tmp/cargo-build-env
|
|
- restore_cache:
|
|
keys:
|
|
- v1-cargo-dev-{{ checksum "/tmp/cargo-build-env" }}-{{ checksum "litellm-rust/Cargo.lock" }}
|
|
- v1-cargo-dev-{{ checksum "/tmp/cargo-build-env" }}-
|
|
- run:
|
|
name: Force a rebuild of the workspace crates restored from the cargo cache
|
|
command: rm -rf litellm-rust/target/debug/.fingerprint/litellm-*
|
|
save_cargo_target:
|
|
steps:
|
|
- when:
|
|
condition:
|
|
equal: [main, << pipeline.git.branch >>]
|
|
steps:
|
|
- save_cache:
|
|
key: v1-cargo-dev-{{ checksum "/tmp/cargo-build-env" }}-{{ checksum "litellm-rust/Cargo.lock" }}
|
|
paths:
|
|
- ~/.cargo/registry
|
|
- ~/project/litellm-rust/target/debug
|
|
start_postgres:
|
|
description: "Start a postgres-db container on port 5432 and wait until it accepts connections."
|
|
parameters:
|
|
db_name:
|
|
type: string
|
|
default: circle_test
|
|
image:
|
|
type: string
|
|
default: postgres:14@sha256:6a70deda415ec296f977890e11aba04a0db9f632a362e3fce45e845e3db74f26
|
|
server_args:
|
|
type: string
|
|
default: ""
|
|
steps:
|
|
- run:
|
|
name: Start PostgreSQL
|
|
command: |
|
|
docker run -d \
|
|
--name postgres-db \
|
|
-e POSTGRES_USER=postgres \
|
|
-e POSTGRES_PASSWORD=postgres \
|
|
-e POSTGRES_DB=<< parameters.db_name >> \
|
|
-p 5432:5432 \
|
|
<< parameters.image >> << parameters.server_args >>
|
|
- wait_for_service:
|
|
url: tcp://localhost:5432
|
|
timeout: "60"
|
|
start_redis:
|
|
description: "Start a redis container on port 6379 and wait until it accepts connections. Use this to isolate a job from the shared remote Redis so concurrent CI pipelines don't contend for pod locks or buffer keys."
|
|
steps:
|
|
- run:
|
|
name: Start Redis
|
|
command: |
|
|
docker run -d \
|
|
--name redis-cache \
|
|
-p 6379:6379 \
|
|
redis:7-alpine@sha256:7aec734b2bb298a1d769fd8729f13b8514a41bf90fcdd1f38ec52267fbaa8ee6
|
|
- wait_for_service:
|
|
url: tcp://localhost:6379
|
|
timeout: "60"
|
|
install_codecov_cli:
|
|
steps:
|
|
- run:
|
|
name: Install Codecov CLI (pinned v11.3.1)
|
|
when: always
|
|
command: |
|
|
curl -sSLf -o /tmp/codecov https://cli.codecov.io/v11.3.1/linux/codecov
|
|
curl -sSLf -o /tmp/codecov.SHA256SUM https://cli.codecov.io/v11.3.1/linux/codecov.SHA256SUM
|
|
[ "$(cat /tmp/codecov.SHA256SUM)" = "ca1d64196d2d34771084afe76ea657d581bf628e31d993ff8e52ea09cc88a56d codecov" ]
|
|
(cd /tmp && sha256sum -c codecov.SHA256SUM)
|
|
chmod +x /tmp/codecov
|
|
mkdir -p "$HOME/.local/bin"
|
|
mv /tmp/codecov "$HOME/.local/bin/codecov"
|
|
start_openai_record_replay_proxy:
|
|
description: "Start the record/replay proxy (tests/_openai_record_replay_proxy.py) on host port 8090 and wait until healthy. Models whose api_base points here replay Redis cache hits. Cache misses call the live provider and store its response, so these legacy E2E jobs still require provider availability and may incur provider charges. The default upstream is OpenAI; a non-OpenAI model must point its api_base at /__recorder_upstream/<host>/ so the recorder forwards there instead of defaulting to OpenAI. Run after uv deps are synced."
|
|
steps:
|
|
- run:
|
|
name: Start record/replay proxy
|
|
background: true
|
|
command: |
|
|
CASSETTE_REDIS_URL="$CASSETTE_REDIS_URL" \
|
|
RECORDER_UPSTREAM_BASE_URL="https://api.openai.com" \
|
|
uv run --no-sync python tests/_openai_record_replay_proxy.py --host 0.0.0.0 --port 8090
|
|
- run:
|
|
name: Wait for record/replay proxy
|
|
command: |
|
|
for i in $(seq 1 30); do
|
|
if curl -sf http://localhost:8090/__recorder_health >/dev/null 2>&1; then
|
|
echo "record/replay proxy is up"
|
|
exit 0
|
|
fi
|
|
sleep 1
|
|
done
|
|
echo "record/replay proxy did not become ready" >&2
|
|
exit 1
|
|
start_fake_openai_endpoint:
|
|
description: "Start the canned OpenAI mock (tests/_fake_openai_endpoint_server.py) on host port 8190 and wait until healthy. Models whose api_base points here (via FAKE_OPENAI_API_BASE) get well-formed chat/text/embedding responses with realistic usage, so the E2E run neither pays for nor depends on the live provider. A request whose model is '429' returns HTTP 429 for rate-limit/cooldown tests. Run after uv deps are synced."
|
|
steps:
|
|
- run:
|
|
name: Start fake OpenAI endpoint
|
|
background: true
|
|
command: |
|
|
uv run --no-sync python tests/_fake_openai_endpoint_server.py --host 0.0.0.0 --port 8190
|
|
- run:
|
|
name: Wait for fake OpenAI endpoint
|
|
command: |
|
|
for i in $(seq 1 30); do
|
|
if curl -sf http://localhost:8190/health >/dev/null 2>&1; then
|
|
echo "fake OpenAI endpoint is up"
|
|
exit 0
|
|
fi
|
|
sleep 1
|
|
done
|
|
echo "fake OpenAI endpoint did not become ready" >&2
|
|
exit 1
|
|
setup_litellm_enterprise_pip:
|
|
steps:
|
|
- run:
|
|
name: "Install local version of litellm-enterprise"
|
|
command: |
|
|
# litellm-enterprise is a uv workspace member and is already installed
|
|
# by the main `uv sync --all-groups --all-extras`. Do NOT run
|
|
# `uv sync --package litellm-enterprise` here — that overwrites the
|
|
# shared .venv and strips out dev/test deps (pytest, prisma, etc.).
|
|
uv run --no-sync python -c "import litellm_enterprise; print('litellm-enterprise OK:', litellm_enterprise.__file__)"
|
|
install_windows_toolchain:
|
|
steps:
|
|
- run:
|
|
name: Install Rust and uv
|
|
no_output_timeout: 30m
|
|
command: |
|
|
$rustupInit = Join-Path $env:TEMP "rustup-init.exe"
|
|
$rustupVersion = "1.28.2"
|
|
$rustupUrl = "https://static.rust-lang.org/rustup/archive/$rustupVersion/x86_64-pc-windows-msvc/rustup-init.exe"
|
|
Invoke-WebRequest -Uri $rustupUrl -OutFile $rustupInit
|
|
$rustupExpected = "88d8258dcf6ae4f7a80c7d1088e1f36fa7025a1cfd1343731b4ee6f385121fc0"
|
|
$rustupActual = (Get-FileHash -Path $rustupInit -Algorithm SHA256).Hash.ToLower()
|
|
if ($rustupActual -ne $rustupExpected) {
|
|
throw "rustup installer hash mismatch: expected $rustupExpected got $rustupActual"
|
|
}
|
|
& $rustupInit -y --profile minimal --default-toolchain 1.98.0
|
|
if ($LASTEXITCODE -ne 0) {
|
|
exit $LASTEXITCODE
|
|
}
|
|
Remove-Item $rustupInit
|
|
$cargoBin = Join-Path $HOME ".cargo\bin"
|
|
$env:Path = "$cargoBin;$env:Path"
|
|
rustc --version
|
|
cargo --version
|
|
$installer = Join-Path $env:TEMP "uv-install.ps1"
|
|
Invoke-WebRequest -Uri https://astral.sh/uv/0.10.9/install.ps1 -OutFile $installer
|
|
$expected = "d43ffff8d28e7d1e7d1831a212465f12b24a43c7f87f386e95e2a5915aee5d7d"
|
|
$actual = (Get-FileHash -Path $installer -Algorithm SHA256).Hash.ToLower()
|
|
if ($actual -ne $expected) {
|
|
throw "uv installer hash mismatch: expected $expected got $actual"
|
|
}
|
|
& $installer
|
|
Remove-Item $installer
|
|
$uvBin = Join-Path $HOME ".local\bin"
|
|
$env:Path = "$uvBin;$env:Path"
|
|
if (!(Test-Path $PROFILE)) {
|
|
New-Item -ItemType File -Force -Path $PROFILE | Out-Null
|
|
}
|
|
if (-not (Select-String -Path $PROFILE -SimpleMatch $uvBin -Quiet)) {
|
|
Add-Content -Path $PROFILE -Value "`$env:Path = `"$uvBin;`$env:Path`""
|
|
}
|
|
if (-not (Select-String -Path $PROFILE -SimpleMatch $cargoBin -Quiet)) {
|
|
Add-Content -Path $PROFILE -Value "`$env:Path = `"$cargoBin;`$env:Path`""
|
|
}
|
|
setup_litellm_test_deps:
|
|
parameters:
|
|
checkout:
|
|
type: boolean
|
|
default: true
|
|
steps:
|
|
- when:
|
|
condition: << parameters.checkout >>
|
|
steps:
|
|
- checkout_over_https
|
|
- setup_google_dns
|
|
- install_uv
|
|
- install_rust
|
|
- restore_cache:
|
|
keys:
|
|
- v3-integration-uv-cache-{{ checksum "uv.lock" }}
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- setup_litellm_enterprise_pip
|
|
- save_cache:
|
|
paths:
|
|
- ~/.cache/uv
|
|
key: v3-integration-uv-cache-{{ checksum "uv.lock" }}
|
|
- save_cargo_target
|
|
|
|
jobs:
|
|
windows_release_wheel:
|
|
executor:
|
|
name: win/default
|
|
shell: powershell.exe
|
|
size: xlarge
|
|
working_directory: ~/project
|
|
environment:
|
|
UV_PYTHON: "3.11"
|
|
CARGO_HTTP_MULTIPLEXING: "false"
|
|
CARGO_NET_RETRY: "5"
|
|
steps:
|
|
- checkout_over_https:
|
|
shell: bash.exe
|
|
- run:
|
|
name: Skip job when no windows-release-relevant files changed
|
|
shell: bash.exe
|
|
command: bash .circleci/scripts/path_filter.sh windows-release
|
|
- run:
|
|
name: Install Python 3.11.9 from python.org
|
|
no_output_timeout: 6m
|
|
command: |
|
|
$ProgressPreference = "SilentlyContinue"
|
|
$installer = Join-Path $env:TEMP "python-3.11.9-amd64.exe"
|
|
Invoke-WebRequest -Uri https://www.python.org/ftp/python/3.11.9/python-3.11.9-amd64.exe -OutFile $installer -TimeoutSec 120
|
|
$expected = "5ee42c4eee1e6b4464bb23722f90b45303f79442df63083f05322f1785f5fdde"
|
|
$actual = (Get-FileHash -Path $installer -Algorithm SHA256).Hash.ToLower()
|
|
if ($actual -ne $expected) {
|
|
throw "Python installer hash mismatch: expected $expected got $actual"
|
|
}
|
|
$arguments = "/quiet InstallAllUsers=1 PrependPath=1 Include_test=0 Include_doc=0 Include_launcher=0 TargetDir=C:\Python311"
|
|
$process = Start-Process -FilePath $installer -ArgumentList $arguments -PassThru
|
|
if (-not $process.WaitForExit(300000)) {
|
|
$process.Kill()
|
|
throw "Python installer did not finish within 5 minutes"
|
|
}
|
|
if ($process.ExitCode -ne 0) {
|
|
throw "Python installer exited with $($process.ExitCode)"
|
|
}
|
|
Remove-Item $installer
|
|
& C:\Python311\python.exe --version
|
|
- install_windows_toolchain
|
|
- run:
|
|
name: Record the Rust build environment for the release cargo cache key
|
|
command: |
|
|
& "$HOME\.cargo\bin\rustc.exe" -vV | Out-File -Encoding ascii .cargo-build-env
|
|
- restore_cache:
|
|
keys:
|
|
- v1-cargo-release-windows-{{ checksum ".cargo-build-env" }}-{{ checksum "litellm-rust/Cargo.lock" }}
|
|
- v1-cargo-release-windows-{{ checksum ".cargo-build-env" }}-
|
|
- run:
|
|
name: Force a rebuild of the workspace crates restored from the cargo cache
|
|
command: |
|
|
$fingerprints = "litellm-rust/target/release/.fingerprint"
|
|
if (Test-Path $fingerprints) {
|
|
Get-ChildItem -Path $fingerprints -Filter "litellm-*" | Remove-Item -Recurse -Force
|
|
}
|
|
- run:
|
|
name: Build the release wheel and install it under a worst-case MAX_PATH prefix
|
|
no_output_timeout: 30m
|
|
environment:
|
|
UV_HTTP_TIMEOUT: "300"
|
|
command: |
|
|
$env:Path = "C:\Python311;C:\Python311\Scripts;$HOME\.cargo\bin;$HOME\.local\bin;$env:Path"
|
|
uv build --wheel --out-dir dist
|
|
if ($LASTEXITCODE -ne 0) {
|
|
exit $LASTEXITCODE
|
|
}
|
|
python tests/windows_tests/check_windows_wheel_install.py
|
|
- when:
|
|
condition:
|
|
equal: [main, << pipeline.git.branch >>]
|
|
steps:
|
|
- save_cache:
|
|
key: v1-cargo-release-windows-{{ checksum ".cargo-build-env" }}-{{ checksum "litellm-rust/Cargo.lock" }}
|
|
paths:
|
|
- ~/.cargo/registry
|
|
- ~/project/litellm-rust/target/release
|
|
|
|
base_sdk_install:
|
|
docker:
|
|
- image: cimg/python:3.12@sha256:9c796c23c84e84a66a964acb508d39dc5433c81a47e07efd56dccbbc2427e07c
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout_over_https
|
|
- setup_google_dns
|
|
- install_uv
|
|
- install_rust
|
|
- run:
|
|
name: Build the wheel
|
|
no_output_timeout: 30m
|
|
environment:
|
|
UV_HTTP_TIMEOUT: "300"
|
|
command: |
|
|
uv build --wheel --out-dir dist
|
|
- run:
|
|
name: Install the wheel with no extras and smoke-check it
|
|
environment:
|
|
UV_HTTP_TIMEOUT: "300"
|
|
command: |
|
|
uv venv /tmp/base-sdk --python 3.12
|
|
VIRTUAL_ENV=/tmp/base-sdk uv pip install dist/*.whl
|
|
/tmp/base-sdk/bin/python tests/base_sdk_tests/check_base_sdk_install.py
|
|
- run:
|
|
name: Guard against MAX_PATH-busting packaged wheel paths
|
|
command: |
|
|
python3 tests/windows_tests/check_windows_wheel_install.py --lengths-only
|
|
|
|
helm_chart_testing:
|
|
machine:
|
|
image: ubuntu-2204:2024.04.1 # Use machine executor instead of docker
|
|
resource_class: medium
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout_over_https
|
|
- skip_if_unrelated_changes
|
|
- attach_workspace:
|
|
at: ~/project
|
|
- setup_google_dns
|
|
- install_helm
|
|
- install_kind
|
|
|
|
# Install kubectl (pinned version with hardcoded checksum)
|
|
- run:
|
|
name: Install kubectl v1.31.4
|
|
command: |
|
|
curl -sSLf -o /tmp/kubectl \
|
|
https://dl.k8s.io/release/v1.31.4/bin/linux/amd64/kubectl
|
|
echo "298e19e9c6c17199011404278f0ff8168a7eca4217edad9097af577023a5620f /tmp/kubectl" | sha256sum -c -
|
|
chmod +x /tmp/kubectl
|
|
sudo mv /tmp/kubectl /usr/local/bin/
|
|
|
|
# Create kind cluster
|
|
- run:
|
|
name: Create Kind Cluster
|
|
command: |
|
|
kind create cluster --name litellm-test
|
|
|
|
- run:
|
|
name: Load Docker Database Image for helm tests
|
|
command: |
|
|
zstd -d litellm-docker-database.tar.zst --stdout | docker load
|
|
IMAGE_TAG=${CIRCLE_SHA1:-ci}
|
|
docker tag litellm-docker-database:ci litellm-ci:${IMAGE_TAG}
|
|
|
|
- run:
|
|
name: Load Docker image into Kind
|
|
command: |
|
|
IMAGE_TAG=${CIRCLE_SHA1:-ci}
|
|
kind load docker-image litellm-ci:${IMAGE_TAG} --name litellm-test
|
|
|
|
# Run helm lint
|
|
- run:
|
|
name: Run helm lint
|
|
command: |
|
|
helm lint ./helm/litellm-helm
|
|
|
|
# Run helm tests
|
|
- run:
|
|
name: Run helm tests
|
|
command: |
|
|
IMAGE_TAG=${CIRCLE_SHA1:-ci}
|
|
helm install litellm ./helm/litellm-helm -f ./helm/litellm-helm/ci/test-values.yaml \
|
|
--set image.repository=litellm-ci \
|
|
--set image.tag=${IMAGE_TAG} \
|
|
--set image.pullPolicy=Never
|
|
if ! kubectl rollout status deployment/litellm --timeout=180s; then
|
|
kubectl describe pods -l app.kubernetes.io/name=litellm
|
|
kubectl logs -l app.kubernetes.io/name=litellm --tail=200
|
|
exit 1
|
|
fi
|
|
|
|
echo "Printing pod logs..."
|
|
kubectl logs $(kubectl get pods -l app.kubernetes.io/name=litellm -o jsonpath="{.items[0].metadata.name}")
|
|
|
|
# Run the helm tests
|
|
helm test litellm --logs
|
|
|
|
# Cleanup
|
|
- run:
|
|
name: Cleanup
|
|
command: |
|
|
kind delete cluster --name litellm-test
|
|
when: always # This ensures cleanup runs even if previous steps fail
|
|
|
|
db_migration_disable_update_check:
|
|
machine:
|
|
image: ubuntu-2204:2024.04.1
|
|
resource_class: medium
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout_over_https
|
|
- run:
|
|
name: Generate LiteLLM master key
|
|
command: |
|
|
key="$(openssl rand -hex 16)"
|
|
printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV"
|
|
- skip_if_unrelated_changes
|
|
- setup_google_dns
|
|
- install_uv
|
|
- install_rust
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- start_fake_openai_endpoint
|
|
- start_postgres:
|
|
db_name: litellm_test
|
|
- attach_workspace:
|
|
at: ~/project
|
|
- run:
|
|
name: Load Docker Database Image
|
|
command: |
|
|
zstd -d litellm-docker-database.tar.zst --stdout | docker load
|
|
docker images | grep litellm-docker-database
|
|
- run:
|
|
name: Seed database with real schema
|
|
command: |
|
|
docker run -d \
|
|
-p 4001:4000 \
|
|
-e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \
|
|
-e LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" \
|
|
--name schema-seed \
|
|
--add-host=host.docker.internal:host-gateway \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \
|
|
litellm-docker-database:ci \
|
|
--config /app/config.yaml \
|
|
--port 4000 \
|
|
--use_prisma_db_push
|
|
- wait_for_service:
|
|
url: http://localhost:4001
|
|
timeout: "300"
|
|
- run:
|
|
name: Stop schema seed container
|
|
command: docker stop schema-seed && docker rm schema-seed
|
|
- run:
|
|
name: Run Docker container with bad schema and disabled updates
|
|
command: |
|
|
docker run -d \
|
|
-p 4000:4000 \
|
|
-e LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true \
|
|
-e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \
|
|
-e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \
|
|
-e DISABLE_SCHEMA_UPDATE="True" \
|
|
-e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
|
|
--name my-app \
|
|
--add-host=host.docker.internal:host-gateway \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/schema.prisma \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/litellm/proxy/schema.prisma \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/disable_schema_update.yaml:/app/config.yaml \
|
|
litellm-docker-database:ci \
|
|
--config /app/config.yaml \
|
|
--port 4000
|
|
- wait_for_service:
|
|
url: http://localhost:4000
|
|
timeout: "60"
|
|
- run:
|
|
name: Check container logs for expected message
|
|
command: |
|
|
echo "=== Printing Full Container Startup Logs ==="
|
|
LOG_OUTPUT="$(docker logs my-app 2>&1)"
|
|
printf '%s\n' "$LOG_OUTPUT"
|
|
echo "=== End of Full Container Startup Logs ==="
|
|
|
|
if printf '%s\n' "$LOG_OUTPUT" | grep -q "prisma schema out of sync with db. Consider running these sql_commands to sync the two"; then
|
|
echo "Expected message found in logs. Test passed."
|
|
else
|
|
echo "Expected message not found in logs. Test failed."
|
|
exit 1
|
|
fi
|
|
- run:
|
|
name: Run Basic Proxy Startup Tests (Health Readiness and Chat Completion)
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/integration/configuration/test_basic_proxy_startup.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
|
|
-v \
|
|
--junitxml=test-results/junit-2.xml \
|
|
--durations=5"
|
|
no_output_timeout: 15m
|
|
- store_test_results:
|
|
path: test-results
|
|
|
|
proxy_logging_guardrails_model_info_tests:
|
|
machine:
|
|
image: ubuntu-2204:2024.04.1
|
|
resource_class: large
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout_over_https
|
|
- run:
|
|
name: Generate LiteLLM master key
|
|
command: |
|
|
key="$(openssl rand -hex 16)"
|
|
printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV"
|
|
- skip_if_unrelated_changes
|
|
- setup_google_dns
|
|
- install_uv
|
|
- install_rust
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- start_postgres
|
|
- attach_workspace:
|
|
at: ~/project
|
|
- run:
|
|
name: Load Docker Database Image
|
|
command: |
|
|
zstd -d litellm-docker-database.tar.zst --stdout | docker load
|
|
docker images | grep litellm-docker-database
|
|
- start_openai_record_replay_proxy
|
|
- start_fake_openai_endpoint
|
|
# Second Docker Container Run with Different Config
|
|
# NOTE: We intentionally pass a "bad" license here. We need to ensure proxy starts and serves request even with bad license
|
|
- run:
|
|
name: Run Second Docker container
|
|
command: |
|
|
docker run -d \
|
|
-p 4000:4000 \
|
|
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
|
|
-e REDIS_HOST=$REDIS_HOST \
|
|
-e REDIS_PASSWORD=$REDIS_PASSWORD \
|
|
-e REDIS_PORT=$REDIS_PORT \
|
|
-e LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" \
|
|
-e OPENAI_API_KEY=$OPENAI_API_KEY \
|
|
-e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
|
|
-e LITELLM_LICENSE="bad-license" \
|
|
-e LITELLM_LOG=ERROR \
|
|
--add-host host.docker.internal:host-gateway \
|
|
--name my-app-3 \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/enterprise_config.yaml:/app/config.yaml \
|
|
litellm-docker-database:ci \
|
|
--config /app/config.yaml \
|
|
--port 4000
|
|
|
|
- run:
|
|
name: Start outputting logs for second container
|
|
command: docker logs -f my-app-3
|
|
background: true
|
|
|
|
- wait_for_service:
|
|
url: http://localhost:4000
|
|
timeout: "300"
|
|
|
|
- run:
|
|
name: Run second round of tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/integration/configuration/test_basic_proxy_startup.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
|
|
-v \
|
|
--junitxml=test-results/junit-2.xml \
|
|
--durations=5"
|
|
no_output_timeout: 15m
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
e2e_ui_build:
|
|
docker:
|
|
- image: cimg/python:3.12-browsers@sha256:b432899af01c9a311bf74f4f22e9ada2e5306d4b1b4383f8d29e1228a5844ef2
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
resource_class: large
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout_over_https
|
|
- skip_if_unrelated_changes:
|
|
category: client
|
|
- setup_google_dns
|
|
- install_node
|
|
- restore_cache:
|
|
keys:
|
|
- ui-dashboard-node-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
|
|
- run:
|
|
name: Install dashboard dependencies
|
|
command: |
|
|
cd ui/litellm-dashboard
|
|
npm ci
|
|
- save_cache:
|
|
key: ui-dashboard-node-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
|
|
paths:
|
|
- ui/litellm-dashboard/node_modules
|
|
- run:
|
|
name: Build UI bundle
|
|
command: |
|
|
cd ui/litellm-dashboard
|
|
npm run build
|
|
mkdir -p ../../ui-bundle
|
|
mv out ../../ui-bundle/out
|
|
find ../../ui-bundle/out -name '*.html' ! -name 'index.html' | while read -r f; do
|
|
d="${f%.html}"; mkdir -p "$d"; mv "$f" "$d/index.html"
|
|
done
|
|
- persist_to_workspace:
|
|
root: ~/project
|
|
paths:
|
|
- ui-bundle/out
|
|
|
|
e2e_ui_testing:
|
|
docker:
|
|
- image: cimg/python:3.12-browsers@sha256:b432899af01c9a311bf74f4f22e9ada2e5306d4b1b4383f8d29e1228a5844ef2
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
- image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
|
|
environment:
|
|
POSTGRES_USER: e2euser
|
|
POSTGRES_PASSWORD: e2epassword
|
|
POSTGRES_DB: litellm_e2e
|
|
resource_class: xlarge
|
|
working_directory: ~/project
|
|
environment:
|
|
DATABASE_URL: "postgresql://e2euser:e2epassword@localhost:5432/litellm_e2e"
|
|
CI: "true"
|
|
# Boot the proxy with an external logout URL so proxyLogoutUrl.spec.ts can
|
|
# assert the redirect. Set at job level so both the proxy boot step and the
|
|
# Playwright step (whose skip guard reads this) see the same value. Safe for
|
|
# the rest of the suite: nothing else performs a logout.
|
|
PROXY_LOGOUT_URL: "https://www.example.com"
|
|
steps:
|
|
- checkout_over_https
|
|
- run:
|
|
name: Generate LiteLLM master key
|
|
command: |
|
|
key="$(openssl rand -hex 16)"
|
|
printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV"
|
|
- skip_if_unrelated_changes:
|
|
category: client
|
|
- attach_workspace:
|
|
at: ~/project
|
|
- setup_google_dns
|
|
- install_node
|
|
- install_uv
|
|
- install_rust
|
|
- restore_cache:
|
|
keys:
|
|
- v1-uv-cache-{{ checksum "uv.lock" }}
|
|
- run:
|
|
name: Install Python dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
uv run --no-sync python -m prisma generate --schema litellm/proxy/schema.prisma
|
|
- save_cache:
|
|
key: v1-uv-cache-{{ checksum "uv.lock" }}
|
|
paths:
|
|
- ~/.cache/uv
|
|
- restore_cache:
|
|
keys:
|
|
- ui-playwright-node-deps-v1-{{ checksum "tests/e2e/ui/package-lock.json" }}
|
|
- run:
|
|
name: Install Node dependencies and Playwright
|
|
# The cimg/python:3.12-browsers image already ships the Chromium system
|
|
# libraries Playwright needs (libnss3, libatk-bridge2.0-0, libcups2, etc.).
|
|
# `--with-deps` triggers a redundant apt-get update + install that adds
|
|
# 5-10 minutes to the job and frequently stalls on flaky Ubuntu mirrors,
|
|
# so we install just the browser binary.
|
|
command: |
|
|
cd tests/e2e/ui
|
|
npm ci
|
|
npx playwright install chromium
|
|
- save_cache:
|
|
key: ui-playwright-node-deps-v1-{{ checksum "tests/e2e/ui/package-lock.json" }}
|
|
paths:
|
|
- tests/e2e/ui/node_modules
|
|
- ~/.cache/ms-playwright
|
|
- run:
|
|
name: Use built UI bundle
|
|
command: |
|
|
rm -rf litellm/proxy/_experimental/out
|
|
cp -r ui-bundle/out litellm/proxy/_experimental/out
|
|
- wait_for_service:
|
|
url: tcp://localhost:5432
|
|
timeout: "30"
|
|
- run:
|
|
name: Push Prisma schema
|
|
command: uv run --no-sync python -m prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
|
|
- run:
|
|
name: Seed database
|
|
command: |
|
|
PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \
|
|
-f tests/e2e/ui/fixtures/seed.sql
|
|
- run:
|
|
name: Start mock LLM server
|
|
command: uv run --no-sync python tests/e2e/ui/fixtures/mock_llm_server/server.py
|
|
background: true
|
|
- run:
|
|
name: Start mock Presidio server
|
|
command: uv run --no-sync python tests/e2e/ui/fixtures/mock_presidio_server/server.py
|
|
background: true
|
|
- run:
|
|
name: Wait for mock Presidio server
|
|
command: |
|
|
for i in $(seq 1 30); do
|
|
if curl -sf http://127.0.0.1:8091/health >/dev/null 2>&1; then exit 0; fi
|
|
sleep 1
|
|
done
|
|
echo "Mock Presidio server never answered /health on port 8091" >&2
|
|
exit 1
|
|
- run:
|
|
name: Start LiteLLM proxy
|
|
environment:
|
|
MOCK_LLM_URL: "http://127.0.0.1:8090/v1"
|
|
DISABLE_SCHEMA_UPDATE: "true"
|
|
SERVER_ROOT_PATH: ""
|
|
# PROXY_LOGOUT_URL is inherited from the job-level environment so the
|
|
# proxy and proxyLogoutUrl.spec.ts agree on the logout target.
|
|
# LITELLM_LICENSE is forwarded from the project env so premium-gated
|
|
# UI flows can be exercised. license.spec.ts asserts the resulting
|
|
# JWT carries premium_user=true; if it ever stops being passed, that
|
|
# test fails loudly rather than silently regressing premium coverage.
|
|
command: |
|
|
LITELLM_LICENSE="$LITELLM_LICENSE" \
|
|
uv run --no-sync python -m litellm.proxy.proxy_cli \
|
|
--config tests/e2e/ui/fixtures/config.yml \
|
|
--port 4000
|
|
background: true
|
|
- run:
|
|
name: Wait for proxy to be ready
|
|
command: |
|
|
for i in $(seq 1 60); do
|
|
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:4000/health -H "Authorization: Bearer $LITELLM_MASTER_KEY" 2>/dev/null || true)
|
|
if [ "$HTTP_CODE" = "200" ]; then
|
|
echo "Proxy is ready"
|
|
exit 0
|
|
fi
|
|
sleep 2
|
|
done
|
|
echo "Proxy failed to start"
|
|
exit 1
|
|
- run:
|
|
name: Run Playwright E2E tests
|
|
# Forward LITELLM_LICENSE so license.spec.ts can detect that the
|
|
# proxy was launched with a license and assert premium_user=true.
|
|
command: |
|
|
cd tests/e2e/ui
|
|
LITELLM_LICENSE="$LITELLM_LICENSE" \
|
|
npx playwright test --config playwright.config.ts --workers=3 --retries=0
|
|
no_output_timeout: 10m
|
|
- store_artifacts:
|
|
path: tests/e2e/ui/test-results
|
|
destination: e2e-test-results
|
|
- store_artifacts:
|
|
path: tests/e2e/ui/playwright-report
|
|
destination: e2e-playwright-report
|
|
|
|
e2e_ui_testing_server_root_path:
|
|
docker:
|
|
- image: cimg/python:3.12-browsers@sha256:b432899af01c9a311bf74f4f22e9ada2e5306d4b1b4383f8d29e1228a5844ef2
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
- image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
|
|
environment:
|
|
POSTGRES_USER: e2euser
|
|
POSTGRES_PASSWORD: e2epassword
|
|
POSTGRES_DB: litellm_e2e
|
|
resource_class: large
|
|
working_directory: ~/project
|
|
environment:
|
|
DATABASE_URL: "postgresql://e2euser:e2epassword@localhost:5432/litellm_e2e"
|
|
CI: "true"
|
|
# The whole job exercises the proxy mounted under a prefix. SERVER_ROOT_PATH
|
|
# is read both by the proxy at boot (to rewrite the built UI bundle in place)
|
|
# and by migration.serverRootPath.config.ts, which refuses to run without it.
|
|
SERVER_ROOT_PATH: "/litellm"
|
|
steps:
|
|
- checkout_over_https
|
|
- run:
|
|
name: Generate LiteLLM master key
|
|
command: |
|
|
key="$(openssl rand -hex 16)"
|
|
printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV"
|
|
- skip_if_unrelated_changes:
|
|
category: client
|
|
- attach_workspace:
|
|
at: ~/project
|
|
- setup_google_dns
|
|
- install_node
|
|
- install_uv
|
|
- install_rust
|
|
- restore_cache:
|
|
keys:
|
|
- v1-uv-cache-{{ checksum "uv.lock" }}
|
|
- run:
|
|
name: Install Python dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
uv run --no-sync python -m prisma generate --schema litellm/proxy/schema.prisma
|
|
- save_cache:
|
|
key: v1-uv-cache-{{ checksum "uv.lock" }}
|
|
paths:
|
|
- ~/.cache/uv
|
|
- restore_cache:
|
|
keys:
|
|
- ui-playwright-node-deps-v1-{{ checksum "tests/e2e/ui/package-lock.json" }}
|
|
- run:
|
|
name: Install Node dependencies and Playwright
|
|
command: |
|
|
cd tests/e2e/ui
|
|
npm ci
|
|
npx playwright install chromium
|
|
- save_cache:
|
|
key: ui-playwright-node-deps-v1-{{ checksum "tests/e2e/ui/package-lock.json" }}
|
|
paths:
|
|
- tests/e2e/ui/node_modules
|
|
- ~/.cache/ms-playwright
|
|
- run:
|
|
name: Use built UI bundle
|
|
command: |
|
|
rm -rf litellm/proxy/_experimental/out
|
|
cp -r ui-bundle/out litellm/proxy/_experimental/out
|
|
- wait_for_service:
|
|
url: tcp://localhost:5432
|
|
timeout: "30"
|
|
- run:
|
|
name: Push Prisma schema
|
|
command: uv run --no-sync python -m prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
|
|
- run:
|
|
name: Seed database
|
|
command: |
|
|
PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \
|
|
-f tests/e2e/ui/fixtures/seed.sql
|
|
- run:
|
|
name: Start mock LLM server
|
|
command: uv run --no-sync python tests/e2e/ui/fixtures/mock_llm_server/server.py
|
|
background: true
|
|
- run:
|
|
name: Start mock Presidio server
|
|
command: uv run --no-sync python tests/e2e/ui/fixtures/mock_presidio_server/server.py
|
|
background: true
|
|
- run:
|
|
name: Wait for mock Presidio server
|
|
command: |
|
|
for i in $(seq 1 30); do
|
|
if curl -sf http://127.0.0.1:8091/health >/dev/null 2>&1; then exit 0; fi
|
|
sleep 1
|
|
done
|
|
echo "Mock Presidio server never answered /health on port 8091" >&2
|
|
exit 1
|
|
- run:
|
|
name: Start LiteLLM proxy under a server root path
|
|
environment:
|
|
MOCK_LLM_URL: "http://127.0.0.1:8090/v1"
|
|
DISABLE_SCHEMA_UPDATE: "true"
|
|
# Output flows to this step's own log, so a boot crash is visible here
|
|
# rather than swallowed by a downstream readiness probe.
|
|
command: |
|
|
LITELLM_LICENSE="$LITELLM_LICENSE" \
|
|
uv run --no-sync python -m litellm.proxy.proxy_cli \
|
|
--config tests/e2e/ui/fixtures/config.yml \
|
|
--port 4000
|
|
background: true
|
|
- run:
|
|
name: Wait for prefixed proxy to be ready
|
|
command: |
|
|
for i in $(seq 1 60); do
|
|
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 -H "Authorization: Bearer $LITELLM_MASTER_KEY" http://127.0.0.1:4000/litellm/health 2>/dev/null || true)
|
|
if [ "$HTTP_CODE" = "200" ]; then
|
|
echo "Prefixed proxy is ready"
|
|
exit 0
|
|
fi
|
|
sleep 2
|
|
done
|
|
echo "Prefixed proxy failed to start; see the 'Start LiteLLM proxy under a server root path' step for the boot log"
|
|
exit 1
|
|
- run:
|
|
name: Run migration smoke under SERVER_ROOT_PATH
|
|
command: |
|
|
cd tests/e2e/ui
|
|
LITELLM_LICENSE="$LITELLM_LICENSE" \
|
|
npx playwright test --config migration.serverRootPath.config.ts
|
|
no_output_timeout: 10m
|
|
- store_artifacts:
|
|
path: tests/e2e/ui/test-results
|
|
destination: e2e-server-root-path-test-results
|
|
- store_artifacts:
|
|
path: tests/e2e/ui/playwright-report
|
|
destination: e2e-server-root-path-playwright-report
|
|
|
|
build_docker_database_image:
|
|
parameters:
|
|
migration_qualification:
|
|
type: boolean
|
|
default: false
|
|
machine:
|
|
image: ubuntu-2204:2024.04.1
|
|
resource_class: large
|
|
working_directory: ~/project
|
|
steps:
|
|
- when:
|
|
condition: << parameters.migration_qualification >>
|
|
steps:
|
|
- checkout_over_https:
|
|
revision: << pipeline.parameters.migration_source_sha >>
|
|
- unless:
|
|
condition: << parameters.migration_qualification >>
|
|
steps:
|
|
- checkout_over_https
|
|
- skip_if_unrelated_changes
|
|
|
|
- run:
|
|
name: Build Docker image
|
|
environment:
|
|
MIGRATION_CANDIDATE_IMAGE: << pipeline.parameters.migration_candidate_image >>
|
|
command: |
|
|
if [ -n "$MIGRATION_CANDIDATE_IMAGE" ]; then
|
|
[[ "$MIGRATION_CANDIDATE_IMAGE" =~ ^ghcr.io/berriai/[a-z0-9._/-]+@sha256:[0-9a-f]{64}$ ]] || exit 1
|
|
docker pull "$MIGRATION_CANDIDATE_IMAGE"
|
|
docker tag "$MIGRATION_CANDIDATE_IMAGE" litellm-docker-database:ci
|
|
else
|
|
docker build \
|
|
--label org.opencontainers.image.revision="$(git rev-parse HEAD)" \
|
|
-t litellm-docker-database:ci \
|
|
-f docker/Dockerfile.database .
|
|
fi
|
|
python3 .circleci/scripts/run_migration_tests.py record-image
|
|
|
|
- run:
|
|
name: Save Docker image to workspace root
|
|
command: |
|
|
docker save litellm-docker-database:ci | zstd -1 -T0 > litellm-docker-database.tar.zst
|
|
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- litellm-docker-database.tar.zst
|
|
- migration-image.json
|
|
|
|
migration_startup_tests:
|
|
parameters:
|
|
suite:
|
|
type: enum
|
|
enum: [startup, recovery, legacy, upgrade, shaped]
|
|
baseline:
|
|
type: boolean
|
|
default: false
|
|
machine:
|
|
image: ubuntu-2204:2024.04.1
|
|
resource_class: large
|
|
working_directory: ~/project
|
|
environment:
|
|
LITELLM_MIGRATION_TESTS: "1"
|
|
LITELLM_MIGRATION_TEST_IMAGE: litellm-docker-database:ci
|
|
LITELLM_MIGRATION_BASELINE_IMAGE: << pipeline.parameters.migration_baseline_image >>
|
|
MIGRATION_TEST_ADMIN_URL: postgresql://postgres:postgres@127.0.0.1:5432/postgres
|
|
MIGRATION_TEST_CONTAINER_ADMIN_URL: postgresql://postgres:postgres@host.docker.internal:5432/postgres
|
|
MIGRATION_TEST_OUTPUT: /tmp/migration-results
|
|
PYTHONPATH: tests/e2e
|
|
steps:
|
|
- checkout_over_https:
|
|
revision: << pipeline.parameters.migration_source_sha >>
|
|
- run:
|
|
name: Generate LiteLLM master key
|
|
command: |
|
|
key="$(openssl rand -hex 16)"
|
|
printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV"
|
|
- install_uv
|
|
- install_rust
|
|
- restore_cache:
|
|
keys:
|
|
- v1-uv-cache-{{ checksum "uv.lock" }}
|
|
- run:
|
|
name: Install test dependencies
|
|
command: uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- attach_workspace:
|
|
at: ~/project
|
|
- run:
|
|
name: Load the shared candidate and start PostgreSQL
|
|
command: |
|
|
zstd -d litellm-docker-database.tar.zst --stdout | docker load
|
|
docker run -d --name migration-postgres \
|
|
-e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres \
|
|
-p 5432:5432 \
|
|
postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
|
|
- wait_for_service:
|
|
url: tcp://localhost:5432
|
|
timeout: "60"
|
|
- when:
|
|
condition: << parameters.baseline >>
|
|
steps:
|
|
- run:
|
|
name: Pull the baseline release the upgrade starts from
|
|
environment:
|
|
BASELINE_IMAGE: << pipeline.parameters.migration_baseline_image >>
|
|
command: |
|
|
[[ "$BASELINE_IMAGE" =~ ^ghcr.io/berriai/[a-z0-9._/-]+(@sha256:[0-9a-f]{64}|:v[0-9][0-9a-z.-]*)$ ]] || exit 1
|
|
docker pull "$BASELINE_IMAGE"
|
|
- run:
|
|
name: Run migration startup regressions
|
|
environment:
|
|
MIGRATION_TEST_SUITE: << parameters.suite >>
|
|
MIGRATION_CANDIDATE_IMAGE: << pipeline.parameters.migration_candidate_image >>
|
|
command: |
|
|
mkdir -p /tmp/migration-results
|
|
uv run --no-sync python .circleci/scripts/run_migration_tests.py
|
|
no_output_timeout: 15m
|
|
- store_test_results:
|
|
path: /tmp/migration-results/junit
|
|
- run:
|
|
name: Package migration diagnostics
|
|
when: always
|
|
command: |
|
|
mkdir -p /tmp/migration-artifacts
|
|
if [ -d /tmp/migration-results ]; then
|
|
tar -czf /tmp/migration-artifacts/diagnostics.tar.gz -C /tmp/migration-results .
|
|
if [ -f /tmp/migration-results/verdict.json ]; then
|
|
cp /tmp/migration-results/verdict.json /tmp/migration-artifacts/verdict.json
|
|
fi
|
|
fi
|
|
- store_artifacts:
|
|
path: /tmp/migration-artifacts
|
|
destination: migration-results
|
|
- run:
|
|
name: Remove migration test containers
|
|
when: always
|
|
command: |
|
|
docker ps -aq --filter label=litellm-migration-test=true | xargs -r docker rm -f
|
|
docker rm -f migration-postgres || true
|
|
|
|
test_bad_database_url:
|
|
machine:
|
|
image: ubuntu-2204:2024.04.1
|
|
resource_class: medium
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout_over_https
|
|
- skip_if_unrelated_changes
|
|
- attach_workspace:
|
|
at: ~/project
|
|
- setup_google_dns
|
|
- start_postgres
|
|
- run:
|
|
name: Load Docker Database Image
|
|
command: |
|
|
zstd -d litellm-docker-database.tar.zst --stdout | docker load
|
|
docker tag litellm-docker-database:ci myapp:latest
|
|
- run:
|
|
name: Run Docker container with bad DATABASE_URL
|
|
command: |
|
|
set +e
|
|
docker run --name my-app \
|
|
-p 4000:4000 \
|
|
-e LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true \
|
|
-e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \
|
|
-e DATABASE_URL="postgresql://wrong:wrong@wrong:5432/wrong" \
|
|
myapp:latest \
|
|
--port 4000 > docker_output.log 2>&1
|
|
echo "$?" > docker_exit_code
|
|
set -e
|
|
- run:
|
|
name: Display Docker logs
|
|
command: cat docker_output.log
|
|
- run:
|
|
name: Proxy must refuse to serve on an unreachable database
|
|
command: |
|
|
fail() { echo "FAILED: $1"; cat docker_output.log; exit 1; }
|
|
exit_code="$(cat docker_exit_code)"
|
|
[ "$exit_code" -ne 0 ] || fail "proxy exited 0 with an unreachable database"
|
|
grep -q "P1001" docker_output.log || fail "log does not name the unreachable database server"
|
|
! grep -q "Application startup complete" docker_output.log || fail "proxy reached serving state"
|
|
! docker exec my-app true 2>/dev/null || fail "container is still running"
|
|
echo "Proxy refused to serve (exit $exit_code) and never reached startup. Test passed."
|
|
|
|
integration_contracts:
|
|
parameters:
|
|
suite:
|
|
type: string
|
|
mode:
|
|
type: enum
|
|
enum: [standard, replica]
|
|
default: standard
|
|
parallelism:
|
|
type: integer
|
|
default: 1
|
|
resource_class:
|
|
type: enum
|
|
enum: [large, xlarge]
|
|
default: large
|
|
machine:
|
|
image: ubuntu-2204:2024.04.1
|
|
resource_class: << parameters.resource_class >>
|
|
working_directory: ~/project
|
|
parallelism: << parameters.parallelism >>
|
|
steps:
|
|
- setup_litellm_test_deps
|
|
- when:
|
|
condition:
|
|
equal: [browser, << parameters.suite >>]
|
|
steps:
|
|
- install_node
|
|
- restore_cache:
|
|
keys:
|
|
- integration-ui-v3-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }}
|
|
- run:
|
|
name: Install locked browser dependencies
|
|
command: |
|
|
cd ui/litellm-dashboard
|
|
npm ci
|
|
cd ../../tests/e2e/ui
|
|
npm ci
|
|
sudo env PATH="$PATH" DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=l \
|
|
timeout --signal=TERM --kill-after=20s 6m node node_modules/@playwright/test/cli.js install-deps chromium
|
|
timeout --signal=TERM --kill-after=20s 3m node node_modules/@playwright/test/cli.js install chromium
|
|
- save_cache:
|
|
key: integration-ui-v3-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }}
|
|
paths:
|
|
- ~/.npm
|
|
- ~/.cache/ms-playwright
|
|
- run:
|
|
name: Build the candidate dashboard
|
|
command: cd ui/litellm-dashboard && NEXT_TELEMETRY_DISABLED=1 npm run build
|
|
- when:
|
|
condition:
|
|
equal: [mcp, << parameters.suite >>]
|
|
steps:
|
|
- run:
|
|
name: Install MCP SDK1 peer
|
|
command: |
|
|
uv venv --python 3.12 .venv-mcp-peer
|
|
uv pip install --no-config --exclude-newer "3 days" --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1'
|
|
echo "export MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$BASH_ENV"
|
|
- when:
|
|
condition:
|
|
equal: [sdk, << parameters.suite >>]
|
|
steps:
|
|
- run:
|
|
name: Install redis-server 7.2.16
|
|
command: |
|
|
mkdir -p /tmp/redis-bin
|
|
cid="$(docker create redis:7.2.16@sha256:0637954999d01b7c9ce9167db2da50656e2590d3b884f1c600c5f63bb6e6773c)"
|
|
docker cp "${cid}:/usr/local/bin/redis-server" /tmp/redis-bin/redis-server
|
|
docker rm "$cid"
|
|
/tmp/redis-bin/redis-server --version | grep -q 'v=7.2.16'
|
|
echo 'export PATH="/tmp/redis-bin:$PATH"' >> "$BASH_ENV"
|
|
- start_postgres:
|
|
image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
|
|
server_args: "-c max_connections=300 -c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c pg_stat_statements.max=20000"
|
|
- start_redis
|
|
- run:
|
|
name: Run owned integration contracts
|
|
command: bash .circleci/scripts/run_integration.sh << parameters.suite >> << parameters.mode >>
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Stop owned database and Redis
|
|
when: always
|
|
command: |
|
|
mkdir -p test-results/services-<< parameters.suite >>-<< parameters.mode >>
|
|
docker logs postgres-db > test-results/services-<< parameters.suite >>-<< parameters.mode >>/postgres.log 2>&1 || true
|
|
docker logs redis-cache > test-results/services-<< parameters.suite >>-<< parameters.mode >>/redis.log 2>&1 || true
|
|
docker rm -f postgres-db redis-cache
|
|
test -z "$(docker ps -aq --filter name=postgres-db --filter name=redis-cache)"
|
|
- when:
|
|
condition:
|
|
not:
|
|
equal: [1, << parameters.parallelism >>]
|
|
steps:
|
|
- run:
|
|
name: Prepare this shard's execution record
|
|
command: |
|
|
results="test-results/integration-<< parameters.suite >>"
|
|
if [ "<< parameters.mode >>" = replica ]; then
|
|
results="$results-replica"
|
|
fi
|
|
mkdir -p "/tmp/integration-shards/<< parameters.suite >>-<< parameters.mode >>/$CIRCLE_NODE_INDEX"
|
|
cp "$results/execution.json" "/tmp/integration-shards/<< parameters.suite >>-<< parameters.mode >>/$CIRCLE_NODE_INDEX/execution.json"
|
|
if [ "<< parameters.suite >>" = mcp ]; then
|
|
mkdir -p "/tmp/integration-shards/mcp-standard/$CIRCLE_NODE_INDEX/coverage"
|
|
cp test-results/integration-mcp/coverage/data "/tmp/integration-shards/mcp-standard/$CIRCLE_NODE_INDEX/coverage/data.$CIRCLE_NODE_INDEX"
|
|
fi
|
|
- persist_to_workspace:
|
|
root: /tmp/integration-shards
|
|
paths:
|
|
- << parameters.suite >>-<< parameters.mode >>
|
|
- store_test_results:
|
|
path: test-results
|
|
- store_artifacts:
|
|
path: test-results
|
|
|
|
integration_shard_coverage:
|
|
parameters:
|
|
suite:
|
|
type: enum
|
|
enum: [management, accounting, database, extensions, providers, mcp]
|
|
mode:
|
|
type: enum
|
|
enum: [standard, replica]
|
|
default: standard
|
|
shards:
|
|
type: integer
|
|
docker:
|
|
- image: cimg/python:3.12@sha256:9c796c23c84e84a66a964acb508d39dc5433c81a47e07efd56dccbbc2427e07c
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
resource_class: small
|
|
working_directory: ~/project
|
|
environment:
|
|
UV_PYTHON_DOWNLOADS: never
|
|
steps:
|
|
- checkout_over_https
|
|
- install_uv
|
|
- run:
|
|
name: Install the locked shard verifier and coverage dependencies
|
|
command: |
|
|
uv export --frozen --all-groups --all-extras --no-emit-project --no-emit-workspace --no-header --no-annotate \
|
|
| awk '/^[^ ]/ { keep = $0 ~ /^(annotated-types|coverage|pydantic|pydantic-core|typing-extensions|typing-inspection)==/ } keep' \
|
|
> /tmp/fan-in-requirements.txt
|
|
uv venv .venv --python 3.12
|
|
uv pip install --no-config --python .venv --require-hashes -r /tmp/fan-in-requirements.txt
|
|
- attach_workspace:
|
|
at: /tmp/integration-shards
|
|
- run:
|
|
name: Verify every << parameters.suite >> case ran exactly once
|
|
command: |
|
|
.venv/bin/python -m tests.integration._support.sharding /tmp/integration-shards/<< parameters.suite >>-<< parameters.mode >> \
|
|
--shards << parameters.shards >>
|
|
- when:
|
|
condition:
|
|
equal: [mcp, << parameters.suite >>]
|
|
steps:
|
|
- run:
|
|
name: Combine coverage from every MCP shard
|
|
command: |
|
|
mkdir -p test-results/integration-mcp/coverage
|
|
export COVERAGE_FILE="$PWD/test-results/integration-mcp/coverage/data"
|
|
.venv/bin/python -m coverage combine --rcfile=tests/integration/mcp_coverage.toml /tmp/integration-shards/mcp-standard/*/coverage
|
|
.venv/bin/python -m coverage report --rcfile=tests/integration/mcp_coverage.toml > test-results/integration-mcp/coverage/coverage.txt
|
|
.venv/bin/python -m coverage html --rcfile=tests/integration/mcp_coverage.toml -d test-results/integration-mcp/coverage/html
|
|
- store_artifacts:
|
|
path: test-results/integration-mcp/coverage
|
|
|
|
routing_parity:
|
|
parameters:
|
|
suite:
|
|
type: string
|
|
machine:
|
|
image: ubuntu-2204:2024.04.1
|
|
resource_class: large
|
|
working_directory: ~/project
|
|
steps:
|
|
- setup_litellm_test_deps
|
|
- run:
|
|
name: Check out base product code
|
|
environment:
|
|
ROUTING_PARITY_BASE: << pipeline.parameters.routing_parity_base >>
|
|
command: |
|
|
[[ "$ROUTING_PARITY_BASE" =~ ^[0-9a-f]{40}$ ]] || exit 1
|
|
git fetch --depth 1 --no-filter origin "$ROUTING_PARITY_BASE"
|
|
git rm -r -f --quiet litellm enterprise litellm-proxy-extras
|
|
git checkout "$ROUTING_PARITY_BASE" -- litellm enterprise litellm-proxy-extras
|
|
git reset --quiet
|
|
test -f litellm/rust_bridge/_native.abi3.so
|
|
- start_postgres:
|
|
image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
|
|
server_args: "-c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c pg_stat_statements.max=20000"
|
|
- start_redis
|
|
- run:
|
|
name: Run base side
|
|
command: bash .circleci/scripts/run_integration.sh << parameters.suite >> parity base
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Stop base database and Redis
|
|
when: always
|
|
command: |
|
|
mkdir -p test-results/services-<< parameters.suite >>-parity-base
|
|
docker logs postgres-db > test-results/services-<< parameters.suite >>-parity-base/postgres.log 2>&1 || true
|
|
docker logs redis-cache > test-results/services-<< parameters.suite >>-parity-base/redis.log 2>&1 || true
|
|
docker rm -f postgres-db redis-cache
|
|
test -z "$(docker ps -aq --filter name=postgres-db --filter name=redis-cache)"
|
|
- run:
|
|
name: Check out head product code
|
|
command: |
|
|
git rm -r -f --quiet litellm enterprise litellm-proxy-extras
|
|
git checkout "$CIRCLE_SHA1" -- litellm enterprise litellm-proxy-extras
|
|
git reset --quiet
|
|
test -f litellm/rust_bridge/_native.abi3.so
|
|
- start_postgres:
|
|
image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
|
|
server_args: "-c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c pg_stat_statements.max=20000"
|
|
- start_redis
|
|
- run:
|
|
name: Run head side
|
|
command: bash .circleci/scripts/run_integration.sh << parameters.suite >> parity head
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Stop head database and Redis
|
|
when: always
|
|
command: |
|
|
mkdir -p test-results/services-<< parameters.suite >>-parity-head
|
|
docker logs postgres-db > test-results/services-<< parameters.suite >>-parity-head/postgres.log 2>&1 || true
|
|
docker logs redis-cache > test-results/services-<< parameters.suite >>-parity-head/redis.log 2>&1 || true
|
|
docker rm -f postgres-db redis-cache
|
|
test -z "$(docker ps -aq --filter name=postgres-db --filter name=redis-cache)"
|
|
- run:
|
|
name: Compare routing parity
|
|
command: PYTHONPATH="$PWD/tests" .venv/bin/python -m integration._support.routing check test-results/parity-<< parameters.suite >>/base test-results/parity-<< parameters.suite >>/head
|
|
- store_test_results:
|
|
path: test-results
|
|
- store_artifacts:
|
|
path: test-results
|
|
|
|
postgres_suite:
|
|
parameters:
|
|
test_path:
|
|
type: string
|
|
seed:
|
|
type: boolean
|
|
coverage_flag:
|
|
type: string
|
|
default: ""
|
|
timeout_minutes:
|
|
type: integer
|
|
workers:
|
|
type: integer
|
|
default: 1
|
|
machine:
|
|
image: ubuntu-2204:2024.04.1
|
|
resource_class: large
|
|
working_directory: ~/project
|
|
environment:
|
|
DATABASE_URL: postgresql://postgres:postgres@localhost:5432/litellm_test
|
|
steps:
|
|
- checkout_over_https
|
|
- skip_if_unrelated_changes
|
|
- setup_litellm_test_deps:
|
|
checkout: false
|
|
- start_postgres:
|
|
db_name: litellm_test
|
|
image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
|
|
- run:
|
|
name: Generate Prisma client
|
|
command: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
|
|
- when:
|
|
condition: << parameters.seed >>
|
|
steps:
|
|
- run:
|
|
name: Seed database schema
|
|
command: uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
|
|
- run:
|
|
name: Run << parameters.test_path >>
|
|
command: |
|
|
mkdir -p test-results
|
|
coverage_args=()
|
|
worker_args=()
|
|
if [ "<< parameters.workers >>" -gt 1 ]; then
|
|
worker_args=(-n << parameters.workers >> --dist=worksteal)
|
|
fi
|
|
if [ -n "<< parameters.coverage_flag >>" ]; then
|
|
coverage_args=(--cov=./litellm --cov-report=xml:coverage.xml)
|
|
fi
|
|
timeout --signal=TERM << parameters.timeout_minutes >>m uv run --no-sync pytest \
|
|
<< parameters.test_path >> -vv --tb=short --durations=10 -o junit_family=xunit1 \
|
|
--junitxml=test-results/junit.xml "${coverage_args[@]}" "${worker_args[@]}"
|
|
- when:
|
|
condition: << parameters.coverage_flag >>
|
|
steps:
|
|
- install_codecov_cli
|
|
- run:
|
|
name: Upload coverage
|
|
when: always
|
|
command: |
|
|
[ -f coverage.xml ] || { echo "no coverage.xml produced"; exit 1; }
|
|
codecov upload-process --disable-search --fail-on-error -f coverage.xml \
|
|
-F << parameters.coverage_flag >> -C "$CIRCLE_SHA1" \
|
|
-n "<< parameters.coverage_flag >>-${CIRCLE_BUILD_NUM}" --git-service github
|
|
- store_test_results:
|
|
path: test-results
|
|
- store_artifacts:
|
|
path: test-results
|
|
|
|
workflows:
|
|
migration_startup_scheduled:
|
|
triggers:
|
|
- schedule:
|
|
cron: "17 0,6,12,18 * * *"
|
|
filters:
|
|
branches:
|
|
only: main
|
|
jobs:
|
|
- build_docker_database_image:
|
|
migration_qualification: true
|
|
- migration_startup_tests:
|
|
name: migration-startup
|
|
suite: startup
|
|
requires: [build_docker_database_image]
|
|
- migration_startup_tests:
|
|
name: migration-recovery
|
|
suite: recovery
|
|
requires: [build_docker_database_image]
|
|
- migration_startup_tests:
|
|
name: migration-legacy-and-pooling
|
|
suite: legacy
|
|
requires: [build_docker_database_image]
|
|
- migration_startup_tests:
|
|
name: migration-upgrade
|
|
suite: upgrade
|
|
baseline: true
|
|
requires: [build_docker_database_image]
|
|
- migration_startup_tests:
|
|
name: migration-upgrade-shaped
|
|
suite: shaped
|
|
baseline: true
|
|
requires: [build_docker_database_image]
|
|
integration:
|
|
jobs:
|
|
- integration_contracts:
|
|
name: integration-<< matrix.suite >>
|
|
filters: &integration_default_run not pipeline.parameters.run_migration_tests and pipeline.parameters.routing_parity_base == ""
|
|
matrix:
|
|
parameters:
|
|
suite: [sdk, cost, browser]
|
|
parallelism: 1
|
|
- integration_contracts:
|
|
name: integration-security
|
|
suite: security
|
|
parallelism: 1
|
|
resource_class: xlarge
|
|
filters: *integration_default_run
|
|
- integration_contracts:
|
|
name: integration-management
|
|
suite: management
|
|
parallelism: 4
|
|
resource_class: xlarge
|
|
filters: *integration_default_run
|
|
- integration_shard_coverage:
|
|
name: integration-management-coverage
|
|
suite: management
|
|
shards: 4
|
|
requires: [integration-management]
|
|
filters: *integration_default_run
|
|
- integration_contracts:
|
|
name: integration-accounting
|
|
suite: accounting
|
|
parallelism: 4
|
|
resource_class: xlarge
|
|
filters: *integration_default_run
|
|
- integration_shard_coverage:
|
|
name: integration-accounting-coverage
|
|
suite: accounting
|
|
shards: 4
|
|
requires: [integration-accounting]
|
|
filters: *integration_default_run
|
|
- integration_contracts:
|
|
name: integration-providers
|
|
suite: providers
|
|
parallelism: 4
|
|
resource_class: xlarge
|
|
filters: *integration_default_run
|
|
- integration_shard_coverage:
|
|
name: integration-providers-coverage
|
|
suite: providers
|
|
shards: 4
|
|
requires:
|
|
- integration-providers
|
|
filters: *integration_default_run
|
|
- integration_contracts:
|
|
name: integration-database
|
|
suite: database
|
|
parallelism: 2
|
|
filters: *integration_default_run
|
|
- integration_shard_coverage:
|
|
name: integration-database-coverage
|
|
suite: database
|
|
shards: 2
|
|
requires: [integration-database]
|
|
filters: *integration_default_run
|
|
- integration_contracts:
|
|
name: integration-mcp
|
|
suite: mcp
|
|
parallelism: 2
|
|
filters: *integration_default_run
|
|
- integration_shard_coverage:
|
|
name: integration-mcp-coverage
|
|
suite: mcp
|
|
shards: 2
|
|
requires:
|
|
- integration-mcp
|
|
filters: *integration_default_run
|
|
- integration_contracts:
|
|
name: integration-extensions
|
|
suite: extensions
|
|
parallelism: 8
|
|
resource_class: xlarge
|
|
filters: *integration_default_run
|
|
- integration_shard_coverage:
|
|
name: integration-extensions-coverage
|
|
suite: extensions
|
|
shards: 8
|
|
requires:
|
|
- integration-extensions
|
|
filters: *integration_default_run
|
|
- integration_contracts:
|
|
name: integration-management-replica
|
|
suite: management
|
|
mode: replica
|
|
parallelism: 4
|
|
resource_class: xlarge
|
|
filters: *integration_default_run
|
|
- integration_shard_coverage:
|
|
name: integration-management-replica-coverage
|
|
suite: management
|
|
mode: replica
|
|
shards: 4
|
|
requires: [integration-management-replica]
|
|
filters: *integration_default_run
|
|
- integration_contracts:
|
|
name: integration-database-replica
|
|
suite: database
|
|
mode: replica
|
|
parallelism: 2
|
|
filters: *integration_default_run
|
|
- integration_shard_coverage:
|
|
name: integration-database-replica-coverage
|
|
suite: database
|
|
mode: replica
|
|
shards: 2
|
|
requires: [integration-database-replica]
|
|
filters: *integration_default_run
|
|
- postgres_suite:
|
|
name: roi-database
|
|
test_path: tests/integration/database/test_roi_observed.py
|
|
seed: false
|
|
coverage_flag: roi-postgres
|
|
timeout_minutes: 10
|
|
filters: *integration_default_run
|
|
- windows_release_wheel:
|
|
filters: *integration_default_run
|
|
- base_sdk_install:
|
|
filters: *integration_default_run
|
|
- build_docker_database_image:
|
|
filters: *integration_default_run
|
|
- proxy_logging_guardrails_model_info_tests:
|
|
requires:
|
|
- build_docker_database_image
|
|
filters: *integration_default_run
|
|
- db_migration_disable_update_check:
|
|
requires:
|
|
- build_docker_database_image
|
|
filters: *integration_default_run
|
|
- test_bad_database_url:
|
|
requires:
|
|
- build_docker_database_image
|
|
filters: *integration_default_run
|
|
- helm_chart_testing:
|
|
requires:
|
|
- build_docker_database_image
|
|
filters: *integration_default_run
|
|
- e2e_ui_build:
|
|
filters: *integration_default_run
|
|
- e2e_ui_testing:
|
|
requires: [e2e_ui_build]
|
|
filters: *integration_default_run
|
|
- e2e_ui_testing_server_root_path:
|
|
requires: [e2e_ui_build]
|
|
filters: *integration_default_run
|
|
- build_docker_database_image:
|
|
name: build_docker_database_image-migration
|
|
migration_qualification: true
|
|
filters: pipeline.parameters.run_migration_tests
|
|
- migration_startup_tests:
|
|
name: migration-startup
|
|
suite: startup
|
|
requires: [build_docker_database_image-migration]
|
|
filters: pipeline.parameters.run_migration_tests
|
|
- migration_startup_tests:
|
|
name: migration-recovery
|
|
suite: recovery
|
|
requires: [build_docker_database_image-migration]
|
|
filters: pipeline.parameters.run_migration_tests
|
|
- migration_startup_tests:
|
|
name: migration-legacy-and-pooling
|
|
suite: legacy
|
|
requires: [build_docker_database_image-migration]
|
|
filters: pipeline.parameters.run_migration_tests
|
|
- migration_startup_tests:
|
|
name: migration-upgrade
|
|
suite: upgrade
|
|
baseline: true
|
|
requires: [build_docker_database_image-migration]
|
|
filters: pipeline.parameters.run_migration_tests
|
|
- migration_startup_tests:
|
|
name: migration-upgrade-shaped
|
|
suite: shaped
|
|
baseline: true
|
|
requires: [build_docker_database_image-migration]
|
|
filters: pipeline.parameters.run_migration_tests
|
|
- routing_parity:
|
|
name: routing-parity-<< matrix.suite >>
|
|
matrix:
|
|
parameters:
|
|
suite: [management, accounting, database, providers, extensions, cost, mcp]
|
|
filters: pipeline.parameters.routing_parity_base != ""
|