mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-07 02:59:05 +00:00
fdaa288607
* ci(circleci): enable Rerun Failed Tests for all pytest suites Migrated every pytest-based CircleCI job that uploads JUnit results to use 'circleci tests run' instead of invoking pytest directly. This is the prerequisite for CircleCI's 'Rerun failed tests' feature to be available on each job in the pipeline. For each job: - Glob test files via 'circleci tests glob' and pipe them into 'circleci tests run --command="xargs ... pytest ..."' so the agent can feed the failed-test subset on rerun. - Preserve all original pytest flags (parallelism, timeouts, retries, coverage, junit output paths). - For jobs that previously lacked 'store_test_results' (proxy spend accuracy, proxy_build_from_pip, db_migration_disable_update_check), add the step so JUnit XML is uploaded and rerun is actually wired up. - Replace the dynamic IGNORE_DIRS shell array in llm_translation_testing with a 'grep -v' filter on the glob output, matching the previous behavior of skipping tests/llm_translation/realtime. - For 'build_and_test', glob 'tests/test_*.py' (top-level only) which matches the prior 'tests/*.py' shell glob; the long list of '--ignore=tests/<subdir>' flags was vestigial and is dropped. Jobs already using 'circleci tests run' (local_testing_part1/2, litellm_router_testing) are unchanged. * fix(ci): convert classnames to file paths on rerun CircleCI's Rerun Failed Tests sends each previously failed test as a JUnit classname (e.g. 'tests.otel_tests.test_key_logging_callbacks'), but pytest needs a file path. Without the awk preprocess step, rerun runs fail with 'file or directory not found'. Mirror the awk transform that local_testing_part1, local_testing_part2, and litellm_router_testing already use, so rerun works in every job that this PR migrated to 'circleci tests run'. * ci: drop -x from OTEL pytest run so all failures are reported --------- Co-authored-by: Cursor Agent <cursoragent@cursor.com>
2689 lines
97 KiB
YAML
2689 lines
97 KiB
YAML
version: 2.1
|
|
orbs:
|
|
codecov: codecov/codecov@4.0.1
|
|
node: circleci/node@5.1.0 # Add this line to declare the node orb
|
|
win: circleci/windows@5.0 # Add Windows orb
|
|
|
|
commands:
|
|
setup_google_dns:
|
|
steps:
|
|
- run:
|
|
name: "Configure Google DNS"
|
|
command: |
|
|
# Backup original resolv.conf
|
|
sudo cp /etc/resolv.conf /etc/resolv.conf.backup
|
|
# Add both local and Google DNS servers
|
|
echo "nameserver 127.0.0.11" | sudo tee /etc/resolv.conf
|
|
echo "nameserver 8.8.8.8" | sudo tee -a /etc/resolv.conf
|
|
echo "nameserver 8.8.4.4" | sudo tee -a /etc/resolv.conf
|
|
wait_for_service:
|
|
description: "Poll a TCP or HTTP endpoint until it responds (replaces dockerize -wait)"
|
|
parameters:
|
|
url:
|
|
type: string
|
|
timeout:
|
|
type: string
|
|
default: "60"
|
|
steps:
|
|
- run:
|
|
name: "Wait for << parameters.url >>"
|
|
command: |
|
|
TIMEOUT=<< parameters.timeout >>
|
|
URL="<< parameters.url >>"
|
|
ELAPSED=0
|
|
echo "Waiting up to ${TIMEOUT}s for ${URL} ..."
|
|
if echo "$URL" | grep -q '^tcp://'; then
|
|
HOST=$(echo "$URL" | sed 's|tcp://||' | cut -d: -f1)
|
|
PORT=$(echo "$URL" | sed 's|tcp://||' | cut -d: -f2)
|
|
while ! bash -c "echo > /dev/tcp/$HOST/$PORT" 2>/dev/null; do
|
|
sleep 2; ELAPSED=$((ELAPSED+2))
|
|
if [ "$ELAPSED" -ge "$TIMEOUT" ]; then echo "Timed out"; exit 1; fi
|
|
done
|
|
else
|
|
while ! curl -sf --max-time 5 "$URL" > /dev/null 2>&1; do
|
|
sleep 2; ELAPSED=$((ELAPSED+2))
|
|
if [ "$ELAPSED" -ge "$TIMEOUT" ]; then echo "Timed out"; exit 1; fi
|
|
done
|
|
fi
|
|
echo "Service ready after ${ELAPSED}s"
|
|
install_helm:
|
|
steps:
|
|
- run:
|
|
name: Install Helm v3.17.3
|
|
command: |
|
|
curl -sSLf -o /tmp/helm.tar.gz \
|
|
https://get.helm.sh/helm-v3.17.3-linux-amd64.tar.gz
|
|
echo "ee88b3c851ae6466a3de507f7be73fe94d54cbf2987cbaa3d1a3832ea331f2cd /tmp/helm.tar.gz" | sha256sum -c -
|
|
sudo tar -C /usr/local/bin --strip-components=1 -xzf /tmp/helm.tar.gz linux-amd64/helm
|
|
rm -f /tmp/helm.tar.gz
|
|
install_kind:
|
|
steps:
|
|
- run:
|
|
name: Install Kind v0.20.0
|
|
command: |
|
|
curl -sSLf -o /tmp/kind \
|
|
https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64
|
|
echo "513a7213d6d3332dd9ef27c24dab35e5ef10a04fa27274fe1c14d8a246493ded /tmp/kind" | sha256sum -c -
|
|
chmod +x /tmp/kind
|
|
sudo mv /tmp/kind /usr/local/bin/kind
|
|
install_uv:
|
|
description: "Install pinned uv (0.10.9) with checksum verification. Adds ~/.local/bin to PATH."
|
|
steps:
|
|
- run:
|
|
name: Install uv (pinned 0.10.9)
|
|
command: |
|
|
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
|
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
|
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
|
rm -f /tmp/uv-install.sh
|
|
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
start_postgres:
|
|
description: "Start a postgres-db container on port 5432 and wait until it accepts connections."
|
|
parameters:
|
|
db_name:
|
|
type: string
|
|
default: circle_test
|
|
steps:
|
|
- run:
|
|
name: Start PostgreSQL
|
|
command: |
|
|
docker run -d \
|
|
--name postgres-db \
|
|
-e POSTGRES_USER=postgres \
|
|
-e POSTGRES_PASSWORD=postgres \
|
|
-e POSTGRES_DB=<< parameters.db_name >> \
|
|
-p 5432:5432 \
|
|
postgres:14@sha256:6a70deda415ec296f977890e11aba04a0db9f632a362e3fce45e845e3db74f26
|
|
- wait_for_service:
|
|
url: tcp://localhost:5432
|
|
timeout: "60"
|
|
start_redis:
|
|
description: "Start a redis container on port 6379 and wait until it accepts connections. Use this to isolate a job from the shared remote Redis so concurrent CI pipelines don't contend for pod locks or buffer keys."
|
|
steps:
|
|
- run:
|
|
name: Start Redis
|
|
command: |
|
|
docker run -d \
|
|
--name redis-cache \
|
|
-p 6379:6379 \
|
|
redis:7-alpine@sha256:7aec734b2bb298a1d769fd8729f13b8514a41bf90fcdd1f38ec52267fbaa8ee6
|
|
- wait_for_service:
|
|
url: tcp://localhost:6379
|
|
timeout: "60"
|
|
setup_litellm_enterprise_pip:
|
|
steps:
|
|
- run:
|
|
name: "Install local version of litellm-enterprise"
|
|
command: |
|
|
# litellm-enterprise is a uv workspace member and is already installed
|
|
# by the main `uv sync --all-groups --all-extras`. Do NOT run
|
|
# `uv sync --package litellm-enterprise` here — that overwrites the
|
|
# shared .venv and strips out dev/test deps (pytest, prisma, etc.).
|
|
uv run --no-sync python -c "import litellm_enterprise; print('litellm-enterprise OK:', litellm_enterprise.__file__)"
|
|
setup_litellm_test_deps:
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- restore_cache:
|
|
keys:
|
|
- v1-uv-cache-{{ checksum "uv.lock" }}
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- setup_litellm_enterprise_pip
|
|
- save_cache:
|
|
paths:
|
|
- ~/.cache/uv
|
|
key: v1-uv-cache-{{ checksum "uv.lock" }}
|
|
|
|
jobs:
|
|
# Add Windows testing job
|
|
using_litellm_on_windows:
|
|
executor:
|
|
name: win/default
|
|
shell: powershell.exe
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- run:
|
|
name: Install Python
|
|
command: |
|
|
choco install python --version=3.11.0 -y --no-progress --force
|
|
refreshenv
|
|
python --version
|
|
environment:
|
|
CHOCOLATEY_CONFIRM_ALL: "true"
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
$installer = Join-Path $env:TEMP "uv-install.ps1"
|
|
Invoke-WebRequest -Uri https://astral.sh/uv/0.10.9/install.ps1 -OutFile $installer
|
|
$expected = "d43ffff8d28e7d1e7d1831a212465f12b24a43c7f87f386e95e2a5915aee5d7d"
|
|
$actual = (Get-FileHash -Path $installer -Algorithm SHA256).Hash.ToLower()
|
|
if ($actual -ne $expected) {
|
|
throw "uv installer hash mismatch: expected $expected got $actual"
|
|
}
|
|
& $installer
|
|
Remove-Item $installer
|
|
$uvBin = Join-Path $HOME ".local\bin"
|
|
$env:Path = "$uvBin;$env:Path"
|
|
if (!(Test-Path $PROFILE)) {
|
|
New-Item -ItemType File -Force -Path $PROFILE | Out-Null
|
|
}
|
|
if (-not (Select-String -Path $PROFILE -SimpleMatch $uvBin -Quiet)) {
|
|
Add-Content -Path $PROFILE -Value "`$env:Path = `"$uvBin;`$env:Path`""
|
|
}
|
|
uv sync --frozen --group dev --python (Get-Command python).Source
|
|
- run:
|
|
name: Run Windows-specific test
|
|
command: |
|
|
uv run --no-sync python -m pytest tests/windows_tests/test_litellm_on_windows.py -v
|
|
|
|
local_testing_part1:
|
|
docker:
|
|
- &python312_image
|
|
image: cimg/python:3.12@sha256:9c796c23c84e84a66a964acb508d39dc5433c81a47e07efd56dccbbc2427e07c
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project
|
|
parallelism: 4
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- restore_cache:
|
|
keys:
|
|
- v1-uv-cache-{{ checksum "uv.lock" }}
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- setup_litellm_enterprise_pip
|
|
- save_cache:
|
|
paths:
|
|
- ~/.cache/uv
|
|
key: v1-uv-cache-{{ checksum "uv.lock" }}
|
|
- run:
|
|
name: Run prisma ./docker/entrypoint.sh
|
|
command: |
|
|
set +e
|
|
chmod +x docker/entrypoint.sh
|
|
./docker/entrypoint.sh
|
|
set -e
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests (Part 1 - A-M)
|
|
command: |
|
|
mkdir test-results
|
|
|
|
# Discover test files (A-M)
|
|
TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_[a-mA-M]*.py")
|
|
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--split-by=timings \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-vv \
|
|
--cov=litellm \
|
|
--cov-report=xml \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=20 \
|
|
-k \"not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache\" \
|
|
-n 4 \
|
|
--timeout=300 \
|
|
--timeout_method=thread"
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml local_testing_part1_coverage.xml
|
|
mv .coverage local_testing_part1_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- local_testing_part1_coverage.xml
|
|
- local_testing_part1_coverage
|
|
local_testing_part2:
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
parallelism: 4
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- restore_cache:
|
|
keys:
|
|
- v1-uv-cache-{{ checksum "uv.lock" }}
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- setup_litellm_enterprise_pip
|
|
- save_cache:
|
|
paths:
|
|
- ~/.cache/uv
|
|
key: v1-uv-cache-{{ checksum "uv.lock" }}
|
|
- run:
|
|
name: Run prisma ./docker/entrypoint.sh
|
|
command: |
|
|
set +e
|
|
chmod +x docker/entrypoint.sh
|
|
./docker/entrypoint.sh
|
|
set -e
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests (Part 2 - N-Z)
|
|
command: |
|
|
mkdir test-results
|
|
|
|
# Discover test files (N-Z)
|
|
TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_[n-zN-Z]*.py")
|
|
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--split-by=timings \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-vv \
|
|
--cov=litellm \
|
|
--cov-report=xml \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=20 \
|
|
-k \"not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache\" \
|
|
-n 4 \
|
|
--timeout=300 \
|
|
--timeout_method=thread"
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml local_testing_part2_coverage.xml
|
|
mv .coverage local_testing_part2_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- local_testing_part2_coverage.xml
|
|
- local_testing_part2_coverage
|
|
langfuse_logging_unit_tests:
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
resource_class: medium
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- restore_cache:
|
|
keys:
|
|
- v1-uv-cache-{{ checksum "uv.lock" }}
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- setup_litellm_enterprise_pip
|
|
- save_cache:
|
|
paths:
|
|
- ~/.cache/uv
|
|
key: v1-uv-cache-{{ checksum "uv.lock" }}
|
|
- run:
|
|
name: Run prisma ./docker/entrypoint.sh
|
|
command: |
|
|
set +e
|
|
chmod +x docker/entrypoint.sh
|
|
./docker/entrypoint.sh
|
|
set -e
|
|
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-v -x \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5 \
|
|
-k \"langfuse\""
|
|
no_output_timeout: 15m
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
auth_ui_unit_tests:
|
|
docker:
|
|
- *python312_image
|
|
- image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
|
|
environment:
|
|
POSTGRES_USER: postgres
|
|
POSTGRES_PASSWORD: postgres
|
|
POSTGRES_DB: litellm_test
|
|
working_directory: ~/project
|
|
environment:
|
|
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- save_cache:
|
|
paths:
|
|
- ~/.cache/uv
|
|
key: v1-uv-cache-{{ checksum "uv.lock" }}
|
|
- wait_for_service:
|
|
url: tcp://localhost:5432
|
|
timeout: "60"
|
|
- run:
|
|
name: Seed DB schema via prisma db push
|
|
command: |
|
|
set +e
|
|
uv run --no-sync litellm --skip_server_startup --use_prisma_db_push
|
|
set -e
|
|
- run:
|
|
name: Generate Prisma Client
|
|
command: uv run --no-sync python -m prisma generate
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/proxy_admin_ui_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-v -x \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5 \
|
|
-n 2"
|
|
no_output_timeout: 15m
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
|
|
litellm_router_testing: # Runs all tests with the "router" keyword
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
resource_class: large
|
|
parallelism: 4
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- restore_cache:
|
|
keys:
|
|
- v1-uv-cache-{{ checksum "uv.lock" }}
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- save_cache:
|
|
paths:
|
|
- ~/.cache/uv
|
|
key: v1-uv-cache-{{ checksum "uv.lock" }}
|
|
# Run pytest and generate JUnit XML report
|
|
- setup_litellm_enterprise_pip
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py")
|
|
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--split-by=timings \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-v \
|
|
-k 'router' \
|
|
-n 4 \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5 \
|
|
--timeout=300 --timeout_method=thread"
|
|
no_output_timeout: 15m
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
|
|
litellm_router_unit_testing: # Runs all tests with the "router" keyword
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
resource_class: large
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- restore_cache:
|
|
keys:
|
|
- v1-uv-cache-{{ checksum "uv.lock" }}
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- save_cache:
|
|
paths:
|
|
- ~/.cache/uv
|
|
key: v1-uv-cache-{{ checksum "uv.lock" }}
|
|
# Run pytest and generate JUnit XML report
|
|
- setup_litellm_enterprise_pip
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/router_unit_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-v -x \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5 \
|
|
-n 4"
|
|
no_output_timeout: 15m
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
litellm_assistants_api_testing: # Runs all tests with the "assistants" keyword
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
resource_class: medium
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
# Run pytest and generate JUnit XML report
|
|
- setup_litellm_enterprise_pip
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-v -x \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5 \
|
|
-k \"assistants\""
|
|
no_output_timeout: 15m
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
llm_translation_testing:
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
resource_class: xlarge
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- restore_cache:
|
|
keys:
|
|
- v1-uv-cache-{{ checksum "uv.lock" }}
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- save_cache:
|
|
paths:
|
|
- ~/.cache/uv
|
|
key: v1-uv-cache-{{ checksum "uv.lock" }}
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
# Add --timeout to kill hanging tests after 120s (2 min)
|
|
# Add --durations=20 to show 20 slowest tests for debugging
|
|
# Subdirectories with dedicated jobs (maintain this list as new jobs are added)
|
|
mkdir -p test-results
|
|
# Glob excludes the realtime/ subdirectory since it has its own job
|
|
TEST_FILES=$(circleci tests glob "tests/llm_translation/**/test_*.py" | grep -v "^tests/llm_translation/realtime/")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-v \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=20 \
|
|
-n 4 \
|
|
--timeout=120 --timeout_method=thread \
|
|
--retries 2 --retry-delay 5 \
|
|
--max-worker-restart=5"
|
|
no_output_timeout: 15m
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
realtime_translation_testing:
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run realtime tests
|
|
command: |
|
|
# Add --timeout to kill hanging tests after 120s (2 min)
|
|
# Add --durations=20 to show 20 slowest tests for debugging
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/llm_translation/realtime/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-vv \
|
|
--cov=litellm --cov-report=xml \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=20 \
|
|
-n 4 \
|
|
--timeout=120 --timeout_method=thread"
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml realtime_translation_coverage.xml
|
|
mv .coverage realtime_translation_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- realtime_translation_coverage.xml
|
|
- realtime_translation_coverage
|
|
agent_testing:
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/agent_tests/**/test_*.py" | grep -v "^tests/agent_tests/local_only_agent_tests/")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-vv -x -s \
|
|
--cov=litellm --cov-report=xml \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5"
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml agent_coverage.xml
|
|
mv .coverage agent_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- agent_coverage.xml
|
|
- agent_coverage
|
|
guardrails_testing:
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
export LITELLM_LOG=WARNING
|
|
TEST_FILES=$(circleci tests glob "tests/guardrails_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-vv \
|
|
--cov=litellm --cov-report=xml \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5 \
|
|
-n 2 \
|
|
--timeout=120 --timeout_method=thread"
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml guardrails_coverage.xml
|
|
mv .coverage guardrails_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- guardrails_coverage.xml
|
|
- guardrails_coverage
|
|
|
|
google_generate_content_endpoint_testing:
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/unified_google_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-vv -x -s \
|
|
--cov=litellm --cov-report=xml \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5 \
|
|
--retries 3 --retry-delay 5"
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml google_generate_content_endpoint_coverage.xml
|
|
mv .coverage google_generate_content_endpoint_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- google_generate_content_endpoint_coverage.xml
|
|
- google_generate_content_endpoint_coverage
|
|
|
|
llm_responses_api_testing:
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
resource_class: large
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- restore_cache:
|
|
keys:
|
|
- v1-uv-cache-{{ checksum "uv.lock" }}
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- save_cache:
|
|
paths:
|
|
- ~/.cache/uv
|
|
key: v1-uv-cache-{{ checksum "uv.lock" }}
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/llm_responses_api_testing/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-v -x \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5 \
|
|
-n 8"
|
|
no_output_timeout: 15m
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
ocr_testing:
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/ocr_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-vv -x \
|
|
--cov=litellm --cov-report=xml \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5 \
|
|
-n 4"
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml ocr_coverage.xml
|
|
mv .coverage ocr_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- ocr_coverage.xml
|
|
- ocr_coverage
|
|
search_testing:
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/search_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-vv -x \
|
|
--cov=litellm --cov-report=xml \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5 \
|
|
-n 4"
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml search_coverage.xml
|
|
mv .coverage search_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- search_coverage.xml
|
|
- search_coverage
|
|
litellm_mapped_enterprise_tests:
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
resource_class: large
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- setup_litellm_enterprise_pip
|
|
- run:
|
|
name: Run enterprise tests
|
|
command: |
|
|
uv run --no-sync python -m prisma generate
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/enterprise/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-v -x \
|
|
--junitxml=test-results/junit-enterprise.xml \
|
|
--durations=10 \
|
|
-n 4"
|
|
no_output_timeout: 15m
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
batches_testing:
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/batches_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-vv -x -s \
|
|
--cov=litellm --cov-report=xml \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5 \
|
|
-n 2"
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml batches_coverage.xml
|
|
mv .coverage batches_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- batches_coverage.xml
|
|
- batches_coverage
|
|
litellm_utils_testing:
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/litellm_utils_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-vv -x -s \
|
|
--cov=litellm --cov-report=xml \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5 \
|
|
-n 2"
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml litellm_utils_coverage.xml
|
|
mv .coverage litellm_utils_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- litellm_utils_coverage.xml
|
|
- litellm_utils_coverage
|
|
|
|
pass_through_unit_testing:
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/pass_through_unit_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-vv -x \
|
|
--cov=litellm --cov-report=xml \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5 \
|
|
-n 4"
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml pass_through_unit_tests_coverage.xml
|
|
mv .coverage pass_through_unit_tests_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- pass_through_unit_tests_coverage.xml
|
|
- pass_through_unit_tests_coverage
|
|
image_gen_testing:
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
resource_class: large
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/image_gen_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-v -x \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5 \
|
|
-n 4"
|
|
no_output_timeout: 15m
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
logging_testing:
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
# Run pytest and generate JUnit XML report
|
|
- setup_litellm_enterprise_pip
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
export LITELLM_LOG=WARNING
|
|
TEST_FILES=$(circleci tests glob "tests/logging_callback_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-vv \
|
|
--cov=litellm --cov-report=xml \
|
|
-n 4 \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5 \
|
|
--timeout=120 --timeout_method=thread"
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml logging_coverage.xml || true
|
|
mv .coverage logging_coverage || true
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- logging_coverage.xml
|
|
- logging_coverage
|
|
audio_testing:
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/audio_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-vv -x -s \
|
|
--cov=litellm --cov-report=xml \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5"
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml audio_coverage.xml
|
|
mv .coverage audio_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- audio_coverage.xml
|
|
- audio_coverage
|
|
redis_caching_unit_tests:
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- restore_cache:
|
|
keys:
|
|
- v1-uv-cache-{{ checksum "uv.lock" }}
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- save_cache:
|
|
paths:
|
|
- ~/.cache/uv
|
|
key: v1-uv-cache-{{ checksum "uv.lock" }}
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(printf "%s\n" \
|
|
tests/local_testing/test_dual_cache.py \
|
|
tests/local_testing/test_redis_batch_optimizations.py \
|
|
tests/local_testing/test_router_utils.py)
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-vv -x -s \
|
|
--cov=litellm --cov-report=xml \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5 -n 2 \
|
|
--reruns 2 --reruns-delay 1"
|
|
no_output_timeout: 20m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml redis_caching_coverage.xml
|
|
mv .coverage redis_caching_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- redis_caching_coverage.xml
|
|
- redis_caching_coverage
|
|
installing_litellm_on_python:
|
|
docker:
|
|
- *python312_image
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- setup_litellm_enterprise_pip
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
uv run --no-sync python -m pytest -vv tests/local_testing/test_basic_python_version.py -k "not v2_resolver"
|
|
|
|
installing_litellm_on_python_3_13:
|
|
docker:
|
|
- image: cimg/python:3.13.1@sha256:87b243ae80d154db75ce5e58af16c72c5dd4b1e23e5c7264a816e85e0c440c13
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project
|
|
resource_class: medium
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.13
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
uv run --no-sync python -m pytest -v tests/local_testing/test_basic_python_version.py -k "not v2_resolver"
|
|
|
|
installing_litellm_on_python_v2_migration_resolver:
|
|
docker:
|
|
- *python312_image
|
|
- image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
|
|
environment:
|
|
POSTGRES_USER: postgres
|
|
POSTGRES_PASSWORD: postgres
|
|
POSTGRES_DB: litellm_test
|
|
working_directory: ~/project
|
|
environment:
|
|
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
|
|
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- setup_litellm_enterprise_pip
|
|
- wait_for_service:
|
|
url: tcp://localhost:5432
|
|
timeout: "60"
|
|
- run:
|
|
name: Run v2 migration resolver proxy smoke test
|
|
command: |
|
|
uv run --no-sync python -m pytest -vv \
|
|
tests/local_testing/test_basic_python_version.py::test_litellm_proxy_server_config_no_general_settings_v2_resolver
|
|
|
|
helm_chart_testing:
|
|
machine:
|
|
image: ubuntu-2204:2024.04.1 # Use machine executor instead of docker
|
|
resource_class: medium
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- attach_workspace:
|
|
at: ~/project
|
|
- setup_google_dns
|
|
- install_helm
|
|
- install_kind
|
|
|
|
# Install kubectl (pinned version with hardcoded checksum)
|
|
- run:
|
|
name: Install kubectl v1.31.4
|
|
command: |
|
|
curl -sSLf -o /tmp/kubectl \
|
|
https://dl.k8s.io/release/v1.31.4/bin/linux/amd64/kubectl
|
|
echo "298e19e9c6c17199011404278f0ff8168a7eca4217edad9097af577023a5620f /tmp/kubectl" | sha256sum -c -
|
|
chmod +x /tmp/kubectl
|
|
sudo mv /tmp/kubectl /usr/local/bin/
|
|
|
|
# Create kind cluster
|
|
- run:
|
|
name: Create Kind Cluster
|
|
command: |
|
|
kind create cluster --name litellm-test
|
|
|
|
- run:
|
|
name: Load Docker Database Image for helm tests
|
|
command: |
|
|
zstd -d litellm-docker-database.tar.zst --stdout | docker load
|
|
IMAGE_TAG=${CIRCLE_SHA1:-ci}
|
|
docker tag litellm-docker-database:ci litellm-ci:${IMAGE_TAG}
|
|
|
|
- run:
|
|
name: Load Docker image into Kind
|
|
command: |
|
|
IMAGE_TAG=${CIRCLE_SHA1:-ci}
|
|
kind load docker-image litellm-ci:${IMAGE_TAG} --name litellm-test
|
|
|
|
# Run helm lint
|
|
- run:
|
|
name: Run helm lint
|
|
command: |
|
|
helm lint ./deploy/charts/litellm-helm
|
|
|
|
# Run helm tests
|
|
- run:
|
|
name: Run helm tests
|
|
command: |
|
|
IMAGE_TAG=${CIRCLE_SHA1:-ci}
|
|
helm install litellm ./deploy/charts/litellm-helm -f ./deploy/charts/litellm-helm/ci/test-values.yaml \
|
|
--set image.repository=litellm-ci \
|
|
--set image.tag=${IMAGE_TAG} \
|
|
--set image.pullPolicy=Never
|
|
# Wait for pod to be ready
|
|
echo "Waiting 30 seconds for pod to be ready..."
|
|
sleep 30
|
|
|
|
# Print pod logs before running tests
|
|
echo "Printing pod logs..."
|
|
kubectl logs $(kubectl get pods -l app.kubernetes.io/name=litellm -o jsonpath="{.items[0].metadata.name}")
|
|
|
|
# Run the helm tests
|
|
helm test litellm --logs
|
|
|
|
# Cleanup
|
|
- run:
|
|
name: Cleanup
|
|
command: |
|
|
kind delete cluster --name litellm-test
|
|
when: always # This ensures cleanup runs even if previous steps fail
|
|
|
|
db_migration_disable_update_check:
|
|
machine:
|
|
image: ubuntu-2204:2024.04.1
|
|
resource_class: medium
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- start_postgres:
|
|
db_name: litellm_test
|
|
- attach_workspace:
|
|
at: ~/project
|
|
- run:
|
|
name: Load Docker Database Image
|
|
command: |
|
|
zstd -d litellm-docker-database.tar.zst --stdout | docker load
|
|
docker images | grep litellm-docker-database
|
|
- run:
|
|
name: Seed database with real schema
|
|
command: |
|
|
docker run -d \
|
|
-p 4001:4000 \
|
|
-e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \
|
|
-e LITELLM_MASTER_KEY="sk-1234" \
|
|
--name schema-seed \
|
|
--add-host=host.docker.internal:host-gateway \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \
|
|
litellm-docker-database:ci \
|
|
--config /app/config.yaml \
|
|
--port 4000 \
|
|
--use_prisma_db_push
|
|
- wait_for_service:
|
|
url: http://localhost:4001
|
|
timeout: "300"
|
|
- run:
|
|
name: Stop schema seed container
|
|
command: docker stop schema-seed && docker rm schema-seed
|
|
- run:
|
|
name: Run Docker container with bad schema and disabled updates
|
|
command: |
|
|
docker run -d \
|
|
-p 4000:4000 \
|
|
-e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \
|
|
-e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \
|
|
-e DISABLE_SCHEMA_UPDATE="True" \
|
|
--name my-app \
|
|
--add-host=host.docker.internal:host-gateway \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/schema.prisma \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/litellm/proxy/schema.prisma \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/disable_schema_update.yaml:/app/config.yaml \
|
|
litellm-docker-database:ci \
|
|
--config /app/config.yaml \
|
|
--port 4000
|
|
- wait_for_service:
|
|
url: http://localhost:4000
|
|
timeout: "60"
|
|
- run:
|
|
name: Check container logs for expected message
|
|
command: |
|
|
echo "=== Printing Full Container Startup Logs ==="
|
|
LOG_OUTPUT="$(docker logs my-app 2>&1)"
|
|
printf '%s\n' "$LOG_OUTPUT"
|
|
echo "=== End of Full Container Startup Logs ==="
|
|
|
|
if printf '%s\n' "$LOG_OUTPUT" | grep -q "prisma schema out of sync with db. Consider running these sql_commands to sync the two"; then
|
|
echo "Expected message found in logs. Test passed."
|
|
else
|
|
echo "Expected message not found in logs. Test failed."
|
|
exit 1
|
|
fi
|
|
- run:
|
|
name: Run Basic Proxy Startup Tests (Health Readiness and Chat Completion)
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-v -x \
|
|
--junitxml=test-results/junit-2.xml \
|
|
--durations=5"
|
|
no_output_timeout: 15m
|
|
- store_test_results:
|
|
path: test-results
|
|
|
|
build_and_test:
|
|
machine:
|
|
image: ubuntu-2204:2024.04.1
|
|
resource_class: large
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- attach_workspace:
|
|
at: ~/project
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- start_postgres
|
|
- run:
|
|
name: Load Docker Database Image
|
|
command: |
|
|
zstd -d litellm-docker-database.tar.zst --stdout | docker load
|
|
docker tag litellm-docker-database:ci my-app:latest
|
|
- run:
|
|
name: Run Docker container
|
|
command: |
|
|
docker run -d \
|
|
-p 4000:4000 \
|
|
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
|
|
-e USE_PRISMA_MIGRATE=True \
|
|
-e AZURE_API_KEY=$AZURE_API_KEY \
|
|
-e REDIS_HOST=$REDIS_HOST \
|
|
-e REDIS_PASSWORD=$REDIS_PASSWORD \
|
|
-e REDIS_PORT=$REDIS_PORT \
|
|
-e AZURE_FRANCE_API_KEY=$AZURE_FRANCE_API_KEY \
|
|
-e AZURE_EUROPE_API_KEY=$AZURE_EUROPE_API_KEY \
|
|
-e MISTRAL_API_KEY=$MISTRAL_API_KEY \
|
|
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
|
|
-e GROQ_API_KEY=$GROQ_API_KEY \
|
|
-e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
|
|
-e COHERE_API_KEY=$COHERE_API_KEY \
|
|
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
|
|
-e AWS_REGION_NAME=$AWS_REGION_NAME \
|
|
-e AUTO_INFER_REGION=True \
|
|
-e OPENAI_API_KEY=$OPENAI_API_KEY \
|
|
-e USE_DDTRACE=True \
|
|
-e DD_API_KEY=$DD_API_KEY \
|
|
-e DD_SITE=$DD_SITE \
|
|
-e LITELLM_LICENSE=$LITELLM_LICENSE \
|
|
-e LANGFUSE_PROJECT1_PUBLIC=$LANGFUSE_PROJECT1_PUBLIC \
|
|
-e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \
|
|
-e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \
|
|
-e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \
|
|
--add-host host.docker.internal:host-gateway \
|
|
--name my-app \
|
|
-v $(pwd)/proxy_server_config.yaml:/app/config.yaml \
|
|
my-app:latest \
|
|
--config /app/config.yaml \
|
|
--port 4000 \
|
|
--detailed_debug \
|
|
- run:
|
|
name: Start outputting logs
|
|
command: docker logs -f my-app
|
|
background: true
|
|
- wait_for_service:
|
|
url: http://localhost:4000
|
|
timeout: "300"
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
# Original used `tests/*.py` (top-level only); the `--ignore=...`
|
|
# flags were vestigial since shell globbing did not descend into
|
|
# subdirectories. Replicate by globbing only top-level test files.
|
|
TEST_FILES=$(circleci tests glob "tests/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-s -v -x \
|
|
--junitxml=test-results/junit.xml \
|
|
-n 4 \
|
|
--durations=5"
|
|
no_output_timeout: 15m
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
e2e_openai_endpoints:
|
|
machine:
|
|
image: ubuntu-2204:2024.04.1
|
|
resource_class: large
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- start_postgres
|
|
- attach_workspace:
|
|
at: ~/project
|
|
- run:
|
|
name: Load Docker Database Image
|
|
command: |
|
|
zstd -d litellm-docker-database.tar.zst --stdout | docker load
|
|
docker images | grep litellm-docker-database
|
|
- run:
|
|
name: Run Docker container
|
|
command: |
|
|
docker run -d \
|
|
-p 4000:4000 \
|
|
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
|
|
-e AZURE_API_KEY=$AZURE_API_KEY \
|
|
-e AZURE_API_BASE=$AZURE_API_BASE \
|
|
-e AZURE_API_VERSION="2024-05-01-preview" \
|
|
-e REDIS_HOST=$REDIS_HOST \
|
|
-e REDIS_PASSWORD=$REDIS_PASSWORD \
|
|
-e REDIS_PORT=$REDIS_PORT \
|
|
-e AZURE_FRANCE_API_KEY=$AZURE_FRANCE_API_KEY \
|
|
-e AZURE_EUROPE_API_KEY=$AZURE_EUROPE_API_KEY \
|
|
-e MISTRAL_API_KEY=$MISTRAL_API_KEY \
|
|
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
|
|
-e GROQ_API_KEY=$GROQ_API_KEY \
|
|
-e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
|
|
-e COHERE_API_KEY=$COHERE_API_KEY \
|
|
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
|
|
-e AWS_REGION_NAME=$AWS_REGION_NAME \
|
|
-e AUTO_INFER_REGION=True \
|
|
-e USE_DDTRACE=True \
|
|
-e DD_API_KEY=$DD_API_KEY \
|
|
-e DD_SITE=$DD_SITE \
|
|
-e OPENAI_API_KEY=$OPENAI_API_KEY \
|
|
-e LITELLM_LICENSE=$LITELLM_LICENSE \
|
|
-e LANGFUSE_PROJECT1_PUBLIC=$LANGFUSE_PROJECT1_PUBLIC \
|
|
-e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \
|
|
-e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \
|
|
-e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \
|
|
--add-host host.docker.internal:host-gateway \
|
|
--name my-app \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/oai_misc_config.yaml:/app/config.yaml \
|
|
litellm-docker-database:ci \
|
|
--config /app/config.yaml \
|
|
--port 4000 \
|
|
--detailed_debug \
|
|
- run:
|
|
name: Start outputting logs
|
|
command: docker logs -f my-app
|
|
background: true
|
|
- wait_for_service:
|
|
url: http://localhost:4000
|
|
timeout: "300"
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/openai_endpoints_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-s -vv \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5"
|
|
no_output_timeout: 15m
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
proxy_logging_guardrails_model_info_tests:
|
|
machine:
|
|
image: ubuntu-2204:2024.04.1
|
|
resource_class: large
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- start_postgres
|
|
- attach_workspace:
|
|
at: ~/project
|
|
- run:
|
|
name: Load Docker Database Image
|
|
command: |
|
|
zstd -d litellm-docker-database.tar.zst --stdout | docker load
|
|
docker images | grep litellm-docker-database
|
|
- run:
|
|
name: Run Docker container
|
|
# intentionally give bad redis credentials here
|
|
# the OTEL test - should get this as a trace
|
|
command: |
|
|
docker run -d \
|
|
-p 4000:4000 \
|
|
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
|
|
-e REDIS_HOST=$REDIS_HOST \
|
|
-e REDIS_PASSWORD=$REDIS_PASSWORD \
|
|
-e REDIS_PORT=$REDIS_PORT \
|
|
-e LITELLM_MASTER_KEY="sk-1234" \
|
|
-e OPENAI_API_KEY=$OPENAI_API_KEY \
|
|
-e LITELLM_LICENSE=$LITELLM_LICENSE \
|
|
-e OTEL_EXPORTER="in_memory" \
|
|
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
|
|
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
|
|
-e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \
|
|
-e USE_DDTRACE=True \
|
|
-e DD_API_KEY=$DD_API_KEY \
|
|
-e DD_SITE=$DD_SITE \
|
|
-e AWS_REGION_NAME=$AWS_REGION_NAME \
|
|
-e COHERE_API_KEY=$COHERE_API_KEY \
|
|
-e GCS_FLUSH_INTERVAL="1" \
|
|
--add-host host.docker.internal:host-gateway \
|
|
--name my-app \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/otel_test_config.yaml:/app/config.yaml \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/custom_guardrail.py:/app/custom_guardrail.py \
|
|
litellm-docker-database:ci \
|
|
--config /app/config.yaml \
|
|
--port 4000 \
|
|
--detailed_debug \
|
|
- run:
|
|
name: Start outputting logs
|
|
command: docker logs -f my-app
|
|
background: true
|
|
- wait_for_service:
|
|
url: http://localhost:4000
|
|
timeout: "300"
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/otel_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-v \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5"
|
|
no_output_timeout: 15m
|
|
# Clean up first container
|
|
- run:
|
|
name: Stop and remove first container
|
|
command: |
|
|
docker stop my-app
|
|
docker rm my-app
|
|
|
|
# Second Docker Container Run with Different Config
|
|
# NOTE: We intentionally pass a "bad" license here. We need to ensure proxy starts and serves request even with bad license
|
|
- run:
|
|
name: Run Second Docker container
|
|
command: |
|
|
docker run -d \
|
|
-p 4000:4000 \
|
|
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
|
|
-e REDIS_HOST=$REDIS_HOST \
|
|
-e REDIS_PASSWORD=$REDIS_PASSWORD \
|
|
-e REDIS_PORT=$REDIS_PORT \
|
|
-e LITELLM_MASTER_KEY="sk-1234" \
|
|
-e OPENAI_API_KEY=$OPENAI_API_KEY \
|
|
-e LITELLM_LICENSE="bad-license" \
|
|
--add-host host.docker.internal:host-gateway \
|
|
--name my-app-3 \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/enterprise_config.yaml:/app/config.yaml \
|
|
litellm-docker-database:ci \
|
|
--config /app/config.yaml \
|
|
--port 4000 \
|
|
--detailed_debug
|
|
|
|
- run:
|
|
name: Start outputting logs for second container
|
|
command: docker logs -f my-app-3
|
|
background: true
|
|
|
|
- wait_for_service:
|
|
url: http://localhost:4000
|
|
timeout: "300"
|
|
|
|
- run:
|
|
name: Run second round of tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-v -x \
|
|
--junitxml=test-results/junit-2.xml \
|
|
--durations=5"
|
|
no_output_timeout: 15m
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
proxy_spend_accuracy_tests:
|
|
machine:
|
|
image: ubuntu-2204:2024.04.1
|
|
resource_class: large
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- start_postgres
|
|
- start_redis
|
|
- attach_workspace:
|
|
at: ~/project
|
|
- run:
|
|
name: Load Docker Database Image
|
|
command: |
|
|
zstd -d litellm-docker-database.tar.zst --stdout | docker load
|
|
docker images | grep litellm-docker-database
|
|
- run:
|
|
name: Run Docker container
|
|
# Point the proxy at the job-local Redis (start_redis) instead of the
|
|
# shared remote Redis. The Redis transaction buffer uses a single
|
|
# global pod-lock key (cronjob_lock:db_spend_update_job) and a single
|
|
# global buffer list (litellm_spend_update_buffer); sharing those
|
|
# across concurrent CI pipelines causes spend flushes to stall or
|
|
# land in the wrong DB, which is what makes this test flaky.
|
|
command: |
|
|
docker run -d \
|
|
-p 4000:4000 \
|
|
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
|
|
-e REDIS_HOST=host.docker.internal \
|
|
-e REDIS_PORT=6379 \
|
|
-e LITELLM_MASTER_KEY="sk-1234" \
|
|
-e OPENAI_API_KEY=$OPENAI_API_KEY \
|
|
-e LITELLM_LICENSE=$LITELLM_LICENSE \
|
|
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
|
|
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
|
|
-e USE_DDTRACE=True \
|
|
-e DD_API_KEY=$DD_API_KEY \
|
|
-e DD_SITE=$DD_SITE \
|
|
-e AWS_REGION_NAME=$AWS_REGION_NAME \
|
|
-e PROXY_BATCH_WRITE_AT=2 \
|
|
--add-host host.docker.internal:host-gateway \
|
|
--name my-app \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/spend_tracking_config.yaml:/app/config.yaml \
|
|
litellm-docker-database:ci \
|
|
--config /app/config.yaml \
|
|
--port 4000 \
|
|
--detailed_debug \
|
|
- run:
|
|
name: Start outputting logs
|
|
command: docker logs -f my-app
|
|
background: true
|
|
- wait_for_service:
|
|
url: http://localhost:4000
|
|
timeout: "300"
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/spend_tracking_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-vv -x \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5"
|
|
no_output_timeout: 15m
|
|
- store_test_results:
|
|
path: test-results
|
|
- run:
|
|
name: Stop and remove first container
|
|
when: always
|
|
command: |
|
|
docker stop my-app
|
|
docker rm my-app
|
|
docker stop redis-cache
|
|
docker rm redis-cache
|
|
|
|
proxy_multi_instance_tests:
|
|
machine:
|
|
image: ubuntu-2204:2024.04.1
|
|
resource_class: large
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- start_postgres
|
|
- attach_workspace:
|
|
at: ~/project
|
|
- run:
|
|
name: Load Docker Database Image
|
|
command: |
|
|
zstd -d litellm-docker-database.tar.zst --stdout | docker load
|
|
docker images | grep litellm-docker-database
|
|
- run:
|
|
name: Run Docker container 1
|
|
# intentionally give bad redis credentials here
|
|
# the OTEL test - should get this as a trace
|
|
command: |
|
|
docker run -d \
|
|
-p 4000:4000 \
|
|
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
|
|
-e REDIS_HOST=$REDIS_HOST \
|
|
-e REDIS_PASSWORD=$REDIS_PASSWORD \
|
|
-e REDIS_PORT=$REDIS_PORT \
|
|
-e LITELLM_MASTER_KEY="sk-1234" \
|
|
-e LITELLM_LICENSE=$LITELLM_LICENSE \
|
|
-e USE_DDTRACE=True \
|
|
-e DD_API_KEY=$DD_API_KEY \
|
|
-e DD_SITE=$DD_SITE \
|
|
--add-host host.docker.internal:host-gateway \
|
|
--name my-app \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml:/app/config.yaml \
|
|
litellm-docker-database:ci \
|
|
--config /app/config.yaml \
|
|
--port 4000 \
|
|
--detailed_debug \
|
|
- run:
|
|
name: Run Docker container 2
|
|
command: |
|
|
docker run -d \
|
|
-p 4001:4001 \
|
|
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
|
|
-e REDIS_HOST=$REDIS_HOST \
|
|
-e REDIS_PASSWORD=$REDIS_PASSWORD \
|
|
-e REDIS_PORT=$REDIS_PORT \
|
|
-e LITELLM_MASTER_KEY="sk-1234" \
|
|
-e LITELLM_LICENSE=$LITELLM_LICENSE \
|
|
-e USE_DDTRACE=True \
|
|
-e DD_API_KEY=$DD_API_KEY \
|
|
-e DD_SITE=$DD_SITE \
|
|
--add-host host.docker.internal:host-gateway \
|
|
--name my-app-2 \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml:/app/config.yaml \
|
|
litellm-docker-database:ci \
|
|
--config /app/config.yaml \
|
|
--port 4001 \
|
|
--detailed_debug
|
|
- run:
|
|
name: Start outputting logs
|
|
command: docker logs -f my-app
|
|
background: true
|
|
- wait_for_service:
|
|
url: http://localhost:4000
|
|
timeout: "300"
|
|
- wait_for_service:
|
|
url: http://localhost:4001
|
|
timeout: "300"
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/multi_instance_e2e_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-vv -x \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5"
|
|
no_output_timeout: 15m
|
|
# Clean up first container
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
|
|
proxy_store_model_in_db_tests:
|
|
machine:
|
|
image: ubuntu-2204:2024.04.1
|
|
resource_class: large
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- start_postgres
|
|
- attach_workspace:
|
|
at: ~/project
|
|
- run:
|
|
name: Load Docker Database Image
|
|
command: |
|
|
zstd -d litellm-docker-database.tar.zst --stdout | docker load
|
|
docker images | grep litellm-docker-database
|
|
- run:
|
|
name: Run Docker container
|
|
# intentionally give bad redis credentials here
|
|
# the OTEL test - should get this as a trace
|
|
command: |
|
|
docker run -d \
|
|
-p 4000:4000 \
|
|
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
|
|
-e STORE_MODEL_IN_DB="True" \
|
|
-e LITELLM_MASTER_KEY="sk-1234" \
|
|
-e LITELLM_LICENSE=$LITELLM_LICENSE \
|
|
--add-host host.docker.internal:host-gateway \
|
|
--name my-app \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/store_model_db_config.yaml:/app/config.yaml \
|
|
litellm-docker-database:ci \
|
|
--config /app/config.yaml \
|
|
--port 4000 \
|
|
--detailed_debug \
|
|
- run:
|
|
name: Start outputting logs
|
|
command: docker logs -f my-app
|
|
background: true
|
|
- wait_for_service:
|
|
url: http://localhost:4000
|
|
timeout: "300"
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/store_model_in_db_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-vv -x \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5"
|
|
no_output_timeout: 15m
|
|
- run:
|
|
name: Stop and remove containers
|
|
command: |
|
|
docker stop my-app || true
|
|
docker rm my-app || true
|
|
docker stop postgres-db || true
|
|
docker rm postgres-db || true
|
|
when: always
|
|
- store_test_results:
|
|
path: test-results
|
|
|
|
proxy_build_from_pip_tests:
|
|
# Change from docker to machine executor
|
|
machine:
|
|
image: ubuntu-2204:2024.04.1
|
|
resource_class: large
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
# Remove Docker CLI installation since it's already available in machine executor
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- run:
|
|
name: Build Docker image
|
|
command: |
|
|
docker build -t my-app:latest -f docker/build_from_pip/Dockerfile.build_from_pip .
|
|
- start_postgres
|
|
- run:
|
|
name: Run Docker container
|
|
# intentionally give bad redis credentials here
|
|
# the OTEL test - should get this as a trace
|
|
command: |
|
|
docker run -d \
|
|
-p 4000:4000 \
|
|
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
|
|
-e REDIS_HOST=$REDIS_HOST \
|
|
-e REDIS_PASSWORD=$REDIS_PASSWORD \
|
|
-e REDIS_PORT=$REDIS_PORT \
|
|
-e LITELLM_MASTER_KEY="sk-1234" \
|
|
-e OPENAI_API_KEY=$OPENAI_API_KEY \
|
|
-e LITELLM_LICENSE=$LITELLM_LICENSE \
|
|
-e OTEL_EXPORTER="in_memory" \
|
|
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
|
|
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
|
|
-e AWS_REGION_NAME=$AWS_REGION_NAME \
|
|
-e COHERE_API_KEY=$COHERE_API_KEY \
|
|
-e USE_DDTRACE=True \
|
|
-e DD_API_KEY=$DD_API_KEY \
|
|
-e DD_SITE=$DD_SITE \
|
|
-e GCS_FLUSH_INTERVAL="1" \
|
|
--add-host host.docker.internal:host-gateway \
|
|
--name my-app \
|
|
-v $(pwd)/docker/build_from_pip/litellm_config.yaml:/app/config.yaml \
|
|
my-app:latest \
|
|
--config /app/config.yaml \
|
|
--port 4000 \
|
|
--detailed_debug \
|
|
- run:
|
|
name: Start outputting logs
|
|
command: docker logs -f my-app
|
|
background: true
|
|
- wait_for_service:
|
|
url: http://localhost:4000
|
|
timeout: "300"
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-vv -x \
|
|
--junitxml=test-results/junit-2.xml \
|
|
--durations=5"
|
|
no_output_timeout: 15m
|
|
# Clean up first container
|
|
- store_test_results:
|
|
path: test-results
|
|
- run:
|
|
name: Stop and remove first container
|
|
command: |
|
|
docker stop my-app || true
|
|
docker rm my-app || true
|
|
docker stop postgres-db || true
|
|
docker rm postgres-db || true
|
|
when: always
|
|
proxy_pass_through_endpoint_tests:
|
|
machine:
|
|
image: ubuntu-2204:2024.04.1
|
|
resource_class: large
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- start_postgres
|
|
- attach_workspace:
|
|
at: ~/project
|
|
- run:
|
|
name: Load Docker Database Image
|
|
command: |
|
|
zstd -d litellm-docker-database.tar.zst --stdout | docker load
|
|
docker images | grep litellm-docker-database
|
|
- run:
|
|
name: Run Docker container
|
|
command: |
|
|
docker run -d \
|
|
-p 4000:4000 \
|
|
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
|
|
-e LITELLM_MASTER_KEY="sk-1234" \
|
|
-e OPENAI_API_KEY=$OPENAI_API_KEY \
|
|
-e GEMINI_API_KEY=$GEMINI_API_KEY \
|
|
-e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
|
|
-e ASSEMBLYAI_API_KEY=$ASSEMBLYAI_API_KEY \
|
|
-e AZURE_API_KEY=$AZURE_API_KEY \
|
|
-e AZURE_API_BASE=$AZURE_API_BASE \
|
|
-e USE_DDTRACE=True \
|
|
-e DD_API_KEY=$DD_API_KEY \
|
|
-e DD_SITE=$DD_SITE \
|
|
-e LITELLM_LICENSE=$LITELLM_LICENSE \
|
|
-e LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES=true \
|
|
--add-host host.docker.internal:host-gateway \
|
|
--name my-app \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/pass_through_config.yaml:/app/config.yaml \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/custom_auth_basic.py:/app/custom_auth_basic.py \
|
|
litellm-docker-database:ci \
|
|
--config /app/config.yaml \
|
|
--port 4000 \
|
|
--detailed_debug \
|
|
- run:
|
|
name: Start outputting logs
|
|
command: docker logs -f my-app
|
|
background: true
|
|
- wait_for_service:
|
|
url: http://localhost:4000
|
|
timeout: "300"
|
|
# Add Ruby installation and testing before the existing Node.js and Python tests
|
|
- run:
|
|
name: Install Ruby and Bundler
|
|
command: |
|
|
# Clone RVM at pinned tag and verify the commit SHA matches the
|
|
# published tag before running its install script.
|
|
RVM_VERSION="1.29.12"
|
|
RVM_EXPECTED_SHA="6bfc9213c9d6914fe756f524eb034a403d51db81"
|
|
git clone --depth 1 --branch "$RVM_VERSION" https://github.com/rvm/rvm.git /tmp/rvm
|
|
RVM_ACTUAL_SHA="$(git -C /tmp/rvm rev-parse HEAD)"
|
|
if [ "$RVM_ACTUAL_SHA" != "$RVM_EXPECTED_SHA" ]; then
|
|
echo "RVM tag $RVM_VERSION resolved to $RVM_ACTUAL_SHA; expected $RVM_EXPECTED_SHA" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Import RVM signing keys (used by `rvm install` to verify Ruby tarballs)
|
|
gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
|
|
|
|
# Install RVM from the verified checkout. The install script
|
|
# sources `scripts/functions/installer` using paths relative to
|
|
# its own working directory, so it must be run from /tmp/rvm.
|
|
(cd /tmp/rvm && ./install --path "$HOME/.rvm")
|
|
source "$HOME/.rvm/scripts/rvm"
|
|
|
|
# Install Ruby 3.2.2 (RVM verifies the tarball PGP signature)
|
|
rvm install 3.2.2
|
|
rvm use 3.2.2 --default
|
|
|
|
# Install latest Bundler
|
|
gem install bundler
|
|
|
|
- run:
|
|
name: Run Ruby tests
|
|
command: |
|
|
source $HOME/.rvm/scripts/rvm
|
|
cd tests/pass_through_tests/ruby_passthrough_tests
|
|
bundle install
|
|
bundle exec rspec
|
|
no_output_timeout: 30m
|
|
# Install Node.js directly from nodejs.org with SHA256 verification,
|
|
# instead of piping NodeSource's setup_18.x apt-repo installer into
|
|
# sudo bash (which runs a mutable upstream script unattended).
|
|
- run:
|
|
name: Install Node.js 18.20.8
|
|
command: |
|
|
NODE_VERSION="18.20.8"
|
|
NODE_TARBALL="node-v${NODE_VERSION}-linux-x64.tar.xz"
|
|
NODE_EXPECTED_SHA="5467ee62d6af1411d46b6a10e3fb5cacc92734dbcef465fea14e7b90993001c9"
|
|
curl -sSLf -o "/tmp/${NODE_TARBALL}" "https://nodejs.org/dist/v${NODE_VERSION}/${NODE_TARBALL}"
|
|
echo "${NODE_EXPECTED_SHA} /tmp/${NODE_TARBALL}" | sha256sum -c -
|
|
sudo tar -xJf "/tmp/${NODE_TARBALL}" -C /usr/local --strip-components=1
|
|
rm -f "/tmp/${NODE_TARBALL}"
|
|
node --version
|
|
npm --version
|
|
|
|
- run:
|
|
name: Install Node.js test dependencies
|
|
command: |
|
|
cd tests/pass_through_tests
|
|
npm ci
|
|
|
|
- run:
|
|
name: Run Vertex AI, Google AI Studio Node.js tests
|
|
command: |
|
|
cd tests/pass_through_tests
|
|
NODE_OPTIONS=--experimental-vm-modules npx jest . --verbose
|
|
no_output_timeout: 30m
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
mkdir -p test-results
|
|
TEST_FILES=$(circleci tests glob "tests/pass_through_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-v -x \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5"
|
|
no_output_timeout: 15m
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
|
|
proxy_e2e_anthropic_messages_tests:
|
|
machine:
|
|
image: ubuntu-2204:2024.04.1
|
|
resource_class: large
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
- start_postgres
|
|
- attach_workspace:
|
|
at: ~/project
|
|
- run:
|
|
name: Load Docker Database Image
|
|
command: |
|
|
zstd -d litellm-docker-database.tar.zst --stdout | docker load
|
|
docker images | grep litellm-docker-database
|
|
- run:
|
|
name: Run Docker container with test config
|
|
command: |
|
|
docker run -d \
|
|
-p 4000:4000 \
|
|
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
|
|
-e LITELLM_MASTER_KEY="sk-1234" \
|
|
-e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
|
|
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
|
|
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
|
|
-e AWS_REGION_NAME="us-east-1" \
|
|
-e LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS="True" \
|
|
--add-host host.docker.internal:host-gateway \
|
|
--name my-app \
|
|
-v $(pwd)/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml:/app/config.yaml \
|
|
litellm-docker-database:ci \
|
|
--config /app/config.yaml \
|
|
--port 4000 \
|
|
--detailed_debug
|
|
- run:
|
|
name: Start outputting logs
|
|
command: docker logs -f my-app
|
|
background: true
|
|
- wait_for_service:
|
|
url: http://localhost:4000
|
|
timeout: "300"
|
|
- run:
|
|
name: Run Claude Agent SDK E2E Tests
|
|
command: |
|
|
mkdir -p test-results
|
|
export LITELLM_PROXY_URL="http://localhost:4000"
|
|
export LITELLM_API_KEY="sk-1234"
|
|
TEST_FILES=$(circleci tests glob "tests/proxy_e2e_anthropic_messages_tests/**/test_*.py")
|
|
echo "$TEST_FILES" | circleci tests run \
|
|
--verbose \
|
|
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
|
-vv -x -s \
|
|
--junitxml=test-results/junit.xml \
|
|
--durations=5"
|
|
no_output_timeout: 15m
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
|
|
upload-coverage:
|
|
docker:
|
|
- *python312_image
|
|
steps:
|
|
- checkout
|
|
- attach_workspace:
|
|
at: .
|
|
# Check file locations
|
|
- run:
|
|
name: Check coverage file location
|
|
command: |
|
|
echo "Current directory:"
|
|
ls -la
|
|
echo "\nContents of tests/llm_translation:"
|
|
ls -la tests/llm_translation
|
|
- install_uv
|
|
- run:
|
|
name: Combine Coverage
|
|
command: |
|
|
uv tool run --from 'coverage[toml]==7.10.6' coverage combine realtime_translation_coverage ocr_coverage search_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage redis_caching_coverage
|
|
uv tool run --from 'coverage[toml]==7.10.6' coverage xml
|
|
- codecov/upload:
|
|
file: ./coverage.xml
|
|
|
|
ui_build:
|
|
docker:
|
|
- image: cimg/node:20.19@sha256:35e64883e8d21bc345b0a7b04c35ee46442c127607ed1d8d7d37d8a1ed76db81
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
resource_class: medium+
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- restore_cache:
|
|
keys:
|
|
- ui-build-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
|
|
- ui-build-deps-v1-
|
|
- restore_cache:
|
|
keys:
|
|
- ui-nextjs-cache-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
|
|
- ui-nextjs-cache-v1-
|
|
- run:
|
|
name: Install dependencies
|
|
command: |
|
|
cd ui/litellm-dashboard
|
|
npm ci
|
|
- save_cache:
|
|
key: ui-build-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
|
|
paths:
|
|
- ui/litellm-dashboard/node_modules
|
|
- run:
|
|
name: Build UI
|
|
command: |
|
|
cd ui/litellm-dashboard
|
|
source ./build_ui.sh
|
|
- save_cache:
|
|
key: ui-nextjs-cache-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
|
|
paths:
|
|
- ui/litellm-dashboard/.next/cache
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- litellm/proxy/_experimental/out
|
|
|
|
ui_unit_tests:
|
|
docker:
|
|
- image: cimg/node:20.19@sha256:35e64883e8d21bc345b0a7b04c35ee46442c127607ed1d8d7d37d8a1ed76db81
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
resource_class: xlarge
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- restore_cache:
|
|
keys:
|
|
- ui-unit-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
|
|
- ui-unit-deps-v1-
|
|
- run:
|
|
name: Install dependencies
|
|
command: |
|
|
cd ui/litellm-dashboard
|
|
npm ci
|
|
- save_cache:
|
|
key: ui-unit-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
|
|
paths:
|
|
- ui/litellm-dashboard/node_modules
|
|
- run:
|
|
name: Run UI unit tests (Vitest)
|
|
command: |
|
|
cd ui/litellm-dashboard
|
|
|
|
CI=true npm run test -- --run \
|
|
--pool forks --poolOptions.forks.maxForks=8
|
|
|
|
e2e_ui_testing:
|
|
docker:
|
|
- image: cimg/python:3.12-browsers@sha256:b432899af01c9a311bf74f4f22e9ada2e5306d4b1b4383f8d29e1228a5844ef2
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
- image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
|
|
environment:
|
|
POSTGRES_USER: e2euser
|
|
POSTGRES_PASSWORD: e2epassword
|
|
POSTGRES_DB: litellm_e2e
|
|
resource_class: large
|
|
working_directory: ~/project
|
|
environment:
|
|
DATABASE_URL: "postgresql://e2euser:e2epassword@localhost:5432/litellm_e2e"
|
|
CI: "true"
|
|
steps:
|
|
- checkout
|
|
- setup_google_dns
|
|
- install_uv
|
|
- restore_cache:
|
|
keys:
|
|
- v1-uv-cache-{{ checksum "uv.lock" }}
|
|
- run:
|
|
name: Install Python dependencies
|
|
command: |
|
|
uv sync --frozen --all-groups --all-extras --python 3.12
|
|
uv run --no-sync python -m prisma generate --schema litellm/proxy/schema.prisma
|
|
- save_cache:
|
|
key: v1-uv-cache-{{ checksum "uv.lock" }}
|
|
paths:
|
|
- ~/.cache/uv
|
|
- restore_cache:
|
|
keys:
|
|
- ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
|
|
- run:
|
|
name: Install Node dependencies and Playwright
|
|
# The cimg/python:3.12-browsers image already ships the Chromium system
|
|
# libraries Playwright needs (libnss3, libatk-bridge2.0-0, libcups2, etc.).
|
|
# `--with-deps` triggers a redundant apt-get update + install that adds
|
|
# 5-10 minutes to the job and frequently stalls on flaky Ubuntu mirrors,
|
|
# so we install just the browser binary.
|
|
command: |
|
|
cd ui/litellm-dashboard
|
|
npm ci
|
|
npx playwright install chromium
|
|
- save_cache:
|
|
key: ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
|
|
paths:
|
|
- ui/litellm-dashboard/node_modules
|
|
- ~/.cache/ms-playwright
|
|
- run:
|
|
name: Build UI from source
|
|
# Prior version used `cp -r out/ ../../litellm/proxy/_experimental/out/`.
|
|
# GNU cp (used on CircleCI's Ubuntu image) interprets that as "copy the
|
|
# source directory as a child of the destination" when the destination
|
|
# already exists — silently creating `_experimental/out/out/` instead of
|
|
# replacing the served bundle. The proxy continued serving whatever was
|
|
# checked into `_experimental/out/*`, so this job was effectively testing
|
|
# the pre-build bundle on every run. Replace-and-move guarantees the
|
|
# freshly built bundle is what the proxy actually serves.
|
|
command: |
|
|
cd ui/litellm-dashboard
|
|
npm run build
|
|
rm -rf ../../litellm/proxy/_experimental/out
|
|
mv out ../../litellm/proxy/_experimental/out
|
|
# Restructure HTML so extensionless routes work (login.html -> login/index.html)
|
|
find ../../litellm/proxy/_experimental/out -name '*.html' ! -name 'index.html' | while read -r f; do
|
|
d="${f%.html}"; mkdir -p "$d"; mv "$f" "$d/index.html"
|
|
done
|
|
- wait_for_service:
|
|
url: tcp://localhost:5432
|
|
timeout: "30"
|
|
- run:
|
|
name: Push Prisma schema
|
|
command: uv run --no-sync python -m prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
|
|
- run:
|
|
name: Seed database
|
|
command: |
|
|
PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \
|
|
-f ui/litellm-dashboard/e2e_tests/fixtures/seed.sql
|
|
- run:
|
|
name: Start mock LLM server
|
|
command: uv run --no-sync python ui/litellm-dashboard/e2e_tests/fixtures/mock_llm_server/server.py
|
|
background: true
|
|
- run:
|
|
name: Start LiteLLM proxy
|
|
environment:
|
|
LITELLM_MASTER_KEY: "sk-1234"
|
|
MOCK_LLM_URL: "http://127.0.0.1:8090/v1"
|
|
DISABLE_SCHEMA_UPDATE: "true"
|
|
SERVER_ROOT_PATH: ""
|
|
PROXY_LOGOUT_URL: ""
|
|
command: |
|
|
uv run --no-sync python -m litellm.proxy.proxy_cli \
|
|
--config ui/litellm-dashboard/e2e_tests/fixtures/config.yml \
|
|
--port 4000
|
|
background: true
|
|
- run:
|
|
name: Wait for proxy to be ready
|
|
command: |
|
|
for i in $(seq 1 60); do
|
|
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:4000/health -H "Authorization: Bearer sk-1234" 2>/dev/null || true)
|
|
if [ "$HTTP_CODE" = "200" ]; then
|
|
echo "Proxy is ready"
|
|
exit 0
|
|
fi
|
|
sleep 2
|
|
done
|
|
echo "Proxy failed to start"
|
|
exit 1
|
|
- run:
|
|
name: Run Playwright E2E tests
|
|
command: |
|
|
cd ui/litellm-dashboard
|
|
npx playwright test --config e2e_tests/playwright.config.ts
|
|
no_output_timeout: 10m
|
|
- store_artifacts:
|
|
path: ui/litellm-dashboard/test-results
|
|
destination: e2e-test-results
|
|
- store_artifacts:
|
|
path: ui/litellm-dashboard/playwright-report
|
|
destination: e2e-playwright-report
|
|
|
|
build_docker_database_image:
|
|
machine:
|
|
image: ubuntu-2204:2024.04.1
|
|
resource_class: large
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
|
|
- run:
|
|
name: Build Docker image
|
|
command: |
|
|
docker build \
|
|
-t litellm-docker-database:ci \
|
|
-f docker/Dockerfile.database .
|
|
|
|
- run:
|
|
name: Save Docker image to workspace root
|
|
command: |
|
|
docker save litellm-docker-database:ci | zstd -1 -T0 > litellm-docker-database.tar.zst
|
|
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- litellm-docker-database.tar.zst
|
|
|
|
|
|
test_bad_database_url:
|
|
machine:
|
|
image: ubuntu-2204:2024.04.1
|
|
resource_class: medium
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- attach_workspace:
|
|
at: ~/project
|
|
- setup_google_dns
|
|
- start_postgres
|
|
- run:
|
|
name: Load Docker Database Image
|
|
command: |
|
|
zstd -d litellm-docker-database.tar.zst --stdout | docker load
|
|
docker tag litellm-docker-database:ci myapp:latest
|
|
- run:
|
|
name: Run Docker container with bad DATABASE_URL
|
|
command: |
|
|
docker run --name my-app \
|
|
-p 4000:4000 \
|
|
-e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \
|
|
-e DATABASE_URL="postgresql://wrong:wrong@wrong:5432/wrong" \
|
|
myapp:latest \
|
|
--port 4000 > docker_output.log 2>&1 || true
|
|
- run:
|
|
name: Display Docker logs
|
|
command: cat docker_output.log
|
|
- run:
|
|
name: Check for expected error
|
|
command: |
|
|
if grep -q "Error: P1001: Can't reach database server at" docker_output.log && \
|
|
(grep -q "Database setup failed after multiple retries" docker_output.log || \
|
|
grep -q "ERROR: Application startup failed. Exiting." docker_output.log); then
|
|
echo "Expected error found. Test passed."
|
|
else
|
|
echo "Expected error not found. Test failed."
|
|
cat docker_output.log
|
|
exit 1
|
|
fi
|
|
|
|
workflows:
|
|
build_and_test:
|
|
jobs:
|
|
- using_litellm_on_windows:
|
|
filters: &main_branches
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- local_testing_part1:
|
|
filters: *main_branches
|
|
- local_testing_part2:
|
|
filters: *main_branches
|
|
- langfuse_logging_unit_tests:
|
|
filters: *main_branches
|
|
- litellm_assistants_api_testing:
|
|
filters: *main_branches
|
|
- litellm_router_testing:
|
|
filters: *main_branches
|
|
- litellm_router_unit_testing:
|
|
filters: *main_branches
|
|
- ui_build:
|
|
filters: *main_branches
|
|
- ui_unit_tests:
|
|
requires:
|
|
- ui_build
|
|
filters: *main_branches
|
|
- auth_ui_unit_tests:
|
|
filters: *main_branches
|
|
- build_docker_database_image:
|
|
filters: *main_branches
|
|
- e2e_ui_testing:
|
|
filters: *main_branches
|
|
- build_and_test:
|
|
requires:
|
|
- build_docker_database_image
|
|
filters: *main_branches
|
|
- e2e_openai_endpoints:
|
|
requires:
|
|
- build_docker_database_image
|
|
filters: *main_branches
|
|
- proxy_logging_guardrails_model_info_tests:
|
|
requires:
|
|
- build_docker_database_image
|
|
filters: *main_branches
|
|
- proxy_spend_accuracy_tests:
|
|
requires:
|
|
- build_docker_database_image
|
|
filters: *main_branches
|
|
- proxy_multi_instance_tests:
|
|
requires:
|
|
- build_docker_database_image
|
|
filters: *main_branches
|
|
- proxy_store_model_in_db_tests:
|
|
requires:
|
|
- build_docker_database_image
|
|
filters: *main_branches
|
|
- proxy_build_from_pip_tests:
|
|
filters: *main_branches
|
|
- proxy_pass_through_endpoint_tests:
|
|
requires:
|
|
- build_docker_database_image
|
|
filters: *main_branches
|
|
- proxy_e2e_anthropic_messages_tests:
|
|
requires:
|
|
- build_docker_database_image
|
|
filters: *main_branches
|
|
- llm_translation_testing:
|
|
filters: *main_branches
|
|
- realtime_translation_testing:
|
|
filters: *main_branches
|
|
- agent_testing:
|
|
filters: *main_branches
|
|
- guardrails_testing:
|
|
filters: *main_branches
|
|
- google_generate_content_endpoint_testing:
|
|
filters: *main_branches
|
|
- llm_responses_api_testing:
|
|
filters: *main_branches
|
|
- ocr_testing:
|
|
filters: *main_branches
|
|
- search_testing:
|
|
filters: *main_branches
|
|
- litellm_mapped_enterprise_tests:
|
|
filters: *main_branches
|
|
- batches_testing:
|
|
filters: *main_branches
|
|
- litellm_utils_testing:
|
|
filters: *main_branches
|
|
- pass_through_unit_testing:
|
|
filters: *main_branches
|
|
- image_gen_testing:
|
|
filters: *main_branches
|
|
- logging_testing:
|
|
filters: *main_branches
|
|
- audio_testing:
|
|
filters: *main_branches
|
|
- redis_caching_unit_tests:
|
|
filters: *main_branches
|
|
- upload-coverage:
|
|
requires:
|
|
- realtime_translation_testing
|
|
- agent_testing
|
|
- google_generate_content_endpoint_testing
|
|
- guardrails_testing
|
|
- ocr_testing
|
|
- search_testing
|
|
- litellm_mapped_enterprise_tests
|
|
- batches_testing
|
|
- litellm_utils_testing
|
|
- pass_through_unit_testing
|
|
- image_gen_testing
|
|
- logging_testing
|
|
- audio_testing
|
|
- redis_caching_unit_tests
|
|
- langfuse_logging_unit_tests
|
|
- local_testing_part1
|
|
- local_testing_part2
|
|
- litellm_assistants_api_testing
|
|
- db_migration_disable_update_check:
|
|
requires:
|
|
- build_docker_database_image
|
|
filters: *main_branches
|
|
- installing_litellm_on_python:
|
|
filters: *main_branches
|
|
- installing_litellm_on_python_3_13:
|
|
filters: *main_branches
|
|
- installing_litellm_on_python_v2_migration_resolver:
|
|
filters: *main_branches
|
|
- helm_chart_testing:
|
|
requires:
|
|
- build_docker_database_image
|
|
filters: *main_branches
|
|
- test_bad_database_url:
|
|
requires:
|
|
- build_docker_database_image
|
|
filters: *main_branches
|