mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-09 08:18:12 +00:00
21cfa0eacd
* ci: run every CircleCI check from the integration workflow * test(ci): keep the startup test CircleCI-owned * test(ui): find reset log rows by request id so concurrent workers cannot page them out * test(ui): retry the key budget window spec until the draft refresh fix lands * ci(ui): set Playwright workers and retries on the CircleCI command only The shared playwright.config.ts also drives the Buildkite UI suite, where 3 workers ran out of memory on one of two runs. Buildkite keeps main's 1 worker and 2 retries --------- Co-authored-by: yuneng <yuneng@berri.ai>
279 lines
10 KiB
YAML
279 lines
10 KiB
YAML
name: smoke
|
|
|
|
on:
|
|
pull_request:
|
|
branches: [main, litellm_internal_staging, litellm_oss_staging, "litellm_**"]
|
|
workflow_dispatch:
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
concurrency:
|
|
group: merge-smoke-${{ github.event.pull_request.number || github.run_id }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
dashboard-build:
|
|
name: dashboard-build
|
|
runs-on: ubuntu-24.04
|
|
timeout-minutes: 30
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- name: Build the dashboard stage
|
|
run: docker build --target ui-builder -f Dockerfile .
|
|
|
|
core-checks:
|
|
name: py${{ matrix.python-version }}
|
|
runs-on: ubuntu-24.04
|
|
timeout-minutes: 30
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
|
|
env:
|
|
LITELLM_LOCAL_MODEL_COST_MAP: "True"
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
|
with:
|
|
python-version: ${{ matrix.python-version }}
|
|
|
|
- name: Set up uv
|
|
uses: ./.github/actions/setup-uv-with-retries
|
|
with:
|
|
version: "0.10.9"
|
|
|
|
- name: Install dependencies
|
|
run: .github/scripts/uv_sync_with_retries.sh --frozen --extra proxy --extra cli --group dev --group proxy-dev --python ${{ matrix.python-version }}
|
|
|
|
- name: Create the loopback-only network namespace
|
|
run: |
|
|
sudo ip netns add smoke
|
|
sudo ip netns exec smoke ip link set lo up
|
|
cat > "${RUNNER_TEMP}/in-netns" <<'WRAP'
|
|
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
exec sudo --preserve-env=LITELLM_LOCAL_MODEL_COST_MAP ip netns exec smoke setpriv --reuid "$(id -u)" --regid "$(id -g)" --init-groups -- env HOME="${HOME}" PATH="${PATH}" "$@"
|
|
WRAP
|
|
chmod +x "${RUNNER_TEMP}/in-netns"
|
|
echo "IN_NETNS=${RUNNER_TEMP}/in-netns" >> "${GITHUB_ENV}"
|
|
|
|
- name: Verify namespace isolation
|
|
run: $IN_NETNS .venv/bin/python .github/scripts/run_merge_smoke.py verify-isolation
|
|
|
|
- name: Verify interpreter version
|
|
run: $IN_NETNS .venv/bin/python .github/scripts/run_merge_smoke.py interpreter --expect ${{ matrix.python-version }}
|
|
|
|
- name: Import and CLI checks
|
|
run: $IN_NETNS .venv/bin/python .github/scripts/run_merge_smoke.py cli
|
|
|
|
- name: Proxy startup check
|
|
run: $IN_NETNS .venv/bin/python .github/scripts/run_merge_smoke.py proxy-startup --diagnostics-dir "${RUNNER_TEMP}/smoke-diagnostics"
|
|
|
|
- name: Upload smoke diagnostics
|
|
if: always()
|
|
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
|
|
with:
|
|
name: merge-smoke-diagnostics-py${{ matrix.python-version }}
|
|
path: ${{ runner.temp }}/smoke-diagnostics
|
|
if-no-files-found: ignore
|
|
|
|
- name: Remove the network namespace
|
|
if: always()
|
|
run: sudo ip netns delete smoke
|
|
|
|
all-extras:
|
|
name: all-extras-py${{ matrix.python-version }}
|
|
runs-on: ubuntu-24.04
|
|
timeout-minutes: 10
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
python-version: ["3.12", "3.13"]
|
|
env:
|
|
LITELLM_LOCAL_MODEL_COST_MAP: "True"
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
|
with:
|
|
python-version: ${{ matrix.python-version }}
|
|
|
|
- name: Set up uv
|
|
uses: ./.github/actions/setup-uv-with-retries
|
|
with:
|
|
version: "0.10.9"
|
|
|
|
- name: Install every extra and dependency group
|
|
run: .github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras --python ${{ matrix.python-version }}
|
|
|
|
- name: Import the SDK, the proxy server and the enterprise package
|
|
run: .venv/bin/python -c "import litellm, litellm.proxy.proxy_server, litellm_enterprise"
|
|
|
|
windows-sdk:
|
|
name: windows-sdk
|
|
runs-on: windows-2025
|
|
timeout-minutes: 10
|
|
env:
|
|
LITELLM_LOCAL_MODEL_COST_MAP: "True"
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
|
with:
|
|
python-version: "3.11"
|
|
|
|
- name: Set up uv
|
|
uses: ./.github/actions/setup-uv-with-retries
|
|
with:
|
|
version: "0.10.9"
|
|
|
|
- name: Install dependencies
|
|
run: .github/scripts/uv_sync_with_retries.sh --frozen --no-install-workspace --group dev --python 3.11
|
|
|
|
- name: Run the Windows tests
|
|
run: uv run --no-sync python -m pytest --tb=short -v tests/windows_tests
|
|
|
|
proxy-build-from-pip:
|
|
name: proxy-build-from-pip
|
|
runs-on: ubuntu-24.04
|
|
timeout-minutes: 10
|
|
services:
|
|
postgres:
|
|
image: postgres:14@sha256:6a70deda415ec296f977890e11aba04a0db9f632a362e3fce45e845e3db74f26
|
|
env:
|
|
POSTGRES_USER: postgres
|
|
POSTGRES_PASSWORD: postgres
|
|
POSTGRES_DB: circle_test
|
|
ports:
|
|
- 5432:5432
|
|
options: >-
|
|
--health-cmd "pg_isready -U postgres"
|
|
--health-interval 2s
|
|
--health-timeout 5s
|
|
--health-retries 30
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
|
with:
|
|
python-version: "3.12"
|
|
|
|
- name: Set up uv
|
|
uses: ./.github/actions/setup-uv-with-retries
|
|
with:
|
|
version: "0.10.9"
|
|
|
|
- name: Install the test client dependencies
|
|
run: .github/scripts/uv_sync_with_retries.sh --frozen --no-install-workspace --extra proxy --group dev --python 3.12
|
|
|
|
- name: Build the image from the published package
|
|
run: docker build -t litellm-build-from-pip:smoke -f docker/build_from_pip/Dockerfile.build_from_pip .
|
|
|
|
- name: Start the fake OpenAI endpoint
|
|
run: |
|
|
nohup .venv/bin/python tests/_fake_openai_endpoint_server.py --host 0.0.0.0 --port 8190 > "${RUNNER_TEMP}/fake-openai.log" 2>&1 &
|
|
for _ in $(seq 1 30); do
|
|
curl -sf http://localhost:8190/health > /dev/null && exit 0
|
|
sleep 1
|
|
done
|
|
cat "${RUNNER_TEMP}/fake-openai.log"
|
|
exit 1
|
|
|
|
- name: Generate a master key
|
|
run: |
|
|
key="sk-$(openssl rand -hex 16)"
|
|
echo "::add-mask::${key}"
|
|
echo "LITELLM_MASTER_KEY=${key}" >> "${GITHUB_ENV}"
|
|
|
|
- name: Start the proxy container
|
|
run: |
|
|
docker run -d \
|
|
-p 4000:4000 \
|
|
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
|
|
-e LITELLM_MASTER_KEY \
|
|
-e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
|
|
-e OTEL_EXPORTER=in_memory \
|
|
-e USE_DDTRACE=True \
|
|
-e GCS_FLUSH_INTERVAL=1 \
|
|
-e LITELLM_LOG=ERROR \
|
|
--add-host host.docker.internal:host-gateway \
|
|
--name litellm-build-from-pip \
|
|
-v "${GITHUB_WORKSPACE}/docker/build_from_pip/litellm_config.yaml:/app/config.yaml" \
|
|
litellm-build-from-pip:smoke \
|
|
--config /app/config.yaml \
|
|
--port 4000
|
|
for _ in $(seq 1 150); do
|
|
curl -sf --max-time 5 -o /dev/null http://localhost:4000 && exit 0
|
|
sleep 2
|
|
done
|
|
exit 1
|
|
|
|
- name: Run the startup tests
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
base_url=http://localhost:4000
|
|
readiness_response="$(curl -sS --fail-with-body --write-out $'\n%{http_code}' "${base_url}/health/readiness")"
|
|
readiness_status="${readiness_response##*$'\n'}"
|
|
readiness_body="${readiness_response%$'\n'*}"
|
|
[[ "${readiness_status}" == "200" ]]
|
|
jq -e '.status == "healthy" or .status == "connected"' <<< "${readiness_body}" > /dev/null
|
|
liveness_status="$(curl -sS --fail-with-body --output /dev/null --write-out '%{http_code}' "${base_url}/health/liveness")"
|
|
[[ "${liveness_status}" == "200" ]]
|
|
completion_response="$(curl -sS --fail-with-body --write-out $'\n%{http_code}' \
|
|
--header "Authorization: Bearer ${LITELLM_MASTER_KEY}" \
|
|
--header 'Content-Type: application/json' \
|
|
--data '{"model":"gpt-4","messages":[{"role":"system","content":"You are a helpful assistant."},{"role":"user","content":"Hello!"}]}' \
|
|
"${base_url}/chat/completions")"
|
|
completion_status="${completion_response##*$'\n'}"
|
|
completion_body="${completion_response%$'\n'*}"
|
|
[[ "${completion_status}" == "200" ]]
|
|
jq -e '
|
|
(.choices | length) == 1 and
|
|
.choices[0].message.role == "assistant" and
|
|
.choices[0].message.content == "Hello! This is a mock response from the fake OpenAI endpoint."
|
|
' <<< "${completion_body}" > /dev/null
|
|
|
|
- name: Show the proxy logs
|
|
if: always()
|
|
run: docker logs litellm-build-from-pip || true
|
|
|
|
smoke-passed:
|
|
name: smoke passed
|
|
needs: [dashboard-build, core-checks, all-extras, windows-sdk, proxy-build-from-pip]
|
|
if: always()
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 2
|
|
permissions: {}
|
|
steps:
|
|
- name: Require every smoke job to succeed
|
|
env:
|
|
NEEDS: ${{ toJSON(needs) }}
|
|
run: |
|
|
jq -r 'to_entries[] | "\(.key): \(.value.result)"' <<< "$NEEDS"
|
|
jq -e 'all(.[]; .result == "success")' <<< "$NEEDS" > /dev/null
|