test(ci): add Claude Code ↔ LiteLLM dockerized E2E workflow

Add a dedicated CircleCI E2E path that builds LiteLLM and Claude client images as separate jobs and validates Claude Code requests through a proxy-backed Anthropic route. Include back-to-back request coverage and harden the harness with explicit env guards, bounded curl timeouts, schema-correct master key placement, and robust response validation.

Made-with: Cursor
This commit is contained in:
Darcy Liu 2026-04-25 22:43:36 +00:00
parent 5db6b83446
commit f5423a3544
7 changed files with 542 additions and 0 deletions

View file

@ -2006,6 +2006,70 @@ jobs:
- store_test_results:
path: test-results
proxy_e2e_claude_code_tests:
machine:
image: ubuntu-2204:2024.04.1
resource_class: large
working_directory: ~/project
steps:
- checkout
- setup_google_dns
- attach_workspace:
at: ~/project
- run:
name: Load Docker Database Image
command: |
zstd -d litellm-docker-database.tar.zst --stdout | docker load
docker images | grep litellm-docker-database
- run:
name: Load Claude Code client image
command: |
zstd -d claude-code-client.tar.zst --stdout | docker load
docker images | grep claude-code-client
- run:
name: Verify required container images stage
command: |
docker image inspect litellm-docker-database:ci >/dev/null
docker image inspect claude-code-client:ci >/dev/null
echo "LiteLLM and Claude Code images are ready"
- run:
name: Run Dockerized Claude Code E2E Tests
command: |
mkdir -p test-results
: "${LITELLM_MASTER_KEY:?Set LITELLM_MASTER_KEY in CircleCI project env vars.}"
: "${ANTHROPIC_API_KEY:?Set ANTHROPIC_API_KEY in CircleCI project env vars.}"
set +e
LITELLM_IMAGE=litellm-docker-database:ci \
CLAUDE_CODE_IMAGE=claude-code-client:ci \
LITELLM_SKIP_BUILD=true \
CLAUDE_CODE_SKIP_BUILD=true \
LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY}" \
tests/proxy_e2e_anthropic_messages_tests/claude_code/run_claude_code_docker_test.sh
status=$?
if [ "$status" -eq 0 ]; then
printf '%s\n' \
'<?xml version="1.0" encoding="UTF-8"?>' \
'<testsuite name="proxy_e2e_anthropic_messages_tests" tests="1" failures="0">' \
' <testcase classname="claude_code_docker" name="claude_code_through_litellm"/>' \
'</testsuite>' \
> test-results/junit.xml
else
printf '%s\n' \
'<?xml version="1.0" encoding="UTF-8"?>' \
'<testsuite name="proxy_e2e_anthropic_messages_tests" tests="1" failures="1">' \
' <testcase classname="claude_code_docker" name="claude_code_through_litellm">' \
' <failure message="Dockerized Claude Code E2E test failed">See CircleCI job output for Docker and LiteLLM logs.</failure>' \
' </testcase>' \
'</testsuite>' \
> test-results/junit.xml
fi
exit "$status"
no_output_timeout: 15m
# Store test results
- store_test_results:
path: test-results
upload-coverage:
docker:
- *python312_image
@ -2247,6 +2311,29 @@ jobs:
paths:
- litellm-docker-database.tar.zst
build_claude_code_client_image:
machine:
image: ubuntu-2204:2024.04.1
resource_class: medium
working_directory: ~/project
steps:
- checkout
- run:
name: Build Claude Code client image
command: |
docker build \
-t claude-code-client:ci \
-f tests/proxy_e2e_anthropic_messages_tests/claude_code/Dockerfile.claude-code \
tests/proxy_e2e_anthropic_messages_tests/claude_code
- run:
name: Save Claude Code image to workspace root
command: |
docker save claude-code-client:ci | zstd -1 -T0 > claude-code-client.tar.zst
- persist_to_workspace:
root: .
paths:
- claude-code-client.tar.zst
test_bad_database_url:
machine:
@ -2320,6 +2407,8 @@ workflows:
filters: *main_branches
- build_docker_database_image:
filters: *main_branches
- build_claude_code_client_image:
filters: *main_branches
- e2e_ui_testing:
filters: *main_branches
- build_and_test:
@ -2356,6 +2445,11 @@ workflows:
requires:
- build_docker_database_image
filters: *main_branches
- proxy_e2e_claude_code_tests:
requires:
- build_docker_database_image
- build_claude_code_client_image
filters: *main_branches
- llm_translation_testing:
filters: *main_branches
- realtime_translation_testing:
@ -2426,3 +2520,4 @@ workflows:
requires:
- build_docker_database_image
filters: *main_branches

View file

@ -0,0 +1,41 @@
ARG CLAUDE_CODE_BASE_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:f26d42a15d09d9a643b231df929fa3cf609bedc58a728eb445be89a9d8d1da9f
FROM ${CLAUDE_CODE_BASE_IMAGE}
ARG CLAUDE_CODE_VERSION=latest
USER root
COPY resolve_claude_code_version.js /tmp/resolve_claude_code_version.js
RUN apk add --no-cache \
bash \
ca-certificates \
curl \
nodejs \
npm \
shadow && \
if [ "${CLAUDE_CODE_VERSION}" = "latest" ]; then \
CLAUDE_CODE_RESOLVED_VERSION="$(node /tmp/resolve_claude_code_version.js)"; \
else \
CLAUDE_CODE_RESOLVED_VERSION="${CLAUDE_CODE_VERSION}"; \
fi && \
echo "Installing @anthropic-ai/claude-code@${CLAUDE_CODE_RESOLVED_VERSION}" && \
npm install -g --ignore-scripts "@anthropic-ai/claude-code@${CLAUDE_CODE_RESOLVED_VERSION}" && \
CLAUDE_CODE_GLOBAL_ROOT="$(npm root -g)" && \
node "${CLAUDE_CODE_GLOBAL_ROOT}/@anthropic-ai/claude-code/install.cjs" && \
npm cache clean --force && \
groupadd --system claude && \
useradd --system --create-home --gid claude --shell /bin/bash claude && \
mkdir -p /workspace /home/claude/.claude && \
chown -R claude:claude /workspace /home/claude
COPY --chown=claude:claude run_claude_code_check.sh /usr/local/bin/run_claude_code_check.sh
RUN chmod 0755 /usr/local/bin/run_claude_code_check.sh
USER claude
WORKDIR /workspace
ENV CI=true \
HOME=/home/claude
ENTRYPOINT ["/usr/local/bin/run_claude_code_check.sh"]

View file

@ -0,0 +1,62 @@
# Claude Code LiteLLM E2E Test
This V0 test validates the customer-critical Claude Code -> LiteLLM -> Anthropic path in CircleCI.
## What It Covers
- Builds/runs the LiteLLM proxy container from the current checkout.
- Starts an isolated Postgres container for proxy database state.
- Builds a separate Claude Code client container.
- Runs Claude Code as a non-root user with only the LiteLLM proxy key.
- Points Claude Code at LiteLLM via `ANTHROPIC_BASE_URL`.
- Verifies the configured Anthropic model is visible from `/v1/models`.
- Sends back-to-back Claude Code prompts through LiteLLM and validates both responses.
- Sends a direct Anthropic Messages API request through LiteLLM and checks proxy response headers:
- `x-litellm-call-id`
- `x-litellm-response-cost`
The direct `/v1/messages` header check gives a clear proxy-side signal that LiteLLM handled and accounted for the request, instead of only proving the client printed text.
## Claude Code Version Policy
The Claude Code image resolves `@anthropic-ai/claude-code@latest` by querying npm publish times, selecting the newest version older than three days, then installing that exact version.
This intentionally follows the newest Claude Code release that has aged at least three days, which catches customer-facing compatibility issues while avoiding just-published releases that are more likely to be pulled or patched.
Set `CLAUDE_CODE_VERSION=<version>` to test a specific version locally or in a follow-up CI job.
## Running Locally
Set `ANTHROPIC_API_KEY` in the environment or in `tests/proxy_e2e_anthropic_messages_tests/claude_code/.env`, then run:
```bash
tests/proxy_e2e_anthropic_messages_tests/claude_code/run_claude_code_docker_test.sh
```
Useful overrides:
```bash
MODEL_NAME=claude-sonnet-4-6
LITELLM_UPSTREAM_MODEL=anthropic/claude-sonnet-4-6
CLAUDE_CODE_IMAGE=litellm-claude-code-client:local
CLAUDE_CODE_VERSION=latest
KEEP_CONTAINERS=1
```
In CircleCI, the job sets `LITELLM_IMAGE=litellm-docker-database:ci` and `LITELLM_SKIP_BUILD=true` so the test uses the LiteLLM image already built from the checked-out commit.
The CircleCI job now has explicit component stages before test execution:
- verify LiteLLM image exists (`litellm-docker-database:ci`)
- use prebuilt Claude Code client image (`claude-code-client:ci`)
- run end-to-end test with both image builds skipped (`LITELLM_SKIP_BUILD=true` and `CLAUDE_CODE_SKIP_BUILD=true`)
## Intentionally Left Out Of V0
- Bedrock and other providers. Those belong in V1 after the Anthropic path is stable.
- Back-to-back session behavior. V0 proves the request path works; V1 can add repeated requests and long-running session checks.
- Tool-use assertions. Claude Code can invoke tools in richer scenarios, but V0 keeps the signal focused on proxy compatibility and real Anthropic request success.
- A Claude Code version matrix. The Dockerfile supports `CLAUDE_CODE_VERSION`, but the default CI path tests one recent version to keep cost and flake surface low.
## Failure Output
The runner prints each phase, dumps LiteLLM logs if the proxy fails readiness, and emits a JUnit result in CircleCI. Header-check failures print the response headers and body so regressions are visible without SSHing into the job.

View file

@ -0,0 +1,57 @@
services:
postgres:
image: postgres:14@sha256:6a70deda415ec296f977890e11aba04a0db9f632a362e3fce45e845e3db74f26
environment:
POSTGRES_DB: litellm
POSTGRES_USER: litellm
POSTGRES_PASSWORD: litellm
healthcheck:
test: ["CMD-SHELL", "pg_isready -d litellm -U litellm"]
interval: 2s
timeout: 5s
retries: 30
litellm:
image: ${LITELLM_IMAGE:-litellm-claude-code-e2e:local}
build:
context: ../../..
dockerfile: Dockerfile
target: runtime
command: ["--config", "/app/config.yaml", "--port", "4000", "--detailed_debug"]
ports:
- "${PROXY_PORT:-4000}:4000"
environment:
DATABASE_URL: postgresql://litellm:litellm@postgres:5432/litellm
LITELLM_MASTER_KEY: ${LITELLM_MASTER_KEY:-sk-1234}
ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:?ANTHROPIC_API_KEY is required}
LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS: "True"
volumes:
- ${CLAUDE_CODE_LITELLM_CONFIG:?CLAUDE_CODE_LITELLM_CONFIG is required}:/app/config.yaml:ro
depends_on:
postgres:
condition: service_healthy
healthcheck:
test:
[
"CMD-SHELL",
"python3 -c \"import urllib.request; urllib.request.urlopen('http://127.0.0.1:4000/health/liveliness', timeout=5)\"",
]
interval: 5s
timeout: 10s
retries: 60
claude-code:
image: ${CLAUDE_CODE_IMAGE:-litellm-claude-code-client:local}
build:
context: .
dockerfile: Dockerfile.claude-code
args:
CLAUDE_CODE_VERSION: ${CLAUDE_CODE_VERSION:-latest}
environment:
ANTHROPIC_BASE_URL: http://litellm:4000
ANTHROPIC_AUTH_TOKEN: ${LITELLM_MASTER_KEY:-sk-1234}
MODEL_NAME: ${MODEL_NAME:-claude-sonnet-4-6}
CLAUDE_CODE_PROMPT: "${CLAUDE_CODE_PROMPT:-Respond with exactly this text and nothing else: Hello from LiteLLM Claude Code.}"
depends_on:
litellm:
condition: service_healthy

View file

@ -0,0 +1,22 @@
const { execFileSync } = require("node:child_process");
const packageName = "@anthropic-ai/claude-code";
const minAgeMs = 3 * 24 * 60 * 60 * 1000;
const cutoff = Date.now() - minAgeMs;
const raw = execFileSync("npm", ["view", packageName, "time", "--json"], {
encoding: "utf8",
});
const time = JSON.parse(raw);
const versions = Object.keys(time)
.filter((version) => /^\d/.test(version))
.filter((version) => new Date(time[version]).getTime() <= cutoff)
.sort((a, b) => a.localeCompare(b, undefined, { numeric: true }));
if (versions.length === 0) {
console.error(
`No ${packageName} version is older than the 3-day exclusion window.`
);
process.exit(1);
}
console.log(versions[versions.length - 1]);

View file

@ -0,0 +1,21 @@
#!/usr/bin/env bash
set -euo pipefail
: "${ANTHROPIC_BASE_URL:?ANTHROPIC_BASE_URL is required}"
: "${ANTHROPIC_AUTH_TOKEN:?ANTHROPIC_AUTH_TOKEN is required}"
: "${MODEL_NAME:?MODEL_NAME is required}"
OUTPUT_FILE="${CLAUDE_CODE_OUTPUT_FILE:-/tmp/claude-code-output.txt}"
PROMPT="${CLAUDE_CODE_PROMPT:-Respond with exactly this text and nothing else: Hello from LiteLLM Claude Code.}"
echo "Running Claude Code against ${ANTHROPIC_BASE_URL} with model ${MODEL_NAME}"
claude -p "${PROMPT}" --model "${MODEL_NAME}" >"${OUTPUT_FILE}"
if [[ ! -s "${OUTPUT_FILE}" ]]; then
echo "Claude Code produced no output."
exit 1
fi
echo "Claude Code output:"
sed -n '1,20p' "${OUTPUT_FILE}"

View file

@ -0,0 +1,244 @@
#!/usr/bin/env bash
set -euo pipefail
# Dockerized Claude Code + LiteLLM integration test.
#
# Required:
# ANTHROPIC_API_KEY or tests/proxy_e2e_anthropic_messages_tests/claude_code/.env
#
# Optional:
# MODEL_NAME=claude-sonnet-4-6
# LITELLM_UPSTREAM_MODEL=anthropic/${MODEL_NAME}
# LITELLM_IMAGE=litellm-claude-code-e2e:local
# CLAUDE_CODE_IMAGE=litellm-claude-code-client:local
# LITELLM_SKIP_BUILD=true
# CLAUDE_CODE_SKIP_BUILD=true
# CLAUDE_CODE_VERSION=latest
# PROXY_PORT=4000
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
COMPOSE_FILE="${SCRIPT_DIR}/docker-compose.yaml"
ENV_FILE="${SCRIPT_DIR}/.env"
MODEL_NAME="${MODEL_NAME:-claude-sonnet-4-6}"
LITELLM_UPSTREAM_MODEL="${LITELLM_UPSTREAM_MODEL:-anthropic/${MODEL_NAME}}"
LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY:-sk-1234}"
PROXY_PORT="${PROXY_PORT:-4000}"
PROXY_URL="http://127.0.0.1:${PROXY_PORT}"
STARTUP_TIMEOUT_S="${STARTUP_TIMEOUT_S:-300}"
COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-litellm-claude-code-e2e}"
TMP_DIR="$(mktemp -d)"
CLAUDE_CODE_LITELLM_CONFIG="${TMP_DIR}/config.yaml"
HEADERS_FILE="${TMP_DIR}/headers.txt"
BODY_FILE="${TMP_DIR}/body.json"
cleanup() {
if [[ "${KEEP_CONTAINERS:-}" == "1" ]]; then
echo "KEEP_CONTAINERS=1 set; leaving docker compose project ${COMPOSE_PROJECT_NAME} running."
echo "Generated config left at ${CLAUDE_CODE_LITELLM_CONFIG}"
return
fi
docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" down -v >/dev/null 2>&1 || true
rm -rf "${TMP_DIR}"
}
trap cleanup EXIT
if [[ -f "${ENV_FILE}" ]]; then
# shellcheck disable=SC1090
source "${ENV_FILE}"
fi
if [[ -z "${ANTHROPIC_API_KEY:-}" ]]; then
echo "ANTHROPIC_API_KEY is required in the environment or ${ENV_FILE}"
exit 1
fi
if ! command -v docker >/dev/null 2>&1; then
echo "docker is required"
exit 1
fi
if ! docker compose version >/dev/null 2>&1; then
echo "docker compose v2 is required"
exit 1
fi
if ! command -v python3 >/dev/null 2>&1; then
echo "python3 is required"
exit 1
fi
is_port_available() {
python3 - "$1" <<'PY'
import socket
import sys
port = int(sys.argv[1])
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.bind(("127.0.0.1", port))
print("yes")
except OSError:
print("no")
finally:
sock.close()
PY
}
if [[ "$(is_port_available "${PROXY_PORT}")" != "yes" ]]; then
echo "Port ${PROXY_PORT} is in use. Searching for a free port..."
FOUND_PORT=""
for candidate in $(seq 4001 4050); do
if [[ "$(is_port_available "${candidate}")" == "yes" ]]; then
FOUND_PORT="${candidate}"
break
fi
done
if [[ -z "${FOUND_PORT}" ]]; then
echo "No free local port found between 4001-4050."
exit 1
fi
PROXY_PORT="${FOUND_PORT}"
PROXY_URL="http://127.0.0.1:${PROXY_PORT}"
echo "Using fallback PROXY_PORT=${PROXY_PORT}"
fi
cat >"${CLAUDE_CODE_LITELLM_CONFIG}" <<EOF
model_list:
- model_name: ${MODEL_NAME}
litellm_params:
model: ${LITELLM_UPSTREAM_MODEL}
api_key: os.environ/ANTHROPIC_API_KEY
general_settings:
forward_client_headers_to_llm_api: true
master_key: "${LITELLM_MASTER_KEY}"
litellm_settings:
drop_params: true
modify_params: true
EOF
export ANTHROPIC_API_KEY
export CLAUDE_CODE_LITELLM_CONFIG
export LITELLM_IMAGE="${LITELLM_IMAGE:-litellm-claude-code-e2e:local}"
export CLAUDE_CODE_IMAGE="${CLAUDE_CODE_IMAGE:-litellm-claude-code-client:local}"
export LITELLM_MASTER_KEY
export MODEL_NAME
export PROXY_PORT
echo "[0/5] Preparing Docker images..."
if [[ "${LITELLM_SKIP_BUILD:-}" == "true" && "${CLAUDE_CODE_SKIP_BUILD:-}" == "true" ]]; then
echo "Skipping image builds (LITELLM_SKIP_BUILD=true and CLAUDE_CODE_SKIP_BUILD=true)."
elif [[ "${LITELLM_SKIP_BUILD:-}" == "true" ]]; then
docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" build claude-code
elif [[ "${CLAUDE_CODE_SKIP_BUILD:-}" == "true" ]]; then
docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" build litellm
else
docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" build litellm claude-code
fi
echo "[1/5] Starting Postgres and LiteLLM proxy..."
docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" up -d postgres litellm
echo "[2/5] Waiting for LiteLLM readiness at ${PROXY_URL}..."
READY=0
for _ in $(seq 1 "${STARTUP_TIMEOUT_S}"); do
if curl -fsS --connect-timeout 5 --max-time 15 \
-H "Authorization: Bearer ${LITELLM_MASTER_KEY}" "${PROXY_URL}/v1/models" >/dev/null 2>&1; then
READY=1
break
fi
sleep 1
done
if [[ "${READY}" != "1" ]]; then
echo "LiteLLM failed to become ready within ${STARTUP_TIMEOUT_S}s"
docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" logs --tail=200 litellm || true
exit 1
fi
echo "[3/5] Checking configured model..."
curl -fsS --connect-timeout 5 --max-time 20 \
-H "Authorization: Bearer ${LITELLM_MASTER_KEY}" "${PROXY_URL}/v1/models" | grep -q "\"${MODEL_NAME}\""
echo "[4/5] Running back-to-back Claude Code requests in an isolated non-root container..."
FIRST_RESPONSE="$(
docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" run --rm \
-e CLAUDE_CODE_PROMPT="Respond with exactly this text and nothing else: Hello from LiteLLM Claude Code request one." \
-e CLAUDE_CODE_OUTPUT_FILE="/tmp/claude-output-1.txt" \
claude-code
)"
SECOND_RESPONSE="$(
docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" run --rm \
-e CLAUDE_CODE_PROMPT="Respond with exactly this text and nothing else: Hello from LiteLLM Claude Code request two." \
-e CLAUDE_CODE_OUTPUT_FILE="/tmp/claude-output-2.txt" \
claude-code
)"
[[ -n "${FIRST_RESPONSE}" ]] || {
echo "First Claude Code request produced no response output."
exit 1
}
[[ -n "${SECOND_RESPONSE}" ]] || {
echo "Second Claude Code request produced no response output."
exit 1
}
printf '%s\n' "${FIRST_RESPONSE}" | grep -qi "request one" || {
echo "First Claude Code response did not contain expected content."
printf '%s\n' "${FIRST_RESPONSE}" | sed -n '1,40p'
exit 1
}
printf '%s\n' "${SECOND_RESPONSE}" | grep -qi "request two" || {
echo "Second Claude Code response did not contain expected content."
printf '%s\n' "${SECOND_RESPONSE}" | sed -n '1,40p'
exit 1
}
echo "[5/5] Verifying LiteLLM request headers on Anthropic messages endpoint..."
REQUEST_BODY="$(python3 - "${MODEL_NAME}" <<'PY'
import json
import sys
print(
json.dumps(
{
"model": sys.argv[1],
"max_tokens": 16,
"messages": [
{
"role": "user",
"content": "Respond with the word ok.",
}
],
}
)
)
PY
)"
curl -fsS --connect-timeout 5 --max-time 30 -D "${HEADERS_FILE}" -o "${BODY_FILE}" \
-X POST "${PROXY_URL}/v1/messages" \
-H "Authorization: Bearer ${LITELLM_MASTER_KEY}" \
-H "Content-Type: application/json" \
-H "anthropic-version: 2023-06-01" \
--data "${REQUEST_BODY}"
grep -qi "^x-litellm-call-id:" "${HEADERS_FILE}" || {
echo "Missing x-litellm-call-id header; proxy record signal not found."
sed -n '1,80p' "${HEADERS_FILE}"
sed -n '1,80p' "${BODY_FILE}"
exit 1
}
if ! grep -qi "^x-litellm-response-cost:" "${HEADERS_FILE}" && \
! grep -qi "^x-litellm-response-cost-original:" "${HEADERS_FILE}"; then
echo "Missing LiteLLM response cost headers; proxy usage signal not found."
echo "Expected one of: x-litellm-response-cost or x-litellm-response-cost-original"
sed -n '1,80p' "${HEADERS_FILE}"
sed -n '1,80p' "${BODY_FILE}"
exit 1
fi
echo "Success: Claude Code ran in its own non-root container through LiteLLM, with Postgres-backed proxy headers present."