diff --git a/.circleci/config.yml b/.circleci/config.yml index 9f01bae3e5b..b3969fd8446 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2006,6 +2006,70 @@ jobs: - store_test_results: path: test-results + proxy_e2e_claude_code_tests: + machine: + image: ubuntu-2204:2024.04.1 + resource_class: large + working_directory: ~/project + steps: + - checkout + - setup_google_dns + - attach_workspace: + at: ~/project + - run: + name: Load Docker Database Image + command: | + zstd -d litellm-docker-database.tar.zst --stdout | docker load + docker images | grep litellm-docker-database + - run: + name: Load Claude Code client image + command: | + zstd -d claude-code-client.tar.zst --stdout | docker load + docker images | grep claude-code-client + - run: + name: Verify required container images stage + command: | + docker image inspect litellm-docker-database:ci >/dev/null + docker image inspect claude-code-client:ci >/dev/null + echo "LiteLLM and Claude Code images are ready" + - run: + name: Run Dockerized Claude Code E2E Tests + command: | + mkdir -p test-results + : "${LITELLM_MASTER_KEY:?Set LITELLM_MASTER_KEY in CircleCI project env vars.}" + : "${ANTHROPIC_API_KEY:?Set ANTHROPIC_API_KEY in CircleCI project env vars.}" + set +e + LITELLM_IMAGE=litellm-docker-database:ci \ + CLAUDE_CODE_IMAGE=claude-code-client:ci \ + LITELLM_SKIP_BUILD=true \ + CLAUDE_CODE_SKIP_BUILD=true \ + LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY}" \ + tests/proxy_e2e_anthropic_messages_tests/claude_code/run_claude_code_docker_test.sh + status=$? + if [ "$status" -eq 0 ]; then + printf '%s\n' \ + '' \ + '' \ + ' ' \ + '' \ + > test-results/junit.xml + else + printf '%s\n' \ + '' \ + '' \ + ' ' \ + ' See CircleCI job output for Docker and LiteLLM logs.' \ + ' ' \ + '' \ + > test-results/junit.xml + fi + exit "$status" + no_output_timeout: 15m + + # Store test results + - store_test_results: + path: test-results + upload-coverage: docker: - *python312_image @@ -2247,6 +2311,29 @@ jobs: paths: - litellm-docker-database.tar.zst + build_claude_code_client_image: + machine: + image: ubuntu-2204:2024.04.1 + resource_class: medium + working_directory: ~/project + steps: + - checkout + - run: + name: Build Claude Code client image + command: | + docker build \ + -t claude-code-client:ci \ + -f tests/proxy_e2e_anthropic_messages_tests/claude_code/Dockerfile.claude-code \ + tests/proxy_e2e_anthropic_messages_tests/claude_code + - run: + name: Save Claude Code image to workspace root + command: | + docker save claude-code-client:ci | zstd -1 -T0 > claude-code-client.tar.zst + - persist_to_workspace: + root: . + paths: + - claude-code-client.tar.zst + test_bad_database_url: machine: @@ -2320,6 +2407,8 @@ workflows: filters: *main_branches - build_docker_database_image: filters: *main_branches + - build_claude_code_client_image: + filters: *main_branches - e2e_ui_testing: filters: *main_branches - build_and_test: @@ -2356,6 +2445,11 @@ workflows: requires: - build_docker_database_image filters: *main_branches + - proxy_e2e_claude_code_tests: + requires: + - build_docker_database_image + - build_claude_code_client_image + filters: *main_branches - llm_translation_testing: filters: *main_branches - realtime_translation_testing: @@ -2426,3 +2520,4 @@ workflows: requires: - build_docker_database_image filters: *main_branches + diff --git a/tests/proxy_e2e_anthropic_messages_tests/claude_code/Dockerfile.claude-code b/tests/proxy_e2e_anthropic_messages_tests/claude_code/Dockerfile.claude-code new file mode 100644 index 00000000000..5ae9938a13c --- /dev/null +++ b/tests/proxy_e2e_anthropic_messages_tests/claude_code/Dockerfile.claude-code @@ -0,0 +1,41 @@ +ARG CLAUDE_CODE_BASE_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:f26d42a15d09d9a643b231df929fa3cf609bedc58a728eb445be89a9d8d1da9f +FROM ${CLAUDE_CODE_BASE_IMAGE} + +ARG CLAUDE_CODE_VERSION=latest + +USER root + +COPY resolve_claude_code_version.js /tmp/resolve_claude_code_version.js + +RUN apk add --no-cache \ + bash \ + ca-certificates \ + curl \ + nodejs \ + npm \ + shadow && \ + if [ "${CLAUDE_CODE_VERSION}" = "latest" ]; then \ + CLAUDE_CODE_RESOLVED_VERSION="$(node /tmp/resolve_claude_code_version.js)"; \ + else \ + CLAUDE_CODE_RESOLVED_VERSION="${CLAUDE_CODE_VERSION}"; \ + fi && \ + echo "Installing @anthropic-ai/claude-code@${CLAUDE_CODE_RESOLVED_VERSION}" && \ + npm install -g --ignore-scripts "@anthropic-ai/claude-code@${CLAUDE_CODE_RESOLVED_VERSION}" && \ + CLAUDE_CODE_GLOBAL_ROOT="$(npm root -g)" && \ + node "${CLAUDE_CODE_GLOBAL_ROOT}/@anthropic-ai/claude-code/install.cjs" && \ + npm cache clean --force && \ + groupadd --system claude && \ + useradd --system --create-home --gid claude --shell /bin/bash claude && \ + mkdir -p /workspace /home/claude/.claude && \ + chown -R claude:claude /workspace /home/claude + +COPY --chown=claude:claude run_claude_code_check.sh /usr/local/bin/run_claude_code_check.sh +RUN chmod 0755 /usr/local/bin/run_claude_code_check.sh + +USER claude +WORKDIR /workspace + +ENV CI=true \ + HOME=/home/claude + +ENTRYPOINT ["/usr/local/bin/run_claude_code_check.sh"] diff --git a/tests/proxy_e2e_anthropic_messages_tests/claude_code/README.md b/tests/proxy_e2e_anthropic_messages_tests/claude_code/README.md new file mode 100644 index 00000000000..5d549769421 --- /dev/null +++ b/tests/proxy_e2e_anthropic_messages_tests/claude_code/README.md @@ -0,0 +1,62 @@ +# Claude Code LiteLLM E2E Test + +This V0 test validates the customer-critical Claude Code -> LiteLLM -> Anthropic path in CircleCI. + +## What It Covers + +- Builds/runs the LiteLLM proxy container from the current checkout. +- Starts an isolated Postgres container for proxy database state. +- Builds a separate Claude Code client container. +- Runs Claude Code as a non-root user with only the LiteLLM proxy key. +- Points Claude Code at LiteLLM via `ANTHROPIC_BASE_URL`. +- Verifies the configured Anthropic model is visible from `/v1/models`. +- Sends back-to-back Claude Code prompts through LiteLLM and validates both responses. +- Sends a direct Anthropic Messages API request through LiteLLM and checks proxy response headers: + - `x-litellm-call-id` + - `x-litellm-response-cost` + +The direct `/v1/messages` header check gives a clear proxy-side signal that LiteLLM handled and accounted for the request, instead of only proving the client printed text. + +## Claude Code Version Policy + +The Claude Code image resolves `@anthropic-ai/claude-code@latest` by querying npm publish times, selecting the newest version older than three days, then installing that exact version. + +This intentionally follows the newest Claude Code release that has aged at least three days, which catches customer-facing compatibility issues while avoiding just-published releases that are more likely to be pulled or patched. + +Set `CLAUDE_CODE_VERSION=` to test a specific version locally or in a follow-up CI job. + +## Running Locally + +Set `ANTHROPIC_API_KEY` in the environment or in `tests/proxy_e2e_anthropic_messages_tests/claude_code/.env`, then run: + +```bash +tests/proxy_e2e_anthropic_messages_tests/claude_code/run_claude_code_docker_test.sh +``` + +Useful overrides: + +```bash +MODEL_NAME=claude-sonnet-4-6 +LITELLM_UPSTREAM_MODEL=anthropic/claude-sonnet-4-6 +CLAUDE_CODE_IMAGE=litellm-claude-code-client:local +CLAUDE_CODE_VERSION=latest +KEEP_CONTAINERS=1 +``` + +In CircleCI, the job sets `LITELLM_IMAGE=litellm-docker-database:ci` and `LITELLM_SKIP_BUILD=true` so the test uses the LiteLLM image already built from the checked-out commit. +The CircleCI job now has explicit component stages before test execution: + +- verify LiteLLM image exists (`litellm-docker-database:ci`) +- use prebuilt Claude Code client image (`claude-code-client:ci`) +- run end-to-end test with both image builds skipped (`LITELLM_SKIP_BUILD=true` and `CLAUDE_CODE_SKIP_BUILD=true`) + +## Intentionally Left Out Of V0 + +- Bedrock and other providers. Those belong in V1 after the Anthropic path is stable. +- Back-to-back session behavior. V0 proves the request path works; V1 can add repeated requests and long-running session checks. +- Tool-use assertions. Claude Code can invoke tools in richer scenarios, but V0 keeps the signal focused on proxy compatibility and real Anthropic request success. +- A Claude Code version matrix. The Dockerfile supports `CLAUDE_CODE_VERSION`, but the default CI path tests one recent version to keep cost and flake surface low. + +## Failure Output + +The runner prints each phase, dumps LiteLLM logs if the proxy fails readiness, and emits a JUnit result in CircleCI. Header-check failures print the response headers and body so regressions are visible without SSHing into the job. diff --git a/tests/proxy_e2e_anthropic_messages_tests/claude_code/docker-compose.yaml b/tests/proxy_e2e_anthropic_messages_tests/claude_code/docker-compose.yaml new file mode 100644 index 00000000000..e4239f40597 --- /dev/null +++ b/tests/proxy_e2e_anthropic_messages_tests/claude_code/docker-compose.yaml @@ -0,0 +1,57 @@ +services: + postgres: + image: postgres:14@sha256:6a70deda415ec296f977890e11aba04a0db9f632a362e3fce45e845e3db74f26 + environment: + POSTGRES_DB: litellm + POSTGRES_USER: litellm + POSTGRES_PASSWORD: litellm + healthcheck: + test: ["CMD-SHELL", "pg_isready -d litellm -U litellm"] + interval: 2s + timeout: 5s + retries: 30 + + litellm: + image: ${LITELLM_IMAGE:-litellm-claude-code-e2e:local} + build: + context: ../../.. + dockerfile: Dockerfile + target: runtime + command: ["--config", "/app/config.yaml", "--port", "4000", "--detailed_debug"] + ports: + - "${PROXY_PORT:-4000}:4000" + environment: + DATABASE_URL: postgresql://litellm:litellm@postgres:5432/litellm + LITELLM_MASTER_KEY: ${LITELLM_MASTER_KEY:-sk-1234} + ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:?ANTHROPIC_API_KEY is required} + LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS: "True" + volumes: + - ${CLAUDE_CODE_LITELLM_CONFIG:?CLAUDE_CODE_LITELLM_CONFIG is required}:/app/config.yaml:ro + depends_on: + postgres: + condition: service_healthy + healthcheck: + test: + [ + "CMD-SHELL", + "python3 -c \"import urllib.request; urllib.request.urlopen('http://127.0.0.1:4000/health/liveliness', timeout=5)\"", + ] + interval: 5s + timeout: 10s + retries: 60 + + claude-code: + image: ${CLAUDE_CODE_IMAGE:-litellm-claude-code-client:local} + build: + context: . + dockerfile: Dockerfile.claude-code + args: + CLAUDE_CODE_VERSION: ${CLAUDE_CODE_VERSION:-latest} + environment: + ANTHROPIC_BASE_URL: http://litellm:4000 + ANTHROPIC_AUTH_TOKEN: ${LITELLM_MASTER_KEY:-sk-1234} + MODEL_NAME: ${MODEL_NAME:-claude-sonnet-4-6} + CLAUDE_CODE_PROMPT: "${CLAUDE_CODE_PROMPT:-Respond with exactly this text and nothing else: Hello from LiteLLM Claude Code.}" + depends_on: + litellm: + condition: service_healthy diff --git a/tests/proxy_e2e_anthropic_messages_tests/claude_code/resolve_claude_code_version.js b/tests/proxy_e2e_anthropic_messages_tests/claude_code/resolve_claude_code_version.js new file mode 100644 index 00000000000..aaf04e64a1f --- /dev/null +++ b/tests/proxy_e2e_anthropic_messages_tests/claude_code/resolve_claude_code_version.js @@ -0,0 +1,22 @@ +const { execFileSync } = require("node:child_process"); + +const packageName = "@anthropic-ai/claude-code"; +const minAgeMs = 3 * 24 * 60 * 60 * 1000; +const cutoff = Date.now() - minAgeMs; +const raw = execFileSync("npm", ["view", packageName, "time", "--json"], { + encoding: "utf8", +}); +const time = JSON.parse(raw); +const versions = Object.keys(time) + .filter((version) => /^\d/.test(version)) + .filter((version) => new Date(time[version]).getTime() <= cutoff) + .sort((a, b) => a.localeCompare(b, undefined, { numeric: true })); + +if (versions.length === 0) { + console.error( + `No ${packageName} version is older than the 3-day exclusion window.` + ); + process.exit(1); +} + +console.log(versions[versions.length - 1]); diff --git a/tests/proxy_e2e_anthropic_messages_tests/claude_code/run_claude_code_check.sh b/tests/proxy_e2e_anthropic_messages_tests/claude_code/run_claude_code_check.sh new file mode 100755 index 00000000000..f4b2891c25a --- /dev/null +++ b/tests/proxy_e2e_anthropic_messages_tests/claude_code/run_claude_code_check.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${ANTHROPIC_BASE_URL:?ANTHROPIC_BASE_URL is required}" +: "${ANTHROPIC_AUTH_TOKEN:?ANTHROPIC_AUTH_TOKEN is required}" +: "${MODEL_NAME:?MODEL_NAME is required}" + +OUTPUT_FILE="${CLAUDE_CODE_OUTPUT_FILE:-/tmp/claude-code-output.txt}" +PROMPT="${CLAUDE_CODE_PROMPT:-Respond with exactly this text and nothing else: Hello from LiteLLM Claude Code.}" + +echo "Running Claude Code against ${ANTHROPIC_BASE_URL} with model ${MODEL_NAME}" + +claude -p "${PROMPT}" --model "${MODEL_NAME}" >"${OUTPUT_FILE}" + +if [[ ! -s "${OUTPUT_FILE}" ]]; then + echo "Claude Code produced no output." + exit 1 +fi + +echo "Claude Code output:" +sed -n '1,20p' "${OUTPUT_FILE}" diff --git a/tests/proxy_e2e_anthropic_messages_tests/claude_code/run_claude_code_docker_test.sh b/tests/proxy_e2e_anthropic_messages_tests/claude_code/run_claude_code_docker_test.sh new file mode 100755 index 00000000000..350a128683a --- /dev/null +++ b/tests/proxy_e2e_anthropic_messages_tests/claude_code/run_claude_code_docker_test.sh @@ -0,0 +1,244 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Dockerized Claude Code + LiteLLM integration test. +# +# Required: +# ANTHROPIC_API_KEY or tests/proxy_e2e_anthropic_messages_tests/claude_code/.env +# +# Optional: +# MODEL_NAME=claude-sonnet-4-6 +# LITELLM_UPSTREAM_MODEL=anthropic/${MODEL_NAME} +# LITELLM_IMAGE=litellm-claude-code-e2e:local +# CLAUDE_CODE_IMAGE=litellm-claude-code-client:local +# LITELLM_SKIP_BUILD=true +# CLAUDE_CODE_SKIP_BUILD=true +# CLAUDE_CODE_VERSION=latest +# PROXY_PORT=4000 + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +COMPOSE_FILE="${SCRIPT_DIR}/docker-compose.yaml" +ENV_FILE="${SCRIPT_DIR}/.env" +MODEL_NAME="${MODEL_NAME:-claude-sonnet-4-6}" +LITELLM_UPSTREAM_MODEL="${LITELLM_UPSTREAM_MODEL:-anthropic/${MODEL_NAME}}" +LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY:-sk-1234}" +PROXY_PORT="${PROXY_PORT:-4000}" +PROXY_URL="http://127.0.0.1:${PROXY_PORT}" +STARTUP_TIMEOUT_S="${STARTUP_TIMEOUT_S:-300}" +COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-litellm-claude-code-e2e}" +TMP_DIR="$(mktemp -d)" +CLAUDE_CODE_LITELLM_CONFIG="${TMP_DIR}/config.yaml" +HEADERS_FILE="${TMP_DIR}/headers.txt" +BODY_FILE="${TMP_DIR}/body.json" + +cleanup() { + if [[ "${KEEP_CONTAINERS:-}" == "1" ]]; then + echo "KEEP_CONTAINERS=1 set; leaving docker compose project ${COMPOSE_PROJECT_NAME} running." + echo "Generated config left at ${CLAUDE_CODE_LITELLM_CONFIG}" + return + fi + docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" down -v >/dev/null 2>&1 || true + rm -rf "${TMP_DIR}" +} +trap cleanup EXIT + +if [[ -f "${ENV_FILE}" ]]; then + # shellcheck disable=SC1090 + source "${ENV_FILE}" +fi + +if [[ -z "${ANTHROPIC_API_KEY:-}" ]]; then + echo "ANTHROPIC_API_KEY is required in the environment or ${ENV_FILE}" + exit 1 +fi + +if ! command -v docker >/dev/null 2>&1; then + echo "docker is required" + exit 1 +fi + +if ! docker compose version >/dev/null 2>&1; then + echo "docker compose v2 is required" + exit 1 +fi + +if ! command -v python3 >/dev/null 2>&1; then + echo "python3 is required" + exit 1 +fi + +is_port_available() { + python3 - "$1" <<'PY' +import socket +import sys + +port = int(sys.argv[1]) +sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +try: + sock.bind(("127.0.0.1", port)) + print("yes") +except OSError: + print("no") +finally: + sock.close() +PY +} + +if [[ "$(is_port_available "${PROXY_PORT}")" != "yes" ]]; then + echo "Port ${PROXY_PORT} is in use. Searching for a free port..." + FOUND_PORT="" + for candidate in $(seq 4001 4050); do + if [[ "$(is_port_available "${candidate}")" == "yes" ]]; then + FOUND_PORT="${candidate}" + break + fi + done + if [[ -z "${FOUND_PORT}" ]]; then + echo "No free local port found between 4001-4050." + exit 1 + fi + PROXY_PORT="${FOUND_PORT}" + PROXY_URL="http://127.0.0.1:${PROXY_PORT}" + echo "Using fallback PROXY_PORT=${PROXY_PORT}" +fi + +cat >"${CLAUDE_CODE_LITELLM_CONFIG}" </dev/null 2>&1; then + READY=1 + break + fi + sleep 1 +done + +if [[ "${READY}" != "1" ]]; then + echo "LiteLLM failed to become ready within ${STARTUP_TIMEOUT_S}s" + docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" logs --tail=200 litellm || true + exit 1 +fi + +echo "[3/5] Checking configured model..." +curl -fsS --connect-timeout 5 --max-time 20 \ + -H "Authorization: Bearer ${LITELLM_MASTER_KEY}" "${PROXY_URL}/v1/models" | grep -q "\"${MODEL_NAME}\"" + +echo "[4/5] Running back-to-back Claude Code requests in an isolated non-root container..." +FIRST_RESPONSE="$( +docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" run --rm \ + -e CLAUDE_CODE_PROMPT="Respond with exactly this text and nothing else: Hello from LiteLLM Claude Code request one." \ + -e CLAUDE_CODE_OUTPUT_FILE="/tmp/claude-output-1.txt" \ + claude-code +)" + +SECOND_RESPONSE="$( +docker compose -p "${COMPOSE_PROJECT_NAME}" -f "${COMPOSE_FILE}" run --rm \ + -e CLAUDE_CODE_PROMPT="Respond with exactly this text and nothing else: Hello from LiteLLM Claude Code request two." \ + -e CLAUDE_CODE_OUTPUT_FILE="/tmp/claude-output-2.txt" \ + claude-code +)" + +[[ -n "${FIRST_RESPONSE}" ]] || { + echo "First Claude Code request produced no response output." + exit 1 +} +[[ -n "${SECOND_RESPONSE}" ]] || { + echo "Second Claude Code request produced no response output." + exit 1 +} + +printf '%s\n' "${FIRST_RESPONSE}" | grep -qi "request one" || { + echo "First Claude Code response did not contain expected content." + printf '%s\n' "${FIRST_RESPONSE}" | sed -n '1,40p' + exit 1 +} +printf '%s\n' "${SECOND_RESPONSE}" | grep -qi "request two" || { + echo "Second Claude Code response did not contain expected content." + printf '%s\n' "${SECOND_RESPONSE}" | sed -n '1,40p' + exit 1 +} + +echo "[5/5] Verifying LiteLLM request headers on Anthropic messages endpoint..." +REQUEST_BODY="$(python3 - "${MODEL_NAME}" <<'PY' +import json +import sys + +print( + json.dumps( + { + "model": sys.argv[1], + "max_tokens": 16, + "messages": [ + { + "role": "user", + "content": "Respond with the word ok.", + } + ], + } + ) +) +PY +)" + +curl -fsS --connect-timeout 5 --max-time 30 -D "${HEADERS_FILE}" -o "${BODY_FILE}" \ + -X POST "${PROXY_URL}/v1/messages" \ + -H "Authorization: Bearer ${LITELLM_MASTER_KEY}" \ + -H "Content-Type: application/json" \ + -H "anthropic-version: 2023-06-01" \ + --data "${REQUEST_BODY}" + +grep -qi "^x-litellm-call-id:" "${HEADERS_FILE}" || { + echo "Missing x-litellm-call-id header; proxy record signal not found." + sed -n '1,80p' "${HEADERS_FILE}" + sed -n '1,80p' "${BODY_FILE}" + exit 1 +} + +if ! grep -qi "^x-litellm-response-cost:" "${HEADERS_FILE}" && \ + ! grep -qi "^x-litellm-response-cost-original:" "${HEADERS_FILE}"; then + echo "Missing LiteLLM response cost headers; proxy usage signal not found." + echo "Expected one of: x-litellm-response-cost or x-litellm-response-cost-original" + sed -n '1,80p' "${HEADERS_FILE}" + sed -n '1,80p' "${BODY_FILE}" + exit 1 +fi + +echo "Success: Claude Code ran in its own non-root container through LiteLLM, with Postgres-backed proxy headers present."