diff --git a/.circleci/config.yml b/.circleci/config.yml index b3969fd8446..1dba9086027 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2041,6 +2041,7 @@ jobs: set +e LITELLM_IMAGE=litellm-docker-database:ci \ CLAUDE_CODE_IMAGE=claude-code-client:ci \ + UPSTREAM_PROVIDER=anthropic \ LITELLM_SKIP_BUILD=true \ CLAUDE_CODE_SKIP_BUILD=true \ LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY}" \ @@ -2070,6 +2071,78 @@ jobs: - store_test_results: path: test-results + proxy_e2e_claude_code_bedrock_tests: + machine: + image: ubuntu-2204:2024.04.1 + resource_class: large + working_directory: ~/project + steps: + - checkout + - setup_google_dns + - attach_workspace: + at: ~/project + - run: + name: Load Docker Database Image + command: | + zstd -d litellm-docker-database.tar.zst --stdout | docker load + docker images | grep litellm-docker-database + - run: + name: Load Claude Code client image + command: | + zstd -d claude-code-client.tar.zst --stdout | docker load + docker images | grep claude-code-client + - run: + name: Verify required container images stage + command: | + docker image inspect litellm-docker-database:ci >/dev/null + docker image inspect claude-code-client:ci >/dev/null + echo "LiteLLM and Claude Code images are ready" + - run: + name: Run Dockerized Claude Code Bedrock E2E Tests + command: | + mkdir -p test-results + : "${LITELLM_MASTER_KEY:?Set LITELLM_MASTER_KEY in CircleCI project env vars.}" + : "${AWS_ACCESS_KEY_ID:?Set AWS_ACCESS_KEY_ID in CircleCI project env vars.}" + : "${AWS_SECRET_ACCESS_KEY:?Set AWS_SECRET_ACCESS_KEY in CircleCI project env vars.}" + : "${AWS_REGION_NAME:?Set AWS_REGION_NAME in CircleCI project env vars.}" + set +e + LITELLM_IMAGE=litellm-docker-database:ci \ + CLAUDE_CODE_IMAGE=claude-code-client:ci \ + UPSTREAM_PROVIDER=bedrock \ + MODEL_NAME=claude-bedrock-sonnet \ + LITELLM_UPSTREAM_MODEL=bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0 \ + LITELLM_SKIP_BUILD=true \ + CLAUDE_CODE_SKIP_BUILD=true \ + LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY}" \ + AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}" \ + AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}" \ + AWS_REGION_NAME="${AWS_REGION_NAME}" \ + tests/proxy_e2e_anthropic_messages_tests/claude_code/run_claude_code_docker_test.sh + status=$? + if [ "$status" -eq 0 ]; then + printf '%s\n' \ + '' \ + '' \ + ' ' \ + '' \ + > test-results/junit.xml + else + printf '%s\n' \ + '' \ + '' \ + ' ' \ + ' See CircleCI job output for Docker and LiteLLM logs.' \ + ' ' \ + '' \ + > test-results/junit.xml + fi + exit "$status" + no_output_timeout: 15m + + # Store test results + - store_test_results: + path: test-results + upload-coverage: docker: - *python312_image @@ -2450,6 +2523,11 @@ workflows: - build_docker_database_image - build_claude_code_client_image filters: *main_branches + - proxy_e2e_claude_code_bedrock_tests: + requires: + - build_docker_database_image + - build_claude_code_client_image + filters: *main_branches - llm_translation_testing: filters: *main_branches - realtime_translation_testing: diff --git a/tests/proxy_e2e_anthropic_messages_tests/claude_code/README.md b/tests/proxy_e2e_anthropic_messages_tests/claude_code/README.md index 5d549769421..0f42521aac5 100644 --- a/tests/proxy_e2e_anthropic_messages_tests/claude_code/README.md +++ b/tests/proxy_e2e_anthropic_messages_tests/claude_code/README.md @@ -1,6 +1,6 @@ # Claude Code LiteLLM E2E Test -This V0 test validates the customer-critical Claude Code -> LiteLLM -> Anthropic path in CircleCI. +This test validates the customer-critical Claude Code -> LiteLLM path in CircleCI for Anthropic and Bedrock upstream providers. ## What It Covers @@ -9,13 +9,14 @@ This V0 test validates the customer-critical Claude Code -> LiteLLM -> Anthropic - Builds a separate Claude Code client container. - Runs Claude Code as a non-root user with only the LiteLLM proxy key. - Points Claude Code at LiteLLM via `ANTHROPIC_BASE_URL`. -- Verifies the configured Anthropic model is visible from `/v1/models`. +- Verifies the configured model alias is visible from `/v1/models`. - Sends back-to-back Claude Code prompts through LiteLLM and validates both responses. - Sends a direct Anthropic Messages API request through LiteLLM and checks proxy response headers: - `x-litellm-call-id` - `x-litellm-response-cost` The direct `/v1/messages` header check gives a clear proxy-side signal that LiteLLM handled and accounted for the request, instead of only proving the client printed text. +For Bedrock, LiteLLM still exposes an Anthropic-compatible interface to Claude Code while routing upstream via Bedrock credentials. ## Claude Code Version Policy @@ -27,7 +28,7 @@ Set `CLAUDE_CODE_VERSION=` to test a specific version locally or in a f ## Running Locally -Set `ANTHROPIC_API_KEY` in the environment or in `tests/proxy_e2e_anthropic_messages_tests/claude_code/.env`, then run: +Set provider credentials in the environment or in `tests/proxy_e2e_anthropic_messages_tests/claude_code/.env`, then run: ```bash tests/proxy_e2e_anthropic_messages_tests/claude_code/run_claude_code_docker_test.sh @@ -38,11 +39,24 @@ Useful overrides: ```bash MODEL_NAME=claude-sonnet-4-6 LITELLM_UPSTREAM_MODEL=anthropic/claude-sonnet-4-6 +UPSTREAM_PROVIDER=anthropic CLAUDE_CODE_IMAGE=litellm-claude-code-client:local CLAUDE_CODE_VERSION=latest KEEP_CONTAINERS=1 ``` +Bedrock example: + +```bash +UPSTREAM_PROVIDER=bedrock +MODEL_NAME=claude-bedrock-sonnet +LITELLM_UPSTREAM_MODEL=bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0 +AWS_ACCESS_KEY_ID=... +AWS_SECRET_ACCESS_KEY=... +AWS_REGION_NAME=us-east-1 +tests/proxy_e2e_anthropic_messages_tests/claude_code/run_claude_code_docker_test.sh +``` + In CircleCI, the job sets `LITELLM_IMAGE=litellm-docker-database:ci` and `LITELLM_SKIP_BUILD=true` so the test uses the LiteLLM image already built from the checked-out commit. The CircleCI job now has explicit component stages before test execution: @@ -52,8 +66,7 @@ The CircleCI job now has explicit component stages before test execution: ## Intentionally Left Out Of V0 -- Bedrock and other providers. Those belong in V1 after the Anthropic path is stable. -- Back-to-back session behavior. V0 proves the request path works; V1 can add repeated requests and long-running session checks. +- Vertex and Azure provider coverage. Those belong in a follow-up matrix after Anthropic + Bedrock. - Tool-use assertions. Claude Code can invoke tools in richer scenarios, but V0 keeps the signal focused on proxy compatibility and real Anthropic request success. - A Claude Code version matrix. The Dockerfile supports `CLAUDE_CODE_VERSION`, but the default CI path tests one recent version to keep cost and flake surface low. diff --git a/tests/proxy_e2e_anthropic_messages_tests/claude_code/docker-compose.yaml b/tests/proxy_e2e_anthropic_messages_tests/claude_code/docker-compose.yaml index 4123cee1ffb..378dff2d73d 100644 --- a/tests/proxy_e2e_anthropic_messages_tests/claude_code/docker-compose.yaml +++ b/tests/proxy_e2e_anthropic_messages_tests/claude_code/docker-compose.yaml @@ -23,7 +23,11 @@ services: environment: DATABASE_URL: postgresql://litellm:litellm@postgres:5432/litellm LITELLM_MASTER_KEY: ${LITELLM_MASTER_KEY:-sk-darcy-sf-onsite-interview} - ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:?ANTHROPIC_API_KEY is required} + ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-} + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-} + AWS_REGION_NAME: ${AWS_REGION_NAME:-} + AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN:-} LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS: "True" volumes: - ${CLAUDE_CODE_LITELLM_CONFIG:?CLAUDE_CODE_LITELLM_CONFIG is required}:/app/config.yaml:ro diff --git a/tests/proxy_e2e_anthropic_messages_tests/claude_code/run_claude_code_docker_test.sh b/tests/proxy_e2e_anthropic_messages_tests/claude_code/run_claude_code_docker_test.sh index e328c311a57..f6485a110a5 100755 --- a/tests/proxy_e2e_anthropic_messages_tests/claude_code/run_claude_code_docker_test.sh +++ b/tests/proxy_e2e_anthropic_messages_tests/claude_code/run_claude_code_docker_test.sh @@ -4,11 +4,13 @@ set -euo pipefail # Dockerized Claude Code + LiteLLM integration test. # # Required: -# ANTHROPIC_API_KEY or tests/proxy_e2e_anthropic_messages_tests/claude_code/.env +# For anthropic: ANTHROPIC_API_KEY or tests/proxy_e2e_anthropic_messages_tests/claude_code/.env +# For bedrock: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_REGION_NAME # # Optional: -# MODEL_NAME=claude-sonnet-4-6 -# LITELLM_UPSTREAM_MODEL=anthropic/${MODEL_NAME} +# UPSTREAM_PROVIDER=anthropic|bedrock (default anthropic) +# MODEL_NAME= +# LITELLM_UPSTREAM_MODEL= # LITELLM_IMAGE=litellm-claude-code-e2e:local # CLAUDE_CODE_IMAGE=litellm-claude-code-client:local # LITELLM_SKIP_BUILD=true @@ -19,8 +21,8 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" COMPOSE_FILE="${SCRIPT_DIR}/docker-compose.yaml" ENV_FILE="${SCRIPT_DIR}/.env" +UPSTREAM_PROVIDER="${UPSTREAM_PROVIDER:-anthropic}" MODEL_NAME="${MODEL_NAME:-claude-sonnet-4-6}" -LITELLM_UPSTREAM_MODEL="${LITELLM_UPSTREAM_MODEL:-anthropic/${MODEL_NAME}}" LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY:-sk-darcy-sf-onsite-interview}" PROXY_PORT="${PROXY_PORT:-4000}" PROXY_URL="http://127.0.0.1:${PROXY_PORT}" @@ -47,8 +49,17 @@ if [[ -f "${ENV_FILE}" ]]; then source "${ENV_FILE}" fi -if [[ -z "${ANTHROPIC_API_KEY:-}" ]]; then - echo "ANTHROPIC_API_KEY is required in the environment or ${ENV_FILE}" +if [[ "${UPSTREAM_PROVIDER}" == "anthropic" ]]; then + if [[ -z "${ANTHROPIC_API_KEY:-}" ]]; then + echo "ANTHROPIC_API_KEY is required for UPSTREAM_PROVIDER=anthropic (env or ${ENV_FILE})" + exit 1 + fi +elif [[ "${UPSTREAM_PROVIDER}" == "bedrock" ]]; then + : "${AWS_ACCESS_KEY_ID:?AWS_ACCESS_KEY_ID is required for UPSTREAM_PROVIDER=bedrock}" + : "${AWS_SECRET_ACCESS_KEY:?AWS_SECRET_ACCESS_KEY is required for UPSTREAM_PROVIDER=bedrock}" + : "${AWS_REGION_NAME:?AWS_REGION_NAME is required for UPSTREAM_PROVIDER=bedrock}" +else + echo "Unsupported UPSTREAM_PROVIDER=${UPSTREAM_PROVIDER}. Use anthropic or bedrock." exit 1 fi @@ -102,12 +113,26 @@ if [[ "$(is_port_available "${PROXY_PORT}")" != "yes" ]]; then echo "Using fallback PROXY_PORT=${PROXY_PORT}" fi -cat >"${CLAUDE_CODE_LITELLM_CONFIG}" <"${CLAUDE_CODE_LITELLM_CONFIG}" <"${CLAUDE_CODE_LITELLM_CONFIG}" <>"${CLAUDE_CODE_LITELLM_CONFIG}" <