From 258328fa0dfcda2d0fbc6e4f8585d0c56fbf1130 Mon Sep 17 00:00:00 2001 From: Darcy Liu Date: Sat, 25 Apr 2026 23:13:57 +0000 Subject: [PATCH] feat(ci): add support for Bedrock provider in Claude Code E2E tests Enhance the CircleCI configuration to include a new job for running end-to-end tests with the Bedrock upstream provider. This includes loading necessary Docker images, verifying their availability, and executing tests with appropriate environment variables for both Anthropic and Bedrock. Update the README to reflect changes in provider support and required credentials. Additionally, modify the Docker test script to handle environment variables for both providers, ensuring proper configuration based on the selected upstream provider. Made-with: Cursor --- .circleci/config.yml | 78 +++++++++++++++++++ .../claude_code/README.md | 23 ++++-- .../claude_code/docker-compose.yaml | 6 +- .../run_claude_code_docker_test.sh | 47 +++++++++-- 4 files changed, 140 insertions(+), 14 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b3969fd8446..1dba9086027 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2041,6 +2041,7 @@ jobs: set +e LITELLM_IMAGE=litellm-docker-database:ci \ CLAUDE_CODE_IMAGE=claude-code-client:ci \ + UPSTREAM_PROVIDER=anthropic \ LITELLM_SKIP_BUILD=true \ CLAUDE_CODE_SKIP_BUILD=true \ LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY}" \ @@ -2070,6 +2071,78 @@ jobs: - store_test_results: path: test-results + proxy_e2e_claude_code_bedrock_tests: + machine: + image: ubuntu-2204:2024.04.1 + resource_class: large + working_directory: ~/project + steps: + - checkout + - setup_google_dns + - attach_workspace: + at: ~/project + - run: + name: Load Docker Database Image + command: | + zstd -d litellm-docker-database.tar.zst --stdout | docker load + docker images | grep litellm-docker-database + - run: + name: Load Claude Code client image + command: | + zstd -d claude-code-client.tar.zst --stdout | docker load + docker images | grep claude-code-client + - run: + name: Verify required container images stage + command: | + docker image inspect litellm-docker-database:ci >/dev/null + docker image inspect claude-code-client:ci >/dev/null + echo "LiteLLM and Claude Code images are ready" + - run: + name: Run Dockerized Claude Code Bedrock E2E Tests + command: | + mkdir -p test-results + : "${LITELLM_MASTER_KEY:?Set LITELLM_MASTER_KEY in CircleCI project env vars.}" + : "${AWS_ACCESS_KEY_ID:?Set AWS_ACCESS_KEY_ID in CircleCI project env vars.}" + : "${AWS_SECRET_ACCESS_KEY:?Set AWS_SECRET_ACCESS_KEY in CircleCI project env vars.}" + : "${AWS_REGION_NAME:?Set AWS_REGION_NAME in CircleCI project env vars.}" + set +e + LITELLM_IMAGE=litellm-docker-database:ci \ + CLAUDE_CODE_IMAGE=claude-code-client:ci \ + UPSTREAM_PROVIDER=bedrock \ + MODEL_NAME=claude-bedrock-sonnet \ + LITELLM_UPSTREAM_MODEL=bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0 \ + LITELLM_SKIP_BUILD=true \ + CLAUDE_CODE_SKIP_BUILD=true \ + LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY}" \ + AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}" \ + AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}" \ + AWS_REGION_NAME="${AWS_REGION_NAME}" \ + tests/proxy_e2e_anthropic_messages_tests/claude_code/run_claude_code_docker_test.sh + status=$? + if [ "$status" -eq 0 ]; then + printf '%s\n' \ + '' \ + '' \ + ' ' \ + '' \ + > test-results/junit.xml + else + printf '%s\n' \ + '' \ + '' \ + ' ' \ + ' See CircleCI job output for Docker and LiteLLM logs.' \ + ' ' \ + '' \ + > test-results/junit.xml + fi + exit "$status" + no_output_timeout: 15m + + # Store test results + - store_test_results: + path: test-results + upload-coverage: docker: - *python312_image @@ -2450,6 +2523,11 @@ workflows: - build_docker_database_image - build_claude_code_client_image filters: *main_branches + - proxy_e2e_claude_code_bedrock_tests: + requires: + - build_docker_database_image + - build_claude_code_client_image + filters: *main_branches - llm_translation_testing: filters: *main_branches - realtime_translation_testing: diff --git a/tests/proxy_e2e_anthropic_messages_tests/claude_code/README.md b/tests/proxy_e2e_anthropic_messages_tests/claude_code/README.md index 5d549769421..0f42521aac5 100644 --- a/tests/proxy_e2e_anthropic_messages_tests/claude_code/README.md +++ b/tests/proxy_e2e_anthropic_messages_tests/claude_code/README.md @@ -1,6 +1,6 @@ # Claude Code LiteLLM E2E Test -This V0 test validates the customer-critical Claude Code -> LiteLLM -> Anthropic path in CircleCI. +This test validates the customer-critical Claude Code -> LiteLLM path in CircleCI for Anthropic and Bedrock upstream providers. ## What It Covers @@ -9,13 +9,14 @@ This V0 test validates the customer-critical Claude Code -> LiteLLM -> Anthropic - Builds a separate Claude Code client container. - Runs Claude Code as a non-root user with only the LiteLLM proxy key. - Points Claude Code at LiteLLM via `ANTHROPIC_BASE_URL`. -- Verifies the configured Anthropic model is visible from `/v1/models`. +- Verifies the configured model alias is visible from `/v1/models`. - Sends back-to-back Claude Code prompts through LiteLLM and validates both responses. - Sends a direct Anthropic Messages API request through LiteLLM and checks proxy response headers: - `x-litellm-call-id` - `x-litellm-response-cost` The direct `/v1/messages` header check gives a clear proxy-side signal that LiteLLM handled and accounted for the request, instead of only proving the client printed text. +For Bedrock, LiteLLM still exposes an Anthropic-compatible interface to Claude Code while routing upstream via Bedrock credentials. ## Claude Code Version Policy @@ -27,7 +28,7 @@ Set `CLAUDE_CODE_VERSION=` to test a specific version locally or in a f ## Running Locally -Set `ANTHROPIC_API_KEY` in the environment or in `tests/proxy_e2e_anthropic_messages_tests/claude_code/.env`, then run: +Set provider credentials in the environment or in `tests/proxy_e2e_anthropic_messages_tests/claude_code/.env`, then run: ```bash tests/proxy_e2e_anthropic_messages_tests/claude_code/run_claude_code_docker_test.sh @@ -38,11 +39,24 @@ Useful overrides: ```bash MODEL_NAME=claude-sonnet-4-6 LITELLM_UPSTREAM_MODEL=anthropic/claude-sonnet-4-6 +UPSTREAM_PROVIDER=anthropic CLAUDE_CODE_IMAGE=litellm-claude-code-client:local CLAUDE_CODE_VERSION=latest KEEP_CONTAINERS=1 ``` +Bedrock example: + +```bash +UPSTREAM_PROVIDER=bedrock +MODEL_NAME=claude-bedrock-sonnet +LITELLM_UPSTREAM_MODEL=bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0 +AWS_ACCESS_KEY_ID=... +AWS_SECRET_ACCESS_KEY=... +AWS_REGION_NAME=us-east-1 +tests/proxy_e2e_anthropic_messages_tests/claude_code/run_claude_code_docker_test.sh +``` + In CircleCI, the job sets `LITELLM_IMAGE=litellm-docker-database:ci` and `LITELLM_SKIP_BUILD=true` so the test uses the LiteLLM image already built from the checked-out commit. The CircleCI job now has explicit component stages before test execution: @@ -52,8 +66,7 @@ The CircleCI job now has explicit component stages before test execution: ## Intentionally Left Out Of V0 -- Bedrock and other providers. Those belong in V1 after the Anthropic path is stable. -- Back-to-back session behavior. V0 proves the request path works; V1 can add repeated requests and long-running session checks. +- Vertex and Azure provider coverage. Those belong in a follow-up matrix after Anthropic + Bedrock. - Tool-use assertions. Claude Code can invoke tools in richer scenarios, but V0 keeps the signal focused on proxy compatibility and real Anthropic request success. - A Claude Code version matrix. The Dockerfile supports `CLAUDE_CODE_VERSION`, but the default CI path tests one recent version to keep cost and flake surface low. diff --git a/tests/proxy_e2e_anthropic_messages_tests/claude_code/docker-compose.yaml b/tests/proxy_e2e_anthropic_messages_tests/claude_code/docker-compose.yaml index 4123cee1ffb..378dff2d73d 100644 --- a/tests/proxy_e2e_anthropic_messages_tests/claude_code/docker-compose.yaml +++ b/tests/proxy_e2e_anthropic_messages_tests/claude_code/docker-compose.yaml @@ -23,7 +23,11 @@ services: environment: DATABASE_URL: postgresql://litellm:litellm@postgres:5432/litellm LITELLM_MASTER_KEY: ${LITELLM_MASTER_KEY:-sk-darcy-sf-onsite-interview} - ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:?ANTHROPIC_API_KEY is required} + ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-} + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-} + AWS_REGION_NAME: ${AWS_REGION_NAME:-} + AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN:-} LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS: "True" volumes: - ${CLAUDE_CODE_LITELLM_CONFIG:?CLAUDE_CODE_LITELLM_CONFIG is required}:/app/config.yaml:ro diff --git a/tests/proxy_e2e_anthropic_messages_tests/claude_code/run_claude_code_docker_test.sh b/tests/proxy_e2e_anthropic_messages_tests/claude_code/run_claude_code_docker_test.sh index e328c311a57..f6485a110a5 100755 --- a/tests/proxy_e2e_anthropic_messages_tests/claude_code/run_claude_code_docker_test.sh +++ b/tests/proxy_e2e_anthropic_messages_tests/claude_code/run_claude_code_docker_test.sh @@ -4,11 +4,13 @@ set -euo pipefail # Dockerized Claude Code + LiteLLM integration test. # # Required: -# ANTHROPIC_API_KEY or tests/proxy_e2e_anthropic_messages_tests/claude_code/.env +# For anthropic: ANTHROPIC_API_KEY or tests/proxy_e2e_anthropic_messages_tests/claude_code/.env +# For bedrock: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_REGION_NAME # # Optional: -# MODEL_NAME=claude-sonnet-4-6 -# LITELLM_UPSTREAM_MODEL=anthropic/${MODEL_NAME} +# UPSTREAM_PROVIDER=anthropic|bedrock (default anthropic) +# MODEL_NAME= +# LITELLM_UPSTREAM_MODEL= # LITELLM_IMAGE=litellm-claude-code-e2e:local # CLAUDE_CODE_IMAGE=litellm-claude-code-client:local # LITELLM_SKIP_BUILD=true @@ -19,8 +21,8 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" COMPOSE_FILE="${SCRIPT_DIR}/docker-compose.yaml" ENV_FILE="${SCRIPT_DIR}/.env" +UPSTREAM_PROVIDER="${UPSTREAM_PROVIDER:-anthropic}" MODEL_NAME="${MODEL_NAME:-claude-sonnet-4-6}" -LITELLM_UPSTREAM_MODEL="${LITELLM_UPSTREAM_MODEL:-anthropic/${MODEL_NAME}}" LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY:-sk-darcy-sf-onsite-interview}" PROXY_PORT="${PROXY_PORT:-4000}" PROXY_URL="http://127.0.0.1:${PROXY_PORT}" @@ -47,8 +49,17 @@ if [[ -f "${ENV_FILE}" ]]; then source "${ENV_FILE}" fi -if [[ -z "${ANTHROPIC_API_KEY:-}" ]]; then - echo "ANTHROPIC_API_KEY is required in the environment or ${ENV_FILE}" +if [[ "${UPSTREAM_PROVIDER}" == "anthropic" ]]; then + if [[ -z "${ANTHROPIC_API_KEY:-}" ]]; then + echo "ANTHROPIC_API_KEY is required for UPSTREAM_PROVIDER=anthropic (env or ${ENV_FILE})" + exit 1 + fi +elif [[ "${UPSTREAM_PROVIDER}" == "bedrock" ]]; then + : "${AWS_ACCESS_KEY_ID:?AWS_ACCESS_KEY_ID is required for UPSTREAM_PROVIDER=bedrock}" + : "${AWS_SECRET_ACCESS_KEY:?AWS_SECRET_ACCESS_KEY is required for UPSTREAM_PROVIDER=bedrock}" + : "${AWS_REGION_NAME:?AWS_REGION_NAME is required for UPSTREAM_PROVIDER=bedrock}" +else + echo "Unsupported UPSTREAM_PROVIDER=${UPSTREAM_PROVIDER}. Use anthropic or bedrock." exit 1 fi @@ -102,12 +113,26 @@ if [[ "$(is_port_available "${PROXY_PORT}")" != "yes" ]]; then echo "Using fallback PROXY_PORT=${PROXY_PORT}" fi -cat >"${CLAUDE_CODE_LITELLM_CONFIG}" <"${CLAUDE_CODE_LITELLM_CONFIG}" <"${CLAUDE_CODE_LITELLM_CONFIG}" <>"${CLAUDE_CODE_LITELLM_CONFIG}" <