diff --git a/.circleci/config.yml b/.circleci/config.yml index 84b15572a66..e5bc82a5967 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3407,6 +3407,110 @@ jobs: - store_test_results: path: test-results + proxy_e2e_anthropic_messages_tests: + machine: + image: ubuntu-2204:2023.10.1 + resource_class: xlarge + working_directory: ~/project + steps: + - checkout + - setup_google_dns + - run: + name: Install Docker CLI (In case it's not already installed) + command: | + curl -fsSL https://get.docker.com | sh + sudo usermod -aG docker $USER + docker version + - run: + name: Install Python 3.10 + command: | + curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh + bash miniconda.sh -b -p $HOME/miniconda + export PATH="$HOME/miniconda/bin:$PATH" + conda init bash + source ~/.bashrc + conda create -n myenv python=3.10 -y + conda activate myenv + python --version + - run: + name: Install Dependencies + command: | + export PATH="$HOME/miniconda/bin:$PATH" + source $HOME/miniconda/etc/profile.d/conda.sh + conda activate myenv + pip install "pytest==7.3.1" + pip install "pytest-asyncio==0.21.1" + pip install "boto3==1.36.0" + pip install "httpx==0.27.0" + pip install "claude-agent-sdk" + pip install -r requirements.txt + - run: + name: Install dockerize + command: | + wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + rm dockerize-linux-amd64-v0.6.1.tar.gz + - run: + name: Start PostgreSQL Database + command: | + docker run -d \ + --name postgres-db \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=circle_test \ + -p 5432:5432 \ + postgres:14 + - run: + name: Wait for PostgreSQL to be ready + command: dockerize -wait tcp://localhost:5432 -timeout 1m + - attach_workspace: + at: ~/project + - run: + name: Load Docker Database Image + command: | + gunzip -c litellm-docker-database.tar.gz | docker load + docker images | grep litellm-docker-database + - run: + name: Run Docker container with test config + command: | + docker run -d \ + -p 4000:4000 \ + -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ + -e LITELLM_MASTER_KEY="sk-1234" \ + -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ + -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ + -e AWS_REGION_NAME="us-east-1" \ + --add-host host.docker.internal:host-gateway \ + --name my-app \ + -v $(pwd)/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml:/app/config.yaml \ + litellm-docker-database:ci \ + --config /app/config.yaml \ + --port 4000 \ + --detailed_debug + - run: + name: Start outputting logs + command: docker logs -f my-app + background: true + - run: + name: Wait for app to be ready + command: dockerize -wait http://localhost:4000 -timeout 5m + - run: + name: Run Claude Agent SDK E2E Tests + command: | + export PATH="$HOME/miniconda/bin:$PATH" + source $HOME/miniconda/etc/profile.d/conda.sh + conda activate myenv + export LITELLM_PROXY_URL="http://localhost:4000" + export LITELLM_API_KEY="sk-1234" + pwd + ls + python -m pytest -vv tests/proxy_e2e_anthropic_messages_tests/ -x -s --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 120m + + # Store test results + - store_test_results: + path: test-results + upload-coverage: docker: - image: cimg/python:3.9 @@ -4075,6 +4179,14 @@ workflows: only: - main - /litellm_.*/ + - proxy_e2e_anthropic_messages_tests: + requires: + - build_docker_database_image + filters: + branches: + only: + - main + - /litellm_.*/ - llm_translation_testing: filters: branches: diff --git a/tests/proxy_e2e_anthropic_messages_tests/test_claude_agent_sdk.py b/tests/proxy_e2e_anthropic_messages_tests/test_claude_agent_sdk.py new file mode 100644 index 00000000000..0838faf4212 --- /dev/null +++ b/tests/proxy_e2e_anthropic_messages_tests/test_claude_agent_sdk.py @@ -0,0 +1,120 @@ +""" +E2E tests for Claude Agent SDK with LiteLLM Proxy using Bedrock models. + +Tests streaming messages across different Bedrock models: +- Regular Bedrock Claude Sonnet 4.5 +- Bedrock Converse Claude Sonnet 4.5 +- AWS Nova Premier +""" + +import os +import pytest +import asyncio +from claude_agent_sdk import ClaudeSDKClient, ClaudeAgentOptions + + +# Test models from proxy_config.yaml +TEST_MODELS = [ + ("bedrock-claude-sonnet-4.5", "Bedrock Invoke API"), + ("bedrock-converse-claude-sonnet-4.5", "Bedrock Converse API"), + ("bedrock-nova-premier", "AWS Nova Premier"), +] + + +@pytest.fixture(scope="module") +def litellm_proxy_config(): + """Configure connection to LiteLLM proxy""" + proxy_url = os.getenv("LITELLM_PROXY_URL", "http://localhost:4000") + api_key = os.getenv("LITELLM_API_KEY", "sk-1234") + + # Set environment variables for Claude Agent SDK + os.environ["ANTHROPIC_BASE_URL"] = proxy_url.rstrip('/') + os.environ["ANTHROPIC_API_KEY"] = api_key + + return { + "proxy_url": proxy_url, + "api_key": api_key, + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model_name,model_description", TEST_MODELS) +async def test_claude_agent_sdk_streaming(litellm_proxy_config, model_name, model_description): + """ + Test streaming messages with Claude Agent SDK through LiteLLM proxy. + + This validates: + 1. Claude Agent SDK can connect to LiteLLM proxy + 2. Streaming works correctly + 3. Different Bedrock models (Invoke, Converse, Nova) work end-to-end + """ + print(f"\n{'='*60}") + print(f"Testing: {model_name} ({model_description})") + print(f"{'='*60}") + + # Configure agent options + options = ClaudeAgentOptions( + system_prompt="You are a helpful AI assistant. Be concise.", + model=model_name, + max_turns=5, + ) + + # Test query + test_query = "Say 'Hello from LiteLLM!' and nothing else." + + # Track streaming + received_chunks = [] + full_response = "" + + try: + async with ClaudeSDKClient(options=options) as client: + await client.query(test_query) + + # Collect streaming response + async for msg in client.receive_response(): + # Handle different message types + if hasattr(msg, 'type'): + if msg.type == 'content_block_delta': + # Streaming text delta + if hasattr(msg, 'delta') and hasattr(msg.delta, 'text'): + chunk_text = msg.delta.text + received_chunks.append(chunk_text) + full_response += chunk_text + elif msg.type == 'content_block_start': + # Start of content block + if hasattr(msg, 'content_block') and hasattr(msg.content_block, 'text'): + chunk_text = msg.content_block.text + received_chunks.append(chunk_text) + full_response += chunk_text + + # Fallback to content handling + if hasattr(msg, 'content'): + for content_block in msg.content: + if hasattr(content_block, 'text'): + chunk_text = content_block.text + received_chunks.append(chunk_text) + full_response += chunk_text + + # Assertions + print(f"\nāœ… Received {len(received_chunks)} chunks") + print(f"šŸ“ Full response: {full_response[:100]}...") + + # Verify we got a response + assert len(full_response) > 0, f"No response received from {model_name}" + + # Verify streaming (should have multiple chunks for most responses) + # Note: Very short responses might come in 1 chunk, so we just verify we got content + assert len(received_chunks) > 0, f"No chunks received from {model_name}" + + # Verify response contains expected content (case insensitive) + assert "hello" in full_response.lower(), f"Response doesn't contain expected greeting: {full_response}" + + print(f"āœ… Test passed for {model_name}") + + except Exception as e: + pytest.fail(f"Test failed for {model_name} ({model_description}): {str(e)}") + + +if __name__ == "__main__": + # Run tests + pytest.main([__file__, "-v", "-s"]) diff --git a/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml b/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml new file mode 100644 index 00000000000..35367860f13 --- /dev/null +++ b/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml @@ -0,0 +1,31 @@ +model_list: + - model_name: bedrock-claude-sonnet-3.5 + litellm_params: + model: "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0" + aws_region_name: "us-east-1" + + - model_name: bedrock-claude-sonnet-4 + litellm_params: + model: "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0" + aws_region_name: "us-east-1" + + - model_name: bedrock-claude-sonnet-4.5 + litellm_params: + model: "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0" + aws_region_name: "us-east-1" + + - model_name: bedrock-claude-opus-4.5 + litellm_params: + model: "bedrock/us.anthropic.claude-opus-4-5-20251101-v1:0" + aws_region_name: "us-east-1" + + - model_name: bedrock-nova-premier + litellm_params: + model: "bedrock/amazon.nova-premier-v1:0" + aws_region_name: "us-east-1" + + # Converse API models + - model_name: bedrock-converse-claude-sonnet-4.5 + litellm_params: + model: "bedrock_converse/us.anthropic.claude-sonnet-4-5-20250929-v1:0" + aws_region_name: "us-east-1"