init: proxy_e2e_anthropic_messages_tests

.circleci/config.yml

@@ -3407,6 +3407,110 @@ jobs:
       - store_test_results:
           path: test-results
 
+  proxy_e2e_anthropic_messages_tests:
+    machine:
+      image: ubuntu-2204:2023.10.1
+    resource_class: xlarge
+    working_directory: ~/project
+    steps:
+      - checkout
+      - setup_google_dns
+      - run:
+          name: Install Docker CLI (In case it's not already installed)
+          command: |
+            curl -fsSL https://get.docker.com | sh
+            sudo usermod -aG docker $USER
+            docker version
+      - run:
+          name: Install Python 3.10
+          command: |
+            curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
+            bash miniconda.sh -b -p $HOME/miniconda
+            export PATH="$HOME/miniconda/bin:$PATH"
+            conda init bash
+            source ~/.bashrc
+            conda create -n myenv python=3.10 -y
+            conda activate myenv
+            python --version
+      - run:
+          name: Install Dependencies
+          command: |
+            export PATH="$HOME/miniconda/bin:$PATH"
+            source $HOME/miniconda/etc/profile.d/conda.sh
+            conda activate myenv
+            pip install "pytest==7.3.1"
+            pip install "pytest-asyncio==0.21.1"
+            pip install "boto3==1.36.0"
+            pip install "httpx==0.27.0"
+            pip install "claude-agent-sdk"
+            pip install -r requirements.txt
+      - run:
+          name: Install dockerize
+          command: |
+            wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
+            sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
+            rm dockerize-linux-amd64-v0.6.1.tar.gz
+      - run:
+          name: Start PostgreSQL Database
+          command: |
+            docker run -d \
+              --name postgres-db \
+              -e POSTGRES_USER=postgres \
+              -e POSTGRES_PASSWORD=postgres \
+              -e POSTGRES_DB=circle_test \
+              -p 5432:5432 \
+              postgres:14
+      - run:
+          name: Wait for PostgreSQL to be ready
+          command: dockerize -wait tcp://localhost:5432 -timeout 1m
+      - attach_workspace:
+          at: ~/project
+      - run:
+          name: Load Docker Database Image
+          command: |
+            gunzip -c litellm-docker-database.tar.gz | docker load
+            docker images | grep litellm-docker-database
+      - run:
+          name: Run Docker container with test config
+          command: |
+            docker run -d \
+              -p 4000:4000 \
+              -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
+              -e LITELLM_MASTER_KEY="sk-1234" \
+              -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
+              -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
+              -e AWS_REGION_NAME="us-east-1" \
+              --add-host host.docker.internal:host-gateway \
+              --name my-app \
+              -v $(pwd)/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml:/app/config.yaml \
+              litellm-docker-database:ci \
+              --config /app/config.yaml \
+              --port 4000 \
+              --detailed_debug
+      - run:
+          name: Start outputting logs
+          command: docker logs -f my-app
+          background: true
+      - run:
+          name: Wait for app to be ready
+          command: dockerize -wait http://localhost:4000 -timeout 5m
+      - run:
+          name: Run Claude Agent SDK E2E Tests
+          command: |
+            export PATH="$HOME/miniconda/bin:$PATH"
+            source $HOME/miniconda/etc/profile.d/conda.sh
+            conda activate myenv
+            export LITELLM_PROXY_URL="http://localhost:4000"
+            export LITELLM_API_KEY="sk-1234"
+            pwd
+            ls
+            python -m pytest -vv tests/proxy_e2e_anthropic_messages_tests/ -x -s --junitxml=test-results/junit.xml --durations=5
+          no_output_timeout: 120m
+
+      # Store test results
+      - store_test_results:
+          path: test-results
+
   upload-coverage:
     docker:
       - image: cimg/python:3.9
@@ -4075,6 +4179,14 @@ workflows:
               only:
                 - main
                 - /litellm_.*/
+      - proxy_e2e_anthropic_messages_tests:
+          requires:
+            - build_docker_database_image
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
       - llm_translation_testing:
           filters:
             branches:

tests/proxy_e2e_anthropic_messages_tests/test_claude_agent_sdk.py

@@ -0,0 +1,120 @@
+"""
+E2E tests for Claude Agent SDK with LiteLLM Proxy using Bedrock models.
+
+Tests streaming messages across different Bedrock models:
+- Regular Bedrock Claude Sonnet 4.5
+- Bedrock Converse Claude Sonnet 4.5
+- AWS Nova Premier
+"""
+
+import os
+import pytest
+import asyncio
+from claude_agent_sdk import ClaudeSDKClient, ClaudeAgentOptions
+
+
+# Test models from proxy_config.yaml
+TEST_MODELS = [
+    ("bedrock-claude-sonnet-4.5", "Bedrock Invoke API"),
+    ("bedrock-converse-claude-sonnet-4.5", "Bedrock Converse API"),
+    ("bedrock-nova-premier", "AWS Nova Premier"),
+]
+
+
+@pytest.fixture(scope="module")
+def litellm_proxy_config():
+    """Configure connection to LiteLLM proxy"""
+    proxy_url = os.getenv("LITELLM_PROXY_URL", "http://localhost:4000")
+    api_key = os.getenv("LITELLM_API_KEY", "sk-1234")
+    
+    # Set environment variables for Claude Agent SDK
+    os.environ["ANTHROPIC_BASE_URL"] = proxy_url.rstrip('/')
+    os.environ["ANTHROPIC_API_KEY"] = api_key
+    
+    return {
+        "proxy_url": proxy_url,
+        "api_key": api_key,
+    }
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("model_name,model_description", TEST_MODELS)
+async def test_claude_agent_sdk_streaming(litellm_proxy_config, model_name, model_description):
+    """
+    Test streaming messages with Claude Agent SDK through LiteLLM proxy.
+    
+    This validates:
+    1. Claude Agent SDK can connect to LiteLLM proxy
+    2. Streaming works correctly
+    3. Different Bedrock models (Invoke, Converse, Nova) work end-to-end
+    """
+    print(f"\n{'='*60}")
+    print(f"Testing: {model_name} ({model_description})")
+    print(f"{'='*60}")
+    
+    # Configure agent options
+    options = ClaudeAgentOptions(
+        system_prompt="You are a helpful AI assistant. Be concise.",
+        model=model_name,
+        max_turns=5,
+    )
+    
+    # Test query
+    test_query = "Say 'Hello from LiteLLM!' and nothing else."
+    
+    # Track streaming
+    received_chunks = []
+    full_response = ""
+    
+    try:
+        async with ClaudeSDKClient(options=options) as client:
+            await client.query(test_query)
+            
+            # Collect streaming response
+            async for msg in client.receive_response():
+                # Handle different message types
+                if hasattr(msg, 'type'):
+                    if msg.type == 'content_block_delta':
+                        # Streaming text delta
+                        if hasattr(msg, 'delta') and hasattr(msg.delta, 'text'):
+                            chunk_text = msg.delta.text
+                            received_chunks.append(chunk_text)
+                            full_response += chunk_text
+                    elif msg.type == 'content_block_start':
+                        # Start of content block
+                        if hasattr(msg, 'content_block') and hasattr(msg.content_block, 'text'):
+                            chunk_text = msg.content_block.text
+                            received_chunks.append(chunk_text)
+                            full_response += chunk_text
+                
+                # Fallback to content handling
+                if hasattr(msg, 'content'):
+                    for content_block in msg.content:
+                        if hasattr(content_block, 'text'):
+                            chunk_text = content_block.text
+                            received_chunks.append(chunk_text)
+                            full_response += chunk_text
+        
+        # Assertions
+        print(f"\n✅ Received {len(received_chunks)} chunks")
+        print(f"📝 Full response: {full_response[:100]}...")
+        
+        # Verify we got a response
+        assert len(full_response) > 0, f"No response received from {model_name}"
+        
+        # Verify streaming (should have multiple chunks for most responses)
+        # Note: Very short responses might come in 1 chunk, so we just verify we got content
+        assert len(received_chunks) > 0, f"No chunks received from {model_name}"
+        
+        # Verify response contains expected content (case insensitive)
+        assert "hello" in full_response.lower(), f"Response doesn't contain expected greeting: {full_response}"
+        
+        print(f"✅ Test passed for {model_name}")
+        
+    except Exception as e:
+        pytest.fail(f"Test failed for {model_name} ({model_description}): {str(e)}")
+
+
+if __name__ == "__main__":
+    # Run tests
+    pytest.main([__file__, "-v", "-s"])

tests/proxy_e2e_anthropic_messages_tests/test_config.yaml

@@ -0,0 +1,31 @@
+model_list:
+  - model_name: bedrock-claude-sonnet-3.5
+    litellm_params:
+      model: "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0"
+      aws_region_name: "us-east-1"
+
+  - model_name: bedrock-claude-sonnet-4
+    litellm_params:
+      model: "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0"
+      aws_region_name: "us-east-1"
+
+  - model_name: bedrock-claude-sonnet-4.5
+    litellm_params:
+      model: "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0"
+      aws_region_name: "us-east-1"
+
+  - model_name: bedrock-claude-opus-4.5
+    litellm_params:
+      model: "bedrock/us.anthropic.claude-opus-4-5-20251101-v1:0"
+      aws_region_name: "us-east-1"
+
+  - model_name: bedrock-nova-premier
+    litellm_params:
+      model: "bedrock/amazon.nova-premier-v1:0"
+      aws_region_name: "us-east-1"
+
+  # Converse API models
+  - model_name: bedrock-converse-claude-sonnet-4.5
+    litellm_params:
+      model: "bedrock_converse/us.anthropic.claude-sonnet-4-5-20250929-v1:0"
+      aws_region_name: "us-east-1"