diff --git a/.github/workflows/test-litellm-matrix.yml b/.github/workflows/test-litellm-matrix.yml index d83fedcb2ae..5590662f5ae 100644 --- a/.github/workflows/test-litellm-matrix.yml +++ b/.github/workflows/test-litellm-matrix.yml @@ -12,7 +12,7 @@ concurrency: jobs: test: runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 20 # Increased from 15 to 20 strategy: fail-fast: false matrix: @@ -20,36 +20,46 @@ jobs: # tests/test_litellm split by subdirectory (~560 files total) - name: "llms" path: "tests/test_litellm/llms" - workers: 4 + workers: 2 # Reduced from 4 to 2 to avoid race conditions + reruns: 2 # Retry flaky tests twice # tests/test_litellm/proxy split by subdirectory (~180 files total) - name: "proxy-guardrails" path: "tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/management_helpers" - workers: 4 + workers: 2 + reruns: 2 - name: "proxy-core" path: "tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine" - workers: 4 + workers: 2 + reruns: 2 - name: "proxy-misc" path: "tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py" - workers: 4 + workers: 2 + reruns: 2 - name: "integrations" path: "tests/test_litellm/integrations" - workers: 4 + workers: 2 + reruns: 3 # Integration tests tend to be flakier - name: "core-utils" path: "tests/test_litellm/litellm_core_utils" workers: 2 + reruns: 1 - name: "other" path: "tests/test_litellm/caching tests/test_litellm/responses tests/test_litellm/secret_managers tests/test_litellm/vector_stores tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/enterprise tests/test_litellm/experimental_mcp_client tests/test_litellm/google_genai tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/router_strategy tests/test_litellm/router_utils tests/test_litellm/types" - workers: 4 + workers: 2 + reruns: 2 - name: "root" path: "tests/test_litellm/test_*.py" - workers: 4 + workers: 2 + reruns: 2 # tests/proxy_unit_tests split alphabetically (~48 files total) - name: "proxy-unit-a" path: "tests/proxy_unit_tests/test_[a-o]*.py" workers: 2 + reruns: 1 - name: "proxy-unit-b" path: "tests/proxy_unit_tests/test_[p-z]*.py" workers: 2 + reruns: 1 name: test (${{ matrix.test-group.name }}) @@ -79,7 +89,8 @@ jobs: run: | poetry config virtualenvs.in-project true poetry install --with dev,proxy-dev --extras "proxy semantic-router" - poetry run pip install pytest-retry==1.6.3 pytest-xdist google-genai==1.22.0 \ + # pytest-rerunfailures and pytest-xdist are in pyproject.toml dev dependencies + poetry run pip install google-genai==1.22.0 \ google-cloud-aiplatform>=1.38 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core - name: Setup litellm-enterprise @@ -92,4 +103,6 @@ jobs: --tb=short -vv \ --maxfail=10 \ -n ${{ matrix.test-group.workers }} \ + --reruns ${{ matrix.test-group.reruns }} \ + --reruns-delay 1 \ --durations=20 diff --git a/docs/my-website/blog/claude_sonnet_4_6/index.md b/docs/my-website/blog/claude_sonnet_4_6/index.md new file mode 100644 index 00000000000..df54fa09792 --- /dev/null +++ b/docs/my-website/blog/claude_sonnet_4_6/index.md @@ -0,0 +1,283 @@ +--- +slug: claude_sonnet_4_6 +title: "Day 0 Support: Claude Sonnet 4.6" +date: 2026-02-17T10:00:00 +authors: + - name: Ishaan Jaff + title: "CTO, LiteLLM" + url: https://www.linkedin.com/in/reffajnaahsi/ + image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - name: Krrish Dholakia + title: "CEO, LiteLLM" + url: https://www.linkedin.com/in/krish-d/ + image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg +description: "Day 0 support for Claude Sonnet 4.6 on LiteLLM AI Gateway - use across Anthropic, Azure, Vertex AI, and Bedrock." +tags: [anthropic, claude, sonnet 4.6] +hide_table_of_contents: false +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +LiteLLM now supports Claude Sonnet 4.6 on Day 0. Use it across Anthropic, Azure, Vertex AI, and Bedrock through the LiteLLM AI Gateway. + +## Docker Image + +```bash +docker pull ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6 +``` + +## Usage - Anthropic + + + + +**1. Setup config.yaml** + +```yaml +model_list: + - model_name: claude-sonnet-4-6 + litellm_params: + model: anthropic/claude-sonnet-4-6 + api_key: os.environ/ANTHROPIC_API_KEY +``` + +**2. Start the proxy** + +```bash +docker run -d \ + -p 4000:4000 \ + -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ + -v $(pwd)/config.yaml:/app/config.yaml \ + ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6 \ + --config /app/config.yaml +``` + +**3. Test it!** + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data '{ + "model": "claude-sonnet-4-6", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] +}' +``` + + + + + +```python +from litellm import completion + +response = completion( + model="anthropic/claude-sonnet-4-6", + messages=[{"role": "user", "content": "what llm are you"}] +) +print(response.choices[0].message.content) +``` + + + + +## Usage - Azure + + + + +**1. Setup config.yaml** + +```yaml +model_list: + - model_name: claude-sonnet-4-6 + litellm_params: + model: azure_ai/claude-sonnet-4-6 + api_key: os.environ/AZURE_AI_API_KEY + api_base: os.environ/AZURE_AI_API_BASE # https://.services.ai.azure.com +``` + +**2. Start the proxy** + +```bash +docker run -d \ + -p 4000:4000 \ + -e AZURE_AI_API_KEY=$AZURE_AI_API_KEY \ + -e AZURE_AI_API_BASE=$AZURE_AI_API_BASE \ + -v $(pwd)/config.yaml:/app/config.yaml \ + ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6 \ + --config /app/config.yaml +``` + +**3. Test it!** + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data '{ + "model": "claude-sonnet-4-6", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] +}' +``` + + + + + +```python +from litellm import completion + +response = completion( + model="azure_ai/claude-sonnet-4-6", + api_key="your-azure-api-key", + api_base="https://.services.ai.azure.com", + messages=[{"role": "user", "content": "what llm are you"}] +) +print(response.choices[0].message.content) +``` + + + + +## Usage - Vertex AI + + + + +**1. Setup config.yaml** + +```yaml +model_list: + - model_name: claude-sonnet-4-6 + litellm_params: + model: vertex_ai/claude-sonnet-4-6 + vertex_project: os.environ/VERTEX_PROJECT + vertex_location: us-east5 +``` + +**2. Start the proxy** + +```bash +docker run -d \ + -p 4000:4000 \ + -e VERTEX_PROJECT=$VERTEX_PROJECT \ + -e GOOGLE_APPLICATION_CREDENTIALS=/app/credentials.json \ + -v $(pwd)/config.yaml:/app/config.yaml \ + -v $(pwd)/credentials.json:/app/credentials.json \ + ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6 \ + --config /app/config.yaml +``` + +**3. Test it!** + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data '{ + "model": "claude-sonnet-4-6", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] +}' +``` + + + + + +```python +from litellm import completion + +response = completion( + model="vertex_ai/claude-sonnet-4-6", + vertex_project="your-project-id", + vertex_location="us-east5", + messages=[{"role": "user", "content": "what llm are you"}] +) +print(response.choices[0].message.content) +``` + + + + +## Usage - Bedrock + + + + +**1. Setup config.yaml** + +```yaml +model_list: + - model_name: claude-sonnet-4-6 + litellm_params: + model: bedrock/anthropic.claude-sonnet-4-6-v1 + aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID + aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY + aws_region_name: us-east-1 +``` + +**2. Start the proxy** + +```bash +docker run -d \ + -p 4000:4000 \ + -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ + -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ + -v $(pwd)/config.yaml:/app/config.yaml \ + ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6 \ + --config /app/config.yaml +``` + +**3. Test it!** + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data '{ + "model": "claude-sonnet-4-6", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] +}' +``` + + + + + +```python +from litellm import completion + +response = completion( + model="bedrock/anthropic.claude-sonnet-4-6-v1", + aws_access_key_id="your-access-key", + aws_secret_access_key="your-secret-key", + aws_region_name="us-east-1", + messages=[{"role": "user", "content": "what llm are you"}] +) +print(response.choices[0].message.content) +``` + + + diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index fcbdd0f0318..f2d7f6f423d 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -1134,6 +1134,11 @@ const sidebars = { type: "category", label: "Blog", items: [ + { + type: "link", + label: "Day 0 Support: Claude Sonnet 4.6", + href: "/blog/claude_sonnet_4_6", + }, { type: "link", label: "Incident: Broken Model Cost Map", diff --git a/poetry.lock b/poetry.lock index 82df007de13..e3f3fea1321 100644 --- a/poetry.lock +++ b/poetry.lock @@ -5524,6 +5524,22 @@ psycopg = ">=3.0.0" pytest = ">=6.2" setuptools = "*" +[[package]] +name = "pytest-rerunfailures" +version = "14.0" +description = "pytest plugin to re-run tests to eliminate flaky failures" +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "pytest-rerunfailures-14.0.tar.gz", hash = "sha256:4a400bcbcd3c7a4ad151ab8afac123d90eca3abe27f98725dc4d9702887d2e92"}, + {file = "pytest_rerunfailures-14.0-py3-none-any.whl", hash = "sha256:4197bdd2eaeffdbf50b5ea6e7236f47ff0e44d1def8dae08e409f536d84e7b32"}, +] + +[package.dependencies] +packaging = ">=17.1" +pytest = ">=7.2" + [[package]] name = "pytest-retry" version = "1.7.0" @@ -7934,4 +7950,4 @@ utils = ["numpydoc"] [metadata] lock-version = "2.1" python-versions = ">=3.9,<4.0" -content-hash = "dfaf1eabfd17db5e30a8dda813872507aa38664fe7681ece2f8fa06ba035d3cf" +content-hash = "d99036fc86de60170dde4a1a9b3f9fdac6eb3610edb6177e70ca86133b71703e" diff --git a/pyproject.toml b/pyproject.toml index 4deb61836b3..52b1b9452f9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -151,7 +151,7 @@ pytest = "^7.4.3" pytest-mock = "^3.12.0" pytest-asyncio = "^0.21.1" pytest-postgresql = "^6.0.0" -pytest-retry = "^1.6.3" +pytest-xdist = "^3.5.0" requests-mock = "^1.12.1" responses = "^0.25.7" respx = "^0.22.0" @@ -166,6 +166,7 @@ opentelemetry-exporter-otlp = "^1.28.0" langfuse = "^2.45.0" fastapi-offline = "^1.7.3" fakeredis = "^2.27.1" +pytest-rerunfailures = "^14.0" [tool.poetry.group.proxy-dev.dependencies] prisma = "0.11.0" @@ -192,8 +193,6 @@ plugins = "pydantic.mypy" [tool.pytest.ini_options] asyncio_mode = "auto" -retries = 20 -retry_delay = 5 markers = [ "asyncio: mark test as an asyncio test", "limit_leaks: mark test with memory limit for leak detection (e.g., '40 MB')",