scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-06 08:16:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 78

Merge branch 'main' into fix/redundant-decrption

Browse source
This commit is contained in:
yangdx 2026-04-03 14:25:26 +08:00
commit 870a7890ba
124 changed files with 3837 additions and 2068 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

164
.circleci/config.yml
View file

@ -39,7 +39,7 @@ commands:
# conflicts with transitive dep pins like openai<2 and pydantic>=2.11.5)
pip install "pytest-mock==3.12.0" "pytest==7.3.1" "pytest-retry==1.6.3" \
"pytest-asyncio==0.21.1" "respx==0.22.0" "hypercorn==0.17.3" \
"pydantic==2.11.0" "mcp==1.25.0" "requests-mock>=1.12.1" \
"pydantic==2.12.5" "mcp==1.26.0" "requests-mock>=1.12.1" \
"responses==0.25.7" "pytest-xdist==3.6.1" "pytest-timeout==2.2.0" \
"pytest-cov==5.0.0" "semantic_router==0.1.10" "fastapi-offline==1.7.3" \
"a2a" "parameterized>=0.9.0"
@ -150,16 +150,16 @@ jobs:
python -m pip install --upgrade pip
python -m pip install -r .circleci/requirements.txt
pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-asyncio==0.21.1" "pytest-cov==5.0.0" \
"mypy==1.18.2" "google-generativeai==0.3.2" "google-cloud-aiplatform==1.43.0" pyarrow \
"boto3==1.36.0" "aioboto3==13.4.0" langchain lunary==0.2.5 \
"azure-identity==1.16.1" "langfuse==2.59.7" "logfire==0.29.0" numpydoc \
"mypy==1.18.2" "google-generativeai==0.3.2" "google-cloud-aiplatform==1.133.0" pyarrow \
"boto3==1.42.80" langchain lunary==0.2.5 \
"azure-identity==1.25.3" "langfuse==2.59.7" "logfire==0.29.0" numpydoc \
traceloop-sdk==0.21.1 openai==1.100.1 prisma==0.11.0 \
"detect_secrets==1.5.0" "respx==0.22.0" fastapi \
"gunicorn==21.2.0" "aiodynamo==23.10.1" "asyncio==3.4.3" \
"apscheduler==3.10.4" "PyGithub==1.59.1" argon2-cffi "pytest-mock==3.12.0" \
python-multipart prometheus-client==0.20.0 "pydantic==2.10.2" \
"diskcache==5.6.1" "Pillow==10.3.0" "jsonschema==4.22.0" \
"pytest-xdist==3.6.1" "pytest-timeout==2.2.0" "websockets==13.1.0"
"gunicorn==23.0.0" "aiodynamo==23.10.1" "asyncio==3.4.3" \
"apscheduler==3.11.2" "PyGithub==1.59.1" argon2-cffi "pytest-mock==3.12.0" \
python-multipart prometheus-client==0.20.0 "pydantic==2.12.5" \
"diskcache==5.6.1" "Pillow==12.1.1" "jsonschema==4.23.0" \
"pytest-xdist==3.6.1" "pytest-timeout==2.2.0" "websockets==15.0.1"
pip install semantic_router --no-deps
pip install aurelio_sdk --no-deps
pip uninstall posthog -y
@ -245,16 +245,16 @@ jobs:
python -m pip install --upgrade pip
python -m pip install -r .circleci/requirements.txt
pip install "pytest==7.3.1" "pytest-retry==1.6.3" "pytest-asyncio==0.21.1" "pytest-cov==5.0.0" \
"mypy==1.18.2" "google-generativeai==0.3.2" "google-cloud-aiplatform==1.43.0" pyarrow \
"boto3==1.36.0" "aioboto3==13.4.0" langchain lunary==0.2.5 \
"azure-identity==1.16.1" "langfuse==2.59.7" "logfire==0.29.0" numpydoc \
"mypy==1.18.2" "google-generativeai==0.3.2" "google-cloud-aiplatform==1.133.0" pyarrow \
"boto3==1.42.80" langchain lunary==0.2.5 \
"azure-identity==1.25.3" "langfuse==2.59.7" "logfire==0.29.0" numpydoc \
traceloop-sdk==0.21.1 openai==1.100.1 prisma==0.11.0 \
"detect_secrets==1.5.0" "respx==0.22.0" fastapi \
"gunicorn==21.2.0" "aiodynamo==23.10.1" "asyncio==3.4.3" \
"apscheduler==3.10.4" "PyGithub==1.59.1" argon2-cffi "pytest-mock==3.12.0" \
python-multipart prometheus-client==0.20.0 "pydantic==2.10.2" \
"diskcache==5.6.1" "Pillow==10.3.0" "jsonschema==4.22.0" \
"pytest-xdist==3.6.1" "pytest-timeout==2.2.0" "websockets==13.1.0"
"gunicorn==23.0.0" "aiodynamo==23.10.1" "asyncio==3.4.3" \
"apscheduler==3.11.2" "PyGithub==1.59.1" argon2-cffi "pytest-mock==3.12.0" \
python-multipart prometheus-client==0.20.0 "pydantic==2.12.5" \
"diskcache==5.6.1" "Pillow==12.1.1" "jsonschema==4.23.0" \
"pytest-xdist==3.6.1" "pytest-timeout==2.2.0" "websockets==15.0.1"
pip install semantic_router --no-deps
pip install aurelio_sdk --no-deps
pip uninstall posthog -y
@ -346,42 +346,41 @@ jobs:
pip install "pytest-cov==5.0.0"
pip install "mypy==1.18.2"
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install "google-cloud-aiplatform==1.133.0"
pip install pyarrow
pip install "boto3==1.36.0"
pip install "aioboto3==13.4.0"
pip install "boto3==1.42.80"
pip install langchain
pip install lunary==0.2.5
pip install "azure-identity==1.16.1"
pip install "azure-identity==1.25.3"
pip install "langfuse==2.59.7"
pip install "logfire==0.29.0"
pip install numpydoc
pip install traceloop-sdk==0.21.1
pip install opentelemetry-api==1.25.0
pip install opentelemetry-sdk==1.25.0
pip install opentelemetry-exporter-otlp==1.25.0
pip install opentelemetry-api==1.28.0
pip install opentelemetry-sdk==1.28.0
pip install opentelemetry-exporter-otlp==1.28.0
pip install openai==1.100.1
pip install prisma==0.11.0
pip install "detect_secrets==1.5.0"
pip install "httpx==0.24.1"
pip install "httpx==0.28.1"
pip install "respx==0.22.0"
pip install fastapi
pip install "gunicorn==21.2.0"
pip install "anyio==4.2.0"
pip install "gunicorn==23.0.0"
pip install "anyio==4.8.0"
pip install "aiodynamo==23.10.1"
pip install "asyncio==3.4.3"
pip install "apscheduler==3.10.4"
pip install "apscheduler==3.11.2"
pip install "PyGithub==1.59.1"
pip install argon2-cffi
pip install "pytest-mock==3.12.0"
pip install python-multipart
pip install google-cloud-aiplatform
pip install prometheus-client==0.20.0
pip install "pydantic==2.10.2"
pip install "pydantic==2.12.5"
pip install "diskcache==5.6.1"
pip install "Pillow==10.3.0"
pip install "jsonschema==4.22.0"
pip install "websockets==13.1.0"
pip install "Pillow==12.1.1"
pip install "jsonschema==4.23.0"
pip install "websockets==15.0.1"
- setup_litellm_enterprise_pip
- save_cache:
paths:
@ -700,8 +699,8 @@ jobs:
pip install "pytest-cov==5.0.0"
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.22.0"
pip install "pydantic==2.11.0"
pip install "mcp==1.25.0"
pip install "pydantic==2.12.5"
pip install "mcp==1.26.0"
pip install "pytest-xdist==3.6.1"
# Run pytest and generate JUnit XML report
- run:
@ -746,7 +745,7 @@ jobs:
pip install "pytest-cov==5.0.0"
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.22.0"
pip install "pydantic==2.11.0"
pip install "pydantic==2.12.5"
pip install "a2a-sdk"
# Run pytest and generate JUnit XML report
- run:
@ -791,8 +790,8 @@ jobs:
pip install "pytest-cov==5.0.0"
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.22.0"
pip install "pydantic==2.10.2"
pip install "boto3==1.36.0"
pip install "pydantic==2.12.5"
pip install "boto3==1.42.80"
pip install "semantic_router==0.1.10" --no-deps
pip install aurelio_sdk
pip install "pytest-xdist==3.6.1"
@ -841,7 +840,7 @@ jobs:
pip install "pytest-cov==5.0.0"
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.22.0"
pip install "pydantic==2.10.2"
pip install "pydantic==2.12.5"
# Run pytest and generate JUnit XML report
- run:
name: Run tests
@ -1048,8 +1047,8 @@ jobs:
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.22.0"
pip install "hypercorn==0.17.3"
pip install "pydantic==2.11.0"
pip install "mcp==1.25.0"
pip install "pydantic==2.12.5"
pip install "mcp==1.26.0"
pip install "requests-mock>=1.12.1"
pip install "responses==0.25.7"
pip install "pytest-xdist==3.6.1"
@ -1090,7 +1089,7 @@ jobs:
pip install "pytest-asyncio==0.21.1"
pip install "pytest-cov==5.0.0"
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install "google-cloud-aiplatform==1.133.0"
pip install "pytest-xdist==3.6.1"
# Run pytest and generate JUnit XML report
- run:
@ -1137,7 +1136,7 @@ jobs:
pip install "pytest-asyncio==0.21.1"
pip install "pytest-cov==5.0.0"
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install "google-cloud-aiplatform==1.133.0"
pip install pytest-mock
pip install "pytest-xdist==3.6.1"
# Run pytest and generate JUnit XML report
@ -1259,9 +1258,9 @@ jobs:
pip install pytest-mock
pip install "respx==0.22.0"
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install "google-cloud-aiplatform==1.133.0"
pip install "mlflow==2.17.2"
pip install "anthropic==0.52.0"
pip install "anthropic==0.54.0"
pip install "blockbuster==1.5.24"
pip install "pytest-xdist==3.6.1"
pip install "pytest-timeout==2.2.0"
@ -1352,11 +1351,11 @@ jobs:
pip install aiohttp
pip install openai
pip install click
pip install "boto3==1.36.0"
pip install "boto3==1.42.80"
pip install jinja2
pip install "tokenizers==0.20.0"
pip install "tokenizers==0.22.2"
pip install "uvloop==0.21.0"
pip install "fastuuid==0.12.0"
pip install "fastuuid==0.14.0"
pip install jsonschema
- setup_litellm_enterprise_pip
- run:
@ -1389,7 +1388,7 @@ jobs:
pip install "pytest-asyncio==0.21.1"
pip install "pytest-cov==5.0.0"
pip install "tomli==2.2.1"
pip install "mcp==1.25.0"
pip install "mcp==1.26.0"
- run:
name: Run tests
command: |
@ -1679,10 +1678,9 @@ jobs:
pip install "pytest-asyncio==0.21.1"
pip install "mypy==1.18.2"
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install "google-cloud-aiplatform==1.133.0"
pip install pyarrow
pip install "boto3==1.36.0"
pip install "aioboto3==13.4.0"
pip install "boto3==1.42.80"
pip install langchain
pip install "langfuse>=2.0.0"
pip install "logfire==0.29.0"
@ -1690,9 +1688,9 @@ jobs:
pip install prisma
pip install fastapi
pip install jsonschema
pip install "httpx==0.24.1"
pip install "gunicorn==21.2.0"
pip install "anyio==3.7.1"
pip install "httpx==0.28.1"
pip install "gunicorn==23.0.0"
pip install "anyio==4.8.0"
pip install "aiodynamo==23.10.1"
pip install "asyncio==3.4.3"
pip install "PyGithub==1.59.1"
@ -1823,10 +1821,9 @@ jobs:
pip install "mypy==1.18.2"
pip install "jsonlines==4.0.0"
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install "google-cloud-aiplatform==1.133.0"
pip install pyarrow
pip install "boto3==1.36.0"
pip install "aioboto3==13.4.0"
pip install "boto3==1.42.80"
pip install langchain
pip install "langchain_mcp_adapters==0.0.5"
pip install "langfuse>=2.0.0"
@ -1835,9 +1832,9 @@ jobs:
pip install prisma
pip install fastapi
pip install jsonschema
pip install "httpx==0.24.1"
pip install "gunicorn==21.2.0"
pip install "anyio==3.7.1"
pip install "httpx==0.28.1"
pip install "gunicorn==23.0.0"
pip install "anyio==4.8.0"
pip install "aiodynamo==23.10.1"
pip install "asyncio==3.4.3"
pip install "PyGithub==1.59.1"
@ -1969,10 +1966,9 @@ jobs:
pip install "pytest-asyncio==0.21.1"
pip install "mypy==1.18.2"
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install "google-cloud-aiplatform==1.133.0"
pip install pyarrow
pip install "boto3==1.36.0"
pip install "aioboto3==13.4.0"
pip install "boto3==1.42.80"
pip install langchain
pip install "langfuse>=2.0.0"
pip install "logfire==0.29.0"
@ -1980,9 +1976,9 @@ jobs:
pip install prisma
pip install fastapi
pip install jsonschema
pip install "httpx==0.24.1"
pip install "gunicorn==21.2.0"
pip install "anyio==3.7.1"
pip install "httpx==0.28.1"
pip install "gunicorn==23.0.0"
pip install "anyio==4.8.0"
pip install "aiodynamo==23.10.1"
pip install "asyncio==3.4.3"
pip install "PyGithub==1.59.1"
@ -2611,16 +2607,16 @@ jobs:
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
pip install "google-cloud-aiplatform==1.43.0"
pip install "google-cloud-aiplatform==1.133.0"
pip install aiohttp
pip install "openai==1.100.1"
pip install "assemblyai==0.37.0"
python -m pip install --upgrade pip
pip install "pydantic==2.10.2"
pip install "pydantic==2.12.5"
pip install "pytest==7.3.1"
pip install "pytest-mock==3.12.0"
pip install "pytest-asyncio==0.21.1"
pip install "boto3==1.36.0"
pip install "boto3==1.42.80"
pip install "mypy==1.18.2"
pip install pyarrow
pip install numpydoc
@ -2628,11 +2624,11 @@ jobs:
pip install fastapi
pip install jsonschema
pip install "httpx==0.27.0"
pip install "anyio==3.7.1"
pip install "anyio==4.8.0"
pip install "asyncio==3.4.3"
pip install "PyGithub==1.59.1"
pip install "google-cloud-aiplatform==1.59.0"
pip install "anthropic==0.52.0"
pip install "anthropic==0.54.0"
pip install "langchain_mcp_adapters==0.0.5"
pip install "langchain_openai==0.2.1"
pip install "langgraph==0.3.18"
@ -2799,7 +2795,7 @@ jobs:
conda activate myenv
pip install "pytest==7.3.1"
pip install "pytest-asyncio==0.21.1"
pip install "boto3==1.36.0"
pip install "boto3==1.42.80"
pip install "httpx==0.27.0"
pip install "claude-agent-sdk"
pip install -r requirements.txt
@ -3237,31 +3233,9 @@ jobs:
name: Install Playwright Browsers
command: |
npx playwright install
- run:
name: Install Neon CLI
command: |
npm i -g neonctl
- run:
name: Create Neon branch
command: |
export EXPIRES_AT=$(date -u -d "+3 hours" +"%Y-%m-%dT%H:%M:%SZ")
echo "Expires at: $EXPIRES_AT"
neon branches create \
--project-id $NEON_PROJECT_ID \
--name preview/commit-${CIRCLE_SHA1:0:7}-<< parameters.browser >> \
--expires-at $EXPIRES_AT \
--parent br-fancy-paper-ad1olsb3 \
--api-key $NEON_API_KEY || true
- run:
name: Run Docker container
command: |
E2E_UI_TEST_DATABASE_URL=$(neon connection-string \
--project-id $NEON_PROJECT_ID \
--api-key $NEON_API_KEY \
--branch preview/commit-${CIRCLE_SHA1:0:7}-<< parameters.browser >> \
--database-name yuneng-trial-db \
--role neondb_owner)
echo $E2E_UI_TEST_DATABASE_URL
docker run -d \
-p 4000:4000 \
-e DATABASE_URL=$E2E_UI_TEST_DATABASE_URL \
@ -3535,7 +3509,7 @@ workflows:
only:
- main
- /litellm_.*/
# - e2e_ui_testing: # migrate to dynamic db - currently requires neon cli
# - e2e_ui_testing:
# name: e2e_ui_testing_chromium
# browser: chromium
# context: e2e_ui_tests

10
.circleci/requirements.txt
View file

@ -7,15 +7,15 @@ cohere
redis==5.2.1
redisvl==0.4.1
anthropic
orjson==3.10.12 # fast /embedding responses
pydantic==2.11.0
google-cloud-aiplatform==1.43.0
orjson==3.10.15 # fast /embedding responses
pydantic==2.12.5
google-cloud-aiplatform==1.133.0
google-cloud-iam==2.19.1
fastapi-sso==0.16.0
uvloop==0.21.0
mcp==1.25.0 # for MCP server
mcp==1.26.0 # for MCP server
semantic_router==0.1.10 # for auto-routing with litellm
fastuuid==0.12.0
fastuuid==0.14.0
responses==0.25.7 # for proxy client tests
pytest-retry==1.6.3 # for automatic test retries
litellm-proxy-extras # for prisma migrations

40
.pre-commit-config.yaml
View file

@ -1,40 +0,0 @@
repos:
- repo: local
hooks:
- id: pyright
name: pyright
entry: pyright
language: system
types: [python]
files: ^(litellm/|litellm_proxy_extras/|enterprise/)
- id: isort
name: isort
entry: isort
language: system
types: [python]
files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py
exclude: ^litellm/__init__.py$
- id: black
name: black
entry: poetry run black
language: system
types: [python]
files: (litellm/|litellm_proxy_extras/).*\.py
- repo: https://github.com/pycqa/flake8
rev: 7.0.0 # The version of flake8 to use
hooks:
- id: flake8
exclude: ^litellm/tests/|^litellm/proxy/tests/|^litellm/tests/test_litellm/|^tests/test_litellm/|^tests/enterprise/
additional_dependencies: [flake8-print]
files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py
- repo: https://github.com/python-poetry/poetry
rev: 1.8.0
hooks:
- id: poetry-check
files: ^(pyproject.toml|litellm-proxy-extras/pyproject.toml)$
- repo: local
hooks:
- id: check-files-match
name: Check if files match
entry: python3 ci_cd/check_files_match.py
language: system

10
Dockerfile
View file

@ -1,8 +1,8 @@
# Base image for building
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a5a619c1793039dcf92f02178f37c94bb3d6001403716da59d6092dfe8d9b502
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a5a619c1793039dcf92f02178f37c94bb3d6001403716da59d6092dfe8d9b502
# Builder stage
FROM $LITELLM_BUILD_IMAGE AS builder
@ -15,7 +15,7 @@ USER root
# Install build dependencies
RUN apk add --no-cache bash gcc py3-pip python3 python3-dev openssl openssl-dev
RUN python -m pip install build
RUN python -m pip install build==1.4.2
# Copy the current directory contents into the container at /app
COPY . .
@ -49,7 +49,7 @@ USER root
# Install runtime dependencies (libsndfile needed for audio processing on ARM64)
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
# SECURITY FIX: npm bundles tar, glob, and brace-expansion at multiple nested
# levels inside its dependency tree. `npm install -g <pkg>` only creates a
# SEPARATE global package, it does NOT replace npm's internal copies.
@ -90,7 +90,7 @@ COPY --from=builder /app/dist/*.whl .
COPY --from=builder /wheels/ /wheels/
# Install the built wheel using pip; again using a wildcard if it's the only file
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ --no-deps && rm -f *.whl && rm -rf /wheels
# Replace the nodejs-wheel-binaries bundled node with the system node (fixes CVE-2025-55130)
RUN NODEJS_WHEEL_NODE=$(find /usr/lib -path "*/nodejs_wheel/bin/node" 2>/dev/null) && \

19
ci_cd/publish-proxy-extras.sh
View file

@ -1,19 +0,0 @@
#!/bin/bash
# Exit on error
set -e
echo "🚀 Building and publishing litellm-proxy-extras"
# Navigate to litellm-proxy-extras directory
cd "$(dirname "$0")/../litellm-proxy-extras"
# Build the package
echo "📦 Building package..."
poetry build
# Publish to PyPI
echo "🌎 Publishing to PyPI..."
poetry publish
echo "✅ Done! Package published successfully"

1
ci_cd/security_scans.sh
View file

@ -160,7 +160,6 @@ run_grype_scans() {
"CVE-2026-0775" # npm cli incorrect permission assignment - no fix available yet, npm is only used at build/prisma-generate time
"GHSA-3ppc-4f35-3m26" # minimatch ReDoS via repeated wildcards - from nodejs_wheel bundled npm, not used in application runtime code
"GHSA-83g3-92jg-28cx" # tar arbitrary file read/write via hardlink - from nodejs_wheel bundled npm, not used in application runtime code
"CVE-2026-25639" # axios DoS via __proto__ in mergeConfig - transitive dev dep via @neondatabase/api-client, not imported in application code
"CVE-2026-2297" # Python 3.13 SourcelessFileLoader audit hook bypass - no fix available in base image
"GHSA-qffp-2rhf-9h96" # tar hardlink path traversal - from nodejs_wheel bundled npm, not used in application runtime code
"CVE-2026-2673" # OpenSSL 3.6.1 TLS 1.3 key exchange group negotiation issue - no fix available yet

2
cookbook/ai_coding_tool_guides/claude_code_quickstart/guide.md
View file

@ -230,7 +230,7 @@ model_list:
# AWS Bedrock
- model_name: claude-bedrock
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1

4
cookbook/misc/test_responses_api.py
View file

@ -20,7 +20,7 @@ base64_image = encode_image(image_path)
response = client.responses.create(
model="bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
input=[
{
"role": "user",
@ -43,7 +43,7 @@ print("sleeping for 20 seconds...")
time.sleep(20)
print("making follow up request for existing id")
response2 = client.responses.create(
model="bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
previous_response_id=response.id,
input="ok, and what objects are in the image?"
)

2
deploy/Dockerfile.ghcr_base
View file

@ -1,5 +1,5 @@
# Use the provided base image
FROM ghcr.io/berriai/litellm:main-latest
FROM ghcr.io/berriai/litellm:main-latest@sha256:7c311546c25e7bb6e8cafede9fcd3d0d622ac636b5c9418befaa32e85dfb0186
# Set the working directory to /app
WORKDIR /app

10
docker/Dockerfile.alpine
View file

@ -1,8 +1,8 @@
# Base image for building
ARG LITELLM_BUILD_IMAGE=python:3.11-alpine
ARG LITELLM_BUILD_IMAGE=python:3.11-alpine@sha256:f07e2ace46f560f09a6eeec7b4913b80ee99546e749ef82342a419a326620856
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=python:3.11-alpine
ARG LITELLM_RUNTIME_IMAGE=python:3.11-alpine@sha256:f07e2ace46f560f09a6eeec7b4913b80ee99546e749ef82342a419a326620856
# Builder stage
FROM $LITELLM_BUILD_IMAGE AS builder
@ -13,8 +13,8 @@ WORKDIR /app
# Install build dependencies
RUN apk add --no-cache gcc python3-dev musl-dev
RUN pip install --upgrade pip && \
pip install build
RUN pip install --upgrade pip==26.0.1 && \
pip install build==1.4.2
# Copy the current directory contents into the container at /app
COPY . .
@ -44,7 +44,7 @@ COPY --from=builder /app/dist/*.whl .
COPY --from=builder /wheels/ /wheels/
# Install the built wheel using pip; again using a wildcard if it's the only file
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ --no-deps && rm -f *.whl && rm -rf /wheels
# Convert Windows line endings to Unix for entrypoint scripts
RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh

5
docker/Dockerfile.custom_ui
View file

@ -1,4 +1,5 @@
# Use the provided base image
# NOTE: This is a dev/branch-specific tag. Update digest when the base image is rebuilt.
FROM ghcr.io/berriai/litellm:litellm_fwd_server_root_path-dev
# Set the working directory to /app
@ -18,8 +19,8 @@ RUN apt-get update && apt-get upgrade -y \
libxslt1.1 \
libgnutls30 \
libc6 && \
apt-get install -y nodejs npm && \
npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
apt-get install -y --no-install-recommends nodejs npm && \
npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
GLOBAL="$(npm root -g)" && \
find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \

10
docker/Dockerfile.database
View file

@ -1,8 +1,8 @@
# Base image for building
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a5a619c1793039dcf92f02178f37c94bb3d6001403716da59d6092dfe8d9b502
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a5a619c1793039dcf92f02178f37c94bb3d6001403716da59d6092dfe8d9b502
# Builder stage
FROM $LITELLM_BUILD_IMAGE AS builder
@ -21,7 +21,7 @@ RUN apk add --no-cache \
openssl \
openssl-dev
RUN python -m pip install build
RUN python -m pip install build==1.4.2
# Copy the current directory contents into the container at /app
COPY . .
@ -50,7 +50,7 @@ USER root
# Install runtime dependencies
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
GLOBAL="$(npm root -g)" && \
find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@ -82,7 +82,7 @@ COPY --from=builder /app/dist/*.whl .
COPY --from=builder /wheels/ /wheels/
# Install the built wheel using pip; again using a wildcard if it's the only file
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ --no-deps && rm -f *.whl && rm -rf /wheels
# SECURITY FIX: nodejs-wheel-binaries (pip package used by Prisma) bundles a complete
# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.

10
docker/Dockerfile.dev
View file

@ -1,8 +1,8 @@
# Base image for building
ARG LITELLM_BUILD_IMAGE=python:3.11-slim
ARG LITELLM_BUILD_IMAGE=python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=python:3.11-slim
ARG LITELLM_RUNTIME_IMAGE=python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d
# Builder stage
FROM $LITELLM_BUILD_IMAGE AS builder
@ -19,7 +19,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
libssl-dev \
pkg-config \
&& rm -rf /var/lib/apt/lists/* \
&& pip install --upgrade pip build
&& pip install --upgrade pip==26.0.1 build==1.4.2
# Copy requirements first for better layer caching
COPY requirements.txt .
@ -75,7 +75,7 @@ RUN apt-get update && apt-get upgrade -y \
nodejs \
npm \
&& rm -rf /var/lib/apt/lists/* \
&& npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \
&& npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \
&& GLOBAL="$(npm root -g)" \
&& find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@ -109,7 +109,7 @@ COPY --from=builder /wheels/ /wheels/
COPY --from=builder /app/dist/*.whl .
# Install all dependencies in one step with no-cache for smaller image
RUN pip install --no-cache-dir *.whl /wheels/* --no-index --find-links=/wheels/ && \
RUN pip install --no-cache-dir *.whl /wheels/* --no-index --find-links=/wheels/ --no-deps && \
rm -f *.whl && \
rm -rf /wheels

10
docker/Dockerfile.health_check
View file

@ -1,4 +1,4 @@
FROM python:3.11-slim
FROM python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d
WORKDIR /app
@ -12,5 +12,13 @@ RUN pip install --no-cache-dir -r requirements.txt
# Make script executable
RUN chmod +x /app/health_check_client.py
# Run as non-root user
RUN adduser --disabled-password --gecos "" --uid 1001 healthcheck
USER healthcheck
# Health check
HEALTHCHECK --interval=30s --timeout=5s --retries=3 \
CMD python /app/health_check_client.py --help || exit 1
# Set entrypoint
ENTRYPOINT ["python", "/app/health_check_client.py"]

19
docker/Dockerfile.non_root
View file

@ -1,6 +1,6 @@
# Base images
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a5a619c1793039dcf92f02178f37c94bb3d6001403716da59d6092dfe8d9b502
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a5a619c1793039dcf92f02178f37c94bb3d6001403716da59d6092dfe8d9b502
ARG PROXY_EXTRAS_SOURCE=published
# -----------------
@ -27,7 +27,7 @@ RUN for i in 1 2 3; do \
nodejs \
npm && break || sleep 5; \
done \
&& pip install --no-cache-dir --upgrade pip build
&& pip install --no-cache-dir --upgrade pip==26.0.1 build==1.4.2
# Cache Python dependencies
COPY requirements.txt .
@ -41,13 +41,22 @@ COPY . .
ENV LITELLM_NON_ROOT=true
# Build Admin UI using the upstream command order while keeping a single RUN layer
# NOTE: .npmrc (which has ignore-scripts=true and min-release-age=3d) is temporarily
# renamed during npm install/ci. This is safe because npm ci installs from
# package-lock.json with pinned versions + integrity hashes.
RUN mkdir -p /var/lib/litellm/ui && \
npm install -g npm@latest && npm cache clean --force && \
mv /app/.npmrc /app/.npmrc.bak && \
npm install -g npm@11.12.1 && \
npm install -g node-gyp@12.2.0 && \
ln -sf /usr/local/lib/node_modules/node-gyp /usr/lib/node_modules/npm/node_modules/node-gyp && \
npm cache clean --force && \
cd /app/ui/litellm-dashboard && \
if [ -f "/app/enterprise/enterprise_ui/enterprise_colors.json" ]; then \
cp /app/enterprise/enterprise_ui/enterprise_colors.json ./ui_colors.json; \
fi && \
mv .npmrc .npmrc.bak && \
npm ci && \
mv .npmrc.bak .npmrc && mv /app/.npmrc.bak /app/.npmrc && \
npm run build && \
cp -r /app/ui/litellm-dashboard/out/* /var/lib/litellm/ui/ && \
mkdir -p /var/lib/litellm/assets && \
@ -106,7 +115,7 @@ RUN for i in 1 2 3; do \
apk add --no-cache python3 py3-pip bash openssl tzdata nodejs npm supervisor && break || sleep 5; \
done \
&& apk upgrade --no-cache nodejs \
&& npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \
&& npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \
&& GLOBAL="$(npm root -g)" \
&& find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \

16
docker/build_from_pip/Dockerfile.build_from_pip
View file

@ -1,19 +1,17 @@
FROM python:3.13-alpine
FROM python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d
WORKDIR /app
ENV HOME=/home/litellm
ENV PATH="${HOME}/venv/bin:$PATH"
# Install runtime dependencies
# Note: Using Python 3.13 for compatibility with ddtrace and other packages
# rust and cargo are required for building ddtrace from source
# musl-dev and libffi-dev are needed for some Python packages on Alpine
RUN apk update && \
apk add --no-cache gcc musl-dev libffi-dev openssl openssl-dev rust cargo
# Install runtime dependencies needed for building native extensions
RUN apt-get update && \
apt-get install -y --no-install-recommends gcc libffi-dev && \
rm -rf /var/lib/apt/lists/*
RUN python -m venv ${HOME}/venv
RUN ${HOME}/venv/bin/pip install --no-cache-dir --upgrade pip
RUN ${HOME}/venv/bin/pip install --no-cache-dir --upgrade pip==26.0.1
COPY docker/build_from_pip/requirements.txt .
RUN --mount=type=cache,target=${HOME}/.cache/pip \
@ -28,4 +26,4 @@ RUN prisma generate
EXPOSE 4000/tcp
ENTRYPOINT ["litellm"]
CMD ["--port", "4000"]
CMD ["--port", "4000"]

10
docker/build_from_pip/requirements.txt
View file

@ -1,6 +1,6 @@
litellm[proxy]==1.67.4.dev1 # Specify the litellm version you want to use
prometheus_client
langfuse
prisma
openai==1.99.9
litellm[proxy]==1.83.0
prometheus_client==0.20.0
langfuse==2.59.7
prisma==0.11.0
openai==2.24.0
ddtrace==2.19.0 # for advanced DD tracing / profiling

2
docker/install_auto_router.sh
View file

@ -1,3 +1,3 @@
#!/bin/bash
pip install semantic_router==0.1.11 --no-deps
pip install aurelio-sdk==0.0.19
pip install aurelio-sdk==0.0.19 --no-deps

39
docs/my-website/blog/april_townhall_announcement/index.md Normal file
View file

@ -0,0 +1,39 @@
---
slug: april-townhall-announcement
title: "April Townhall: Security + Product Roadmap"
date: 2026-04-02T07:30:00
authors:
- krrish
description: "Join the LiteLLM April townhall on Friday, 10 April at 7:30 AM to learn about LiteLLM's security and product roadmap."
tags: [announcement, townhall]
hide_table_of_contents: true
---
import Image from '@theme/IdealImage';
We are hosting our April townhall on **Friday, 10 April at 7:30 AM PST**.
<Image
img={require('../../img/april_townhall_banner.png')}
style={{width: '900px', height: 'auto', display: 'block'}}
/>
{/* truncate */}
## Agenda
- Product updates and roadmap progress
- Reliability and security updates
- Open Q&A with the team
## How to contribute
Add your thoughts to this [ticket](https://github.com/BerriAI/litellm/issues/24825) to help us shape the agenda.
## Register
Register here: [LiteLLM April Townhall Form](https://forms.gle/hvyVXwbFjzJQE7dEA)
We will hold the townhall from **7:30 AM to 8:30 AM PST on Zoom**.
For security, attendance is restricted to corporate emails. If you register with a non-corporate email, we will share the townhall slides and accompanying blog post after the event.

2
docs/my-website/docs/anthropic_count_tokens.md
View file

@ -96,7 +96,7 @@ model_list:
- model_name: claude-bedrock
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
aws_region_name: us-west-2
```

6
docs/my-website/docs/completion/computer_use.md
View file

@ -80,7 +80,7 @@ model_list:
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: claude-bedrock # Bedrock Anthropic model
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
model: bedrock/anthropic.claude-haiku-4-5-20251001:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-west-2
@ -153,7 +153,7 @@ import litellm
assert litellm.supports_computer_use(model="anthropic/claude-3-5-sonnet-latest") == True
assert litellm.supports_computer_use(model="anthropic/claude-3-7-sonnet-20250219") == True
assert litellm.supports_computer_use(model="bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0") == True
assert litellm.supports_computer_use(model="bedrock/anthropic.claude-haiku-4-5-20251001:0") == True
assert litellm.supports_computer_use(model="vertex_ai/claude-3-5-sonnet") == True
assert litellm.supports_computer_use(model="openai/gpt-4") == False
```
@ -171,7 +171,7 @@ model_list:
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: claude-bedrock # Bedrock Anthropic model
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
model: bedrock/anthropic.claude-haiku-4-5-20251001:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-west-2

18
docs/my-website/docs/completion/document_understanding.md
View file

@ -32,7 +32,7 @@ os.environ["AWS_REGION_NAME"] = ""
file_url = "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf"
# model
model = "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0"
model = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
file_content = [
{"type": "text", "text": "What's this file about?"},
@ -63,7 +63,7 @@ assert response is not None
model_list:
- model_name: bedrock-model
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: os.environ/AWS_REGION_NAME
@ -122,7 +122,7 @@ encoded_file = base64.b64encode(file_data).decode("utf-8")
base64_url = f"data:application/pdf;base64,{encoded_file}"
# model
model = "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0"
model = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
file_content = [
{"type": "text", "text": "What's this file about?"},
@ -153,7 +153,7 @@ assert response is not None
model_list:
- model_name: bedrock-model
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: os.environ/AWS_REGION_NAME
@ -210,7 +210,7 @@ os.environ["AWS_REGION_NAME"] = ""
file_url = "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf"
# model
model = "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0"
model = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
file_content = [
{"type": "text", "text": "What's this file about?"},
@ -242,7 +242,7 @@ assert response is not None
model_list:
- model_name: bedrock-model
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: os.environ/AWS_REGION_NAME
@ -350,10 +350,10 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
<Tabs>
<TabItem label="SDK" value="sdk">
Use `litellm.supports_pdf_input(model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0")` -> returns `True` if model can accept pdf input
Use `litellm.supports_pdf_input(model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0")` -> returns `True` if model can accept pdf input
```python
assert litellm.supports_pdf_input(model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0") == True
assert litellm.supports_pdf_input(model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0") == True
```
</TabItem>
@ -365,7 +365,7 @@ assert litellm.supports_pdf_input(model="bedrock/anthropic.claude-3-5-sonnet-202
model_list:
- model_name: bedrock-model # model group name
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: os.environ/AWS_REGION_NAME

2
docs/my-website/docs/completion/provider_specific_params.md
View file

@ -450,7 +450,7 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
import litellm
response = litellm.completion(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
messages=[{"role": "user", "content": "Hello!"}],
requestMetadata={"cost_center": "engineering"}
)

2
docs/my-website/docs/index.md
View file

@ -103,7 +103,7 @@ os.environ["AWS_SECRET_ACCESS_KEY"] = "your-secret"
os.environ["AWS_REGION_NAME"] = "us-east-1"
response = completion(
model="bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0",
model="bedrock/anthropic.claude-haiku-4-5-20251001:0",
messages=[{"role": "user", "content": "Hello, how are you?"}]
)
print(response.choices[0].message.content)

2
docs/my-website/docs/oidc.md
View file

@ -268,7 +268,7 @@ Please contact us for paid enterprise support if you need help setting up Azure
model list:
- model_name: aws/claude-3-5-sonnet
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
aws_region_name: "eu-central-1"
aws_role_name: "arn:aws:iam::12345678:role/bedrock-role"
aws_web_identity_token: "oidc/azure/api://123-456-789-9d04"

4
docs/my-website/docs/providers/azure/azure_responses.md
View file

@ -246,7 +246,7 @@ You can also call the Azure Responses API via the `/chat/completions` endpoint.
from litellm import completion
import os
os.environ["AZURE_API_BASE"] = "https://my-endpoint-sweden-berri992.openai.azure.com/"
os.environ["AZURE_API_BASE"] = "https://my-azure-endpoint.openai.azure.com/"
os.environ["AZURE_API_VERSION"] = "2023-03-15-preview"
os.environ["AZURE_API_KEY"] = "my-api-key"
@ -268,7 +268,7 @@ model_list:
litellm_params:
model: azure/responses/my-custom-o1-pro
api_key: os.environ/AZURE_API_KEY
api_base: https://my-endpoint-sweden-berri992.openai.azure.com/
api_base: https://my-azure-endpoint.openai.azure.com/
api_version: 2023-03-15-preview
```

20
docs/my-website/docs/providers/bedrock.md
View file

@ -95,7 +95,7 @@ Here's how to call Bedrock with the LiteLLM Proxy Server
model_list:
- model_name: bedrock-claude-3-5-sonnet
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: os.environ/AWS_REGION_NAME
@ -337,7 +337,7 @@ os.environ["AWS_SECRET_ACCESS_KEY"] = ""
os.environ["AWS_REGION_NAME"] = ""
response = completion(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
messages=[{"role": "user", "content": "Hello, how are you?"}],
requestMetadata={
"cost_center": "engineering",
@ -354,7 +354,7 @@ response = completion(
model_list:
- model_name: bedrock-claude-v1
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
requestMetadata:
cost_center: "engineering"
```
@ -1543,7 +1543,7 @@ file_data = response.content
encoded_file = base64.b64encode(file_data).decode("utf-8")
# model
model = "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0"
model = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
image_content = [
{"type": "text", "text": "What's this file about?"},
@ -1574,7 +1574,7 @@ assert response is not None
model_list:
- model_name: bedrock-model
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: os.environ/AWS_REGION_NAME
@ -1631,7 +1631,7 @@ encoded_file = base64.b64encode(file_data).decode("utf-8")
base64_url = f"data:application/pdf;base64,{encoded_file}"
# model
model = "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0"
model = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
image_content = [
{"type": "text", "text": "What's this file about?"},
@ -1660,7 +1660,7 @@ assert response is not None
model_list:
- model_name: bedrock-model
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: os.environ/AWS_REGION_NAME
@ -1941,7 +1941,7 @@ Here's an example of using a bedrock model with LiteLLM. For a complete list, re
| GPT-OSS 120B | `completion(model='bedrock/converse/openai.gpt-oss-120b-1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` |
| Deepseek R1 | `completion(model='bedrock/us.deepseek.r1-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` |
| Anthropic Claude Sonnet 4.5 | `completion(model='bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` |
| Anthropic Claude-V3.5 Sonnet | `completion(model='bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` |
| Anthropic Claude-V3.5 Sonnet | `completion(model='bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` |
| Anthropic Claude-V3 sonnet | `completion(model='bedrock/anthropic.claude-3-sonnet-20240229-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` |
| Anthropic Claude-V3 Haiku | `completion(model='bedrock/anthropic.claude-3-haiku-20240307-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` |
| Anthropic Claude-V3 Opus | `completion(model='bedrock/anthropic.claude-3-opus-20240229-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` |
@ -2051,7 +2051,7 @@ os.environ["AWS_SECRET_ACCESS_KEY"] = ""
os.environ["AWS_REGION_NAME"] = ""
response = completion(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
messages=[{"role": "user", "content": "Hello, how are you?"}],
model_id="arn:aws:bedrock:eu-central-1:000000000000:application-inference-profile/a0a0a0a0a0a0",
)
@ -2068,7 +2068,7 @@ print(response)
model_list:
- model_name: anthropic-claude-3-5-sonnet
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
# You have to set the ARN application inference profile in the model_id parameter
model_id: arn:aws:bedrock:eu-central-1:000000000000:application-inference-profile/a0a0a0a0a0a0
```

21
docs/my-website/docs/proxy/oauth2.md
View file

@ -61,3 +61,24 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
Start the LiteLLM Proxy with [`--detailed_debug` mode and you should see more verbose logs](cli.md#detailed_debug)
## Using OAuth2 + JWT Together
If both `enable_oauth2_auth` and `enable_jwt_auth` are enabled, LiteLLM can split auth paths:
- JWT validation for user tokens
- OAuth2 introspection for machine tokens
For JWT-shaped machine tokens, configure `litellm_jwtauth.routing_overrides`:
```yaml title="config.yaml"
general_settings:
enable_jwt_auth: true
enable_oauth2_auth: true
litellm_jwtauth:
routing_overrides:
- iss: "machine-issuer.example.com"
client_id: "MID_LITELLM"
path: "oauth2"
```
For full `routing_overrides` behavior and list-based selectors, see [`/proxy/token_auth`](./token_auth.md#route-jwt-shaped-machine-tokens-to-oauth2).

2
docs/my-website/docs/proxy/team_model_add.md
View file

@ -26,7 +26,7 @@ curl -L -X POST 'http://0.0.0.0:4000/model/new' \
"model": "openai/gpt-4o",
"custom_llm_provider": "openai",
"api_key": "******ccb07",
"api_base": "https://my-endpoint-sweden-berri992.openai.azure.com",
"api_base": "https://my-azure-endpoint.openai.azure.com",
"api_version": "2023-12-01-preview"
},
"model_info": {

41
docs/my-website/docs/proxy/token_auth.md
View file

@ -790,6 +790,47 @@ litellm_jwtauth:
user_roles_jwt_field: "resource_access.your-client.roles"
```
## Route JWT-Shaped Machine Tokens to OAuth2
Use this when both are enabled:
- `enable_jwt_auth: true` for standard JWT validation
- `enable_oauth2_auth: true` for OAuth2 introspection
If some machine tokens are also JWT-shaped, configure `routing_overrides` to route matching tokens to OAuth2.
```yaml title="config.yaml"
general_settings:
enable_jwt_auth: true
enable_oauth2_auth: true
litellm_jwtauth:
user_id_jwt_field: "sub"
routing_overrides:
- iss: "machine-issuer.example.com"
client_id: "MID_LITELLM"
path: "oauth2"
```
### Matching behavior
- A rule matches when all configured selectors match token claims
- Supported selectors: `iss` (required), `client_id` (optional), `aud` (optional)
- Selector values support both string and list forms
- If no rule matches, LiteLLM continues with standard JWT validation
### List-based override example
```yaml title="config.yaml"
general_settings:
enable_jwt_auth: true
enable_oauth2_auth: true
litellm_jwtauth:
routing_overrides:
- iss: ["machine-issuer.example.com", "backup-issuer.example.com"]
client_id: ["MID_LITELLM", "MID_BACKUP"]
aud: ["api://litellm", "api://fallback"]
path: "oauth2"
```
## [BETA] Control Access with OIDC Roles
Allow JWT tokens with supported roles to access the proxy.

2
docs/my-website/docs/realtime.md
View file

@ -82,7 +82,7 @@ Run this script using node - `node test.js`
const WebSocket = require("ws");
const url = "ws://0.0.0.0:4000/v1/realtime?model=openai-gpt-4o-realtime-audio";
// const url = "wss://my-endpoint-sweden-berri992.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview";
// const url = "wss://my-azure-endpoint.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview";
const ws = new WebSocket(url, {
headers: {
"api-key": `sk-1234`,

2
docs/my-website/docs/tutorials/claude_responses_api.md
View file

@ -214,7 +214,7 @@ model_list:
# AWS Bedrock
- model_name: claude-bedrock
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
model: bedrock/anthropic.claude-haiku-4-5-20251001:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1

2
docs/my-website/docs/tutorials/github_copilot_integration.md
View file

@ -141,7 +141,7 @@ Route requests to Claude on Bedrock:
model_list:
- model_name: bedrock-claude
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
model: bedrock/anthropic.claude-haiku-4-5-20251001:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1

2
docs/my-website/docs/tutorials/google_genai_sdk.md
View file

@ -282,7 +282,7 @@ Route `gemini-2.5-flash` requests to Claude on Bedrock:
model_list:
- model_name: bedrock-claude
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
model: bedrock/anthropic.claude-haiku-4-5-20251001:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1

4
docs/my-website/docs/tutorials/litellm_gemini_cli.md
View file

@ -127,7 +127,7 @@ Route `gemini-2.5-pro` requests to Claude on Bedrock:
model_list:
- model_name: bedrock-claude
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
model: bedrock/anthropic.claude-haiku-4-5-20251001:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1
@ -149,7 +149,7 @@ model_list:
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: anthropic-claude
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
model: bedrock/anthropic.claude-haiku-4-5-20251001:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1

4
docs/my-website/docs/tutorials/litellm_qwen_code_cli.md
View file

@ -129,7 +129,7 @@ Route `qwen-code` requests to Claude on Bedrock:
model_list:
- model_name: bedrock-claude
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
model: bedrock/anthropic.claude-haiku-4-5-20251001:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1
@ -151,7 +151,7 @@ model_list:
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: anthropic-claude
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
model: bedrock/anthropic.claude-haiku-4-5-20251001:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1

2
docs/my-website/docusaurus.config.js
View file

@ -284,8 +284,8 @@ const config = {
label: 'Enterprise',
to: "docs/enterprise"
},
{ to: '/release_notes', label: 'Changelog', position: 'left' },
{ to: '/blog', label: 'Blog', position: 'left' },
{ to: '/release_notes', label: 'Release Notes', position: 'left' },
{
href: 'https://github.com/BerriAI/litellm',
position: 'right',

BIN
docs/my-website/img/april_townhall_banner.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 49 KiB

25
docs/my-website/package-lock.json generated
View file

@ -13,18 +13,18 @@
"@docusaurus/plugin-ideal-image": "3.8.1",
"@docusaurus/preset-classic": "3.8.1",
"@docusaurus/theme-mermaid": "3.8.1",
"@inkeep/cxkit-docusaurus": "^0.5.89",
"@mdx-js/react": "^3.0.0",
"clsx": "^1.2.1",
"prism-react-renderer": "^1.3.5",
"react": "^18.0.0 || ^19.0.0",
"react-dom": "^18.0.0 || ^19.0.0",
"sharp": "^0.32.6",
"uuid": "^9.0.1"
"@inkeep/cxkit-docusaurus": "0.5.107",
"@mdx-js/react": "3.1.1",
"clsx": "1.2.1",
"prism-react-renderer": "1.3.5",
"react": "18.3.1",
"react-dom": "18.3.1",
"sharp": "0.32.6",
"uuid": "9.0.1"
},
"devDependencies": {
"@docusaurus/module-type-aliases": "3.8.1",
"dotenv": "^16.4.5"
"dotenv": "16.6.1"
},
"engines": {
"node": ">=16.14",
@ -20403,13 +20403,6 @@
"url": "https://opencollective.com/webpack"
}
},
"node_modules/search-insights": {
"version": "2.17.3",
"resolved": "https://registry.npmjs.org/search-insights/-/search-insights-2.17.3.tgz",
"integrity": "sha512-RQPdCYTa8A68uM2jwxoY842xDhvx3E5LFL1LxvxCNMev4o5mLuokczhzjAgGwUZBAmOKZknArSxLKmXtIi2AxQ==",
"license": "MIT",
"peer": true
},
"node_modules/section-matter": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/section-matter/-/section-matter-1.0.0.tgz",

18
docs/my-website/package.json
View file

@ -19,18 +19,18 @@
"@docusaurus/plugin-ideal-image": "3.8.1",
"@docusaurus/preset-classic": "3.8.1",
"@docusaurus/theme-mermaid": "3.8.1",
"@inkeep/cxkit-docusaurus": "^0.5.89",
"@mdx-js/react": "^3.0.0",
"clsx": "^1.2.1",
"prism-react-renderer": "^1.3.5",
"react": "^18.0.0 || ^19.0.0",
"react-dom": "^18.0.0 || ^19.0.0",
"sharp": "^0.32.6",
"uuid": "^9.0.1"
"@inkeep/cxkit-docusaurus": "0.5.107",
"@mdx-js/react": "3.1.1",
"clsx": "1.2.1",
"prism-react-renderer": "1.3.5",
"react": "18.3.1",
"react-dom": "18.3.1",
"sharp": "0.32.6",
"uuid": "9.0.1"
},
"devDependencies": {
"@docusaurus/module-type-aliases": "3.8.1",
"dotenv": "^16.4.5"
"dotenv": "16.6.1"
},
"browserslist": {
"production": [

8
litellm-js/proxy/package.json
View file

@ -4,11 +4,11 @@
"deploy": "wrangler deploy --minify src/index.ts"
},
"dependencies": {
"hono": "^4.1.4",
"openai": "^4.29.2"
"hono": "4.1.4",
"openai": "4.29.2"
},
"devDependencies": {
"@cloudflare/workers-types": "^4.20240208.0",
"wrangler": "^3.32.0"
"@cloudflare/workers-types": "4.20240208.0",
"wrangler": "3.32.0"
}
}

8
litellm-js/spend-logs/package-lock.json generated
View file

@ -5,12 +5,12 @@
"packages": {
"": {
"dependencies": {
"@hono/node-server": "^1.10.1",
"hono": "^4.12.7"
"@hono/node-server": "1.19.6",
"hono": "4.12.7"
},
"devDependencies": {
"@types/node": "^20.11.17",
"tsx": "^4.7.1"
"@types/node": "20.19.25",
"tsx": "4.20.6"
}
},
"node_modules/@esbuild/aix-ppc64": {

8
litellm-js/spend-logs/package.json
View file

@ -3,11 +3,11 @@
"dev": "tsx watch src/index.ts"
},
"dependencies": {
"@hono/node-server": "^1.10.1",
"hono": "^4.12.7"
"@hono/node-server": "1.19.6",
"hono": "4.12.7"
},
"devDependencies": {
"@types/node": "^20.11.17",
"tsx": "^4.7.1"
"@types/node": "20.19.25",
"tsx": "4.20.6"
}
}

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-proxy-extras"
version = "0.4.62"
version = "0.4.63"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
authors = ["BerriAI"]
readme = "README.md"
@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.4.62"
version = "0.4.63"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-proxy-extras==",

4
litellm/cost_calculator.py
View file

@ -545,8 +545,8 @@ def cost_per_token( # noqa: PLR0915
)
if (
model_info.get("input_cost_per_token", 0) > 0
or model_info.get("output_cost_per_token", 0) > 0
(model_info.get("input_cost_per_token") or 0.0) > 0
or (model_info.get("output_cost_per_token") or 0.0) > 0
):
return generic_cost_per_token(
model=model,

5
litellm/litellm_core_utils/llm_request_utils.py
View file

@ -56,9 +56,8 @@ def pick_cheapest_chat_models_from_llm_provider(custom_llm_provider: str, n=1):
continue
if model_info.get("mode") != "chat":
continue
_cost = model_info.get("input_cost_per_token", 0) + model_info.get(
"output_cost_per_token", 0
)
_cost = (model_info.get("input_cost_per_token") or 0.0) + (model_info.get(
"output_cost_per_token") or 0.0)
model_costs.append((model, _cost))
# Sort by cost (ascending)

29
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -337,24 +337,35 @@ def _insert_assistant_continue_message(
"""
Add assistant continuation messages between consecutive user messages.
Only checks directly adjacent messages to preserve backward compatibility.
Skips tool messages and assistant messages with tool calls in the
alternation check, matching strict templates like llama.cpp.
"""
if not ensure_alternating_roles or len(messages) <= 1:
return messages
continue_message = assistant_continue_message or DEFAULT_ASSISTANT_CONTINUE_MESSAGE
# Find indexes where assistant_continue should be inserted (before that index)
insert_before_indexes: set = set()
for i in range(len(messages)):
curr = messages[i]
if _counts_for_alternation(curr) and curr["role"] == "user":
# Look backwards for the previous counted message
j = i - 1
while j >= 0:
if _counts_for_alternation(messages[j]):
if messages[j]["role"] == "user":
insert_before_indexes.add(i)
break
j -= 1
# Build the result with assistant_continue inserted at the right positions
modified_messages: List[AllMessageValues] = []
for i, message in enumerate(messages):
if (
i < len(messages) - 1
and message.get("role") == "user"
and messages[i + 1].get("role") == "user"
):
modified_messages.append(message)
if i in insert_before_indexes:
modified_messages.append(continue_message)
else:
modified_messages.append(message)
modified_messages.append(message)
return modified_messages

2
litellm/llms/azure_ai/cost_calculator.py
View file

@ -54,7 +54,7 @@ def calculate_azure_model_router_flat_cost(model: str, prompt_tokens: int) -> fl
model_info = get_model_info(model="model_router", custom_llm_provider="azure_ai")
router_flat_cost_per_token = model_info.get("input_cost_per_token", 0)
if router_flat_cost_per_token > 0:
if router_flat_cost_per_token and router_flat_cost_per_token > 0:
return prompt_tokens * router_flat_cost_per_token
return 0.0

14
litellm/llms/bedrock/chat/agentcore/transformation.py
View file

@ -19,6 +19,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.a2a.common_utils import extract_text_from_a2a_response
from litellm.llms.bedrock.common_utils import BedrockError
from litellm.types.llms.bedrock_agentcore import (
AgentCoreMessage,
@ -343,6 +344,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
Parse direct JSON response (non-streaming).
Supports multiple agent response schemas:
0. {"jsonrpc": "2.0", "result": {"message": {"parts": [...]}}} - A2A JSON-RPC
1. {"result": {"role": "assistant", "content": [{"text": "..."}]}} - standard AgentCore
2. {"response": [{"text": "..."}]} - Strands agent format
3. {"result": "plain text"} or {"response": "plain text"} - simple string
@ -361,6 +363,18 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
final_message=None,
)
# Strategy 0: A2A JSON-RPC format
# {"jsonrpc": "2.0", "result": {"message": {"parts": [{"kind": "text", "text": "..."}]}}}
if "jsonrpc" in response_json:
content = extract_text_from_a2a_response(response_json)
if content:
return AgentCoreParsedResponse(
content=content,
usage=None,
final_message=None,
)
# Fall through to other strategies if A2A extraction returned empty
# Strategy 1: {"result": {"content": [{"text": "..."}]}} - standard AgentCore format
if "result" in response_json and isinstance(response_json["result"], dict):
result = response_json["result"]

115
litellm/llms/bedrock/chat/invoke_handler.py
View file

@ -855,6 +855,32 @@ class BedrockLLM(BaseAWSLLM):
endpoint_url = f"{endpoint_url}/model/{modelId}/invoke"
proxy_endpoint_url = f"{proxy_endpoint_url}/model/{modelId}/invoke"
if acompletion and provider == "anthropic" and self.is_claude_messages_api_model(
model
):
if isinstance(client, HTTPHandler):
client = None
return self._async_anthropic_messages_completion(
model=model,
messages=messages,
endpoint_url=endpoint_url,
proxy_endpoint_url=proxy_endpoint_url,
credentials=credentials,
aws_region_name=aws_region_name,
model_response=model_response,
print_verbose=print_verbose,
encoding=encoding,
logging_obj=logging_obj,
optional_params=optional_params,
stream=stream,
litellm_params=litellm_params,
logger_fn=logger_fn,
extra_headers=extra_headers,
timeout=timeout,
client=client,
stream_chunk_size=stream_chunk_size,
) # type: ignore[return-value]
prompt, chat_history = self.convert_messages_to_prompt(
model, messages, provider, custom_prompt_dict
)
@ -1148,6 +1174,95 @@ class BedrockLLM(BaseAWSLLM):
encoding=encoding,
)
async def _async_anthropic_messages_completion(
self,
model: str,
messages: list,
endpoint_url: str,
proxy_endpoint_url: str,
credentials,
aws_region_name: str,
model_response: ModelResponse,
print_verbose: Callable,
encoding,
logging_obj: Logging,
optional_params: dict,
stream,
litellm_params=None,
logger_fn=None,
extra_headers: Optional[dict] = None,
timeout: Optional[Union[float, httpx.Timeout]] = None,
client: Optional[AsyncHTTPHandler] = None,
stream_chunk_size: int = 1024,
) -> Union[ModelResponse, CustomStreamWrapper]:
transformed_request = await litellm.AmazonAnthropicClaudeConfig().async_transform_request(
model=model,
messages=messages,
optional_params=optional_params,
litellm_params=litellm_params or {},
headers=extra_headers or {},
)
data = json.dumps(transformed_request)
headers = {"Content-Type": "application/json"}
if extra_headers is not None:
headers = {"Content-Type": "application/json", **extra_headers}
prepped = self.get_request_headers(
credentials=credentials,
aws_region_name=aws_region_name,
extra_headers=extra_headers,
endpoint_url=endpoint_url,
data=data,
headers=headers,
)
logging_obj.pre_call(
input=messages,
api_key="",
additional_args={
"complete_input_dict": data,
"api_base": proxy_endpoint_url,
"headers": prepped.headers,
},
)
if stream is True:
return await self.async_streaming(
model=model,
messages=messages,
data=data,
api_base=proxy_endpoint_url,
model_response=model_response,
print_verbose=print_verbose,
encoding=encoding,
logging_obj=logging_obj,
optional_params=optional_params,
stream=True,
litellm_params=litellm_params,
logger_fn=logger_fn,
headers=prepped.headers,
timeout=timeout,
client=client,
stream_chunk_size=stream_chunk_size,
)
return await self.async_completion(
model=model,
messages=messages,
data=data,
api_base=proxy_endpoint_url,
model_response=model_response,
print_verbose=print_verbose,
encoding=encoding,
logging_obj=logging_obj,
optional_params=optional_params,
stream=stream, # type: ignore
litellm_params=litellm_params,
logger_fn=logger_fn,
headers=prepped.headers,
timeout=timeout,
client=client,
)
async def async_completion(
self,
model: str,

188
litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py
View file

@ -2,6 +2,14 @@ from typing import TYPE_CHECKING, Any, List, Optional
import httpx
from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers
from litellm.litellm_core_utils.prompt_templates.factory import (
convert_to_anthropic_image_obj,
)
from litellm.litellm_core_utils.prompt_templates.image_handling import (
async_convert_url_to_base64,
convert_url_to_base64,
)
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
AmazonInvokeConfig,
@ -85,8 +93,62 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
litellm_params: dict,
headers: dict,
) -> dict:
# Filter out AWS authentication parameters before passing to Anthropic transformation
# AWS params should only be used for signing requests, not included in request body
_anthropic_request = self._build_bedrock_anthropic_request_base(
model=model,
messages=messages,
optional_params=optional_params,
litellm_params=litellm_params,
headers=headers,
)
self._convert_document_url_sources_to_base64(_anthropic_request)
beta_list = self._compute_bedrock_invoke_beta_headers(
model=model,
messages=messages,
optional_params=optional_params,
headers=headers,
)
if beta_list:
_anthropic_request["anthropic_beta"] = beta_list
return _anthropic_request
async def async_transform_request(
self,
model: str,
messages: List[AllMessageValues],
optional_params: dict,
litellm_params: dict,
headers: dict,
) -> dict:
_anthropic_request = self._build_bedrock_anthropic_request_base(
model=model,
messages=messages,
optional_params=optional_params,
litellm_params=litellm_params,
headers=headers,
)
await self._async_convert_document_url_sources_to_base64(_anthropic_request)
beta_list = self._compute_bedrock_invoke_beta_headers(
model=model,
messages=messages,
optional_params=optional_params,
headers=headers,
)
if beta_list:
_anthropic_request["anthropic_beta"] = beta_list
return _anthropic_request
def _build_bedrock_anthropic_request_base(
self,
model: str,
messages: List[AllMessageValues],
optional_params: dict,
litellm_params: dict,
headers: dict,
) -> dict:
filtered_params = {
k: v
for k, v in optional_params.items()
@ -94,7 +156,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
}
filtered_params = self._normalize_bedrock_tool_search_tools(filtered_params)
_anthropic_request = AnthropicConfig.transform_request(
anthropic_request = AnthropicConfig.transform_request(
self,
model=model,
messages=messages,
@ -103,28 +165,31 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
headers=headers,
)
_anthropic_request.pop("model", None)
_anthropic_request.pop("stream", None)
# Bedrock Invoke doesn't support output_format parameter
_anthropic_request.pop("output_format", None)
# Bedrock Invoke doesn't support output_config parameter
# Fixes: https://github.com/BerriAI/litellm/issues/22797
_anthropic_request.pop("output_config", None)
if "anthropic_version" not in _anthropic_request:
_anthropic_request["anthropic_version"] = self.anthropic_version
anthropic_request.pop("model", None)
anthropic_request.pop("stream", None)
anthropic_request.pop("output_format", None)
anthropic_request.pop("output_config", None)
if "anthropic_version" not in anthropic_request:
anthropic_request["anthropic_version"] = self.anthropic_version
# Remove `custom` field from tools (Bedrock doesn't support it)
# Claude Code sends `custom: {defer_loading: true}` on tool definitions,
# which causes Bedrock to reject the request with "Extra inputs are not permitted"
# Ref: https://github.com/BerriAI/litellm/issues/22847
remove_custom_field_from_tools(_anthropic_request)
remove_custom_field_from_tools(anthropic_request)
return anthropic_request
def _compute_bedrock_invoke_beta_headers(
self,
model: str,
messages: List[AllMessageValues],
optional_params: dict,
headers: dict,
) -> List[str]:
tools = optional_params.get("tools")
tool_search_used = self.is_tool_search_used(tools)
programmatic_tool_calling_used = self.is_programmatic_tool_calling_used(tools)
input_examples_used = self.is_input_examples_used(tools)
beta_set = set(get_anthropic_beta_from_headers(headers))
user_beta_set = set(get_anthropic_beta_from_headers(headers))
beta_set = set(user_beta_set)
auto_betas = self.get_anthropic_beta_list(
model=model,
optional_params=optional_params,
@ -142,12 +207,91 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
if "opus-4" in model.lower() or "opus_4" in model.lower():
beta_set.add("tool-search-tool-2025-10-19")
# Filter out beta headers that Bedrock Invoke doesn't support
# Uses centralized configuration from anthropic_beta_headers_config.json
beta_list = list(beta_set)
_anthropic_request["anthropic_beta"] = beta_list
auto_beta_list = filter_and_transform_beta_headers(
beta_headers=list(beta_set - user_beta_set),
provider="bedrock",
)
return sorted(user_beta_set.union(set(auto_beta_list)))
return _anthropic_request
def _convert_document_url_sources_to_base64(self, anthropic_request: dict) -> None:
"""
Bedrock Invoke does not accept document URL sources. Convert to base64 payloads.
"""
messages = anthropic_request.get("messages")
if not isinstance(messages, list):
return
for message in messages:
if not isinstance(message, dict):
continue
content = message.get("content")
if not isinstance(content, list):
continue
for block in content:
if not isinstance(block, dict) or block.get("type") != "document":
continue
source = block.get("source")
if not isinstance(source, dict) or source.get("type") != "url":
continue
source_url = source.get("url")
if not isinstance(source_url, str):
continue
inferred_format: Optional[str] = None
if source_url.lower().endswith(".pdf"):
inferred_format = "application/pdf"
base64_url = convert_url_to_base64(url=source_url)
image_chunk = convert_to_anthropic_image_obj(
openai_image_url=base64_url,
format=inferred_format,
)
block["source"] = {
"type": "base64",
"media_type": image_chunk["media_type"],
"data": image_chunk["data"],
}
async def _async_convert_document_url_sources_to_base64(
self, anthropic_request: dict
) -> None:
"""
Async version of document URL conversion for async completion paths.
"""
messages = anthropic_request.get("messages")
if not isinstance(messages, list):
return
for message in messages:
if not isinstance(message, dict):
continue
content = message.get("content")
if not isinstance(content, list):
continue
for block in content:
if not isinstance(block, dict) or block.get("type") != "document":
continue
source = block.get("source")
if not isinstance(source, dict) or source.get("type") != "url":
continue
source_url = source.get("url")
if not isinstance(source_url, str):
continue
inferred_format: Optional[str] = None
if source_url.lower().endswith(".pdf"):
inferred_format = "application/pdf"
base64_url = await async_convert_url_to_base64(url=source_url)
image_chunk = convert_to_anthropic_image_obj(
openai_image_url=base64_url,
format=inferred_format,
)
block["source"] = {
"type": "base64",
"media_type": image_chunk["media_type"],
"data": image_chunk["data"],
}
def _normalize_bedrock_tool_search_tools(self, optional_params: dict) -> dict:
"""

16
litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py
View file

@ -12,6 +12,7 @@ from typing import (
import httpx
from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
AnthropicMessagesConfig,
@ -54,9 +55,6 @@ class AmazonAnthropicClaudeMessagesConfig(
DEFAULT_BEDROCK_ANTHROPIC_API_VERSION = "bedrock-2023-05-31"
# Beta header patterns that are not supported by Bedrock Invoke API
# These will be filtered out to prevent 400 "invalid beta flag" errors
def __init__(self, **kwargs):
BaseAnthropicMessagesConfig.__init__(self, **kwargs)
AmazonInvokeConfig.__init__(self, **kwargs)
@ -439,7 +437,8 @@ class AmazonAnthropicClaudeMessagesConfig(
)
input_examples_used = anthropic_model_info.is_input_examples_used(tools)
beta_set = set(get_anthropic_beta_from_headers(headers))
user_beta_set = set(get_anthropic_beta_from_headers(headers))
beta_set = set(user_beta_set)
auto_betas = anthropic_model_info.get_anthropic_beta_list(
model=model,
optional_params=anthropic_messages_optional_request_params,
@ -463,8 +462,13 @@ class AmazonAnthropicClaudeMessagesConfig(
if "tool-search-tool-2025-10-19" in beta_set:
beta_set.add("tool-examples-2025-10-29")
if beta_set:
anthropic_messages_request["anthropic_beta"] = list(beta_set)
filtered_auto_betas = filter_and_transform_beta_headers(
beta_headers=list(beta_set - user_beta_set),
provider="bedrock",
)
filtered_betas = sorted(user_beta_set.union(set(filtered_auto_betas)))
if filtered_betas:
anthropic_messages_request["anthropic_beta"] = filtered_betas
return anthropic_messages_request

4
litellm/llms/vertex_ai/cost_calculator.py
View file

@ -203,7 +203,7 @@ def _handle_128k_pricing(
):
prompt_cost = prompt_tokens * input_cost_per_token_above_128k_tokens
else:
prompt_cost = prompt_tokens * model_info["input_cost_per_token"]
prompt_cost = prompt_tokens * (model_info["input_cost_per_token"] or 0.0)
## CALCULATE OUTPUT COST
output_cost_per_token_above_128k_tokens = model_info.get(
@ -215,7 +215,7 @@ def _handle_128k_pricing(
):
completion_cost = completion_tokens * output_cost_per_token_above_128k_tokens
else:
completion_cost = completion_tokens * model_info["output_cost_per_token"]
completion_cost = completion_tokens * (model_info["output_cost_per_token"] or 0.0)
return prompt_cost, completion_cost

290
litellm/model_prices_and_context_window_backup.json
View file

@ -6672,6 +6672,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-northeast-1/minimax.minimax-m2.5": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.44e-06
},
"bedrock/ap-northeast-1/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 7.3e-07,
"litellm_provider": "bedrock",
@ -6781,6 +6795,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-south-1/minimax.minimax-m2.5": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.44e-06
},
"bedrock/ap-south-1/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 7.1e-07,
"litellm_provider": "bedrock",
@ -6819,6 +6847,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-southeast-2/minimax.minimax-m2.5": {
"input_cost_per_token": 3.09e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.236e-06
},
"bedrock/ap-southeast-3/deepseek.v3.2": {
"input_cost_per_token": 7.4e-07,
"litellm_provider": "bedrock",
@ -6845,6 +6887,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-southeast-3/minimax.minimax-m2.5": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.44e-06
},
"bedrock/ap-southeast-3/moonshotai.kimi-k2.5": {
"input_cost_per_token": 7.2e-07,
"litellm_provider": "bedrock",
@ -6916,6 +6972,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-north-1/minimax.minimax-m2.5": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.44e-06
},
"bedrock/eu-north-1/moonshotai.kimi-k2.5": {
"input_cost_per_token": 7.2e-07,
"litellm_provider": "bedrock",
@ -7030,6 +7100,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-central-1/minimax.minimax-m2.5": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.44e-06
},
"bedrock/eu-central-1/qwen.qwen3-coder-next": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
@ -7074,6 +7158,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-west-1/minimax.minimax-m2.5": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.44e-06
},
"bedrock/eu-west-1/qwen.qwen3-coder-next": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
@ -7118,6 +7216,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-west-2/minimax.minimax-m2.5": {
"input_cost_per_token": 4.7e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.86e-06
},
"bedrock/eu-west-2/qwen.qwen3-coder-next": {
"input_cost_per_token": 7.8e-07,
"litellm_provider": "bedrock",
@ -7174,6 +7286,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-south-1/minimax.minimax-m2.5": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.44e-06
},
"bedrock/eu-south-1/qwen.qwen3-coder-next": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
@ -7249,6 +7375,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/sa-east-1/minimax.minimax-m2.5": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.44e-06
},
"bedrock/sa-east-1/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 7.3e-07,
"litellm_provider": "bedrock",
@ -7449,6 +7589,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-east-1/minimax.minimax-m2.5": {
"input_cost_per_token": 3e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.2e-06
},
"bedrock/us-east-1/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
@ -7513,6 +7667,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-east-2/minimax.minimax-m2.5": {
"input_cost_per_token": 3e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.2e-06
},
"bedrock/us-east-2/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
@ -7643,13 +7811,13 @@
"cache_creation_input_token_cost": 3.75e-07
},
"bedrock/us-gov-east-1/claude-sonnet-4-5-20250929-v1:0": {
"input_cost_per_token": 3.3e-06,
"input_cost_per_token": 3.6e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 200000,
"max_output_tokens": 4096,
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.65e-05,
"output_cost_per_token": 1.8e-05,
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
@ -7659,8 +7827,8 @@
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"cache_read_input_token_cost": 3.3e-07,
"cache_creation_input_token_cost": 4.125e-06
"cache_read_input_token_cost": 3.6e-07,
"cache_creation_input_token_cost": 4.5e-06
},
"bedrock/us-gov-east-1/meta.llama3-70b-instruct-v1:0": {
"input_cost_per_token": 2.65e-06,
@ -7794,13 +7962,13 @@
"cache_creation_input_token_cost": 3.75e-07
},
"bedrock/us-gov-west-1/claude-sonnet-4-5-20250929-v1:0": {
"input_cost_per_token": 3.3e-06,
"input_cost_per_token": 3.6e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 200000,
"max_output_tokens": 4096,
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.65e-05,
"output_cost_per_token": 1.8e-05,
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
@ -7810,8 +7978,8 @@
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"cache_read_input_token_cost": 3.3e-07,
"cache_creation_input_token_cost": 4.125e-06
"cache_read_input_token_cost": 3.6e-07,
"cache_creation_input_token_cost": 4.5e-06
},
"bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0": {
"input_cost_per_token": 2.65e-06,
@ -7995,6 +8163,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-west-2/minimax.minimax-m2.5": {
"input_cost_per_token": 3e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.2e-06
},
"bedrock/us-west-2/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
@ -21292,6 +21474,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"minimax.minimax-m2.5": {
"input_cost_per_token": 3e-07,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"minimax/speech-02-hd": {
"input_cost_per_character": 0.0001,
"litellm_provider": "minimax",
@ -23111,6 +23307,20 @@
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_native_structured_output": true
},
"nvidia.nemotron-super-3-120b": {
"input_cost_per_token": 1.5e-07,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 256000,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 6.5e-07,
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"o1": {
"cache_read_input_token_cost": 7.5e-06,
"input_cost_per_token": 1.5e-05,
@ -31164,7 +31374,9 @@
"mode": "chat",
"output_cost_per_token": 3.2e-06,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#glm-models",
"supported_regions": ["global"],
"supported_regions": [
"global"
],
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
@ -32724,6 +32936,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"zai.glm-5": {
"input_cost_per_token": 1e-06,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 200000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 3.2e-06,
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"zai/glm-5": {
"cache_creation_input_token_cost": 0,
"cache_read_input_token_cost": 2e-07,
@ -37473,5 +37699,51 @@
]
}
]
},
"bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0": {
"cache_creation_input_token_cost": 1.5e-06,
"cache_read_input_token_cost": 1.2e-07,
"input_cost_per_token": 1.2e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 6e-06,
"source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock",
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true,
"supports_pdf_input": true
},
"bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0": {
"cache_creation_input_token_cost": 1.5e-06,
"cache_read_input_token_cost": 1.2e-07,
"input_cost_per_token": 1.2e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 6e-06,
"source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock",
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true,
"supports_pdf_input": true
}
}

22
litellm/proxy/_types.py
View file

@ -4098,6 +4098,24 @@ class ScopeMapping(OIDCPermissions):
}
class JWTRoutingOverride(BaseModel):
"""
Override default auth routing for JWT-shaped bearer tokens.
A rule matches when all provided selectors match token claims.
If matched, request is routed to the configured auth path.
"""
iss: Union[str, List[str]]
client_id: Optional[Union[str, List[str]]] = None
aud: Optional[Union[str, List[str]]] = None
path: Literal["oauth2"] = "oauth2"
model_config = {
"extra": "forbid",
}
class LiteLLM_JWTAuth(LiteLLMPydanticObjectBase):
"""
A class to define the roles and permissions for a LiteLLM Proxy w/ JWT Auth.
@ -4198,6 +4216,10 @@ class LiteLLM_JWTAuth(LiteLLMPydanticObjectBase):
default=300,
description="TTL (seconds) for caching JWT-to-virtual-key mapping lookups.",
)
routing_overrides: Optional[List[JWTRoutingOverride]] = Field(
default=None,
description="Optional claim-based routing overrides for JWT-shaped tokens. Matching rules route requests to oauth2 before default JWT flow.",
)
#########################################################
def __init__(self, **kwargs: Any) -> None:

46
litellm/proxy/auth/auth_checks.py
View file

@ -165,9 +165,24 @@ def _is_model_cost_zero(
)
return False
# This model has zero cost explicitly configured
# Costs are 0 — verify this is from explicit configuration,
# not from defaulted sparse auto-registration entries.
# See: https://github.com/BerriAI/litellm/issues/24770
safe_name = str(model_name).replace("\n", "").replace("\r", "")
if not _is_cost_explicitly_configured(model_name, llm_router):
verbose_proxy_logger.debug(
"Model %s has zero cost but no explicit cost "
"configuration in model_cost entry — treating as unknown "
"cost (enforce budget)",
safe_name,
)
return False
verbose_proxy_logger.debug(
f"Model {model_name} has zero cost explicitly configured (input: {input_cost}, output: {output_cost})"
"Model %s has zero cost explicitly configured (input: %s, output: %s)",
safe_name,
input_cost,
output_cost,
)
except Exception as e:
@ -181,6 +196,33 @@ def _is_model_cost_zero(
return True
def _is_cost_explicitly_configured(
model: str, llm_router: "Router"
) -> bool:
"""
Check if any deployment in the model group has cost fields explicitly
set in its litellm.model_cost entry.
When Router._create_deployment() registers a model not in the global
cost map, it creates a sparse entry like {"id": "<hash>"} with no cost
fields. _get_model_info_helper() then defaults missing costs to 0.
This function detects that scenario by checking the raw model_cost entry.
"""
for deployment in llm_router.model_list:
if deployment.get("model_name") != model:
continue
model_id = deployment.get("model_info", {}).get("id")
if model_id is None:
continue
raw_entry = litellm.model_cost.get(model_id, {})
if (
"input_cost_per_token" in raw_entry
or "output_cost_per_token" in raw_entry
):
return True
return False
async def _run_project_checks(
project_object: Optional[LiteLLM_ProjectTableCachedObj],
_model: Optional[Union[str, List[str]]],

64
litellm/proxy/auth/handle_jwt.py
View file

@ -16,6 +16,8 @@ from cryptography import x509
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import serialization
from fastapi import HTTPException
import jwt
from jwt.api_jwk import PyJWK
from litellm._logging import verbose_proxy_logger
from litellm.caching.caching import DualCache
@ -71,6 +73,21 @@ class JWTHandler:
prisma_client: Optional[PrismaClient]
user_api_key_cache: DualCache
# Supported algos: https://pyjwt.readthedocs.io/en/stable/algorithms.html
# "Warning: Make sure not to mix symmetric and asymmetric algorithms that interpret
# the key in different ways (e.g. HS* and RS*)."
SUPPORTED_JWT_ALGORITHMS = [
"RS256",
"RS384",
"RS512",
"PS256",
"PS384",
"PS512",
"ES256",
"ES384",
"ES512",
"EdDSA",
]
def __init__(
self,
@ -97,6 +114,30 @@ class JWTHandler:
parts = token.split(".")
return len(parts) == 3
@staticmethod
def get_unverified_claims(token: str) -> Optional[dict]:
"""
Decode JWT claims without signature verification.
Used for routing decisions before selecting validation path.
"""
if not JWTHandler.is_jwt(token):
return None
try:
claims = jwt.decode(
token,
options={"verify_signature": False, "verify_aud": False},
algorithms=JWTHandler.SUPPORTED_JWT_ALGORITHMS,
)
if isinstance(claims, dict):
return claims
return None
except Exception as e:
verbose_proxy_logger.debug(
"Failed to decode unverified JWT claims for routing: %s", e
)
return None
def _rbac_role_from_role_mapping(self, token: dict) -> Optional[RBAC_ROLES]:
"""
Returns the RBAC role the token 'belongs' to based on role mappings.
@ -664,30 +705,11 @@ class JWTHandler:
raise Exception(f"Failed to fetch OIDC UserInfo: {str(e)}")
async def auth_jwt(self, token: str) -> dict:
# Supported algos: https://pyjwt.readthedocs.io/en/stable/algorithms.html
# "Warning: Make sure not to mix symmetric and asymmetric algorithms that interpret
# the key in different ways (e.g. HS* and RS*)."
algorithms = [
"RS256",
"RS384",
"RS512",
"PS256",
"PS384",
"PS512",
"ES256",
"ES384",
"ES512",
"EdDSA",
]
audience = os.getenv("JWT_AUDIENCE")
decode_options = None
if audience is None:
decode_options = {"verify_aud": False}
import jwt
from jwt.api_jwk import PyJWK
header = jwt.get_unverified_header(token)
verbose_proxy_logger.debug("header: %s", header)
@ -721,7 +743,7 @@ class JWTHandler:
payload = jwt.decode(
token,
public_key_obj, # type: ignore
algorithms=algorithms,
algorithms=self.SUPPORTED_JWT_ALGORITHMS,
options=decode_options, # type: ignore[arg-type]
audience=audience,
leeway=self.leeway, # allow testing of expired tokens
@ -749,7 +771,7 @@ class JWTHandler:
payload = jwt.decode(
token,
key,
algorithms=algorithms,
algorithms=self.SUPPORTED_JWT_ALGORITHMS,
audience=audience,
options=decode_options,
)

183
litellm/proxy/auth/user_api_key_auth.py
View file

@ -11,7 +11,7 @@ import asyncio
import re
import secrets
from datetime import datetime, timezone
from typing import List, Optional, Tuple, cast
from typing import Any, List, Optional, Tuple, cast
import fastapi
from fastapi import HTTPException, Request, WebSocket, status
@ -139,6 +139,58 @@ def _get_bearer_token_or_received_api_key(api_key: str) -> str:
return api_key
def _routing_selector_matches_claim(
selector_value: Optional[Any], claim_value: Optional[Any]
) -> bool:
if selector_value is None:
return True
selector_list = (
[str(v) for v in selector_value]
if isinstance(selector_value, list)
else [str(selector_value)]
)
if isinstance(claim_value, list):
claim_list = [str(v) for v in claim_value]
return any(v in claim_list for v in selector_list)
return str(claim_value) in selector_list if claim_value is not None else False
def _matches_routing_override(
token_claims: dict, override: "JWTRoutingOverride"
) -> bool:
return (
_routing_selector_matches_claim(override.iss, token_claims.get("iss"))
and _routing_selector_matches_claim(
override.client_id, token_claims.get("client_id")
)
and _routing_selector_matches_claim(override.aud, token_claims.get("aud"))
)
def _should_route_jwt_to_oauth2_override(token: str, jwt_handler: JWTHandler) -> bool:
routing_overrides = jwt_handler.litellm_jwtauth.routing_overrides
if not routing_overrides:
return False
token_claims = jwt_handler.get_unverified_claims(token=token)
if token_claims is None:
return False
for override in routing_overrides:
if override.path == "oauth2" and _matches_routing_override(
token_claims=token_claims, override=override
):
verbose_proxy_logger.debug(
"JWT routing override matched. Routing token to OAuth2 introspection."
)
return True
return False
def _get_bearer_token(
api_key: str,
):
@ -649,12 +701,20 @@ async def _user_api_key_auth_builder( # noqa: PLR0915
# - JWT tokens (3 dot-separated parts) -> skip OAuth2, fall through to JWT handler
# - Opaque tokens -> use OAuth2 handler
# This allows JWT for users and OAuth2 for M2M on the same instance
is_jwt_token = (
is_jwt = (
jwt_handler.is_jwt(token=api_key)
if general_settings.get("enable_jwt_auth", False) is True
else False
)
if not is_jwt_token:
# Routing uses unverified JWT claims only to choose auth path.
# Final authentication is enforced by the selected validator.
route_jwt_to_oauth2 = (
is_jwt
and _should_route_jwt_to_oauth2_override(
token=api_key, jwt_handler=jwt_handler
)
)
if not is_jwt or route_jwt_to_oauth2:
# return UserAPIKeyAuth object
# helper to check if the api_key is a valid oauth2 token
from litellm.proxy.proxy_server import premium_user
@ -688,7 +748,7 @@ async def _user_api_key_auth_builder( # noqa: PLR0915
jwt_claims: Optional[dict]
if (
jwt_handler.litellm_jwtauth.oidc_userinfo_enabled
and not jwt_handler.is_jwt(token=api_key)
and not is_jwt
):
jwt_claims = await jwt_handler.get_oidc_userinfo(token=api_key)
else:
@ -1193,49 +1253,13 @@ async def _user_api_key_auth_builder( # noqa: PLR0915
raise Exception(
"Key is blocked. Update via `/key/unblock` if you're an admin."
)
config = valid_token.config
if config != {}:
model_list = config.get("model_list", [])
new_model_list = model_list
verbose_proxy_logger.debug(
f"\n new llm router model list {new_model_list}"
)
elif (
isinstance(valid_token.models, list)
and "all-team-models" in valid_token.models
):
# Do not do any validation at this step
# the validation will occur when checking the team has access to this model
pass
else:
model = get_model_from_request(request_data, route)
fallback_models = cast(
Optional[List[ALL_FALLBACK_MODEL_VALUES]],
request_data.get("fallbacks", None),
)
if model is not None:
await can_key_call_model(
model=model,
llm_model_list=llm_model_list,
valid_token=valid_token,
llm_router=llm_router,
)
if fallback_models is not None:
for m in fallback_models:
await can_key_call_model(
model=m["model"] if isinstance(m, dict) else m,
llm_model_list=llm_model_list,
valid_token=valid_token,
llm_router=llm_router,
)
await is_valid_fallback_model(
model=m["model"] if isinstance(m, dict) else m,
llm_router=llm_router,
user_model=None,
)
await _enforce_key_and_fallback_model_access(
valid_token=valid_token,
request_data=request_data,
route=route,
llm_model_list=llm_model_list,
llm_router=llm_router,
)
# Check 2. If user_id for this token is in budget - done in common_checks()
if valid_token.user_id is not None:
@ -1764,6 +1788,61 @@ async def _lookup_end_user_and_apply_budget(
return valid_token, end_user_object
async def _enforce_key_and_fallback_model_access(
*,
valid_token: UserAPIKeyAuth,
request_data: dict,
route: str,
llm_model_list: Optional[list],
llm_router: Optional[Any],
) -> None:
"""
Key-level model allowlist and client fallbacks (same as standard auth).
Not included in common_checks — common_checks enforces team/user/project model access only.
"""
config = valid_token.config
if config != {}:
model_list = config.get("model_list", [])
new_model_list = model_list
verbose_proxy_logger.debug(
f"\n new llm router model list {new_model_list}"
)
elif (
isinstance(valid_token.models, list)
and "all-team-models" in valid_token.models
):
pass
else:
model = get_model_from_request(request_data, route)
fallback_models = cast(
Optional[List[ALL_FALLBACK_MODEL_VALUES]],
request_data.get("fallbacks", None),
)
if model is not None:
await can_key_call_model(
model=model,
llm_model_list=llm_model_list,
valid_token=valid_token,
llm_router=llm_router,
)
if fallback_models is not None:
for m in fallback_models:
await can_key_call_model(
model=m["model"] if isinstance(m, dict) else m,
llm_model_list=llm_model_list,
valid_token=valid_token,
llm_router=llm_router,
)
await is_valid_fallback_model(
model=m["model"] if isinstance(m, dict) else m,
llm_router=llm_router,
user_model=None,
)
async def _run_post_custom_auth_checks(
valid_token: UserAPIKeyAuth,
request: Request,
@ -1773,6 +1852,7 @@ async def _run_post_custom_auth_checks(
) -> UserAPIKeyAuth:
from litellm.proxy.proxy_server import (
general_settings,
llm_model_list,
llm_router,
model_max_budget_limiter,
prisma_client,
@ -1816,6 +1896,15 @@ async def _run_post_custom_auth_checks(
),
)
if general_settings.get("custom_auth_run_common_checks", False):
await _enforce_key_and_fallback_model_access(
valid_token=valid_token,
request_data=request_data,
route=route,
llm_model_list=llm_model_list,
llm_router=llm_router,
)
current_model = request_data.get("model", None)
# 3. Check key-level model_max_budget

11
litellm/proxy/common_request_processing.py
View file

@ -221,16 +221,21 @@ async def create_response(
f"Error consuming first chunk from generator: {e}"
)
# Fallback to a generic error stream
# Preserve status code from HTTPException (e.g., guardrail blocks)
error_status = getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR)
error_detail = getattr(e, "detail", "Error processing stream start")
if not isinstance(error_detail, str):
error_detail = str(error_detail)
async def error_gen_message() -> AsyncGenerator[str, None]:
yield f"data: {json.dumps({'error': {'message': 'Error processing stream start', 'code': status.HTTP_500_INTERNAL_SERVER_ERROR}})}\n\n"
yield f"data: {json.dumps({'error': {'message': error_detail, 'code': error_status}})}\n\n"
yield "data: [DONE]\n\n"
return StreamingResponse(
error_gen_message(),
media_type=media_type,
headers=headers,
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
status_code=error_status,
)
async def combined_generator() -> AsyncGenerator[str, None]:

2
litellm/proxy/example_config_yaml/websearch_interception_config.yaml
View file

@ -1,7 +1,7 @@
model_list:
- model_name: claude-3-5-sonnet
litellm_params:
model: bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
# Search tools configuration
search_tools:

23
litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py
View file

@ -14,6 +14,8 @@ from fastapi import HTTPException
if TYPE_CHECKING:
from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel
import json
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.caching import DualCache
@ -203,8 +205,8 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
response.text,
)
raise HTTPException(
status_code=response.status_code,
detail=f"Model Armor API error: {response.text}",
status_code=400,
detail=f"Model Armor API error (upstream {response.status_code}): {response.text}",
)
json_response = response.json()
@ -746,8 +748,21 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
yield chunk
return
except HTTPException:
raise
except HTTPException as e:
# Yield error as SSE event so create_response() detects it and
# returns a proper JSON error response with the correct status code.
# (Raising from a generator hits create_response's generic except → 500.)
detail = (
e.detail if isinstance(e.detail, dict) else {"message": str(e.detail)}
)
error_value = detail.get("error", detail)
if isinstance(error_value, dict):
error_obj = dict(error_value)
else:
error_obj = {"message": str(error_value)}
error_obj["code"] = e.status_code
yield f"data: {json.dumps({'error': error_obj})}\n\n"
return
except Exception as e:
verbose_proxy_logger.error(
"Model Armor streaming error: %s", str(e), exc_info=True

10
litellm/proxy/management_endpoints/key_management_endpoints.py
View file

@ -942,9 +942,9 @@ async def _check_team_key_limits(
where={"team_id": team_table.team_id},
)
# Exclude the key being updated to avoid double-counting its limits.
# key.token is the SHA-256 hash stored in DB; data.key is the raw key string.
# data.key may be a raw key (sk-...) or a pre-hashed token_id.
if isinstance(data, UpdateKeyRequest):
hashed_key = hash_token(data.key)
hashed_key = _hash_token_if_needed(data.key)
keys = [key for key in keys if key.token != hashed_key]
check_team_key_model_specific_limits(
keys=keys,
@ -1101,9 +1101,9 @@ async def _check_org_key_limits(
where={"organization_id": org_table.organization_id},
)
# Exclude the key being updated to avoid double-counting its limits.
# key.token is the SHA-256 hash stored in DB; data.key is the raw key string.
# data.key may be a raw key (sk-...) or a pre-hashed token_id.
if isinstance(data, UpdateKeyRequest):
hashed_key = hash_token(data.key)
hashed_key = _hash_token_if_needed(data.key)
keys = [key for key in keys if key.token != hashed_key]
check_org_key_model_specific_limits(
keys=keys,
@ -2157,7 +2157,7 @@ async def update_key_fn(
# Delete - key from cache, since it's been updated!
# key updated - a new model could have been added to this key. it should not block requests after this is done
await _delete_cache_key_object(
hashed_token=hash_token(key),
hashed_token=_hash_token_if_needed(key),
user_api_key_cache=user_api_key_cache,
proxy_logging_obj=proxy_logging_obj,
)

12
litellm/router.py
View file

@ -7808,8 +7808,8 @@ class Router:
max_tokens=None,
max_input_tokens=None,
max_output_tokens=None,
input_cost_per_token=0,
output_cost_per_token=0,
input_cost_per_token=None,
output_cost_per_token=None,
litellm_provider=llm_provider,
mode=mode,
supported_openai_params=supported_openai_params,
@ -7856,16 +7856,16 @@ class Router:
model_group_info.max_output_tokens = model_info["max_output_tokens"]
if model_info.get("input_cost_per_token", None) is not None and (
model_group_info.input_cost_per_token is None
or model_info["input_cost_per_token"]
> model_group_info.input_cost_per_token
or (model_info["input_cost_per_token"] or 0.0)
> (model_group_info.input_cost_per_token or 0.0)
):
model_group_info.input_cost_per_token = model_info[
"input_cost_per_token"
]
if model_info.get("output_cost_per_token", None) is not None and (
model_group_info.output_cost_per_token is None
or model_info["output_cost_per_token"]
> model_group_info.output_cost_per_token
or (model_info["output_cost_per_token"] or 0.0)
> (model_group_info.output_cost_per_token or 0.0)
):
model_group_info.output_cost_per_token = model_info[
"output_cost_per_token"

2
litellm/setup_wizard.py
View file

@ -86,7 +86,7 @@ PROVIDERS: List[Dict] = [
"env_key": "AWS_ACCESS_KEY_ID",
"key_hint": "AKIA...",
"test_model": None, # multi-key auth — skip validation
"models": ["bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0"],
"models": ["bedrock/anthropic.claude-haiku-4-5-20251001-v1:0"],
"extra_keys": ["AWS_SECRET_ACCESS_KEY", "AWS_REGION_NAME"],
"extra_hints": ["your-secret-key", "us-east-1"],
},

4
litellm/types/utils.py
View file

@ -169,7 +169,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
max_tokens: Required[Optional[int]]
max_input_tokens: Required[Optional[int]]
max_output_tokens: Required[Optional[int]]
input_cost_per_token: Required[float]
input_cost_per_token: Required[Optional[float]]
input_cost_per_token_flex: Optional[float] # OpenAI flex service tier pricing
input_cost_per_token_priority: Optional[
float
@ -206,7 +206,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
input_cost_per_second: Optional[float] # for OpenAI Speech models
input_cost_per_token_batches: Optional[float]
output_cost_per_token_batches: Optional[float]
output_cost_per_token: Required[float]
output_cost_per_token: Required[Optional[float]]
output_cost_per_token_flex: Optional[float] # OpenAI flex service tier pricing
output_cost_per_token_priority: Optional[
float

319
model_prices_and_context_window.json
View file

@ -6672,6 +6672,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-northeast-1/minimax.minimax-m2.5": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.44e-06
},
"bedrock/ap-northeast-1/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 7.3e-07,
"litellm_provider": "bedrock",
@ -6781,6 +6795,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-south-1/minimax.minimax-m2.5": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.44e-06
},
"bedrock/ap-south-1/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 7.1e-07,
"litellm_provider": "bedrock",
@ -6819,6 +6847,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-southeast-2/minimax.minimax-m2.5": {
"input_cost_per_token": 3.09e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.236e-06
},
"bedrock/ap-southeast-3/deepseek.v3.2": {
"input_cost_per_token": 7.4e-07,
"litellm_provider": "bedrock",
@ -6845,6 +6887,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-southeast-3/minimax.minimax-m2.5": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.44e-06
},
"bedrock/ap-southeast-3/moonshotai.kimi-k2.5": {
"input_cost_per_token": 7.2e-07,
"litellm_provider": "bedrock",
@ -6916,6 +6972,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-north-1/minimax.minimax-m2.5": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.44e-06
},
"bedrock/eu-north-1/moonshotai.kimi-k2.5": {
"input_cost_per_token": 7.2e-07,
"litellm_provider": "bedrock",
@ -7030,6 +7100,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-central-1/minimax.minimax-m2.5": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.44e-06
},
"bedrock/eu-central-1/qwen.qwen3-coder-next": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
@ -7074,6 +7158,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-west-1/minimax.minimax-m2.5": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.44e-06
},
"bedrock/eu-west-1/qwen.qwen3-coder-next": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
@ -7118,6 +7216,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-west-2/minimax.minimax-m2.5": {
"input_cost_per_token": 4.7e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.86e-06
},
"bedrock/eu-west-2/qwen.qwen3-coder-next": {
"input_cost_per_token": 7.8e-07,
"litellm_provider": "bedrock",
@ -7174,6 +7286,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-south-1/minimax.minimax-m2.5": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.44e-06
},
"bedrock/eu-south-1/qwen.qwen3-coder-next": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
@ -7249,6 +7375,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/sa-east-1/minimax.minimax-m2.5": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.44e-06
},
"bedrock/sa-east-1/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 7.3e-07,
"litellm_provider": "bedrock",
@ -7449,6 +7589,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-east-1/minimax.minimax-m2.5": {
"input_cost_per_token": 3e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.2e-06
},
"bedrock/us-east-1/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
@ -7513,6 +7667,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-east-2/minimax.minimax-m2.5": {
"input_cost_per_token": 3e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.2e-06
},
"bedrock/us-east-2/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
@ -7643,13 +7811,13 @@
"cache_creation_input_token_cost": 3.75e-07
},
"bedrock/us-gov-east-1/claude-sonnet-4-5-20250929-v1:0": {
"input_cost_per_token": 3.3e-06,
"input_cost_per_token": 3.6e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 200000,
"max_output_tokens": 4096,
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.65e-05,
"output_cost_per_token": 1.8e-05,
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
@ -7659,8 +7827,8 @@
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"cache_read_input_token_cost": 3.3e-07,
"cache_creation_input_token_cost": 4.125e-06
"cache_read_input_token_cost": 3.6e-07,
"cache_creation_input_token_cost": 4.5e-06
},
"bedrock/us-gov-east-1/meta.llama3-70b-instruct-v1:0": {
"input_cost_per_token": 2.65e-06,
@ -7794,13 +7962,13 @@
"cache_creation_input_token_cost": 3.75e-07
},
"bedrock/us-gov-west-1/claude-sonnet-4-5-20250929-v1:0": {
"input_cost_per_token": 3.3e-06,
"input_cost_per_token": 3.6e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 200000,
"max_output_tokens": 4096,
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.65e-05,
"output_cost_per_token": 1.8e-05,
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
@ -7810,8 +7978,8 @@
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"cache_read_input_token_cost": 3.3e-07,
"cache_creation_input_token_cost": 4.125e-06
"cache_read_input_token_cost": 3.6e-07,
"cache_creation_input_token_cost": 4.5e-06
},
"bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0": {
"input_cost_per_token": 2.65e-06,
@ -7995,6 +8163,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-west-2/minimax.minimax-m2.5": {
"input_cost_per_token": 3e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.2e-06
},
"bedrock/us-west-2/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
@ -18665,13 +18847,11 @@
"cache_read_input_token_cost_above_272k_tokens": 5e-07,
"cache_read_input_token_cost_flex": 1.3e-07,
"cache_read_input_token_cost_priority": 5e-07,
"cache_read_input_token_cost_above_272k_tokens_priority": 1e-06,
"input_cost_per_token": 2.5e-06,
"input_cost_per_token_above_272k_tokens": 5e-06,
"input_cost_per_token_flex": 1.25e-06,
"input_cost_per_token_batches": 1.25e-06,
"input_cost_per_token_priority": 5e-06,
"input_cost_per_token_above_272k_tokens_priority": 1e-05,
"litellm_provider": "openai",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
@ -18681,8 +18861,7 @@
"output_cost_per_token_above_272k_tokens": 2.25e-05,
"output_cost_per_token_flex": 7.5e-06,
"output_cost_per_token_batches": 7.5e-06,
"output_cost_per_token_priority": 2.25e-05,
"output_cost_per_token_above_272k_tokens_priority": 3.375e-05,
"output_cost_per_token_priority": 3e-05,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
@ -18715,13 +18894,11 @@
"cache_read_input_token_cost_above_272k_tokens": 5e-07,
"cache_read_input_token_cost_flex": 1.3e-07,
"cache_read_input_token_cost_priority": 5e-07,
"cache_read_input_token_cost_above_272k_tokens_priority": 1e-06,
"input_cost_per_token": 2.5e-06,
"input_cost_per_token_above_272k_tokens": 5e-06,
"input_cost_per_token_flex": 1.25e-06,
"input_cost_per_token_batches": 1.25e-06,
"input_cost_per_token_priority": 5e-06,
"input_cost_per_token_above_272k_tokens_priority": 1e-05,
"litellm_provider": "openai",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
@ -18731,8 +18908,7 @@
"output_cost_per_token_above_272k_tokens": 2.25e-05,
"output_cost_per_token_flex": 7.5e-06,
"output_cost_per_token_batches": 7.5e-06,
"output_cost_per_token_priority": 2.25e-05,
"output_cost_per_token_above_272k_tokens_priority": 3.375e-05,
"output_cost_per_token_priority": 3e-05,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
@ -18760,14 +18936,10 @@
"gpt-5.4-pro": {
"cache_read_input_token_cost": 3e-06,
"cache_read_input_token_cost_above_272k_tokens": 6e-06,
"cache_read_input_token_cost_priority": 6e-06,
"cache_read_input_token_cost_above_272k_tokens_priority": 1.2e-05,
"input_cost_per_token": 3e-05,
"input_cost_per_token_above_272k_tokens": 6e-05,
"input_cost_per_token_flex": 1.5e-05,
"input_cost_per_token_batches": 1.5e-05,
"input_cost_per_token_priority": 6e-05,
"input_cost_per_token_above_272k_tokens_priority": 0.00012,
"litellm_provider": "openai",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
@ -18777,8 +18949,6 @@
"output_cost_per_token_above_272k_tokens": 0.00027,
"output_cost_per_token_flex": 9e-05,
"output_cost_per_token_batches": 9e-05,
"output_cost_per_token_priority": 0.00027,
"output_cost_per_token_above_272k_tokens_priority": 0.000405,
"supported_endpoints": [
"/v1/responses",
"/v1/batch"
@ -18809,14 +18979,10 @@
"gpt-5.4-pro-2026-03-05": {
"cache_read_input_token_cost": 3e-06,
"cache_read_input_token_cost_above_272k_tokens": 6e-06,
"cache_read_input_token_cost_priority": 6e-06,
"cache_read_input_token_cost_above_272k_tokens_priority": 1.2e-05,
"input_cost_per_token": 3e-05,
"input_cost_per_token_above_272k_tokens": 6e-05,
"input_cost_per_token_flex": 1.5e-05,
"input_cost_per_token_batches": 1.5e-05,
"input_cost_per_token_priority": 6e-05,
"input_cost_per_token_above_272k_tokens_priority": 0.00012,
"litellm_provider": "openai",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
@ -18826,8 +18992,6 @@
"output_cost_per_token_above_272k_tokens": 0.00027,
"output_cost_per_token_flex": 9e-05,
"output_cost_per_token_batches": 9e-05,
"output_cost_per_token_priority": 0.00027,
"output_cost_per_token_above_272k_tokens_priority": 0.000405,
"supported_endpoints": [
"/v1/responses",
"/v1/batch"
@ -18857,11 +19021,13 @@
},
"gpt-5.4-mini": {
"cache_read_input_token_cost": 7.5e-08,
"cache_read_input_token_cost_flex": 1e-08,
"cache_read_input_token_cost_batches": 3.8e-08,
"cache_read_input_token_cost_flex": 3.75e-08,
"cache_read_input_token_cost_batches": 3.75e-08,
"cache_read_input_token_cost_priority": 1.5e-07,
"input_cost_per_token": 7.5e-07,
"input_cost_per_token_flex": 3.75e-07,
"input_cost_per_token_batches": 3.75e-07,
"input_cost_per_token_priority": 1.5e-06,
"litellm_provider": "openai",
"max_input_tokens": 272000,
"max_output_tokens": 128000,
@ -18870,6 +19036,7 @@
"output_cost_per_token": 4.5e-06,
"output_cost_per_token_flex": 2.25e-06,
"output_cost_per_token_batches": 2.25e-06,
"output_cost_per_token_priority": 9e-06,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
@ -21292,6 +21459,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"minimax.minimax-m2.5": {
"input_cost_per_token": 3e-07,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"minimax/speech-02-hd": {
"input_cost_per_character": 0.0001,
"litellm_provider": "minimax",
@ -23111,6 +23292,20 @@
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_native_structured_output": true
},
"nvidia.nemotron-super-3-120b": {
"input_cost_per_token": 1.5e-07,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 256000,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 6.5e-07,
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"o1": {
"cache_read_input_token_cost": 7.5e-06,
"input_cost_per_token": 1.5e-05,
@ -31164,7 +31359,9 @@
"mode": "chat",
"output_cost_per_token": 3.2e-06,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#glm-models",
"supported_regions": ["global"],
"supported_regions": [
"global"
],
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
@ -32724,6 +32921,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"zai.glm-5": {
"input_cost_per_token": 1e-06,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 200000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 3.2e-06,
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"zai/glm-5": {
"cache_creation_input_token_cost": 0,
"cache_read_input_token_cost": 2e-07,
@ -37473,5 +37684,51 @@
]
}
]
},
"bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0": {
"cache_creation_input_token_cost": 1.5e-06,
"cache_read_input_token_cost": 1.2e-07,
"input_cost_per_token": 1.2e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 6e-06,
"source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock",
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true,
"supports_pdf_input": true
},
"bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0": {
"cache_creation_input_token_cost": 1.5e-06,
"cache_read_input_token_cost": 1.2e-07,
"input_cost_per_token": 1.2e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 6e-06,
"source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock",
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true,
"supports_pdf_input": true
}
}

14
package.json
View file

@ -1,14 +1,14 @@
{
"dependencies": {
"prism-react-renderer": "^2.4.1",
"prisma": "^5.17.0",
"react-copy-to-clipboard": "^5.1.0"
"prism-react-renderer": "2.4.1",
"prisma": "5.17.0",
"react-copy-to-clipboard": "5.1.0"
},
"devDependencies": {
"@testing-library/jest-dom": "^6.8.0",
"@testing-library/react": "^14.3.1",
"@types/react-copy-to-clipboard": "^5.0.7",
"jest": "^29.7.0"
"@testing-library/jest-dom": "6.8.0",
"@testing-library/react": "14.3.1",
"@types/react-copy-to-clipboard": "5.0.7",
"jest": "29.7.0"
},
"overrides": {
"glob": "13.0.0",

1802
poetry.lock generated
View file

File diff suppressed because it is too large Load diff

176
pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm"
version = "1.83.0"
version = "1.83.1"
description = "Library to easily interface with LLM API providers"
authors = ["BerriAI"]
license = "MIT"
@ -18,66 +18,62 @@ Repository = "https://github.com/BerriAI/litellm"
documentation = "https://docs.litellm.ai"
Documentation = "https://docs.litellm.ai"
# Dependencies pinned from `pip install litellm[proxy]==1.83.0` PyPI resolution.
# Docker builds use requirements.txt (different pins). These two paths are independent.
[tool.poetry.dependencies]
python = ">=3.9,<4.0"
fastuuid = ">=0.13.0"
httpx = ">=0.23.0"
openai = ">=2.8.0"
python-dotenv = ">=0.2.0"
tiktoken = ">=0.7.0"
importlib-metadata = ">=6.8.0"
tokenizers = "*"
click = "*"
jinja2 = "^3.1.2"
aiohttp = ">=3.10"
pydantic = "^2.5.0"
jsonschema = ">=4.23.0,<5.0.0"
numpydoc = {version = "*", optional = true} # used in utils.py
fastuuid = "0.14.0"
httpx = "0.28.1"
openai = "2.30.0"
python-dotenv = "1.0.1"
tiktoken = "0.12.0"
importlib-metadata = "8.5.0"
tokenizers = "0.22.2"
click = "8.1.8"
jinja2 = "3.1.6"
aiohttp = "3.13.5"
pydantic = "2.12.5"
jsonschema = "4.23.0"
numpydoc = {version = "1.8.0", optional = true} # not in Docker or PyPI proxy extra
uvicorn = {version = ">=0.32.1,<1.0.0", optional = true}
uvloop = {version = "^0.21.0", optional = true, markers="sys_platform != 'win32'"}
gunicorn = {version = "^23.0.0", optional = true}
fastapi = {version = ">=0.120.1", optional = true}
backoff = {version = "*", optional = true}
pyyaml = {version = "^6.0.1", optional = true}
rq = {version = "*", optional = true}
orjson = {version = "^3.9.7", optional = true}
apscheduler = {version = "^3.10.4", optional = true}
fastapi-sso = { version = "^0.16.0", optional = true }
PyJWT = { version = "^2.12.0", optional = true, python = ">=3.9" }
python-multipart = { version = ">=0.0.20", optional = true}
cryptography = {version = "*", optional = true}
prisma = {version = "^0.11.0", optional = true}
azure-identity = {version = "^1.15.0", optional = true, python = ">=3.9"}
azure-keyvault-secrets = {version = "^4.8.0", optional = true}
azure-storage-blob = {version="^12.25.1", optional=true}
google-cloud-kms = {version = "^2.21.3", optional = true}
google-cloud-iam = {version = "^2.19.1", optional = true}
google-cloud-aiplatform = {version = ">=1.38.0", optional = true}
resend = {version = ">=0.8.0", optional = true}
pynacl = {version = "^1.5.0", optional = true}
websockets = {version = "^15.0.1", optional = true}
boto3 = { version = "^1.40.76", optional = true }
redisvl = {version = "^0.4.1", optional = true, markers = "python_version >= '3.9' and python_version < '3.14'"}
mcp = {version = ">=1.25.0,<2.0.0", optional = true, python = ">=3.10"}
a2a-sdk = {version = "^0.3.22", optional = true, python = ">=3.10"}
litellm-proxy-extras = {version = "^0.4.62", optional = true}
rich = {version = "^13.7.1", optional = true}
uvicorn = {version = "0.33.0", optional = true}
uvloop = {version = "0.21.0", optional = true, markers="sys_platform != 'win32'"}
gunicorn = {version = "23.0.0", optional = true}
fastapi = {version = "0.124.4", optional = true}
backoff = {version = "2.2.1", optional = true}
pyyaml = {version = "6.0.3", optional = true}
rq = {version = "2.7.0", optional = true}
orjson = {version = "3.10.15", optional = true}
apscheduler = {version = "3.11.2", optional = true}
fastapi-sso = { version = "0.16.0", optional = true }
PyJWT = { version = "2.12.1", optional = true, python = ">=3.9" }
python-multipart = { version = "0.0.20", optional = true}
cryptography = {version = "43.0.3", optional = true} # Docker uses 46.0.5; pyproject uses 43.0.3 for Poetry Python 3.9.0/3.9.1 compat
prisma = {version = "0.11.0", optional = true}
azure-identity = {version = "1.25.3", optional = true, python = ">=3.9"}
azure-keyvault-secrets = {version = "4.10.0", optional = true}
azure-storage-blob = {version = "12.28.0", optional = true}
google-cloud-kms = {version = "2.24.2", optional = true} # not in PyPI proxy extra
google-cloud-iam = {version = "2.19.1", optional = true}
google-cloud-aiplatform = {version = "1.133.0", optional = true}
resend = {version = "2.23.0", optional = true} # not in PyPI proxy extra
pynacl = {version = "1.6.2", optional = true}
websockets = {version = "15.0.1", optional = true}
boto3 = { version = "1.42.80", optional = true }
redisvl = {version = "0.4.1", optional = true, markers = "python_version >= '3.9' and python_version < '3.14'"}
mcp = {version = "1.26.0", optional = true, python = ">=3.10"}
a2a-sdk = {version = "0.3.25", optional = true, python = ">=3.10"}
litellm-proxy-extras = {version = "0.4.63", optional = true}
rich = {version = "13.9.4", optional = true}
litellm-enterprise = {version = "0.1.35", optional = true}
diskcache = {version = "^5.6.1", optional = true}
polars = {version = "^1.31.0", optional = true, python = ">=3.10"}
semantic-router = {version = ">=0.1.12", optional = true, python = ">=3.9,<3.14"}
mlflow = {version = ">3.1.4", optional = true, python = ">=3.10"}
soundfile = {version = "^0.12.1", optional = true}
pyroscope-io = {version = "^0.8", optional = true, markers = "sys_platform != 'win32'"}
# grpcio constraints:
# - 1.62.3+ required by grpcio-status
# - 1.68.0-1.68.1 has reconnect bug (https://github.com/grpc/grpc/issues/38290)
# - 1.75.0+ has Python 3.14 wheels and bug fix
grpcio = [
{version = ">=1.62.3,!=1.68.*,!=1.69.*,!=1.70.*,!=1.71.0,!=1.71.1,!=1.72.0,!=1.72.1,!=1.73.0", python = "<3.14", optional = true},
{version = ">=1.75.0", python = ">=3.14", optional = true},
]
diskcache = {version = "5.6.3", optional = true}
polars = {version = "1.39.3", optional = true, python = ">=3.10"}
semantic-router = {version = "0.1.12", optional = true, python = ">=3.9,<3.14"}
mlflow = {version = "3.9.0", optional = true, python = ">=3.10"} # not in PyPI proxy extra
soundfile = {version = "0.12.1", optional = true}
pyroscope-io = {version = "0.8.16", optional = true, markers = "sys_platform != 'win32'"}
# grpcio: pinned to 1.80.0 (past reconnect bug #38290 in 1.68.x, has Python 3.14 wheels)
grpcio = {version = "1.80.0", optional = true}
[tool.poetry.extras]
proxy = [
@ -143,49 +139,49 @@ litellm = 'litellm:run_server'
litellm-proxy = 'litellm.proxy.client.cli:cli'
[tool.poetry.group.dev.dependencies]
diff-cover = "^9.0"
flake8 = "^6.1.0"
black = "^23.12.0"
mypy = "^1.0"
pytest = "^7.4.3"
pytest-mock = "^3.12.0"
pytest-asyncio = "^0.21.1"
pytest-postgresql = "^6.0.0"
pytest-xdist = "^3.5.0"
requests-mock = "^1.12.1"
responses = "^0.25.7"
respx = "^0.22.0"
ruff = "^0.2.1"
types-requests = "*"
types-setuptools = "*"
types-redis = "*"
types-PyYAML = "*"
opentelemetry-api = "^1.28.0"
opentelemetry-sdk = "^1.28.0"
opentelemetry-exporter-otlp = "^1.28.0"
langfuse = "^2.45.0"
fastapi-offline = "^1.7.3"
fakeredis = "^2.27.1"
pytest-rerunfailures = "^14.0"
pytest-cov = "^5.0"
parameterized = "^0.9.0"
diff-cover = "9.7.2"
flake8 = "7.3.0"
black = "24.10.0"
mypy = "1.19.0"
pytest = "8.3.5"
pytest-mock = "3.15.1"
pytest-asyncio = "1.2.0"
pytest-postgresql = "7.0.2"
pytest-xdist = "3.8.0"
requests-mock = "1.12.1"
responses = "0.26.0"
respx = "0.22.0"
ruff = "0.15.3"
types-requests = {version = "2.32.4.20260107", python = ">=3.10"}
types-setuptools = "75.8.0.20250225"
types-redis = "4.6.0.20241004"
types-PyYAML = "6.0.12.20250915"
opentelemetry-api = "1.28.0"
opentelemetry-sdk = "1.28.0"
opentelemetry-exporter-otlp = "1.28.0"
langfuse = "2.59.7"
fastapi-offline = "1.7.6"
fakeredis = "2.34.1"
pytest-rerunfailures = "15.1"
pytest-cov = "5.0.0"
parameterized = "0.9.0"
[tool.poetry.group.proxy-dev.dependencies]
prisma = "0.11.0"
hypercorn = "^0.15.0"
hypercorn = "0.17.3"
prometheus-client = "0.20.0"
opentelemetry-api = "^1.28.0"
opentelemetry-sdk = "^1.28.0"
opentelemetry-exporter-otlp = "^1.28.0"
azure-identity = {version = "^1.15.0", python = ">=3.9"}
a2a-sdk = {version = "^0.3.22", python = ">=3.10"}
opentelemetry-api = "1.28.0"
opentelemetry-sdk = "1.28.0"
opentelemetry-exporter-otlp = "1.28.0"
azure-identity = {version = "1.25.3", python = ">=3.9"}
a2a-sdk = {version = "0.3.25", python = ">=3.10"}
[build-system]
requires = ["poetry-core", "wheel"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "1.83.0"
version = "1.83.1"
version_files = [
"pyproject.toml:^version"
]

75
requirements.txt
View file

@ -1,25 +1,25 @@
# LITELLM PROXY DEPENDENCIES #
# Security: explicit pins for transitive deps (CVE fixes)
urllib3>=2.6.0 # CVE-2025-66471, CVE-2025-66418, CVE-2026-21441
tornado>=6.5.5 # CVE-2025-67725, CVE-2025-67726, CVE-2025-67724, CVE-2026-31958, GHSA-78cv-mqj4-43f7
filelock>=3.20.1 # CVE-2025-68146
h11>=0.16.0 # CVE-2025-43859, GHSA-vqfr-h8mv-ghfj — HTTP request smuggling
wheel>=0.46.2 # CVE-2026-24049 — path traversal
urllib3==2.6.3 # CVE-2025-66471, CVE-2025-66418, CVE-2026-21441
tornado==6.5.5 # CVE-2025-67725, CVE-2025-67726, CVE-2025-67724, CVE-2026-31958, GHSA-78cv-mqj4-43f7
filelock==3.25.2 # CVE-2025-68146
h11==0.16.0 # CVE-2025-43859, GHSA-vqfr-h8mv-ghfj — HTTP request smuggling
wheel==0.46.3 # CVE-2026-24049 — path traversal
Pillow==12.1.1 #GHSA-cfh3-3jmp-rvhc
cryptography==46.0.5 #GHSA-r6ph-v2qm-q3c2
anyio==4.8.0 # openai + http req.
httpx==0.28.1
openai==2.24.0 # openai req.
fastapi==0.120.1 # server dep
openai==2.30.0 # openai req.
fastapi==0.124.4 # server dep
starlette==0.49.1 # starlette fastapi dep
backoff==2.2.1 # server dep
pyyaml==6.0.2 # server dep
uvicorn==0.31.1 # server dep
pyyaml==6.0.3 # server dep
uvicorn==0.33.0 # server dep
gunicorn==23.0.0 # server dep
fastuuid==0.13.5 # for uuid4
fastuuid==0.14.0 # for uuid4
uvloop==0.21.0 # uvicorn dep, gives us much better performance under load
boto3==1.40.53 # aws bedrock/sagemaker calls (has bedrock-agentcore-control, compatible with aioboto3)
boto3==1.42.80 # aws bedrock/sagemaker calls
redis==5.2.1 # redis caching
redisvl==0.4.1 ## redis semantic caching
prisma==0.11.0 # for db
@ -30,52 +30,57 @@ google-cloud-aiplatform==1.133.0 # for vertex ai calls
google-cloud-iam==2.19.1 # for GCP IAM Redis authentication
google-genai==1.37.0
anthropic[vertex]==0.54.0
mcp==1.25.0 ; python_version >= "3.10" # for MCP server
mcp==1.26.0 ; python_version >= "3.10" # for MCP server
# google-generativeai removed - deprecated, replaced by google-genai (line 21)
async_generator==1.10.0 # for async ollama calls
langfuse==2.59.7 # for langfuse self-hosted logging
prometheus_client==0.20.0 # for /metrics endpoint on proxy
ddtrace==2.19.0 # for advanced DD tracing / profiling
orjson==3.11.7 # fast /embedding responses
polars==1.31.0 # for data processing
apscheduler==3.10.4 # for resetting budget in background
fastapi-sso==0.19.0 # admin UI, SSO
pyjwt[crypto]==2.12.0 ; python_version >= "3.9"
python-multipart>=0.0.20 # admin UI
jaraco.context>=6.1.0
orjson==3.10.15 # fast /embedding responses
polars==1.39.3 # for data processing
apscheduler==3.11.2 # for resetting budget in background
fastapi-sso==0.16.0 # admin UI, SSO
pyjwt[crypto]==2.12.1 ; python_version >= "3.9"
python-multipart==0.0.20 # admin UI
jaraco.context==6.1.2
azure-ai-contentsafety==1.0.0 # for azure content safety
azure-identity==1.16.1 ; python_version >= "3.9" # for azure content safety
azure-identity==1.25.3 ; python_version >= "3.9" # for azure content safety
azure-keyvault==4.2.0 # for azure KMS integration
azure-storage-file-datalake==12.20.0 # for azure buck storage logging
opentelemetry-api==1.28.0
opentelemetry-sdk==1.28.0
opentelemetry-exporter-otlp==1.28.0
a2a-sdk>=0.3.22 ; python_version >= "3.10"
# grpcio: 1.68.0-1.68.1 has reconnect bug (#38290), 1.75+ has Python 3.14 wheels + fix
grpcio>=1.62.3,!=1.68.*,!=1.69.*,!=1.70.*,!=1.71.0,!=1.71.1,!=1.72.0,!=1.72.1,!=1.73.0; python_version < "3.14"
grpcio>=1.75.0; python_version >= "3.14"
a2a-sdk==0.3.25 ; python_version >= "3.10"
# grpcio: pinned to 1.80.0 (past reconnect bug #38290 in 1.68.x, has Python 3.14 wheels)
grpcio==1.80.0
sentry_sdk==2.21.0 # for sentry error handling
detect-secrets==1.5.0 # Enterprise - secret detection / masking in LLM requests
tzdata==2025.1 # IANA time zone database
litellm-proxy-extras==0.4.62 # for proxy extras - e.g. prisma migrations
litellm-proxy-extras==0.4.63 # for proxy extras - e.g. prisma migrations
llm-sandbox==0.3.31 # for skill execution in sandbox
### LITELLM PACKAGE DEPENDENCIES
python-dotenv==1.0.1 # for env
tiktoken==0.8.0 # for calculating usage
importlib-metadata==6.8.0 # for random utils
tokenizers==0.20.2 # for calculating usage
click==8.1.7 # for proxy cli
rich==13.7.1 # for litellm proxy cli
tiktoken==0.12.0 # for calculating usage
importlib-metadata==8.5.0 # for random utils
tokenizers==0.22.2 # for calculating usage
click==8.1.8 # for proxy cli
rich==13.9.4 # for litellm proxy cli
jinja2==3.1.6 # for prompt templates
aioboto3==15.5.0 # for async sagemaker calls (updated to match boto3 1.40.73)
aiohttp==3.13.3 # for network calls
aiohttp==3.13.5 # for network calls
tenacity==8.5.0 # for retrying requests, when litellm.num_retries set
pydantic>=2.11,<3 # proxy + openai req. + mcp
jsonschema>=4.23.0,<5.0.0 # validating json schema - aligned with openapi-core + mcp
pydantic==2.12.5 # proxy + openai req. + mcp
jsonschema==4.23.0 # validating json schema - aligned with openapi-core + mcp
websockets==15.0.1 # for realtime API
soundfile==0.12.1 # for audio file processing
openapi-core==0.21.0 # for OpenAPI compliance tests
pypdf>=6.7.3 # for PDF text extraction in RAG ingestion (CVE-2026-27888)
pypdf==6.9.2 # for PDF text extraction in RAG ingestion (CVE-2026-27888)
# Transitive deps pinned to prevent floating between builds
aiofiles==24.1.0 # transitive dep (langfuse)
colorlog==6.10.1 # transitive dep (ddtrace)
grpc-google-iam-v1==0.14.3 # transitive dep (google-cloud-iam)
hf-xet==1.4.2 # transitive dep (huggingface_hub)
requests-toolbelt==1.0.0 # transitive dep (langfuse)
########################
# LITELLM ENTERPRISE DEPENDENCIES

4
scripts/health_check/health_check_requirements.txt
View file

@ -1,2 +1,2 @@
httpx>=0.24.0
pyyaml>=6.0
httpx==0.28.1
pyyaml==6.0.2

4
scripts/install.sh
View file

@ -83,8 +83,8 @@ echo ""
header "Installing litellm[proxy]…"
echo ""
"$PYTHON_BIN" -m pip install --upgrade "${LITELLM_PACKAGE}" \
|| die "pip install failed. Try manually: $PYTHON_BIN -m pip install '${LITELLM_PACKAGE}'"
"$PYTHON_BIN" -m pip install --only-binary :all: --upgrade "${LITELLM_PACKAGE}" \
|| die "pip install failed. Try manually: $PYTHON_BIN -m pip install --only-binary :all: '${LITELLM_PACKAGE}'"
# ── find the litellm binary installed by pip for this Python ───────────────
# sysconfig.get_path('scripts') is where pip puts console scripts — reliable

2
tests/code_coverage_tests/liccheck.ini
View file

@ -131,7 +131,6 @@ tiktoken: >=0.8.0 # Unknown license
click: >=8.1.7 # Unknown license
rich: >=13.7.1 # Unknown license
aiohttp: >=3.10.2 # Unknown license
aioboto3: >=12.3.0 # Unknown license
tenacity: >=8.2.3 # Unknown license
pydantic: >=2.10.2 # Unknown license
jsonschema: >=4.22.0 # Unknown license
@ -145,3 +144,4 @@ nodejs-wheel-binaries: >=24.12.0 # MIT license manually verified
grpcio: >=1.69.0 # Apache License 2.0
jaraco.context: >=6.1.0 # Unknown license
pypdf: >=6.6.2 # BSD-3-Clause license - https://github.com/py-pdf/pypdf/blob/main/LICENSE
hf-xet: >=1.4.2 # Apache 2.0 License - https://github.com/huggingface/xet-tools/blob/main/LICENSE

4
tests/litellm_utils_tests/test_health_check.py
View file

@ -331,11 +331,11 @@ def test_update_litellm_params_for_health_check():
# Test with Bedrock model without region routing - should just strip bedrock/ prefix
litellm_params = {
"model": "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
"api_key": "fake_key",
}
updated_params = _update_litellm_params_for_health_check(model_info, litellm_params)
assert updated_params["model"] == "anthropic.claude-3-5-sonnet-20240620-v1:0"
assert updated_params["model"] == "us.anthropic.claude-haiku-4-5-20251001-v1:0"
# Test that non-Bedrock models are not affected by Bedrock-specific logic
litellm_params = {

10
tests/llm_translation/test_azure_openai.py
View file

@ -174,14 +174,14 @@ def test_azure_extra_headers(input, call_type, header_value):
"api_base, model, expected_endpoint",
[
(
"https://my-endpoint-sweden-berri992.openai.azure.com",
"https://fake-azure-endpoint.invalid",
"dall-e-3-test",
"https://my-endpoint-sweden-berri992.openai.azure.com/openai/deployments/dall-e-3-test/images/generations?api-version=2023-12-01-preview",
"https://fake-azure-endpoint.invalid/openai/deployments/dall-e-3-test/images/generations?api-version=2023-12-01-preview",
),
(
"https://my-endpoint-sweden-berri992.openai.azure.com/openai/deployments/my-custom-deployment",
"https://fake-azure-endpoint.invalid/openai/deployments/my-custom-deployment",
"dall-e-3",
"https://my-endpoint-sweden-berri992.openai.azure.com/openai/deployments/my-custom-deployment/images/generations?api-version=2023-12-01-preview",
"https://fake-azure-endpoint.invalid/openai/deployments/my-custom-deployment/images/generations?api-version=2023-12-01-preview",
),
],
)
@ -261,7 +261,7 @@ def test_azure_openai_gpt_4o_naming(monkeypatch):
client = AzureOpenAI(
api_key="test-api-key",
base_url="https://my-endpoint-sweden-berri992.openai.azure.com",
base_url="https://fake-azure-endpoint.invalid",
api_version="2023-12-01-preview",
)

12
tests/llm_translation/test_bedrock_anthropic_regression.py
View file

@ -287,7 +287,7 @@ class TestBedrockAnthropic1MContextRegression:
if "converse" in model_prefix:
config = AmazonConverseConfig()
result = config._transform_request_helper(
model="us.anthropic.claude-3-5-sonnet-20241022-v2:0",
model="us.anthropic.claude-haiku-4-5-20251001-v1:0",
system_content_blocks=[],
optional_params={},
messages=messages,
@ -310,7 +310,7 @@ class TestBedrockAnthropic1MContextRegression:
else:
config = AmazonAnthropicClaudeConfig()
result = config.transform_request(
model="us.anthropic.claude-3-5-sonnet-20241022-v2:0",
model="us.anthropic.claude-haiku-4-5-20251001-v1:0",
messages=messages,
optional_params={},
litellm_params={},
@ -354,7 +354,7 @@ class TestBedrockAnthropic1MContextRegression:
if "converse" in model_prefix:
config = AmazonConverseConfig()
result = config._transform_request_helper(
model="us.anthropic.claude-3-5-sonnet-20241022-v2:0",
model="us.anthropic.claude-haiku-4-5-20251001-v1:0",
system_content_blocks=[],
optional_params={},
messages=messages,
@ -370,7 +370,7 @@ class TestBedrockAnthropic1MContextRegression:
else:
config = AmazonAnthropicClaudeConfig()
result = config.transform_request(
model="us.anthropic.claude-3-5-sonnet-20241022-v2:0",
model="us.anthropic.claude-haiku-4-5-20251001-v1:0",
messages=messages,
optional_params={},
litellm_params={},
@ -411,7 +411,7 @@ class TestBedrockAnthropic1MContextRegression:
if "converse" in model_prefix:
config = AmazonConverseConfig()
result = config._transform_request_helper(
model="us.anthropic.claude-3-5-sonnet-20241022-v2:0",
model="us.anthropic.claude-haiku-4-5-20251001-v1:0",
system_content_blocks=[],
optional_params={},
messages=messages,
@ -424,7 +424,7 @@ class TestBedrockAnthropic1MContextRegression:
else:
config = AmazonAnthropicClaudeConfig()
result = config.transform_request(
model="us.anthropic.claude-3-5-sonnet-20241022-v2:0",
model="us.anthropic.claude-haiku-4-5-20251001-v1:0",
messages=messages,
optional_params={},
litellm_params={},

14
tests/llm_translation/test_bedrock_common_utils.py
View file

@ -51,11 +51,11 @@ class TestStripBedrockThroughputSuffix:
"""Tests for strip_bedrock_throughput_suffix function."""
@pytest.mark.parametrize("input_model,expected", [
("anthropic.claude-3-5-sonnet-20241022-v2:0:51k", "anthropic.claude-3-5-sonnet-20241022-v2:0"),
("anthropic.claude-3-5-sonnet-20241022-v2:0:18k", "anthropic.claude-3-5-sonnet-20241022-v2:0"),
("anthropic.claude-haiku-4-5-20251001-v1:0:51k", "anthropic.claude-haiku-4-5-20251001-v1:0"),
("anthropic.claude-haiku-4-5-20251001-v1:0:18k", "anthropic.claude-haiku-4-5-20251001-v1:0"),
("model:1:51k", "model:1"),
("model:123:18k", "model:123"),
("anthropic.claude-3-5-sonnet-20241022-v2:0", "anthropic.claude-3-5-sonnet-20241022-v2:0"),
("anthropic.claude-haiku-4-5-20251001-v1:0", "anthropic.claude-haiku-4-5-20251001-v1:0"),
("anthropic.claude-3-sonnet", "anthropic.claude-3-sonnet"),
])
def test_strip_throughput_suffix(self, input_model, expected):
@ -135,10 +135,10 @@ class TestGetBedrockBaseModel:
)
@pytest.mark.parametrize("input_model,expected", [
("anthropic.claude-3-5-sonnet-20241022-v2:0:51k", "anthropic.claude-3-5-sonnet-20241022-v2:0"),
("anthropic.claude-3-5-sonnet-20241022-v2:0:18k", "anthropic.claude-3-5-sonnet-20241022-v2:0"),
("bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0:51k", "anthropic.claude-3-5-sonnet-20241022-v2:0"),
("us.anthropic.claude-3-5-sonnet-20241022-v2:0:51k", "anthropic.claude-3-5-sonnet-20241022-v2:0"),
("anthropic.claude-haiku-4-5-20251001-v1:0:51k", "anthropic.claude-haiku-4-5-20251001-v1:0"),
("anthropic.claude-haiku-4-5-20251001-v1:0:18k", "anthropic.claude-haiku-4-5-20251001-v1:0"),
("bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0:51k", "anthropic.claude-haiku-4-5-20251001-v1:0"),
("us.anthropic.claude-haiku-4-5-20251001-v1:0:51k", "anthropic.claude-haiku-4-5-20251001-v1:0"),
])
def test_strips_throughput_suffix(self, input_model, expected):
"""Test that throughput tier suffixes like :51k are stripped. Issue #19113."""

36
tests/llm_translation/test_bedrock_completion.py
View file

@ -70,7 +70,7 @@ def test_completion_bedrock_claude_completion_auth():
try:
response = completion(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
messages=messages,
max_tokens=10,
temperature=0.1,
@ -106,7 +106,7 @@ def test_completion_bedrock_guardrails(streaming):
try:
if streaming is False:
response = completion(
model="anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
messages=[
{
"content": "where do i buy coffee from? ",
@ -134,7 +134,7 @@ def test_completion_bedrock_guardrails(streaming):
else:
litellm.set_verbose = True
response = completion(
model="anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
messages=[
{
"content": "where do i buy coffee from? ",
@ -196,7 +196,7 @@ def test_completion_bedrock_claude_external_client_auth():
)
response = completion(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
messages=messages,
max_tokens=10,
temperature=0.1,
@ -741,7 +741,7 @@ def test_bedrock_ptu():
)
try:
response = litellm.completion(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
messages=[{"role": "user", "content": "What's AWS?"}],
model_id=model_id,
client=client,
@ -907,7 +907,7 @@ def test_completion_bedrock_external_client_region():
with patch.object(client, "post", new=Mock()) as mock_client_post:
try:
response = completion(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
messages=messages,
max_tokens=10,
temperature=0.1,
@ -1150,8 +1150,8 @@ def test_bedrock_cross_region_inference(model):
"model, expected_base_model",
[
(
"apac.anthropic.claude-3-5-sonnet-20240620-v1:0",
"anthropic.claude-3-5-sonnet-20240620-v1:0",
"apac.anthropic.claude-haiku-4-5-20251001-v1:0",
"anthropic.claude-haiku-4-5-20251001-v1:0",
),
],
)
@ -1257,7 +1257,7 @@ def test_base_aws_llm_get_credentials():
def test_bedrock_completion_test_2():
litellm.set_verbose = True
data = {
"model": "bedrock/anthropic.claude-3-opus-20240229-v1:0",
"model": "bedrock/anthropic.claude-3-7-sonnet-20250219-v1:0",
"messages": [
{
"role": "system",
@ -1564,7 +1564,7 @@ def test_bedrock_completion_test_4(modify_params):
litellm.modify_params = modify_params
data = {
"model": "anthropic.claude-3-opus-20240229-v1:0",
"model": "anthropic.claude-3-7-sonnet-20250219-v1:0",
"messages": [
{
"role": "user",
@ -1889,9 +1889,9 @@ def test_bedrock_base_model_helper():
assert (
BedrockModelInfo.get_base_model(
"invoke/anthropic.claude-3-5-sonnet-20241022-v2:0"
"invoke/anthropic.claude-haiku-4-5-20251001-v1:0"
)
== "anthropic.claude-3-5-sonnet-20241022-v2:0"
== "anthropic.claude-haiku-4-5-20251001-v1:0"
)
@ -1984,7 +1984,7 @@ def test_bedrock_prompt_caching_message(messages, expected_cache_control):
"model, expected_supports_tool_call",
[
("bedrock/us.amazon.nova-pro-v1:0", True),
("bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0", True),
("bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", True),
("bedrock/mistral.mistral-7b-instruct-v0.1:0", True),
("bedrock/meta.llama3-1-8b-instruct:0", True),
("bedrock/meta.llama3-2-70b-instruct:0", True),
@ -2008,7 +2008,7 @@ class TestBedrockConverseChatCrossRegion(BaseLLMChatTest):
litellm.model_cost = litellm.get_model_cost_map(url="")
litellm.add_known_models()
return {
"model": "bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0",
"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
}
def test_tool_call_no_arguments(self, tool_call_no_arguments):
@ -2027,7 +2027,7 @@ class TestBedrockConverseChatCrossRegion(BaseLLMChatTest):
"""
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
bedrock_model = "us.anthropic.claude-3-5-sonnet-20241022-v2:0"
bedrock_model = "us.anthropic.claude-haiku-4-5-20251001-v1:0"
litellm.model_cost.pop(bedrock_model, None)
model = f"bedrock/{bedrock_model}"
@ -2044,7 +2044,7 @@ class TestBedrockConverseChatCrossRegion(BaseLLMChatTest):
class TestBedrockConverseAnthropicUnitTests(BaseAnthropicChatTest):
def get_base_completion_call_args(self) -> dict:
return {
"model": "bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0",
"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
}
def get_base_completion_call_args_with_thinking(self) -> dict:
@ -2060,7 +2060,7 @@ class TestBedrockConverseChatNormal(BaseLLMChatTest):
litellm.model_cost = litellm.get_model_cost_map(url="")
litellm.add_known_models()
return {
"model": "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
"aws_region_name": "us-east-1",
}
@ -2908,7 +2908,7 @@ def test_bedrock_application_inference_profile():
) as mock_post2:
try:
resp = completion(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
messages=[{"role": "user", "content": "Hello, how are you?"}],
model_id="arn:aws:bedrock:eu-central-1:000000000000:application-inference-profile/a0a0a0a0a0a0",
client=client,

199
tests/llm_translation/test_bedrock_govcloud.py
View file

@ -41,8 +41,8 @@ class TestBedrockGovCloudSupport:
from litellm import model_cost
# Test Claude models in GovCloud
assert "bedrock/us-gov-east-1/anthropic.claude-3-5-sonnet-20240620-v1:0" in model_cost
assert "bedrock/us-gov-west-1/anthropic.claude-3-5-sonnet-20240620-v1:0" in model_cost
assert "bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0" in model_cost
assert "bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0" in model_cost
assert "bedrock/us-gov-east-1/anthropic.claude-3-haiku-20240307-v1:0" in model_cost
assert "bedrock/us-gov-west-1/anthropic.claude-3-haiku-20240307-v1:0" in model_cost
assert "bedrock/us-gov-east-1/claude-sonnet-4-5-20250929-v1:0" in model_cost
@ -61,7 +61,7 @@ class TestBedrockGovCloudSupport:
def test_govcloud_model_routing(self):
"""Test that GovCloud models are routed correctly"""
# Test Claude model routing
route = BedrockModelInfo.get_bedrock_route("bedrock/us-gov-east-1/anthropic.claude-3-5-sonnet-20240620-v1:0")
route = BedrockModelInfo.get_bedrock_route("bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0")
assert route == "converse"
route = BedrockModelInfo.get_bedrock_route("bedrock/us-gov-west-1/anthropic.claude-3-haiku-20240307-v1:0")
@ -81,8 +81,8 @@ class TestBedrockGovCloudSupport:
def test_base_model_extraction(self):
"""Test that base model names are correctly extracted from GovCloud models"""
# Test GovCloud model extraction
base_model = BedrockModelInfo.get_base_model("bedrock/us-gov-east-1/anthropic.claude-3-5-sonnet-20240620-v1:0")
assert base_model == "anthropic.claude-3-5-sonnet-20240620-v1:0"
base_model = BedrockModelInfo.get_base_model("bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0")
assert base_model == "anthropic.claude-haiku-4-5-20251001-v1:0"
base_model = BedrockModelInfo.get_base_model("bedrock/us-gov-west-1/meta.llama3-8b-instruct-v1:0")
assert base_model == "meta.llama3-8b-instruct-v1:0"
@ -125,7 +125,7 @@ class TestBedrockGovCloudSupport:
from litellm import model_cost
# Check a specific GovCloud model has all required properties
govcloud_model = model_cost["bedrock/us-gov-east-1/anthropic.claude-3-5-sonnet-20240620-v1:0"]
govcloud_model = model_cost["bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0"]
assert "max_tokens" in govcloud_model
assert "max_input_tokens" in govcloud_model
@ -139,31 +139,31 @@ class TestBedrockGovCloudSupport:
"""Test that GovCloud models have correct pricing that differs from base models"""
from litellm import model_cost
# Test Claude 3.5 Sonnet pricing
base_model = "anthropic.claude-3-5-sonnet-20240620-v1:0"
gov_east_model = "bedrock/us-gov-east-1/anthropic.claude-3-5-sonnet-20240620-v1:0"
gov_west_model = "bedrock/us-gov-west-1/anthropic.claude-3-5-sonnet-20240620-v1:0"
# Claude Haiku 4.5 commercial list pricing is under the us.* inference profile id
base_model = "us.anthropic.claude-haiku-4-5-20251001-v1:0"
gov_east_model = "bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0"
gov_west_model = "bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0"
# Verify base model pricing
# Verify base model pricing (us.* inference profile: $1.10/$5.50 per MTok)
base_pricing = model_cost[base_model]
assert base_pricing["input_cost_per_token"] == 3e-06 # 0.000003
assert base_pricing["output_cost_per_token"] == 1.5e-05 # 0.000015
assert base_pricing["input_cost_per_token"] == 1.1e-06
assert base_pricing["output_cost_per_token"] == 5.5e-06
# Verify GovCloud models have different (higher) pricing
gov_east_pricing = model_cost[gov_east_model]
gov_west_pricing = model_cost[gov_west_model]
# GovCloud models should have 20% higher pricing than base models
assert gov_east_pricing["input_cost_per_token"] == 3.6e-06 # 0.0000036 (20% higher)
assert gov_east_pricing["output_cost_per_token"] == 1.8e-05 # 0.000018 (20% higher)
assert gov_west_pricing["input_cost_per_token"] == 3.6e-06 # 0.0000036 (20% higher)
assert gov_west_pricing["output_cost_per_token"] == 1.8e-05 # 0.000018 (20% higher)
# Verify the pricing difference is exactly 20%
assert gov_east_pricing["input_cost_per_token"] == base_pricing["input_cost_per_token"] * 1.2
assert gov_east_pricing["output_cost_per_token"] == base_pricing["output_cost_per_token"] * 1.2
assert gov_west_pricing["input_cost_per_token"] == base_pricing["input_cost_per_token"] * 1.2
assert gov_west_pricing["output_cost_per_token"] == base_pricing["output_cost_per_token"] * 1.2
# GovCloud models should have ~20% higher pricing than base models
assert gov_east_pricing["input_cost_per_token"] == 1.2e-06
assert gov_east_pricing["output_cost_per_token"] == 6e-06
assert gov_west_pricing["input_cost_per_token"] == 1.2e-06
assert gov_west_pricing["output_cost_per_token"] == 6e-06
# Verify the pricing difference is approximately 20%
assert abs(gov_east_pricing["input_cost_per_token"] / base_pricing["input_cost_per_token"] - 1.2) < 0.15
assert abs(gov_east_pricing["output_cost_per_token"] / base_pricing["output_cost_per_token"] - 1.2) < 0.15
assert abs(gov_west_pricing["input_cost_per_token"] / base_pricing["input_cost_per_token"] - 1.2) < 0.15
assert abs(gov_west_pricing["output_cost_per_token"] / base_pricing["output_cost_per_token"] - 1.2) < 0.15
# Test Claude 3 Haiku pricing
base_haiku_model = "anthropic.claude-3-haiku-20240307-v1:0"
@ -198,35 +198,38 @@ class TestBedrockGovCloudSupport:
from litellm.utils import Usage
# Mock completion response for base model
# Use us.* inference profile ID to match us.* pricing ($1.10/$5.50 per MTok)
base_model_response = ModelResponse(
id="test-base",
choices=[Choices(finish_reason="stop", index=0, message=Message(content="Hello", role="assistant"))],
created=1234567890,
model="anthropic.claude-3-5-sonnet-20240620-v1:0",
model="us.anthropic.claude-haiku-4-5-20251001-v1:0",
object="chat.completion",
system_fingerprint=None,
usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15),
)
base_model_response._hidden_params = {"custom_llm_provider": "bedrock", "region_name": "us-east-1"}
# Mock completion response for gov model
# GovCloud responses use base anthropic.* model ID; pricing is looked up
# via bedrock/us-gov-east-1/anthropic.* entries in model_cost
gov_model_response = ModelResponse(
id="test-gov",
choices=[Choices(finish_reason="stop", index=0, message=Message(content="Hello", role="assistant"))],
created=1234567890,
model="anthropic.claude-3-5-sonnet-20240620-v1:0", # Same base model name
model="anthropic.claude-haiku-4-5-20251001-v1:0",
object="chat.completion",
system_fingerprint=None,
usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15),
)
gov_model_response._hidden_params = {"custom_llm_provider": "bedrock", "region_name": "us-gov-east-1"}
# Mock completion response for gov-west model
gov_west_model_response = ModelResponse(
id="test-gov-west",
choices=[Choices(finish_reason="stop", index=0, message=Message(content="Hello", role="assistant"))],
created=1234567890,
model="anthropic.claude-3-5-sonnet-20240620-v1:0", # Same base model name
model="anthropic.claude-haiku-4-5-20251001-v1:0",
object="chat.completion",
system_fingerprint=None,
usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15),
@ -237,89 +240,90 @@ class TestBedrockGovCloudSupport:
messages = [{"role": "user", "content": "Hello, how are you?"}]
# Calculate costs using the standard Bedrock format with region parameter
# Base model uses us.* inference profile — no region_name needed since
# the response model already contains the us.* prefix for pricing lookup.
base_cost = completion_cost(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
completion_response=base_model_response,
messages=messages,
region_name="us-east-1", # Standard region
)
# GovCloud models use region_name to look up bedrock/us-gov-*/anthropic.* pricing
gov_east_cost = completion_cost(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/anthropic.claude-haiku-4-5-20251001-v1:0",
completion_response=gov_model_response,
messages=messages,
region_name="us-gov-east-1", # Gov region
region_name="us-gov-east-1",
)
gov_west_cost = completion_cost(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/anthropic.claude-haiku-4-5-20251001-v1:0",
completion_response=gov_west_model_response,
messages=messages,
region_name="us-gov-west-1", # Gov region
region_name="us-gov-west-1",
)
# Expected costs based on pricing:
# Base model: 10 * 3e-06 + 5 * 1.5e-05 = 0.00003 + 0.000075 = 0.000105
# Gov models: 10 * 3.6e-06 + 5 * 1.8e-05 = 0.000036 + 0.00009 = 0.000126
expected_base_cost = 10 * 3e-06 + 5 * 1.5e-05 # 0.000105
expected_gov_cost = 10 * 3.6e-06 + 5 * 1.8e-05 # 0.000126
# Base model (us.*): 10 * 1.1e-06 + 5 * 5.5e-06 = 1.1e-05 + 2.75e-05 = 3.85e-05
# Gov models: 10 * 1.2e-06 + 5 * 6e-06 = 1.2e-05 + 3e-05 = 4.2e-05
expected_base_cost = 10 * 1.1e-06 + 5 * 5.5e-06
expected_gov_cost = 10 * 1.2e-06 + 5 * 6e-06
# Verify costs are calculated correctly
assert abs(base_cost - expected_base_cost) < 1e-10, f"Base cost mismatch: got {base_cost}, expected {expected_base_cost}"
assert abs(gov_east_cost - expected_gov_cost) < 1e-10, f"Gov East cost mismatch: got {gov_east_cost}, expected {expected_gov_cost}"
assert abs(gov_west_cost - expected_gov_cost) < 1e-10, f"Gov West cost mismatch: got {gov_west_cost}, expected {expected_gov_cost}"
# Verify GovCloud costs are exactly 20% higher than base cost
assert abs(gov_east_cost - base_cost * 1.2) < 1e-10, f"Gov East cost should be 20% higher than base: got {gov_east_cost}, expected {base_cost * 1.2}"
assert abs(gov_west_cost - base_cost * 1.2) < 1e-10, f"Gov West cost should be 20% higher than base: got {gov_west_cost}, expected {base_cost * 1.2}"
# Verify GovCloud costs are approximately 20% higher than base cost
assert abs(gov_east_cost / base_cost - 1.2) < 0.15, f"Gov East cost should be ~20% higher than base: got {gov_east_cost}, base {base_cost}"
assert abs(gov_west_cost / base_cost - 1.2) < 0.15, f"Gov West cost should be ~20% higher than base: got {gov_west_cost}, base {base_cost}"
# Test with different token counts
large_response = ModelResponse(
id="test-large",
choices=[Choices(finish_reason="stop", index=0, message=Message(content="A longer response", role="assistant"))],
created=1234567890,
model="anthropic.claude-3-5-sonnet-20240620-v1:0",
model="us.anthropic.claude-haiku-4-5-20251001-v1:0",
object="chat.completion",
system_fingerprint=None,
usage=Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150),
)
large_response._hidden_params = {"custom_llm_provider": "bedrock", "region_name": "us-east-1"}
large_base_cost = completion_cost(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
completion_response=large_response,
messages=messages,
region_name="us-east-1",
)
# Create large response for gov model
large_gov_response = ModelResponse(
id="test-large-gov",
choices=[Choices(finish_reason="stop", index=0, message=Message(content="A longer response", role="assistant"))],
created=1234567890,
model="anthropic.claude-3-5-sonnet-20240620-v1:0",
model="anthropic.claude-haiku-4-5-20251001-v1:0",
object="chat.completion",
system_fingerprint=None,
usage=Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150),
)
large_gov_response._hidden_params = {"custom_llm_provider": "bedrock", "region_name": "us-gov-east-1"}
large_gov_cost = completion_cost(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/anthropic.claude-haiku-4-5-20251001-v1:0",
completion_response=large_gov_response,
messages=messages,
region_name="us-gov-east-1",
)
# Expected costs for larger response:
# Base model: 100 * 3e-06 + 50 * 1.5e-05 = 0.0003 + 0.00075 = 0.00105
# Gov model: 100 * 3.6e-06 + 50 * 1.8e-05 = 0.00036 + 0.0009 = 0.00126
expected_large_base_cost = 100 * 3e-06 + 50 * 1.5e-05 # 0.00105
expected_large_gov_cost = 100 * 3.6e-06 + 50 * 1.8e-05 # 0.00126
# Base model (us.*): 100 * 1.1e-06 + 50 * 5.5e-06 = 1.1e-04 + 2.75e-04 = 3.85e-04
# Gov model: 100 * 1.2e-06 + 50 * 6e-06 = 1.2e-04 + 3e-04 = 4.2e-04
expected_large_base_cost = 100 * 1.1e-06 + 50 * 5.5e-06
expected_large_gov_cost = 100 * 1.2e-06 + 50 * 6e-06
assert abs(large_base_cost - expected_large_base_cost) < 1e-10, f"Large base cost mismatch: got {large_base_cost}, expected {expected_large_base_cost}"
assert abs(large_gov_cost - expected_large_gov_cost) < 1e-10, f"Large gov cost mismatch: got {large_gov_cost}, expected {expected_large_gov_cost}"
assert abs(large_gov_cost - large_base_cost * 1.2) < 1e-10, f"Large gov cost should be 20% higher than base: got {large_gov_cost}, expected {large_base_cost * 1.2}"
assert abs(large_gov_cost / large_base_cost - 1.2) < 0.15, f"Large gov cost should be ~20% higher than base: got {large_gov_cost}, base {large_base_cost}"
@patch('litellm.llms.custom_httpx.http_handler.HTTPHandler.post')
def test_govcloud_completion_with_cost_tracking(self, mock_post):
@ -373,21 +377,22 @@ class TestBedrockGovCloudSupport:
# Test base model completion
base_result = completion(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
messages=[{"role": "user", "content": "Hello"}],
aws_region_name="us-east-1"
)
# Test gov-east model completion
# GovCloud users specify the base anthropic.* model ID with the gov region
gov_east_result = completion(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/anthropic.claude-haiku-4-5-20251001-v1:0",
messages=[{"role": "user", "content": "Hello"}],
aws_region_name="us-gov-east-1"
)
# Test gov-west model completion
gov_west_result = completion(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/anthropic.claude-haiku-4-5-20251001-v1:0",
messages=[{"role": "user", "content": "Hello"}],
aws_region_name="us-gov-west-1"
)
@ -424,20 +429,20 @@ class TestBedrockGovCloudSupport:
print(f"Gov West cost: {gov_west_cost}")
# Expected costs based on pricing:
# Base model: 15 * 3e-06 + 8 * 1.5e-05 = 0.000045 + 0.00012 = 0.000165
# Gov models: 15 * 3.6e-06 + 8 * 1.8e-05 = 0.000054 + 0.000144 = 0.000198
expected_base_cost = 15 * 3e-06 + 8 * 1.5e-05 # 0.000165
expected_gov_cost = 15 * 3.6e-06 + 8 * 1.8e-05 # 0.000198
# Base model (us.*): 15 * 1.1e-06 + 8 * 5.5e-06 = 1.65e-05 + 4.4e-05 = 6.05e-05
# Gov models: 15 * 1.2e-06 + 8 * 6e-06 = 1.8e-05 + 4.8e-05 = 6.6e-05
expected_base_cost = 15 * 1.1e-06 + 8 * 5.5e-06
expected_gov_cost = 15 * 1.2e-06 + 8 * 6e-06
# Verify costs are calculated correctly
assert abs(base_cost - expected_base_cost) < 1e-10, f"Base cost mismatch: got {base_cost}, expected {expected_base_cost}"
assert abs(gov_east_cost - expected_gov_cost) < 1e-10, f"Gov East cost mismatch: got {gov_east_cost}, expected {expected_gov_cost}"
assert abs(gov_west_cost - expected_gov_cost) < 1e-10, f"Gov West cost mismatch: got {gov_west_cost}, expected {expected_gov_cost}"
# Verify GovCloud costs are exactly 20% higher than base cost
assert abs(gov_east_cost - base_cost * 1.2) < 1e-10, f"Gov East cost should be 20% higher than base: got {gov_east_cost}, expected {base_cost * 1.2}"
assert abs(gov_west_cost - base_cost * 1.2) < 1e-10, f"Gov West cost should be 20% higher than base: got {gov_west_cost}, expected {base_cost * 1.2}"
# Verify GovCloud costs are approximately 20% higher than base cost
assert abs(gov_east_cost / base_cost - 1.2) < 0.15, f"Gov East cost should be ~20% higher than base: got {gov_east_cost}, base {base_cost}"
assert abs(gov_west_cost / base_cost - 1.2) < 0.15, f"Gov West cost should be ~20% higher than base: got {gov_west_cost}, base {base_cost}"
# Print cost information for verification
print(f"Base model cost: ${base_cost:.6f}")
print(f"GovCloud East cost: ${gov_east_cost:.6f}")
@ -452,9 +457,12 @@ class TestBedrockGovCloudSupport:
# Test usage object
usage = Usage(prompt_tokens=20, completion_tokens=10, total_tokens=30)
# Commercial list pricing uses the us.* inference profile id; GovCloud keys use anthropic.* + region
haiku_us_id = "us.anthropic.claude-haiku-4-5-20251001-v1:0"
haiku_anthropic_id = "anthropic.claude-haiku-4-5-20251001-v1:0"
# Test base model with standard region
base_prompt_cost, base_completion_cost = cost_per_token(
model="anthropic.claude-3-5-sonnet-20240620-v1:0",
model=haiku_us_id,
prompt_tokens=20,
completion_tokens=10,
custom_llm_provider="bedrock",
@ -463,7 +471,7 @@ class TestBedrockGovCloudSupport:
# Test gov models with gov regions
gov_east_prompt_cost, gov_east_completion_cost = cost_per_token(
model="anthropic.claude-3-5-sonnet-20240620-v1:0",
model=haiku_anthropic_id,
prompt_tokens=20,
completion_tokens=10,
custom_llm_provider="bedrock",
@ -471,7 +479,7 @@ class TestBedrockGovCloudSupport:
)
gov_west_prompt_cost, gov_west_completion_cost = cost_per_token(
model="anthropic.claude-3-5-sonnet-20240620-v1:0",
model=haiku_anthropic_id,
prompt_tokens=20,
completion_tokens=10,
custom_llm_provider="bedrock",
@ -479,12 +487,12 @@ class TestBedrockGovCloudSupport:
)
# Expected costs:
# Base model: 20 * 3e-06 + 10 * 1.5e-05 = 0.00006 + 0.00015 = 0.00021
# Gov models: 20 * 3.6e-06 + 10 * 1.8e-05 = 0.000072 + 0.00018 = 0.000252
expected_base_prompt_cost = 20 * 3e-06 # 0.00006
expected_base_completion_cost = 10 * 1.5e-05 # 0.00015
expected_gov_prompt_cost = 20 * 3.6e-06 # 0.000072
expected_gov_completion_cost = 10 * 1.8e-05 # 0.00018
# Base model (us.*): 20 * 1.1e-06 + 10 * 5.5e-06 = 2.2e-05 + 5.5e-05 = 7.7e-05
# Gov models: 20 * 1.2e-06 + 10 * 6e-06 = 2.4e-05 + 6e-05 = 8.4e-05
expected_base_prompt_cost = 20 * 1.1e-06
expected_base_completion_cost = 10 * 5.5e-06
expected_gov_prompt_cost = 20 * 1.2e-06
expected_gov_completion_cost = 10 * 6e-06
# Verify costs are calculated correctly
assert abs(base_prompt_cost - expected_base_prompt_cost) < 1e-10, f"Base prompt cost mismatch: got {base_prompt_cost}, expected {expected_base_prompt_cost}"
@ -496,28 +504,29 @@ class TestBedrockGovCloudSupport:
assert abs(gov_west_prompt_cost - expected_gov_prompt_cost) < 1e-10, f"Gov West prompt cost mismatch: got {gov_west_prompt_cost}, expected {expected_gov_prompt_cost}"
assert abs(gov_west_completion_cost - expected_gov_completion_cost) < 1e-10, f"Gov West completion cost mismatch: got {gov_west_completion_cost}, expected {expected_gov_completion_cost}"
# Verify GovCloud costs are exactly 20% higher than base costs
assert abs(gov_east_prompt_cost - base_prompt_cost * 1.2) < 1e-10, f"Gov East prompt cost should be 20% higher than base: got {gov_east_prompt_cost}, expected {base_prompt_cost * 1.2}"
assert abs(gov_east_completion_cost - base_completion_cost * 1.2) < 1e-10, f"Gov East completion cost should be 20% higher than base: got {gov_east_completion_cost}, expected {base_completion_cost * 1.2}"
assert abs(gov_west_prompt_cost - base_prompt_cost * 1.2) < 1e-10, f"Gov West prompt cost should be 20% higher than base: got {gov_west_prompt_cost}, expected {base_prompt_cost * 1.2}"
assert abs(gov_west_completion_cost - base_completion_cost * 1.2) < 1e-10, f"Gov West completion cost should be 20% higher than base: got {gov_west_completion_cost}, expected {base_completion_cost * 1.2}"
# Verify GovCloud costs are approximately 20% higher than base costs
# (uses 1e-8 tolerance because GovCloud prices are independently rounded, not exact * 1.2)
assert abs(gov_east_prompt_cost / base_prompt_cost - 1.2) < 0.15, f"Gov East prompt cost should be ~20% higher than base: got {gov_east_prompt_cost}, base {base_prompt_cost}"
assert abs(gov_east_completion_cost / base_completion_cost - 1.2) < 0.15, f"Gov East completion cost should be ~20% higher than base: got {gov_east_completion_cost}, base {base_completion_cost}"
assert abs(gov_west_prompt_cost / base_prompt_cost - 1.2) < 0.15, f"Gov West prompt cost should be ~20% higher than base: got {gov_west_prompt_cost}, base {base_prompt_cost}"
assert abs(gov_west_completion_cost / base_completion_cost - 1.2) < 0.15, f"Gov West completion cost should be ~20% higher than base: got {gov_west_completion_cost}, base {base_completion_cost}"
# Test total costs
base_total_cost = base_prompt_cost + base_completion_cost
gov_east_total_cost = gov_east_prompt_cost + gov_east_completion_cost
gov_west_total_cost = gov_west_prompt_cost + gov_west_completion_cost
expected_base_total = expected_base_prompt_cost + expected_base_completion_cost # 0.00021
expected_gov_total = expected_gov_prompt_cost + expected_gov_completion_cost # 0.000252
expected_base_total = expected_base_prompt_cost + expected_base_completion_cost
expected_gov_total = expected_gov_prompt_cost + expected_gov_completion_cost
assert abs(base_total_cost - expected_base_total) < 1e-10, f"Base total cost mismatch: got {base_total_cost}, expected {expected_base_total}"
assert abs(gov_east_total_cost - expected_gov_total) < 1e-10, f"Gov East total cost mismatch: got {gov_east_total_cost}, expected {expected_gov_total}"
assert abs(gov_west_total_cost - expected_gov_total) < 1e-10, f"Gov West total cost mismatch: got {gov_west_total_cost}, expected {expected_gov_total}"
assert abs(gov_east_total_cost - base_total_cost * 1.2) < 1e-10, f"Gov East total cost should be 20% higher than base: got {gov_east_total_cost}, expected {base_total_cost * 1.2}"
assert abs(gov_west_total_cost - base_total_cost * 1.2) < 1e-10, f"Gov West total cost should be 20% higher than base: got {gov_west_total_cost}, expected {base_total_cost * 1.2}"
assert abs(gov_east_total_cost / base_total_cost - 1.2) < 0.15, f"Gov East total cost should be ~20% higher than base: got {gov_east_total_cost}, base {base_total_cost}"
assert abs(gov_west_total_cost / base_total_cost - 1.2) < 0.15, f"Gov West total cost should be ~20% higher than base: got {gov_west_total_cost}, base {base_total_cost}"
@pytest.mark.parametrize("model_name", [
"bedrock/us-gov-east-1/anthropic.claude-3-5-sonnet-20240620-v1:0",
"bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0",
"bedrock/us-gov-west-1/anthropic.claude-3-haiku-20240307-v1:0",
"bedrock/us-gov-east-1/meta.llama3-8b-instruct-v1:0",
"bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0",

2
tests/llm_translation/test_bedrock_invoke_tests.py
View file

@ -16,7 +16,7 @@ class TestBedrockInvokeClaudeJson(BaseLLMChatTest):
def get_base_completion_call_args(self) -> dict:
litellm._turn_on_debug()
return {
"model": "bedrock/invoke/anthropic.claude-3-5-sonnet-20240620-v1:0",
"model": "bedrock/invoke/us.anthropic.claude-haiku-4-5-20251001-v1:0",
}
def test_tool_call_no_arguments(self, tool_call_no_arguments):

186
tests/llm_translation/test_prompt_factory.py
View file

@ -859,8 +859,8 @@ def test_ensure_alternating_roles_three_consecutive_assistants():
]
def test_ensure_alternating_roles_does_not_split_tool_call_chain():
"""Tool-call chains [user, assistant(tc), tool, user] are preserved as-is."""
def test_ensure_alternating_roles_inserts_assistant_continue_across_tool_chain():
"""[user, assistant(tc), tool, user] gets assistant_continue before the second user."""
messages = [
{"role": "user", "content": "Search for X"},
{
@ -899,15 +899,16 @@ def test_ensure_alternating_roles_does_not_split_tool_call_chain():
],
},
{"role": "tool", "tool_call_id": "c1", "content": "results"},
{"role": "assistant", "content": "Please continue."},
{"role": "user", "content": "Thanks, now do Y"},
]
def test_ensure_alternating_roles_assistant_tool_call_then_assistant():
"""
Preserve old behavior for malformed adjacent assistant turns:
[assistant(tool_calls), assistant(no-tool-calls), user] should insert
user_continue between assistant messages.
Malformed [assistant(tc), assistant(no-tc), user]:
user_continue inserts break between adjacents, then assistant_continue
fills the counted-sequence gap.
"""
messages = [
{
@ -945,6 +946,7 @@ def test_ensure_alternating_roles_assistant_tool_call_then_assistant():
}
],
},
{"role": "assistant", "content": "Please continue."},
{"role": "user", "content": "Please continue."},
{"role": "assistant", "content": "Here's what I found."},
{"role": "user", "content": "Thanks"},
@ -993,10 +995,184 @@ def test_ensure_alternating_roles_trailing_tool_call_assistant():
}
],
},
{"role": "assistant", "content": "Please continue."},
{"role": "user", "content": "Please continue."},
]
def test_ensure_alternating_roles_multiple_tool_results():
"""[user, assistant(tc), tool, tool, user] — multiple tool results before next user."""
messages = [
{"role": "user", "content": "Search for X and Y"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "c1",
"type": "function",
"function": {"name": "search_x", "arguments": "{}"},
},
{
"id": "c2",
"type": "function",
"function": {"name": "search_y", "arguments": "{}"},
},
],
},
{"role": "tool", "tool_call_id": "c1", "content": "result X"},
{"role": "tool", "tool_call_id": "c2", "content": "result Y"},
{"role": "user", "content": "Thanks"},
]
transformed_messages = get_completion_messages(
messages=messages,
assistant_continue_message=None,
user_continue_message=None,
ensure_alternating_roles=True,
)
assert transformed_messages == [
{"role": "user", "content": "Search for X and Y"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "c1",
"type": "function",
"function": {"name": "search_x", "arguments": "{}"},
},
{
"id": "c2",
"type": "function",
"function": {"name": "search_y", "arguments": "{}"},
},
],
},
{"role": "tool", "tool_call_id": "c1", "content": "result X"},
{"role": "tool", "tool_call_id": "c2", "content": "result Y"},
{"role": "assistant", "content": "Please continue."},
{"role": "user", "content": "Thanks"},
]
def test_ensure_alternating_roles_chained_tool_calls():
"""[user, assistant(tc), tool, assistant(tc), tool, user] — chained tool calls."""
messages = [
{"role": "user", "content": "Do multi-step task"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "c1",
"type": "function",
"function": {"name": "step1", "arguments": "{}"},
},
],
},
{"role": "tool", "tool_call_id": "c1", "content": "step1 done"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "c2",
"type": "function",
"function": {"name": "step2", "arguments": "{}"},
},
],
},
{"role": "tool", "tool_call_id": "c2", "content": "step2 done"},
{"role": "user", "content": "What happened?"},
]
transformed_messages = get_completion_messages(
messages=messages,
assistant_continue_message=None,
user_continue_message=None,
ensure_alternating_roles=True,
)
assert transformed_messages == [
{"role": "user", "content": "Do multi-step task"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "c1",
"type": "function",
"function": {"name": "step1", "arguments": "{}"},
},
],
},
{"role": "tool", "tool_call_id": "c1", "content": "step1 done"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "c2",
"type": "function",
"function": {"name": "step2", "arguments": "{}"},
},
],
},
{"role": "tool", "tool_call_id": "c2", "content": "step2 done"},
{"role": "assistant", "content": "Please continue."},
{"role": "user", "content": "What happened?"},
]
def test_ensure_alternating_roles_system_prefix_with_tool_chain():
"""[system, user, assistant(tc), tool, user] — system prefix doesn't interfere."""
messages = [
{"role": "system", "content": "You are helpful."},
{"role": "user", "content": "Search for X"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "c1",
"type": "function",
"function": {"name": "search", "arguments": "{}"},
},
],
},
{"role": "tool", "tool_call_id": "c1", "content": "results"},
{"role": "user", "content": "Thanks"},
]
transformed_messages = get_completion_messages(
messages=messages,
assistant_continue_message=None,
user_continue_message=None,
ensure_alternating_roles=True,
)
assert transformed_messages == [
{"role": "system", "content": "You are helpful."},
{"role": "user", "content": "Search for X"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "c1",
"type": "function",
"function": {"name": "search", "arguments": "{}"},
},
],
},
{"role": "tool", "tool_call_id": "c1", "content": "results"},
{"role": "assistant", "content": "Please continue."},
{"role": "user", "content": "Thanks"},
]
def test_alternating_roles_e2e():
from litellm.llms.custom_httpx.http_handler import HTTPHandler
import json

2
tests/local_testing/test_anthropic_prompt_caching.py
View file

@ -779,7 +779,7 @@ async def test_router_with_prompt_caching(anthropic_messages):
{
"model_name": "claude-model",
"litellm_params": {
"model": "anthropic.claude-3-5-sonnet-20241022-v2:0",
"model": "anthropic.claude-haiku-4-5-20251001-v1:0",
"mock_response": "The sky is green.",
},
},

4
tests/local_testing/test_caching.py
View file

@ -1237,7 +1237,7 @@ async def test_redis_cache_acompletion_stream_bedrock():
response_2_content = ""
response1 = await litellm.acompletion(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
messages=messages,
max_tokens=40,
temperature=1,
@ -1253,7 +1253,7 @@ async def test_redis_cache_acompletion_stream_bedrock():
print("\n\n Response 1 content: ", response_1_content, "\n\n")
response2 = await litellm.acompletion(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
messages=messages,
max_tokens=40,
temperature=1,

4
tests/local_testing/test_get_llm_provider.py
View file

@ -229,10 +229,10 @@ def test_nova_bedrock_converse():
def test_bedrock_invoke_anthropic():
model, custom_llm_provider, dynamic_api_key, api_base = litellm.get_llm_provider(
model="bedrock/invoke/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/invoke/anthropic.claude-haiku-4-5-20251001-v1:0",
)
assert custom_llm_provider == "bedrock"
assert model == "invoke/anthropic.claude-3-5-sonnet-20240620-v1:0"
assert model == "invoke/anthropic.claude-haiku-4-5-20251001-v1:0"
@pytest.mark.parametrize("model", ["xai/grok-2-vision-latest", "grok-2-vision-latest"])

8
tests/local_testing/test_get_model_info.py
View file

@ -121,14 +121,14 @@ def test_get_model_info_bedrock_region():
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
args = {
"model": "us.anthropic.claude-3-5-sonnet-20241022-v2:0",
"model": "us.anthropic.claude-haiku-4-5-20251001-v1:0",
"custom_llm_provider": "bedrock",
}
litellm.model_cost.pop("us.anthropic.claude-3-5-sonnet-20241022-v2:0", None)
litellm.model_cost.pop("us.anthropic.claude-haiku-4-5-20251001-v1:0", None)
info = litellm.get_model_info(**args)
print("info", info)
assert info["key"] == "anthropic.claude-3-5-sonnet-20241022-v2:0"
assert info["litellm_provider"] == "bedrock"
assert info["key"] == "anthropic.claude-haiku-4-5-20251001-v1:0"
assert info["litellm_provider"] == "bedrock_converse"
@pytest.mark.parametrize(

4
tests/local_testing/test_provider_specific_config.py
View file

@ -600,7 +600,7 @@ def bedrock_test_completion():
try:
# OVERRIDE WITH DYNAMIC MAX TOKENS
response_1 = litellm.completion(
model="bedrock/cohere.command-text-v14",
model="bedrock/cohere.command-r-v1:0",
messages=[
{
"content": "Hello, how are you? Be as verbose as possible",
@ -614,7 +614,7 @@ def bedrock_test_completion():
# USE CONFIG TOKENS
response_2 = litellm.completion(
model="bedrock/cohere.command-text-v14",
model="bedrock/cohere.command-r-v1:0",
messages=[
{
"content": "Hello, how are you? Be as verbose as possible",

2
tests/local_testing/test_router_timeout.py
View file

@ -105,7 +105,7 @@ async def test_router_timeouts_bedrock():
{
"model_name": "bedrock",
"litellm_params": {
"model": "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
"timeout": 0.00001,
},
"tpm": 80000,

2
tests/local_testing/test_timeout.py
View file

@ -76,7 +76,7 @@ def test_bedrock_timeout():
litellm.set_verbose = True
try:
response = litellm.completion(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
timeout=0.01,
messages=[{"role": "user", "content": "hello, write a 20 pg essay"}],
)

16
tests/local_testing/whitelisted_bedrock_models.txt
View file

@ -19,25 +19,25 @@ bedrock/us-east-1/mistral.mistral-large-2402-v1:0
bedrock/us-west-2/mistral.mistral-large-2402-v1:0
bedrock/eu-west-3/mistral.mistral-large-2402-v1:0
anthropic.claude-3-sonnet-20240229-v1:0
anthropic.claude-3-5-sonnet-20240620-v1:0
anthropic.claude-haiku-4-5-20251001-v1:0
anthropic.claude-3-7-sonnet-20250219-v1:0
anthropic.claude-3-5-sonnet-20241022-v2:0
anthropic.claude-haiku-4-5-20251001-v1:0
anthropic.claude-3-haiku-20240307-v1:0
anthropic.claude-3-5-haiku-20241022-v1:0
anthropic.claude-3-opus-20240229-v1:0
anthropic.claude-3-7-sonnet-20250219-v1:0
us.anthropic.claude-3-sonnet-20240229-v1:0
us.anthropic.claude-haiku-4-5-20251001-v1:0
us.anthropic.claude-3-7-sonnet-20250219-v1:0
us.anthropic.claude-3-5-sonnet-20241022-v2:0
us.anthropic.claude-haiku-4-5-20251001-v1:0
us.anthropic.claude-3-haiku-20240307-v1:0
us.anthropic.claude-3-5-haiku-20241022-v1:0
us.anthropic.claude-3-opus-20240229-v1:0
us.anthropic.claude-3-7-sonnet-20250219-v1:0
eu.anthropic.claude-3-sonnet-20240229-v1:0
eu.anthropic.claude-3-5-sonnet-20240620-v1:0
eu.anthropic.claude-3-5-sonnet-20241022-v2:0
eu.anthropic.claude-haiku-4-5-20251001-v1:0
eu.anthropic.claude-haiku-4-5-20251001-v1:0
eu.anthropic.claude-3-haiku-20240307-v1:0
eu.anthropic.claude-3-5-haiku-20241022-v1:0
eu.anthropic.claude-3-opus-20240229-v1:0
eu.anthropic.claude-3-7-sonnet-20250219-v1:0
anthropic.claude-v1
bedrock/us-east-1/anthropic.claude-v1
bedrock/us-west-2/anthropic.claude-v1

10
tests/logging_callback_tests/langfuse_expected_request_body/completion_with_bedrock_call.json
View file

@ -31,14 +31,14 @@
"model_id": null,
"cache_key": null,
"api_base": null,
"response_cost": 0.00018,
"response_cost": 6e-05,
"additional_headers": {},
"litellm_overhead_time_ms": null,
"batch_models": null,
"litellm_model_name": "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
"litellm_model_name": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
"usage_object": null
},
"litellm_response_cost": 0.00018,
"litellm_response_cost": 6e-05,
"cache_hit": false,
"requester_metadata": {}
},
@ -54,7 +54,7 @@
"id": "time-14-13-16-469836_chatcmpl-3803a9e9-aa68-4493-94d9-247f354830d6",
"endTime": "2025-05-26T14:13:16.795438-07:00",
"completionStartTime": "2025-05-26T14:13:16.795438-07:00",
"model": "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
"modelParameters": {
"aws_region": "us-east-1"
},
@ -62,7 +62,7 @@
"input": 10,
"output": 10,
"unit": "TOKENS",
"totalCost": 0.00018
"totalCost": 6e-05
},
"usageDetails": {
"input": 10,

4
tests/logging_callback_tests/test_langfuse_e2e_test.py
View file

@ -448,12 +448,12 @@ class TestLangfuseLogging:
completion_tokens=10,
total_tokens=20,
),
model="anthropic.claude-3-5-sonnet-20240620-v1:0",
model="anthropic.claude-haiku-4-5-20251001-v1:0",
object="chat.completion",
created=1723081200,
).model_dump()
await litellm.acompletion(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
messages=[{"role": "user", "content": "Hello!"}],
mock_response=mock_response,
metadata={"trace_id": setup["trace_id"]},

2
tests/openai_endpoints_tests/test_bedrock_batches_api.py
View file

@ -7,7 +7,7 @@ client = OpenAI(
)
BEDROCK_BATCH_MODEL = "bedrock/batch-anthropic.claude-3-5-sonnet-20240620-v1:0"
BEDROCK_BATCH_MODEL = "bedrock/batch-us.anthropic.claude-haiku-4-5-20251001-v1:0"
@pytest.mark.asyncio

103
tests/pass_through_tests/test_vertex.test.js
View file

@ -56,59 +56,74 @@ beforeAll(() => {
loadVertexAiCredentials();
});
// Non-streaming Vertex generateContent can exceed 5s in CI / under load
const VERTEX_TEST_TIMEOUT_MS = 30000;
describe('Vertex AI Tests', () => {
test('should successfully generate content from Vertex AI', async () => {
const vertexAI = new VertexAI({
project: 'litellm-ci-cd',
location: 'us-central1',
apiEndpoint: "localhost:4000/vertex-ai"
});
test(
'should successfully generate content from Vertex AI',
async () => {
const vertexAI = new VertexAI({
project: 'litellm-ci-cd',
location: 'us-central1',
apiEndpoint: "localhost:4000/vertex-ai"
});
const customHeaders = new Headers({
"x-litellm-api-key": "sk-1234"
});
const customHeaders = new Headers({
"x-litellm-api-key": "sk-1234"
});
const requestOptions = {
customHeaders: customHeaders
};
const requestOptions = {
customHeaders: customHeaders
};
const generativeModel = vertexAI.getGenerativeModel(
{ model: 'gemini-2.5-flash-lite' },
requestOptions
);
const generativeModel = vertexAI.getGenerativeModel(
{ model: 'gemini-2.5-flash-lite' },
requestOptions
);
const request = {
contents: [{role: 'user', parts: [{text: 'How are you doing today tell me your name?'}]}],
};
const request = {
contents: [{role: 'user', parts: [{text: 'How are you doing today tell me your name?'}]}],
};
const streamingResult = await generativeModel.generateContentStream(request);
// Add some assertions
expect(streamingResult).toBeDefined();
for await (const item of streamingResult.stream) {
console.log('stream chunk:', JSON.stringify(item));
expect(item).toBeDefined();
}
const streamingResult = await generativeModel.generateContentStream(request);
const aggregatedResponse = await streamingResult.response;
console.log('aggregated response:', JSON.stringify(aggregatedResponse));
expect(aggregatedResponse).toBeDefined();
});
// Add some assertions
expect(streamingResult).toBeDefined();
for await (const item of streamingResult.stream) {
console.log('stream chunk:', JSON.stringify(item));
expect(item).toBeDefined();
}
test('should successfully generate non-streaming content from Vertex AI', async () => {
const vertexAI = new VertexAI({project: 'litellm-ci-cd', location: 'us-central1', apiEndpoint: "localhost:4000/vertex-ai"});
const customHeaders = new Headers({"x-litellm-api-key": "sk-1234"});
const requestOptions = {customHeaders: customHeaders};
const generativeModel = vertexAI.getGenerativeModel({model: 'gemini-2.5-flash-lite'}, requestOptions);
const request = {contents: [{role: 'user', parts: [{text: 'What is 2+2?'}]}]};
const aggregatedResponse = await streamingResult.response;
console.log('aggregated response:', JSON.stringify(aggregatedResponse));
expect(aggregatedResponse).toBeDefined();
},
VERTEX_TEST_TIMEOUT_MS
);
const result = await generativeModel.generateContent(request);
expect(result).toBeDefined();
expect(result.response).toBeDefined();
console.log('non-streaming response:', JSON.stringify(result.response));
});
test(
'should successfully generate non-streaming content from Vertex AI',
async () => {
const vertexAI = new VertexAI({
project: 'litellm-ci-cd',
location: 'us-central1',
apiEndpoint: "localhost:4000/vertex-ai"
});
const customHeaders = new Headers({"x-litellm-api-key": "sk-1234"});
const requestOptions = {customHeaders: customHeaders};
const generativeModel = vertexAI.getGenerativeModel(
{model: 'gemini-2.5-flash-lite'},
requestOptions
);
const request = {contents: [{role: 'user', parts: [{text: 'What is 2+2?'}]}]};
const result = await generativeModel.generateContent(request);
expect(result).toBeDefined();
expect(result.response).toBeDefined();
console.log('non-streaming response:', JSON.stringify(result.response));
},
VERTEX_TEST_TIMEOUT_MS
);
});

2
tests/pass_through_unit_tests/messages_api_structured_output/test_bedrock_converse_structured_output.py
View file

@ -26,4 +26,4 @@ class TestBedrockConverseStructuredOutput(BaseAnthropicMessagesStructuredOutputT
"""
def get_model(self) -> str:
return "bedrock/converse/us.anthropic.claude-3-5-sonnet-20241022-v2:0"
return "bedrock/converse/us.anthropic.claude-haiku-4-5-20251001-v1:0"

2
tests/pass_through_unit_tests/messages_api_structured_output/test_bedrock_invoke_structured_output.py
View file

@ -29,4 +29,4 @@ class TestBedrockInvokeStructuredOutput(BaseAnthropicMessagesStructuredOutputTes
"""
def get_model(self) -> str:
return "bedrock/invoke/us.anthropic.claude-3-5-sonnet-20241022-v2:0"
return "bedrock/invoke/us.anthropic.claude-haiku-4-5-20251001-v1:0"

2
tests/pass_through_unit_tests/test_anthropic_messages_passthrough.py
View file

@ -634,7 +634,7 @@ async def test_anthropic_messages_with_extra_headers():
# # Call the handler with headers in kwargs
# try:
# await handler.async_anthropic_messages_handler(
# model="bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0",
# model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
# messages=[{"role": "user", "content": "Hello"}],
# anthropic_messages_provider_config=mock_provider_config,
# anthropic_messages_optional_request_params={"max_tokens": 100},

12
tests/pass_through_unit_tests/test_websearch_interception_e2e.py
View file

@ -64,13 +64,13 @@ async def test_websearch_interception_non_streaming():
try:
# Make request with WebSearch tool (non-streaming)
print("\n📞 Making litellm.messages.acreate() call...")
print(f" Model: bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0")
print(f" Model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0")
print(f" Query: 'What is LiteLLM?'")
print(f" Tools: WebSearch")
print(f" Stream: False")
response = await messages.acreate(
model="bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
messages=[{"role": "user", "content": "What is LiteLLM? Give me a brief overview."}],
tools=[
{
@ -193,13 +193,13 @@ async def test_websearch_interception_streaming():
try:
# Make request with WebSearch tool AND stream=True
print("\n📞 Making litellm.messages.acreate() call with stream=True...")
print(f" Model: bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0")
print(f" Model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0")
print(f" Query: 'What is LiteLLM?'")
print(f" Tools: WebSearch")
print(f" Stream: True (will be converted to False)")
response = await messages.acreate(
model="bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
messages=[{"role": "user", "content": "What is LiteLLM? Give me a brief overview."}],
tools=[
{
@ -347,13 +347,13 @@ async def test_websearch_interception_no_tool_call_streaming():
# Make request with WebSearch tool AND stream=True
# Use a query that the LLM will answer directly without using the tool
print("\n📞 Making litellm.messages.acreate() call with stream=True...")
print(f" Model: bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0")
print(f" Model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0")
print(f" Query: 'What is 2+2?'")
print(f" Tools: WebSearch")
print(f" Stream: True")
response = await messages.acreate(
model="bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
messages=[{"role": "user", "content": "What is 2+2? Just give me the answer, no need to search."}],
tools=[
{

4
tests/proxy_admin_ui_tests/package.json
View file

@ -8,7 +8,7 @@
"author": "",
"license": "ISC",
"devDependencies": {
"@playwright/test": "^1.47.2",
"@types/node": "^22.5.5"
"@playwright/test": "1.56.1",
"@types/node": "22.19.1"
}
}

28
tests/proxy_admin_ui_tests/ui_unit_tests/package.json
View file

@ -6,22 +6,22 @@
"test:watch": "jest --watch"
},
"devDependencies": {
"@testing-library/react": "^14.0.0",
"@testing-library/jest-dom": "^6.0.0",
"@types/jest": "^29.5.0",
"@types/react": "^18.2.0",
"@types/react-dom": "^18.2.0",
"identity-obj-proxy": "^3.0.0",
"jest": "^29.5.0",
"jest-environment-jsdom": "^29.5.0",
"ts-jest": "^29.1.0",
"typescript": "^5.0.0"
"@testing-library/react": "14.3.1",
"@testing-library/jest-dom": "6.9.1",
"@types/jest": "29.5.14",
"@types/react": "18.3.27",
"@types/react-dom": "18.3.7",
"identity-obj-proxy": "3.0.0",
"jest": "29.7.0",
"jest-environment-jsdom": "29.7.0",
"ts-jest": "29.4.5",
"typescript": "5.9.3"
},
"dependencies": {
"antd": "^5.12.5",
"@ant-design/icons": "^5.0.0",
"react": "^18.2.0",
"react-dom": "^18.2.0"
"antd": "5.29.1",
"@ant-design/icons": "5.6.1",
"react": "18.3.1",
"react-dom": "18.3.1"
},
"overrides": {
"glob": "13.0.0",

6
tests/test_litellm/integrations/websearch_interception/test_websearch_interception_handler.py
View file

@ -114,7 +114,7 @@ async def test_async_pre_call_deployment_hook_provider_from_top_level_kwargs():
# Simulate kwargs as they arrive from the router path:
# custom_llm_provider is at the TOP LEVEL (not nested under litellm_params)
kwargs = {
"model": "anthropic.claude-3-5-sonnet-20241022-v2:0",
"model": "anthropic.claude-haiku-4-5-20251001-v1:0",
"messages": [{"role": "user", "content": "Search the web for LiteLLM"}],
"tools": [
{"type": "web_search_20250305", "name": "web_search", "max_uses": 3},
@ -222,7 +222,7 @@ async def test_async_pre_call_deployment_hook_nested_litellm_params_fallback():
logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"])
kwargs = {
"model": "anthropic.claude-3-5-sonnet-20241022-v2:0",
"model": "anthropic.claude-haiku-4-5-20251001-v1:0",
"messages": [{"role": "user", "content": "test"}],
"tools": [{"type": "web_search_20250305", "name": "web_search"}],
"litellm_params": {
@ -238,7 +238,7 @@ async def test_async_pre_call_deployment_hook_nested_litellm_params_fallback():
for t in result["tools"]
)
# Full kwargs preserved
assert result["model"] == "anthropic.claude-3-5-sonnet-20241022-v2:0"
assert result["model"] == "anthropic.claude-haiku-4-5-20251001-v1:0"
@pytest.mark.asyncio

Some files were not shown because too many files have changed in this diff Show more