scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-07 02:59:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

Merge remote-tracking branch 'origin/main' into litellm_ui_e2e_psql_pr

Browse source
This commit is contained in:
Yuneng Jiang 2026-04-03 23:49:39 -07:00
commit f53db7f113
No known key found for this signature in database
156 changed files with 6400 additions and 1405 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
.circleci/config.yml
View file

@ -3233,31 +3233,9 @@ jobs:
name: Install Playwright Browsers
command: |
npx playwright install
- run:
name: Install Neon CLI
command: |
npm i -g neonctl
- run:
name: Create Neon branch
command: |
export EXPIRES_AT=$(date -u -d "+3 hours" +"%Y-%m-%dT%H:%M:%SZ")
echo "Expires at: $EXPIRES_AT"
neon branches create \
--project-id $NEON_PROJECT_ID \
--name preview/commit-${CIRCLE_SHA1:0:7}-<< parameters.browser >> \
--expires-at $EXPIRES_AT \
--parent br-fancy-paper-ad1olsb3 \
--api-key $NEON_API_KEY || true
- run:
name: Run Docker container
command: |
E2E_UI_TEST_DATABASE_URL=$(neon connection-string \
--project-id $NEON_PROJECT_ID \
--api-key $NEON_API_KEY \
--branch preview/commit-${CIRCLE_SHA1:0:7}-<< parameters.browser >> \
--database-name yuneng-trial-db \
--role neondb_owner)
echo $E2E_UI_TEST_DATABASE_URL
docker run -d \
-p 4000:4000 \
-e DATABASE_URL=$E2E_UI_TEST_DATABASE_URL \
@ -3531,7 +3509,7 @@ workflows:
only:
- main
- /litellm_.*/
# - e2e_ui_testing: # migrate to dynamic db - currently requires neon cli
# - e2e_ui_testing:
# name: e2e_ui_testing_chromium
# browser: chromium
# context: e2e_ui_tests

2
.github/workflows/scorecard.yml vendored
View file

@ -42,6 +42,6 @@ jobs:
retention-days: 5
- name: Upload to code scanning
uses: github/codeql-action/upload-sarif@c10b806170c8ee63ea24152429041b5624f0baf5 # v4.35.1
uses: github/codeql-action/upload-sarif@c10b8064de6f491fea524254123dbe5e09572f13 # v4.35.1
with:
sarif_file: results.sarif

40
.pre-commit-config.yaml
View file

@ -1,40 +0,0 @@
repos:
- repo: local
hooks:
- id: pyright
name: pyright
entry: pyright
language: system
types: [python]
files: ^(litellm/|litellm_proxy_extras/|enterprise/)
- id: isort
name: isort
entry: isort
language: system
types: [python]
files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py
exclude: ^litellm/__init__.py$
- id: black
name: black
entry: poetry run black
language: system
types: [python]
files: (litellm/|litellm_proxy_extras/).*\.py
- repo: https://github.com/pycqa/flake8
rev: 7.0.0 # The version of flake8 to use
hooks:
- id: flake8
exclude: ^litellm/tests/|^litellm/proxy/tests/|^litellm/tests/test_litellm/|^tests/test_litellm/|^tests/enterprise/
additional_dependencies: [flake8-print]
files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py
- repo: https://github.com/python-poetry/poetry
rev: 1.8.0
hooks:
- id: poetry-check
files: ^(pyproject.toml|litellm-proxy-extras/pyproject.toml)$
- repo: local
hooks:
- id: check-files-match
name: Check if files match
entry: python3 ci_cd/check_files_match.py
language: system

19
ci_cd/publish-proxy-extras.sh
View file

@ -1,19 +0,0 @@
#!/bin/bash
# Exit on error
set -e
echo "🚀 Building and publishing litellm-proxy-extras"
# Navigate to litellm-proxy-extras directory
cd "$(dirname "$0")/../litellm-proxy-extras"
# Build the package
echo "📦 Building package..."
poetry build
# Publish to PyPI
echo "🌎 Publishing to PyPI..."
poetry publish
echo "✅ Done! Package published successfully"

1
ci_cd/security_scans.sh
View file

@ -160,7 +160,6 @@ run_grype_scans() {
"CVE-2026-0775" # npm cli incorrect permission assignment - no fix available yet, npm is only used at build/prisma-generate time
"GHSA-3ppc-4f35-3m26" # minimatch ReDoS via repeated wildcards - from nodejs_wheel bundled npm, not used in application runtime code
"GHSA-83g3-92jg-28cx" # tar arbitrary file read/write via hardlink - from nodejs_wheel bundled npm, not used in application runtime code
"CVE-2026-25639" # axios DoS via __proto__ in mergeConfig - transitive dev dep via @neondatabase/api-client, not imported in application code
"CVE-2026-2297" # Python 3.13 SourcelessFileLoader audit hook bypass - no fix available in base image
"GHSA-qffp-2rhf-9h96" # tar hardlink path traversal - from nodejs_wheel bundled npm, not used in application runtime code
"CVE-2026-2673" # OpenSSL 3.6.1 TLS 1.3 key exchange group negotiation issue - no fix available yet

2
cookbook/ai_coding_tool_guides/claude_code_quickstart/guide.md
View file

@ -230,7 +230,7 @@ model_list:
# AWS Bedrock
- model_name: claude-bedrock
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1

4
cookbook/misc/test_responses_api.py
View file

@ -20,7 +20,7 @@ base64_image = encode_image(image_path)
response = client.responses.create(
model="bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
input=[
{
"role": "user",
@ -43,7 +43,7 @@ print("sleeping for 20 seconds...")
time.sleep(20)
print("making follow up request for existing id")
response2 = client.responses.create(
model="bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
previous_response_id=response.id,
input="ok, and what objects are in the image?"
)

5
docker/Dockerfile.non_root
View file

@ -46,7 +46,10 @@ ENV LITELLM_NON_ROOT=true
# package-lock.json with pinned versions + integrity hashes.
RUN mkdir -p /var/lib/litellm/ui && \
mv /app/.npmrc /app/.npmrc.bak && \
npm install -g npm@11.12.1 && npm cache clean --force && \
npm install -g npm@11.12.1 && \
npm install -g node-gyp@12.2.0 && \
ln -sf /usr/local/lib/node_modules/node-gyp /usr/lib/node_modules/npm/node_modules/node-gyp && \
npm cache clean --force && \
cd /app/ui/litellm-dashboard && \
if [ -f "/app/enterprise/enterprise_ui/enterprise_colors.json" ]; then \
cp /app/enterprise/enterprise_ui/enterprise_colors.json ./ui_colors.json; \

39
docs/my-website/blog/april_townhall_announcement/index.md Normal file
View file

@ -0,0 +1,39 @@
---
slug: april-townhall-announcement
title: "April Townhall: Security + Product Roadmap"
date: 2026-04-02T07:30:00
authors:
- krrish
description: "Join the LiteLLM April townhall on Friday, 10 April at 7:30 AM to learn about LiteLLM's security and product roadmap."
tags: [announcement, townhall]
hide_table_of_contents: true
---
import Image from '@theme/IdealImage';
We are hosting our April townhall on **Friday, 10 April at 7:30 AM PST**.
<Image
img={require('../../img/april_townhall_banner.png')}
style={{width: '900px', height: 'auto', display: 'block'}}
/>
{/* truncate */}
## Agenda
- Product updates and roadmap progress
- Reliability and security updates
- Open Q&A with the team
## How to contribute
Add your thoughts to this [ticket](https://github.com/BerriAI/litellm/issues/24825) to help us shape the agenda.
## Register
Register here: [LiteLLM April Townhall Form](https://forms.gle/hvyVXwbFjzJQE7dEA)
We will hold the townhall from **7:30 AM to 8:30 AM PST on Zoom**.
For security, attendance is restricted to corporate emails. If you register with a non-corporate email, we will share the townhall slides and accompanying blog post after the event.

66
docs/my-website/blog/security_hardening_april_2026/index.md Normal file
View file

@ -0,0 +1,66 @@
---
slug: security-hardening-april-2026
title: "Security Update: Vulnerability Disclosures and Ongoing Hardening"
date: 2026-04-03T12:00:00
authors:
- krrish
- ishaan-alt
description: "Disclosure of security vulnerabilities fixed in LiteLLM v1.83.0, and the launch of our bug bounty program."
tags: [security]
hide_table_of_contents: false
---
After the [supply chain incident](https://docs.litellm.ai/blog/security-update-march-2026) in March, we brought in [Veria Labs](https://verialabs.com/) to audit the LiteLLM proxy and fixed a number of vulnerability reports from independent researchers. All issues below are fixed in v1.83.0. If you are affected, particularly if you have JWT auth enabled, we recommend upgrading.
We've also launched a [bug bounty program](#bug-bounty-program) and Veria Labs is continuing to audit the proxy. More fixes will ship in upcoming versions.
The two high-severity issues ([CVE-2026-35029](https://github.com/BerriAI/litellm/security/advisories/GHSA-53mr-6c8q-9789) and [GHSA-69x8-hrgq-fjj8](https://github.com/BerriAI/litellm/security/advisories/GHSA-69x8-hrgq-fjj8)) **both require the attacker to already have a valid API key for the proxy**. These are not exploitable by unauthenticated users.
The critical-severity issue ([CVE-2026-35030](https://github.com/BerriAI/litellm/security/advisories/GHSA-jjhc-v7c2-5hh6)) is an authentication bypass, but only affects deployments with `enable_jwt_auth` explicitly enabled, which is off by default. **The default LiteLLM configuration is not affected, and no LiteLLM Cloud customers had this feature enabled.**
{/* truncate */}
## Vulnerabilities
### CVE-2026-35030: Authentication bypass via OIDC cache collision (Critical)
Found by Veria Labs.
When `enable_jwt_auth` is enabled, LiteLLM cached OIDC userinfo using `token[:20]` as the cache key. JWTs from the same signing algorithm share the same header prefix, so an attacker could forge a token that hits another user's cache entry and inherit their session. We fixed this by keying the cache on `sha256(token)` instead.
**Most deployments are not affected.** This requires `enable_jwt_auth: true`, which is off by default. If you can't upgrade, disable JWT auth as a workaround.
Full advisory: [GHSA-jjhc-v7c2-5hh6](https://github.com/BerriAI/litellm/security/advisories/GHSA-jjhc-v7c2-5hh6)
### CVE-2026-35029: Privilege escalation via `/config/update` (High)
Found by Lakera.
`/config/update` didn't check the caller's role. Any authenticated user could modify the proxy's runtime configuration, which could lead to arbitrary file read, admin account takeover, or remote code execution. We now require the `proxy_admin` role on this endpoint.
Full advisory: [GHSA-53mr-6c8q-9789](https://github.com/BerriAI/litellm/security/advisories/GHSA-53mr-6c8q-9789)
### Password hash exposure and pass-the-hash login (High)
Weak hashing originally reported by GitHub user [hamzayevmaqsud](https://github.com/hamzayevmaqsud) ([#15484](https://github.com/BerriAI/litellm/issues/15484)). The full chain was identified by Luca Vandenweghe and Maarten De Rammelaere of [iO Digital](https://www.iodigital.com/).
Passwords were stored as unsalted SHA-256 hashes, and in some cases plaintext. Several API endpoints returned the hash to any authenticated user, and `/v2/login` accepted the raw hash as a credential without re-hashing it, so a stolen hash was as good as the password itself. We've moved to scrypt with random salts and stripped hashes from all API responses.
Full advisory: [GHSA-69x8-hrgq-fjj8](https://github.com/BerriAI/litellm/security/advisories/GHSA-69x8-hrgq-fjj8)
## Bug bounty program
After the supply chain incident and these disclosures it was clear we needed more external eyes on the project. We've set up a bug bounty program so researchers have a way to report issues.
Bounties are currently paid for P0 (supply chain) and P1 (unauthenticated proxy access) vulnerabilities:
| Severity | Bounty | Example |
|----------|--------|---------|
| Critical | $1,500 – $3,000 | Supply chain compromise |
| High | $500 – $1,500 | Unauthenticated access to protected data |
We plan on expanding the program further in the coming months. More info about the bug bounty program is available [here](https://github.com/BerriAI/litellm/security).
## What's next
Veria Labs is continuing to work with us on a broader audit of the proxy. Security advisories sent through Github will be responded to within five business days. We'll publish advisories as issues are confirmed and fixed.

2
docs/my-website/docs/anthropic_count_tokens.md
View file

@ -96,7 +96,7 @@ model_list:
- model_name: claude-bedrock
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
aws_region_name: us-west-2
```

6
docs/my-website/docs/completion/computer_use.md
View file

@ -80,7 +80,7 @@ model_list:
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: claude-bedrock # Bedrock Anthropic model
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
model: bedrock/anthropic.claude-haiku-4-5-20251001:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-west-2
@ -153,7 +153,7 @@ import litellm
assert litellm.supports_computer_use(model="anthropic/claude-3-5-sonnet-latest") == True
assert litellm.supports_computer_use(model="anthropic/claude-3-7-sonnet-20250219") == True
assert litellm.supports_computer_use(model="bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0") == True
assert litellm.supports_computer_use(model="bedrock/anthropic.claude-haiku-4-5-20251001:0") == True
assert litellm.supports_computer_use(model="vertex_ai/claude-3-5-sonnet") == True
assert litellm.supports_computer_use(model="openai/gpt-4") == False
```
@ -171,7 +171,7 @@ model_list:
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: claude-bedrock # Bedrock Anthropic model
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
model: bedrock/anthropic.claude-haiku-4-5-20251001:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-west-2

18
docs/my-website/docs/completion/document_understanding.md
View file

@ -32,7 +32,7 @@ os.environ["AWS_REGION_NAME"] = ""
file_url = "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf"
# model
model = "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0"
model = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
file_content = [
{"type": "text", "text": "What's this file about?"},
@ -63,7 +63,7 @@ assert response is not None
model_list:
- model_name: bedrock-model
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: os.environ/AWS_REGION_NAME
@ -122,7 +122,7 @@ encoded_file = base64.b64encode(file_data).decode("utf-8")
base64_url = f"data:application/pdf;base64,{encoded_file}"
# model
model = "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0"
model = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
file_content = [
{"type": "text", "text": "What's this file about?"},
@ -153,7 +153,7 @@ assert response is not None
model_list:
- model_name: bedrock-model
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: os.environ/AWS_REGION_NAME
@ -210,7 +210,7 @@ os.environ["AWS_REGION_NAME"] = ""
file_url = "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf"
# model
model = "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0"
model = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
file_content = [
{"type": "text", "text": "What's this file about?"},
@ -242,7 +242,7 @@ assert response is not None
model_list:
- model_name: bedrock-model
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: os.environ/AWS_REGION_NAME
@ -350,10 +350,10 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
<Tabs>
<TabItem label="SDK" value="sdk">
Use `litellm.supports_pdf_input(model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0")` -> returns `True` if model can accept pdf input
Use `litellm.supports_pdf_input(model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0")` -> returns `True` if model can accept pdf input
```python
assert litellm.supports_pdf_input(model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0") == True
assert litellm.supports_pdf_input(model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0") == True
```
</TabItem>
@ -365,7 +365,7 @@ assert litellm.supports_pdf_input(model="bedrock/anthropic.claude-3-5-sonnet-202
model_list:
- model_name: bedrock-model # model group name
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: os.environ/AWS_REGION_NAME

2
docs/my-website/docs/completion/provider_specific_params.md
View file

@ -450,7 +450,7 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
import litellm
response = litellm.completion(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
messages=[{"role": "user", "content": "Hello!"}],
requestMetadata={"cost_center": "engineering"}
)

2
docs/my-website/docs/index.md
View file

@ -103,7 +103,7 @@ os.environ["AWS_SECRET_ACCESS_KEY"] = "your-secret"
os.environ["AWS_REGION_NAME"] = "us-east-1"
response = completion(
model="bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0",
model="bedrock/anthropic.claude-haiku-4-5-20251001:0",
messages=[{"role": "user", "content": "Hello, how are you?"}]
)
print(response.choices[0].message.content)

2
docs/my-website/docs/oidc.md
View file

@ -268,7 +268,7 @@ Please contact us for paid enterprise support if you need help setting up Azure
model list:
- model_name: aws/claude-3-5-sonnet
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
aws_region_name: "eu-central-1"
aws_role_name: "arn:aws:iam::12345678:role/bedrock-role"
aws_web_identity_token: "oidc/azure/api://123-456-789-9d04"

4
docs/my-website/docs/providers/azure/azure_responses.md
View file

@ -246,7 +246,7 @@ You can also call the Azure Responses API via the `/chat/completions` endpoint.
from litellm import completion
import os
os.environ["AZURE_API_BASE"] = "https://my-endpoint-sweden-berri992.openai.azure.com/"
os.environ["AZURE_API_BASE"] = "https://my-azure-endpoint.openai.azure.com/"
os.environ["AZURE_API_VERSION"] = "2023-03-15-preview"
os.environ["AZURE_API_KEY"] = "my-api-key"
@ -268,7 +268,7 @@ model_list:
litellm_params:
model: azure/responses/my-custom-o1-pro
api_key: os.environ/AZURE_API_KEY
api_base: https://my-endpoint-sweden-berri992.openai.azure.com/
api_base: https://my-azure-endpoint.openai.azure.com/
api_version: 2023-03-15-preview
```

20
docs/my-website/docs/providers/bedrock.md
View file

@ -95,7 +95,7 @@ Here's how to call Bedrock with the LiteLLM Proxy Server
model_list:
- model_name: bedrock-claude-3-5-sonnet
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: os.environ/AWS_REGION_NAME
@ -337,7 +337,7 @@ os.environ["AWS_SECRET_ACCESS_KEY"] = ""
os.environ["AWS_REGION_NAME"] = ""
response = completion(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
messages=[{"role": "user", "content": "Hello, how are you?"}],
requestMetadata={
"cost_center": "engineering",
@ -354,7 +354,7 @@ response = completion(
model_list:
- model_name: bedrock-claude-v1
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
requestMetadata:
cost_center: "engineering"
```
@ -1543,7 +1543,7 @@ file_data = response.content
encoded_file = base64.b64encode(file_data).decode("utf-8")
# model
model = "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0"
model = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
image_content = [
{"type": "text", "text": "What's this file about?"},
@ -1574,7 +1574,7 @@ assert response is not None
model_list:
- model_name: bedrock-model
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: os.environ/AWS_REGION_NAME
@ -1631,7 +1631,7 @@ encoded_file = base64.b64encode(file_data).decode("utf-8")
base64_url = f"data:application/pdf;base64,{encoded_file}"
# model
model = "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0"
model = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
image_content = [
{"type": "text", "text": "What's this file about?"},
@ -1660,7 +1660,7 @@ assert response is not None
model_list:
- model_name: bedrock-model
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: os.environ/AWS_REGION_NAME
@ -1941,7 +1941,7 @@ Here's an example of using a bedrock model with LiteLLM. For a complete list, re
| GPT-OSS 120B | `completion(model='bedrock/converse/openai.gpt-oss-120b-1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` |
| Deepseek R1 | `completion(model='bedrock/us.deepseek.r1-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` |
| Anthropic Claude Sonnet 4.5 | `completion(model='bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` |
| Anthropic Claude-V3.5 Sonnet | `completion(model='bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` |
| Anthropic Claude-V3.5 Sonnet | `completion(model='bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` |
| Anthropic Claude-V3 sonnet | `completion(model='bedrock/anthropic.claude-3-sonnet-20240229-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` |
| Anthropic Claude-V3 Haiku | `completion(model='bedrock/anthropic.claude-3-haiku-20240307-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` |
| Anthropic Claude-V3 Opus | `completion(model='bedrock/anthropic.claude-3-opus-20240229-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` |
@ -2051,7 +2051,7 @@ os.environ["AWS_SECRET_ACCESS_KEY"] = ""
os.environ["AWS_REGION_NAME"] = ""
response = completion(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
messages=[{"role": "user", "content": "Hello, how are you?"}],
model_id="arn:aws:bedrock:eu-central-1:000000000000:application-inference-profile/a0a0a0a0a0a0",
)
@ -2068,7 +2068,7 @@ print(response)
model_list:
- model_name: anthropic-claude-3-5-sonnet
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
# You have to set the ARN application inference profile in the model_id parameter
model_id: arn:aws:bedrock:eu-central-1:000000000000:application-inference-profile/a0a0a0a0a0a0
```

24
docs/my-website/docs/providers/bedrock_image_gen.md
View file

@ -111,6 +111,29 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/images/generations' \
</TabItem>
</Tabs>
## Amazon Nova Canvas - Image Edit
Use OpenAI-compatible `image_edit()` with Bedrock Nova Canvas (`amazon.nova-canvas-v1:0`). Requests use the same `InvokeModel` API as generation; LiteLLM maps inputs to [Nova Canvas task types](https://docs.aws.amazon.com/nova/latest/userguide/image-gen-access.html):
| Scenario | `taskType` sent to Bedrock |
|----------|----------------------------|
| Image + prompt (no mask) | `IMAGE_VARIATION` |
| Image + prompt + mask | `INPAINTING` (`inPaintingParams.image`, `maskImage` or `maskPrompt`) |
| `taskType: OUTPAINTING` + `mask` or `maskPrompt` | `OUTPAINTING` (Bedrock requires one; LiteLLM raises a clear error if both are missing) |
| `taskType: BACKGROUND_REMOVAL` | `BACKGROUND_REMOVAL` |
```python
from litellm import image_edit
response = image_edit(
image=open("photo.png", "rb"),
prompt="Add soft sunset lighting",
model="bedrock/amazon.nova-canvas-v1:0",
)
```
For **`BACKGROUND_REMOVAL`**, the AWS request must not include `imageGenerationConfig`; LiteLLM omits it for that task even if you pass `size`, `n`, `seed`, etc. Additional Nova Canvas inference IDs for image edit should set **`supports_nova_canvas_image_edit`: true** in `model_prices_and_context_window.json` (see `amazon.nova-canvas-v1:0`).
## Using Inference Profiles with Image Generation
For AWS Bedrock Application Inference Profiles with image generation, use the `model_id` parameter to specify the inference profile ARN:
@ -147,4 +170,3 @@ model_list:
## Authentication
All standard Bedrock authentication methods are supported for image generation. See [Bedrock Authentication](./bedrock#boto3---authentication) for details.

21
docs/my-website/docs/proxy/oauth2.md
View file

@ -61,3 +61,24 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
Start the LiteLLM Proxy with [`--detailed_debug` mode and you should see more verbose logs](cli.md#detailed_debug)
## Using OAuth2 + JWT Together
If both `enable_oauth2_auth` and `enable_jwt_auth` are enabled, LiteLLM can split auth paths:
- JWT validation for user tokens
- OAuth2 introspection for machine tokens
For JWT-shaped machine tokens, configure `litellm_jwtauth.routing_overrides`:
```yaml title="config.yaml"
general_settings:
enable_jwt_auth: true
enable_oauth2_auth: true
litellm_jwtauth:
routing_overrides:
- iss: "machine-issuer.example.com"
client_id: "MID_LITELLM"
path: "oauth2"
```
For full `routing_overrides` behavior and list-based selectors, see [`/proxy/token_auth`](./token_auth.md#route-jwt-shaped-machine-tokens-to-oauth2).

2
docs/my-website/docs/proxy/team_model_add.md
View file

@ -26,7 +26,7 @@ curl -L -X POST 'http://0.0.0.0:4000/model/new' \
"model": "openai/gpt-4o",
"custom_llm_provider": "openai",
"api_key": "******ccb07",
"api_base": "https://my-endpoint-sweden-berri992.openai.azure.com",
"api_base": "https://my-azure-endpoint.openai.azure.com",
"api_version": "2023-12-01-preview"
},
"model_info": {

41
docs/my-website/docs/proxy/token_auth.md
View file

@ -790,6 +790,47 @@ litellm_jwtauth:
user_roles_jwt_field: "resource_access.your-client.roles"
```
## Route JWT-Shaped Machine Tokens to OAuth2
Use this when both are enabled:
- `enable_jwt_auth: true` for standard JWT validation
- `enable_oauth2_auth: true` for OAuth2 introspection
If some machine tokens are also JWT-shaped, configure `routing_overrides` to route matching tokens to OAuth2.
```yaml title="config.yaml"
general_settings:
enable_jwt_auth: true
enable_oauth2_auth: true
litellm_jwtauth:
user_id_jwt_field: "sub"
routing_overrides:
- iss: "machine-issuer.example.com"
client_id: "MID_LITELLM"
path: "oauth2"
```
### Matching behavior
- A rule matches when all configured selectors match token claims
- Supported selectors: `iss` (required), `client_id` (optional), `aud` (optional)
- Selector values support both string and list forms
- If no rule matches, LiteLLM continues with standard JWT validation
### List-based override example
```yaml title="config.yaml"
general_settings:
enable_jwt_auth: true
enable_oauth2_auth: true
litellm_jwtauth:
routing_overrides:
- iss: ["machine-issuer.example.com", "backup-issuer.example.com"]
client_id: ["MID_LITELLM", "MID_BACKUP"]
aud: ["api://litellm", "api://fallback"]
path: "oauth2"
```
## [BETA] Control Access with OIDC Roles
Allow JWT tokens with supported roles to access the proxy.

2
docs/my-website/docs/realtime.md
View file

@ -82,7 +82,7 @@ Run this script using node - `node test.js`
const WebSocket = require("ws");
const url = "ws://0.0.0.0:4000/v1/realtime?model=openai-gpt-4o-realtime-audio";
// const url = "wss://my-endpoint-sweden-berri992.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview";
// const url = "wss://my-azure-endpoint.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview";
const ws = new WebSocket(url, {
headers: {
"api-key": `sk-1234`,

2
docs/my-website/docs/tutorials/claude_responses_api.md
View file

@ -214,7 +214,7 @@ model_list:
# AWS Bedrock
- model_name: claude-bedrock
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
model: bedrock/anthropic.claude-haiku-4-5-20251001:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1

2
docs/my-website/docs/tutorials/github_copilot_integration.md
View file

@ -141,7 +141,7 @@ Route requests to Claude on Bedrock:
model_list:
- model_name: bedrock-claude
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
model: bedrock/anthropic.claude-haiku-4-5-20251001:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1

2
docs/my-website/docs/tutorials/google_genai_sdk.md
View file

@ -282,7 +282,7 @@ Route `gemini-2.5-flash` requests to Claude on Bedrock:
model_list:
- model_name: bedrock-claude
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
model: bedrock/anthropic.claude-haiku-4-5-20251001:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1

4
docs/my-website/docs/tutorials/litellm_gemini_cli.md
View file

@ -127,7 +127,7 @@ Route `gemini-2.5-pro` requests to Claude on Bedrock:
model_list:
- model_name: bedrock-claude
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
model: bedrock/anthropic.claude-haiku-4-5-20251001:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1
@ -149,7 +149,7 @@ model_list:
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: anthropic-claude
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
model: bedrock/anthropic.claude-haiku-4-5-20251001:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1

4
docs/my-website/docs/tutorials/litellm_qwen_code_cli.md
View file

@ -129,7 +129,7 @@ Route `qwen-code` requests to Claude on Bedrock:
model_list:
- model_name: bedrock-claude
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
model: bedrock/anthropic.claude-haiku-4-5-20251001:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1
@ -151,7 +151,7 @@ model_list:
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: anthropic-claude
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
model: bedrock/anthropic.claude-haiku-4-5-20251001:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1

2
docs/my-website/docusaurus.config.js
View file

@ -284,8 +284,8 @@ const config = {
label: 'Enterprise',
to: "docs/enterprise"
},
{ to: '/release_notes', label: 'Changelog', position: 'left' },
{ to: '/blog', label: 'Blog', position: 'left' },
{ to: '/release_notes', label: 'Release Notes', position: 'left' },
{
href: 'https://github.com/BerriAI/litellm',
position: 'right',

BIN
docs/my-website/img/april_townhall_banner.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 49 KiB

25
docs/my-website/package-lock.json generated
View file

@ -13,18 +13,18 @@
"@docusaurus/plugin-ideal-image": "3.8.1",
"@docusaurus/preset-classic": "3.8.1",
"@docusaurus/theme-mermaid": "3.8.1",
"@inkeep/cxkit-docusaurus": "^0.5.89",
"@mdx-js/react": "^3.0.0",
"clsx": "^1.2.1",
"prism-react-renderer": "^1.3.5",
"react": "^18.0.0 || ^19.0.0",
"react-dom": "^18.0.0 || ^19.0.0",
"sharp": "^0.32.6",
"uuid": "^9.0.1"
"@inkeep/cxkit-docusaurus": "0.5.107",
"@mdx-js/react": "3.1.1",
"clsx": "1.2.1",
"prism-react-renderer": "1.3.5",
"react": "18.3.1",
"react-dom": "18.3.1",
"sharp": "0.32.6",
"uuid": "9.0.1"
},
"devDependencies": {
"@docusaurus/module-type-aliases": "3.8.1",
"dotenv": "^16.4.5"
"dotenv": "16.6.1"
},
"engines": {
"node": ">=16.14",
@ -20403,13 +20403,6 @@
"url": "https://opencollective.com/webpack"
}
},
"node_modules/search-insights": {
"version": "2.17.3",
"resolved": "https://registry.npmjs.org/search-insights/-/search-insights-2.17.3.tgz",
"integrity": "sha512-RQPdCYTa8A68uM2jwxoY842xDhvx3E5LFL1LxvxCNMev4o5mLuokczhzjAgGwUZBAmOKZknArSxLKmXtIi2AxQ==",
"license": "MIT",
"peer": true
},
"node_modules/section-matter": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/section-matter/-/section-matter-1.0.0.tgz",

18
docs/my-website/package.json
View file

@ -19,18 +19,18 @@
"@docusaurus/plugin-ideal-image": "3.8.1",
"@docusaurus/preset-classic": "3.8.1",
"@docusaurus/theme-mermaid": "3.8.1",
"@inkeep/cxkit-docusaurus": "^0.5.89",
"@mdx-js/react": "^3.0.0",
"clsx": "^1.2.1",
"prism-react-renderer": "^1.3.5",
"react": "^18.0.0 || ^19.0.0",
"react-dom": "^18.0.0 || ^19.0.0",
"sharp": "^0.32.6",
"uuid": "^9.0.1"
"@inkeep/cxkit-docusaurus": "0.5.107",
"@mdx-js/react": "3.1.1",
"clsx": "1.2.1",
"prism-react-renderer": "1.3.5",
"react": "18.3.1",
"react-dom": "18.3.1",
"sharp": "0.32.6",
"uuid": "9.0.1"
},
"devDependencies": {
"@docusaurus/module-type-aliases": "3.8.1",
"dotenv": "^16.4.5"
"dotenv": "16.6.1"
},
"browserslist": {
"production": [

8
litellm-js/proxy/package.json
View file

@ -4,11 +4,11 @@
"deploy": "wrangler deploy --minify src/index.ts"
},
"dependencies": {
"hono": "^4.1.4",
"openai": "^4.29.2"
"hono": "4.1.4",
"openai": "4.29.2"
},
"devDependencies": {
"@cloudflare/workers-types": "^4.20240208.0",
"wrangler": "^3.32.0"
"@cloudflare/workers-types": "4.20240208.0",
"wrangler": "3.32.0"
}
}

8
litellm-js/spend-logs/package-lock.json generated
View file

@ -5,12 +5,12 @@
"packages": {
"": {
"dependencies": {
"@hono/node-server": "^1.10.1",
"hono": "^4.12.7"
"@hono/node-server": "1.19.6",
"hono": "4.12.7"
},
"devDependencies": {
"@types/node": "^20.11.17",
"tsx": "^4.7.1"
"@types/node": "20.19.25",
"tsx": "4.20.6"
}
},
"node_modules/@esbuild/aix-ppc64": {

8
litellm-js/spend-logs/package.json
View file

@ -3,11 +3,11 @@
"dev": "tsx watch src/index.ts"
},
"dependencies": {
"@hono/node-server": "^1.10.1",
"hono": "^4.12.7"
"@hono/node-server": "1.19.6",
"hono": "4.12.7"
},
"devDependencies": {
"@types/node": "^20.11.17",
"tsx": "^4.7.1"
"@types/node": "20.19.25",
"tsx": "4.20.6"
}
}

12
litellm-proxy-extras/litellm_proxy_extras/migrations/20260331000000_add_prompt_environment_and_created_by/migration.sql Normal file
View file

@ -0,0 +1,12 @@
-- AlterTable
ALTER TABLE "LiteLLM_PromptTable" ADD COLUMN "environment" TEXT NOT NULL DEFAULT 'development';
ALTER TABLE "LiteLLM_PromptTable" ADD COLUMN "created_by" TEXT;
-- DropIndex (old unique constraint)
DROP INDEX IF EXISTS "LiteLLM_PromptTable_prompt_id_version_key";
-- CreateIndex (new unique constraint)
CREATE UNIQUE INDEX "LiteLLM_PromptTable_prompt_id_version_environment_key" ON "LiteLLM_PromptTable"("prompt_id", "version", "environment");
-- CreateIndex (new composite index)
CREATE INDEX "LiteLLM_PromptTable_prompt_id_environment_idx" ON "LiteLLM_PromptTable"("prompt_id", "environment");

16
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -321,11 +321,12 @@ model LiteLLM_MCPServerTable {
byok_description String[] @default([])
byok_api_key_help_url String?
source_url String?
approval_status String? @default("active")
submitted_by String?
submitted_at DateTime?
reviewed_at DateTime?
review_notes String?
// BYOM submission lifecycle
approval_status String? @default("active")
submitted_by String?
submitted_at DateTime?
reviewed_at DateTime?
review_notes String?
@@index([approval_status])
}
@ -1001,12 +1002,15 @@ model LiteLLM_PromptTable {
id String @id @default(uuid())
prompt_id String
version Int @default(1)
environment String @default("development")
created_by String?
litellm_params Json
prompt_info Json?
created_at DateTime @default(now())
updated_at DateTime @updatedAt
@@unique([prompt_id, version])
@@unique([prompt_id, version, environment])
@@index([prompt_id, environment])
@@index([prompt_id])
}

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-proxy-extras"
version = "0.4.62"
version = "0.4.64"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
authors = ["BerriAI"]
readme = "README.md"
@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.4.62"
version = "0.4.64"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-proxy-extras==",

14
litellm/a2a_protocol/litellm_completion_bridge/handler.py
View file

@ -48,20 +48,19 @@ class A2ACompletionBridgeHandler:
# Get provider config for custom_llm_provider
custom_llm_provider = litellm_params.get("custom_llm_provider")
a2a_provider_config = A2AProviderConfigManager.get_provider_config(
custom_llm_provider=custom_llm_provider
custom_llm_provider=custom_llm_provider,
model=litellm_params.get("model"),
)
# If provider config exists, use it
if a2a_provider_config is not None:
if api_base is None:
raise ValueError(f"api_base is required for {custom_llm_provider}")
verbose_logger.info(f"A2A: Using provider config for {custom_llm_provider}")
response_data = await a2a_provider_config.handle_non_streaming(
request_id=request_id,
params=params,
api_base=api_base,
litellm_params=litellm_params,
)
return response_data
@ -147,14 +146,12 @@ class A2ACompletionBridgeHandler:
# Get provider config for custom_llm_provider
custom_llm_provider = litellm_params.get("custom_llm_provider")
a2a_provider_config = A2AProviderConfigManager.get_provider_config(
custom_llm_provider=custom_llm_provider
custom_llm_provider=custom_llm_provider,
model=litellm_params.get("model"),
)
# If provider config exists, use it
if a2a_provider_config is not None:
if api_base is None:
raise ValueError(f"api_base is required for {custom_llm_provider}")
verbose_logger.info(
f"A2A: Using provider config for {custom_llm_provider} (streaming)"
)
@ -163,6 +160,7 @@ class A2ACompletionBridgeHandler:
request_id=request_id,
params=params,
api_base=api_base,
litellm_params=litellm_params,
):
yield chunk

6
litellm/a2a_protocol/providers/base.py
View file

@ -3,7 +3,7 @@ Base configuration for A2A protocol providers.
"""
from abc import ABC, abstractmethod
from typing import Any, AsyncIterator, Dict
from typing import Any, AsyncIterator, Dict, Optional
class BaseA2AProviderConfig(ABC):
@ -19,7 +19,7 @@ class BaseA2AProviderConfig(ABC):
self,
request_id: str,
params: Dict[str, Any],
api_base: str,
api_base: Optional[str] = None,
**kwargs,
) -> Dict[str, Any]:
"""
@ -41,7 +41,7 @@ class BaseA2AProviderConfig(ABC):
self,
request_id: str,
params: Dict[str, Any],
api_base: str,
api_base: Optional[str] = None,
**kwargs,
) -> AsyncIterator[Dict[str, Any]]:
"""

22
litellm/a2a_protocol/providers/bedrock_agentcore/__init__.py Normal file
View file

@ -0,0 +1,22 @@
"""
Bedrock AgentCore A2A provider.
Preserves JSON-RPC envelopes for AgentCore agents that speak A2A natively,
bypassing the completion bridge that would otherwise strip the envelope.
"""
from litellm.a2a_protocol.providers.bedrock_agentcore.config import (
BedrockAgentCoreA2AConfig,
)
from litellm.a2a_protocol.providers.bedrock_agentcore.handler import (
BedrockAgentCoreA2AHandler,
)
from litellm.a2a_protocol.providers.bedrock_agentcore.transformation import (
BedrockAgentCoreA2ATransformation,
)
__all__ = [
"BedrockAgentCoreA2AConfig",
"BedrockAgentCoreA2AHandler",
"BedrockAgentCoreA2ATransformation",
]

61
litellm/a2a_protocol/providers/bedrock_agentcore/config.py Normal file
View file

@ -0,0 +1,61 @@
"""
Bedrock AgentCore A2A provider configuration.
"""
from typing import Any, AsyncIterator, Dict, Optional
from litellm.a2a_protocol.providers.base import BaseA2AProviderConfig
from litellm.a2a_protocol.providers.bedrock_agentcore.handler import (
BedrockAgentCoreA2AHandler,
)
class BedrockAgentCoreA2AConfig(BaseA2AProviderConfig):
"""
Provider configuration for Bedrock AgentCore A2A-native agents.
AgentCore agents that speak A2A natively expect the full JSON-RPC envelope.
This config bypasses the completion bridge and forwards requests directly,
deriving the endpoint URL from the model ARN and signing with SigV4/JWT.
"""
async def handle_non_streaming(
self,
request_id: str,
params: Dict[str, Any],
api_base: Optional[str] = None,
**kwargs,
) -> Dict[str, Any]:
"""Handle non-streaming request to AgentCore A2A agent."""
litellm_params = kwargs.get("litellm_params")
if not litellm_params:
raise ValueError(
"litellm_params is required for BedrockAgentCoreA2AConfig "
"(must contain model with AgentCore ARN)"
)
return await BedrockAgentCoreA2AHandler.handle_non_streaming(
request_id=request_id,
params=params,
litellm_params=litellm_params,
)
async def handle_streaming(
self,
request_id: str,
params: Dict[str, Any],
api_base: Optional[str] = None,
**kwargs,
) -> AsyncIterator[Dict[str, Any]]:
"""Handle streaming request to AgentCore A2A agent."""
litellm_params = kwargs.get("litellm_params")
if not litellm_params:
raise ValueError(
"litellm_params is required for BedrockAgentCoreA2AConfig "
"(must contain model with AgentCore ARN)"
)
async for chunk in BedrockAgentCoreA2AHandler.handle_streaming(
request_id=request_id,
params=params,
litellm_params=litellm_params,
):
yield chunk

134
litellm/a2a_protocol/providers/bedrock_agentcore/handler.py Normal file
View file

@ -0,0 +1,134 @@
"""
Handler for Bedrock AgentCore A2A-native agents.
Sends JSON-RPC envelopes directly to AgentCore endpoints, bypassing the
completion bridge that would otherwise strip the envelope.
"""
import json
from typing import Any, AsyncIterator, Dict, cast
from litellm._logging import verbose_logger
from litellm.a2a_protocol.providers.bedrock_agentcore.transformation import (
BedrockAgentCoreA2ATransformation,
)
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.types.llms.custom_http import httpxSpecialProvider
class BedrockAgentCoreA2AHandler:
"""
Handler for Bedrock AgentCore A2A requests.
Constructs JSON-RPC envelopes, signs them via AmazonAgentCoreConfig,
and POSTs directly to the AgentCore endpoint.
"""
@staticmethod
async def handle_non_streaming(
request_id: str,
params: Dict[str, Any],
litellm_params: Dict[str, Any],
) -> Dict[str, Any]:
"""
Handle non-streaming A2A request to AgentCore.
Args:
request_id: A2A JSON-RPC request ID
params: A2A MessageSendParams containing the message
litellm_params: Agent's litellm_params (model, api_key, etc.)
Returns:
A2A JSON-RPC response dict from the AgentCore agent
"""
url, headers, body = (
BedrockAgentCoreA2ATransformation.get_url_and_signed_request(
request_id=request_id,
params=params,
litellm_params=litellm_params,
method="message/send",
)
)
verbose_logger.info(
f"BedrockAgentCore A2A: Sending non-streaming request to {url}"
)
client = get_async_httpx_client(
llm_provider=cast(Any, httpxSpecialProvider.A2AProvider),
)
response = await client.post(
url,
headers=headers,
data=body,
)
response.raise_for_status()
response_data = response.json()
if "error" in response_data:
verbose_logger.warning(
f"BedrockAgentCore A2A: Agent returned error: {response_data['error']}"
)
return response_data
@staticmethod
async def handle_streaming(
request_id: str,
params: Dict[str, Any],
litellm_params: Dict[str, Any],
) -> AsyncIterator[Dict[str, Any]]:
"""
Handle streaming A2A request to AgentCore.
Args:
request_id: A2A JSON-RPC request ID
params: A2A MessageSendParams containing the message
litellm_params: Agent's litellm_params (model, api_key, etc.)
Yields:
A2A streaming response events from the AgentCore agent
"""
url, headers, body = (
BedrockAgentCoreA2ATransformation.get_url_and_signed_request(
request_id=request_id,
params=params,
litellm_params=litellm_params,
method="message/send",
stream=True,
)
)
verbose_logger.info(
f"BedrockAgentCore A2A: Sending streaming request to {url}"
)
client = get_async_httpx_client(
llm_provider=cast(Any, httpxSpecialProvider.A2AProvider),
)
response = await client.post(
url,
headers=headers,
data=body,
stream=True,
)
response.raise_for_status()
# Check content type — AgentCore may return JSON instead of SSE
content_type = response.headers.get("content-type", "").lower()
if "application/json" in content_type:
# Single JSON response fallback (not SSE)
verbose_logger.debug(
"BedrockAgentCore A2A streaming: received JSON instead of SSE, "
"yielding as single event"
)
response_body = await response.aread()
response_data = json.loads(response_body)
yield response_data
else:
# SSE stream — parse data: lines
async for event in BedrockAgentCoreA2ATransformation.parse_sse_events(
response
):
yield event

134
litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py Normal file
View file

@ -0,0 +1,134 @@
"""
Transformation layer for Bedrock AgentCore A2A provider.
Constructs JSON-RPC envelopes, derives AgentCore URLs from model ARNs,
and signs requests via AmazonAgentCoreConfig (SigV4 or JWT).
"""
import json
from typing import Any, AsyncIterator, Dict, Tuple
from litellm._logging import verbose_logger
from litellm.llms.bedrock.chat.agentcore.transformation import AmazonAgentCoreConfig
class BedrockAgentCoreA2ATransformation:
"""
Request/response transformation for Bedrock AgentCore A2A agents.
Reuses AmazonAgentCoreConfig for URL construction, ARN parsing,
and request signing. No logic is duplicated.
"""
@staticmethod
def get_url_and_signed_request(
request_id: str,
params: Dict[str, Any],
litellm_params: Dict[str, Any],
method: str = "message/send",
stream: bool = False,
) -> Tuple[str, dict, bytes]:
"""
Build the AgentCore URL, construct a JSON-RPC envelope, and sign the request.
Args:
request_id: A2A JSON-RPC request ID
params: A2A MessageSendParams
litellm_params: Agent's litellm_params (model, api_key, etc.)
method: JSON-RPC method name (default: "message/send")
stream: Whether this is a streaming request
Returns:
Tuple of (url, signed_headers, signed_body_bytes)
"""
# Extract model and strip the "bedrock/" prefix
# "bedrock/agentcore/arn:aws:..." → "agentcore/arn:aws:..."
model = litellm_params.get("model", "")
if model.startswith("bedrock/"):
agentcore_model = model[len("bedrock/") :]
else:
agentcore_model = model
# Build optional_params from litellm_params (everything except model and custom_llm_provider)
optional_params = {
k: v
for k, v in litellm_params.items()
if k not in ("model", "custom_llm_provider")
}
agentcore_config = AmazonAgentCoreConfig()
# Derive URL from ARN
url = agentcore_config.get_complete_url(
api_base=optional_params.get("api_base"),
api_key=optional_params.get("api_key"),
model=agentcore_model,
optional_params=optional_params,
litellm_params=litellm_params,
stream=stream,
)
# Construct JSON-RPC 2.0 envelope
json_rpc_body = {
"jsonrpc": "2.0",
"method": method,
"id": request_id,
"params": params,
}
# Set required AgentCore session headers (normally set by transform_request,
# which we skip because it also builds {"prompt": "..."})
headers: dict = {}
session_id = agentcore_config._get_runtime_session_id(optional_params)
headers["X-Amzn-Bedrock-AgentCore-Runtime-Session-Id"] = session_id
runtime_user_id = agentcore_config._get_runtime_user_id(optional_params)
if runtime_user_id:
headers["X-Amzn-Bedrock-AgentCore-Runtime-User-Id"] = runtime_user_id
# Sign the request (SigV4 or JWT depending on api_key presence)
signed_headers, signed_body = agentcore_config.sign_request(
headers=headers,
optional_params=optional_params,
request_data=json_rpc_body,
api_base=url,
api_key=optional_params.get("api_key"),
model=agentcore_model,
stream=stream,
)
# sign_request returns Optional[bytes] — ensure we have bytes
if signed_body is None:
signed_body = json.dumps(json_rpc_body).encode()
return url, signed_headers, signed_body
@staticmethod
async def parse_sse_events(response: Any) -> AsyncIterator[Dict[str, Any]]:
"""
Parse SSE events from an httpx streaming response.
Reads line-by-line, parses `data:` lines as JSON, and yields each parsed dict.
Args:
response: httpx streaming response
Yields:
Parsed JSON dicts from SSE data lines
"""
async for line in response.aiter_lines():
line = line.strip()
if not line:
continue
if line.startswith("data:"):
data_str = line[len("data:") :].strip()
if not data_str:
continue
try:
event = json.loads(data_str)
yield event
except json.JSONDecodeError:
verbose_logger.debug(
f"BedrockAgentCore A2A: Skipping non-JSON SSE line: {data_str[:100]}"
)
continue

12
litellm/a2a_protocol/providers/config_manager.py
View file

@ -19,12 +19,14 @@ class A2AProviderConfigManager:
@staticmethod
def get_provider_config(
custom_llm_provider: Optional[str],
model: Optional[str] = None,
) -> Optional[BaseA2AProviderConfig]:
"""
Get the provider configuration for a given custom_llm_provider.
Args:
custom_llm_provider: The provider identifier (e.g., "pydantic_ai_agents")
model: The model string (used to distinguish sub-providers, e.g. agentcore vs other bedrock)
Returns:
Provider configuration instance or None if not found
@ -39,9 +41,11 @@ class A2AProviderConfigManager:
return PydanticAIProviderConfig()
# Add more providers here as needed
# elif custom_llm_provider == "another_provider":
# from litellm.a2a_protocol.providers.another_provider.config import AnotherProviderConfig
# return AnotherProviderConfig()
if custom_llm_provider == "bedrock" and model and "agentcore" in model:
from litellm.a2a_protocol.providers.bedrock_agentcore.config import (
BedrockAgentCoreA2AConfig,
)
return BedrockAgentCoreA2AConfig()
return None

4
litellm/cost_calculator.py
View file

@ -545,8 +545,8 @@ def cost_per_token( # noqa: PLR0915
)
if (
model_info.get("input_cost_per_token", 0) > 0
or model_info.get("output_cost_per_token", 0) > 0
(model_info.get("input_cost_per_token") or 0.0) > 0
or (model_info.get("output_cost_per_token") or 0.0) > 0
):
return generic_cost_per_token(
model=model,

5
litellm/litellm_core_utils/llm_request_utils.py
View file

@ -56,9 +56,8 @@ def pick_cheapest_chat_models_from_llm_provider(custom_llm_provider: str, n=1):
continue
if model_info.get("mode") != "chat":
continue
_cost = model_info.get("input_cost_per_token", 0) + model_info.get(
"output_cost_per_token", 0
)
_cost = (model_info.get("input_cost_per_token") or 0.0) + (model_info.get(
"output_cost_per_token") or 0.0)
model_costs.append((model, _cost))
# Sort by cost (ascending)

29
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -337,24 +337,35 @@ def _insert_assistant_continue_message(
"""
Add assistant continuation messages between consecutive user messages.
Only checks directly adjacent messages to preserve backward compatibility.
Skips tool messages and assistant messages with tool calls in the
alternation check, matching strict templates like llama.cpp.
"""
if not ensure_alternating_roles or len(messages) <= 1:
return messages
continue_message = assistant_continue_message or DEFAULT_ASSISTANT_CONTINUE_MESSAGE
# Find indexes where assistant_continue should be inserted (before that index)
insert_before_indexes: set = set()
for i in range(len(messages)):
curr = messages[i]
if _counts_for_alternation(curr) and curr["role"] == "user":
# Look backwards for the previous counted message
j = i - 1
while j >= 0:
if _counts_for_alternation(messages[j]):
if messages[j]["role"] == "user":
insert_before_indexes.add(i)
break
j -= 1
# Build the result with assistant_continue inserted at the right positions
modified_messages: List[AllMessageValues] = []
for i, message in enumerate(messages):
if (
i < len(messages) - 1
and message.get("role") == "user"
and messages[i + 1].get("role") == "user"
):
modified_messages.append(message)
if i in insert_before_indexes:
modified_messages.append(continue_message)
else:
modified_messages.append(message)
modified_messages.append(message)
return modified_messages

254
litellm/llms/anthropic/experimental_pass_through/messages/fake_stream_iterator.py
View file

@ -38,6 +38,102 @@ class FakeAnthropicMessagesStreamIterator:
self.chunks = self._create_streaming_chunks()
self.current_index = 0
def _create_content_block_chunks(
self, block_dict: Dict[str, Any], index: int
) -> List[bytes]:
"""Build SSE chunks for a single content block."""
chunks = []
block_type = block_dict.get("type")
if block_type == "text":
content_block_start = {
"type": "content_block_start",
"index": index,
"content_block": {"type": "text", "text": ""},
}
chunks.append(
f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode()
)
text = block_dict.get("text", "")
content_block_delta = {
"type": "content_block_delta",
"index": index,
"delta": {"type": "text_delta", "text": text},
}
chunks.append(
f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode()
)
elif block_type == "thinking":
content_block_start = {
"type": "content_block_start",
"index": index,
"content_block": {"type": "thinking", "thinking": "", "signature": ""},
}
chunks.append(
f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode()
)
thinking_text = block_dict.get("thinking", "")
if thinking_text:
content_block_delta = {
"type": "content_block_delta",
"index": index,
"delta": {"type": "thinking_delta", "thinking": thinking_text},
}
chunks.append(
f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode()
)
signature = block_dict.get("signature", "")
if signature:
signature_delta = {
"type": "content_block_delta",
"index": index,
"delta": {"type": "signature_delta", "signature": signature},
}
chunks.append(
f"event: content_block_delta\ndata: {json.dumps(signature_delta)}\n\n".encode()
)
elif block_type == "redacted_thinking":
content_block_start = {
"type": "content_block_start",
"index": index,
"content_block": {"type": "redacted_thinking"},
}
chunks.append(
f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode()
)
elif block_type == "tool_use":
content_block_start = {
"type": "content_block_start",
"index": index,
"content_block": {
"type": "tool_use",
"id": block_dict.get("id"),
"name": block_dict.get("name"),
"input": {},
},
}
chunks.append(
f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode()
)
input_data = block_dict.get("input", {})
content_block_delta = {
"type": "content_block_delta",
"index": index,
"delta": {"type": "input_json_delta", "partial_json": json.dumps(input_data)},
}
chunks.append(
f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode()
)
content_block_stop = {"type": "content_block_stop", "index": index}
chunks.append(
f"event: content_block_stop\ndata: {json.dumps(content_block_stop)}\n\n".encode()
)
return chunks
def _create_streaming_chunks(self) -> List[bytes]:
"""Convert the non-streaming response to streaming chunks"""
chunks = []
@ -69,152 +165,34 @@ class FakeAnthropicMessagesStreamIterator:
# 2-4. For each content block, send start/delta/stop events
content_blocks = response_dict.get("content", [])
if content_blocks:
for index, block in enumerate(content_blocks):
# Cast block to dict for easier access
block_dict = cast(Dict[str, Any], block)
block_type = block_dict.get("type")
if block_type == "text":
# content_block_start
content_block_start = {
"type": "content_block_start",
"index": index,
"content_block": {"type": "text", "text": ""},
}
chunks.append(
f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode()
)
# content_block_delta (send full text as one delta for simplicity)
text = block_dict.get("text", "")
content_block_delta = {
"type": "content_block_delta",
"index": index,
"delta": {"type": "text_delta", "text": text},
}
chunks.append(
f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode()
)
# content_block_stop
content_block_stop = {"type": "content_block_stop", "index": index}
chunks.append(
f"event: content_block_stop\ndata: {json.dumps(content_block_stop)}\n\n".encode()
)
elif block_type == "thinking":
# content_block_start for thinking
content_block_start = {
"type": "content_block_start",
"index": index,
"content_block": {
"type": "thinking",
"thinking": "",
"signature": "",
},
}
chunks.append(
f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode()
)
# content_block_delta for thinking text
thinking_text = block_dict.get("thinking", "")
if thinking_text:
content_block_delta = {
"type": "content_block_delta",
"index": index,
"delta": {
"type": "thinking_delta",
"thinking": thinking_text,
},
}
chunks.append(
f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode()
)
# content_block_delta for signature (if present)
signature = block_dict.get("signature", "")
if signature:
signature_delta = {
"type": "content_block_delta",
"index": index,
"delta": {
"type": "signature_delta",
"signature": signature,
},
}
chunks.append(
f"event: content_block_delta\ndata: {json.dumps(signature_delta)}\n\n".encode()
)
# content_block_stop
content_block_stop = {"type": "content_block_stop", "index": index}
chunks.append(
f"event: content_block_stop\ndata: {json.dumps(content_block_stop)}\n\n".encode()
)
elif block_type == "redacted_thinking":
# content_block_start for redacted_thinking
content_block_start = {
"type": "content_block_start",
"index": index,
"content_block": {"type": "redacted_thinking"},
}
chunks.append(
f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode()
)
# content_block_stop (no delta for redacted thinking)
content_block_stop = {"type": "content_block_stop", "index": index}
chunks.append(
f"event: content_block_stop\ndata: {json.dumps(content_block_stop)}\n\n".encode()
)
elif block_type == "tool_use":
# content_block_start
content_block_start = {
"type": "content_block_start",
"index": index,
"content_block": {
"type": "tool_use",
"id": block_dict.get("id"),
"name": block_dict.get("name"),
"input": {},
},
}
chunks.append(
f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode()
)
# content_block_delta (send input as JSON delta)
input_data = block_dict.get("input", {})
content_block_delta = {
"type": "content_block_delta",
"index": index,
"delta": {
"type": "input_json_delta",
"partial_json": json.dumps(input_data),
},
}
chunks.append(
f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode()
)
# content_block_stop
content_block_stop = {"type": "content_block_stop", "index": index}
chunks.append(
f"event: content_block_stop\ndata: {json.dumps(content_block_stop)}\n\n".encode()
)
for index, block in enumerate(content_blocks):
block_dict = cast(Dict[str, Any], block)
chunks.extend(self._create_content_block_chunks(block_dict, index))
# 5. message_delta event (with final usage and stop_reason)
# Include cache usage fields so clients that only read message_delta
# (like Claude Code's SDK) see the full input token breakdown.
delta_usage: Dict[str, Any] = {
"output_tokens": usage.get("output_tokens", 0) if usage else 0,
}
if usage:
if usage.get("input_tokens") is not None:
delta_usage["input_tokens"] = usage["input_tokens"]
if usage.get("cache_creation_input_tokens") is not None:
delta_usage["cache_creation_input_tokens"] = usage[
"cache_creation_input_tokens"
]
if usage.get("cache_read_input_tokens") is not None:
delta_usage["cache_read_input_tokens"] = usage[
"cache_read_input_tokens"
]
message_delta = {
"type": "message_delta",
"delta": {
"stop_reason": response_dict.get("stop_reason"),
"stop_sequence": response_dict.get("stop_sequence"),
},
"usage": {"output_tokens": usage.get("output_tokens", 0) if usage else 0},
"usage": delta_usage,
}
chunks.append(
f"event: message_delta\ndata: {json.dumps(message_delta)}\n\n".encode()

2
litellm/llms/azure_ai/cost_calculator.py
View file

@ -54,7 +54,7 @@ def calculate_azure_model_router_flat_cost(model: str, prompt_tokens: int) -> fl
model_info = get_model_info(model="model_router", custom_llm_provider="azure_ai")
router_flat_cost_per_token = model_info.get("input_cost_per_token", 0)
if router_flat_cost_per_token > 0:
if router_flat_cost_per_token and router_flat_cost_per_token > 0:
return prompt_tokens * router_flat_cost_per_token
return 0.0

14
litellm/llms/bedrock/chat/agentcore/transformation.py
View file

@ -19,6 +19,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.a2a.common_utils import extract_text_from_a2a_response
from litellm.llms.bedrock.common_utils import BedrockError
from litellm.types.llms.bedrock_agentcore import (
AgentCoreMessage,
@ -343,6 +344,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
Parse direct JSON response (non-streaming).
Supports multiple agent response schemas:
0. {"jsonrpc": "2.0", "result": {"message": {"parts": [...]}}} - A2A JSON-RPC
1. {"result": {"role": "assistant", "content": [{"text": "..."}]}} - standard AgentCore
2. {"response": [{"text": "..."}]} - Strands agent format
3. {"result": "plain text"} or {"response": "plain text"} - simple string
@ -361,6 +363,18 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
final_message=None,
)
# Strategy 0: A2A JSON-RPC format
# {"jsonrpc": "2.0", "result": {"message": {"parts": [{"kind": "text", "text": "..."}]}}}
if "jsonrpc" in response_json:
content = extract_text_from_a2a_response(response_json)
if content:
return AgentCoreParsedResponse(
content=content,
usage=None,
final_message=None,
)
# Fall through to other strategies if A2A extraction returned empty
# Strategy 1: {"result": {"content": [{"text": "..."}]}} - standard AgentCore format
if "result" in response_json and isinstance(response_json["result"], dict):
result = response_json["result"]

115
litellm/llms/bedrock/chat/invoke_handler.py
View file

@ -855,6 +855,32 @@ class BedrockLLM(BaseAWSLLM):
endpoint_url = f"{endpoint_url}/model/{modelId}/invoke"
proxy_endpoint_url = f"{proxy_endpoint_url}/model/{modelId}/invoke"
if acompletion and provider == "anthropic" and self.is_claude_messages_api_model(
model
):
if isinstance(client, HTTPHandler):
client = None
return self._async_anthropic_messages_completion(
model=model,
messages=messages,
endpoint_url=endpoint_url,
proxy_endpoint_url=proxy_endpoint_url,
credentials=credentials,
aws_region_name=aws_region_name,
model_response=model_response,
print_verbose=print_verbose,
encoding=encoding,
logging_obj=logging_obj,
optional_params=optional_params,
stream=stream,
litellm_params=litellm_params,
logger_fn=logger_fn,
extra_headers=extra_headers,
timeout=timeout,
client=client,
stream_chunk_size=stream_chunk_size,
) # type: ignore[return-value]
prompt, chat_history = self.convert_messages_to_prompt(
model, messages, provider, custom_prompt_dict
)
@ -1148,6 +1174,95 @@ class BedrockLLM(BaseAWSLLM):
encoding=encoding,
)
async def _async_anthropic_messages_completion(
self,
model: str,
messages: list,
endpoint_url: str,
proxy_endpoint_url: str,
credentials,
aws_region_name: str,
model_response: ModelResponse,
print_verbose: Callable,
encoding,
logging_obj: Logging,
optional_params: dict,
stream,
litellm_params=None,
logger_fn=None,
extra_headers: Optional[dict] = None,
timeout: Optional[Union[float, httpx.Timeout]] = None,
client: Optional[AsyncHTTPHandler] = None,
stream_chunk_size: int = 1024,
) -> Union[ModelResponse, CustomStreamWrapper]:
transformed_request = await litellm.AmazonAnthropicClaudeConfig().async_transform_request(
model=model,
messages=messages,
optional_params=optional_params,
litellm_params=litellm_params or {},
headers=extra_headers or {},
)
data = json.dumps(transformed_request)
headers = {"Content-Type": "application/json"}
if extra_headers is not None:
headers = {"Content-Type": "application/json", **extra_headers}
prepped = self.get_request_headers(
credentials=credentials,
aws_region_name=aws_region_name,
extra_headers=extra_headers,
endpoint_url=endpoint_url,
data=data,
headers=headers,
)
logging_obj.pre_call(
input=messages,
api_key="",
additional_args={
"complete_input_dict": data,
"api_base": proxy_endpoint_url,
"headers": prepped.headers,
},
)
if stream is True:
return await self.async_streaming(
model=model,
messages=messages,
data=data,
api_base=proxy_endpoint_url,
model_response=model_response,
print_verbose=print_verbose,
encoding=encoding,
logging_obj=logging_obj,
optional_params=optional_params,
stream=True,
litellm_params=litellm_params,
logger_fn=logger_fn,
headers=prepped.headers,
timeout=timeout,
client=client,
stream_chunk_size=stream_chunk_size,
)
return await self.async_completion(
model=model,
messages=messages,
data=data,
api_base=proxy_endpoint_url,
model_response=model_response,
print_verbose=print_verbose,
encoding=encoding,
logging_obj=logging_obj,
optional_params=optional_params,
stream=stream, # type: ignore
litellm_params=litellm_params,
logger_fn=logger_fn,
headers=prepped.headers,
timeout=timeout,
client=client,
)
async def async_completion(
self,
model: str,

188
litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py
View file

@ -2,6 +2,14 @@ from typing import TYPE_CHECKING, Any, List, Optional
import httpx
from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers
from litellm.litellm_core_utils.prompt_templates.factory import (
convert_to_anthropic_image_obj,
)
from litellm.litellm_core_utils.prompt_templates.image_handling import (
async_convert_url_to_base64,
convert_url_to_base64,
)
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
AmazonInvokeConfig,
@ -85,8 +93,62 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
litellm_params: dict,
headers: dict,
) -> dict:
# Filter out AWS authentication parameters before passing to Anthropic transformation
# AWS params should only be used for signing requests, not included in request body
_anthropic_request = self._build_bedrock_anthropic_request_base(
model=model,
messages=messages,
optional_params=optional_params,
litellm_params=litellm_params,
headers=headers,
)
self._convert_document_url_sources_to_base64(_anthropic_request)
beta_list = self._compute_bedrock_invoke_beta_headers(
model=model,
messages=messages,
optional_params=optional_params,
headers=headers,
)
if beta_list:
_anthropic_request["anthropic_beta"] = beta_list
return _anthropic_request
async def async_transform_request(
self,
model: str,
messages: List[AllMessageValues],
optional_params: dict,
litellm_params: dict,
headers: dict,
) -> dict:
_anthropic_request = self._build_bedrock_anthropic_request_base(
model=model,
messages=messages,
optional_params=optional_params,
litellm_params=litellm_params,
headers=headers,
)
await self._async_convert_document_url_sources_to_base64(_anthropic_request)
beta_list = self._compute_bedrock_invoke_beta_headers(
model=model,
messages=messages,
optional_params=optional_params,
headers=headers,
)
if beta_list:
_anthropic_request["anthropic_beta"] = beta_list
return _anthropic_request
def _build_bedrock_anthropic_request_base(
self,
model: str,
messages: List[AllMessageValues],
optional_params: dict,
litellm_params: dict,
headers: dict,
) -> dict:
filtered_params = {
k: v
for k, v in optional_params.items()
@ -94,7 +156,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
}
filtered_params = self._normalize_bedrock_tool_search_tools(filtered_params)
_anthropic_request = AnthropicConfig.transform_request(
anthropic_request = AnthropicConfig.transform_request(
self,
model=model,
messages=messages,
@ -103,28 +165,31 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
headers=headers,
)
_anthropic_request.pop("model", None)
_anthropic_request.pop("stream", None)
# Bedrock Invoke doesn't support output_format parameter
_anthropic_request.pop("output_format", None)
# Bedrock Invoke doesn't support output_config parameter
# Fixes: https://github.com/BerriAI/litellm/issues/22797
_anthropic_request.pop("output_config", None)
if "anthropic_version" not in _anthropic_request:
_anthropic_request["anthropic_version"] = self.anthropic_version
anthropic_request.pop("model", None)
anthropic_request.pop("stream", None)
anthropic_request.pop("output_format", None)
anthropic_request.pop("output_config", None)
if "anthropic_version" not in anthropic_request:
anthropic_request["anthropic_version"] = self.anthropic_version
# Remove `custom` field from tools (Bedrock doesn't support it)
# Claude Code sends `custom: {defer_loading: true}` on tool definitions,
# which causes Bedrock to reject the request with "Extra inputs are not permitted"
# Ref: https://github.com/BerriAI/litellm/issues/22847
remove_custom_field_from_tools(_anthropic_request)
remove_custom_field_from_tools(anthropic_request)
return anthropic_request
def _compute_bedrock_invoke_beta_headers(
self,
model: str,
messages: List[AllMessageValues],
optional_params: dict,
headers: dict,
) -> List[str]:
tools = optional_params.get("tools")
tool_search_used = self.is_tool_search_used(tools)
programmatic_tool_calling_used = self.is_programmatic_tool_calling_used(tools)
input_examples_used = self.is_input_examples_used(tools)
beta_set = set(get_anthropic_beta_from_headers(headers))
user_beta_set = set(get_anthropic_beta_from_headers(headers))
beta_set = set(user_beta_set)
auto_betas = self.get_anthropic_beta_list(
model=model,
optional_params=optional_params,
@ -142,12 +207,91 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
if "opus-4" in model.lower() or "opus_4" in model.lower():
beta_set.add("tool-search-tool-2025-10-19")
# Filter out beta headers that Bedrock Invoke doesn't support
# Uses centralized configuration from anthropic_beta_headers_config.json
beta_list = list(beta_set)
_anthropic_request["anthropic_beta"] = beta_list
auto_beta_list = filter_and_transform_beta_headers(
beta_headers=list(beta_set - user_beta_set),
provider="bedrock",
)
return sorted(user_beta_set.union(set(auto_beta_list)))
return _anthropic_request
def _convert_document_url_sources_to_base64(self, anthropic_request: dict) -> None:
"""
Bedrock Invoke does not accept document URL sources. Convert to base64 payloads.
"""
messages = anthropic_request.get("messages")
if not isinstance(messages, list):
return
for message in messages:
if not isinstance(message, dict):
continue
content = message.get("content")
if not isinstance(content, list):
continue
for block in content:
if not isinstance(block, dict) or block.get("type") != "document":
continue
source = block.get("source")
if not isinstance(source, dict) or source.get("type") != "url":
continue
source_url = source.get("url")
if not isinstance(source_url, str):
continue
inferred_format: Optional[str] = None
if source_url.lower().endswith(".pdf"):
inferred_format = "application/pdf"
base64_url = convert_url_to_base64(url=source_url)
image_chunk = convert_to_anthropic_image_obj(
openai_image_url=base64_url,
format=inferred_format,
)
block["source"] = {
"type": "base64",
"media_type": image_chunk["media_type"],
"data": image_chunk["data"],
}
async def _async_convert_document_url_sources_to_base64(
self, anthropic_request: dict
) -> None:
"""
Async version of document URL conversion for async completion paths.
"""
messages = anthropic_request.get("messages")
if not isinstance(messages, list):
return
for message in messages:
if not isinstance(message, dict):
continue
content = message.get("content")
if not isinstance(content, list):
continue
for block in content:
if not isinstance(block, dict) or block.get("type") != "document":
continue
source = block.get("source")
if not isinstance(source, dict) or source.get("type") != "url":
continue
source_url = source.get("url")
if not isinstance(source_url, str):
continue
inferred_format: Optional[str] = None
if source_url.lower().endswith(".pdf"):
inferred_format = "application/pdf"
base64_url = await async_convert_url_to_base64(url=source_url)
image_chunk = convert_to_anthropic_image_obj(
openai_image_url=base64_url,
format=inferred_format,
)
block["source"] = {
"type": "base64",
"media_type": image_chunk["media_type"],
"data": image_chunk["data"],
}
def _normalize_bedrock_tool_search_tools(self, optional_params: dict) -> dict:
"""

515
litellm/llms/bedrock/image_edit/amazon_nova_canvas_image_edit_transformation.py Normal file
View file

@ -0,0 +1,515 @@
"""
Amazon Nova Canvas image edit on Bedrock (InvokeModel).
Maps OpenAI-style image edit (image + prompt, optional mask) to Nova Canvas task types:
- With mask: INPAINTING (inPaintingParams per AWS docs)
- Without mask: IMAGE_VARIATION (imageVariationParams)
Refs:
- https://docs.aws.amazon.com/nova/latest/userguide/image-gen-access.html
- https://docs.aws.amazon.com/nova/latest/userguide/image-gen-req-resp-structure.html
"""
from __future__ import annotations
import base64
import os
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
import httpx
from litellm._logging import verbose_logger
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
from litellm.types.images.main import ImageEditOptionalRequestParams
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import FileTypes, ImageObject, ImageResponse
from litellm.utils import (
_get_model_cost_key,
_get_potential_model_names,
get_model_info,
)
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
LiteLLMLoggingObj = Any
def _nova_canvas_task_body(
*,
image_b64: str,
mask_b64: Optional[str],
text: str,
negative_text: Optional[str],
similarity_strength: Optional[float],
task_type: Optional[str],
mask_prompt: Optional[str],
out_painting_mode: Optional[str],
) -> Dict[str, Any]:
"""Build InvokeModel body task section (without imageGenerationConfig)."""
if task_type == "BACKGROUND_REMOVAL":
return {
"taskType": "BACKGROUND_REMOVAL",
"backgroundRemovalParams": {"image": image_b64},
}
if task_type == "OUTPAINTING":
if mask_prompt is None and mask_b64 is None:
raise ValueError(
"OUTPAINTING requires either a mask image or a mask prompt. "
"Pass mask=<file> or maskPrompt=<str> in the request."
)
out_params: Dict[str, Any] = {
"image": image_b64,
"text": text,
}
if mask_prompt is not None:
out_params["maskPrompt"] = mask_prompt
elif mask_b64 is not None:
out_params["maskImage"] = mask_b64
if negative_text is not None:
out_params["negativeText"] = negative_text
if out_painting_mode is not None:
out_params["outPaintingMode"] = out_painting_mode
return {
"taskType": "OUTPAINTING",
"outPaintingParams": out_params,
}
# Honour explicit IMAGE_VARIATION even when a mask is present (mask is ignored
# for this task type; callers use INPAINTING when they want mask semantics).
if task_type == "IMAGE_VARIATION":
var_params_explicit: Dict[str, Any] = {
"images": [image_b64],
"text": text,
}
if negative_text is not None:
var_params_explicit["negativeText"] = negative_text
if similarity_strength is not None:
var_params_explicit["similarityStrength"] = similarity_strength
return {
"taskType": "IMAGE_VARIATION",
"imageVariationParams": var_params_explicit,
}
# Explicit taskType must be INPAINTING or omitted from here on; anything else is invalid.
if task_type is not None and str(task_type).strip() != "":
if task_type != "INPAINTING":
raise ValueError(
f"Unsupported Amazon Nova Canvas taskType: {task_type!r}. "
"Use BACKGROUND_REMOVAL, OUTPAINTING, IMAGE_VARIATION, INPAINTING, "
"or omit taskType for automatic routing (mask → INPAINTING, else IMAGE_VARIATION)."
)
if mask_b64 is not None or mask_prompt is not None or task_type == "INPAINTING":
in_params: Dict[str, Any] = {"image": image_b64, "text": text}
if mask_prompt is not None:
in_params["maskPrompt"] = mask_prompt
elif mask_b64 is not None:
in_params["maskImage"] = mask_b64
if negative_text is not None:
in_params["negativeText"] = negative_text
if "maskPrompt" not in in_params and "maskImage" not in in_params:
raise ValueError(
"Amazon Nova Canvas INPAINTING requires either maskPrompt or maskImage "
"(use OpenAI mask= for maskImage, or pass maskPrompt in optional params). "
"See https://docs.aws.amazon.com/nova/latest/userguide/image-gen-req-resp-structure.html"
)
return {"taskType": "INPAINTING", "inPaintingParams": in_params}
var_params: Dict[str, Any] = {
"images": [image_b64],
"text": text,
}
if negative_text is not None:
var_params["negativeText"] = negative_text
if similarity_strength is not None:
var_params["similarityStrength"] = similarity_strength
return {
"taskType": "IMAGE_VARIATION",
"imageVariationParams": var_params,
}
def _file_types_to_b64(image: Optional[FileTypes]) -> str:
"""Encode OpenAI image input to base64 string for Nova Canvas."""
if image is None:
raise ValueError("Nova Canvas image edit requires an image input")
if hasattr(image, "read") and callable(getattr(image, "read", None)):
if hasattr(image, "seek"):
image.seek(0) # type: ignore[union-attr]
image_bytes = image.read() # type: ignore[union-attr]
return base64.b64encode(image_bytes).decode("utf-8")
if isinstance(image, bytes):
return base64.b64encode(image).decode("utf-8")
if isinstance(image, str):
return image
if isinstance(image, os.PathLike):
with open(image, "rb") as f:
return base64.b64encode(f.read()).decode("utf-8")
if isinstance(image, tuple):
raise ValueError(
"Nova Canvas image edit does not support tuple FileTypes. "
"Pass a file-like object, bytes, or a base64-encoded string."
)
return base64.b64encode(bytes(image)).decode("utf-8") # type: ignore[arg-type]
def _supports_nova_canvas_image_edit_from_model_cost(model: str) -> bool:
"""
True when model_cost has supports_nova_canvas_image_edit for a resolved catalog key.
get_model_info / ModelInfoBase omit arbitrary JSON keys, so we read model_cost
directly (same idea as supports_* bare_entry fallback).
"""
import litellm as _litellm
if not model:
return False
seen: set[str] = set()
candidates: List[str] = []
def _add(name: Optional[str]) -> None:
if name and name not in seen:
seen.add(name)
candidates.append(name)
_add(model)
if "/" in model:
suffix = model.split("/")[-1]
_add(suffix)
_add(f"bedrock/{suffix}")
# Cross-region inference ids (e.g. us.amazon.nova-canvas-v1:0) share pricing with
# the base model id (amazon.nova-canvas-v1:0) in model_cost.
try:
from litellm.llms.bedrock.common_utils import BedrockModelInfo
base_model = BedrockModelInfo.get_base_model(model)
if base_model and base_model != model:
_add(base_model)
_add(f"bedrock/{base_model}")
except Exception:
pass
try:
potential = _get_potential_model_names(model=model, custom_llm_provider=None)
for field in (
"combined_model_name",
"combined_stripped_model_name",
"stripped_model_name",
"split_model",
):
raw = potential.get(field)
if isinstance(raw, str):
_add(raw)
except Exception:
pass
for name in candidates:
key = _get_model_cost_key(name)
if key is None:
continue
entry = _litellm.model_cost.get(key) or {}
if entry.get("supports_nova_canvas_image_edit") is True:
return True
return False
class BedrockAmazonNovaCanvasImageEditConfig(BaseImageEditConfig):
"""
Bedrock InvokeModel image edit for amazon.nova-canvas-v1:0 and regional variants.
"""
@classmethod
def _is_nova_canvas_image_edit_model(cls, model: Optional[str] = None) -> bool:
"""
Use model_cost.supports_nova_canvas_image_edit so new Nova Canvas inference IDs
are added via model_prices_and_context_window.json only (not get_model_info, which
drops keys not on ModelInfoBase).
"""
return _supports_nova_canvas_image_edit_from_model_cost(model or "")
def get_supported_openai_params(self, model: str) -> list:
return [
"n",
"size",
"response_format",
"mask",
"negativeText",
"similarityStrength",
"cfgScale",
"seed",
"quality",
"taskType",
"maskPrompt",
"outPaintingMode",
"imageGenerationConfig",
]
def map_openai_params(
self,
image_edit_optional_params: ImageEditOptionalRequestParams,
model: str,
drop_params: bool,
) -> Dict[str, Any]:
supported = set(self.get_supported_openai_params(model))
mapped: Dict[str, Any] = dict(image_edit_optional_params)
_size = mapped.pop("size", None)
if _size is not None and isinstance(_size, str) and "x" in _size:
w, h = _size.split("x", 1)
try:
mapped["width"], mapped["height"] = int(w), int(h)
except ValueError:
pass
_n = mapped.pop("n", None)
if _n is not None:
mapped["numberOfImages"] = _n
_quality = mapped.pop("quality", None)
if _quality is not None:
if _quality in ("hd", "premium"):
mapped["quality"] = "premium"
elif _quality == "standard":
mapped["quality"] = "standard"
else:
# Re-emit unknown values (e.g. OpenAI "auto") so transform_image_edit_request
# forwards them and the API can reject, or drop_params can still apply upstream.
mapped["quality"] = _quality
# Accepted for OpenAI compatibility but ignored for Nova Canvas image edit;
# Bedrock returns base64 images only (no URL mode).
response_format = mapped.pop("response_format", None)
if response_format not in (None, "b64_json"):
verbose_logger.debug(
"Nova Canvas image edit ignores response_format=%s and returns base64 images",
response_format,
)
# Drop unknown keys if drop_params
if drop_params:
for k in list(mapped.keys()):
if k.startswith("_"):
continue
if k not in supported and k not in (
"width",
"height",
"numberOfImages",
"mask",
):
mapped.pop(k, None)
return mapped
def transform_image_edit_request(
self,
model: str,
prompt: Optional[str],
image: Optional[FileTypes],
image_edit_optional_request_params: Dict,
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> Tuple[Dict, Any]:
op = dict(image_edit_optional_request_params)
image_b64 = _file_types_to_b64(image)
mask_raw = op.pop("mask", None)
mask_b64: Optional[str] = None
if mask_raw is not None:
mask_b64 = _file_types_to_b64(mask_raw) # type: ignore[arg-type]
_size = op.pop("size", None)
width = op.pop("width", None)
height = op.pop("height", None)
if (
width is None
and height is None
and _size is not None
and isinstance(_size, str)
and "x" in _size
):
w, h = _size.split("x", 1)
try:
width, height = int(w), int(h)
except ValueError:
pass
number_of_images = op.pop("numberOfImages", None)
quality = op.pop("quality", None)
cfg_scale = op.pop("cfgScale", None)
seed = op.pop("seed", None)
image_generation_config: Dict[str, Any] = {}
nested_igc = op.pop("imageGenerationConfig", None)
if isinstance(nested_igc, dict):
image_generation_config.update(nested_igc)
if width is not None:
image_generation_config["width"] = width
if height is not None:
image_generation_config["height"] = height
if number_of_images is not None:
image_generation_config["numberOfImages"] = number_of_images
if quality is not None:
image_generation_config["quality"] = quality
if cfg_scale is not None:
image_generation_config["cfgScale"] = cfg_scale
if seed is not None:
image_generation_config["seed"] = seed
task_type = op.pop("taskType", None)
if (prompt is None or prompt == "") and task_type in (
"INPAINTING",
"OUTPAINTING",
):
raise ValueError(
f"Amazon Nova Canvas {task_type} requires a text prompt. "
"Pass a non-empty `prompt` in your request."
)
text = prompt if prompt is not None and prompt != "" else " "
negative_text = op.pop("negativeText", None)
similarity_strength = op.pop("similarityStrength", None)
mask_prompt = op.pop("maskPrompt", None)
out_painting_mode = op.pop("outPaintingMode", None)
body = _nova_canvas_task_body(
image_b64=image_b64,
mask_b64=mask_b64,
text=text,
negative_text=negative_text,
similarity_strength=similarity_strength,
task_type=task_type,
mask_prompt=mask_prompt,
out_painting_mode=out_painting_mode,
)
# BACKGROUND_REMOVAL InvokeModel body must not include imageGenerationConfig (AWS rejects it).
if image_generation_config and body.get("taskType") != "BACKGROUND_REMOVAL":
body["imageGenerationConfig"] = image_generation_config
return body, {}
def transform_image_edit_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
api_key: Optional[str] = None,
json_mode: Optional[bool] = None,
) -> ImageResponse:
try:
response_data = raw_response.json()
except Exception as e:
raise self.get_error_class(
error_message=f"Error parsing Nova Canvas image edit response: {e}",
status_code=raw_response.status_code,
headers=raw_response.headers,
)
if raw_response.status_code not in (200,):
raise self.get_error_class(
error_message=f"Nova Canvas image edit error: {response_data}",
status_code=raw_response.status_code,
headers=raw_response.headers,
)
images: List[str] = response_data.get("images") or []
if "errors" in response_data and not images:
raise self.get_error_class(
error_message=f"Nova Canvas image edit error: {response_data['errors']}",
status_code=raw_response.status_code,
headers=raw_response.headers,
)
# Nova Canvas InvokeModel success body uses "images" and optional "error" (AWS docs);
# it does not use Stability-style "finish_reasons".
error_msg = response_data.get("message") or response_data.get("error")
if error_msg and not images:
if not isinstance(error_msg, str):
error_msg = str(error_msg)
raise self.get_error_class(
error_message=f"Nova Canvas image edit error: {error_msg}",
status_code=raw_response.status_code,
headers=raw_response.headers,
)
model_response = ImageResponse()
model_response.data = []
for image_b64 in images:
if image_b64:
model_response.data.append(
ImageObject(
b64_json=image_b64,
url=None,
revised_prompt=None,
)
)
if not model_response.data:
raise self.get_error_class(
error_message="Nova Canvas image edit returned no images",
status_code=raw_response.status_code,
headers=raw_response.headers,
)
if not hasattr(model_response, "_hidden_params"):
model_response._hidden_params = {}
if "additional_headers" not in model_response._hidden_params:
model_response._hidden_params["additional_headers"] = {}
try:
model_info = get_model_info(model, custom_llm_provider="bedrock")
cost_per_image = model_info.get("output_cost_per_image", 0)
if cost_per_image is not None and model_response.data:
model_response._hidden_params["additional_headers"][
"llm_provider-x-litellm-response-cost"
] = float(cost_per_image) * len(model_response.data)
except Exception:
pass
return model_response
def use_multipart_form_data(self) -> bool:
return False
def get_complete_url(
self,
model: str,
api_base: Optional[str],
litellm_params: dict,
) -> str:
raise NotImplementedError(
"Nova Canvas image edit URLs are built in BedrockImageEdit._prepare_request "
"(AWS runtime endpoint + model invoke path). Do not use get_complete_url for "
"this config."
)
def validate_environment(
self,
headers: dict,
model: str,
api_key: Optional[str] = None,
) -> dict:
if headers is None:
headers = {}
if "Content-Type" not in headers:
headers["Content-Type"] = "application/json"
return headers
def get_bedrock_image_edit_config_for_model(
model: str,
) -> BaseImageEditConfig:
"""
Return the correct Bedrock image-edit config for the model id.
Same routing as ``BedrockImageEdit.get_config_class``: Stability edit models,
Nova Canvas when marked in model_cost; otherwise raises ``ValueError``.
"""
from litellm.llms.bedrock.image_edit.stability_transformation import (
BedrockStabilityImageEditConfig,
)
if BedrockStabilityImageEditConfig._is_stability_edit_model(model):
return BedrockStabilityImageEditConfig()
if BedrockAmazonNovaCanvasImageEditConfig._is_nova_canvas_image_edit_model(model):
return BedrockAmazonNovaCanvasImageEditConfig()
raise ValueError(
f"Unsupported Bedrock image-edit model: {model!r}. "
"Use a stability.* image-edit model id or add supports_nova_canvas_image_edit "
"in model_prices for this id."
)

14
litellm/llms/bedrock/image_edit/handler.py
View file

@ -15,6 +15,9 @@ from pydantic import BaseModel
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging
from litellm.llms.bedrock.image_edit.amazon_nova_canvas_image_edit_transformation import (
BedrockAmazonNovaCanvasImageEditConfig,
)
from litellm.llms.bedrock.image_edit.stability_transformation import (
BedrockStabilityImageEditConfig,
)
@ -55,8 +58,15 @@ class BedrockImageEdit(BaseAWSLLM):
def get_config_class(cls, model: str | None):
if BedrockStabilityImageEditConfig._is_stability_edit_model(model):
return BedrockStabilityImageEditConfig
else:
raise ValueError(f"Unsupported model for bedrock image edit: {model}")
if BedrockAmazonNovaCanvasImageEditConfig._is_nova_canvas_image_edit_model(
model
):
return BedrockAmazonNovaCanvasImageEditConfig
raise ValueError(
f"Unsupported Bedrock image-edit model: {model!r}. "
"Use a stability.* image-edit model id or add supports_nova_canvas_image_edit "
"in model_prices for this id."
)
def image_edit(
self,

96
litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py
View file

@ -12,6 +12,7 @@ from typing import (
import httpx
from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
AnthropicMessagesConfig,
@ -54,9 +55,6 @@ class AmazonAnthropicClaudeMessagesConfig(
DEFAULT_BEDROCK_ANTHROPIC_API_VERSION = "bedrock-2023-05-31"
# Beta header patterns that are not supported by Bedrock Invoke API
# These will be filtered out to prevent 400 "invalid beta flag" errors
def __init__(self, **kwargs):
BaseAnthropicMessagesConfig.__init__(self, **kwargs)
AmazonInvokeConfig.__init__(self, **kwargs)
@ -439,7 +437,8 @@ class AmazonAnthropicClaudeMessagesConfig(
)
input_examples_used = anthropic_model_info.is_input_examples_used(tools)
beta_set = set(get_anthropic_beta_from_headers(headers))
user_beta_set = set(get_anthropic_beta_from_headers(headers))
beta_set = set(user_beta_set)
auto_betas = anthropic_model_info.get_anthropic_beta_list(
model=model,
optional_params=anthropic_messages_optional_request_params,
@ -463,8 +462,13 @@ class AmazonAnthropicClaudeMessagesConfig(
if "tool-search-tool-2025-10-19" in beta_set:
beta_set.add("tool-examples-2025-10-29")
if beta_set:
anthropic_messages_request["anthropic_beta"] = list(beta_set)
filtered_auto_betas = filter_and_transform_beta_headers(
beta_headers=list(beta_set - user_beta_set),
provider="bedrock",
)
filtered_betas = sorted(user_beta_set.union(set(filtered_auto_betas)))
if filtered_betas:
anthropic_messages_request["anthropic_beta"] = filtered_betas
return anthropic_messages_request
@ -498,6 +502,12 @@ class AmazonAnthropicClaudeMessagesConfig(
):
"""
Bedrock invoke does not return SSE formatted data. This function is a wrapper to ensure litellm chunks are SSE formatted.
Bedrock's Anthropic-compatible streaming puts cache usage fields
(cache_creation_input_tokens, cache_read_input_tokens) only on
message_stop, not on message_start or message_delta. Claude Code's
SDK only merges usage from message_delta, so we promote those fields
from message_stop onto message_delta before yielding.
"""
from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import (
BaseAnthropicMessagesStreamingIterator,
@ -508,9 +518,81 @@ class AmazonAnthropicClaudeMessagesConfig(
request_body=request_body,
)
async for chunk in handler.async_sse_wrapper(completion_stream):
patched_stream = self._promote_message_stop_usage(completion_stream)
async for chunk in handler.async_sse_wrapper(patched_stream):
yield chunk
@staticmethod
async def _promote_message_stop_usage(
completion_stream: AsyncIterator[
Union[bytes, GenericStreamingChunk, ModelResponseStream, dict]
],
) -> AsyncIterator[Union[bytes, GenericStreamingChunk, ModelResponseStream, dict]]:
"""
Promote cache usage fields from message_stop onto message_delta.
Bedrock reports input_tokens (uncached only) on message_start, and
the full breakdown (input_tokens, cache_creation_input_tokens,
cache_read_input_tokens) only on message_stop. Claude Code's SDK
merges usage from message_start and message_delta but ignores
message_stop. This method buffers message_delta and, when
message_stop arrives with cache usage, merges those fields into the
message_delta usage and also updates the input_tokens on
message_delta to include the full count (uncached + cache_creation +
cache_read).
"""
_CACHE_FIELDS = ("cache_creation_input_tokens", "cache_read_input_tokens")
pending_delta = None
async for chunk in completion_stream:
if not isinstance(chunk, dict):
if pending_delta is not None:
yield pending_delta
pending_delta = None
yield chunk
continue
chunk_type = chunk.get("type")
if chunk_type == "message_delta":
pending_delta = chunk
continue
if chunk_type == "message_stop" and pending_delta is not None:
stop_usage = dict(chunk.get("usage") or {})
delta_usage = dict(pending_delta.get("usage") or {})
for field in _CACHE_FIELDS:
if field in stop_usage:
delta_usage[field] = stop_usage[field]
raw_input = stop_usage.get("input_tokens")
if raw_input is not None:
uncached = raw_input if isinstance(raw_input, int) else 0
raw_cc = delta_usage.get("cache_creation_input_tokens", 0)
cache_creation = raw_cc if isinstance(raw_cc, int) else 0
raw_cr = delta_usage.get("cache_read_input_tokens", 0)
cache_read = raw_cr if isinstance(raw_cr, int) else 0
delta_usage["input_tokens"] = uncached + cache_creation + cache_read
if delta_usage:
pending_delta["usage"] = delta_usage # type: ignore[arg-type]
yield pending_delta
pending_delta = None
yield chunk
continue
if pending_delta is not None:
yield pending_delta
pending_delta = None
yield chunk
if pending_delta is not None:
yield pending_delta
class AmazonAnthropicClaudeMessagesStreamDecoder(AWSEventStreamDecoder):
def __init__(

80
litellm/llms/firecrawl/search/transformation.py
View file

@ -159,15 +159,13 @@ class FirecrawlSearchConfig(BaseSearchConfig):
"""
Transform Firecrawl API response to LiteLLM unified SearchResponse format.
Firecrawl → LiteLLM mappings:
- data.web[].title → SearchResult.title
- data.web[].url → SearchResult.url
- data.web[].description OR data.web[].markdown → SearchResult.snippet
- No date field in web results (set to None)
- No last_updated field in Firecrawl response (set to None)
Supports both response formats:
Note: Firecrawl v2 returns results organized by source type (web, images, news).
We primarily use web results for the unified format.
Firecrawl Cloud (v2):
{"data": {"web": [...], "news": [...]}}
Firecrawl Self-Hosted (v1):
{"success": true, "data": [{"url": "...", "title": "...", ...}, ...]}
Args:
raw_response: Raw httpx response from Firecrawl API
@ -181,36 +179,52 @@ class FirecrawlSearchConfig(BaseSearchConfig):
# Transform results to SearchResult objects
results = []
# Process web results (primary source)
data = response_json.get("data", {})
web_results = data.get("web", [])
for result in web_results:
# Use markdown if available, otherwise fall back to description
snippet = result.get("markdown") or result.get("description", "")
if isinstance(data, list):
# Self-hosted Firecrawl (v1) format: data is a flat list of results
for result in data:
snippet = (
result.get("markdown") or result.get("description", "")
)
search_result = SearchResult(
title=result.get("title", ""),
url=result.get("url", ""),
snippet=snippet,
date=None,
last_updated=None,
)
results.append(search_result)
elif isinstance(data, dict):
# Firecrawl Cloud (v2) format: data is a dict with web/news keys
web_results = data.get("web", [])
search_result = SearchResult(
title=result.get("title", ""),
url=result.get("url", ""),
snippet=snippet,
date=None, # Web results don't include date
last_updated=None, # Firecrawl doesn't provide last_updated in response
)
results.append(search_result)
for result in web_results:
# Use markdown if available, otherwise fall back to description
snippet = result.get("markdown") or result.get("description", "")
# Process news results if available (they have date field)
news_results = data.get("news", [])
for result in news_results:
snippet = result.get("markdown") or result.get("snippet", "")
search_result = SearchResult(
title=result.get("title", ""),
url=result.get("url", ""),
snippet=snippet,
date=None,
last_updated=None,
)
results.append(search_result)
search_result = SearchResult(
title=result.get("title", ""),
url=result.get("url", ""),
snippet=snippet,
date=result.get("date"), # News results include date
last_updated=None,
)
results.append(search_result)
# Process news results if available (they have date field)
news_results = data.get("news", [])
for result in news_results:
snippet = result.get("markdown") or result.get("snippet", "")
search_result = SearchResult(
title=result.get("title", ""),
url=result.get("url", ""),
snippet=snippet,
date=result.get("date"), # News results include date
last_updated=None,
)
results.append(search_result)
return SearchResponse(
results=results,

4
litellm/llms/vertex_ai/cost_calculator.py
View file

@ -203,7 +203,7 @@ def _handle_128k_pricing(
):
prompt_cost = prompt_tokens * input_cost_per_token_above_128k_tokens
else:
prompt_cost = prompt_tokens * model_info["input_cost_per_token"]
prompt_cost = prompt_tokens * (model_info["input_cost_per_token"] or 0.0)
## CALCULATE OUTPUT COST
output_cost_per_token_above_128k_tokens = model_info.get(
@ -215,7 +215,7 @@ def _handle_128k_pricing(
):
completion_cost = completion_tokens * output_cost_per_token_above_128k_tokens
else:
completion_cost = completion_tokens * model_info["output_cost_per_token"]
completion_cost = completion_tokens * (model_info["output_cost_per_token"] or 0.0)
return prompt_cost, completion_cost

300
litellm/model_prices_and_context_window_backup.json
View file

@ -277,7 +277,15 @@
"litellm_provider": "bedrock",
"max_input_tokens": 2600,
"mode": "image_generation",
"output_cost_per_image": 0.06
"output_cost_per_image": 0.06,
"supports_nova_canvas_image_edit": true
},
"us.amazon.nova-canvas-v1:0": {
"litellm_provider": "bedrock",
"max_input_tokens": 2600,
"mode": "image_generation",
"output_cost_per_image": 0.06,
"supports_nova_canvas_image_edit": true
},
"us.writer.palmyra-x4-v1:0": {
"input_cost_per_token": 2.5e-06,
@ -6672,6 +6680,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-northeast-1/minimax.minimax-m2.5": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.44e-06
},
"bedrock/ap-northeast-1/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 7.3e-07,
"litellm_provider": "bedrock",
@ -6781,6 +6803,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-south-1/minimax.minimax-m2.5": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.44e-06
},
"bedrock/ap-south-1/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 7.1e-07,
"litellm_provider": "bedrock",
@ -6819,6 +6855,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-southeast-2/minimax.minimax-m2.5": {
"input_cost_per_token": 3.09e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.236e-06
},
"bedrock/ap-southeast-3/deepseek.v3.2": {
"input_cost_per_token": 7.4e-07,
"litellm_provider": "bedrock",
@ -6845,6 +6895,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-southeast-3/minimax.minimax-m2.5": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.44e-06
},
"bedrock/ap-southeast-3/moonshotai.kimi-k2.5": {
"input_cost_per_token": 7.2e-07,
"litellm_provider": "bedrock",
@ -6916,6 +6980,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-north-1/minimax.minimax-m2.5": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.44e-06
},
"bedrock/eu-north-1/moonshotai.kimi-k2.5": {
"input_cost_per_token": 7.2e-07,
"litellm_provider": "bedrock",
@ -7030,6 +7108,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-central-1/minimax.minimax-m2.5": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.44e-06
},
"bedrock/eu-central-1/qwen.qwen3-coder-next": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
@ -7074,6 +7166,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-west-1/minimax.minimax-m2.5": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.44e-06
},
"bedrock/eu-west-1/qwen.qwen3-coder-next": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
@ -7118,6 +7224,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-west-2/minimax.minimax-m2.5": {
"input_cost_per_token": 4.7e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.86e-06
},
"bedrock/eu-west-2/qwen.qwen3-coder-next": {
"input_cost_per_token": 7.8e-07,
"litellm_provider": "bedrock",
@ -7174,6 +7294,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-south-1/minimax.minimax-m2.5": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.44e-06
},
"bedrock/eu-south-1/qwen.qwen3-coder-next": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
@ -7249,6 +7383,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/sa-east-1/minimax.minimax-m2.5": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.44e-06
},
"bedrock/sa-east-1/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 7.3e-07,
"litellm_provider": "bedrock",
@ -7449,6 +7597,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-east-1/minimax.minimax-m2.5": {
"input_cost_per_token": 3e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.2e-06
},
"bedrock/us-east-1/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
@ -7513,6 +7675,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-east-2/minimax.minimax-m2.5": {
"input_cost_per_token": 3e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.2e-06
},
"bedrock/us-east-2/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
@ -7643,13 +7819,13 @@
"cache_creation_input_token_cost": 3.75e-07
},
"bedrock/us-gov-east-1/claude-sonnet-4-5-20250929-v1:0": {
"input_cost_per_token": 3.3e-06,
"input_cost_per_token": 3.6e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 200000,
"max_output_tokens": 4096,
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.65e-05,
"output_cost_per_token": 1.8e-05,
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
@ -7659,8 +7835,8 @@
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"cache_read_input_token_cost": 3.3e-07,
"cache_creation_input_token_cost": 4.125e-06
"cache_read_input_token_cost": 3.6e-07,
"cache_creation_input_token_cost": 4.5e-06
},
"bedrock/us-gov-east-1/meta.llama3-70b-instruct-v1:0": {
"input_cost_per_token": 2.65e-06,
@ -7794,13 +7970,13 @@
"cache_creation_input_token_cost": 3.75e-07
},
"bedrock/us-gov-west-1/claude-sonnet-4-5-20250929-v1:0": {
"input_cost_per_token": 3.3e-06,
"input_cost_per_token": 3.6e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 200000,
"max_output_tokens": 4096,
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.65e-05,
"output_cost_per_token": 1.8e-05,
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
@ -7810,8 +7986,8 @@
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"cache_read_input_token_cost": 3.3e-07,
"cache_creation_input_token_cost": 4.125e-06
"cache_read_input_token_cost": 3.6e-07,
"cache_creation_input_token_cost": 4.5e-06
},
"bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0": {
"input_cost_per_token": 2.65e-06,
@ -7995,6 +8171,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-west-2/minimax.minimax-m2.5": {
"input_cost_per_token": 3e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.2e-06
},
"bedrock/us-west-2/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
@ -21292,6 +21482,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"minimax.minimax-m2.5": {
"input_cost_per_token": 3e-07,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"minimax/speech-02-hd": {
"input_cost_per_character": 0.0001,
"litellm_provider": "minimax",
@ -23111,6 +23315,20 @@
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_native_structured_output": true
},
"nvidia.nemotron-super-3-120b": {
"input_cost_per_token": 1.5e-07,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 256000,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 6.5e-07,
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"o1": {
"cache_read_input_token_cost": 7.5e-06,
"input_cost_per_token": 1.5e-05,
@ -31164,7 +31382,9 @@
"mode": "chat",
"output_cost_per_token": 3.2e-06,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#glm-models",
"supported_regions": ["global"],
"supported_regions": [
"global"
],
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
@ -32724,6 +32944,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"zai.glm-5": {
"input_cost_per_token": 1e-06,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 200000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 3.2e-06,
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"zai/glm-5": {
"cache_creation_input_token_cost": 0,
"cache_read_input_token_cost": 2e-07,
@ -37473,5 +37707,51 @@
]
}
]
},
"bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0": {
"cache_creation_input_token_cost": 1.5e-06,
"cache_read_input_token_cost": 1.2e-07,
"input_cost_per_token": 1.2e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 6e-06,
"source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock",
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true,
"supports_pdf_input": true
},
"bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0": {
"cache_creation_input_token_cost": 1.5e-06,
"cache_read_input_token_cost": 1.2e-07,
"input_cost_per_token": 1.2e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 6e-06,
"source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock",
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true,
"supports_pdf_input": true
}
}

26
litellm/proxy/_types.py
View file

@ -2753,6 +2753,8 @@ class NewProjectRequest(LiteLLM_BudgetTable):
budget_id: Optional[str] = None
metadata: Optional[dict] = None
tags: Optional[List[str]] = None
guardrails: Optional[List[str]] = None
policies: Optional[List[str]] = None
models: List[str] = []
model_rpm_limit: Optional[dict] = None
model_tpm_limit: Optional[dict] = None
@ -2785,6 +2787,8 @@ class UpdateProjectRequest(LiteLLM_BudgetTable):
team_id: Optional[str] = None
metadata: Optional[dict] = None
tags: Optional[List[str]] = None
guardrails: Optional[List[str]] = None
policies: Optional[List[str]] = None
models: Optional[List[str]] = None
model_rpm_limit: Optional[dict] = None
model_tpm_limit: Optional[dict] = None
@ -4098,6 +4102,24 @@ class ScopeMapping(OIDCPermissions):
}
class JWTRoutingOverride(BaseModel):
"""
Override default auth routing for JWT-shaped bearer tokens.
A rule matches when all provided selectors match token claims.
If matched, request is routed to the configured auth path.
"""
iss: Union[str, List[str]]
client_id: Optional[Union[str, List[str]]] = None
aud: Optional[Union[str, List[str]]] = None
path: Literal["oauth2"] = "oauth2"
model_config = {
"extra": "forbid",
}
class LiteLLM_JWTAuth(LiteLLMPydanticObjectBase):
"""
A class to define the roles and permissions for a LiteLLM Proxy w/ JWT Auth.
@ -4198,6 +4220,10 @@ class LiteLLM_JWTAuth(LiteLLMPydanticObjectBase):
default=300,
description="TTL (seconds) for caching JWT-to-virtual-key mapping lookups.",
)
routing_overrides: Optional[List[JWTRoutingOverride]] = Field(
default=None,
description="Optional claim-based routing overrides for JWT-shaped tokens. Matching rules route requests to oauth2 before default JWT flow.",
)
#########################################################
def __init__(self, **kwargs: Any) -> None:

46
litellm/proxy/auth/auth_checks.py
View file

@ -165,9 +165,24 @@ def _is_model_cost_zero(
)
return False
# This model has zero cost explicitly configured
# Costs are 0 — verify this is from explicit configuration,
# not from defaulted sparse auto-registration entries.
# See: https://github.com/BerriAI/litellm/issues/24770
safe_name = str(model_name).replace("\n", "").replace("\r", "")
if not _is_cost_explicitly_configured(model_name, llm_router):
verbose_proxy_logger.debug(
"Model %s has zero cost but no explicit cost "
"configuration in model_cost entry — treating as unknown "
"cost (enforce budget)",
safe_name,
)
return False
verbose_proxy_logger.debug(
f"Model {model_name} has zero cost explicitly configured (input: {input_cost}, output: {output_cost})"
"Model %s has zero cost explicitly configured (input: %s, output: %s)",
safe_name,
input_cost,
output_cost,
)
except Exception as e:
@ -181,6 +196,33 @@ def _is_model_cost_zero(
return True
def _is_cost_explicitly_configured(
model: str, llm_router: "Router"
) -> bool:
"""
Check if any deployment in the model group has cost fields explicitly
set in its litellm.model_cost entry.
When Router._create_deployment() registers a model not in the global
cost map, it creates a sparse entry like {"id": "<hash>"} with no cost
fields. _get_model_info_helper() then defaults missing costs to 0.
This function detects that scenario by checking the raw model_cost entry.
"""
for deployment in llm_router.model_list:
if deployment.get("model_name") != model:
continue
model_id = deployment.get("model_info", {}).get("id")
if model_id is None:
continue
raw_entry = litellm.model_cost.get(model_id, {})
if (
"input_cost_per_token" in raw_entry
or "output_cost_per_token" in raw_entry
):
return True
return False
async def _run_project_checks(
project_object: Optional[LiteLLM_ProjectTableCachedObj],
_model: Optional[Union[str, List[str]]],

64
litellm/proxy/auth/handle_jwt.py
View file

@ -16,6 +16,8 @@ from cryptography import x509
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import serialization
from fastapi import HTTPException
import jwt
from jwt.api_jwk import PyJWK
from litellm._logging import verbose_proxy_logger
from litellm.caching.caching import DualCache
@ -71,6 +73,21 @@ class JWTHandler:
prisma_client: Optional[PrismaClient]
user_api_key_cache: DualCache
# Supported algos: https://pyjwt.readthedocs.io/en/stable/algorithms.html
# "Warning: Make sure not to mix symmetric and asymmetric algorithms that interpret
# the key in different ways (e.g. HS* and RS*)."
SUPPORTED_JWT_ALGORITHMS = [
"RS256",
"RS384",
"RS512",
"PS256",
"PS384",
"PS512",
"ES256",
"ES384",
"ES512",
"EdDSA",
]
def __init__(
self,
@ -97,6 +114,30 @@ class JWTHandler:
parts = token.split(".")
return len(parts) == 3
@staticmethod
def get_unverified_claims(token: str) -> Optional[dict]:
"""
Decode JWT claims without signature verification.
Used for routing decisions before selecting validation path.
"""
if not JWTHandler.is_jwt(token):
return None
try:
claims = jwt.decode(
token,
options={"verify_signature": False, "verify_aud": False},
algorithms=JWTHandler.SUPPORTED_JWT_ALGORITHMS,
)
if isinstance(claims, dict):
return claims
return None
except Exception as e:
verbose_proxy_logger.debug(
"Failed to decode unverified JWT claims for routing: %s", e
)
return None
def _rbac_role_from_role_mapping(self, token: dict) -> Optional[RBAC_ROLES]:
"""
Returns the RBAC role the token 'belongs' to based on role mappings.
@ -664,30 +705,11 @@ class JWTHandler:
raise Exception(f"Failed to fetch OIDC UserInfo: {str(e)}")
async def auth_jwt(self, token: str) -> dict:
# Supported algos: https://pyjwt.readthedocs.io/en/stable/algorithms.html
# "Warning: Make sure not to mix symmetric and asymmetric algorithms that interpret
# the key in different ways (e.g. HS* and RS*)."
algorithms = [
"RS256",
"RS384",
"RS512",
"PS256",
"PS384",
"PS512",
"ES256",
"ES384",
"ES512",
"EdDSA",
]
audience = os.getenv("JWT_AUDIENCE")
decode_options = None
if audience is None:
decode_options = {"verify_aud": False}
import jwt
from jwt.api_jwk import PyJWK
header = jwt.get_unverified_header(token)
verbose_proxy_logger.debug("header: %s", header)
@ -721,7 +743,7 @@ class JWTHandler:
payload = jwt.decode(
token,
public_key_obj, # type: ignore
algorithms=algorithms,
algorithms=self.SUPPORTED_JWT_ALGORITHMS,
options=decode_options, # type: ignore[arg-type]
audience=audience,
leeway=self.leeway, # allow testing of expired tokens
@ -749,7 +771,7 @@ class JWTHandler:
payload = jwt.decode(
token,
key,
algorithms=algorithms,
algorithms=self.SUPPORTED_JWT_ALGORITHMS,
audience=audience,
options=decode_options,
)

183
litellm/proxy/auth/user_api_key_auth.py
View file

@ -11,7 +11,7 @@ import asyncio
import re
import secrets
from datetime import datetime, timezone
from typing import List, Optional, Tuple, cast
from typing import Any, List, Optional, Tuple, cast
import fastapi
from fastapi import HTTPException, Request, WebSocket, status
@ -139,6 +139,58 @@ def _get_bearer_token_or_received_api_key(api_key: str) -> str:
return api_key
def _routing_selector_matches_claim(
selector_value: Optional[Any], claim_value: Optional[Any]
) -> bool:
if selector_value is None:
return True
selector_list = (
[str(v) for v in selector_value]
if isinstance(selector_value, list)
else [str(selector_value)]
)
if isinstance(claim_value, list):
claim_list = [str(v) for v in claim_value]
return any(v in claim_list for v in selector_list)
return str(claim_value) in selector_list if claim_value is not None else False
def _matches_routing_override(
token_claims: dict, override: "JWTRoutingOverride"
) -> bool:
return (
_routing_selector_matches_claim(override.iss, token_claims.get("iss"))
and _routing_selector_matches_claim(
override.client_id, token_claims.get("client_id")
)
and _routing_selector_matches_claim(override.aud, token_claims.get("aud"))
)
def _should_route_jwt_to_oauth2_override(token: str, jwt_handler: JWTHandler) -> bool:
routing_overrides = jwt_handler.litellm_jwtauth.routing_overrides
if not routing_overrides:
return False
token_claims = jwt_handler.get_unverified_claims(token=token)
if token_claims is None:
return False
for override in routing_overrides:
if override.path == "oauth2" and _matches_routing_override(
token_claims=token_claims, override=override
):
verbose_proxy_logger.debug(
"JWT routing override matched. Routing token to OAuth2 introspection."
)
return True
return False
def _get_bearer_token(
api_key: str,
):
@ -649,12 +701,20 @@ async def _user_api_key_auth_builder( # noqa: PLR0915
# - JWT tokens (3 dot-separated parts) -> skip OAuth2, fall through to JWT handler
# - Opaque tokens -> use OAuth2 handler
# This allows JWT for users and OAuth2 for M2M on the same instance
is_jwt_token = (
is_jwt = (
jwt_handler.is_jwt(token=api_key)
if general_settings.get("enable_jwt_auth", False) is True
else False
)
if not is_jwt_token:
# Routing uses unverified JWT claims only to choose auth path.
# Final authentication is enforced by the selected validator.
route_jwt_to_oauth2 = (
is_jwt
and _should_route_jwt_to_oauth2_override(
token=api_key, jwt_handler=jwt_handler
)
)
if not is_jwt or route_jwt_to_oauth2:
# return UserAPIKeyAuth object
# helper to check if the api_key is a valid oauth2 token
from litellm.proxy.proxy_server import premium_user
@ -688,7 +748,7 @@ async def _user_api_key_auth_builder( # noqa: PLR0915
jwt_claims: Optional[dict]
if (
jwt_handler.litellm_jwtauth.oidc_userinfo_enabled
and not jwt_handler.is_jwt(token=api_key)
and not is_jwt
):
jwt_claims = await jwt_handler.get_oidc_userinfo(token=api_key)
else:
@ -1193,49 +1253,13 @@ async def _user_api_key_auth_builder( # noqa: PLR0915
raise Exception(
"Key is blocked. Update via `/key/unblock` if you're an admin."
)
config = valid_token.config
if config != {}:
model_list = config.get("model_list", [])
new_model_list = model_list
verbose_proxy_logger.debug(
f"\n new llm router model list {new_model_list}"
)
elif (
isinstance(valid_token.models, list)
and "all-team-models" in valid_token.models
):
# Do not do any validation at this step
# the validation will occur when checking the team has access to this model
pass
else:
model = get_model_from_request(request_data, route)
fallback_models = cast(
Optional[List[ALL_FALLBACK_MODEL_VALUES]],
request_data.get("fallbacks", None),
)
if model is not None:
await can_key_call_model(
model=model,
llm_model_list=llm_model_list,
valid_token=valid_token,
llm_router=llm_router,
)
if fallback_models is not None:
for m in fallback_models:
await can_key_call_model(
model=m["model"] if isinstance(m, dict) else m,
llm_model_list=llm_model_list,
valid_token=valid_token,
llm_router=llm_router,
)
await is_valid_fallback_model(
model=m["model"] if isinstance(m, dict) else m,
llm_router=llm_router,
user_model=None,
)
await _enforce_key_and_fallback_model_access(
valid_token=valid_token,
request_data=request_data,
route=route,
llm_model_list=llm_model_list,
llm_router=llm_router,
)
# Check 2. If user_id for this token is in budget - done in common_checks()
if valid_token.user_id is not None:
@ -1764,6 +1788,61 @@ async def _lookup_end_user_and_apply_budget(
return valid_token, end_user_object
async def _enforce_key_and_fallback_model_access(
*,
valid_token: UserAPIKeyAuth,
request_data: dict,
route: str,
llm_model_list: Optional[list],
llm_router: Optional[Any],
) -> None:
"""
Key-level model allowlist and client fallbacks (same as standard auth).
Not included in common_checks — common_checks enforces team/user/project model access only.
"""
config = valid_token.config
if config != {}:
model_list = config.get("model_list", [])
new_model_list = model_list
verbose_proxy_logger.debug(
f"\n new llm router model list {new_model_list}"
)
elif (
isinstance(valid_token.models, list)
and "all-team-models" in valid_token.models
):
pass
else:
model = get_model_from_request(request_data, route)
fallback_models = cast(
Optional[List[ALL_FALLBACK_MODEL_VALUES]],
request_data.get("fallbacks", None),
)
if model is not None:
await can_key_call_model(
model=model,
llm_model_list=llm_model_list,
valid_token=valid_token,
llm_router=llm_router,
)
if fallback_models is not None:
for m in fallback_models:
await can_key_call_model(
model=m["model"] if isinstance(m, dict) else m,
llm_model_list=llm_model_list,
valid_token=valid_token,
llm_router=llm_router,
)
await is_valid_fallback_model(
model=m["model"] if isinstance(m, dict) else m,
llm_router=llm_router,
user_model=None,
)
async def _run_post_custom_auth_checks(
valid_token: UserAPIKeyAuth,
request: Request,
@ -1773,6 +1852,7 @@ async def _run_post_custom_auth_checks(
) -> UserAPIKeyAuth:
from litellm.proxy.proxy_server import (
general_settings,
llm_model_list,
llm_router,
model_max_budget_limiter,
prisma_client,
@ -1816,6 +1896,15 @@ async def _run_post_custom_auth_checks(
),
)
if general_settings.get("custom_auth_run_common_checks", False):
await _enforce_key_and_fallback_model_access(
valid_token=valid_token,
request_data=request_data,
route=route,
llm_model_list=llm_model_list,
llm_router=llm_router,
)
current_model = request_data.get("model", None)
# 3. Check key-level model_max_budget

11
litellm/proxy/common_request_processing.py
View file

@ -221,16 +221,21 @@ async def create_response(
f"Error consuming first chunk from generator: {e}"
)
# Fallback to a generic error stream
# Preserve status code from HTTPException (e.g., guardrail blocks)
error_status = getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR)
error_detail = getattr(e, "detail", "Error processing stream start")
if not isinstance(error_detail, str):
error_detail = str(error_detail)
async def error_gen_message() -> AsyncGenerator[str, None]:
yield f"data: {json.dumps({'error': {'message': 'Error processing stream start', 'code': status.HTTP_500_INTERNAL_SERVER_ERROR}})}\n\n"
yield f"data: {json.dumps({'error': {'message': error_detail, 'code': error_status}})}\n\n"
yield "data: [DONE]\n\n"
return StreamingResponse(
error_gen_message(),
media_type=media_type,
headers=headers,
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
status_code=error_status,
)
async def combined_generator() -> AsyncGenerator[str, None]:

2
litellm/proxy/example_config_yaml/websearch_interception_config.yaml
View file

@ -1,7 +1,7 @@
model_list:
- model_name: claude-3-5-sonnet
litellm_params:
model: bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
# Search tools configuration
search_tools:

23
litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py
View file

@ -14,6 +14,8 @@ from fastapi import HTTPException
if TYPE_CHECKING:
from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel
import json
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.caching import DualCache
@ -203,8 +205,8 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
response.text,
)
raise HTTPException(
status_code=response.status_code,
detail=f"Model Armor API error: {response.text}",
status_code=400,
detail=f"Model Armor API error (upstream {response.status_code}): {response.text}",
)
json_response = response.json()
@ -746,8 +748,21 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
yield chunk
return
except HTTPException:
raise
except HTTPException as e:
# Yield error as SSE event so create_response() detects it and
# returns a proper JSON error response with the correct status code.
# (Raising from a generator hits create_response's generic except → 500.)
detail = (
e.detail if isinstance(e.detail, dict) else {"message": str(e.detail)}
)
error_value = detail.get("error", detail)
if isinstance(error_value, dict):
error_obj = dict(error_value)
else:
error_obj = {"message": str(error_value)}
error_obj["code"] = str(e.status_code)
yield f"data: {json.dumps({'error': error_obj})}\n\n" # type: ignore[misc]
return
except Exception as e:
verbose_proxy_logger.error(
"Model Armor streaming error: %s", str(e), exc_info=True

10
litellm/proxy/hooks/proxy_track_cost_callback.py
View file

@ -112,6 +112,14 @@ class _ProxyDBLogger(CustomLogger):
_litellm_logging_obj, "litellm_trace_id", None
)
# Use the actual request start time from the logging object so that
# failed requests record the real duration instead of 0.
actual_start_time = datetime.now()
if _litellm_logging_obj is not None:
obj_start = getattr(_litellm_logging_obj, "start_time", None)
if obj_start is not None:
actual_start_time = obj_start
await proxy_logging_obj.db_spend_update_writer.update_database(
token=user_api_key_dict.api_key,
response_cost=0.0,
@ -120,7 +128,7 @@ class _ProxyDBLogger(CustomLogger):
team_id=user_api_key_dict.team_id,
kwargs=request_data,
completion_response=original_exception,
start_time=datetime.now(),
start_time=actual_start_time,
end_time=datetime.now(),
org_id=user_api_key_dict.org_id,
)

42
litellm/proxy/litellm_pre_call_utils.py
View file

@ -1472,17 +1472,19 @@ def _add_guardrails_from_key_or_team_metadata(
team_metadata: Optional[dict],
data: dict,
metadata_variable_name: str,
project_metadata: Optional[dict] = None,
) -> None:
"""
Helper add guardrails from key or team metadata to request data
Helper add guardrails from key, team, or project metadata to request data
Key guardrails are set first, then team guardrails are appended (without duplicates).
Key guardrails are set first, then team and project guardrails are appended (without duplicates).
Args:
key_metadata: The key metadata dictionary to check for guardrails
team_metadata: The team metadata dictionary to check for guardrails
data: The request data to update
metadata_variable_name: The name of the metadata field in data
project_metadata: The project metadata dictionary to check for guardrails
"""
from litellm.proxy.utils import _premium_user_check
@ -1508,6 +1510,15 @@ def _add_guardrails_from_key_or_team_metadata(
_premium_user_check()
combined_guardrails.update(team_metadata["guardrails"])
# Add project-level guardrails (set automatically handles duplicates)
if project_metadata and "guardrails" in project_metadata:
if (
isinstance(project_metadata["guardrails"], list)
and len(project_metadata["guardrails"]) > 0
):
_premium_user_check()
combined_guardrails.update(project_metadata["guardrails"])
# Set combined guardrails in metadata as list
if combined_guardrails:
data[metadata_variable_name]["guardrails"] = list(combined_guardrails)
@ -1518,12 +1529,13 @@ def _add_guardrails_from_policies_in_metadata(
team_metadata: Optional[dict],
data: dict,
metadata_variable_name: str,
project_metadata: Optional[dict] = None,
) -> None:
"""
Helper to resolve guardrails from policies attached to key/team metadata.
Helper to resolve guardrails from policies attached to key/team/project metadata.
This function:
1. Gets policy names from key and team metadata
1. Gets policy names from key, team, and project metadata
2. Resolves guardrails from those policies (including inheritance)
3. Adds resolved guardrails to request metadata
@ -1532,6 +1544,7 @@ def _add_guardrails_from_policies_in_metadata(
team_metadata: The team metadata dictionary to check for policies
data: The request data to update
metadata_variable_name: The name of the metadata field in data
project_metadata: The project metadata dictionary to check for policies
"""
from litellm._logging import verbose_proxy_logger
from litellm.proxy.policy_engine.policy_registry import get_policy_registry
@ -1560,6 +1573,15 @@ def _add_guardrails_from_policies_in_metadata(
_premium_user_check()
policy_names.update(team_metadata["policies"])
# Add project-level policies
if project_metadata and "policies" in project_metadata:
if (
isinstance(project_metadata["policies"], list)
and len(project_metadata["policies"]) > 0
):
_premium_user_check()
policy_names.update(project_metadata["policies"])
if not policy_names:
return
@ -1641,6 +1663,7 @@ async def move_guardrails_to_metadata(
# Early-out: skip all guardrails processing when nothing is configured
key_metadata = user_api_key_dict.metadata
team_metadata = user_api_key_dict.team_metadata
project_metadata = user_api_key_dict.project_metadata or {}
has_key_config = key_metadata and (
"guardrails" in key_metadata or "policies" in key_metadata
@ -1648,12 +1671,15 @@ async def move_guardrails_to_metadata(
has_team_config = team_metadata and (
"guardrails" in team_metadata or "policies" in team_metadata
)
has_project_config = project_metadata and (
"guardrails" in project_metadata or "policies" in project_metadata
)
has_request_config = (
"guardrails" in data or "guardrail_config" in data or "policies" in data
)
# Only check policy engine if no local config (avoid import + registry lookup)
if not (has_key_config or has_team_config or has_request_config):
if not (has_key_config or has_team_config or has_project_config or has_request_config):
from litellm.proxy.policy_engine.policy_registry import get_policy_registry
if not get_policy_registry().is_initialized():
@ -1661,20 +1687,22 @@ async def move_guardrails_to_metadata(
data.pop("policies", None)
return
# Check key-level guardrails
# Check key/team/project-level guardrails
_add_guardrails_from_key_or_team_metadata(
key_metadata=user_api_key_dict.metadata,
team_metadata=user_api_key_dict.team_metadata,
project_metadata=project_metadata,
data=data,
metadata_variable_name=_metadata_variable_name,
)
#########################################################################################
# Add guardrails from policies attached to key/team metadata
# Add guardrails from policies attached to key/team/project metadata
#########################################################################################
_add_guardrails_from_policies_in_metadata(
key_metadata=user_api_key_dict.metadata,
team_metadata=user_api_key_dict.team_metadata,
project_metadata=project_metadata,
data=data,
metadata_variable_name=_metadata_variable_name,
)

147
litellm/proxy/management_endpoints/key_management_endpoints.py
View file

@ -18,7 +18,7 @@ import re
import secrets
import traceback
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, List, Literal, Optional, Tuple, cast
from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, cast
import fastapi
import yaml
@ -487,6 +487,59 @@ async def validate_team_id_used_in_service_account_request(
return True
def _enforce_upperbound_key_params(
data: Union[GenerateKeyRequest, UpdateKeyRequest],
fill_defaults: bool = True,
) -> None:
"""
Enforce upperbound limits on key parameters.
For key generation (fill_defaults=True): fills None values with upperbound defaults.
For key update (fill_defaults=False): only validates explicitly provided values.
"""
if litellm.upperbound_key_generate_params is None:
return
for elem in data:
key, value = elem
upperbound_value = getattr(
litellm.upperbound_key_generate_params, key, None
)
if upperbound_value is not None:
if value is None:
if fill_defaults:
setattr(data, key, upperbound_value)
else:
if key in [
"max_budget",
"max_parallel_requests",
"tpm_limit",
"rpm_limit",
]:
if value > upperbound_value:
raise HTTPException(
status_code=400,
detail={
"error": f"{key} is over max limit set in config - user_value={value}; max_value={upperbound_value}"
},
)
elif key in ["budget_duration", "duration"]:
upperbound_duration = duration_in_seconds(
duration=upperbound_value
)
if value == "-1":
user_duration = float("inf")
else:
user_duration = duration_in_seconds(duration=value)
if user_duration > upperbound_duration:
raise HTTPException(
status_code=400,
detail={
"error": f"{key} is over max limit set in config - user_value={value}; max_value={upperbound_value}"
},
)
async def _common_key_generation_helper( # noqa: PLR0915
data: GenerateKeyRequest,
user_api_key_dict: UserAPIKeyAuth,
@ -537,49 +590,8 @@ async def _common_key_generation_helper( # noqa: PLR0915
elif key == "metadata" and value == {}:
setattr(data, key, litellm.default_key_generate_params.get(key, {}))
# check if user set default key/generate params on config.yaml
if litellm.upperbound_key_generate_params is not None:
for elem in data:
key, value = elem
upperbound_value = getattr(
litellm.upperbound_key_generate_params, key, None
)
if upperbound_value is not None:
if value is None:
# Use the upperbound value if user didn't provide a value
setattr(data, key, upperbound_value)
else:
# Compare with upperbound for numeric fields
if key in [
"max_budget",
"max_parallel_requests",
"tpm_limit",
"rpm_limit",
]:
if value > upperbound_value:
raise HTTPException(
status_code=400,
detail={
"error": f"{key} is over max limit set in config - user_value={value}; max_value={upperbound_value}"
},
)
# Compare durations
elif key in ["budget_duration", "duration"]:
upperbound_duration = duration_in_seconds(
duration=upperbound_value
)
# Handle special case where duration is None or "-1" (never expires)
if value is None or value == "-1":
user_duration = float("inf") # Infinite duration
else:
user_duration = duration_in_seconds(duration=value)
if user_duration > upperbound_duration:
raise HTTPException(
status_code=400,
detail={
"error": f"{key} is over max limit set in config - user_value={value}; max_value={upperbound_value}"
},
)
# check if user set upperbound key/generate params on config.yaml
_enforce_upperbound_key_params(data, fill_defaults=True)
# APPLY ENTERPRISE KEY MANAGEMENT PARAMS
try:
@ -942,9 +954,9 @@ async def _check_team_key_limits(
where={"team_id": team_table.team_id},
)
# Exclude the key being updated to avoid double-counting its limits.
# key.token is the SHA-256 hash stored in DB; data.key is the raw key string.
# data.key may be a raw key (sk-...) or a pre-hashed token_id.
if isinstance(data, UpdateKeyRequest):
hashed_key = hash_token(data.key)
hashed_key = _hash_token_if_needed(data.key)
keys = [key for key in keys if key.token != hashed_key]
check_team_key_model_specific_limits(
keys=keys,
@ -1101,9 +1113,9 @@ async def _check_org_key_limits(
where={"organization_id": org_table.organization_id},
)
# Exclude the key being updated to avoid double-counting its limits.
# key.token is the SHA-256 hash stored in DB; data.key is the raw key string.
# data.key may be a raw key (sk-...) or a pre-hashed token_id.
if isinstance(data, UpdateKeyRequest):
hashed_key = hash_token(data.key)
hashed_key = _hash_token_if_needed(data.key)
keys = [key for key in keys if key.token != hashed_key]
check_org_key_model_specific_limits(
keys=keys,
@ -1687,6 +1699,7 @@ async def _process_single_key_update(
user_api_key_cache: DualCache,
proxy_logging_obj: Any,
llm_router: Optional[Router],
user_custom_key_update: Optional[Callable] = None,
) -> Dict[str, Any]:
"""
Process a single key update with all validations and checks.
@ -1737,6 +1750,22 @@ async def _process_single_key_update(
tags=key_update_item.tags,
)
# Custom key update hook
if user_custom_key_update is not None:
if inspect.iscoroutinefunction(user_custom_key_update):
result = await user_custom_key_update(update_key_request)
else:
raise ValueError("user_custom_key_update must be a coroutine")
decision = result.get("decision", True)
message = result.get("message", "Authentication Failed - Custom Auth Rule")
if not decision:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN, detail=message
)
# Enforce upperbound key params on update (don't fill defaults)
_enforce_upperbound_key_params(update_key_request, fill_defaults=False)
# Get team object and check team limits if team_id is provided
team_obj: Optional[LiteLLM_TeamTableCachedObj] = None
if update_key_request.team_id is not None:
@ -2014,7 +2043,7 @@ async def _validate_update_key_data(
"/key/update", tags=["key management"], dependencies=[Depends(user_api_key_auth)]
)
@management_endpoint_wrapper
async def update_key_fn(
async def update_key_fn( # noqa: PLR0915
request: Request,
data: UpdateKeyRequest,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
@ -2095,6 +2124,7 @@ async def update_key_fn(
prisma_client,
proxy_logging_obj,
user_api_key_cache,
user_custom_key_update,
)
try:
@ -2126,6 +2156,21 @@ async def update_key_fn(
user_api_key_cache=user_api_key_cache,
)
# Custom key update hook
if user_custom_key_update is not None:
if inspect.iscoroutinefunction(user_custom_key_update):
result = await user_custom_key_update(data)
else:
raise ValueError("user_custom_key_update must be a coroutine")
decision = result.get("decision", True)
message = result.get("message", "Authentication Failed - Custom Auth Rule")
if not decision:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN, detail=message
)
# Enforce upperbound key params on update (don't fill defaults)
_enforce_upperbound_key_params(data, fill_defaults=False)
non_default_values = await prepare_key_update_data(
data=data, existing_key_row=existing_key_row
)
@ -2157,7 +2202,7 @@ async def update_key_fn(
# Delete - key from cache, since it's been updated!
# key updated - a new model could have been added to this key. it should not block requests after this is done
await _delete_cache_key_object(
hashed_token=hash_token(key),
hashed_token=_hash_token_if_needed(key),
user_api_key_cache=user_api_key_cache,
proxy_logging_obj=proxy_logging_obj,
)
@ -2261,6 +2306,7 @@ async def bulk_update_keys(
prisma_client,
proxy_logging_obj,
user_api_key_cache,
user_custom_key_update,
)
if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN.value:
@ -2304,6 +2350,7 @@ async def bulk_update_keys(
user_api_key_cache=user_api_key_cache,
proxy_logging_obj=proxy_logging_obj,
llm_router=llm_router,
user_custom_key_update=user_custom_key_update,
)
successful_updates.append(

447
litellm/proxy/prompts/prompt_endpoints.py
View file

@ -192,19 +192,22 @@ def get_latest_prompt_versions(prompts: List[PromptSpec]) -> List[PromptSpec]:
return list(latest_prompts.values())
async def get_next_version_for_prompt(prisma_client, prompt_id: str) -> int:
async def get_next_version_for_prompt(
prisma_client, prompt_id: str, environment: str = "development"
) -> int:
"""
Get the next version number for a prompt.
Get the next version number for a prompt in a specific environment.
Args:
prisma_client: Prisma database client
prompt_id: Base prompt ID
environment: The environment to check versions for
Returns:
Next version number (1 if no versions exist, max_version + 1 otherwise)
"""
existing_prompts = await prisma_client.db.litellm_prompttable.find_many(
where={"prompt_id": prompt_id}
where={"prompt_id": prompt_id, "environment": environment}
)
if existing_prompts:
@ -231,6 +234,8 @@ def create_versioned_prompt_spec(db_prompt) -> PromptSpec:
prompt_dict = db_prompt.model_dump()
base_prompt_id = prompt_dict["prompt_id"]
version = prompt_dict.get("version", 1)
environment = prompt_dict.get("environment", "development")
created_by = prompt_dict.get("created_by")
# Parse litellm_params
litellm_params_data = prompt_dict.get("litellm_params")
@ -256,6 +261,8 @@ def create_versioned_prompt_spec(db_prompt) -> PromptSpec:
prompt_info=prompt_info,
created_at=prompt_dict.get("created_at"),
updated_at=prompt_dict.get("updated_at"),
environment=environment,
created_by=created_by,
)
@ -277,6 +284,7 @@ class PatchPromptRequest(BaseModel):
response_model=ListPromptsResponse,
)
async def list_prompts(
environment: Optional[str] = None,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
):
"""
@ -318,23 +326,26 @@ async def list_prompts(
if key_metadata is not None:
prompts = cast(Optional[List[str]], key_metadata.get("prompts", None))
if prompts is not None:
all_prompts = [
IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS[prompt_id]
for prompt_id in prompts
if prompt_id in IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS
]
if environment:
all_prompts = [p for p in all_prompts if p.environment == environment]
prompt_list = []
for prompt_id in prompts:
if prompt_id in IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS:
original_prompt = IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS[
prompt_id
]
# Create a copy with base prompt_id (without version suffix)
prompt_copy = PromptSpec(
prompt_id=get_base_prompt_id(
prompt_id=original_prompt.prompt_id
),
litellm_params=original_prompt.litellm_params,
prompt_info=original_prompt.prompt_info,
created_at=original_prompt.created_at,
updated_at=original_prompt.updated_at,
)
prompt_list.append(prompt_copy)
for original_prompt in all_prompts:
# Create a copy with base prompt_id (without version suffix)
prompt_copy = PromptSpec(
prompt_id=get_base_prompt_id(prompt_id=original_prompt.prompt_id),
litellm_params=original_prompt.litellm_params,
prompt_info=original_prompt.prompt_info,
created_at=original_prompt.created_at,
updated_at=original_prompt.updated_at,
environment=original_prompt.environment,
created_by=original_prompt.created_by,
)
prompt_list.append(prompt_copy)
return ListPromptsResponse(prompts=prompt_list)
# check if user is proxy admin - show all prompts
if user_api_key_dict.user_role is not None and (
@ -343,6 +354,8 @@ async def list_prompts(
):
# Get all prompts and filter to show only the latest version of each
all_prompts = list(IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS.values())
if environment:
all_prompts = [p for p in all_prompts if p.environment == environment]
latest_prompts = get_latest_prompt_versions(prompts=all_prompts)
# Create copies with base prompt_id (without version suffix) for display
prompts_for_display = []
@ -353,6 +366,8 @@ async def list_prompts(
prompt_info=original_prompt.prompt_info,
created_at=original_prompt.created_at,
updated_at=original_prompt.updated_at,
environment=original_prompt.environment,
created_by=original_prompt.created_by,
)
prompts_for_display.append(prompt_copy)
return ListPromptsResponse(prompts=prompts_for_display)
@ -368,6 +383,7 @@ async def list_prompts(
)
async def get_prompt_versions(
prompt_id: str,
environment: Optional[str] = None,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
):
"""
@ -404,6 +420,7 @@ async def get_prompt_versions(
```
"""
from litellm.proxy.prompts.prompt_registry import IN_MEMORY_PROMPT_REGISTRY
from litellm.proxy.proxy_server import prisma_client
# Only allow proxy admins to view version history
if user_api_key_dict.user_role is None or (
@ -414,49 +431,112 @@ async def get_prompt_versions(
status_code=403, detail="Only proxy admins can view prompt versions"
)
# Strip version suffix if provided (e.g., "jack_success.v1" -> "jack_success")
base_prompt_id = get_base_prompt_id(prompt_id=prompt_id)
# Get all prompts and filter by base_prompt_id
all_prompts = list(IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS.values())
prompt_versions = [
prompt
for prompt in all_prompts
if get_base_prompt_id(prompt_id=prompt.prompt_id) == base_prompt_id
]
# Query DB for versions
versioned_prompts = []
if prisma_client is not None:
where_clause: Dict[str, Any] = {"prompt_id": base_prompt_id}
if environment:
where_clause["environment"] = environment
db_prompts = await prisma_client.db.litellm_prompttable.find_many(
where=where_clause,
order={"version": "desc"},
)
for db_prompt in db_prompts:
spec = create_versioned_prompt_spec(db_prompt=db_prompt)
versioned_prompts.append(
PromptSpec(
prompt_id=base_prompt_id,
litellm_params=spec.litellm_params,
prompt_info=spec.prompt_info,
created_at=spec.created_at,
updated_at=spec.updated_at,
version=get_version_number(prompt_id=spec.prompt_id),
environment=spec.environment,
created_by=spec.created_by,
)
)
else:
# Fallback: in-memory registry (no DB)
all_prompts = list(IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS.values())
prompt_versions = [
prompt
for prompt in all_prompts
if get_base_prompt_id(prompt_id=prompt.prompt_id) == base_prompt_id
and (environment is None or prompt.environment == environment)
]
for prompt in prompt_versions:
version_number = get_version_number(prompt_id=prompt.prompt_id)
versioned_prompts.append(
PromptSpec(
prompt_id=base_prompt_id,
litellm_params=prompt.litellm_params,
prompt_info=prompt.prompt_info,
created_at=prompt.created_at,
updated_at=prompt.updated_at,
version=version_number,
environment=prompt.environment,
created_by=prompt.created_by,
)
)
versioned_prompts.sort(key=lambda p: p.version or 1, reverse=True)
if not prompt_versions:
if not versioned_prompts:
raise HTTPException(
status_code=404, detail=f"No versions found for prompt ID {base_prompt_id}"
)
# Create response with explicit version field for each prompt
versioned_prompts = []
for prompt in prompt_versions:
# Extract version number from the root prompt_id which has version suffix
# (e.g., "jack-sparrow.v3" -> 3)
version_number = get_version_number(prompt_id=prompt.prompt_id)
# Strip version from prompt_id for clean display
base_prompt_id = get_base_prompt_id(prompt_id=prompt.prompt_id)
# Create a copy with explicit version field and clean prompt_id
versioned_prompt = PromptSpec(
prompt_id=base_prompt_id, # Clean ID without version (e.g., "jack-sparrow")
litellm_params=prompt.litellm_params,
prompt_info=prompt.prompt_info,
created_at=prompt.created_at,
updated_at=prompt.updated_at,
version=version_number, # Explicit version field (e.g., 3)
)
versioned_prompts.append(versioned_prompt)
# Sort by version number (descending - newest first)
versioned_prompts.sort(key=lambda p: p.version or 1, reverse=True)
return ListPromptsResponse(prompts=versioned_prompts)
def _get_prompt_template(
prompt_spec: PromptSpec, base_prompt_id: str
) -> Optional[PromptTemplateBase]:
"""Resolve the raw prompt template from dotprompt content or the in-memory registry."""
from litellm.proxy.prompts.prompt_registry import IN_MEMORY_PROMPT_REGISTRY
try:
dotprompt_content = prompt_spec.litellm_params.dotprompt_content
if dotprompt_content:
from litellm.integrations.dotprompt import (
_get_prompt_data_from_dotprompt_content,
)
parsed = _get_prompt_data_from_dotprompt_content(dotprompt_content)
if parsed:
return PromptTemplateBase(
litellm_prompt_id=base_prompt_id,
content=parsed.get("content", ""),
metadata=parsed.get("metadata"),
)
else:
prompt_callback = IN_MEMORY_PROMPT_REGISTRY.get_prompt_callback_by_id(
prompt_spec.prompt_id
)
if prompt_callback is not None:
integration_name = prompt_callback.integration_name
if integration_name == "dotprompt":
from litellm.integrations.dotprompt.dotprompt_manager import (
DotpromptManager,
)
if isinstance(prompt_callback, DotpromptManager):
template = (
prompt_callback.prompt_manager.get_all_prompts_as_json()
)
if template is not None and len(template) == 1:
template_id = list(template.keys())[0]
return PromptTemplateBase(
litellm_prompt_id=template_id,
content=template[template_id]["content"],
metadata=template[template_id]["metadata"],
)
except Exception:
pass
return None
@router.get(
"/prompts/{prompt_id}",
tags=["Prompt Management"],
@ -471,6 +551,7 @@ async def get_prompt_versions(
)
async def get_prompt_info(
prompt_id: str,
environment: Optional[str] = None,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
):
"""
@ -503,6 +584,7 @@ async def get_prompt_info(
```
"""
from litellm.proxy.prompts.prompt_registry import IN_MEMORY_PROMPT_REGISTRY
from litellm.proxy.proxy_server import prisma_client
## CHECK IF USER HAS ACCESS TO PROMPT
prompts: Optional[List[str]] = None
@ -523,68 +605,80 @@ async def get_prompt_info(
detail=f"You are not authorized to access this prompt. Your role - {user_api_key_dict.user_role}, Your key's prompts - {prompts}",
)
# Try to get prompt directly first
prompt_spec = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(prompt_id)
base_prompt_id = get_base_prompt_id(prompt_id=prompt_id)
# If not found, try to find the latest version
if prompt_spec is None:
latest_prompt_id = get_latest_version_prompt_id(
prompt_id=prompt_id,
all_prompt_ids=IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS,
# Query all environments this prompt exists in (lightweight: distinct on environment)
all_environments: List[str] = []
if prisma_client is not None:
all_prompt_rows = await prisma_client.db.litellm_prompttable.find_many(
where={"prompt_id": base_prompt_id},
distinct=["environment"],
)
prompt_spec = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(latest_prompt_id)
all_environments = sorted(
set(row.environment for row in all_prompt_rows if row.environment)
)
# If environment is specified, find the version in that environment from DB
# If prompt_id has a version suffix (e.g., "testprompt.v2"), fetch that specific version
# Otherwise fetch the latest version in that environment
prompt_spec = None
requested_version = (
get_version_number(prompt_id=prompt_id) if prompt_id != base_prompt_id else None
)
if environment and prisma_client is not None:
where_clause: Dict[str, Any] = {
"prompt_id": base_prompt_id,
"environment": environment,
}
if requested_version is not None:
where_clause["version"] = requested_version
env_prompts = await prisma_client.db.litellm_prompttable.find_many(
where=where_clause,
order={"version": "desc"},
take=1,
)
if env_prompts:
prompt_spec = create_versioned_prompt_spec(db_prompt=env_prompts[0])
# Fallback: use in-memory registry (no environment filter)
if prompt_spec is None and environment is None:
prompt_spec = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(prompt_id)
if prompt_spec is None:
latest_prompt_id = get_latest_version_prompt_id(
prompt_id=prompt_id,
all_prompt_ids=IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS,
)
prompt_spec = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(latest_prompt_id)
if prompt_spec is None:
raise HTTPException(status_code=400, detail=f"Prompt {prompt_id} not found")
raise HTTPException(
status_code=400,
detail=f"Prompt {prompt_id} not found"
+ (f" in environment {environment}" if environment else ""),
)
# Extract version number from the prompt_id
version_number = get_version_number(prompt_id=prompt_spec.prompt_id)
# Create a copy of the prompt spec with the base prompt ID (stripped of version)
# and explicit version field for consistency with list_prompts and versions endpoints
prompt_spec_response = PromptSpec(
prompt_id=get_base_prompt_id(prompt_id=prompt_spec.prompt_id),
litellm_params=prompt_spec.litellm_params, # This preserves the versioned ID
litellm_params=prompt_spec.litellm_params,
prompt_info=prompt_spec.prompt_info,
created_at=prompt_spec.created_at,
updated_at=prompt_spec.updated_at,
version=version_number, # Explicit version field
version=version_number,
environment=prompt_spec.environment,
created_by=prompt_spec.created_by,
)
# Get prompt content from the callback
prompt_template: Optional[PromptTemplateBase] = None
try:
prompt_callback = IN_MEMORY_PROMPT_REGISTRY.get_prompt_callback_by_id(
prompt_spec.prompt_id
)
if prompt_callback is not None:
# Extract content based on integration type
integration_name = prompt_callback.integration_name
# Get prompt content
prompt_template = _get_prompt_template(prompt_spec, base_prompt_id)
if integration_name == "dotprompt":
# For dotprompt integration, get content from the prompt manager
from litellm.integrations.dotprompt.dotprompt_manager import (
DotpromptManager,
)
if isinstance(prompt_callback, DotpromptManager):
template = prompt_callback.prompt_manager.get_all_prompts_as_json()
if template is not None and len(template) == 1:
template_id = list(template.keys())[0]
prompt_template = PromptTemplateBase(
litellm_prompt_id=template_id, # id sent to prompt management tool
content=template[template_id]["content"],
metadata=template[template_id]["metadata"],
)
except Exception:
# If content extraction fails, continue without content
pass
# Create response with content
return PromptInfoResponse(
prompt_spec=prompt_spec_response,
raw_prompt_template=prompt_template,
environments=all_environments,
)
@ -641,9 +735,18 @@ async def create_prompt(
)
try:
# Extract environment from request
environment = (
request.prompt_info.environment
if request.prompt_info and request.prompt_info.environment
else "development"
)
# Get next version number
new_version = await get_next_version_for_prompt(
prisma_client=prisma_client, prompt_id=request.prompt_id
prisma_client=prisma_client,
prompt_id=request.prompt_id,
environment=environment,
)
# Store prompt in db with version
@ -651,6 +754,8 @@ async def create_prompt(
data={
"prompt_id": request.prompt_id,
"version": new_version,
"environment": environment,
"created_by": user_api_key_dict.user_id,
"litellm_params": request.litellm_params.model_dump_json(),
"prompt_info": (
request.prompt_info.model_dump_json()
@ -733,14 +838,22 @@ async def update_prompt(
# Strip version suffix from prompt_id if present (e.g., "jack_success.v1" -> "jack_success")
base_prompt_id = get_base_prompt_id(prompt_id=prompt_id)
# Check if any version exists
# Extract environment from request
environment = (
request.prompt_info.environment
if request.prompt_info and request.prompt_info.environment
else "development"
)
# Check if any version of this prompt exists (in any environment)
existing_prompts = await prisma_client.db.litellm_prompttable.find_many(
where={"prompt_id": base_prompt_id}
)
if not existing_prompts:
raise HTTPException(
status_code=404, detail=f"Prompt with ID {base_prompt_id} not found"
status_code=404,
detail=f"Prompt with ID {base_prompt_id} not found",
)
# Check if it's a config prompt
@ -756,7 +869,9 @@ async def update_prompt(
# Get next version number (UPDATE creates a new version)
new_version = await get_next_version_for_prompt(
prisma_client=prisma_client, prompt_id=base_prompt_id
prisma_client=prisma_client,
prompt_id=base_prompt_id,
environment=environment,
)
# Store new version in db
@ -764,6 +879,8 @@ async def update_prompt(
data={
"prompt_id": base_prompt_id,
"version": new_version,
"environment": environment,
"created_by": user_api_key_dict.user_id,
"litellm_params": request.litellm_params.model_dump_json(),
"prompt_info": (
request.prompt_info.model_dump_json()
@ -800,6 +917,7 @@ async def update_prompt(
)
async def delete_prompt(
prompt_id: str,
environment: Optional[str] = None,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
):
"""
@ -867,15 +985,31 @@ async def delete_prompt(
# Get the base prompt ID (without version suffix) for database deletion
base_prompt_id = get_base_prompt_id(prompt_id=prompt_id)
# Delete all versions of the prompt from the database
await prisma_client.db.litellm_prompttable.delete_many(
where={"prompt_id": base_prompt_id}
)
# Build delete filter; scope to environment if provided
delete_where: Dict[str, Any] = {"prompt_id": base_prompt_id}
if environment:
delete_where["environment"] = environment
# Remove all versions of the prompt from memory
IN_MEMORY_PROMPT_REGISTRY.delete_prompts_by_base_id(base_prompt_id)
# Delete versions from the database (scoped to environment if provided)
await prisma_client.db.litellm_prompttable.delete_many(where=delete_where)
return {"message": f"Prompt {base_prompt_id} deleted successfully"}
# Remove matching prompts from memory — scope to environment if provided
if environment:
prompts_to_delete = [
pid
for pid, prompt in IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS.items()
if get_base_prompt_id(prompt_id=pid) == base_prompt_id
and prompt.environment == environment
]
for pid in prompts_to_delete:
del IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS[pid]
if pid in IN_MEMORY_PROMPT_REGISTRY.prompt_id_to_custom_prompt:
del IN_MEMORY_PROMPT_REGISTRY.prompt_id_to_custom_prompt[pid]
else:
IN_MEMORY_PROMPT_REGISTRY.delete_prompts_by_base_id(base_prompt_id)
env_msg = f" from {environment}" if environment else ""
return {"message": f"Prompt {base_prompt_id} deleted successfully{env_msg}"}
except HTTPException as e:
raise e
@ -884,6 +1018,22 @@ async def delete_prompt(
raise HTTPException(status_code=500, detail=str(e))
def _reload_prompt_in_registry(
registry: Any, versioned_id: str, updated_prompt_spec: PromptSpec
) -> PromptSpec:
"""Remove stale entry and re-initialize the prompt in the in-memory registry."""
if versioned_id in registry.IN_MEMORY_PROMPTS:
del registry.IN_MEMORY_PROMPTS[versioned_id]
if versioned_id in registry.prompt_id_to_custom_prompt:
del registry.prompt_id_to_custom_prompt[versioned_id]
initialized = registry.initialize_prompt(
prompt=updated_prompt_spec, config_file_path=None
)
if initialized is None:
raise HTTPException(status_code=500, detail="Failed to patch prompt")
return initialized
@router.patch(
"/prompts/{prompt_id}",
tags=["Prompt Management"],
@ -892,6 +1042,7 @@ async def delete_prompt(
async def patch_prompt(
prompt_id: str,
request: PatchPromptRequest,
environment: Optional[str] = None,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
):
"""
@ -935,61 +1086,93 @@ async def patch_prompt(
)
try:
# Check if prompt exists and get current data
existing_prompt = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(prompt_id)
if existing_prompt is None:
# Resolve the target row: find the latest version in the given environment
base_prompt_id = get_base_prompt_id(prompt_id=prompt_id)
env = environment or "development"
requested_version = (
get_version_number(prompt_id=prompt_id)
if prompt_id != base_prompt_id
else None
)
# Build query to find the exact row by composite unique key
find_where: Dict[str, Any] = {
"prompt_id": base_prompt_id,
"environment": env,
}
if requested_version is not None:
find_where["version"] = requested_version
db_rows = await prisma_client.db.litellm_prompttable.find_many(
where=find_where,
order={"version": "desc"},
take=1,
)
if not db_rows:
raise HTTPException(
status_code=404, detail=f"Prompt with ID {prompt_id} not found"
status_code=404,
detail=f"Prompt with ID {base_prompt_id} not found in environment {env}",
)
if existing_prompt.prompt_info.prompt_type == "config":
target_row = db_rows[0]
# Check if prompt exists in memory
versioned_id = f"{base_prompt_id}.v{target_row.version}"
existing_prompt = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(versioned_id)
if existing_prompt and existing_prompt.prompt_info.prompt_type == "config":
raise HTTPException(
status_code=400,
detail="Cannot update config prompts.",
)
# Use existing prompt from memory or build from DB row for field merging
if existing_prompt:
current_litellm_params = existing_prompt.litellm_params
current_prompt_info = existing_prompt.prompt_info
else:
current_spec = create_versioned_prompt_spec(db_prompt=target_row)
current_litellm_params = current_spec.litellm_params
current_prompt_info = current_spec.prompt_info
# Update fields if provided
updated_litellm_params = (
request.litellm_params
if request.litellm_params is not None
else existing_prompt.litellm_params
else current_litellm_params
)
updated_prompt_info = (
request.prompt_info
if request.prompt_info is not None
else existing_prompt.prompt_info
else current_prompt_info
)
# Ensure we have valid litellm_params
if updated_litellm_params is None:
raise HTTPException(status_code=400, detail="litellm_params cannot be None")
# Create updated prompt spec - cast to satisfy typing
# Build update data dict
update_data: Dict[str, Any] = {
"litellm_params": updated_litellm_params.model_dump_json(),
"prompt_info": updated_prompt_info.model_dump_json(),
}
if user_api_key_dict.user_id:
update_data["created_by"] = user_api_key_dict.user_id
# Update by primary key (id) to target exactly one row
updated_prompt_db_entry = await prisma_client.db.litellm_prompttable.update(
where={"prompt_id": prompt_id},
data={
"litellm_params": updated_litellm_params.model_dump_json(),
"prompt_info": updated_prompt_info.model_dump_json(),
},
where={"id": target_row.id},
data=update_data,
)
updated_prompt_spec = PromptSpec(**updated_prompt_db_entry.model_dump())
# Remove the old prompt from memory
del IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS[prompt_id]
if prompt_id in IN_MEMORY_PROMPT_REGISTRY.prompt_id_to_custom_prompt:
del IN_MEMORY_PROMPT_REGISTRY.prompt_id_to_custom_prompt[prompt_id]
# Initialize the updated prompt
initialized_prompt = IN_MEMORY_PROMPT_REGISTRY.initialize_prompt(
prompt=updated_prompt_spec, config_file_path=None
updated_prompt_spec = create_versioned_prompt_spec(
db_prompt=updated_prompt_db_entry
)
if initialized_prompt is None:
raise HTTPException(status_code=500, detail="Failed to patch prompt")
return initialized_prompt
return _reload_prompt_in_registry(
IN_MEMORY_PROMPT_REGISTRY, versioned_id, updated_prompt_spec
)
except HTTPException as e:
raise e

14
litellm/proxy/proxy_server.py
View file

@ -690,7 +690,7 @@ _description = (
def cleanup_router_config_variables():
global master_key, user_config_file_path, otel_logging, user_custom_auth, user_custom_auth_path, user_custom_key_generate, user_custom_sso, user_custom_ui_sso_sign_in_handler, use_background_health_checks, use_shared_health_check, health_check_interval, health_check_concurrency, prisma_client
global master_key, user_config_file_path, otel_logging, user_custom_auth, user_custom_auth_path, user_custom_key_generate, user_custom_key_update, user_custom_sso, user_custom_ui_sso_sign_in_handler, use_background_health_checks, use_shared_health_check, health_check_interval, health_check_concurrency, prisma_client
# Set all variables to None
master_key = None
@ -699,6 +699,7 @@ def cleanup_router_config_variables():
user_custom_auth = None
user_custom_auth_path = None
user_custom_key_generate = None
user_custom_key_update = None
user_custom_sso = None
user_custom_ui_sso_sign_in_handler = None
use_background_health_checks = None
@ -709,7 +710,7 @@ def cleanup_router_config_variables():
async def proxy_shutdown_event():
global prisma_client, master_key, user_custom_auth, user_custom_key_generate
global prisma_client, master_key, user_custom_auth, user_custom_key_generate, user_custom_key_update
verbose_proxy_logger.info("Shutting down LiteLLM Proxy Server")
if prisma_client:
verbose_proxy_logger.debug("Disconnecting from Prisma")
@ -1564,6 +1565,7 @@ user_custom_key_generate = None
# Tests that need to reset it can patch 'litellm.proxy.proxy_server._pkce_no_redis_warning_emitted'.
_pkce_no_redis_warning_emitted: bool = False
_cp_no_redis_warning_emitted: bool = False
user_custom_key_update = None
user_custom_sso = None
user_custom_ui_sso_sign_in_handler = None
use_background_health_checks = None
@ -2935,7 +2937,7 @@ class ProxyConfig:
"""
Load config values into proxy global state
"""
global master_key, user_config_file_path, otel_logging, user_custom_auth, user_custom_auth_path, user_custom_key_generate, user_custom_sso, user_custom_ui_sso_sign_in_handler, use_background_health_checks, use_shared_health_check, health_check_interval, health_check_concurrency, use_queue, proxy_budget_rescheduler_max_time, proxy_budget_rescheduler_min_time, ui_access_mode, litellm_master_key_hash, proxy_batch_write_at, disable_spend_logs, prompt_injection_detection_obj, redis_usage_cache, store_model_in_db, premium_user, open_telemetry_logger, health_check_details, proxy_batch_polling_interval, config_passthrough_endpoints
global master_key, user_config_file_path, otel_logging, user_custom_auth, user_custom_auth_path, user_custom_key_generate, user_custom_key_update, user_custom_sso, user_custom_ui_sso_sign_in_handler, use_background_health_checks, use_shared_health_check, health_check_interval, health_check_concurrency, use_queue, proxy_budget_rescheduler_max_time, proxy_budget_rescheduler_min_time, ui_access_mode, litellm_master_key_hash, proxy_batch_write_at, disable_spend_logs, prompt_injection_detection_obj, redis_usage_cache, store_model_in_db, premium_user, open_telemetry_logger, health_check_details, proxy_batch_polling_interval, config_passthrough_endpoints
config: dict = await self.get_config(config_file_path=config_file_path)
@ -3356,6 +3358,12 @@ class ProxyConfig:
value=custom_key_generate, config_file_path=config_file_path
)
custom_key_update = general_settings.get("custom_key_update", None)
if custom_key_update is not None:
user_custom_key_update = get_instance_fn(
value=custom_key_update, config_file_path=config_file_path
)
custom_sso = general_settings.get("custom_sso", None)
if custom_sso is not None:
user_custom_sso = get_instance_fn(

5
litellm/proxy/schema.prisma
View file

@ -1002,12 +1002,15 @@ model LiteLLM_PromptTable {
id String @id @default(uuid())
prompt_id String
version Int @default(1)
environment String @default("development")
created_by String?
litellm_params Json
prompt_info Json?
created_at DateTime @default(now())
updated_at DateTime @updatedAt
@@unique([prompt_id, version])
@@unique([prompt_id, version, environment])
@@index([prompt_id, environment])
@@index([prompt_id])
}

12
litellm/router.py
View file

@ -7808,8 +7808,8 @@ class Router:
max_tokens=None,
max_input_tokens=None,
max_output_tokens=None,
input_cost_per_token=0,
output_cost_per_token=0,
input_cost_per_token=None,
output_cost_per_token=None,
litellm_provider=llm_provider,
mode=mode,
supported_openai_params=supported_openai_params,
@ -7856,16 +7856,16 @@ class Router:
model_group_info.max_output_tokens = model_info["max_output_tokens"]
if model_info.get("input_cost_per_token", None) is not None and (
model_group_info.input_cost_per_token is None
or model_info["input_cost_per_token"]
> model_group_info.input_cost_per_token
or (model_info["input_cost_per_token"] or 0.0)
> (model_group_info.input_cost_per_token or 0.0)
):
model_group_info.input_cost_per_token = model_info[
"input_cost_per_token"
]
if model_info.get("output_cost_per_token", None) is not None and (
model_group_info.output_cost_per_token is None
or model_info["output_cost_per_token"]
> model_group_info.output_cost_per_token
or (model_info["output_cost_per_token"] or 0.0)
> (model_group_info.output_cost_per_token or 0.0)
):
model_group_info.output_cost_per_token = model_info[
"output_cost_per_token"

2
litellm/setup_wizard.py
View file

@ -86,7 +86,7 @@ PROVIDERS: List[Dict] = [
"env_key": "AWS_ACCESS_KEY_ID",
"key_hint": "AKIA...",
"test_model": None, # multi-key auth — skip validation
"models": ["bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0"],
"models": ["bedrock/anthropic.claude-haiku-4-5-20251001-v1:0"],
"extra_keys": ["AWS_SECRET_ACCESS_KEY", "AWS_REGION_NAME"],
"extra_hints": ["your-secret-key", "us-east-1"],
},

6
litellm/types/prompts/init_prompts.py
View file

@ -17,6 +17,7 @@ class SupportedPromptIntegrations(str, Enum):
class PromptInfo(BaseModel):
prompt_type: Literal["config", "db"]
environment: Optional[str] = "development"
model_config = ConfigDict(extra="allow", protected_namespaces=())
@ -48,6 +49,8 @@ class PromptSpec(BaseModel):
created_at: Optional[datetime] = None
updated_at: Optional[datetime] = None
version: Optional[int] = None # Version number for version history
environment: Optional[str] = "development"
created_by: Optional[str] = None
def __init__(self, **data):
if "prompt_info" not in data:
@ -70,6 +73,9 @@ class PromptTemplateBase(BaseModel):
class PromptInfoResponse(BaseModel):
prompt_spec: PromptSpec
raw_prompt_template: Optional[PromptTemplateBase] = None
environments: Optional[
List[str]
] = None # All environments this prompt is deployed to
class ListPromptsResponse(BaseModel):

4
litellm/types/utils.py
View file

@ -169,7 +169,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
max_tokens: Required[Optional[int]]
max_input_tokens: Required[Optional[int]]
max_output_tokens: Required[Optional[int]]
input_cost_per_token: Required[float]
input_cost_per_token: Required[Optional[float]]
input_cost_per_token_flex: Optional[float] # OpenAI flex service tier pricing
input_cost_per_token_priority: Optional[
float
@ -206,7 +206,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
input_cost_per_second: Optional[float] # for OpenAI Speech models
input_cost_per_token_batches: Optional[float]
output_cost_per_token_batches: Optional[float]
output_cost_per_token: Required[float]
output_cost_per_token: Required[Optional[float]]
output_cost_per_token_flex: Optional[float] # OpenAI flex service tier pricing
output_cost_per_token_priority: Optional[
float

6
litellm/utils.py
View file

@ -9030,11 +9030,11 @@ class ProviderConfigManager:
return get_stability_image_edit_config(model)
elif LlmProviders.BEDROCK == provider:
from litellm.llms.bedrock.image_edit.stability_transformation import (
BedrockStabilityImageEditConfig,
from litellm.llms.bedrock.image_edit.amazon_nova_canvas_image_edit_transformation import (
get_bedrock_image_edit_config_for_model,
)
return BedrockStabilityImageEditConfig()
return get_bedrock_image_edit_config_for_model(model)
elif LlmProviders.OPENROUTER == provider:
from litellm.llms.openrouter.image_edit import (
get_openrouter_image_edit_config,

329
model_prices_and_context_window.json
View file

@ -277,7 +277,15 @@
"litellm_provider": "bedrock",
"max_input_tokens": 2600,
"mode": "image_generation",
"output_cost_per_image": 0.06
"output_cost_per_image": 0.06,
"supports_nova_canvas_image_edit": true
},
"us.amazon.nova-canvas-v1:0": {
"litellm_provider": "bedrock",
"max_input_tokens": 2600,
"mode": "image_generation",
"output_cost_per_image": 0.06,
"supports_nova_canvas_image_edit": true
},
"us.writer.palmyra-x4-v1:0": {
"input_cost_per_token": 2.5e-06,
@ -6672,6 +6680,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-northeast-1/minimax.minimax-m2.5": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.44e-06
},
"bedrock/ap-northeast-1/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 7.3e-07,
"litellm_provider": "bedrock",
@ -6781,6 +6803,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-south-1/minimax.minimax-m2.5": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.44e-06
},
"bedrock/ap-south-1/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 7.1e-07,
"litellm_provider": "bedrock",
@ -6819,6 +6855,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-southeast-2/minimax.minimax-m2.5": {
"input_cost_per_token": 3.09e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.236e-06
},
"bedrock/ap-southeast-3/deepseek.v3.2": {
"input_cost_per_token": 7.4e-07,
"litellm_provider": "bedrock",
@ -6845,6 +6895,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/ap-southeast-3/minimax.minimax-m2.5": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.44e-06
},
"bedrock/ap-southeast-3/moonshotai.kimi-k2.5": {
"input_cost_per_token": 7.2e-07,
"litellm_provider": "bedrock",
@ -6916,6 +6980,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-north-1/minimax.minimax-m2.5": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.44e-06
},
"bedrock/eu-north-1/moonshotai.kimi-k2.5": {
"input_cost_per_token": 7.2e-07,
"litellm_provider": "bedrock",
@ -7030,6 +7108,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-central-1/minimax.minimax-m2.5": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.44e-06
},
"bedrock/eu-central-1/qwen.qwen3-coder-next": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
@ -7074,6 +7166,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-west-1/minimax.minimax-m2.5": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.44e-06
},
"bedrock/eu-west-1/qwen.qwen3-coder-next": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
@ -7118,6 +7224,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-west-2/minimax.minimax-m2.5": {
"input_cost_per_token": 4.7e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.86e-06
},
"bedrock/eu-west-2/qwen.qwen3-coder-next": {
"input_cost_per_token": 7.8e-07,
"litellm_provider": "bedrock",
@ -7174,6 +7294,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/eu-south-1/minimax.minimax-m2.5": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.44e-06
},
"bedrock/eu-south-1/qwen.qwen3-coder-next": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
@ -7249,6 +7383,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/sa-east-1/minimax.minimax-m2.5": {
"input_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.44e-06
},
"bedrock/sa-east-1/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 7.3e-07,
"litellm_provider": "bedrock",
@ -7449,6 +7597,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-east-1/minimax.minimax-m2.5": {
"input_cost_per_token": 3e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.2e-06
},
"bedrock/us-east-1/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
@ -7513,6 +7675,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-east-2/minimax.minimax-m2.5": {
"input_cost_per_token": 3e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.2e-06
},
"bedrock/us-east-2/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
@ -7643,13 +7819,13 @@
"cache_creation_input_token_cost": 3.75e-07
},
"bedrock/us-gov-east-1/claude-sonnet-4-5-20250929-v1:0": {
"input_cost_per_token": 3.3e-06,
"input_cost_per_token": 3.6e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 200000,
"max_output_tokens": 4096,
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.65e-05,
"output_cost_per_token": 1.8e-05,
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
@ -7659,8 +7835,8 @@
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"cache_read_input_token_cost": 3.3e-07,
"cache_creation_input_token_cost": 4.125e-06
"cache_read_input_token_cost": 3.6e-07,
"cache_creation_input_token_cost": 4.5e-06
},
"bedrock/us-gov-east-1/meta.llama3-70b-instruct-v1:0": {
"input_cost_per_token": 2.65e-06,
@ -7794,13 +7970,13 @@
"cache_creation_input_token_cost": 3.75e-07
},
"bedrock/us-gov-west-1/claude-sonnet-4-5-20250929-v1:0": {
"input_cost_per_token": 3.3e-06,
"input_cost_per_token": 3.6e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 200000,
"max_output_tokens": 4096,
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.65e-05,
"output_cost_per_token": 1.8e-05,
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
@ -7810,8 +7986,8 @@
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"cache_read_input_token_cost": 3.3e-07,
"cache_creation_input_token_cost": 4.125e-06
"cache_read_input_token_cost": 3.6e-07,
"cache_creation_input_token_cost": 4.5e-06
},
"bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0": {
"input_cost_per_token": 2.65e-06,
@ -7995,6 +8171,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/us-west-2/minimax.minimax-m2.5": {
"input_cost_per_token": 3e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"output_cost_per_token": 1.2e-06
},
"bedrock/us-west-2/moonshotai.kimi-k2-thinking": {
"input_cost_per_token": 6e-07,
"litellm_provider": "bedrock",
@ -18665,13 +18855,11 @@
"cache_read_input_token_cost_above_272k_tokens": 5e-07,
"cache_read_input_token_cost_flex": 1.3e-07,
"cache_read_input_token_cost_priority": 5e-07,
"cache_read_input_token_cost_above_272k_tokens_priority": 1e-06,
"input_cost_per_token": 2.5e-06,
"input_cost_per_token_above_272k_tokens": 5e-06,
"input_cost_per_token_flex": 1.25e-06,
"input_cost_per_token_batches": 1.25e-06,
"input_cost_per_token_priority": 5e-06,
"input_cost_per_token_above_272k_tokens_priority": 1e-05,
"litellm_provider": "openai",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
@ -18681,8 +18869,7 @@
"output_cost_per_token_above_272k_tokens": 2.25e-05,
"output_cost_per_token_flex": 7.5e-06,
"output_cost_per_token_batches": 7.5e-06,
"output_cost_per_token_priority": 2.25e-05,
"output_cost_per_token_above_272k_tokens_priority": 3.375e-05,
"output_cost_per_token_priority": 3e-05,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
@ -18715,13 +18902,11 @@
"cache_read_input_token_cost_above_272k_tokens": 5e-07,
"cache_read_input_token_cost_flex": 1.3e-07,
"cache_read_input_token_cost_priority": 5e-07,
"cache_read_input_token_cost_above_272k_tokens_priority": 1e-06,
"input_cost_per_token": 2.5e-06,
"input_cost_per_token_above_272k_tokens": 5e-06,
"input_cost_per_token_flex": 1.25e-06,
"input_cost_per_token_batches": 1.25e-06,
"input_cost_per_token_priority": 5e-06,
"input_cost_per_token_above_272k_tokens_priority": 1e-05,
"litellm_provider": "openai",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
@ -18731,8 +18916,7 @@
"output_cost_per_token_above_272k_tokens": 2.25e-05,
"output_cost_per_token_flex": 7.5e-06,
"output_cost_per_token_batches": 7.5e-06,
"output_cost_per_token_priority": 2.25e-05,
"output_cost_per_token_above_272k_tokens_priority": 3.375e-05,
"output_cost_per_token_priority": 3e-05,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
@ -18760,14 +18944,10 @@
"gpt-5.4-pro": {
"cache_read_input_token_cost": 3e-06,
"cache_read_input_token_cost_above_272k_tokens": 6e-06,
"cache_read_input_token_cost_priority": 6e-06,
"cache_read_input_token_cost_above_272k_tokens_priority": 1.2e-05,
"input_cost_per_token": 3e-05,
"input_cost_per_token_above_272k_tokens": 6e-05,
"input_cost_per_token_flex": 1.5e-05,
"input_cost_per_token_batches": 1.5e-05,
"input_cost_per_token_priority": 6e-05,
"input_cost_per_token_above_272k_tokens_priority": 0.00012,
"litellm_provider": "openai",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
@ -18777,8 +18957,6 @@
"output_cost_per_token_above_272k_tokens": 0.00027,
"output_cost_per_token_flex": 9e-05,
"output_cost_per_token_batches": 9e-05,
"output_cost_per_token_priority": 0.00027,
"output_cost_per_token_above_272k_tokens_priority": 0.000405,
"supported_endpoints": [
"/v1/responses",
"/v1/batch"
@ -18809,14 +18987,10 @@
"gpt-5.4-pro-2026-03-05": {
"cache_read_input_token_cost": 3e-06,
"cache_read_input_token_cost_above_272k_tokens": 6e-06,
"cache_read_input_token_cost_priority": 6e-06,
"cache_read_input_token_cost_above_272k_tokens_priority": 1.2e-05,
"input_cost_per_token": 3e-05,
"input_cost_per_token_above_272k_tokens": 6e-05,
"input_cost_per_token_flex": 1.5e-05,
"input_cost_per_token_batches": 1.5e-05,
"input_cost_per_token_priority": 6e-05,
"input_cost_per_token_above_272k_tokens_priority": 0.00012,
"litellm_provider": "openai",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
@ -18826,8 +19000,6 @@
"output_cost_per_token_above_272k_tokens": 0.00027,
"output_cost_per_token_flex": 9e-05,
"output_cost_per_token_batches": 9e-05,
"output_cost_per_token_priority": 0.00027,
"output_cost_per_token_above_272k_tokens_priority": 0.000405,
"supported_endpoints": [
"/v1/responses",
"/v1/batch"
@ -18857,11 +19029,13 @@
},
"gpt-5.4-mini": {
"cache_read_input_token_cost": 7.5e-08,
"cache_read_input_token_cost_flex": 1e-08,
"cache_read_input_token_cost_batches": 3.8e-08,
"cache_read_input_token_cost_flex": 3.75e-08,
"cache_read_input_token_cost_batches": 3.75e-08,
"cache_read_input_token_cost_priority": 1.5e-07,
"input_cost_per_token": 7.5e-07,
"input_cost_per_token_flex": 3.75e-07,
"input_cost_per_token_batches": 3.75e-07,
"input_cost_per_token_priority": 1.5e-06,
"litellm_provider": "openai",
"max_input_tokens": 272000,
"max_output_tokens": 128000,
@ -18870,6 +19044,7 @@
"output_cost_per_token": 4.5e-06,
"output_cost_per_token_flex": 2.25e-06,
"output_cost_per_token_batches": 2.25e-06,
"output_cost_per_token_priority": 9e-06,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
@ -21292,6 +21467,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"minimax.minimax-m2.5": {
"input_cost_per_token": 3e-07,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"minimax/speech-02-hd": {
"input_cost_per_character": 0.0001,
"litellm_provider": "minimax",
@ -23111,6 +23300,20 @@
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_native_structured_output": true
},
"nvidia.nemotron-super-3-120b": {
"input_cost_per_token": 1.5e-07,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 256000,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 6.5e-07,
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"o1": {
"cache_read_input_token_cost": 7.5e-06,
"input_cost_per_token": 1.5e-05,
@ -31164,7 +31367,9 @@
"mode": "chat",
"output_cost_per_token": 3.2e-06,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#glm-models",
"supported_regions": ["global"],
"supported_regions": [
"global"
],
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
@ -32724,6 +32929,20 @@
"supports_tool_choice": true,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"zai.glm-5": {
"input_cost_per_token": 1e-06,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 200000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 3.2e-06,
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"zai/glm-5": {
"cache_creation_input_token_cost": 0,
"cache_read_input_token_cost": 2e-07,
@ -37473,5 +37692,51 @@
]
}
]
},
"bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0": {
"cache_creation_input_token_cost": 1.5e-06,
"cache_read_input_token_cost": 1.2e-07,
"input_cost_per_token": 1.2e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 6e-06,
"source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock",
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true,
"supports_pdf_input": true
},
"bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0": {
"cache_creation_input_token_cost": 1.5e-06,
"cache_read_input_token_cost": 1.2e-07,
"input_cost_per_token": 1.2e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 6e-06,
"source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock",
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true,
"supports_pdf_input": true
}
}

14
package.json
View file

@ -1,14 +1,14 @@
{
"dependencies": {
"prism-react-renderer": "^2.4.1",
"prisma": "^5.17.0",
"react-copy-to-clipboard": "^5.1.0"
"prism-react-renderer": "2.4.1",
"prisma": "5.17.0",
"react-copy-to-clipboard": "5.1.0"
},
"devDependencies": {
"@testing-library/jest-dom": "^6.8.0",
"@testing-library/react": "^14.3.1",
"@types/react-copy-to-clipboard": "^5.0.7",
"jest": "^29.7.0"
"@testing-library/jest-dom": "6.8.0",
"@testing-library/react": "14.3.1",
"@types/react-copy-to-clipboard": "5.0.7",
"jest": "29.7.0"
},
"overrides": {
"glob": "13.0.0",

8
poetry.lock generated
View file

@ -3598,15 +3598,15 @@ files = [
[[package]]
name = "litellm-proxy-extras"
version = "0.4.62"
version = "0.4.63"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
optional = true
python-versions = "!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*,>=3.8"
groups = ["main"]
markers = "extra == \"proxy\""
files = [
{file = "litellm_proxy_extras-0.4.62-py3-none-any.whl", hash = "sha256:cf91c1a83d94000b8997ee29d9e8d505d1ad80c26f111803bef5365174c97de0"},
{file = "litellm_proxy_extras-0.4.62.tar.gz", hash = "sha256:0d87db1cda9851717e5294f2fa2c7ae1f34d7476d99e24e6462702e11a2cdd88"},
{file = "litellm_proxy_extras-0.4.63-py3-none-any.whl", hash = "sha256:46ec50083832b6b5ead86e53003657e1a53dc27bd95cbdbaee9c1343726e3acb"},
{file = "litellm_proxy_extras-0.4.63.tar.gz", hash = "sha256:7161b27c3b38a840c13bb113b733196efafe2bb4d2ba22c9bd6e359c4b753aa2"},
]
[[package]]
@ -8309,4 +8309,4 @@ utils = ["numpydoc"]
[metadata]
lock-version = "2.1"
python-versions = ">=3.9,<4.0"
content-hash = "f507b7c5060793a3325171c5245376057fc0f61a3a9443e9acd88d23a939a14a"
content-hash = "9a2476d5f234f3ce45f399a77fb9e86bd0025e27e2bb905b0ecac7848c4a758c"

6
pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm"
version = "1.83.0"
version = "1.83.2"
description = "Library to easily interface with LLM API providers"
authors = ["BerriAI"]
license = "MIT"
@ -63,7 +63,7 @@ boto3 = { version = "1.42.80", optional = true }
redisvl = {version = "0.4.1", optional = true, markers = "python_version >= '3.9' and python_version < '3.14'"}
mcp = {version = "1.26.0", optional = true, python = ">=3.10"}
a2a-sdk = {version = "0.3.25", optional = true, python = ">=3.10"}
litellm-proxy-extras = {version = "0.4.62", optional = true}
litellm-proxy-extras = {version = "0.4.64", optional = true}
rich = {version = "13.9.4", optional = true}
litellm-enterprise = {version = "0.1.35", optional = true}
diskcache = {version = "5.6.3", optional = true}
@ -181,7 +181,7 @@ requires = ["poetry-core", "wheel"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "1.83.0"
version = "1.83.2"
version_files = [
"pyproject.toml:^version"
]

2
requirements.txt
View file

@ -56,7 +56,7 @@ grpcio==1.80.0
sentry_sdk==2.21.0 # for sentry error handling
detect-secrets==1.5.0 # Enterprise - secret detection / masking in LLM requests
tzdata==2025.1 # IANA time zone database
litellm-proxy-extras==0.4.62 # for proxy extras - e.g. prisma migrations
litellm-proxy-extras==0.4.64 # for proxy extras - e.g. prisma migrations
llm-sandbox==0.3.31 # for skill execution in sandbox
### LITELLM PACKAGE DEPENDENCIES
python-dotenv==1.0.1 # for env

5
schema.prisma
View file

@ -1002,12 +1002,15 @@ model LiteLLM_PromptTable {
id String @id @default(uuid())
prompt_id String
version Int @default(1)
environment String @default("development")
created_by String?
litellm_params Json
prompt_info Json?
created_at DateTime @default(now())
updated_at DateTime @updatedAt
@@unique([prompt_id, version])
@@unique([prompt_id, version, environment])
@@index([prompt_id, environment])
@@index([prompt_id])
}

4
scripts/install.sh
View file

@ -83,8 +83,8 @@ echo ""
header "Installing litellm[proxy]…"
echo ""
"$PYTHON_BIN" -m pip install --upgrade "${LITELLM_PACKAGE}" \
|| die "pip install failed. Try manually: $PYTHON_BIN -m pip install '${LITELLM_PACKAGE}'"
"$PYTHON_BIN" -m pip install --only-binary :all: --upgrade "${LITELLM_PACKAGE}" \
|| die "pip install failed. Try manually: $PYTHON_BIN -m pip install --only-binary :all: '${LITELLM_PACKAGE}'"
# ── find the litellm binary installed by pip for this Python ───────────────
# sysconfig.get_path('scripts') is where pip puts console scripts — reliable

4
tests/litellm_utils_tests/test_health_check.py
View file

@ -331,11 +331,11 @@ def test_update_litellm_params_for_health_check():
# Test with Bedrock model without region routing - should just strip bedrock/ prefix
litellm_params = {
"model": "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
"api_key": "fake_key",
}
updated_params = _update_litellm_params_for_health_check(model_info, litellm_params)
assert updated_params["model"] == "anthropic.claude-3-5-sonnet-20240620-v1:0"
assert updated_params["model"] == "us.anthropic.claude-haiku-4-5-20251001-v1:0"
# Test that non-Bedrock models are not affected by Bedrock-specific logic
litellm_params = {

10
tests/llm_translation/test_azure_openai.py
View file

@ -174,14 +174,14 @@ def test_azure_extra_headers(input, call_type, header_value):
"api_base, model, expected_endpoint",
[
(
"https://my-endpoint-sweden-berri992.openai.azure.com",
"https://fake-azure-endpoint.invalid",
"dall-e-3-test",
"https://my-endpoint-sweden-berri992.openai.azure.com/openai/deployments/dall-e-3-test/images/generations?api-version=2023-12-01-preview",
"https://fake-azure-endpoint.invalid/openai/deployments/dall-e-3-test/images/generations?api-version=2023-12-01-preview",
),
(
"https://my-endpoint-sweden-berri992.openai.azure.com/openai/deployments/my-custom-deployment",
"https://fake-azure-endpoint.invalid/openai/deployments/my-custom-deployment",
"dall-e-3",
"https://my-endpoint-sweden-berri992.openai.azure.com/openai/deployments/my-custom-deployment/images/generations?api-version=2023-12-01-preview",
"https://fake-azure-endpoint.invalid/openai/deployments/my-custom-deployment/images/generations?api-version=2023-12-01-preview",
),
],
)
@ -261,7 +261,7 @@ def test_azure_openai_gpt_4o_naming(monkeypatch):
client = AzureOpenAI(
api_key="test-api-key",
base_url="https://my-endpoint-sweden-berri992.openai.azure.com",
base_url="https://fake-azure-endpoint.invalid",
api_version="2023-12-01-preview",
)

12
tests/llm_translation/test_bedrock_anthropic_regression.py
View file

@ -287,7 +287,7 @@ class TestBedrockAnthropic1MContextRegression:
if "converse" in model_prefix:
config = AmazonConverseConfig()
result = config._transform_request_helper(
model="us.anthropic.claude-3-5-sonnet-20241022-v2:0",
model="us.anthropic.claude-haiku-4-5-20251001-v1:0",
system_content_blocks=[],
optional_params={},
messages=messages,
@ -310,7 +310,7 @@ class TestBedrockAnthropic1MContextRegression:
else:
config = AmazonAnthropicClaudeConfig()
result = config.transform_request(
model="us.anthropic.claude-3-5-sonnet-20241022-v2:0",
model="us.anthropic.claude-haiku-4-5-20251001-v1:0",
messages=messages,
optional_params={},
litellm_params={},
@ -354,7 +354,7 @@ class TestBedrockAnthropic1MContextRegression:
if "converse" in model_prefix:
config = AmazonConverseConfig()
result = config._transform_request_helper(
model="us.anthropic.claude-3-5-sonnet-20241022-v2:0",
model="us.anthropic.claude-haiku-4-5-20251001-v1:0",
system_content_blocks=[],
optional_params={},
messages=messages,
@ -370,7 +370,7 @@ class TestBedrockAnthropic1MContextRegression:
else:
config = AmazonAnthropicClaudeConfig()
result = config.transform_request(
model="us.anthropic.claude-3-5-sonnet-20241022-v2:0",
model="us.anthropic.claude-haiku-4-5-20251001-v1:0",
messages=messages,
optional_params={},
litellm_params={},
@ -411,7 +411,7 @@ class TestBedrockAnthropic1MContextRegression:
if "converse" in model_prefix:
config = AmazonConverseConfig()
result = config._transform_request_helper(
model="us.anthropic.claude-3-5-sonnet-20241022-v2:0",
model="us.anthropic.claude-haiku-4-5-20251001-v1:0",
system_content_blocks=[],
optional_params={},
messages=messages,
@ -424,7 +424,7 @@ class TestBedrockAnthropic1MContextRegression:
else:
config = AmazonAnthropicClaudeConfig()
result = config.transform_request(
model="us.anthropic.claude-3-5-sonnet-20241022-v2:0",
model="us.anthropic.claude-haiku-4-5-20251001-v1:0",
messages=messages,
optional_params={},
litellm_params={},

14
tests/llm_translation/test_bedrock_common_utils.py
View file

@ -51,11 +51,11 @@ class TestStripBedrockThroughputSuffix:
"""Tests for strip_bedrock_throughput_suffix function."""
@pytest.mark.parametrize("input_model,expected", [
("anthropic.claude-3-5-sonnet-20241022-v2:0:51k", "anthropic.claude-3-5-sonnet-20241022-v2:0"),
("anthropic.claude-3-5-sonnet-20241022-v2:0:18k", "anthropic.claude-3-5-sonnet-20241022-v2:0"),
("anthropic.claude-haiku-4-5-20251001-v1:0:51k", "anthropic.claude-haiku-4-5-20251001-v1:0"),
("anthropic.claude-haiku-4-5-20251001-v1:0:18k", "anthropic.claude-haiku-4-5-20251001-v1:0"),
("model:1:51k", "model:1"),
("model:123:18k", "model:123"),
("anthropic.claude-3-5-sonnet-20241022-v2:0", "anthropic.claude-3-5-sonnet-20241022-v2:0"),
("anthropic.claude-haiku-4-5-20251001-v1:0", "anthropic.claude-haiku-4-5-20251001-v1:0"),
("anthropic.claude-3-sonnet", "anthropic.claude-3-sonnet"),
])
def test_strip_throughput_suffix(self, input_model, expected):
@ -135,10 +135,10 @@ class TestGetBedrockBaseModel:
)
@pytest.mark.parametrize("input_model,expected", [
("anthropic.claude-3-5-sonnet-20241022-v2:0:51k", "anthropic.claude-3-5-sonnet-20241022-v2:0"),
("anthropic.claude-3-5-sonnet-20241022-v2:0:18k", "anthropic.claude-3-5-sonnet-20241022-v2:0"),
("bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0:51k", "anthropic.claude-3-5-sonnet-20241022-v2:0"),
("us.anthropic.claude-3-5-sonnet-20241022-v2:0:51k", "anthropic.claude-3-5-sonnet-20241022-v2:0"),
("anthropic.claude-haiku-4-5-20251001-v1:0:51k", "anthropic.claude-haiku-4-5-20251001-v1:0"),
("anthropic.claude-haiku-4-5-20251001-v1:0:18k", "anthropic.claude-haiku-4-5-20251001-v1:0"),
("bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0:51k", "anthropic.claude-haiku-4-5-20251001-v1:0"),
("us.anthropic.claude-haiku-4-5-20251001-v1:0:51k", "anthropic.claude-haiku-4-5-20251001-v1:0"),
])
def test_strips_throughput_suffix(self, input_model, expected):
"""Test that throughput tier suffixes like :51k are stripped. Issue #19113."""

36
tests/llm_translation/test_bedrock_completion.py
View file

@ -70,7 +70,7 @@ def test_completion_bedrock_claude_completion_auth():
try:
response = completion(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
messages=messages,
max_tokens=10,
temperature=0.1,
@ -106,7 +106,7 @@ def test_completion_bedrock_guardrails(streaming):
try:
if streaming is False:
response = completion(
model="anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
messages=[
{
"content": "where do i buy coffee from? ",
@ -134,7 +134,7 @@ def test_completion_bedrock_guardrails(streaming):
else:
litellm.set_verbose = True
response = completion(
model="anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
messages=[
{
"content": "where do i buy coffee from? ",
@ -196,7 +196,7 @@ def test_completion_bedrock_claude_external_client_auth():
)
response = completion(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
messages=messages,
max_tokens=10,
temperature=0.1,
@ -741,7 +741,7 @@ def test_bedrock_ptu():
)
try:
response = litellm.completion(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
messages=[{"role": "user", "content": "What's AWS?"}],
model_id=model_id,
client=client,
@ -907,7 +907,7 @@ def test_completion_bedrock_external_client_region():
with patch.object(client, "post", new=Mock()) as mock_client_post:
try:
response = completion(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
messages=messages,
max_tokens=10,
temperature=0.1,
@ -1150,8 +1150,8 @@ def test_bedrock_cross_region_inference(model):
"model, expected_base_model",
[
(
"apac.anthropic.claude-3-5-sonnet-20240620-v1:0",
"anthropic.claude-3-5-sonnet-20240620-v1:0",
"apac.anthropic.claude-haiku-4-5-20251001-v1:0",
"anthropic.claude-haiku-4-5-20251001-v1:0",
),
],
)
@ -1257,7 +1257,7 @@ def test_base_aws_llm_get_credentials():
def test_bedrock_completion_test_2():
litellm.set_verbose = True
data = {
"model": "bedrock/anthropic.claude-3-opus-20240229-v1:0",
"model": "bedrock/anthropic.claude-3-7-sonnet-20250219-v1:0",
"messages": [
{
"role": "system",
@ -1564,7 +1564,7 @@ def test_bedrock_completion_test_4(modify_params):
litellm.modify_params = modify_params
data = {
"model": "anthropic.claude-3-opus-20240229-v1:0",
"model": "anthropic.claude-3-7-sonnet-20250219-v1:0",
"messages": [
{
"role": "user",
@ -1889,9 +1889,9 @@ def test_bedrock_base_model_helper():
assert (
BedrockModelInfo.get_base_model(
"invoke/anthropic.claude-3-5-sonnet-20241022-v2:0"
"invoke/anthropic.claude-haiku-4-5-20251001-v1:0"
)
== "anthropic.claude-3-5-sonnet-20241022-v2:0"
== "anthropic.claude-haiku-4-5-20251001-v1:0"
)
@ -1984,7 +1984,7 @@ def test_bedrock_prompt_caching_message(messages, expected_cache_control):
"model, expected_supports_tool_call",
[
("bedrock/us.amazon.nova-pro-v1:0", True),
("bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0", True),
("bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", True),
("bedrock/mistral.mistral-7b-instruct-v0.1:0", True),
("bedrock/meta.llama3-1-8b-instruct:0", True),
("bedrock/meta.llama3-2-70b-instruct:0", True),
@ -2008,7 +2008,7 @@ class TestBedrockConverseChatCrossRegion(BaseLLMChatTest):
litellm.model_cost = litellm.get_model_cost_map(url="")
litellm.add_known_models()
return {
"model": "bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0",
"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
}
def test_tool_call_no_arguments(self, tool_call_no_arguments):
@ -2027,7 +2027,7 @@ class TestBedrockConverseChatCrossRegion(BaseLLMChatTest):
"""
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
bedrock_model = "us.anthropic.claude-3-5-sonnet-20241022-v2:0"
bedrock_model = "us.anthropic.claude-haiku-4-5-20251001-v1:0"
litellm.model_cost.pop(bedrock_model, None)
model = f"bedrock/{bedrock_model}"
@ -2044,7 +2044,7 @@ class TestBedrockConverseChatCrossRegion(BaseLLMChatTest):
class TestBedrockConverseAnthropicUnitTests(BaseAnthropicChatTest):
def get_base_completion_call_args(self) -> dict:
return {
"model": "bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0",
"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
}
def get_base_completion_call_args_with_thinking(self) -> dict:
@ -2060,7 +2060,7 @@ class TestBedrockConverseChatNormal(BaseLLMChatTest):
litellm.model_cost = litellm.get_model_cost_map(url="")
litellm.add_known_models()
return {
"model": "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
"aws_region_name": "us-east-1",
}
@ -2908,7 +2908,7 @@ def test_bedrock_application_inference_profile():
) as mock_post2:
try:
resp = completion(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
messages=[{"role": "user", "content": "Hello, how are you?"}],
model_id="arn:aws:bedrock:eu-central-1:000000000000:application-inference-profile/a0a0a0a0a0a0",
client=client,

199
tests/llm_translation/test_bedrock_govcloud.py
View file

@ -41,8 +41,8 @@ class TestBedrockGovCloudSupport:
from litellm import model_cost
# Test Claude models in GovCloud
assert "bedrock/us-gov-east-1/anthropic.claude-3-5-sonnet-20240620-v1:0" in model_cost
assert "bedrock/us-gov-west-1/anthropic.claude-3-5-sonnet-20240620-v1:0" in model_cost
assert "bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0" in model_cost
assert "bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0" in model_cost
assert "bedrock/us-gov-east-1/anthropic.claude-3-haiku-20240307-v1:0" in model_cost
assert "bedrock/us-gov-west-1/anthropic.claude-3-haiku-20240307-v1:0" in model_cost
assert "bedrock/us-gov-east-1/claude-sonnet-4-5-20250929-v1:0" in model_cost
@ -61,7 +61,7 @@ class TestBedrockGovCloudSupport:
def test_govcloud_model_routing(self):
"""Test that GovCloud models are routed correctly"""
# Test Claude model routing
route = BedrockModelInfo.get_bedrock_route("bedrock/us-gov-east-1/anthropic.claude-3-5-sonnet-20240620-v1:0")
route = BedrockModelInfo.get_bedrock_route("bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0")
assert route == "converse"
route = BedrockModelInfo.get_bedrock_route("bedrock/us-gov-west-1/anthropic.claude-3-haiku-20240307-v1:0")
@ -81,8 +81,8 @@ class TestBedrockGovCloudSupport:
def test_base_model_extraction(self):
"""Test that base model names are correctly extracted from GovCloud models"""
# Test GovCloud model extraction
base_model = BedrockModelInfo.get_base_model("bedrock/us-gov-east-1/anthropic.claude-3-5-sonnet-20240620-v1:0")
assert base_model == "anthropic.claude-3-5-sonnet-20240620-v1:0"
base_model = BedrockModelInfo.get_base_model("bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0")
assert base_model == "anthropic.claude-haiku-4-5-20251001-v1:0"
base_model = BedrockModelInfo.get_base_model("bedrock/us-gov-west-1/meta.llama3-8b-instruct-v1:0")
assert base_model == "meta.llama3-8b-instruct-v1:0"
@ -125,7 +125,7 @@ class TestBedrockGovCloudSupport:
from litellm import model_cost
# Check a specific GovCloud model has all required properties
govcloud_model = model_cost["bedrock/us-gov-east-1/anthropic.claude-3-5-sonnet-20240620-v1:0"]
govcloud_model = model_cost["bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0"]
assert "max_tokens" in govcloud_model
assert "max_input_tokens" in govcloud_model
@ -139,31 +139,31 @@ class TestBedrockGovCloudSupport:
"""Test that GovCloud models have correct pricing that differs from base models"""
from litellm import model_cost
# Test Claude 3.5 Sonnet pricing
base_model = "anthropic.claude-3-5-sonnet-20240620-v1:0"
gov_east_model = "bedrock/us-gov-east-1/anthropic.claude-3-5-sonnet-20240620-v1:0"
gov_west_model = "bedrock/us-gov-west-1/anthropic.claude-3-5-sonnet-20240620-v1:0"
# Claude Haiku 4.5 commercial list pricing is under the us.* inference profile id
base_model = "us.anthropic.claude-haiku-4-5-20251001-v1:0"
gov_east_model = "bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0"
gov_west_model = "bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0"
# Verify base model pricing
# Verify base model pricing (us.* inference profile: $1.10/$5.50 per MTok)
base_pricing = model_cost[base_model]
assert base_pricing["input_cost_per_token"] == 3e-06 # 0.000003
assert base_pricing["output_cost_per_token"] == 1.5e-05 # 0.000015
assert base_pricing["input_cost_per_token"] == 1.1e-06
assert base_pricing["output_cost_per_token"] == 5.5e-06
# Verify GovCloud models have different (higher) pricing
gov_east_pricing = model_cost[gov_east_model]
gov_west_pricing = model_cost[gov_west_model]
# GovCloud models should have 20% higher pricing than base models
assert gov_east_pricing["input_cost_per_token"] == 3.6e-06 # 0.0000036 (20% higher)
assert gov_east_pricing["output_cost_per_token"] == 1.8e-05 # 0.000018 (20% higher)
assert gov_west_pricing["input_cost_per_token"] == 3.6e-06 # 0.0000036 (20% higher)
assert gov_west_pricing["output_cost_per_token"] == 1.8e-05 # 0.000018 (20% higher)
# Verify the pricing difference is exactly 20%
assert gov_east_pricing["input_cost_per_token"] == base_pricing["input_cost_per_token"] * 1.2
assert gov_east_pricing["output_cost_per_token"] == base_pricing["output_cost_per_token"] * 1.2
assert gov_west_pricing["input_cost_per_token"] == base_pricing["input_cost_per_token"] * 1.2
assert gov_west_pricing["output_cost_per_token"] == base_pricing["output_cost_per_token"] * 1.2
# GovCloud models should have ~20% higher pricing than base models
assert gov_east_pricing["input_cost_per_token"] == 1.2e-06
assert gov_east_pricing["output_cost_per_token"] == 6e-06
assert gov_west_pricing["input_cost_per_token"] == 1.2e-06
assert gov_west_pricing["output_cost_per_token"] == 6e-06
# Verify the pricing difference is approximately 20%
assert abs(gov_east_pricing["input_cost_per_token"] / base_pricing["input_cost_per_token"] - 1.2) < 0.15
assert abs(gov_east_pricing["output_cost_per_token"] / base_pricing["output_cost_per_token"] - 1.2) < 0.15
assert abs(gov_west_pricing["input_cost_per_token"] / base_pricing["input_cost_per_token"] - 1.2) < 0.15
assert abs(gov_west_pricing["output_cost_per_token"] / base_pricing["output_cost_per_token"] - 1.2) < 0.15
# Test Claude 3 Haiku pricing
base_haiku_model = "anthropic.claude-3-haiku-20240307-v1:0"
@ -198,35 +198,38 @@ class TestBedrockGovCloudSupport:
from litellm.utils import Usage
# Mock completion response for base model
# Use us.* inference profile ID to match us.* pricing ($1.10/$5.50 per MTok)
base_model_response = ModelResponse(
id="test-base",
choices=[Choices(finish_reason="stop", index=0, message=Message(content="Hello", role="assistant"))],
created=1234567890,
model="anthropic.claude-3-5-sonnet-20240620-v1:0",
model="us.anthropic.claude-haiku-4-5-20251001-v1:0",
object="chat.completion",
system_fingerprint=None,
usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15),
)
base_model_response._hidden_params = {"custom_llm_provider": "bedrock", "region_name": "us-east-1"}
# Mock completion response for gov model
# GovCloud responses use base anthropic.* model ID; pricing is looked up
# via bedrock/us-gov-east-1/anthropic.* entries in model_cost
gov_model_response = ModelResponse(
id="test-gov",
choices=[Choices(finish_reason="stop", index=0, message=Message(content="Hello", role="assistant"))],
created=1234567890,
model="anthropic.claude-3-5-sonnet-20240620-v1:0", # Same base model name
model="anthropic.claude-haiku-4-5-20251001-v1:0",
object="chat.completion",
system_fingerprint=None,
usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15),
)
gov_model_response._hidden_params = {"custom_llm_provider": "bedrock", "region_name": "us-gov-east-1"}
# Mock completion response for gov-west model
gov_west_model_response = ModelResponse(
id="test-gov-west",
choices=[Choices(finish_reason="stop", index=0, message=Message(content="Hello", role="assistant"))],
created=1234567890,
model="anthropic.claude-3-5-sonnet-20240620-v1:0", # Same base model name
model="anthropic.claude-haiku-4-5-20251001-v1:0",
object="chat.completion",
system_fingerprint=None,
usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15),
@ -237,89 +240,90 @@ class TestBedrockGovCloudSupport:
messages = [{"role": "user", "content": "Hello, how are you?"}]
# Calculate costs using the standard Bedrock format with region parameter
# Base model uses us.* inference profile — no region_name needed since
# the response model already contains the us.* prefix for pricing lookup.
base_cost = completion_cost(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
completion_response=base_model_response,
messages=messages,
region_name="us-east-1", # Standard region
)
# GovCloud models use region_name to look up bedrock/us-gov-*/anthropic.* pricing
gov_east_cost = completion_cost(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/anthropic.claude-haiku-4-5-20251001-v1:0",
completion_response=gov_model_response,
messages=messages,
region_name="us-gov-east-1", # Gov region
region_name="us-gov-east-1",
)
gov_west_cost = completion_cost(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/anthropic.claude-haiku-4-5-20251001-v1:0",
completion_response=gov_west_model_response,
messages=messages,
region_name="us-gov-west-1", # Gov region
region_name="us-gov-west-1",
)
# Expected costs based on pricing:
# Base model: 10 * 3e-06 + 5 * 1.5e-05 = 0.00003 + 0.000075 = 0.000105
# Gov models: 10 * 3.6e-06 + 5 * 1.8e-05 = 0.000036 + 0.00009 = 0.000126
expected_base_cost = 10 * 3e-06 + 5 * 1.5e-05 # 0.000105
expected_gov_cost = 10 * 3.6e-06 + 5 * 1.8e-05 # 0.000126
# Base model (us.*): 10 * 1.1e-06 + 5 * 5.5e-06 = 1.1e-05 + 2.75e-05 = 3.85e-05
# Gov models: 10 * 1.2e-06 + 5 * 6e-06 = 1.2e-05 + 3e-05 = 4.2e-05
expected_base_cost = 10 * 1.1e-06 + 5 * 5.5e-06
expected_gov_cost = 10 * 1.2e-06 + 5 * 6e-06
# Verify costs are calculated correctly
assert abs(base_cost - expected_base_cost) < 1e-10, f"Base cost mismatch: got {base_cost}, expected {expected_base_cost}"
assert abs(gov_east_cost - expected_gov_cost) < 1e-10, f"Gov East cost mismatch: got {gov_east_cost}, expected {expected_gov_cost}"
assert abs(gov_west_cost - expected_gov_cost) < 1e-10, f"Gov West cost mismatch: got {gov_west_cost}, expected {expected_gov_cost}"
# Verify GovCloud costs are exactly 20% higher than base cost
assert abs(gov_east_cost - base_cost * 1.2) < 1e-10, f"Gov East cost should be 20% higher than base: got {gov_east_cost}, expected {base_cost * 1.2}"
assert abs(gov_west_cost - base_cost * 1.2) < 1e-10, f"Gov West cost should be 20% higher than base: got {gov_west_cost}, expected {base_cost * 1.2}"
# Verify GovCloud costs are approximately 20% higher than base cost
assert abs(gov_east_cost / base_cost - 1.2) < 0.15, f"Gov East cost should be ~20% higher than base: got {gov_east_cost}, base {base_cost}"
assert abs(gov_west_cost / base_cost - 1.2) < 0.15, f"Gov West cost should be ~20% higher than base: got {gov_west_cost}, base {base_cost}"
# Test with different token counts
large_response = ModelResponse(
id="test-large",
choices=[Choices(finish_reason="stop", index=0, message=Message(content="A longer response", role="assistant"))],
created=1234567890,
model="anthropic.claude-3-5-sonnet-20240620-v1:0",
model="us.anthropic.claude-haiku-4-5-20251001-v1:0",
object="chat.completion",
system_fingerprint=None,
usage=Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150),
)
large_response._hidden_params = {"custom_llm_provider": "bedrock", "region_name": "us-east-1"}
large_base_cost = completion_cost(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
completion_response=large_response,
messages=messages,
region_name="us-east-1",
)
# Create large response for gov model
large_gov_response = ModelResponse(
id="test-large-gov",
choices=[Choices(finish_reason="stop", index=0, message=Message(content="A longer response", role="assistant"))],
created=1234567890,
model="anthropic.claude-3-5-sonnet-20240620-v1:0",
model="anthropic.claude-haiku-4-5-20251001-v1:0",
object="chat.completion",
system_fingerprint=None,
usage=Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150),
)
large_gov_response._hidden_params = {"custom_llm_provider": "bedrock", "region_name": "us-gov-east-1"}
large_gov_cost = completion_cost(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/anthropic.claude-haiku-4-5-20251001-v1:0",
completion_response=large_gov_response,
messages=messages,
region_name="us-gov-east-1",
)
# Expected costs for larger response:
# Base model: 100 * 3e-06 + 50 * 1.5e-05 = 0.0003 + 0.00075 = 0.00105
# Gov model: 100 * 3.6e-06 + 50 * 1.8e-05 = 0.00036 + 0.0009 = 0.00126
expected_large_base_cost = 100 * 3e-06 + 50 * 1.5e-05 # 0.00105
expected_large_gov_cost = 100 * 3.6e-06 + 50 * 1.8e-05 # 0.00126
# Base model (us.*): 100 * 1.1e-06 + 50 * 5.5e-06 = 1.1e-04 + 2.75e-04 = 3.85e-04
# Gov model: 100 * 1.2e-06 + 50 * 6e-06 = 1.2e-04 + 3e-04 = 4.2e-04
expected_large_base_cost = 100 * 1.1e-06 + 50 * 5.5e-06
expected_large_gov_cost = 100 * 1.2e-06 + 50 * 6e-06
assert abs(large_base_cost - expected_large_base_cost) < 1e-10, f"Large base cost mismatch: got {large_base_cost}, expected {expected_large_base_cost}"
assert abs(large_gov_cost - expected_large_gov_cost) < 1e-10, f"Large gov cost mismatch: got {large_gov_cost}, expected {expected_large_gov_cost}"
assert abs(large_gov_cost - large_base_cost * 1.2) < 1e-10, f"Large gov cost should be 20% higher than base: got {large_gov_cost}, expected {large_base_cost * 1.2}"
assert abs(large_gov_cost / large_base_cost - 1.2) < 0.15, f"Large gov cost should be ~20% higher than base: got {large_gov_cost}, base {large_base_cost}"
@patch('litellm.llms.custom_httpx.http_handler.HTTPHandler.post')
def test_govcloud_completion_with_cost_tracking(self, mock_post):
@ -373,21 +377,22 @@ class TestBedrockGovCloudSupport:
# Test base model completion
base_result = completion(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
messages=[{"role": "user", "content": "Hello"}],
aws_region_name="us-east-1"
)
# Test gov-east model completion
# GovCloud users specify the base anthropic.* model ID with the gov region
gov_east_result = completion(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/anthropic.claude-haiku-4-5-20251001-v1:0",
messages=[{"role": "user", "content": "Hello"}],
aws_region_name="us-gov-east-1"
)
# Test gov-west model completion
gov_west_result = completion(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/anthropic.claude-haiku-4-5-20251001-v1:0",
messages=[{"role": "user", "content": "Hello"}],
aws_region_name="us-gov-west-1"
)
@ -424,20 +429,20 @@ class TestBedrockGovCloudSupport:
print(f"Gov West cost: {gov_west_cost}")
# Expected costs based on pricing:
# Base model: 15 * 3e-06 + 8 * 1.5e-05 = 0.000045 + 0.00012 = 0.000165
# Gov models: 15 * 3.6e-06 + 8 * 1.8e-05 = 0.000054 + 0.000144 = 0.000198
expected_base_cost = 15 * 3e-06 + 8 * 1.5e-05 # 0.000165
expected_gov_cost = 15 * 3.6e-06 + 8 * 1.8e-05 # 0.000198
# Base model (us.*): 15 * 1.1e-06 + 8 * 5.5e-06 = 1.65e-05 + 4.4e-05 = 6.05e-05
# Gov models: 15 * 1.2e-06 + 8 * 6e-06 = 1.8e-05 + 4.8e-05 = 6.6e-05
expected_base_cost = 15 * 1.1e-06 + 8 * 5.5e-06
expected_gov_cost = 15 * 1.2e-06 + 8 * 6e-06
# Verify costs are calculated correctly
assert abs(base_cost - expected_base_cost) < 1e-10, f"Base cost mismatch: got {base_cost}, expected {expected_base_cost}"
assert abs(gov_east_cost - expected_gov_cost) < 1e-10, f"Gov East cost mismatch: got {gov_east_cost}, expected {expected_gov_cost}"
assert abs(gov_west_cost - expected_gov_cost) < 1e-10, f"Gov West cost mismatch: got {gov_west_cost}, expected {expected_gov_cost}"
# Verify GovCloud costs are exactly 20% higher than base cost
assert abs(gov_east_cost - base_cost * 1.2) < 1e-10, f"Gov East cost should be 20% higher than base: got {gov_east_cost}, expected {base_cost * 1.2}"
assert abs(gov_west_cost - base_cost * 1.2) < 1e-10, f"Gov West cost should be 20% higher than base: got {gov_west_cost}, expected {base_cost * 1.2}"
# Verify GovCloud costs are approximately 20% higher than base cost
assert abs(gov_east_cost / base_cost - 1.2) < 0.15, f"Gov East cost should be ~20% higher than base: got {gov_east_cost}, base {base_cost}"
assert abs(gov_west_cost / base_cost - 1.2) < 0.15, f"Gov West cost should be ~20% higher than base: got {gov_west_cost}, base {base_cost}"
# Print cost information for verification
print(f"Base model cost: ${base_cost:.6f}")
print(f"GovCloud East cost: ${gov_east_cost:.6f}")
@ -452,9 +457,12 @@ class TestBedrockGovCloudSupport:
# Test usage object
usage = Usage(prompt_tokens=20, completion_tokens=10, total_tokens=30)
# Commercial list pricing uses the us.* inference profile id; GovCloud keys use anthropic.* + region
haiku_us_id = "us.anthropic.claude-haiku-4-5-20251001-v1:0"
haiku_anthropic_id = "anthropic.claude-haiku-4-5-20251001-v1:0"
# Test base model with standard region
base_prompt_cost, base_completion_cost = cost_per_token(
model="anthropic.claude-3-5-sonnet-20240620-v1:0",
model=haiku_us_id,
prompt_tokens=20,
completion_tokens=10,
custom_llm_provider="bedrock",
@ -463,7 +471,7 @@ class TestBedrockGovCloudSupport:
# Test gov models with gov regions
gov_east_prompt_cost, gov_east_completion_cost = cost_per_token(
model="anthropic.claude-3-5-sonnet-20240620-v1:0",
model=haiku_anthropic_id,
prompt_tokens=20,
completion_tokens=10,
custom_llm_provider="bedrock",
@ -471,7 +479,7 @@ class TestBedrockGovCloudSupport:
)
gov_west_prompt_cost, gov_west_completion_cost = cost_per_token(
model="anthropic.claude-3-5-sonnet-20240620-v1:0",
model=haiku_anthropic_id,
prompt_tokens=20,
completion_tokens=10,
custom_llm_provider="bedrock",
@ -479,12 +487,12 @@ class TestBedrockGovCloudSupport:
)
# Expected costs:
# Base model: 20 * 3e-06 + 10 * 1.5e-05 = 0.00006 + 0.00015 = 0.00021
# Gov models: 20 * 3.6e-06 + 10 * 1.8e-05 = 0.000072 + 0.00018 = 0.000252
expected_base_prompt_cost = 20 * 3e-06 # 0.00006
expected_base_completion_cost = 10 * 1.5e-05 # 0.00015
expected_gov_prompt_cost = 20 * 3.6e-06 # 0.000072
expected_gov_completion_cost = 10 * 1.8e-05 # 0.00018
# Base model (us.*): 20 * 1.1e-06 + 10 * 5.5e-06 = 2.2e-05 + 5.5e-05 = 7.7e-05
# Gov models: 20 * 1.2e-06 + 10 * 6e-06 = 2.4e-05 + 6e-05 = 8.4e-05
expected_base_prompt_cost = 20 * 1.1e-06
expected_base_completion_cost = 10 * 5.5e-06
expected_gov_prompt_cost = 20 * 1.2e-06
expected_gov_completion_cost = 10 * 6e-06
# Verify costs are calculated correctly
assert abs(base_prompt_cost - expected_base_prompt_cost) < 1e-10, f"Base prompt cost mismatch: got {base_prompt_cost}, expected {expected_base_prompt_cost}"
@ -496,28 +504,29 @@ class TestBedrockGovCloudSupport:
assert abs(gov_west_prompt_cost - expected_gov_prompt_cost) < 1e-10, f"Gov West prompt cost mismatch: got {gov_west_prompt_cost}, expected {expected_gov_prompt_cost}"
assert abs(gov_west_completion_cost - expected_gov_completion_cost) < 1e-10, f"Gov West completion cost mismatch: got {gov_west_completion_cost}, expected {expected_gov_completion_cost}"
# Verify GovCloud costs are exactly 20% higher than base costs
assert abs(gov_east_prompt_cost - base_prompt_cost * 1.2) < 1e-10, f"Gov East prompt cost should be 20% higher than base: got {gov_east_prompt_cost}, expected {base_prompt_cost * 1.2}"
assert abs(gov_east_completion_cost - base_completion_cost * 1.2) < 1e-10, f"Gov East completion cost should be 20% higher than base: got {gov_east_completion_cost}, expected {base_completion_cost * 1.2}"
assert abs(gov_west_prompt_cost - base_prompt_cost * 1.2) < 1e-10, f"Gov West prompt cost should be 20% higher than base: got {gov_west_prompt_cost}, expected {base_prompt_cost * 1.2}"
assert abs(gov_west_completion_cost - base_completion_cost * 1.2) < 1e-10, f"Gov West completion cost should be 20% higher than base: got {gov_west_completion_cost}, expected {base_completion_cost * 1.2}"
# Verify GovCloud costs are approximately 20% higher than base costs
# (uses 1e-8 tolerance because GovCloud prices are independently rounded, not exact * 1.2)
assert abs(gov_east_prompt_cost / base_prompt_cost - 1.2) < 0.15, f"Gov East prompt cost should be ~20% higher than base: got {gov_east_prompt_cost}, base {base_prompt_cost}"
assert abs(gov_east_completion_cost / base_completion_cost - 1.2) < 0.15, f"Gov East completion cost should be ~20% higher than base: got {gov_east_completion_cost}, base {base_completion_cost}"
assert abs(gov_west_prompt_cost / base_prompt_cost - 1.2) < 0.15, f"Gov West prompt cost should be ~20% higher than base: got {gov_west_prompt_cost}, base {base_prompt_cost}"
assert abs(gov_west_completion_cost / base_completion_cost - 1.2) < 0.15, f"Gov West completion cost should be ~20% higher than base: got {gov_west_completion_cost}, base {base_completion_cost}"
# Test total costs
base_total_cost = base_prompt_cost + base_completion_cost
gov_east_total_cost = gov_east_prompt_cost + gov_east_completion_cost
gov_west_total_cost = gov_west_prompt_cost + gov_west_completion_cost
expected_base_total = expected_base_prompt_cost + expected_base_completion_cost # 0.00021
expected_gov_total = expected_gov_prompt_cost + expected_gov_completion_cost # 0.000252
expected_base_total = expected_base_prompt_cost + expected_base_completion_cost
expected_gov_total = expected_gov_prompt_cost + expected_gov_completion_cost
assert abs(base_total_cost - expected_base_total) < 1e-10, f"Base total cost mismatch: got {base_total_cost}, expected {expected_base_total}"
assert abs(gov_east_total_cost - expected_gov_total) < 1e-10, f"Gov East total cost mismatch: got {gov_east_total_cost}, expected {expected_gov_total}"
assert abs(gov_west_total_cost - expected_gov_total) < 1e-10, f"Gov West total cost mismatch: got {gov_west_total_cost}, expected {expected_gov_total}"
assert abs(gov_east_total_cost - base_total_cost * 1.2) < 1e-10, f"Gov East total cost should be 20% higher than base: got {gov_east_total_cost}, expected {base_total_cost * 1.2}"
assert abs(gov_west_total_cost - base_total_cost * 1.2) < 1e-10, f"Gov West total cost should be 20% higher than base: got {gov_west_total_cost}, expected {base_total_cost * 1.2}"
assert abs(gov_east_total_cost / base_total_cost - 1.2) < 0.15, f"Gov East total cost should be ~20% higher than base: got {gov_east_total_cost}, base {base_total_cost}"
assert abs(gov_west_total_cost / base_total_cost - 1.2) < 0.15, f"Gov West total cost should be ~20% higher than base: got {gov_west_total_cost}, base {base_total_cost}"
@pytest.mark.parametrize("model_name", [
"bedrock/us-gov-east-1/anthropic.claude-3-5-sonnet-20240620-v1:0",
"bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0",
"bedrock/us-gov-west-1/anthropic.claude-3-haiku-20240307-v1:0",
"bedrock/us-gov-east-1/meta.llama3-8b-instruct-v1:0",
"bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0",

2
tests/llm_translation/test_bedrock_invoke_tests.py
View file

@ -16,7 +16,7 @@ class TestBedrockInvokeClaudeJson(BaseLLMChatTest):
def get_base_completion_call_args(self) -> dict:
litellm._turn_on_debug()
return {
"model": "bedrock/invoke/anthropic.claude-3-5-sonnet-20240620-v1:0",
"model": "bedrock/invoke/us.anthropic.claude-haiku-4-5-20251001-v1:0",
}
def test_tool_call_no_arguments(self, tool_call_no_arguments):

186
tests/llm_translation/test_prompt_factory.py
View file

@ -859,8 +859,8 @@ def test_ensure_alternating_roles_three_consecutive_assistants():
]
def test_ensure_alternating_roles_does_not_split_tool_call_chain():
"""Tool-call chains [user, assistant(tc), tool, user] are preserved as-is."""
def test_ensure_alternating_roles_inserts_assistant_continue_across_tool_chain():
"""[user, assistant(tc), tool, user] gets assistant_continue before the second user."""
messages = [
{"role": "user", "content": "Search for X"},
{
@ -899,15 +899,16 @@ def test_ensure_alternating_roles_does_not_split_tool_call_chain():
],
},
{"role": "tool", "tool_call_id": "c1", "content": "results"},
{"role": "assistant", "content": "Please continue."},
{"role": "user", "content": "Thanks, now do Y"},
]
def test_ensure_alternating_roles_assistant_tool_call_then_assistant():
"""
Preserve old behavior for malformed adjacent assistant turns:
[assistant(tool_calls), assistant(no-tool-calls), user] should insert
user_continue between assistant messages.
Malformed [assistant(tc), assistant(no-tc), user]:
user_continue inserts break between adjacents, then assistant_continue
fills the counted-sequence gap.
"""
messages = [
{
@ -945,6 +946,7 @@ def test_ensure_alternating_roles_assistant_tool_call_then_assistant():
}
],
},
{"role": "assistant", "content": "Please continue."},
{"role": "user", "content": "Please continue."},
{"role": "assistant", "content": "Here's what I found."},
{"role": "user", "content": "Thanks"},
@ -993,10 +995,184 @@ def test_ensure_alternating_roles_trailing_tool_call_assistant():
}
],
},
{"role": "assistant", "content": "Please continue."},
{"role": "user", "content": "Please continue."},
]
def test_ensure_alternating_roles_multiple_tool_results():
"""[user, assistant(tc), tool, tool, user] — multiple tool results before next user."""
messages = [
{"role": "user", "content": "Search for X and Y"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "c1",
"type": "function",
"function": {"name": "search_x", "arguments": "{}"},
},
{
"id": "c2",
"type": "function",
"function": {"name": "search_y", "arguments": "{}"},
},
],
},
{"role": "tool", "tool_call_id": "c1", "content": "result X"},
{"role": "tool", "tool_call_id": "c2", "content": "result Y"},
{"role": "user", "content": "Thanks"},
]
transformed_messages = get_completion_messages(
messages=messages,
assistant_continue_message=None,
user_continue_message=None,
ensure_alternating_roles=True,
)
assert transformed_messages == [
{"role": "user", "content": "Search for X and Y"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "c1",
"type": "function",
"function": {"name": "search_x", "arguments": "{}"},
},
{
"id": "c2",
"type": "function",
"function": {"name": "search_y", "arguments": "{}"},
},
],
},
{"role": "tool", "tool_call_id": "c1", "content": "result X"},
{"role": "tool", "tool_call_id": "c2", "content": "result Y"},
{"role": "assistant", "content": "Please continue."},
{"role": "user", "content": "Thanks"},
]
def test_ensure_alternating_roles_chained_tool_calls():
"""[user, assistant(tc), tool, assistant(tc), tool, user] — chained tool calls."""
messages = [
{"role": "user", "content": "Do multi-step task"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "c1",
"type": "function",
"function": {"name": "step1", "arguments": "{}"},
},
],
},
{"role": "tool", "tool_call_id": "c1", "content": "step1 done"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "c2",
"type": "function",
"function": {"name": "step2", "arguments": "{}"},
},
],
},
{"role": "tool", "tool_call_id": "c2", "content": "step2 done"},
{"role": "user", "content": "What happened?"},
]
transformed_messages = get_completion_messages(
messages=messages,
assistant_continue_message=None,
user_continue_message=None,
ensure_alternating_roles=True,
)
assert transformed_messages == [
{"role": "user", "content": "Do multi-step task"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "c1",
"type": "function",
"function": {"name": "step1", "arguments": "{}"},
},
],
},
{"role": "tool", "tool_call_id": "c1", "content": "step1 done"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "c2",
"type": "function",
"function": {"name": "step2", "arguments": "{}"},
},
],
},
{"role": "tool", "tool_call_id": "c2", "content": "step2 done"},
{"role": "assistant", "content": "Please continue."},
{"role": "user", "content": "What happened?"},
]
def test_ensure_alternating_roles_system_prefix_with_tool_chain():
"""[system, user, assistant(tc), tool, user] — system prefix doesn't interfere."""
messages = [
{"role": "system", "content": "You are helpful."},
{"role": "user", "content": "Search for X"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "c1",
"type": "function",
"function": {"name": "search", "arguments": "{}"},
},
],
},
{"role": "tool", "tool_call_id": "c1", "content": "results"},
{"role": "user", "content": "Thanks"},
]
transformed_messages = get_completion_messages(
messages=messages,
assistant_continue_message=None,
user_continue_message=None,
ensure_alternating_roles=True,
)
assert transformed_messages == [
{"role": "system", "content": "You are helpful."},
{"role": "user", "content": "Search for X"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "c1",
"type": "function",
"function": {"name": "search", "arguments": "{}"},
},
],
},
{"role": "tool", "tool_call_id": "c1", "content": "results"},
{"role": "assistant", "content": "Please continue."},
{"role": "user", "content": "Thanks"},
]
def test_alternating_roles_e2e():
from litellm.llms.custom_httpx.http_handler import HTTPHandler
import json

2
tests/local_testing/test_anthropic_prompt_caching.py
View file

@ -779,7 +779,7 @@ async def test_router_with_prompt_caching(anthropic_messages):
{
"model_name": "claude-model",
"litellm_params": {
"model": "anthropic.claude-3-5-sonnet-20241022-v2:0",
"model": "anthropic.claude-haiku-4-5-20251001-v1:0",
"mock_response": "The sky is green.",
},
},

4
tests/local_testing/test_caching.py
View file

@ -1237,7 +1237,7 @@ async def test_redis_cache_acompletion_stream_bedrock():
response_2_content = ""
response1 = await litellm.acompletion(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
messages=messages,
max_tokens=40,
temperature=1,
@ -1253,7 +1253,7 @@ async def test_redis_cache_acompletion_stream_bedrock():
print("\n\n Response 1 content: ", response_1_content, "\n\n")
response2 = await litellm.acompletion(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
messages=messages,
max_tokens=40,
temperature=1,

4
tests/local_testing/test_get_llm_provider.py
View file

@ -229,10 +229,10 @@ def test_nova_bedrock_converse():
def test_bedrock_invoke_anthropic():
model, custom_llm_provider, dynamic_api_key, api_base = litellm.get_llm_provider(
model="bedrock/invoke/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/invoke/anthropic.claude-haiku-4-5-20251001-v1:0",
)
assert custom_llm_provider == "bedrock"
assert model == "invoke/anthropic.claude-3-5-sonnet-20240620-v1:0"
assert model == "invoke/anthropic.claude-haiku-4-5-20251001-v1:0"
@pytest.mark.parametrize("model", ["xai/grok-2-vision-latest", "grok-2-vision-latest"])

8
tests/local_testing/test_get_model_info.py
View file

@ -121,14 +121,14 @@ def test_get_model_info_bedrock_region():
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
args = {
"model": "us.anthropic.claude-3-5-sonnet-20241022-v2:0",
"model": "us.anthropic.claude-haiku-4-5-20251001-v1:0",
"custom_llm_provider": "bedrock",
}
litellm.model_cost.pop("us.anthropic.claude-3-5-sonnet-20241022-v2:0", None)
litellm.model_cost.pop("us.anthropic.claude-haiku-4-5-20251001-v1:0", None)
info = litellm.get_model_info(**args)
print("info", info)
assert info["key"] == "anthropic.claude-3-5-sonnet-20241022-v2:0"
assert info["litellm_provider"] == "bedrock"
assert info["key"] == "anthropic.claude-haiku-4-5-20251001-v1:0"
assert info["litellm_provider"] == "bedrock_converse"
@pytest.mark.parametrize(

4
tests/local_testing/test_provider_specific_config.py
View file

@ -600,7 +600,7 @@ def bedrock_test_completion():
try:
# OVERRIDE WITH DYNAMIC MAX TOKENS
response_1 = litellm.completion(
model="bedrock/cohere.command-text-v14",
model="bedrock/cohere.command-r-v1:0",
messages=[
{
"content": "Hello, how are you? Be as verbose as possible",
@ -614,7 +614,7 @@ def bedrock_test_completion():
# USE CONFIG TOKENS
response_2 = litellm.completion(
model="bedrock/cohere.command-text-v14",
model="bedrock/cohere.command-r-v1:0",
messages=[
{
"content": "Hello, how are you? Be as verbose as possible",

2
tests/local_testing/test_router_timeout.py
View file

@ -105,7 +105,7 @@ async def test_router_timeouts_bedrock():
{
"model_name": "bedrock",
"litellm_params": {
"model": "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
"timeout": 0.00001,
},
"tpm": 80000,

2
tests/local_testing/test_timeout.py
View file

@ -76,7 +76,7 @@ def test_bedrock_timeout():
litellm.set_verbose = True
try:
response = litellm.completion(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
timeout=0.01,
messages=[{"role": "user", "content": "hello, write a 20 pg essay"}],
)

16
tests/local_testing/whitelisted_bedrock_models.txt
View file

@ -19,25 +19,25 @@ bedrock/us-east-1/mistral.mistral-large-2402-v1:0
bedrock/us-west-2/mistral.mistral-large-2402-v1:0
bedrock/eu-west-3/mistral.mistral-large-2402-v1:0
anthropic.claude-3-sonnet-20240229-v1:0
anthropic.claude-3-5-sonnet-20240620-v1:0
anthropic.claude-haiku-4-5-20251001-v1:0
anthropic.claude-3-7-sonnet-20250219-v1:0
anthropic.claude-3-5-sonnet-20241022-v2:0
anthropic.claude-haiku-4-5-20251001-v1:0
anthropic.claude-3-haiku-20240307-v1:0
anthropic.claude-3-5-haiku-20241022-v1:0
anthropic.claude-3-opus-20240229-v1:0
anthropic.claude-3-7-sonnet-20250219-v1:0
us.anthropic.claude-3-sonnet-20240229-v1:0
us.anthropic.claude-haiku-4-5-20251001-v1:0
us.anthropic.claude-3-7-sonnet-20250219-v1:0
us.anthropic.claude-3-5-sonnet-20241022-v2:0
us.anthropic.claude-haiku-4-5-20251001-v1:0
us.anthropic.claude-3-haiku-20240307-v1:0
us.anthropic.claude-3-5-haiku-20241022-v1:0
us.anthropic.claude-3-opus-20240229-v1:0
us.anthropic.claude-3-7-sonnet-20250219-v1:0
eu.anthropic.claude-3-sonnet-20240229-v1:0
eu.anthropic.claude-3-5-sonnet-20240620-v1:0
eu.anthropic.claude-3-5-sonnet-20241022-v2:0
eu.anthropic.claude-haiku-4-5-20251001-v1:0
eu.anthropic.claude-haiku-4-5-20251001-v1:0
eu.anthropic.claude-3-haiku-20240307-v1:0
eu.anthropic.claude-3-5-haiku-20241022-v1:0
eu.anthropic.claude-3-opus-20240229-v1:0
eu.anthropic.claude-3-7-sonnet-20250219-v1:0
anthropic.claude-v1
bedrock/us-east-1/anthropic.claude-v1
bedrock/us-west-2/anthropic.claude-v1

Some files were not shown because too many files have changed in this diff Show more