diff --git a/.circleci/config.yml b/.circleci/config.yml index 5bbabad9107..307247651e7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3233,31 +3233,9 @@ jobs: name: Install Playwright Browsers command: | npx playwright install - - run: - name: Install Neon CLI - command: | - npm i -g neonctl - - run: - name: Create Neon branch - command: | - export EXPIRES_AT=$(date -u -d "+3 hours" +"%Y-%m-%dT%H:%M:%SZ") - echo "Expires at: $EXPIRES_AT" - neon branches create \ - --project-id $NEON_PROJECT_ID \ - --name preview/commit-${CIRCLE_SHA1:0:7}-<< parameters.browser >> \ - --expires-at $EXPIRES_AT \ - --parent br-fancy-paper-ad1olsb3 \ - --api-key $NEON_API_KEY || true - run: name: Run Docker container command: | - E2E_UI_TEST_DATABASE_URL=$(neon connection-string \ - --project-id $NEON_PROJECT_ID \ - --api-key $NEON_API_KEY \ - --branch preview/commit-${CIRCLE_SHA1:0:7}-<< parameters.browser >> \ - --database-name yuneng-trial-db \ - --role neondb_owner) - echo $E2E_UI_TEST_DATABASE_URL docker run -d \ -p 4000:4000 \ -e DATABASE_URL=$E2E_UI_TEST_DATABASE_URL \ @@ -3531,7 +3509,7 @@ workflows: only: - main - /litellm_.*/ - # - e2e_ui_testing: # migrate to dynamic db - currently requires neon cli + # - e2e_ui_testing: # name: e2e_ui_testing_chromium # browser: chromium # context: e2e_ui_tests diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 7cd12bb219c..3a00064c3bd 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -42,6 +42,6 @@ jobs: retention-days: 5 - name: Upload to code scanning - uses: github/codeql-action/upload-sarif@c10b806170c8ee63ea24152429041b5624f0baf5 # v4.35.1 + uses: github/codeql-action/upload-sarif@c10b8064de6f491fea524254123dbe5e09572f13 # v4.35.1 with: sarif_file: results.sarif diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml deleted file mode 100644 index 2bc361bc48f..00000000000 --- a/.pre-commit-config.yaml +++ /dev/null @@ -1,40 +0,0 @@ -repos: -- repo: local - hooks: - - id: pyright - name: pyright - entry: pyright - language: system - types: [python] - files: ^(litellm/|litellm_proxy_extras/|enterprise/) - - id: isort - name: isort - entry: isort - language: system - types: [python] - files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py - exclude: ^litellm/__init__.py$ - - id: black - name: black - entry: poetry run black - language: system - types: [python] - files: (litellm/|litellm_proxy_extras/).*\.py -- repo: https://github.com/pycqa/flake8 - rev: 7.0.0 # The version of flake8 to use - hooks: - - id: flake8 - exclude: ^litellm/tests/|^litellm/proxy/tests/|^litellm/tests/test_litellm/|^tests/test_litellm/|^tests/enterprise/ - additional_dependencies: [flake8-print] - files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py -- repo: https://github.com/python-poetry/poetry - rev: 1.8.0 - hooks: - - id: poetry-check - files: ^(pyproject.toml|litellm-proxy-extras/pyproject.toml)$ -- repo: local - hooks: - - id: check-files-match - name: Check if files match - entry: python3 ci_cd/check_files_match.py - language: system \ No newline at end of file diff --git a/ci_cd/publish-proxy-extras.sh b/ci_cd/publish-proxy-extras.sh deleted file mode 100644 index 6c83d1f9212..00000000000 --- a/ci_cd/publish-proxy-extras.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -# Exit on error -set -e - -echo "🚀 Building and publishing litellm-proxy-extras" - -# Navigate to litellm-proxy-extras directory -cd "$(dirname "$0")/../litellm-proxy-extras" - -# Build the package -echo "📦 Building package..." -poetry build - -# Publish to PyPI -echo "🌎 Publishing to PyPI..." -poetry publish - -echo "✅ Done! Package published successfully" \ No newline at end of file diff --git a/ci_cd/security_scans.sh b/ci_cd/security_scans.sh index ee33393c8b1..2138fca6cd5 100755 --- a/ci_cd/security_scans.sh +++ b/ci_cd/security_scans.sh @@ -160,7 +160,6 @@ run_grype_scans() { "CVE-2026-0775" # npm cli incorrect permission assignment - no fix available yet, npm is only used at build/prisma-generate time "GHSA-3ppc-4f35-3m26" # minimatch ReDoS via repeated wildcards - from nodejs_wheel bundled npm, not used in application runtime code "GHSA-83g3-92jg-28cx" # tar arbitrary file read/write via hardlink - from nodejs_wheel bundled npm, not used in application runtime code - "CVE-2026-25639" # axios DoS via __proto__ in mergeConfig - transitive dev dep via @neondatabase/api-client, not imported in application code "CVE-2026-2297" # Python 3.13 SourcelessFileLoader audit hook bypass - no fix available in base image "GHSA-qffp-2rhf-9h96" # tar hardlink path traversal - from nodejs_wheel bundled npm, not used in application runtime code "CVE-2026-2673" # OpenSSL 3.6.1 TLS 1.3 key exchange group negotiation issue - no fix available yet diff --git a/cookbook/ai_coding_tool_guides/claude_code_quickstart/guide.md b/cookbook/ai_coding_tool_guides/claude_code_quickstart/guide.md index 3d6c75498b1..b2d81be25bb 100644 --- a/cookbook/ai_coding_tool_guides/claude_code_quickstart/guide.md +++ b/cookbook/ai_coding_tool_guides/claude_code_quickstart/guide.md @@ -230,7 +230,7 @@ model_list: # AWS Bedrock - model_name: claude-bedrock litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0 + model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: us-east-1 diff --git a/cookbook/misc/test_responses_api.py b/cookbook/misc/test_responses_api.py index 5fd19c6f66f..62e4e2cf62e 100644 --- a/cookbook/misc/test_responses_api.py +++ b/cookbook/misc/test_responses_api.py @@ -20,7 +20,7 @@ base64_image = encode_image(image_path) response = client.responses.create( - model="bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", input=[ { "role": "user", @@ -43,7 +43,7 @@ print("sleeping for 20 seconds...") time.sleep(20) print("making follow up request for existing id") response2 = client.responses.create( - model="bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", previous_response_id=response.id, input="ok, and what objects are in the image?" ) diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index 1cf6c0dfd15..bf1af9ed756 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -46,7 +46,10 @@ ENV LITELLM_NON_ROOT=true # package-lock.json with pinned versions + integrity hashes. RUN mkdir -p /var/lib/litellm/ui && \ mv /app/.npmrc /app/.npmrc.bak && \ - npm install -g npm@11.12.1 && npm cache clean --force && \ + npm install -g npm@11.12.1 && \ + npm install -g node-gyp@12.2.0 && \ + ln -sf /usr/local/lib/node_modules/node-gyp /usr/lib/node_modules/npm/node_modules/node-gyp && \ + npm cache clean --force && \ cd /app/ui/litellm-dashboard && \ if [ -f "/app/enterprise/enterprise_ui/enterprise_colors.json" ]; then \ cp /app/enterprise/enterprise_ui/enterprise_colors.json ./ui_colors.json; \ diff --git a/docs/my-website/blog/april_townhall_announcement/index.md b/docs/my-website/blog/april_townhall_announcement/index.md new file mode 100644 index 00000000000..466d9e845f0 --- /dev/null +++ b/docs/my-website/blog/april_townhall_announcement/index.md @@ -0,0 +1,39 @@ +--- +slug: april-townhall-announcement +title: "April Townhall: Security + Product Roadmap" +date: 2026-04-02T07:30:00 +authors: + - krrish +description: "Join the LiteLLM April townhall on Friday, 10 April at 7:30 AM to learn about LiteLLM's security and product roadmap." +tags: [announcement, townhall] +hide_table_of_contents: true +--- + +import Image from '@theme/IdealImage'; + +We are hosting our April townhall on **Friday, 10 April at 7:30 AM PST**. + + + +{/* truncate */} + +## Agenda + +- Product updates and roadmap progress +- Reliability and security updates +- Open Q&A with the team + +## How to contribute + +Add your thoughts to this [ticket](https://github.com/BerriAI/litellm/issues/24825) to help us shape the agenda. + +## Register + +Register here: [LiteLLM April Townhall Form](https://forms.gle/hvyVXwbFjzJQE7dEA) + +We will hold the townhall from **7:30 AM to 8:30 AM PST on Zoom**. + +For security, attendance is restricted to corporate emails. If you register with a non-corporate email, we will share the townhall slides and accompanying blog post after the event. diff --git a/docs/my-website/blog/security_hardening_april_2026/index.md b/docs/my-website/blog/security_hardening_april_2026/index.md new file mode 100644 index 00000000000..1af4caa3e1f --- /dev/null +++ b/docs/my-website/blog/security_hardening_april_2026/index.md @@ -0,0 +1,66 @@ +--- +slug: security-hardening-april-2026 +title: "Security Update: Vulnerability Disclosures and Ongoing Hardening" +date: 2026-04-03T12:00:00 +authors: + - krrish + - ishaan-alt +description: "Disclosure of security vulnerabilities fixed in LiteLLM v1.83.0, and the launch of our bug bounty program." +tags: [security] +hide_table_of_contents: false +--- + +After the [supply chain incident](https://docs.litellm.ai/blog/security-update-march-2026) in March, we brought in [Veria Labs](https://verialabs.com/) to audit the LiteLLM proxy and fixed a number of vulnerability reports from independent researchers. All issues below are fixed in v1.83.0. If you are affected, particularly if you have JWT auth enabled, we recommend upgrading. + +We've also launched a [bug bounty program](#bug-bounty-program) and Veria Labs is continuing to audit the proxy. More fixes will ship in upcoming versions. + +The two high-severity issues ([CVE-2026-35029](https://github.com/BerriAI/litellm/security/advisories/GHSA-53mr-6c8q-9789) and [GHSA-69x8-hrgq-fjj8](https://github.com/BerriAI/litellm/security/advisories/GHSA-69x8-hrgq-fjj8)) **both require the attacker to already have a valid API key for the proxy**. These are not exploitable by unauthenticated users. + +The critical-severity issue ([CVE-2026-35030](https://github.com/BerriAI/litellm/security/advisories/GHSA-jjhc-v7c2-5hh6)) is an authentication bypass, but only affects deployments with `enable_jwt_auth` explicitly enabled, which is off by default. **The default LiteLLM configuration is not affected, and no LiteLLM Cloud customers had this feature enabled.** + +{/* truncate */} + +## Vulnerabilities + +### CVE-2026-35030: Authentication bypass via OIDC cache collision (Critical) + +Found by Veria Labs. + +When `enable_jwt_auth` is enabled, LiteLLM cached OIDC userinfo using `token[:20]` as the cache key. JWTs from the same signing algorithm share the same header prefix, so an attacker could forge a token that hits another user's cache entry and inherit their session. We fixed this by keying the cache on `sha256(token)` instead. + +**Most deployments are not affected.** This requires `enable_jwt_auth: true`, which is off by default. If you can't upgrade, disable JWT auth as a workaround. + +Full advisory: [GHSA-jjhc-v7c2-5hh6](https://github.com/BerriAI/litellm/security/advisories/GHSA-jjhc-v7c2-5hh6) + +### CVE-2026-35029: Privilege escalation via `/config/update` (High) + +Found by Lakera. + +`/config/update` didn't check the caller's role. Any authenticated user could modify the proxy's runtime configuration, which could lead to arbitrary file read, admin account takeover, or remote code execution. We now require the `proxy_admin` role on this endpoint. + +Full advisory: [GHSA-53mr-6c8q-9789](https://github.com/BerriAI/litellm/security/advisories/GHSA-53mr-6c8q-9789) + +### Password hash exposure and pass-the-hash login (High) + +Weak hashing originally reported by GitHub user [hamzayevmaqsud](https://github.com/hamzayevmaqsud) ([#15484](https://github.com/BerriAI/litellm/issues/15484)). The full chain was identified by Luca Vandenweghe and Maarten De Rammelaere of [iO Digital](https://www.iodigital.com/). + +Passwords were stored as unsalted SHA-256 hashes, and in some cases plaintext. Several API endpoints returned the hash to any authenticated user, and `/v2/login` accepted the raw hash as a credential without re-hashing it, so a stolen hash was as good as the password itself. We've moved to scrypt with random salts and stripped hashes from all API responses. + +Full advisory: [GHSA-69x8-hrgq-fjj8](https://github.com/BerriAI/litellm/security/advisories/GHSA-69x8-hrgq-fjj8) + +## Bug bounty program + +After the supply chain incident and these disclosures it was clear we needed more external eyes on the project. We've set up a bug bounty program so researchers have a way to report issues. + +Bounties are currently paid for P0 (supply chain) and P1 (unauthenticated proxy access) vulnerabilities: + +| Severity | Bounty | Example | +|----------|--------|---------| +| Critical | $1,500 – $3,000 | Supply chain compromise | +| High | $500 – $1,500 | Unauthenticated access to protected data | + +We plan on expanding the program further in the coming months. More info about the bug bounty program is available [here](https://github.com/BerriAI/litellm/security). + +## What's next + +Veria Labs is continuing to work with us on a broader audit of the proxy. Security advisories sent through Github will be responded to within five business days. We'll publish advisories as issues are confirmed and fixed. diff --git a/docs/my-website/docs/anthropic_count_tokens.md b/docs/my-website/docs/anthropic_count_tokens.md index 5985516d69c..a62e46f156a 100644 --- a/docs/my-website/docs/anthropic_count_tokens.md +++ b/docs/my-website/docs/anthropic_count_tokens.md @@ -96,7 +96,7 @@ model_list: - model_name: claude-bedrock litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0 + model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 aws_region_name: us-west-2 ``` diff --git a/docs/my-website/docs/completion/computer_use.md b/docs/my-website/docs/completion/computer_use.md index ed09a73b219..400f108f97e 100644 --- a/docs/my-website/docs/completion/computer_use.md +++ b/docs/my-website/docs/completion/computer_use.md @@ -80,7 +80,7 @@ model_list: api_key: os.environ/ANTHROPIC_API_KEY - model_name: claude-bedrock # Bedrock Anthropic model litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0 + model: bedrock/anthropic.claude-haiku-4-5-20251001:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: us-west-2 @@ -153,7 +153,7 @@ import litellm assert litellm.supports_computer_use(model="anthropic/claude-3-5-sonnet-latest") == True assert litellm.supports_computer_use(model="anthropic/claude-3-7-sonnet-20250219") == True -assert litellm.supports_computer_use(model="bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0") == True +assert litellm.supports_computer_use(model="bedrock/anthropic.claude-haiku-4-5-20251001:0") == True assert litellm.supports_computer_use(model="vertex_ai/claude-3-5-sonnet") == True assert litellm.supports_computer_use(model="openai/gpt-4") == False ``` @@ -171,7 +171,7 @@ model_list: api_key: os.environ/ANTHROPIC_API_KEY - model_name: claude-bedrock # Bedrock Anthropic model litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0 + model: bedrock/anthropic.claude-haiku-4-5-20251001:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: us-west-2 diff --git a/docs/my-website/docs/completion/document_understanding.md b/docs/my-website/docs/completion/document_understanding.md index 172e0792801..f510a33f79a 100644 --- a/docs/my-website/docs/completion/document_understanding.md +++ b/docs/my-website/docs/completion/document_understanding.md @@ -32,7 +32,7 @@ os.environ["AWS_REGION_NAME"] = "" file_url = "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf" # model -model = "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0" +model = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" file_content = [ {"type": "text", "text": "What's this file about?"}, @@ -63,7 +63,7 @@ assert response is not None model_list: - model_name: bedrock-model litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0 + model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: os.environ/AWS_REGION_NAME @@ -122,7 +122,7 @@ encoded_file = base64.b64encode(file_data).decode("utf-8") base64_url = f"data:application/pdf;base64,{encoded_file}" # model -model = "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0" +model = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" file_content = [ {"type": "text", "text": "What's this file about?"}, @@ -153,7 +153,7 @@ assert response is not None model_list: - model_name: bedrock-model litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0 + model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: os.environ/AWS_REGION_NAME @@ -210,7 +210,7 @@ os.environ["AWS_REGION_NAME"] = "" file_url = "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf" # model -model = "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0" +model = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" file_content = [ {"type": "text", "text": "What's this file about?"}, @@ -242,7 +242,7 @@ assert response is not None model_list: - model_name: bedrock-model litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0 + model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: os.environ/AWS_REGION_NAME @@ -350,10 +350,10 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \ -Use `litellm.supports_pdf_input(model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0")` -> returns `True` if model can accept pdf input +Use `litellm.supports_pdf_input(model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0")` -> returns `True` if model can accept pdf input ```python -assert litellm.supports_pdf_input(model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0") == True +assert litellm.supports_pdf_input(model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0") == True ``` @@ -365,7 +365,7 @@ assert litellm.supports_pdf_input(model="bedrock/anthropic.claude-3-5-sonnet-202 model_list: - model_name: bedrock-model # model group name litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0 + model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: os.environ/AWS_REGION_NAME diff --git a/docs/my-website/docs/completion/provider_specific_params.md b/docs/my-website/docs/completion/provider_specific_params.md index 250b410c9c4..791153d2bc8 100644 --- a/docs/my-website/docs/completion/provider_specific_params.md +++ b/docs/my-website/docs/completion/provider_specific_params.md @@ -450,7 +450,7 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \ import litellm response = litellm.completion( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "Hello!"}], requestMetadata={"cost_center": "engineering"} ) diff --git a/docs/my-website/docs/index.md b/docs/my-website/docs/index.md index ca63c9e39ff..6410e052b05 100644 --- a/docs/my-website/docs/index.md +++ b/docs/my-website/docs/index.md @@ -103,7 +103,7 @@ os.environ["AWS_SECRET_ACCESS_KEY"] = "your-secret" os.environ["AWS_REGION_NAME"] = "us-east-1" response = completion( - model="bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0", + model="bedrock/anthropic.claude-haiku-4-5-20251001:0", messages=[{"role": "user", "content": "Hello, how are you?"}] ) print(response.choices[0].message.content) diff --git a/docs/my-website/docs/oidc.md b/docs/my-website/docs/oidc.md index b541329aa38..23eb431b7e0 100644 --- a/docs/my-website/docs/oidc.md +++ b/docs/my-website/docs/oidc.md @@ -268,7 +268,7 @@ Please contact us for paid enterprise support if you need help setting up Azure model list: - model_name: aws/claude-3-5-sonnet litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0 + model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 aws_region_name: "eu-central-1" aws_role_name: "arn:aws:iam::12345678:role/bedrock-role" aws_web_identity_token: "oidc/azure/api://123-456-789-9d04" diff --git a/docs/my-website/docs/providers/azure/azure_responses.md b/docs/my-website/docs/providers/azure/azure_responses.md index 34ec0e194f7..de085001ba1 100644 --- a/docs/my-website/docs/providers/azure/azure_responses.md +++ b/docs/my-website/docs/providers/azure/azure_responses.md @@ -246,7 +246,7 @@ You can also call the Azure Responses API via the `/chat/completions` endpoint. from litellm import completion import os -os.environ["AZURE_API_BASE"] = "https://my-endpoint-sweden-berri992.openai.azure.com/" +os.environ["AZURE_API_BASE"] = "https://my-azure-endpoint.openai.azure.com/" os.environ["AZURE_API_VERSION"] = "2023-03-15-preview" os.environ["AZURE_API_KEY"] = "my-api-key" @@ -268,7 +268,7 @@ model_list: litellm_params: model: azure/responses/my-custom-o1-pro api_key: os.environ/AZURE_API_KEY - api_base: https://my-endpoint-sweden-berri992.openai.azure.com/ + api_base: https://my-azure-endpoint.openai.azure.com/ api_version: 2023-03-15-preview ``` diff --git a/docs/my-website/docs/providers/bedrock.md b/docs/my-website/docs/providers/bedrock.md index bb07216a295..e5942cc1119 100644 --- a/docs/my-website/docs/providers/bedrock.md +++ b/docs/my-website/docs/providers/bedrock.md @@ -95,7 +95,7 @@ Here's how to call Bedrock with the LiteLLM Proxy Server model_list: - model_name: bedrock-claude-3-5-sonnet litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0 + model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: os.environ/AWS_REGION_NAME @@ -337,7 +337,7 @@ os.environ["AWS_SECRET_ACCESS_KEY"] = "" os.environ["AWS_REGION_NAME"] = "" response = completion( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "Hello, how are you?"}], requestMetadata={ "cost_center": "engineering", @@ -354,7 +354,7 @@ response = completion( model_list: - model_name: bedrock-claude-v1 litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0 + model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 requestMetadata: cost_center: "engineering" ``` @@ -1543,7 +1543,7 @@ file_data = response.content encoded_file = base64.b64encode(file_data).decode("utf-8") # model -model = "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0" +model = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" image_content = [ {"type": "text", "text": "What's this file about?"}, @@ -1574,7 +1574,7 @@ assert response is not None model_list: - model_name: bedrock-model litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0 + model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: os.environ/AWS_REGION_NAME @@ -1631,7 +1631,7 @@ encoded_file = base64.b64encode(file_data).decode("utf-8") base64_url = f"data:application/pdf;base64,{encoded_file}" # model -model = "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0" +model = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" image_content = [ {"type": "text", "text": "What's this file about?"}, @@ -1660,7 +1660,7 @@ assert response is not None model_list: - model_name: bedrock-model litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0 + model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: os.environ/AWS_REGION_NAME @@ -1941,7 +1941,7 @@ Here's an example of using a bedrock model with LiteLLM. For a complete list, re | GPT-OSS 120B | `completion(model='bedrock/converse/openai.gpt-oss-120b-1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` | | Deepseek R1 | `completion(model='bedrock/us.deepseek.r1-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` | | Anthropic Claude Sonnet 4.5 | `completion(model='bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` | -| Anthropic Claude-V3.5 Sonnet | `completion(model='bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` | +| Anthropic Claude-V3.5 Sonnet | `completion(model='bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` | | Anthropic Claude-V3 sonnet | `completion(model='bedrock/anthropic.claude-3-sonnet-20240229-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` | | Anthropic Claude-V3 Haiku | `completion(model='bedrock/anthropic.claude-3-haiku-20240307-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` | | Anthropic Claude-V3 Opus | `completion(model='bedrock/anthropic.claude-3-opus-20240229-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` | @@ -2051,7 +2051,7 @@ os.environ["AWS_SECRET_ACCESS_KEY"] = "" os.environ["AWS_REGION_NAME"] = "" response = completion( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "Hello, how are you?"}], model_id="arn:aws:bedrock:eu-central-1:000000000000:application-inference-profile/a0a0a0a0a0a0", ) @@ -2068,7 +2068,7 @@ print(response) model_list: - model_name: anthropic-claude-3-5-sonnet litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0 + model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 # You have to set the ARN application inference profile in the model_id parameter model_id: arn:aws:bedrock:eu-central-1:000000000000:application-inference-profile/a0a0a0a0a0a0 ``` diff --git a/docs/my-website/docs/providers/bedrock_image_gen.md b/docs/my-website/docs/providers/bedrock_image_gen.md index 799c6d46437..e6e8429817d 100644 --- a/docs/my-website/docs/providers/bedrock_image_gen.md +++ b/docs/my-website/docs/providers/bedrock_image_gen.md @@ -111,6 +111,29 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/images/generations' \ +## Amazon Nova Canvas - Image Edit + +Use OpenAI-compatible `image_edit()` with Bedrock Nova Canvas (`amazon.nova-canvas-v1:0`). Requests use the same `InvokeModel` API as generation; LiteLLM maps inputs to [Nova Canvas task types](https://docs.aws.amazon.com/nova/latest/userguide/image-gen-access.html): + +| Scenario | `taskType` sent to Bedrock | +|----------|----------------------------| +| Image + prompt (no mask) | `IMAGE_VARIATION` | +| Image + prompt + mask | `INPAINTING` (`inPaintingParams.image`, `maskImage` or `maskPrompt`) | +| `taskType: OUTPAINTING` + `mask` or `maskPrompt` | `OUTPAINTING` (Bedrock requires one; LiteLLM raises a clear error if both are missing) | +| `taskType: BACKGROUND_REMOVAL` | `BACKGROUND_REMOVAL` | + +```python +from litellm import image_edit + +response = image_edit( + image=open("photo.png", "rb"), + prompt="Add soft sunset lighting", + model="bedrock/amazon.nova-canvas-v1:0", +) +``` + +For **`BACKGROUND_REMOVAL`**, the AWS request must not include `imageGenerationConfig`; LiteLLM omits it for that task even if you pass `size`, `n`, `seed`, etc. Additional Nova Canvas inference IDs for image edit should set **`supports_nova_canvas_image_edit`: true** in `model_prices_and_context_window.json` (see `amazon.nova-canvas-v1:0`). + ## Using Inference Profiles with Image Generation For AWS Bedrock Application Inference Profiles with image generation, use the `model_id` parameter to specify the inference profile ARN: @@ -147,4 +170,3 @@ model_list: ## Authentication All standard Bedrock authentication methods are supported for image generation. See [Bedrock Authentication](./bedrock#boto3---authentication) for details. - diff --git a/docs/my-website/docs/proxy/oauth2.md b/docs/my-website/docs/proxy/oauth2.md index 204a01538cc..c0597058cfd 100644 --- a/docs/my-website/docs/proxy/oauth2.md +++ b/docs/my-website/docs/proxy/oauth2.md @@ -61,3 +61,24 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ Start the LiteLLM Proxy with [`--detailed_debug` mode and you should see more verbose logs](cli.md#detailed_debug) +## Using OAuth2 + JWT Together + +If both `enable_oauth2_auth` and `enable_jwt_auth` are enabled, LiteLLM can split auth paths: +- JWT validation for user tokens +- OAuth2 introspection for machine tokens + +For JWT-shaped machine tokens, configure `litellm_jwtauth.routing_overrides`: + +```yaml title="config.yaml" +general_settings: + enable_jwt_auth: true + enable_oauth2_auth: true + litellm_jwtauth: + routing_overrides: + - iss: "machine-issuer.example.com" + client_id: "MID_LITELLM" + path: "oauth2" +``` + +For full `routing_overrides` behavior and list-based selectors, see [`/proxy/token_auth`](./token_auth.md#route-jwt-shaped-machine-tokens-to-oauth2). + diff --git a/docs/my-website/docs/proxy/team_model_add.md b/docs/my-website/docs/proxy/team_model_add.md index bb4238055b5..4aa286f3e5f 100644 --- a/docs/my-website/docs/proxy/team_model_add.md +++ b/docs/my-website/docs/proxy/team_model_add.md @@ -26,7 +26,7 @@ curl -L -X POST 'http://0.0.0.0:4000/model/new' \ "model": "openai/gpt-4o", "custom_llm_provider": "openai", "api_key": "******ccb07", - "api_base": "https://my-endpoint-sweden-berri992.openai.azure.com", + "api_base": "https://my-azure-endpoint.openai.azure.com", "api_version": "2023-12-01-preview" }, "model_info": { diff --git a/docs/my-website/docs/proxy/token_auth.md b/docs/my-website/docs/proxy/token_auth.md index c287ab03644..d37b05391b6 100644 --- a/docs/my-website/docs/proxy/token_auth.md +++ b/docs/my-website/docs/proxy/token_auth.md @@ -790,6 +790,47 @@ litellm_jwtauth: user_roles_jwt_field: "resource_access.your-client.roles" ``` +## Route JWT-Shaped Machine Tokens to OAuth2 + +Use this when both are enabled: +- `enable_jwt_auth: true` for standard JWT validation +- `enable_oauth2_auth: true` for OAuth2 introspection + +If some machine tokens are also JWT-shaped, configure `routing_overrides` to route matching tokens to OAuth2. + +```yaml title="config.yaml" +general_settings: + enable_jwt_auth: true + enable_oauth2_auth: true + litellm_jwtauth: + user_id_jwt_field: "sub" + routing_overrides: + - iss: "machine-issuer.example.com" + client_id: "MID_LITELLM" + path: "oauth2" +``` + +### Matching behavior + +- A rule matches when all configured selectors match token claims +- Supported selectors: `iss` (required), `client_id` (optional), `aud` (optional) +- Selector values support both string and list forms +- If no rule matches, LiteLLM continues with standard JWT validation + +### List-based override example + +```yaml title="config.yaml" +general_settings: + enable_jwt_auth: true + enable_oauth2_auth: true + litellm_jwtauth: + routing_overrides: + - iss: ["machine-issuer.example.com", "backup-issuer.example.com"] + client_id: ["MID_LITELLM", "MID_BACKUP"] + aud: ["api://litellm", "api://fallback"] + path: "oauth2" +``` + ## [BETA] Control Access with OIDC Roles Allow JWT tokens with supported roles to access the proxy. diff --git a/docs/my-website/docs/realtime.md b/docs/my-website/docs/realtime.md index 15a838bb7d7..08f1e47fa73 100644 --- a/docs/my-website/docs/realtime.md +++ b/docs/my-website/docs/realtime.md @@ -82,7 +82,7 @@ Run this script using node - `node test.js` const WebSocket = require("ws"); const url = "ws://0.0.0.0:4000/v1/realtime?model=openai-gpt-4o-realtime-audio"; -// const url = "wss://my-endpoint-sweden-berri992.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview"; +// const url = "wss://my-azure-endpoint.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview"; const ws = new WebSocket(url, { headers: { "api-key": `sk-1234`, diff --git a/docs/my-website/docs/tutorials/claude_responses_api.md b/docs/my-website/docs/tutorials/claude_responses_api.md index 03ac9935fd2..2a6a1236ab1 100644 --- a/docs/my-website/docs/tutorials/claude_responses_api.md +++ b/docs/my-website/docs/tutorials/claude_responses_api.md @@ -214,7 +214,7 @@ model_list: # AWS Bedrock - model_name: claude-bedrock litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0 + model: bedrock/anthropic.claude-haiku-4-5-20251001:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: us-east-1 diff --git a/docs/my-website/docs/tutorials/github_copilot_integration.md b/docs/my-website/docs/tutorials/github_copilot_integration.md index fc2682df6f9..87171ca2909 100644 --- a/docs/my-website/docs/tutorials/github_copilot_integration.md +++ b/docs/my-website/docs/tutorials/github_copilot_integration.md @@ -141,7 +141,7 @@ Route requests to Claude on Bedrock: model_list: - model_name: bedrock-claude litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0 + model: bedrock/anthropic.claude-haiku-4-5-20251001:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: us-east-1 diff --git a/docs/my-website/docs/tutorials/google_genai_sdk.md b/docs/my-website/docs/tutorials/google_genai_sdk.md index b0538795c4d..fdb55ee1a26 100644 --- a/docs/my-website/docs/tutorials/google_genai_sdk.md +++ b/docs/my-website/docs/tutorials/google_genai_sdk.md @@ -282,7 +282,7 @@ Route `gemini-2.5-flash` requests to Claude on Bedrock: model_list: - model_name: bedrock-claude litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0 + model: bedrock/anthropic.claude-haiku-4-5-20251001:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: us-east-1 diff --git a/docs/my-website/docs/tutorials/litellm_gemini_cli.md b/docs/my-website/docs/tutorials/litellm_gemini_cli.md index a36d898d7da..542d2237758 100644 --- a/docs/my-website/docs/tutorials/litellm_gemini_cli.md +++ b/docs/my-website/docs/tutorials/litellm_gemini_cli.md @@ -127,7 +127,7 @@ Route `gemini-2.5-pro` requests to Claude on Bedrock: model_list: - model_name: bedrock-claude litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0 + model: bedrock/anthropic.claude-haiku-4-5-20251001:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: us-east-1 @@ -149,7 +149,7 @@ model_list: api_key: os.environ/ANTHROPIC_API_KEY - model_name: anthropic-claude litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0 + model: bedrock/anthropic.claude-haiku-4-5-20251001:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: us-east-1 diff --git a/docs/my-website/docs/tutorials/litellm_qwen_code_cli.md b/docs/my-website/docs/tutorials/litellm_qwen_code_cli.md index 06b46a6f895..00eaa58abbd 100644 --- a/docs/my-website/docs/tutorials/litellm_qwen_code_cli.md +++ b/docs/my-website/docs/tutorials/litellm_qwen_code_cli.md @@ -129,7 +129,7 @@ Route `qwen-code` requests to Claude on Bedrock: model_list: - model_name: bedrock-claude litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0 + model: bedrock/anthropic.claude-haiku-4-5-20251001:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: us-east-1 @@ -151,7 +151,7 @@ model_list: api_key: os.environ/ANTHROPIC_API_KEY - model_name: anthropic-claude litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0 + model: bedrock/anthropic.claude-haiku-4-5-20251001:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: us-east-1 diff --git a/docs/my-website/docusaurus.config.js b/docs/my-website/docusaurus.config.js index 81c08609e5d..0102d96ee6f 100644 --- a/docs/my-website/docusaurus.config.js +++ b/docs/my-website/docusaurus.config.js @@ -284,8 +284,8 @@ const config = { label: 'Enterprise', to: "docs/enterprise" }, + { to: '/release_notes', label: 'Changelog', position: 'left' }, { to: '/blog', label: 'Blog', position: 'left' }, - { to: '/release_notes', label: 'Release Notes', position: 'left' }, { href: 'https://github.com/BerriAI/litellm', position: 'right', diff --git a/docs/my-website/img/april_townhall_banner.png b/docs/my-website/img/april_townhall_banner.png new file mode 100644 index 00000000000..e589101f2fc Binary files /dev/null and b/docs/my-website/img/april_townhall_banner.png differ diff --git a/docs/my-website/package-lock.json b/docs/my-website/package-lock.json index a3e9cb61428..56684b737de 100644 --- a/docs/my-website/package-lock.json +++ b/docs/my-website/package-lock.json @@ -13,18 +13,18 @@ "@docusaurus/plugin-ideal-image": "3.8.1", "@docusaurus/preset-classic": "3.8.1", "@docusaurus/theme-mermaid": "3.8.1", - "@inkeep/cxkit-docusaurus": "^0.5.89", - "@mdx-js/react": "^3.0.0", - "clsx": "^1.2.1", - "prism-react-renderer": "^1.3.5", - "react": "^18.0.0 || ^19.0.0", - "react-dom": "^18.0.0 || ^19.0.0", - "sharp": "^0.32.6", - "uuid": "^9.0.1" + "@inkeep/cxkit-docusaurus": "0.5.107", + "@mdx-js/react": "3.1.1", + "clsx": "1.2.1", + "prism-react-renderer": "1.3.5", + "react": "18.3.1", + "react-dom": "18.3.1", + "sharp": "0.32.6", + "uuid": "9.0.1" }, "devDependencies": { "@docusaurus/module-type-aliases": "3.8.1", - "dotenv": "^16.4.5" + "dotenv": "16.6.1" }, "engines": { "node": ">=16.14", @@ -20403,13 +20403,6 @@ "url": "https://opencollective.com/webpack" } }, - "node_modules/search-insights": { - "version": "2.17.3", - "resolved": "https://registry.npmjs.org/search-insights/-/search-insights-2.17.3.tgz", - "integrity": "sha512-RQPdCYTa8A68uM2jwxoY842xDhvx3E5LFL1LxvxCNMev4o5mLuokczhzjAgGwUZBAmOKZknArSxLKmXtIi2AxQ==", - "license": "MIT", - "peer": true - }, "node_modules/section-matter": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/section-matter/-/section-matter-1.0.0.tgz", diff --git a/docs/my-website/package.json b/docs/my-website/package.json index 2802d24b57e..73ff62dcb43 100644 --- a/docs/my-website/package.json +++ b/docs/my-website/package.json @@ -19,18 +19,18 @@ "@docusaurus/plugin-ideal-image": "3.8.1", "@docusaurus/preset-classic": "3.8.1", "@docusaurus/theme-mermaid": "3.8.1", - "@inkeep/cxkit-docusaurus": "^0.5.89", - "@mdx-js/react": "^3.0.0", - "clsx": "^1.2.1", - "prism-react-renderer": "^1.3.5", - "react": "^18.0.0 || ^19.0.0", - "react-dom": "^18.0.0 || ^19.0.0", - "sharp": "^0.32.6", - "uuid": "^9.0.1" + "@inkeep/cxkit-docusaurus": "0.5.107", + "@mdx-js/react": "3.1.1", + "clsx": "1.2.1", + "prism-react-renderer": "1.3.5", + "react": "18.3.1", + "react-dom": "18.3.1", + "sharp": "0.32.6", + "uuid": "9.0.1" }, "devDependencies": { "@docusaurus/module-type-aliases": "3.8.1", - "dotenv": "^16.4.5" + "dotenv": "16.6.1" }, "browserslist": { "production": [ diff --git a/litellm-js/proxy/package.json b/litellm-js/proxy/package.json index f63cf36d2ed..7195c6b3601 100644 --- a/litellm-js/proxy/package.json +++ b/litellm-js/proxy/package.json @@ -4,11 +4,11 @@ "deploy": "wrangler deploy --minify src/index.ts" }, "dependencies": { - "hono": "^4.1.4", - "openai": "^4.29.2" + "hono": "4.1.4", + "openai": "4.29.2" }, "devDependencies": { - "@cloudflare/workers-types": "^4.20240208.0", - "wrangler": "^3.32.0" + "@cloudflare/workers-types": "4.20240208.0", + "wrangler": "3.32.0" } } diff --git a/litellm-js/spend-logs/package-lock.json b/litellm-js/spend-logs/package-lock.json index b24ff0a4940..3784fb78658 100644 --- a/litellm-js/spend-logs/package-lock.json +++ b/litellm-js/spend-logs/package-lock.json @@ -5,12 +5,12 @@ "packages": { "": { "dependencies": { - "@hono/node-server": "^1.10.1", - "hono": "^4.12.7" + "@hono/node-server": "1.19.6", + "hono": "4.12.7" }, "devDependencies": { - "@types/node": "^20.11.17", - "tsx": "^4.7.1" + "@types/node": "20.19.25", + "tsx": "4.20.6" } }, "node_modules/@esbuild/aix-ppc64": { diff --git a/litellm-js/spend-logs/package.json b/litellm-js/spend-logs/package.json index 8bccbdba27b..742dafe2b8e 100644 --- a/litellm-js/spend-logs/package.json +++ b/litellm-js/spend-logs/package.json @@ -3,11 +3,11 @@ "dev": "tsx watch src/index.ts" }, "dependencies": { - "@hono/node-server": "^1.10.1", - "hono": "^4.12.7" + "@hono/node-server": "1.19.6", + "hono": "4.12.7" }, "devDependencies": { - "@types/node": "^20.11.17", - "tsx": "^4.7.1" + "@types/node": "20.19.25", + "tsx": "4.20.6" } } diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260331000000_add_prompt_environment_and_created_by/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260331000000_add_prompt_environment_and_created_by/migration.sql new file mode 100644 index 00000000000..74357814d8d --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260331000000_add_prompt_environment_and_created_by/migration.sql @@ -0,0 +1,12 @@ +-- AlterTable +ALTER TABLE "LiteLLM_PromptTable" ADD COLUMN "environment" TEXT NOT NULL DEFAULT 'development'; +ALTER TABLE "LiteLLM_PromptTable" ADD COLUMN "created_by" TEXT; + +-- DropIndex (old unique constraint) +DROP INDEX IF EXISTS "LiteLLM_PromptTable_prompt_id_version_key"; + +-- CreateIndex (new unique constraint) +CREATE UNIQUE INDEX "LiteLLM_PromptTable_prompt_id_version_environment_key" ON "LiteLLM_PromptTable"("prompt_id", "version", "environment"); + +-- CreateIndex (new composite index) +CREATE INDEX "LiteLLM_PromptTable_prompt_id_environment_idx" ON "LiteLLM_PromptTable"("prompt_id", "environment"); diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index d8d6015ce87..5d804d75b6a 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -321,11 +321,12 @@ model LiteLLM_MCPServerTable { byok_description String[] @default([]) byok_api_key_help_url String? source_url String? - approval_status String? @default("active") - submitted_by String? - submitted_at DateTime? - reviewed_at DateTime? - review_notes String? + // BYOM submission lifecycle + approval_status String? @default("active") + submitted_by String? + submitted_at DateTime? + reviewed_at DateTime? + review_notes String? @@index([approval_status]) } @@ -1001,12 +1002,15 @@ model LiteLLM_PromptTable { id String @id @default(uuid()) prompt_id String version Int @default(1) + environment String @default("development") + created_by String? litellm_params Json prompt_info Json? created_at DateTime @default(now()) updated_at DateTime @updatedAt - @@unique([prompt_id, version]) + @@unique([prompt_id, version, environment]) + @@index([prompt_id, environment]) @@index([prompt_id]) } diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml index 55c964557fd..6d35ed0c14d 100644 --- a/litellm-proxy-extras/pyproject.toml +++ b/litellm-proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm-proxy-extras" -version = "0.4.62" +version = "0.4.64" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." authors = ["BerriAI"] readme = "README.md" @@ -22,7 +22,7 @@ requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "0.4.62" +version = "0.4.64" version_files = [ "pyproject.toml:version", "../requirements.txt:litellm-proxy-extras==", diff --git a/litellm/a2a_protocol/litellm_completion_bridge/handler.py b/litellm/a2a_protocol/litellm_completion_bridge/handler.py index c3d2e415237..53aac1d3e6a 100644 --- a/litellm/a2a_protocol/litellm_completion_bridge/handler.py +++ b/litellm/a2a_protocol/litellm_completion_bridge/handler.py @@ -48,20 +48,19 @@ class A2ACompletionBridgeHandler: # Get provider config for custom_llm_provider custom_llm_provider = litellm_params.get("custom_llm_provider") a2a_provider_config = A2AProviderConfigManager.get_provider_config( - custom_llm_provider=custom_llm_provider + custom_llm_provider=custom_llm_provider, + model=litellm_params.get("model"), ) # If provider config exists, use it if a2a_provider_config is not None: - if api_base is None: - raise ValueError(f"api_base is required for {custom_llm_provider}") - verbose_logger.info(f"A2A: Using provider config for {custom_llm_provider}") response_data = await a2a_provider_config.handle_non_streaming( request_id=request_id, params=params, api_base=api_base, + litellm_params=litellm_params, ) return response_data @@ -147,14 +146,12 @@ class A2ACompletionBridgeHandler: # Get provider config for custom_llm_provider custom_llm_provider = litellm_params.get("custom_llm_provider") a2a_provider_config = A2AProviderConfigManager.get_provider_config( - custom_llm_provider=custom_llm_provider + custom_llm_provider=custom_llm_provider, + model=litellm_params.get("model"), ) # If provider config exists, use it if a2a_provider_config is not None: - if api_base is None: - raise ValueError(f"api_base is required for {custom_llm_provider}") - verbose_logger.info( f"A2A: Using provider config for {custom_llm_provider} (streaming)" ) @@ -163,6 +160,7 @@ class A2ACompletionBridgeHandler: request_id=request_id, params=params, api_base=api_base, + litellm_params=litellm_params, ): yield chunk diff --git a/litellm/a2a_protocol/providers/base.py b/litellm/a2a_protocol/providers/base.py index a2354b3495e..3ac1cb47fc8 100644 --- a/litellm/a2a_protocol/providers/base.py +++ b/litellm/a2a_protocol/providers/base.py @@ -3,7 +3,7 @@ Base configuration for A2A protocol providers. """ from abc import ABC, abstractmethod -from typing import Any, AsyncIterator, Dict +from typing import Any, AsyncIterator, Dict, Optional class BaseA2AProviderConfig(ABC): @@ -19,7 +19,7 @@ class BaseA2AProviderConfig(ABC): self, request_id: str, params: Dict[str, Any], - api_base: str, + api_base: Optional[str] = None, **kwargs, ) -> Dict[str, Any]: """ @@ -41,7 +41,7 @@ class BaseA2AProviderConfig(ABC): self, request_id: str, params: Dict[str, Any], - api_base: str, + api_base: Optional[str] = None, **kwargs, ) -> AsyncIterator[Dict[str, Any]]: """ diff --git a/litellm/a2a_protocol/providers/bedrock_agentcore/__init__.py b/litellm/a2a_protocol/providers/bedrock_agentcore/__init__.py new file mode 100644 index 00000000000..a61d8f98b39 --- /dev/null +++ b/litellm/a2a_protocol/providers/bedrock_agentcore/__init__.py @@ -0,0 +1,22 @@ +""" +Bedrock AgentCore A2A provider. + +Preserves JSON-RPC envelopes for AgentCore agents that speak A2A natively, +bypassing the completion bridge that would otherwise strip the envelope. +""" + +from litellm.a2a_protocol.providers.bedrock_agentcore.config import ( + BedrockAgentCoreA2AConfig, +) +from litellm.a2a_protocol.providers.bedrock_agentcore.handler import ( + BedrockAgentCoreA2AHandler, +) +from litellm.a2a_protocol.providers.bedrock_agentcore.transformation import ( + BedrockAgentCoreA2ATransformation, +) + +__all__ = [ + "BedrockAgentCoreA2AConfig", + "BedrockAgentCoreA2AHandler", + "BedrockAgentCoreA2ATransformation", +] diff --git a/litellm/a2a_protocol/providers/bedrock_agentcore/config.py b/litellm/a2a_protocol/providers/bedrock_agentcore/config.py new file mode 100644 index 00000000000..679e19c23cd --- /dev/null +++ b/litellm/a2a_protocol/providers/bedrock_agentcore/config.py @@ -0,0 +1,61 @@ +""" +Bedrock AgentCore A2A provider configuration. +""" + +from typing import Any, AsyncIterator, Dict, Optional + +from litellm.a2a_protocol.providers.base import BaseA2AProviderConfig +from litellm.a2a_protocol.providers.bedrock_agentcore.handler import ( + BedrockAgentCoreA2AHandler, +) + + +class BedrockAgentCoreA2AConfig(BaseA2AProviderConfig): + """ + Provider configuration for Bedrock AgentCore A2A-native agents. + + AgentCore agents that speak A2A natively expect the full JSON-RPC envelope. + This config bypasses the completion bridge and forwards requests directly, + deriving the endpoint URL from the model ARN and signing with SigV4/JWT. + """ + + async def handle_non_streaming( + self, + request_id: str, + params: Dict[str, Any], + api_base: Optional[str] = None, + **kwargs, + ) -> Dict[str, Any]: + """Handle non-streaming request to AgentCore A2A agent.""" + litellm_params = kwargs.get("litellm_params") + if not litellm_params: + raise ValueError( + "litellm_params is required for BedrockAgentCoreA2AConfig " + "(must contain model with AgentCore ARN)" + ) + return await BedrockAgentCoreA2AHandler.handle_non_streaming( + request_id=request_id, + params=params, + litellm_params=litellm_params, + ) + + async def handle_streaming( + self, + request_id: str, + params: Dict[str, Any], + api_base: Optional[str] = None, + **kwargs, + ) -> AsyncIterator[Dict[str, Any]]: + """Handle streaming request to AgentCore A2A agent.""" + litellm_params = kwargs.get("litellm_params") + if not litellm_params: + raise ValueError( + "litellm_params is required for BedrockAgentCoreA2AConfig " + "(must contain model with AgentCore ARN)" + ) + async for chunk in BedrockAgentCoreA2AHandler.handle_streaming( + request_id=request_id, + params=params, + litellm_params=litellm_params, + ): + yield chunk diff --git a/litellm/a2a_protocol/providers/bedrock_agentcore/handler.py b/litellm/a2a_protocol/providers/bedrock_agentcore/handler.py new file mode 100644 index 00000000000..d7445dfc252 --- /dev/null +++ b/litellm/a2a_protocol/providers/bedrock_agentcore/handler.py @@ -0,0 +1,134 @@ +""" +Handler for Bedrock AgentCore A2A-native agents. + +Sends JSON-RPC envelopes directly to AgentCore endpoints, bypassing the +completion bridge that would otherwise strip the envelope. +""" + +import json +from typing import Any, AsyncIterator, Dict, cast + +from litellm._logging import verbose_logger +from litellm.a2a_protocol.providers.bedrock_agentcore.transformation import ( + BedrockAgentCoreA2ATransformation, +) +from litellm.llms.custom_httpx.http_handler import get_async_httpx_client +from litellm.types.llms.custom_http import httpxSpecialProvider + + +class BedrockAgentCoreA2AHandler: + """ + Handler for Bedrock AgentCore A2A requests. + + Constructs JSON-RPC envelopes, signs them via AmazonAgentCoreConfig, + and POSTs directly to the AgentCore endpoint. + """ + + @staticmethod + async def handle_non_streaming( + request_id: str, + params: Dict[str, Any], + litellm_params: Dict[str, Any], + ) -> Dict[str, Any]: + """ + Handle non-streaming A2A request to AgentCore. + + Args: + request_id: A2A JSON-RPC request ID + params: A2A MessageSendParams containing the message + litellm_params: Agent's litellm_params (model, api_key, etc.) + + Returns: + A2A JSON-RPC response dict from the AgentCore agent + """ + url, headers, body = ( + BedrockAgentCoreA2ATransformation.get_url_and_signed_request( + request_id=request_id, + params=params, + litellm_params=litellm_params, + method="message/send", + ) + ) + + verbose_logger.info( + f"BedrockAgentCore A2A: Sending non-streaming request to {url}" + ) + + client = get_async_httpx_client( + llm_provider=cast(Any, httpxSpecialProvider.A2AProvider), + ) + response = await client.post( + url, + headers=headers, + data=body, + ) + response.raise_for_status() + response_data = response.json() + + if "error" in response_data: + verbose_logger.warning( + f"BedrockAgentCore A2A: Agent returned error: {response_data['error']}" + ) + + return response_data + + @staticmethod + async def handle_streaming( + request_id: str, + params: Dict[str, Any], + litellm_params: Dict[str, Any], + ) -> AsyncIterator[Dict[str, Any]]: + """ + Handle streaming A2A request to AgentCore. + + Args: + request_id: A2A JSON-RPC request ID + params: A2A MessageSendParams containing the message + litellm_params: Agent's litellm_params (model, api_key, etc.) + + Yields: + A2A streaming response events from the AgentCore agent + """ + url, headers, body = ( + BedrockAgentCoreA2ATransformation.get_url_and_signed_request( + request_id=request_id, + params=params, + litellm_params=litellm_params, + method="message/send", + stream=True, + ) + ) + + verbose_logger.info( + f"BedrockAgentCore A2A: Sending streaming request to {url}" + ) + + client = get_async_httpx_client( + llm_provider=cast(Any, httpxSpecialProvider.A2AProvider), + ) + response = await client.post( + url, + headers=headers, + data=body, + stream=True, + ) + response.raise_for_status() + + # Check content type — AgentCore may return JSON instead of SSE + content_type = response.headers.get("content-type", "").lower() + + if "application/json" in content_type: + # Single JSON response fallback (not SSE) + verbose_logger.debug( + "BedrockAgentCore A2A streaming: received JSON instead of SSE, " + "yielding as single event" + ) + response_body = await response.aread() + response_data = json.loads(response_body) + yield response_data + else: + # SSE stream — parse data: lines + async for event in BedrockAgentCoreA2ATransformation.parse_sse_events( + response + ): + yield event diff --git a/litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py b/litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py new file mode 100644 index 00000000000..44dc10fe2b7 --- /dev/null +++ b/litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py @@ -0,0 +1,134 @@ +""" +Transformation layer for Bedrock AgentCore A2A provider. + +Constructs JSON-RPC envelopes, derives AgentCore URLs from model ARNs, +and signs requests via AmazonAgentCoreConfig (SigV4 or JWT). +""" + +import json +from typing import Any, AsyncIterator, Dict, Tuple + +from litellm._logging import verbose_logger +from litellm.llms.bedrock.chat.agentcore.transformation import AmazonAgentCoreConfig + + +class BedrockAgentCoreA2ATransformation: + """ + Request/response transformation for Bedrock AgentCore A2A agents. + + Reuses AmazonAgentCoreConfig for URL construction, ARN parsing, + and request signing. No logic is duplicated. + """ + + @staticmethod + def get_url_and_signed_request( + request_id: str, + params: Dict[str, Any], + litellm_params: Dict[str, Any], + method: str = "message/send", + stream: bool = False, + ) -> Tuple[str, dict, bytes]: + """ + Build the AgentCore URL, construct a JSON-RPC envelope, and sign the request. + + Args: + request_id: A2A JSON-RPC request ID + params: A2A MessageSendParams + litellm_params: Agent's litellm_params (model, api_key, etc.) + method: JSON-RPC method name (default: "message/send") + stream: Whether this is a streaming request + + Returns: + Tuple of (url, signed_headers, signed_body_bytes) + """ + # Extract model and strip the "bedrock/" prefix + # "bedrock/agentcore/arn:aws:..." → "agentcore/arn:aws:..." + model = litellm_params.get("model", "") + if model.startswith("bedrock/"): + agentcore_model = model[len("bedrock/") :] + else: + agentcore_model = model + + # Build optional_params from litellm_params (everything except model and custom_llm_provider) + optional_params = { + k: v + for k, v in litellm_params.items() + if k not in ("model", "custom_llm_provider") + } + + agentcore_config = AmazonAgentCoreConfig() + + # Derive URL from ARN + url = agentcore_config.get_complete_url( + api_base=optional_params.get("api_base"), + api_key=optional_params.get("api_key"), + model=agentcore_model, + optional_params=optional_params, + litellm_params=litellm_params, + stream=stream, + ) + + # Construct JSON-RPC 2.0 envelope + json_rpc_body = { + "jsonrpc": "2.0", + "method": method, + "id": request_id, + "params": params, + } + + # Set required AgentCore session headers (normally set by transform_request, + # which we skip because it also builds {"prompt": "..."}) + headers: dict = {} + session_id = agentcore_config._get_runtime_session_id(optional_params) + headers["X-Amzn-Bedrock-AgentCore-Runtime-Session-Id"] = session_id + runtime_user_id = agentcore_config._get_runtime_user_id(optional_params) + if runtime_user_id: + headers["X-Amzn-Bedrock-AgentCore-Runtime-User-Id"] = runtime_user_id + + # Sign the request (SigV4 or JWT depending on api_key presence) + signed_headers, signed_body = agentcore_config.sign_request( + headers=headers, + optional_params=optional_params, + request_data=json_rpc_body, + api_base=url, + api_key=optional_params.get("api_key"), + model=agentcore_model, + stream=stream, + ) + + # sign_request returns Optional[bytes] — ensure we have bytes + if signed_body is None: + signed_body = json.dumps(json_rpc_body).encode() + + return url, signed_headers, signed_body + + @staticmethod + async def parse_sse_events(response: Any) -> AsyncIterator[Dict[str, Any]]: + """ + Parse SSE events from an httpx streaming response. + + Reads line-by-line, parses `data:` lines as JSON, and yields each parsed dict. + + Args: + response: httpx streaming response + + Yields: + Parsed JSON dicts from SSE data lines + """ + async for line in response.aiter_lines(): + line = line.strip() + if not line: + continue + + if line.startswith("data:"): + data_str = line[len("data:") :].strip() + if not data_str: + continue + try: + event = json.loads(data_str) + yield event + except json.JSONDecodeError: + verbose_logger.debug( + f"BedrockAgentCore A2A: Skipping non-JSON SSE line: {data_str[:100]}" + ) + continue diff --git a/litellm/a2a_protocol/providers/config_manager.py b/litellm/a2a_protocol/providers/config_manager.py index a8b9566c171..d684efd4756 100644 --- a/litellm/a2a_protocol/providers/config_manager.py +++ b/litellm/a2a_protocol/providers/config_manager.py @@ -19,12 +19,14 @@ class A2AProviderConfigManager: @staticmethod def get_provider_config( custom_llm_provider: Optional[str], + model: Optional[str] = None, ) -> Optional[BaseA2AProviderConfig]: """ Get the provider configuration for a given custom_llm_provider. Args: custom_llm_provider: The provider identifier (e.g., "pydantic_ai_agents") + model: The model string (used to distinguish sub-providers, e.g. agentcore vs other bedrock) Returns: Provider configuration instance or None if not found @@ -39,9 +41,11 @@ class A2AProviderConfigManager: return PydanticAIProviderConfig() - # Add more providers here as needed - # elif custom_llm_provider == "another_provider": - # from litellm.a2a_protocol.providers.another_provider.config import AnotherProviderConfig - # return AnotherProviderConfig() + if custom_llm_provider == "bedrock" and model and "agentcore" in model: + from litellm.a2a_protocol.providers.bedrock_agentcore.config import ( + BedrockAgentCoreA2AConfig, + ) + + return BedrockAgentCoreA2AConfig() return None diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 29d28b8c896..3b73b853eca 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -545,8 +545,8 @@ def cost_per_token( # noqa: PLR0915 ) if ( - model_info.get("input_cost_per_token", 0) > 0 - or model_info.get("output_cost_per_token", 0) > 0 + (model_info.get("input_cost_per_token") or 0.0) > 0 + or (model_info.get("output_cost_per_token") or 0.0) > 0 ): return generic_cost_per_token( model=model, diff --git a/litellm/litellm_core_utils/llm_request_utils.py b/litellm/litellm_core_utils/llm_request_utils.py index a2292d6e00f..dc70069ac5a 100644 --- a/litellm/litellm_core_utils/llm_request_utils.py +++ b/litellm/litellm_core_utils/llm_request_utils.py @@ -56,9 +56,8 @@ def pick_cheapest_chat_models_from_llm_provider(custom_llm_provider: str, n=1): continue if model_info.get("mode") != "chat": continue - _cost = model_info.get("input_cost_per_token", 0) + model_info.get( - "output_cost_per_token", 0 - ) + _cost = (model_info.get("input_cost_per_token") or 0.0) + (model_info.get( + "output_cost_per_token") or 0.0) model_costs.append((model, _cost)) # Sort by cost (ascending) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index cec61405ebb..46e60c24d39 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -337,24 +337,35 @@ def _insert_assistant_continue_message( """ Add assistant continuation messages between consecutive user messages. - Only checks directly adjacent messages to preserve backward compatibility. + Skips tool messages and assistant messages with tool calls in the + alternation check, matching strict templates like llama.cpp. """ if not ensure_alternating_roles or len(messages) <= 1: return messages continue_message = assistant_continue_message or DEFAULT_ASSISTANT_CONTINUE_MESSAGE + # Find indexes where assistant_continue should be inserted (before that index) + insert_before_indexes: set = set() + + for i in range(len(messages)): + curr = messages[i] + if _counts_for_alternation(curr) and curr["role"] == "user": + # Look backwards for the previous counted message + j = i - 1 + while j >= 0: + if _counts_for_alternation(messages[j]): + if messages[j]["role"] == "user": + insert_before_indexes.add(i) + break + j -= 1 + + # Build the result with assistant_continue inserted at the right positions modified_messages: List[AllMessageValues] = [] for i, message in enumerate(messages): - if ( - i < len(messages) - 1 - and message.get("role") == "user" - and messages[i + 1].get("role") == "user" - ): - modified_messages.append(message) + if i in insert_before_indexes: modified_messages.append(continue_message) - else: - modified_messages.append(message) + modified_messages.append(message) return modified_messages diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/fake_stream_iterator.py b/litellm/llms/anthropic/experimental_pass_through/messages/fake_stream_iterator.py index 80afea78504..7fc9b00f2c7 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/fake_stream_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/fake_stream_iterator.py @@ -38,6 +38,102 @@ class FakeAnthropicMessagesStreamIterator: self.chunks = self._create_streaming_chunks() self.current_index = 0 + def _create_content_block_chunks( + self, block_dict: Dict[str, Any], index: int + ) -> List[bytes]: + """Build SSE chunks for a single content block.""" + chunks = [] + block_type = block_dict.get("type") + + if block_type == "text": + content_block_start = { + "type": "content_block_start", + "index": index, + "content_block": {"type": "text", "text": ""}, + } + chunks.append( + f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode() + ) + text = block_dict.get("text", "") + content_block_delta = { + "type": "content_block_delta", + "index": index, + "delta": {"type": "text_delta", "text": text}, + } + chunks.append( + f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode() + ) + + elif block_type == "thinking": + content_block_start = { + "type": "content_block_start", + "index": index, + "content_block": {"type": "thinking", "thinking": "", "signature": ""}, + } + chunks.append( + f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode() + ) + thinking_text = block_dict.get("thinking", "") + if thinking_text: + content_block_delta = { + "type": "content_block_delta", + "index": index, + "delta": {"type": "thinking_delta", "thinking": thinking_text}, + } + chunks.append( + f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode() + ) + signature = block_dict.get("signature", "") + if signature: + signature_delta = { + "type": "content_block_delta", + "index": index, + "delta": {"type": "signature_delta", "signature": signature}, + } + chunks.append( + f"event: content_block_delta\ndata: {json.dumps(signature_delta)}\n\n".encode() + ) + + elif block_type == "redacted_thinking": + content_block_start = { + "type": "content_block_start", + "index": index, + "content_block": {"type": "redacted_thinking"}, + } + chunks.append( + f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode() + ) + + elif block_type == "tool_use": + content_block_start = { + "type": "content_block_start", + "index": index, + "content_block": { + "type": "tool_use", + "id": block_dict.get("id"), + "name": block_dict.get("name"), + "input": {}, + }, + } + chunks.append( + f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode() + ) + input_data = block_dict.get("input", {}) + content_block_delta = { + "type": "content_block_delta", + "index": index, + "delta": {"type": "input_json_delta", "partial_json": json.dumps(input_data)}, + } + chunks.append( + f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode() + ) + + content_block_stop = {"type": "content_block_stop", "index": index} + chunks.append( + f"event: content_block_stop\ndata: {json.dumps(content_block_stop)}\n\n".encode() + ) + return chunks + def _create_streaming_chunks(self) -> List[bytes]: """Convert the non-streaming response to streaming chunks""" chunks = [] @@ -69,152 +165,34 @@ class FakeAnthropicMessagesStreamIterator: # 2-4. For each content block, send start/delta/stop events content_blocks = response_dict.get("content", []) - if content_blocks: - for index, block in enumerate(content_blocks): - # Cast block to dict for easier access - block_dict = cast(Dict[str, Any], block) - block_type = block_dict.get("type") - - if block_type == "text": - # content_block_start - content_block_start = { - "type": "content_block_start", - "index": index, - "content_block": {"type": "text", "text": ""}, - } - chunks.append( - f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode() - ) - - # content_block_delta (send full text as one delta for simplicity) - text = block_dict.get("text", "") - content_block_delta = { - "type": "content_block_delta", - "index": index, - "delta": {"type": "text_delta", "text": text}, - } - chunks.append( - f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode() - ) - - # content_block_stop - content_block_stop = {"type": "content_block_stop", "index": index} - chunks.append( - f"event: content_block_stop\ndata: {json.dumps(content_block_stop)}\n\n".encode() - ) - - elif block_type == "thinking": - # content_block_start for thinking - content_block_start = { - "type": "content_block_start", - "index": index, - "content_block": { - "type": "thinking", - "thinking": "", - "signature": "", - }, - } - chunks.append( - f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode() - ) - - # content_block_delta for thinking text - thinking_text = block_dict.get("thinking", "") - if thinking_text: - content_block_delta = { - "type": "content_block_delta", - "index": index, - "delta": { - "type": "thinking_delta", - "thinking": thinking_text, - }, - } - chunks.append( - f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode() - ) - - # content_block_delta for signature (if present) - signature = block_dict.get("signature", "") - if signature: - signature_delta = { - "type": "content_block_delta", - "index": index, - "delta": { - "type": "signature_delta", - "signature": signature, - }, - } - chunks.append( - f"event: content_block_delta\ndata: {json.dumps(signature_delta)}\n\n".encode() - ) - - # content_block_stop - content_block_stop = {"type": "content_block_stop", "index": index} - chunks.append( - f"event: content_block_stop\ndata: {json.dumps(content_block_stop)}\n\n".encode() - ) - - elif block_type == "redacted_thinking": - # content_block_start for redacted_thinking - content_block_start = { - "type": "content_block_start", - "index": index, - "content_block": {"type": "redacted_thinking"}, - } - chunks.append( - f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode() - ) - - # content_block_stop (no delta for redacted thinking) - content_block_stop = {"type": "content_block_stop", "index": index} - chunks.append( - f"event: content_block_stop\ndata: {json.dumps(content_block_stop)}\n\n".encode() - ) - - elif block_type == "tool_use": - # content_block_start - content_block_start = { - "type": "content_block_start", - "index": index, - "content_block": { - "type": "tool_use", - "id": block_dict.get("id"), - "name": block_dict.get("name"), - "input": {}, - }, - } - chunks.append( - f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode() - ) - - # content_block_delta (send input as JSON delta) - input_data = block_dict.get("input", {}) - content_block_delta = { - "type": "content_block_delta", - "index": index, - "delta": { - "type": "input_json_delta", - "partial_json": json.dumps(input_data), - }, - } - chunks.append( - f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode() - ) - - # content_block_stop - content_block_stop = {"type": "content_block_stop", "index": index} - chunks.append( - f"event: content_block_stop\ndata: {json.dumps(content_block_stop)}\n\n".encode() - ) + for index, block in enumerate(content_blocks): + block_dict = cast(Dict[str, Any], block) + chunks.extend(self._create_content_block_chunks(block_dict, index)) # 5. message_delta event (with final usage and stop_reason) + # Include cache usage fields so clients that only read message_delta + # (like Claude Code's SDK) see the full input token breakdown. + delta_usage: Dict[str, Any] = { + "output_tokens": usage.get("output_tokens", 0) if usage else 0, + } + if usage: + if usage.get("input_tokens") is not None: + delta_usage["input_tokens"] = usage["input_tokens"] + if usage.get("cache_creation_input_tokens") is not None: + delta_usage["cache_creation_input_tokens"] = usage[ + "cache_creation_input_tokens" + ] + if usage.get("cache_read_input_tokens") is not None: + delta_usage["cache_read_input_tokens"] = usage[ + "cache_read_input_tokens" + ] message_delta = { "type": "message_delta", "delta": { "stop_reason": response_dict.get("stop_reason"), "stop_sequence": response_dict.get("stop_sequence"), }, - "usage": {"output_tokens": usage.get("output_tokens", 0) if usage else 0}, + "usage": delta_usage, } chunks.append( f"event: message_delta\ndata: {json.dumps(message_delta)}\n\n".encode() diff --git a/litellm/llms/azure_ai/cost_calculator.py b/litellm/llms/azure_ai/cost_calculator.py index 3cca61b2186..067181b946a 100644 --- a/litellm/llms/azure_ai/cost_calculator.py +++ b/litellm/llms/azure_ai/cost_calculator.py @@ -54,7 +54,7 @@ def calculate_azure_model_router_flat_cost(model: str, prompt_tokens: int) -> fl model_info = get_model_info(model="model_router", custom_llm_provider="azure_ai") router_flat_cost_per_token = model_info.get("input_cost_per_token", 0) - if router_flat_cost_per_token > 0: + if router_flat_cost_per_token and router_flat_cost_per_token > 0: return prompt_tokens * router_flat_cost_per_token return 0.0 diff --git a/litellm/llms/bedrock/chat/agentcore/transformation.py b/litellm/llms/bedrock/chat/agentcore/transformation.py index d6eb5a734c4..f066322b814 100644 --- a/litellm/llms/bedrock/chat/agentcore/transformation.py +++ b/litellm/llms/bedrock/chat/agentcore/transformation.py @@ -19,6 +19,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM +from litellm.llms.a2a.common_utils import extract_text_from_a2a_response from litellm.llms.bedrock.common_utils import BedrockError from litellm.types.llms.bedrock_agentcore import ( AgentCoreMessage, @@ -343,6 +344,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): Parse direct JSON response (non-streaming). Supports multiple agent response schemas: + 0. {"jsonrpc": "2.0", "result": {"message": {"parts": [...]}}} - A2A JSON-RPC 1. {"result": {"role": "assistant", "content": [{"text": "..."}]}} - standard AgentCore 2. {"response": [{"text": "..."}]} - Strands agent format 3. {"result": "plain text"} or {"response": "plain text"} - simple string @@ -361,6 +363,18 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): final_message=None, ) + # Strategy 0: A2A JSON-RPC format + # {"jsonrpc": "2.0", "result": {"message": {"parts": [{"kind": "text", "text": "..."}]}}} + if "jsonrpc" in response_json: + content = extract_text_from_a2a_response(response_json) + if content: + return AgentCoreParsedResponse( + content=content, + usage=None, + final_message=None, + ) + # Fall through to other strategies if A2A extraction returned empty + # Strategy 1: {"result": {"content": [{"text": "..."}]}} - standard AgentCore format if "result" in response_json and isinstance(response_json["result"], dict): result = response_json["result"] diff --git a/litellm/llms/bedrock/chat/invoke_handler.py b/litellm/llms/bedrock/chat/invoke_handler.py index 1077731779d..67bba28e4c5 100644 --- a/litellm/llms/bedrock/chat/invoke_handler.py +++ b/litellm/llms/bedrock/chat/invoke_handler.py @@ -855,6 +855,32 @@ class BedrockLLM(BaseAWSLLM): endpoint_url = f"{endpoint_url}/model/{modelId}/invoke" proxy_endpoint_url = f"{proxy_endpoint_url}/model/{modelId}/invoke" + if acompletion and provider == "anthropic" and self.is_claude_messages_api_model( + model + ): + if isinstance(client, HTTPHandler): + client = None + return self._async_anthropic_messages_completion( + model=model, + messages=messages, + endpoint_url=endpoint_url, + proxy_endpoint_url=proxy_endpoint_url, + credentials=credentials, + aws_region_name=aws_region_name, + model_response=model_response, + print_verbose=print_verbose, + encoding=encoding, + logging_obj=logging_obj, + optional_params=optional_params, + stream=stream, + litellm_params=litellm_params, + logger_fn=logger_fn, + extra_headers=extra_headers, + timeout=timeout, + client=client, + stream_chunk_size=stream_chunk_size, + ) # type: ignore[return-value] + prompt, chat_history = self.convert_messages_to_prompt( model, messages, provider, custom_prompt_dict ) @@ -1148,6 +1174,95 @@ class BedrockLLM(BaseAWSLLM): encoding=encoding, ) + async def _async_anthropic_messages_completion( + self, + model: str, + messages: list, + endpoint_url: str, + proxy_endpoint_url: str, + credentials, + aws_region_name: str, + model_response: ModelResponse, + print_verbose: Callable, + encoding, + logging_obj: Logging, + optional_params: dict, + stream, + litellm_params=None, + logger_fn=None, + extra_headers: Optional[dict] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + client: Optional[AsyncHTTPHandler] = None, + stream_chunk_size: int = 1024, + ) -> Union[ModelResponse, CustomStreamWrapper]: + transformed_request = await litellm.AmazonAnthropicClaudeConfig().async_transform_request( + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params or {}, + headers=extra_headers or {}, + ) + data = json.dumps(transformed_request) + + headers = {"Content-Type": "application/json"} + if extra_headers is not None: + headers = {"Content-Type": "application/json", **extra_headers} + prepped = self.get_request_headers( + credentials=credentials, + aws_region_name=aws_region_name, + extra_headers=extra_headers, + endpoint_url=endpoint_url, + data=data, + headers=headers, + ) + + logging_obj.pre_call( + input=messages, + api_key="", + additional_args={ + "complete_input_dict": data, + "api_base": proxy_endpoint_url, + "headers": prepped.headers, + }, + ) + + if stream is True: + return await self.async_streaming( + model=model, + messages=messages, + data=data, + api_base=proxy_endpoint_url, + model_response=model_response, + print_verbose=print_verbose, + encoding=encoding, + logging_obj=logging_obj, + optional_params=optional_params, + stream=True, + litellm_params=litellm_params, + logger_fn=logger_fn, + headers=prepped.headers, + timeout=timeout, + client=client, + stream_chunk_size=stream_chunk_size, + ) + return await self.async_completion( + model=model, + messages=messages, + data=data, + api_base=proxy_endpoint_url, + model_response=model_response, + print_verbose=print_verbose, + encoding=encoding, + logging_obj=logging_obj, + optional_params=optional_params, + stream=stream, # type: ignore + litellm_params=litellm_params, + logger_fn=logger_fn, + headers=prepped.headers, + timeout=timeout, + client=client, + ) + async def async_completion( self, model: str, diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py index 7936b6ea644..2b28473ad3e 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py @@ -2,6 +2,14 @@ from typing import TYPE_CHECKING, Any, List, Optional import httpx +from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers +from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_anthropic_image_obj, +) +from litellm.litellm_core_utils.prompt_templates.image_handling import ( + async_convert_url_to_base64, + convert_url_to_base64, +) from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( AmazonInvokeConfig, @@ -85,8 +93,62 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): litellm_params: dict, headers: dict, ) -> dict: - # Filter out AWS authentication parameters before passing to Anthropic transformation - # AWS params should only be used for signing requests, not included in request body + _anthropic_request = self._build_bedrock_anthropic_request_base( + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers=headers, + ) + + self._convert_document_url_sources_to_base64(_anthropic_request) + beta_list = self._compute_bedrock_invoke_beta_headers( + model=model, + messages=messages, + optional_params=optional_params, + headers=headers, + ) + if beta_list: + _anthropic_request["anthropic_beta"] = beta_list + + return _anthropic_request + + async def async_transform_request( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + _anthropic_request = self._build_bedrock_anthropic_request_base( + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers=headers, + ) + + await self._async_convert_document_url_sources_to_base64(_anthropic_request) + beta_list = self._compute_bedrock_invoke_beta_headers( + model=model, + messages=messages, + optional_params=optional_params, + headers=headers, + ) + if beta_list: + _anthropic_request["anthropic_beta"] = beta_list + + return _anthropic_request + + def _build_bedrock_anthropic_request_base( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: filtered_params = { k: v for k, v in optional_params.items() @@ -94,7 +156,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): } filtered_params = self._normalize_bedrock_tool_search_tools(filtered_params) - _anthropic_request = AnthropicConfig.transform_request( + anthropic_request = AnthropicConfig.transform_request( self, model=model, messages=messages, @@ -103,28 +165,31 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): headers=headers, ) - _anthropic_request.pop("model", None) - _anthropic_request.pop("stream", None) - # Bedrock Invoke doesn't support output_format parameter - _anthropic_request.pop("output_format", None) - # Bedrock Invoke doesn't support output_config parameter - # Fixes: https://github.com/BerriAI/litellm/issues/22797 - _anthropic_request.pop("output_config", None) - if "anthropic_version" not in _anthropic_request: - _anthropic_request["anthropic_version"] = self.anthropic_version + anthropic_request.pop("model", None) + anthropic_request.pop("stream", None) + anthropic_request.pop("output_format", None) + anthropic_request.pop("output_config", None) + if "anthropic_version" not in anthropic_request: + anthropic_request["anthropic_version"] = self.anthropic_version # Remove `custom` field from tools (Bedrock doesn't support it) - # Claude Code sends `custom: {defer_loading: true}` on tool definitions, - # which causes Bedrock to reject the request with "Extra inputs are not permitted" - # Ref: https://github.com/BerriAI/litellm/issues/22847 - remove_custom_field_from_tools(_anthropic_request) + remove_custom_field_from_tools(anthropic_request) + return anthropic_request + def _compute_bedrock_invoke_beta_headers( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + headers: dict, + ) -> List[str]: tools = optional_params.get("tools") tool_search_used = self.is_tool_search_used(tools) programmatic_tool_calling_used = self.is_programmatic_tool_calling_used(tools) input_examples_used = self.is_input_examples_used(tools) - beta_set = set(get_anthropic_beta_from_headers(headers)) + user_beta_set = set(get_anthropic_beta_from_headers(headers)) + beta_set = set(user_beta_set) auto_betas = self.get_anthropic_beta_list( model=model, optional_params=optional_params, @@ -142,12 +207,91 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): if "opus-4" in model.lower() or "opus_4" in model.lower(): beta_set.add("tool-search-tool-2025-10-19") - # Filter out beta headers that Bedrock Invoke doesn't support - # Uses centralized configuration from anthropic_beta_headers_config.json - beta_list = list(beta_set) - _anthropic_request["anthropic_beta"] = beta_list + auto_beta_list = filter_and_transform_beta_headers( + beta_headers=list(beta_set - user_beta_set), + provider="bedrock", + ) + return sorted(user_beta_set.union(set(auto_beta_list))) - return _anthropic_request + def _convert_document_url_sources_to_base64(self, anthropic_request: dict) -> None: + """ + Bedrock Invoke does not accept document URL sources. Convert to base64 payloads. + """ + messages = anthropic_request.get("messages") + if not isinstance(messages, list): + return + + for message in messages: + if not isinstance(message, dict): + continue + content = message.get("content") + if not isinstance(content, list): + continue + + for block in content: + if not isinstance(block, dict) or block.get("type") != "document": + continue + source = block.get("source") + if not isinstance(source, dict) or source.get("type") != "url": + continue + source_url = source.get("url") + if not isinstance(source_url, str): + continue + + inferred_format: Optional[str] = None + if source_url.lower().endswith(".pdf"): + inferred_format = "application/pdf" + base64_url = convert_url_to_base64(url=source_url) + image_chunk = convert_to_anthropic_image_obj( + openai_image_url=base64_url, + format=inferred_format, + ) + block["source"] = { + "type": "base64", + "media_type": image_chunk["media_type"], + "data": image_chunk["data"], + } + + async def _async_convert_document_url_sources_to_base64( + self, anthropic_request: dict + ) -> None: + """ + Async version of document URL conversion for async completion paths. + """ + messages = anthropic_request.get("messages") + if not isinstance(messages, list): + return + + for message in messages: + if not isinstance(message, dict): + continue + content = message.get("content") + if not isinstance(content, list): + continue + + for block in content: + if not isinstance(block, dict) or block.get("type") != "document": + continue + source = block.get("source") + if not isinstance(source, dict) or source.get("type") != "url": + continue + source_url = source.get("url") + if not isinstance(source_url, str): + continue + + inferred_format: Optional[str] = None + if source_url.lower().endswith(".pdf"): + inferred_format = "application/pdf" + base64_url = await async_convert_url_to_base64(url=source_url) + image_chunk = convert_to_anthropic_image_obj( + openai_image_url=base64_url, + format=inferred_format, + ) + block["source"] = { + "type": "base64", + "media_type": image_chunk["media_type"], + "data": image_chunk["data"], + } def _normalize_bedrock_tool_search_tools(self, optional_params: dict) -> dict: """ diff --git a/litellm/llms/bedrock/image_edit/amazon_nova_canvas_image_edit_transformation.py b/litellm/llms/bedrock/image_edit/amazon_nova_canvas_image_edit_transformation.py new file mode 100644 index 00000000000..f806cd2a81a --- /dev/null +++ b/litellm/llms/bedrock/image_edit/amazon_nova_canvas_image_edit_transformation.py @@ -0,0 +1,515 @@ +""" +Amazon Nova Canvas image edit on Bedrock (InvokeModel). + +Maps OpenAI-style image edit (image + prompt, optional mask) to Nova Canvas task types: +- With mask: INPAINTING (inPaintingParams per AWS docs) +- Without mask: IMAGE_VARIATION (imageVariationParams) + +Refs: +- https://docs.aws.amazon.com/nova/latest/userguide/image-gen-access.html +- https://docs.aws.amazon.com/nova/latest/userguide/image-gen-req-resp-structure.html +""" + +from __future__ import annotations + +import base64 +import os +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple + +import httpx + +from litellm._logging import verbose_logger +from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig +from litellm.types.images.main import ImageEditOptionalRequestParams +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import FileTypes, ImageObject, ImageResponse +from litellm.utils import ( + _get_model_cost_key, + _get_potential_model_names, + get_model_info, +) + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +def _nova_canvas_task_body( + *, + image_b64: str, + mask_b64: Optional[str], + text: str, + negative_text: Optional[str], + similarity_strength: Optional[float], + task_type: Optional[str], + mask_prompt: Optional[str], + out_painting_mode: Optional[str], +) -> Dict[str, Any]: + """Build InvokeModel body task section (without imageGenerationConfig).""" + if task_type == "BACKGROUND_REMOVAL": + return { + "taskType": "BACKGROUND_REMOVAL", + "backgroundRemovalParams": {"image": image_b64}, + } + if task_type == "OUTPAINTING": + if mask_prompt is None and mask_b64 is None: + raise ValueError( + "OUTPAINTING requires either a mask image or a mask prompt. " + "Pass mask= or maskPrompt= in the request." + ) + out_params: Dict[str, Any] = { + "image": image_b64, + "text": text, + } + if mask_prompt is not None: + out_params["maskPrompt"] = mask_prompt + elif mask_b64 is not None: + out_params["maskImage"] = mask_b64 + if negative_text is not None: + out_params["negativeText"] = negative_text + if out_painting_mode is not None: + out_params["outPaintingMode"] = out_painting_mode + return { + "taskType": "OUTPAINTING", + "outPaintingParams": out_params, + } + # Honour explicit IMAGE_VARIATION even when a mask is present (mask is ignored + # for this task type; callers use INPAINTING when they want mask semantics). + if task_type == "IMAGE_VARIATION": + var_params_explicit: Dict[str, Any] = { + "images": [image_b64], + "text": text, + } + if negative_text is not None: + var_params_explicit["negativeText"] = negative_text + if similarity_strength is not None: + var_params_explicit["similarityStrength"] = similarity_strength + return { + "taskType": "IMAGE_VARIATION", + "imageVariationParams": var_params_explicit, + } + # Explicit taskType must be INPAINTING or omitted from here on; anything else is invalid. + if task_type is not None and str(task_type).strip() != "": + if task_type != "INPAINTING": + raise ValueError( + f"Unsupported Amazon Nova Canvas taskType: {task_type!r}. " + "Use BACKGROUND_REMOVAL, OUTPAINTING, IMAGE_VARIATION, INPAINTING, " + "or omit taskType for automatic routing (mask → INPAINTING, else IMAGE_VARIATION)." + ) + if mask_b64 is not None or mask_prompt is not None or task_type == "INPAINTING": + in_params: Dict[str, Any] = {"image": image_b64, "text": text} + if mask_prompt is not None: + in_params["maskPrompt"] = mask_prompt + elif mask_b64 is not None: + in_params["maskImage"] = mask_b64 + if negative_text is not None: + in_params["negativeText"] = negative_text + if "maskPrompt" not in in_params and "maskImage" not in in_params: + raise ValueError( + "Amazon Nova Canvas INPAINTING requires either maskPrompt or maskImage " + "(use OpenAI mask= for maskImage, or pass maskPrompt in optional params). " + "See https://docs.aws.amazon.com/nova/latest/userguide/image-gen-req-resp-structure.html" + ) + return {"taskType": "INPAINTING", "inPaintingParams": in_params} + var_params: Dict[str, Any] = { + "images": [image_b64], + "text": text, + } + if negative_text is not None: + var_params["negativeText"] = negative_text + if similarity_strength is not None: + var_params["similarityStrength"] = similarity_strength + return { + "taskType": "IMAGE_VARIATION", + "imageVariationParams": var_params, + } + + +def _file_types_to_b64(image: Optional[FileTypes]) -> str: + """Encode OpenAI image input to base64 string for Nova Canvas.""" + if image is None: + raise ValueError("Nova Canvas image edit requires an image input") + if hasattr(image, "read") and callable(getattr(image, "read", None)): + if hasattr(image, "seek"): + image.seek(0) # type: ignore[union-attr] + image_bytes = image.read() # type: ignore[union-attr] + return base64.b64encode(image_bytes).decode("utf-8") + if isinstance(image, bytes): + return base64.b64encode(image).decode("utf-8") + if isinstance(image, str): + return image + if isinstance(image, os.PathLike): + with open(image, "rb") as f: + return base64.b64encode(f.read()).decode("utf-8") + if isinstance(image, tuple): + raise ValueError( + "Nova Canvas image edit does not support tuple FileTypes. " + "Pass a file-like object, bytes, or a base64-encoded string." + ) + return base64.b64encode(bytes(image)).decode("utf-8") # type: ignore[arg-type] + + +def _supports_nova_canvas_image_edit_from_model_cost(model: str) -> bool: + """ + True when model_cost has supports_nova_canvas_image_edit for a resolved catalog key. + + get_model_info / ModelInfoBase omit arbitrary JSON keys, so we read model_cost + directly (same idea as supports_* bare_entry fallback). + """ + import litellm as _litellm + + if not model: + return False + + seen: set[str] = set() + candidates: List[str] = [] + + def _add(name: Optional[str]) -> None: + if name and name not in seen: + seen.add(name) + candidates.append(name) + + _add(model) + if "/" in model: + suffix = model.split("/")[-1] + _add(suffix) + _add(f"bedrock/{suffix}") + + # Cross-region inference ids (e.g. us.amazon.nova-canvas-v1:0) share pricing with + # the base model id (amazon.nova-canvas-v1:0) in model_cost. + try: + from litellm.llms.bedrock.common_utils import BedrockModelInfo + + base_model = BedrockModelInfo.get_base_model(model) + if base_model and base_model != model: + _add(base_model) + _add(f"bedrock/{base_model}") + except Exception: + pass + + try: + potential = _get_potential_model_names(model=model, custom_llm_provider=None) + for field in ( + "combined_model_name", + "combined_stripped_model_name", + "stripped_model_name", + "split_model", + ): + raw = potential.get(field) + if isinstance(raw, str): + _add(raw) + except Exception: + pass + + for name in candidates: + key = _get_model_cost_key(name) + if key is None: + continue + entry = _litellm.model_cost.get(key) or {} + if entry.get("supports_nova_canvas_image_edit") is True: + return True + return False + + +class BedrockAmazonNovaCanvasImageEditConfig(BaseImageEditConfig): + """ + Bedrock InvokeModel image edit for amazon.nova-canvas-v1:0 and regional variants. + """ + + @classmethod + def _is_nova_canvas_image_edit_model(cls, model: Optional[str] = None) -> bool: + """ + Use model_cost.supports_nova_canvas_image_edit so new Nova Canvas inference IDs + are added via model_prices_and_context_window.json only (not get_model_info, which + drops keys not on ModelInfoBase). + """ + return _supports_nova_canvas_image_edit_from_model_cost(model or "") + + def get_supported_openai_params(self, model: str) -> list: + return [ + "n", + "size", + "response_format", + "mask", + "negativeText", + "similarityStrength", + "cfgScale", + "seed", + "quality", + "taskType", + "maskPrompt", + "outPaintingMode", + "imageGenerationConfig", + ] + + def map_openai_params( + self, + image_edit_optional_params: ImageEditOptionalRequestParams, + model: str, + drop_params: bool, + ) -> Dict[str, Any]: + supported = set(self.get_supported_openai_params(model)) + mapped: Dict[str, Any] = dict(image_edit_optional_params) + _size = mapped.pop("size", None) + if _size is not None and isinstance(_size, str) and "x" in _size: + w, h = _size.split("x", 1) + try: + mapped["width"], mapped["height"] = int(w), int(h) + except ValueError: + pass + _n = mapped.pop("n", None) + if _n is not None: + mapped["numberOfImages"] = _n + _quality = mapped.pop("quality", None) + if _quality is not None: + if _quality in ("hd", "premium"): + mapped["quality"] = "premium" + elif _quality == "standard": + mapped["quality"] = "standard" + else: + # Re-emit unknown values (e.g. OpenAI "auto") so transform_image_edit_request + # forwards them and the API can reject, or drop_params can still apply upstream. + mapped["quality"] = _quality + # Accepted for OpenAI compatibility but ignored for Nova Canvas image edit; + # Bedrock returns base64 images only (no URL mode). + response_format = mapped.pop("response_format", None) + if response_format not in (None, "b64_json"): + verbose_logger.debug( + "Nova Canvas image edit ignores response_format=%s and returns base64 images", + response_format, + ) + # Drop unknown keys if drop_params + if drop_params: + for k in list(mapped.keys()): + if k.startswith("_"): + continue + if k not in supported and k not in ( + "width", + "height", + "numberOfImages", + "mask", + ): + mapped.pop(k, None) + return mapped + + def transform_image_edit_request( + self, + model: str, + prompt: Optional[str], + image: Optional[FileTypes], + image_edit_optional_request_params: Dict, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[Dict, Any]: + op = dict(image_edit_optional_request_params) + image_b64 = _file_types_to_b64(image) + + mask_raw = op.pop("mask", None) + mask_b64: Optional[str] = None + if mask_raw is not None: + mask_b64 = _file_types_to_b64(mask_raw) # type: ignore[arg-type] + + _size = op.pop("size", None) + width = op.pop("width", None) + height = op.pop("height", None) + if ( + width is None + and height is None + and _size is not None + and isinstance(_size, str) + and "x" in _size + ): + w, h = _size.split("x", 1) + try: + width, height = int(w), int(h) + except ValueError: + pass + + number_of_images = op.pop("numberOfImages", None) + quality = op.pop("quality", None) + cfg_scale = op.pop("cfgScale", None) + seed = op.pop("seed", None) + + image_generation_config: Dict[str, Any] = {} + nested_igc = op.pop("imageGenerationConfig", None) + if isinstance(nested_igc, dict): + image_generation_config.update(nested_igc) + if width is not None: + image_generation_config["width"] = width + if height is not None: + image_generation_config["height"] = height + if number_of_images is not None: + image_generation_config["numberOfImages"] = number_of_images + if quality is not None: + image_generation_config["quality"] = quality + if cfg_scale is not None: + image_generation_config["cfgScale"] = cfg_scale + if seed is not None: + image_generation_config["seed"] = seed + + task_type = op.pop("taskType", None) + if (prompt is None or prompt == "") and task_type in ( + "INPAINTING", + "OUTPAINTING", + ): + raise ValueError( + f"Amazon Nova Canvas {task_type} requires a text prompt. " + "Pass a non-empty `prompt` in your request." + ) + text = prompt if prompt is not None and prompt != "" else " " + negative_text = op.pop("negativeText", None) + similarity_strength = op.pop("similarityStrength", None) + mask_prompt = op.pop("maskPrompt", None) + out_painting_mode = op.pop("outPaintingMode", None) + + body = _nova_canvas_task_body( + image_b64=image_b64, + mask_b64=mask_b64, + text=text, + negative_text=negative_text, + similarity_strength=similarity_strength, + task_type=task_type, + mask_prompt=mask_prompt, + out_painting_mode=out_painting_mode, + ) + + # BACKGROUND_REMOVAL InvokeModel body must not include imageGenerationConfig (AWS rejects it). + if image_generation_config and body.get("taskType") != "BACKGROUND_REMOVAL": + body["imageGenerationConfig"] = image_generation_config + + return body, {} + + def transform_image_edit_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> ImageResponse: + try: + response_data = raw_response.json() + except Exception as e: + raise self.get_error_class( + error_message=f"Error parsing Nova Canvas image edit response: {e}", + status_code=raw_response.status_code, + headers=raw_response.headers, + ) + + if raw_response.status_code not in (200,): + raise self.get_error_class( + error_message=f"Nova Canvas image edit error: {response_data}", + status_code=raw_response.status_code, + headers=raw_response.headers, + ) + + images: List[str] = response_data.get("images") or [] + + if "errors" in response_data and not images: + raise self.get_error_class( + error_message=f"Nova Canvas image edit error: {response_data['errors']}", + status_code=raw_response.status_code, + headers=raw_response.headers, + ) + + # Nova Canvas InvokeModel success body uses "images" and optional "error" (AWS docs); + # it does not use Stability-style "finish_reasons". + error_msg = response_data.get("message") or response_data.get("error") + if error_msg and not images: + if not isinstance(error_msg, str): + error_msg = str(error_msg) + raise self.get_error_class( + error_message=f"Nova Canvas image edit error: {error_msg}", + status_code=raw_response.status_code, + headers=raw_response.headers, + ) + + model_response = ImageResponse() + model_response.data = [] + for image_b64 in images: + if image_b64: + model_response.data.append( + ImageObject( + b64_json=image_b64, + url=None, + revised_prompt=None, + ) + ) + + if not model_response.data: + raise self.get_error_class( + error_message="Nova Canvas image edit returned no images", + status_code=raw_response.status_code, + headers=raw_response.headers, + ) + + if not hasattr(model_response, "_hidden_params"): + model_response._hidden_params = {} + if "additional_headers" not in model_response._hidden_params: + model_response._hidden_params["additional_headers"] = {} + + try: + model_info = get_model_info(model, custom_llm_provider="bedrock") + cost_per_image = model_info.get("output_cost_per_image", 0) + if cost_per_image is not None and model_response.data: + model_response._hidden_params["additional_headers"][ + "llm_provider-x-litellm-response-cost" + ] = float(cost_per_image) * len(model_response.data) + except Exception: + pass + + return model_response + + def use_multipart_form_data(self) -> bool: + return False + + def get_complete_url( + self, + model: str, + api_base: Optional[str], + litellm_params: dict, + ) -> str: + raise NotImplementedError( + "Nova Canvas image edit URLs are built in BedrockImageEdit._prepare_request " + "(AWS runtime endpoint + model invoke path). Do not use get_complete_url for " + "this config." + ) + + def validate_environment( + self, + headers: dict, + model: str, + api_key: Optional[str] = None, + ) -> dict: + if headers is None: + headers = {} + if "Content-Type" not in headers: + headers["Content-Type"] = "application/json" + return headers + + +def get_bedrock_image_edit_config_for_model( + model: str, +) -> BaseImageEditConfig: + """ + Return the correct Bedrock image-edit config for the model id. + + Same routing as ``BedrockImageEdit.get_config_class``: Stability edit models, + Nova Canvas when marked in model_cost; otherwise raises ``ValueError``. + """ + from litellm.llms.bedrock.image_edit.stability_transformation import ( + BedrockStabilityImageEditConfig, + ) + + if BedrockStabilityImageEditConfig._is_stability_edit_model(model): + return BedrockStabilityImageEditConfig() + if BedrockAmazonNovaCanvasImageEditConfig._is_nova_canvas_image_edit_model(model): + return BedrockAmazonNovaCanvasImageEditConfig() + raise ValueError( + f"Unsupported Bedrock image-edit model: {model!r}. " + "Use a stability.* image-edit model id or add supports_nova_canvas_image_edit " + "in model_prices for this id." + ) diff --git a/litellm/llms/bedrock/image_edit/handler.py b/litellm/llms/bedrock/image_edit/handler.py index 867944f8796..90344310746 100644 --- a/litellm/llms/bedrock/image_edit/handler.py +++ b/litellm/llms/bedrock/image_edit/handler.py @@ -15,6 +15,9 @@ from pydantic import BaseModel import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging +from litellm.llms.bedrock.image_edit.amazon_nova_canvas_image_edit_transformation import ( + BedrockAmazonNovaCanvasImageEditConfig, +) from litellm.llms.bedrock.image_edit.stability_transformation import ( BedrockStabilityImageEditConfig, ) @@ -55,8 +58,15 @@ class BedrockImageEdit(BaseAWSLLM): def get_config_class(cls, model: str | None): if BedrockStabilityImageEditConfig._is_stability_edit_model(model): return BedrockStabilityImageEditConfig - else: - raise ValueError(f"Unsupported model for bedrock image edit: {model}") + if BedrockAmazonNovaCanvasImageEditConfig._is_nova_canvas_image_edit_model( + model + ): + return BedrockAmazonNovaCanvasImageEditConfig + raise ValueError( + f"Unsupported Bedrock image-edit model: {model!r}. " + "Use a stability.* image-edit model id or add supports_nova_canvas_image_edit " + "in model_prices for this id." + ) def image_edit( self, diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index e31820d7631..c1eccaebd04 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -12,6 +12,7 @@ from typing import ( import httpx +from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers from litellm.llms.anthropic.common_utils import AnthropicModelInfo from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( AnthropicMessagesConfig, @@ -54,9 +55,6 @@ class AmazonAnthropicClaudeMessagesConfig( DEFAULT_BEDROCK_ANTHROPIC_API_VERSION = "bedrock-2023-05-31" - # Beta header patterns that are not supported by Bedrock Invoke API - # These will be filtered out to prevent 400 "invalid beta flag" errors - def __init__(self, **kwargs): BaseAnthropicMessagesConfig.__init__(self, **kwargs) AmazonInvokeConfig.__init__(self, **kwargs) @@ -439,7 +437,8 @@ class AmazonAnthropicClaudeMessagesConfig( ) input_examples_used = anthropic_model_info.is_input_examples_used(tools) - beta_set = set(get_anthropic_beta_from_headers(headers)) + user_beta_set = set(get_anthropic_beta_from_headers(headers)) + beta_set = set(user_beta_set) auto_betas = anthropic_model_info.get_anthropic_beta_list( model=model, optional_params=anthropic_messages_optional_request_params, @@ -463,8 +462,13 @@ class AmazonAnthropicClaudeMessagesConfig( if "tool-search-tool-2025-10-19" in beta_set: beta_set.add("tool-examples-2025-10-29") - if beta_set: - anthropic_messages_request["anthropic_beta"] = list(beta_set) + filtered_auto_betas = filter_and_transform_beta_headers( + beta_headers=list(beta_set - user_beta_set), + provider="bedrock", + ) + filtered_betas = sorted(user_beta_set.union(set(filtered_auto_betas))) + if filtered_betas: + anthropic_messages_request["anthropic_beta"] = filtered_betas return anthropic_messages_request @@ -498,6 +502,12 @@ class AmazonAnthropicClaudeMessagesConfig( ): """ Bedrock invoke does not return SSE formatted data. This function is a wrapper to ensure litellm chunks are SSE formatted. + + Bedrock's Anthropic-compatible streaming puts cache usage fields + (cache_creation_input_tokens, cache_read_input_tokens) only on + message_stop, not on message_start or message_delta. Claude Code's + SDK only merges usage from message_delta, so we promote those fields + from message_stop onto message_delta before yielding. """ from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import ( BaseAnthropicMessagesStreamingIterator, @@ -508,9 +518,81 @@ class AmazonAnthropicClaudeMessagesConfig( request_body=request_body, ) - async for chunk in handler.async_sse_wrapper(completion_stream): + patched_stream = self._promote_message_stop_usage(completion_stream) + + async for chunk in handler.async_sse_wrapper(patched_stream): yield chunk + @staticmethod + async def _promote_message_stop_usage( + completion_stream: AsyncIterator[ + Union[bytes, GenericStreamingChunk, ModelResponseStream, dict] + ], + ) -> AsyncIterator[Union[bytes, GenericStreamingChunk, ModelResponseStream, dict]]: + """ + Promote cache usage fields from message_stop onto message_delta. + + Bedrock reports input_tokens (uncached only) on message_start, and + the full breakdown (input_tokens, cache_creation_input_tokens, + cache_read_input_tokens) only on message_stop. Claude Code's SDK + merges usage from message_start and message_delta but ignores + message_stop. This method buffers message_delta and, when + message_stop arrives with cache usage, merges those fields into the + message_delta usage and also updates the input_tokens on + message_delta to include the full count (uncached + cache_creation + + cache_read). + """ + _CACHE_FIELDS = ("cache_creation_input_tokens", "cache_read_input_tokens") + pending_delta = None + + async for chunk in completion_stream: + if not isinstance(chunk, dict): + if pending_delta is not None: + yield pending_delta + pending_delta = None + yield chunk + continue + + chunk_type = chunk.get("type") + + if chunk_type == "message_delta": + pending_delta = chunk + continue + + if chunk_type == "message_stop" and pending_delta is not None: + stop_usage = dict(chunk.get("usage") or {}) + delta_usage = dict(pending_delta.get("usage") or {}) + + for field in _CACHE_FIELDS: + if field in stop_usage: + delta_usage[field] = stop_usage[field] + + raw_input = stop_usage.get("input_tokens") + if raw_input is not None: + uncached = raw_input if isinstance(raw_input, int) else 0 + raw_cc = delta_usage.get("cache_creation_input_tokens", 0) + cache_creation = raw_cc if isinstance(raw_cc, int) else 0 + raw_cr = delta_usage.get("cache_read_input_tokens", 0) + cache_read = raw_cr if isinstance(raw_cr, int) else 0 + delta_usage["input_tokens"] = uncached + cache_creation + cache_read + + if delta_usage: + pending_delta["usage"] = delta_usage # type: ignore[arg-type] + + yield pending_delta + pending_delta = None + yield chunk + continue + + if pending_delta is not None: + yield pending_delta + pending_delta = None + + yield chunk + + if pending_delta is not None: + yield pending_delta + class AmazonAnthropicClaudeMessagesStreamDecoder(AWSEventStreamDecoder): def __init__( diff --git a/litellm/llms/firecrawl/search/transformation.py b/litellm/llms/firecrawl/search/transformation.py index 61b589218cc..71136e1d3b3 100644 --- a/litellm/llms/firecrawl/search/transformation.py +++ b/litellm/llms/firecrawl/search/transformation.py @@ -159,15 +159,13 @@ class FirecrawlSearchConfig(BaseSearchConfig): """ Transform Firecrawl API response to LiteLLM unified SearchResponse format. - Firecrawl → LiteLLM mappings: - - data.web[].title → SearchResult.title - - data.web[].url → SearchResult.url - - data.web[].description OR data.web[].markdown → SearchResult.snippet - - No date field in web results (set to None) - - No last_updated field in Firecrawl response (set to None) + Supports both response formats: - Note: Firecrawl v2 returns results organized by source type (web, images, news). - We primarily use web results for the unified format. + Firecrawl Cloud (v2): + {"data": {"web": [...], "news": [...]}} + + Firecrawl Self-Hosted (v1): + {"success": true, "data": [{"url": "...", "title": "...", ...}, ...]} Args: raw_response: Raw httpx response from Firecrawl API @@ -181,36 +179,52 @@ class FirecrawlSearchConfig(BaseSearchConfig): # Transform results to SearchResult objects results = [] - # Process web results (primary source) data = response_json.get("data", {}) - web_results = data.get("web", []) - for result in web_results: - # Use markdown if available, otherwise fall back to description - snippet = result.get("markdown") or result.get("description", "") + if isinstance(data, list): + # Self-hosted Firecrawl (v1) format: data is a flat list of results + for result in data: + snippet = ( + result.get("markdown") or result.get("description", "") + ) + search_result = SearchResult( + title=result.get("title", ""), + url=result.get("url", ""), + snippet=snippet, + date=None, + last_updated=None, + ) + results.append(search_result) + elif isinstance(data, dict): + # Firecrawl Cloud (v2) format: data is a dict with web/news keys + web_results = data.get("web", []) - search_result = SearchResult( - title=result.get("title", ""), - url=result.get("url", ""), - snippet=snippet, - date=None, # Web results don't include date - last_updated=None, # Firecrawl doesn't provide last_updated in response - ) - results.append(search_result) + for result in web_results: + # Use markdown if available, otherwise fall back to description + snippet = result.get("markdown") or result.get("description", "") - # Process news results if available (they have date field) - news_results = data.get("news", []) - for result in news_results: - snippet = result.get("markdown") or result.get("snippet", "") + search_result = SearchResult( + title=result.get("title", ""), + url=result.get("url", ""), + snippet=snippet, + date=None, + last_updated=None, + ) + results.append(search_result) - search_result = SearchResult( - title=result.get("title", ""), - url=result.get("url", ""), - snippet=snippet, - date=result.get("date"), # News results include date - last_updated=None, - ) - results.append(search_result) + # Process news results if available (they have date field) + news_results = data.get("news", []) + for result in news_results: + snippet = result.get("markdown") or result.get("snippet", "") + + search_result = SearchResult( + title=result.get("title", ""), + url=result.get("url", ""), + snippet=snippet, + date=result.get("date"), # News results include date + last_updated=None, + ) + results.append(search_result) return SearchResponse( results=results, diff --git a/litellm/llms/vertex_ai/cost_calculator.py b/litellm/llms/vertex_ai/cost_calculator.py index e7ac453e949..4ca3d29e7d2 100644 --- a/litellm/llms/vertex_ai/cost_calculator.py +++ b/litellm/llms/vertex_ai/cost_calculator.py @@ -203,7 +203,7 @@ def _handle_128k_pricing( ): prompt_cost = prompt_tokens * input_cost_per_token_above_128k_tokens else: - prompt_cost = prompt_tokens * model_info["input_cost_per_token"] + prompt_cost = prompt_tokens * (model_info["input_cost_per_token"] or 0.0) ## CALCULATE OUTPUT COST output_cost_per_token_above_128k_tokens = model_info.get( @@ -215,7 +215,7 @@ def _handle_128k_pricing( ): completion_cost = completion_tokens * output_cost_per_token_above_128k_tokens else: - completion_cost = completion_tokens * model_info["output_cost_per_token"] + completion_cost = completion_tokens * (model_info["output_cost_per_token"] or 0.0) return prompt_cost, completion_cost diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index d4f986edd9b..4e1c7f4ac80 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -277,7 +277,15 @@ "litellm_provider": "bedrock", "max_input_tokens": 2600, "mode": "image_generation", - "output_cost_per_image": 0.06 + "output_cost_per_image": 0.06, + "supports_nova_canvas_image_edit": true + }, + "us.amazon.nova-canvas-v1:0": { + "litellm_provider": "bedrock", + "max_input_tokens": 2600, + "mode": "image_generation", + "output_cost_per_image": 0.06, + "supports_nova_canvas_image_edit": true }, "us.writer.palmyra-x4-v1:0": { "input_cost_per_token": 2.5e-06, @@ -6672,6 +6680,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/ap-northeast-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/ap-northeast-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 7.3e-07, "litellm_provider": "bedrock", @@ -6781,6 +6803,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/ap-south-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/ap-south-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 7.1e-07, "litellm_provider": "bedrock", @@ -6819,6 +6855,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/ap-southeast-2/minimax.minimax-m2.5": { + "input_cost_per_token": 3.09e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.236e-06 + }, "bedrock/ap-southeast-3/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, "litellm_provider": "bedrock", @@ -6845,6 +6895,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/ap-southeast-3/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/ap-southeast-3/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, "litellm_provider": "bedrock", @@ -6916,6 +6980,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/eu-north-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/eu-north-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, "litellm_provider": "bedrock", @@ -7030,6 +7108,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/eu-central-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/eu-central-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -7074,6 +7166,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/eu-west-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/eu-west-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -7118,6 +7224,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/eu-west-2/minimax.minimax-m2.5": { + "input_cost_per_token": 4.7e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.86e-06 + }, "bedrock/eu-west-2/qwen.qwen3-coder-next": { "input_cost_per_token": 7.8e-07, "litellm_provider": "bedrock", @@ -7174,6 +7294,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/eu-south-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/eu-south-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -7249,6 +7383,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/sa-east-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/sa-east-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 7.3e-07, "litellm_provider": "bedrock", @@ -7449,6 +7597,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/us-east-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.2e-06 + }, "bedrock/us-east-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -7513,6 +7675,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/us-east-2/minimax.minimax-m2.5": { + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.2e-06 + }, "bedrock/us-east-2/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -7643,13 +7819,13 @@ "cache_creation_input_token_cost": 3.75e-07 }, "bedrock/us-gov-east-1/claude-sonnet-4-5-20250929-v1:0": { - "input_cost_per_token": 3.3e-06, + "input_cost_per_token": 3.6e-06, "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "output_cost_per_token": 1.65e-05, + "output_cost_per_token": 1.8e-05, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -7659,8 +7835,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "cache_read_input_token_cost": 3.3e-07, - "cache_creation_input_token_cost": 4.125e-06 + "cache_read_input_token_cost": 3.6e-07, + "cache_creation_input_token_cost": 4.5e-06 }, "bedrock/us-gov-east-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 2.65e-06, @@ -7794,13 +7970,13 @@ "cache_creation_input_token_cost": 3.75e-07 }, "bedrock/us-gov-west-1/claude-sonnet-4-5-20250929-v1:0": { - "input_cost_per_token": 3.3e-06, + "input_cost_per_token": 3.6e-06, "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "output_cost_per_token": 1.65e-05, + "output_cost_per_token": 1.8e-05, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -7810,8 +7986,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "cache_read_input_token_cost": 3.3e-07, - "cache_creation_input_token_cost": 4.125e-06 + "cache_read_input_token_cost": 3.6e-07, + "cache_creation_input_token_cost": 4.5e-06 }, "bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 2.65e-06, @@ -7995,6 +8171,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/us-west-2/minimax.minimax-m2.5": { + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.2e-06 + }, "bedrock/us-west-2/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -21292,6 +21482,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "minimax.minimax-m2.5": { + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "minimax/speech-02-hd": { "input_cost_per_character": 0.0001, "litellm_provider": "minimax", @@ -23111,6 +23315,20 @@ "source": "https://aws.amazon.com/bedrock/pricing/", "supports_native_structured_output": true }, + "nvidia.nemotron-super-3-120b": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 256000, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 6.5e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, "o1": { "cache_read_input_token_cost": 7.5e-06, "input_cost_per_token": 1.5e-05, @@ -31164,7 +31382,9 @@ "mode": "chat", "output_cost_per_token": 3.2e-06, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#glm-models", - "supported_regions": ["global"], + "supported_regions": [ + "global" + ], "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -32724,6 +32944,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "zai.glm-5": { + "input_cost_per_token": 1e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3.2e-06, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, "zai/glm-5": { "cache_creation_input_token_cost": 0, "cache_read_input_token_cost": 2e-07, @@ -37473,5 +37707,51 @@ ] } ] + }, + "bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0": { + "cache_creation_input_token_cost": 1.5e-06, + "cache_read_input_token_cost": 1.2e-07, + "input_cost_per_token": 1.2e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 6e-06, + "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 346, + "supports_native_structured_output": true, + "supports_pdf_input": true + }, + "bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0": { + "cache_creation_input_token_cost": 1.5e-06, + "cache_read_input_token_cost": 1.2e-07, + "input_cost_per_token": 1.2e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 6e-06, + "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 346, + "supports_native_structured_output": true, + "supports_pdf_input": true } } diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 8faf36df4c6..4f3e51a2a32 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2753,6 +2753,8 @@ class NewProjectRequest(LiteLLM_BudgetTable): budget_id: Optional[str] = None metadata: Optional[dict] = None tags: Optional[List[str]] = None + guardrails: Optional[List[str]] = None + policies: Optional[List[str]] = None models: List[str] = [] model_rpm_limit: Optional[dict] = None model_tpm_limit: Optional[dict] = None @@ -2785,6 +2787,8 @@ class UpdateProjectRequest(LiteLLM_BudgetTable): team_id: Optional[str] = None metadata: Optional[dict] = None tags: Optional[List[str]] = None + guardrails: Optional[List[str]] = None + policies: Optional[List[str]] = None models: Optional[List[str]] = None model_rpm_limit: Optional[dict] = None model_tpm_limit: Optional[dict] = None @@ -4098,6 +4102,24 @@ class ScopeMapping(OIDCPermissions): } +class JWTRoutingOverride(BaseModel): + """ + Override default auth routing for JWT-shaped bearer tokens. + + A rule matches when all provided selectors match token claims. + If matched, request is routed to the configured auth path. + """ + + iss: Union[str, List[str]] + client_id: Optional[Union[str, List[str]]] = None + aud: Optional[Union[str, List[str]]] = None + path: Literal["oauth2"] = "oauth2" + + model_config = { + "extra": "forbid", + } + + class LiteLLM_JWTAuth(LiteLLMPydanticObjectBase): """ A class to define the roles and permissions for a LiteLLM Proxy w/ JWT Auth. @@ -4198,6 +4220,10 @@ class LiteLLM_JWTAuth(LiteLLMPydanticObjectBase): default=300, description="TTL (seconds) for caching JWT-to-virtual-key mapping lookups.", ) + routing_overrides: Optional[List[JWTRoutingOverride]] = Field( + default=None, + description="Optional claim-based routing overrides for JWT-shaped tokens. Matching rules route requests to oauth2 before default JWT flow.", + ) ######################################################### def __init__(self, **kwargs: Any) -> None: diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index c00a351bdd8..68bde8434a6 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -165,9 +165,24 @@ def _is_model_cost_zero( ) return False - # This model has zero cost explicitly configured + # Costs are 0 — verify this is from explicit configuration, + # not from defaulted sparse auto-registration entries. + # See: https://github.com/BerriAI/litellm/issues/24770 + safe_name = str(model_name).replace("\n", "").replace("\r", "") + if not _is_cost_explicitly_configured(model_name, llm_router): + verbose_proxy_logger.debug( + "Model %s has zero cost but no explicit cost " + "configuration in model_cost entry — treating as unknown " + "cost (enforce budget)", + safe_name, + ) + return False + verbose_proxy_logger.debug( - f"Model {model_name} has zero cost explicitly configured (input: {input_cost}, output: {output_cost})" + "Model %s has zero cost explicitly configured (input: %s, output: %s)", + safe_name, + input_cost, + output_cost, ) except Exception as e: @@ -181,6 +196,33 @@ def _is_model_cost_zero( return True +def _is_cost_explicitly_configured( + model: str, llm_router: "Router" +) -> bool: + """ + Check if any deployment in the model group has cost fields explicitly + set in its litellm.model_cost entry. + + When Router._create_deployment() registers a model not in the global + cost map, it creates a sparse entry like {"id": ""} with no cost + fields. _get_model_info_helper() then defaults missing costs to 0. + This function detects that scenario by checking the raw model_cost entry. + """ + for deployment in llm_router.model_list: + if deployment.get("model_name") != model: + continue + model_id = deployment.get("model_info", {}).get("id") + if model_id is None: + continue + raw_entry = litellm.model_cost.get(model_id, {}) + if ( + "input_cost_per_token" in raw_entry + or "output_cost_per_token" in raw_entry + ): + return True + return False + + async def _run_project_checks( project_object: Optional[LiteLLM_ProjectTableCachedObj], _model: Optional[Union[str, List[str]]], diff --git a/litellm/proxy/auth/handle_jwt.py b/litellm/proxy/auth/handle_jwt.py index 6ad372f3b5a..4a6856b6d14 100644 --- a/litellm/proxy/auth/handle_jwt.py +++ b/litellm/proxy/auth/handle_jwt.py @@ -16,6 +16,8 @@ from cryptography import x509 from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization from fastapi import HTTPException +import jwt +from jwt.api_jwk import PyJWK from litellm._logging import verbose_proxy_logger from litellm.caching.caching import DualCache @@ -71,6 +73,21 @@ class JWTHandler: prisma_client: Optional[PrismaClient] user_api_key_cache: DualCache + # Supported algos: https://pyjwt.readthedocs.io/en/stable/algorithms.html + # "Warning: Make sure not to mix symmetric and asymmetric algorithms that interpret + # the key in different ways (e.g. HS* and RS*)." + SUPPORTED_JWT_ALGORITHMS = [ + "RS256", + "RS384", + "RS512", + "PS256", + "PS384", + "PS512", + "ES256", + "ES384", + "ES512", + "EdDSA", + ] def __init__( self, @@ -97,6 +114,30 @@ class JWTHandler: parts = token.split(".") return len(parts) == 3 + @staticmethod + def get_unverified_claims(token: str) -> Optional[dict]: + """ + Decode JWT claims without signature verification. + Used for routing decisions before selecting validation path. + """ + if not JWTHandler.is_jwt(token): + return None + + try: + claims = jwt.decode( + token, + options={"verify_signature": False, "verify_aud": False}, + algorithms=JWTHandler.SUPPORTED_JWT_ALGORITHMS, + ) + if isinstance(claims, dict): + return claims + return None + except Exception as e: + verbose_proxy_logger.debug( + "Failed to decode unverified JWT claims for routing: %s", e + ) + return None + def _rbac_role_from_role_mapping(self, token: dict) -> Optional[RBAC_ROLES]: """ Returns the RBAC role the token 'belongs' to based on role mappings. @@ -664,30 +705,11 @@ class JWTHandler: raise Exception(f"Failed to fetch OIDC UserInfo: {str(e)}") async def auth_jwt(self, token: str) -> dict: - # Supported algos: https://pyjwt.readthedocs.io/en/stable/algorithms.html - # "Warning: Make sure not to mix symmetric and asymmetric algorithms that interpret - # the key in different ways (e.g. HS* and RS*)." - algorithms = [ - "RS256", - "RS384", - "RS512", - "PS256", - "PS384", - "PS512", - "ES256", - "ES384", - "ES512", - "EdDSA", - ] - audience = os.getenv("JWT_AUDIENCE") decode_options = None if audience is None: decode_options = {"verify_aud": False} - import jwt - from jwt.api_jwk import PyJWK - header = jwt.get_unverified_header(token) verbose_proxy_logger.debug("header: %s", header) @@ -721,7 +743,7 @@ class JWTHandler: payload = jwt.decode( token, public_key_obj, # type: ignore - algorithms=algorithms, + algorithms=self.SUPPORTED_JWT_ALGORITHMS, options=decode_options, # type: ignore[arg-type] audience=audience, leeway=self.leeway, # allow testing of expired tokens @@ -749,7 +771,7 @@ class JWTHandler: payload = jwt.decode( token, key, - algorithms=algorithms, + algorithms=self.SUPPORTED_JWT_ALGORITHMS, audience=audience, options=decode_options, ) diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 53ae08aefb1..046c39a9101 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -11,7 +11,7 @@ import asyncio import re import secrets from datetime import datetime, timezone -from typing import List, Optional, Tuple, cast +from typing import Any, List, Optional, Tuple, cast import fastapi from fastapi import HTTPException, Request, WebSocket, status @@ -139,6 +139,58 @@ def _get_bearer_token_or_received_api_key(api_key: str) -> str: return api_key +def _routing_selector_matches_claim( + selector_value: Optional[Any], claim_value: Optional[Any] +) -> bool: + if selector_value is None: + return True + + selector_list = ( + [str(v) for v in selector_value] + if isinstance(selector_value, list) + else [str(selector_value)] + ) + + if isinstance(claim_value, list): + claim_list = [str(v) for v in claim_value] + return any(v in claim_list for v in selector_list) + + return str(claim_value) in selector_list if claim_value is not None else False + + +def _matches_routing_override( + token_claims: dict, override: "JWTRoutingOverride" +) -> bool: + return ( + _routing_selector_matches_claim(override.iss, token_claims.get("iss")) + and _routing_selector_matches_claim( + override.client_id, token_claims.get("client_id") + ) + and _routing_selector_matches_claim(override.aud, token_claims.get("aud")) + ) + + +def _should_route_jwt_to_oauth2_override(token: str, jwt_handler: JWTHandler) -> bool: + routing_overrides = jwt_handler.litellm_jwtauth.routing_overrides + if not routing_overrides: + return False + + token_claims = jwt_handler.get_unverified_claims(token=token) + if token_claims is None: + return False + + for override in routing_overrides: + if override.path == "oauth2" and _matches_routing_override( + token_claims=token_claims, override=override + ): + verbose_proxy_logger.debug( + "JWT routing override matched. Routing token to OAuth2 introspection." + ) + return True + + return False + + def _get_bearer_token( api_key: str, ): @@ -649,12 +701,20 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 # - JWT tokens (3 dot-separated parts) -> skip OAuth2, fall through to JWT handler # - Opaque tokens -> use OAuth2 handler # This allows JWT for users and OAuth2 for M2M on the same instance - is_jwt_token = ( + is_jwt = ( jwt_handler.is_jwt(token=api_key) if general_settings.get("enable_jwt_auth", False) is True else False ) - if not is_jwt_token: + # Routing uses unverified JWT claims only to choose auth path. + # Final authentication is enforced by the selected validator. + route_jwt_to_oauth2 = ( + is_jwt + and _should_route_jwt_to_oauth2_override( + token=api_key, jwt_handler=jwt_handler + ) + ) + if not is_jwt or route_jwt_to_oauth2: # return UserAPIKeyAuth object # helper to check if the api_key is a valid oauth2 token from litellm.proxy.proxy_server import premium_user @@ -688,7 +748,7 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 jwt_claims: Optional[dict] if ( jwt_handler.litellm_jwtauth.oidc_userinfo_enabled - and not jwt_handler.is_jwt(token=api_key) + and not is_jwt ): jwt_claims = await jwt_handler.get_oidc_userinfo(token=api_key) else: @@ -1193,49 +1253,13 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 raise Exception( "Key is blocked. Update via `/key/unblock` if you're an admin." ) - config = valid_token.config - - if config != {}: - model_list = config.get("model_list", []) - new_model_list = model_list - verbose_proxy_logger.debug( - f"\n new llm router model list {new_model_list}" - ) - elif ( - isinstance(valid_token.models, list) - and "all-team-models" in valid_token.models - ): - # Do not do any validation at this step - # the validation will occur when checking the team has access to this model - pass - else: - model = get_model_from_request(request_data, route) - fallback_models = cast( - Optional[List[ALL_FALLBACK_MODEL_VALUES]], - request_data.get("fallbacks", None), - ) - - if model is not None: - await can_key_call_model( - model=model, - llm_model_list=llm_model_list, - valid_token=valid_token, - llm_router=llm_router, - ) - - if fallback_models is not None: - for m in fallback_models: - await can_key_call_model( - model=m["model"] if isinstance(m, dict) else m, - llm_model_list=llm_model_list, - valid_token=valid_token, - llm_router=llm_router, - ) - await is_valid_fallback_model( - model=m["model"] if isinstance(m, dict) else m, - llm_router=llm_router, - user_model=None, - ) + await _enforce_key_and_fallback_model_access( + valid_token=valid_token, + request_data=request_data, + route=route, + llm_model_list=llm_model_list, + llm_router=llm_router, + ) # Check 2. If user_id for this token is in budget - done in common_checks() if valid_token.user_id is not None: @@ -1764,6 +1788,61 @@ async def _lookup_end_user_and_apply_budget( return valid_token, end_user_object +async def _enforce_key_and_fallback_model_access( + *, + valid_token: UserAPIKeyAuth, + request_data: dict, + route: str, + llm_model_list: Optional[list], + llm_router: Optional[Any], +) -> None: + """ + Key-level model allowlist and client fallbacks (same as standard auth). + Not included in common_checks — common_checks enforces team/user/project model access only. + """ + config = valid_token.config + + if config != {}: + model_list = config.get("model_list", []) + new_model_list = model_list + verbose_proxy_logger.debug( + f"\n new llm router model list {new_model_list}" + ) + elif ( + isinstance(valid_token.models, list) + and "all-team-models" in valid_token.models + ): + pass + else: + model = get_model_from_request(request_data, route) + fallback_models = cast( + Optional[List[ALL_FALLBACK_MODEL_VALUES]], + request_data.get("fallbacks", None), + ) + + if model is not None: + await can_key_call_model( + model=model, + llm_model_list=llm_model_list, + valid_token=valid_token, + llm_router=llm_router, + ) + + if fallback_models is not None: + for m in fallback_models: + await can_key_call_model( + model=m["model"] if isinstance(m, dict) else m, + llm_model_list=llm_model_list, + valid_token=valid_token, + llm_router=llm_router, + ) + await is_valid_fallback_model( + model=m["model"] if isinstance(m, dict) else m, + llm_router=llm_router, + user_model=None, + ) + + async def _run_post_custom_auth_checks( valid_token: UserAPIKeyAuth, request: Request, @@ -1773,6 +1852,7 @@ async def _run_post_custom_auth_checks( ) -> UserAPIKeyAuth: from litellm.proxy.proxy_server import ( general_settings, + llm_model_list, llm_router, model_max_budget_limiter, prisma_client, @@ -1816,6 +1896,15 @@ async def _run_post_custom_auth_checks( ), ) + if general_settings.get("custom_auth_run_common_checks", False): + await _enforce_key_and_fallback_model_access( + valid_token=valid_token, + request_data=request_data, + route=route, + llm_model_list=llm_model_list, + llm_router=llm_router, + ) + current_model = request_data.get("model", None) # 3. Check key-level model_max_budget diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 386b9d27723..09200c96841 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -221,16 +221,21 @@ async def create_response( f"Error consuming first chunk from generator: {e}" ) - # Fallback to a generic error stream + # Preserve status code from HTTPException (e.g., guardrail blocks) + error_status = getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR) + error_detail = getattr(e, "detail", "Error processing stream start") + if not isinstance(error_detail, str): + error_detail = str(error_detail) + async def error_gen_message() -> AsyncGenerator[str, None]: - yield f"data: {json.dumps({'error': {'message': 'Error processing stream start', 'code': status.HTTP_500_INTERNAL_SERVER_ERROR}})}\n\n" + yield f"data: {json.dumps({'error': {'message': error_detail, 'code': error_status}})}\n\n" yield "data: [DONE]\n\n" return StreamingResponse( error_gen_message(), media_type=media_type, headers=headers, - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + status_code=error_status, ) async def combined_generator() -> AsyncGenerator[str, None]: diff --git a/litellm/proxy/example_config_yaml/websearch_interception_config.yaml b/litellm/proxy/example_config_yaml/websearch_interception_config.yaml index 2c1cd623c30..89c35c9c9d3 100644 --- a/litellm/proxy/example_config_yaml/websearch_interception_config.yaml +++ b/litellm/proxy/example_config_yaml/websearch_interception_config.yaml @@ -1,7 +1,7 @@ model_list: - model_name: claude-3-5-sonnet litellm_params: - model: bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0 + model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 # Search tools configuration search_tools: diff --git a/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py b/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py index 2d3f048f81b..3250e0bb7cf 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py +++ b/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py @@ -14,6 +14,8 @@ from fastapi import HTTPException if TYPE_CHECKING: from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel +import json + import litellm from litellm._logging import verbose_proxy_logger from litellm.caching import DualCache @@ -203,8 +205,8 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): response.text, ) raise HTTPException( - status_code=response.status_code, - detail=f"Model Armor API error: {response.text}", + status_code=400, + detail=f"Model Armor API error (upstream {response.status_code}): {response.text}", ) json_response = response.json() @@ -746,8 +748,21 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): yield chunk return - except HTTPException: - raise + except HTTPException as e: + # Yield error as SSE event so create_response() detects it and + # returns a proper JSON error response with the correct status code. + # (Raising from a generator hits create_response's generic except → 500.) + detail = ( + e.detail if isinstance(e.detail, dict) else {"message": str(e.detail)} + ) + error_value = detail.get("error", detail) + if isinstance(error_value, dict): + error_obj = dict(error_value) + else: + error_obj = {"message": str(error_value)} + error_obj["code"] = str(e.status_code) + yield f"data: {json.dumps({'error': error_obj})}\n\n" # type: ignore[misc] + return except Exception as e: verbose_proxy_logger.error( "Model Armor streaming error: %s", str(e), exc_info=True diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 948d6dd33af..46000f4fe6e 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -112,6 +112,14 @@ class _ProxyDBLogger(CustomLogger): _litellm_logging_obj, "litellm_trace_id", None ) + # Use the actual request start time from the logging object so that + # failed requests record the real duration instead of 0. + actual_start_time = datetime.now() + if _litellm_logging_obj is not None: + obj_start = getattr(_litellm_logging_obj, "start_time", None) + if obj_start is not None: + actual_start_time = obj_start + await proxy_logging_obj.db_spend_update_writer.update_database( token=user_api_key_dict.api_key, response_cost=0.0, @@ -120,7 +128,7 @@ class _ProxyDBLogger(CustomLogger): team_id=user_api_key_dict.team_id, kwargs=request_data, completion_response=original_exception, - start_time=datetime.now(), + start_time=actual_start_time, end_time=datetime.now(), org_id=user_api_key_dict.org_id, ) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index ba9577f35d7..fcb8b6db80a 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -1472,17 +1472,19 @@ def _add_guardrails_from_key_or_team_metadata( team_metadata: Optional[dict], data: dict, metadata_variable_name: str, + project_metadata: Optional[dict] = None, ) -> None: """ - Helper add guardrails from key or team metadata to request data + Helper add guardrails from key, team, or project metadata to request data - Key guardrails are set first, then team guardrails are appended (without duplicates). + Key guardrails are set first, then team and project guardrails are appended (without duplicates). Args: key_metadata: The key metadata dictionary to check for guardrails team_metadata: The team metadata dictionary to check for guardrails data: The request data to update metadata_variable_name: The name of the metadata field in data + project_metadata: The project metadata dictionary to check for guardrails """ from litellm.proxy.utils import _premium_user_check @@ -1508,6 +1510,15 @@ def _add_guardrails_from_key_or_team_metadata( _premium_user_check() combined_guardrails.update(team_metadata["guardrails"]) + # Add project-level guardrails (set automatically handles duplicates) + if project_metadata and "guardrails" in project_metadata: + if ( + isinstance(project_metadata["guardrails"], list) + and len(project_metadata["guardrails"]) > 0 + ): + _premium_user_check() + combined_guardrails.update(project_metadata["guardrails"]) + # Set combined guardrails in metadata as list if combined_guardrails: data[metadata_variable_name]["guardrails"] = list(combined_guardrails) @@ -1518,12 +1529,13 @@ def _add_guardrails_from_policies_in_metadata( team_metadata: Optional[dict], data: dict, metadata_variable_name: str, + project_metadata: Optional[dict] = None, ) -> None: """ - Helper to resolve guardrails from policies attached to key/team metadata. + Helper to resolve guardrails from policies attached to key/team/project metadata. This function: - 1. Gets policy names from key and team metadata + 1. Gets policy names from key, team, and project metadata 2. Resolves guardrails from those policies (including inheritance) 3. Adds resolved guardrails to request metadata @@ -1532,6 +1544,7 @@ def _add_guardrails_from_policies_in_metadata( team_metadata: The team metadata dictionary to check for policies data: The request data to update metadata_variable_name: The name of the metadata field in data + project_metadata: The project metadata dictionary to check for policies """ from litellm._logging import verbose_proxy_logger from litellm.proxy.policy_engine.policy_registry import get_policy_registry @@ -1560,6 +1573,15 @@ def _add_guardrails_from_policies_in_metadata( _premium_user_check() policy_names.update(team_metadata["policies"]) + # Add project-level policies + if project_metadata and "policies" in project_metadata: + if ( + isinstance(project_metadata["policies"], list) + and len(project_metadata["policies"]) > 0 + ): + _premium_user_check() + policy_names.update(project_metadata["policies"]) + if not policy_names: return @@ -1641,6 +1663,7 @@ async def move_guardrails_to_metadata( # Early-out: skip all guardrails processing when nothing is configured key_metadata = user_api_key_dict.metadata team_metadata = user_api_key_dict.team_metadata + project_metadata = user_api_key_dict.project_metadata or {} has_key_config = key_metadata and ( "guardrails" in key_metadata or "policies" in key_metadata @@ -1648,12 +1671,15 @@ async def move_guardrails_to_metadata( has_team_config = team_metadata and ( "guardrails" in team_metadata or "policies" in team_metadata ) + has_project_config = project_metadata and ( + "guardrails" in project_metadata or "policies" in project_metadata + ) has_request_config = ( "guardrails" in data or "guardrail_config" in data or "policies" in data ) # Only check policy engine if no local config (avoid import + registry lookup) - if not (has_key_config or has_team_config or has_request_config): + if not (has_key_config or has_team_config or has_project_config or has_request_config): from litellm.proxy.policy_engine.policy_registry import get_policy_registry if not get_policy_registry().is_initialized(): @@ -1661,20 +1687,22 @@ async def move_guardrails_to_metadata( data.pop("policies", None) return - # Check key-level guardrails + # Check key/team/project-level guardrails _add_guardrails_from_key_or_team_metadata( key_metadata=user_api_key_dict.metadata, team_metadata=user_api_key_dict.team_metadata, + project_metadata=project_metadata, data=data, metadata_variable_name=_metadata_variable_name, ) ######################################################################################### - # Add guardrails from policies attached to key/team metadata + # Add guardrails from policies attached to key/team/project metadata ######################################################################################### _add_guardrails_from_policies_in_metadata( key_metadata=user_api_key_dict.metadata, team_metadata=user_api_key_dict.team_metadata, + project_metadata=project_metadata, data=data, metadata_variable_name=_metadata_variable_name, ) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index c140de98191..6f9166909c0 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -18,7 +18,7 @@ import re import secrets import traceback from datetime import datetime, timedelta, timezone -from typing import Any, Dict, List, Literal, Optional, Tuple, cast +from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, cast import fastapi import yaml @@ -487,6 +487,59 @@ async def validate_team_id_used_in_service_account_request( return True +def _enforce_upperbound_key_params( + data: Union[GenerateKeyRequest, UpdateKeyRequest], + fill_defaults: bool = True, +) -> None: + """ + Enforce upperbound limits on key parameters. + + For key generation (fill_defaults=True): fills None values with upperbound defaults. + For key update (fill_defaults=False): only validates explicitly provided values. + """ + if litellm.upperbound_key_generate_params is None: + return + + for elem in data: + key, value = elem + upperbound_value = getattr( + litellm.upperbound_key_generate_params, key, None + ) + if upperbound_value is not None: + if value is None: + if fill_defaults: + setattr(data, key, upperbound_value) + else: + if key in [ + "max_budget", + "max_parallel_requests", + "tpm_limit", + "rpm_limit", + ]: + if value > upperbound_value: + raise HTTPException( + status_code=400, + detail={ + "error": f"{key} is over max limit set in config - user_value={value}; max_value={upperbound_value}" + }, + ) + elif key in ["budget_duration", "duration"]: + upperbound_duration = duration_in_seconds( + duration=upperbound_value + ) + if value == "-1": + user_duration = float("inf") + else: + user_duration = duration_in_seconds(duration=value) + if user_duration > upperbound_duration: + raise HTTPException( + status_code=400, + detail={ + "error": f"{key} is over max limit set in config - user_value={value}; max_value={upperbound_value}" + }, + ) + + async def _common_key_generation_helper( # noqa: PLR0915 data: GenerateKeyRequest, user_api_key_dict: UserAPIKeyAuth, @@ -537,49 +590,8 @@ async def _common_key_generation_helper( # noqa: PLR0915 elif key == "metadata" and value == {}: setattr(data, key, litellm.default_key_generate_params.get(key, {})) - # check if user set default key/generate params on config.yaml - if litellm.upperbound_key_generate_params is not None: - for elem in data: - key, value = elem - upperbound_value = getattr( - litellm.upperbound_key_generate_params, key, None - ) - if upperbound_value is not None: - if value is None: - # Use the upperbound value if user didn't provide a value - setattr(data, key, upperbound_value) - else: - # Compare with upperbound for numeric fields - if key in [ - "max_budget", - "max_parallel_requests", - "tpm_limit", - "rpm_limit", - ]: - if value > upperbound_value: - raise HTTPException( - status_code=400, - detail={ - "error": f"{key} is over max limit set in config - user_value={value}; max_value={upperbound_value}" - }, - ) - # Compare durations - elif key in ["budget_duration", "duration"]: - upperbound_duration = duration_in_seconds( - duration=upperbound_value - ) - # Handle special case where duration is None or "-1" (never expires) - if value is None or value == "-1": - user_duration = float("inf") # Infinite duration - else: - user_duration = duration_in_seconds(duration=value) - if user_duration > upperbound_duration: - raise HTTPException( - status_code=400, - detail={ - "error": f"{key} is over max limit set in config - user_value={value}; max_value={upperbound_value}" - }, - ) + # check if user set upperbound key/generate params on config.yaml + _enforce_upperbound_key_params(data, fill_defaults=True) # APPLY ENTERPRISE KEY MANAGEMENT PARAMS try: @@ -942,9 +954,9 @@ async def _check_team_key_limits( where={"team_id": team_table.team_id}, ) # Exclude the key being updated to avoid double-counting its limits. - # key.token is the SHA-256 hash stored in DB; data.key is the raw key string. + # data.key may be a raw key (sk-...) or a pre-hashed token_id. if isinstance(data, UpdateKeyRequest): - hashed_key = hash_token(data.key) + hashed_key = _hash_token_if_needed(data.key) keys = [key for key in keys if key.token != hashed_key] check_team_key_model_specific_limits( keys=keys, @@ -1101,9 +1113,9 @@ async def _check_org_key_limits( where={"organization_id": org_table.organization_id}, ) # Exclude the key being updated to avoid double-counting its limits. - # key.token is the SHA-256 hash stored in DB; data.key is the raw key string. + # data.key may be a raw key (sk-...) or a pre-hashed token_id. if isinstance(data, UpdateKeyRequest): - hashed_key = hash_token(data.key) + hashed_key = _hash_token_if_needed(data.key) keys = [key for key in keys if key.token != hashed_key] check_org_key_model_specific_limits( keys=keys, @@ -1687,6 +1699,7 @@ async def _process_single_key_update( user_api_key_cache: DualCache, proxy_logging_obj: Any, llm_router: Optional[Router], + user_custom_key_update: Optional[Callable] = None, ) -> Dict[str, Any]: """ Process a single key update with all validations and checks. @@ -1737,6 +1750,22 @@ async def _process_single_key_update( tags=key_update_item.tags, ) + # Custom key update hook + if user_custom_key_update is not None: + if inspect.iscoroutinefunction(user_custom_key_update): + result = await user_custom_key_update(update_key_request) + else: + raise ValueError("user_custom_key_update must be a coroutine") + decision = result.get("decision", True) + message = result.get("message", "Authentication Failed - Custom Auth Rule") + if not decision: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, detail=message + ) + + # Enforce upperbound key params on update (don't fill defaults) + _enforce_upperbound_key_params(update_key_request, fill_defaults=False) + # Get team object and check team limits if team_id is provided team_obj: Optional[LiteLLM_TeamTableCachedObj] = None if update_key_request.team_id is not None: @@ -2014,7 +2043,7 @@ async def _validate_update_key_data( "/key/update", tags=["key management"], dependencies=[Depends(user_api_key_auth)] ) @management_endpoint_wrapper -async def update_key_fn( +async def update_key_fn( # noqa: PLR0915 request: Request, data: UpdateKeyRequest, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), @@ -2095,6 +2124,7 @@ async def update_key_fn( prisma_client, proxy_logging_obj, user_api_key_cache, + user_custom_key_update, ) try: @@ -2126,6 +2156,21 @@ async def update_key_fn( user_api_key_cache=user_api_key_cache, ) + # Custom key update hook + if user_custom_key_update is not None: + if inspect.iscoroutinefunction(user_custom_key_update): + result = await user_custom_key_update(data) + else: + raise ValueError("user_custom_key_update must be a coroutine") + decision = result.get("decision", True) + message = result.get("message", "Authentication Failed - Custom Auth Rule") + if not decision: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, detail=message + ) + + # Enforce upperbound key params on update (don't fill defaults) + _enforce_upperbound_key_params(data, fill_defaults=False) non_default_values = await prepare_key_update_data( data=data, existing_key_row=existing_key_row ) @@ -2157,7 +2202,7 @@ async def update_key_fn( # Delete - key from cache, since it's been updated! # key updated - a new model could have been added to this key. it should not block requests after this is done await _delete_cache_key_object( - hashed_token=hash_token(key), + hashed_token=_hash_token_if_needed(key), user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, ) @@ -2261,6 +2306,7 @@ async def bulk_update_keys( prisma_client, proxy_logging_obj, user_api_key_cache, + user_custom_key_update, ) if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN.value: @@ -2304,6 +2350,7 @@ async def bulk_update_keys( user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, llm_router=llm_router, + user_custom_key_update=user_custom_key_update, ) successful_updates.append( diff --git a/litellm/proxy/prompts/prompt_endpoints.py b/litellm/proxy/prompts/prompt_endpoints.py index 73e0ece3e2c..483107a3759 100644 --- a/litellm/proxy/prompts/prompt_endpoints.py +++ b/litellm/proxy/prompts/prompt_endpoints.py @@ -192,19 +192,22 @@ def get_latest_prompt_versions(prompts: List[PromptSpec]) -> List[PromptSpec]: return list(latest_prompts.values()) -async def get_next_version_for_prompt(prisma_client, prompt_id: str) -> int: +async def get_next_version_for_prompt( + prisma_client, prompt_id: str, environment: str = "development" +) -> int: """ - Get the next version number for a prompt. + Get the next version number for a prompt in a specific environment. Args: prisma_client: Prisma database client prompt_id: Base prompt ID + environment: The environment to check versions for Returns: Next version number (1 if no versions exist, max_version + 1 otherwise) """ existing_prompts = await prisma_client.db.litellm_prompttable.find_many( - where={"prompt_id": prompt_id} + where={"prompt_id": prompt_id, "environment": environment} ) if existing_prompts: @@ -231,6 +234,8 @@ def create_versioned_prompt_spec(db_prompt) -> PromptSpec: prompt_dict = db_prompt.model_dump() base_prompt_id = prompt_dict["prompt_id"] version = prompt_dict.get("version", 1) + environment = prompt_dict.get("environment", "development") + created_by = prompt_dict.get("created_by") # Parse litellm_params litellm_params_data = prompt_dict.get("litellm_params") @@ -256,6 +261,8 @@ def create_versioned_prompt_spec(db_prompt) -> PromptSpec: prompt_info=prompt_info, created_at=prompt_dict.get("created_at"), updated_at=prompt_dict.get("updated_at"), + environment=environment, + created_by=created_by, ) @@ -277,6 +284,7 @@ class PatchPromptRequest(BaseModel): response_model=ListPromptsResponse, ) async def list_prompts( + environment: Optional[str] = None, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ @@ -318,23 +326,26 @@ async def list_prompts( if key_metadata is not None: prompts = cast(Optional[List[str]], key_metadata.get("prompts", None)) if prompts is not None: + all_prompts = [ + IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS[prompt_id] + for prompt_id in prompts + if prompt_id in IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS + ] + if environment: + all_prompts = [p for p in all_prompts if p.environment == environment] prompt_list = [] - for prompt_id in prompts: - if prompt_id in IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS: - original_prompt = IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS[ - prompt_id - ] - # Create a copy with base prompt_id (without version suffix) - prompt_copy = PromptSpec( - prompt_id=get_base_prompt_id( - prompt_id=original_prompt.prompt_id - ), - litellm_params=original_prompt.litellm_params, - prompt_info=original_prompt.prompt_info, - created_at=original_prompt.created_at, - updated_at=original_prompt.updated_at, - ) - prompt_list.append(prompt_copy) + for original_prompt in all_prompts: + # Create a copy with base prompt_id (without version suffix) + prompt_copy = PromptSpec( + prompt_id=get_base_prompt_id(prompt_id=original_prompt.prompt_id), + litellm_params=original_prompt.litellm_params, + prompt_info=original_prompt.prompt_info, + created_at=original_prompt.created_at, + updated_at=original_prompt.updated_at, + environment=original_prompt.environment, + created_by=original_prompt.created_by, + ) + prompt_list.append(prompt_copy) return ListPromptsResponse(prompts=prompt_list) # check if user is proxy admin - show all prompts if user_api_key_dict.user_role is not None and ( @@ -343,6 +354,8 @@ async def list_prompts( ): # Get all prompts and filter to show only the latest version of each all_prompts = list(IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS.values()) + if environment: + all_prompts = [p for p in all_prompts if p.environment == environment] latest_prompts = get_latest_prompt_versions(prompts=all_prompts) # Create copies with base prompt_id (without version suffix) for display prompts_for_display = [] @@ -353,6 +366,8 @@ async def list_prompts( prompt_info=original_prompt.prompt_info, created_at=original_prompt.created_at, updated_at=original_prompt.updated_at, + environment=original_prompt.environment, + created_by=original_prompt.created_by, ) prompts_for_display.append(prompt_copy) return ListPromptsResponse(prompts=prompts_for_display) @@ -368,6 +383,7 @@ async def list_prompts( ) async def get_prompt_versions( prompt_id: str, + environment: Optional[str] = None, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ @@ -404,6 +420,7 @@ async def get_prompt_versions( ``` """ from litellm.proxy.prompts.prompt_registry import IN_MEMORY_PROMPT_REGISTRY + from litellm.proxy.proxy_server import prisma_client # Only allow proxy admins to view version history if user_api_key_dict.user_role is None or ( @@ -414,49 +431,112 @@ async def get_prompt_versions( status_code=403, detail="Only proxy admins can view prompt versions" ) - # Strip version suffix if provided (e.g., "jack_success.v1" -> "jack_success") base_prompt_id = get_base_prompt_id(prompt_id=prompt_id) - # Get all prompts and filter by base_prompt_id - all_prompts = list(IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS.values()) - prompt_versions = [ - prompt - for prompt in all_prompts - if get_base_prompt_id(prompt_id=prompt.prompt_id) == base_prompt_id - ] + # Query DB for versions + versioned_prompts = [] + if prisma_client is not None: + where_clause: Dict[str, Any] = {"prompt_id": base_prompt_id} + if environment: + where_clause["environment"] = environment + db_prompts = await prisma_client.db.litellm_prompttable.find_many( + where=where_clause, + order={"version": "desc"}, + ) + for db_prompt in db_prompts: + spec = create_versioned_prompt_spec(db_prompt=db_prompt) + versioned_prompts.append( + PromptSpec( + prompt_id=base_prompt_id, + litellm_params=spec.litellm_params, + prompt_info=spec.prompt_info, + created_at=spec.created_at, + updated_at=spec.updated_at, + version=get_version_number(prompt_id=spec.prompt_id), + environment=spec.environment, + created_by=spec.created_by, + ) + ) + else: + # Fallback: in-memory registry (no DB) + all_prompts = list(IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS.values()) + prompt_versions = [ + prompt + for prompt in all_prompts + if get_base_prompt_id(prompt_id=prompt.prompt_id) == base_prompt_id + and (environment is None or prompt.environment == environment) + ] + for prompt in prompt_versions: + version_number = get_version_number(prompt_id=prompt.prompt_id) + versioned_prompts.append( + PromptSpec( + prompt_id=base_prompt_id, + litellm_params=prompt.litellm_params, + prompt_info=prompt.prompt_info, + created_at=prompt.created_at, + updated_at=prompt.updated_at, + version=version_number, + environment=prompt.environment, + created_by=prompt.created_by, + ) + ) + versioned_prompts.sort(key=lambda p: p.version or 1, reverse=True) - if not prompt_versions: + if not versioned_prompts: raise HTTPException( status_code=404, detail=f"No versions found for prompt ID {base_prompt_id}" ) - # Create response with explicit version field for each prompt - versioned_prompts = [] - for prompt in prompt_versions: - # Extract version number from the root prompt_id which has version suffix - # (e.g., "jack-sparrow.v3" -> 3) - version_number = get_version_number(prompt_id=prompt.prompt_id) - - # Strip version from prompt_id for clean display - base_prompt_id = get_base_prompt_id(prompt_id=prompt.prompt_id) - - # Create a copy with explicit version field and clean prompt_id - versioned_prompt = PromptSpec( - prompt_id=base_prompt_id, # Clean ID without version (e.g., "jack-sparrow") - litellm_params=prompt.litellm_params, - prompt_info=prompt.prompt_info, - created_at=prompt.created_at, - updated_at=prompt.updated_at, - version=version_number, # Explicit version field (e.g., 3) - ) - versioned_prompts.append(versioned_prompt) - - # Sort by version number (descending - newest first) - versioned_prompts.sort(key=lambda p: p.version or 1, reverse=True) - return ListPromptsResponse(prompts=versioned_prompts) +def _get_prompt_template( + prompt_spec: PromptSpec, base_prompt_id: str +) -> Optional[PromptTemplateBase]: + """Resolve the raw prompt template from dotprompt content or the in-memory registry.""" + from litellm.proxy.prompts.prompt_registry import IN_MEMORY_PROMPT_REGISTRY + + try: + dotprompt_content = prompt_spec.litellm_params.dotprompt_content + if dotprompt_content: + from litellm.integrations.dotprompt import ( + _get_prompt_data_from_dotprompt_content, + ) + + parsed = _get_prompt_data_from_dotprompt_content(dotprompt_content) + if parsed: + return PromptTemplateBase( + litellm_prompt_id=base_prompt_id, + content=parsed.get("content", ""), + metadata=parsed.get("metadata"), + ) + else: + prompt_callback = IN_MEMORY_PROMPT_REGISTRY.get_prompt_callback_by_id( + prompt_spec.prompt_id + ) + if prompt_callback is not None: + integration_name = prompt_callback.integration_name + if integration_name == "dotprompt": + from litellm.integrations.dotprompt.dotprompt_manager import ( + DotpromptManager, + ) + + if isinstance(prompt_callback, DotpromptManager): + template = ( + prompt_callback.prompt_manager.get_all_prompts_as_json() + ) + if template is not None and len(template) == 1: + template_id = list(template.keys())[0] + return PromptTemplateBase( + litellm_prompt_id=template_id, + content=template[template_id]["content"], + metadata=template[template_id]["metadata"], + ) + except Exception: + pass + return None + + @router.get( "/prompts/{prompt_id}", tags=["Prompt Management"], @@ -471,6 +551,7 @@ async def get_prompt_versions( ) async def get_prompt_info( prompt_id: str, + environment: Optional[str] = None, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ @@ -503,6 +584,7 @@ async def get_prompt_info( ``` """ from litellm.proxy.prompts.prompt_registry import IN_MEMORY_PROMPT_REGISTRY + from litellm.proxy.proxy_server import prisma_client ## CHECK IF USER HAS ACCESS TO PROMPT prompts: Optional[List[str]] = None @@ -523,68 +605,80 @@ async def get_prompt_info( detail=f"You are not authorized to access this prompt. Your role - {user_api_key_dict.user_role}, Your key's prompts - {prompts}", ) - # Try to get prompt directly first - prompt_spec = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(prompt_id) + base_prompt_id = get_base_prompt_id(prompt_id=prompt_id) - # If not found, try to find the latest version - if prompt_spec is None: - latest_prompt_id = get_latest_version_prompt_id( - prompt_id=prompt_id, - all_prompt_ids=IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS, + # Query all environments this prompt exists in (lightweight: distinct on environment) + all_environments: List[str] = [] + if prisma_client is not None: + all_prompt_rows = await prisma_client.db.litellm_prompttable.find_many( + where={"prompt_id": base_prompt_id}, + distinct=["environment"], ) - prompt_spec = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(latest_prompt_id) + all_environments = sorted( + set(row.environment for row in all_prompt_rows if row.environment) + ) + + # If environment is specified, find the version in that environment from DB + # If prompt_id has a version suffix (e.g., "testprompt.v2"), fetch that specific version + # Otherwise fetch the latest version in that environment + prompt_spec = None + requested_version = ( + get_version_number(prompt_id=prompt_id) if prompt_id != base_prompt_id else None + ) + if environment and prisma_client is not None: + where_clause: Dict[str, Any] = { + "prompt_id": base_prompt_id, + "environment": environment, + } + if requested_version is not None: + where_clause["version"] = requested_version + env_prompts = await prisma_client.db.litellm_prompttable.find_many( + where=where_clause, + order={"version": "desc"}, + take=1, + ) + if env_prompts: + prompt_spec = create_versioned_prompt_spec(db_prompt=env_prompts[0]) + + # Fallback: use in-memory registry (no environment filter) + if prompt_spec is None and environment is None: + prompt_spec = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(prompt_id) + if prompt_spec is None: + latest_prompt_id = get_latest_version_prompt_id( + prompt_id=prompt_id, + all_prompt_ids=IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS, + ) + prompt_spec = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(latest_prompt_id) if prompt_spec is None: - raise HTTPException(status_code=400, detail=f"Prompt {prompt_id} not found") + raise HTTPException( + status_code=400, + detail=f"Prompt {prompt_id} not found" + + (f" in environment {environment}" if environment else ""), + ) # Extract version number from the prompt_id version_number = get_version_number(prompt_id=prompt_spec.prompt_id) # Create a copy of the prompt spec with the base prompt ID (stripped of version) - # and explicit version field for consistency with list_prompts and versions endpoints prompt_spec_response = PromptSpec( prompt_id=get_base_prompt_id(prompt_id=prompt_spec.prompt_id), - litellm_params=prompt_spec.litellm_params, # This preserves the versioned ID + litellm_params=prompt_spec.litellm_params, prompt_info=prompt_spec.prompt_info, created_at=prompt_spec.created_at, updated_at=prompt_spec.updated_at, - version=version_number, # Explicit version field + version=version_number, + environment=prompt_spec.environment, + created_by=prompt_spec.created_by, ) - # Get prompt content from the callback - prompt_template: Optional[PromptTemplateBase] = None - try: - prompt_callback = IN_MEMORY_PROMPT_REGISTRY.get_prompt_callback_by_id( - prompt_spec.prompt_id - ) - if prompt_callback is not None: - # Extract content based on integration type - integration_name = prompt_callback.integration_name + # Get prompt content + prompt_template = _get_prompt_template(prompt_spec, base_prompt_id) - if integration_name == "dotprompt": - # For dotprompt integration, get content from the prompt manager - from litellm.integrations.dotprompt.dotprompt_manager import ( - DotpromptManager, - ) - - if isinstance(prompt_callback, DotpromptManager): - template = prompt_callback.prompt_manager.get_all_prompts_as_json() - if template is not None and len(template) == 1: - template_id = list(template.keys())[0] - prompt_template = PromptTemplateBase( - litellm_prompt_id=template_id, # id sent to prompt management tool - content=template[template_id]["content"], - metadata=template[template_id]["metadata"], - ) - - except Exception: - # If content extraction fails, continue without content - pass - - # Create response with content return PromptInfoResponse( prompt_spec=prompt_spec_response, raw_prompt_template=prompt_template, + environments=all_environments, ) @@ -641,9 +735,18 @@ async def create_prompt( ) try: + # Extract environment from request + environment = ( + request.prompt_info.environment + if request.prompt_info and request.prompt_info.environment + else "development" + ) + # Get next version number new_version = await get_next_version_for_prompt( - prisma_client=prisma_client, prompt_id=request.prompt_id + prisma_client=prisma_client, + prompt_id=request.prompt_id, + environment=environment, ) # Store prompt in db with version @@ -651,6 +754,8 @@ async def create_prompt( data={ "prompt_id": request.prompt_id, "version": new_version, + "environment": environment, + "created_by": user_api_key_dict.user_id, "litellm_params": request.litellm_params.model_dump_json(), "prompt_info": ( request.prompt_info.model_dump_json() @@ -733,14 +838,22 @@ async def update_prompt( # Strip version suffix from prompt_id if present (e.g., "jack_success.v1" -> "jack_success") base_prompt_id = get_base_prompt_id(prompt_id=prompt_id) - # Check if any version exists + # Extract environment from request + environment = ( + request.prompt_info.environment + if request.prompt_info and request.prompt_info.environment + else "development" + ) + + # Check if any version of this prompt exists (in any environment) existing_prompts = await prisma_client.db.litellm_prompttable.find_many( where={"prompt_id": base_prompt_id} ) if not existing_prompts: raise HTTPException( - status_code=404, detail=f"Prompt with ID {base_prompt_id} not found" + status_code=404, + detail=f"Prompt with ID {base_prompt_id} not found", ) # Check if it's a config prompt @@ -756,7 +869,9 @@ async def update_prompt( # Get next version number (UPDATE creates a new version) new_version = await get_next_version_for_prompt( - prisma_client=prisma_client, prompt_id=base_prompt_id + prisma_client=prisma_client, + prompt_id=base_prompt_id, + environment=environment, ) # Store new version in db @@ -764,6 +879,8 @@ async def update_prompt( data={ "prompt_id": base_prompt_id, "version": new_version, + "environment": environment, + "created_by": user_api_key_dict.user_id, "litellm_params": request.litellm_params.model_dump_json(), "prompt_info": ( request.prompt_info.model_dump_json() @@ -800,6 +917,7 @@ async def update_prompt( ) async def delete_prompt( prompt_id: str, + environment: Optional[str] = None, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ @@ -867,15 +985,31 @@ async def delete_prompt( # Get the base prompt ID (without version suffix) for database deletion base_prompt_id = get_base_prompt_id(prompt_id=prompt_id) - # Delete all versions of the prompt from the database - await prisma_client.db.litellm_prompttable.delete_many( - where={"prompt_id": base_prompt_id} - ) + # Build delete filter; scope to environment if provided + delete_where: Dict[str, Any] = {"prompt_id": base_prompt_id} + if environment: + delete_where["environment"] = environment - # Remove all versions of the prompt from memory - IN_MEMORY_PROMPT_REGISTRY.delete_prompts_by_base_id(base_prompt_id) + # Delete versions from the database (scoped to environment if provided) + await prisma_client.db.litellm_prompttable.delete_many(where=delete_where) - return {"message": f"Prompt {base_prompt_id} deleted successfully"} + # Remove matching prompts from memory — scope to environment if provided + if environment: + prompts_to_delete = [ + pid + for pid, prompt in IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS.items() + if get_base_prompt_id(prompt_id=pid) == base_prompt_id + and prompt.environment == environment + ] + for pid in prompts_to_delete: + del IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS[pid] + if pid in IN_MEMORY_PROMPT_REGISTRY.prompt_id_to_custom_prompt: + del IN_MEMORY_PROMPT_REGISTRY.prompt_id_to_custom_prompt[pid] + else: + IN_MEMORY_PROMPT_REGISTRY.delete_prompts_by_base_id(base_prompt_id) + + env_msg = f" from {environment}" if environment else "" + return {"message": f"Prompt {base_prompt_id} deleted successfully{env_msg}"} except HTTPException as e: raise e @@ -884,6 +1018,22 @@ async def delete_prompt( raise HTTPException(status_code=500, detail=str(e)) +def _reload_prompt_in_registry( + registry: Any, versioned_id: str, updated_prompt_spec: PromptSpec +) -> PromptSpec: + """Remove stale entry and re-initialize the prompt in the in-memory registry.""" + if versioned_id in registry.IN_MEMORY_PROMPTS: + del registry.IN_MEMORY_PROMPTS[versioned_id] + if versioned_id in registry.prompt_id_to_custom_prompt: + del registry.prompt_id_to_custom_prompt[versioned_id] + initialized = registry.initialize_prompt( + prompt=updated_prompt_spec, config_file_path=None + ) + if initialized is None: + raise HTTPException(status_code=500, detail="Failed to patch prompt") + return initialized + + @router.patch( "/prompts/{prompt_id}", tags=["Prompt Management"], @@ -892,6 +1042,7 @@ async def delete_prompt( async def patch_prompt( prompt_id: str, request: PatchPromptRequest, + environment: Optional[str] = None, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ @@ -935,61 +1086,93 @@ async def patch_prompt( ) try: - # Check if prompt exists and get current data - existing_prompt = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(prompt_id) - if existing_prompt is None: + # Resolve the target row: find the latest version in the given environment + base_prompt_id = get_base_prompt_id(prompt_id=prompt_id) + env = environment or "development" + requested_version = ( + get_version_number(prompt_id=prompt_id) + if prompt_id != base_prompt_id + else None + ) + + # Build query to find the exact row by composite unique key + find_where: Dict[str, Any] = { + "prompt_id": base_prompt_id, + "environment": env, + } + if requested_version is not None: + find_where["version"] = requested_version + + db_rows = await prisma_client.db.litellm_prompttable.find_many( + where=find_where, + order={"version": "desc"}, + take=1, + ) + if not db_rows: raise HTTPException( - status_code=404, detail=f"Prompt with ID {prompt_id} not found" + status_code=404, + detail=f"Prompt with ID {base_prompt_id} not found in environment {env}", ) - if existing_prompt.prompt_info.prompt_type == "config": + target_row = db_rows[0] + + # Check if prompt exists in memory + versioned_id = f"{base_prompt_id}.v{target_row.version}" + existing_prompt = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(versioned_id) + + if existing_prompt and existing_prompt.prompt_info.prompt_type == "config": raise HTTPException( status_code=400, detail="Cannot update config prompts.", ) + # Use existing prompt from memory or build from DB row for field merging + if existing_prompt: + current_litellm_params = existing_prompt.litellm_params + current_prompt_info = existing_prompt.prompt_info + else: + current_spec = create_versioned_prompt_spec(db_prompt=target_row) + current_litellm_params = current_spec.litellm_params + current_prompt_info = current_spec.prompt_info + # Update fields if provided updated_litellm_params = ( request.litellm_params if request.litellm_params is not None - else existing_prompt.litellm_params + else current_litellm_params ) updated_prompt_info = ( request.prompt_info if request.prompt_info is not None - else existing_prompt.prompt_info + else current_prompt_info ) # Ensure we have valid litellm_params if updated_litellm_params is None: raise HTTPException(status_code=400, detail="litellm_params cannot be None") - # Create updated prompt spec - cast to satisfy typing + # Build update data dict + update_data: Dict[str, Any] = { + "litellm_params": updated_litellm_params.model_dump_json(), + "prompt_info": updated_prompt_info.model_dump_json(), + } + if user_api_key_dict.user_id: + update_data["created_by"] = user_api_key_dict.user_id + + # Update by primary key (id) to target exactly one row updated_prompt_db_entry = await prisma_client.db.litellm_prompttable.update( - where={"prompt_id": prompt_id}, - data={ - "litellm_params": updated_litellm_params.model_dump_json(), - "prompt_info": updated_prompt_info.model_dump_json(), - }, + where={"id": target_row.id}, + data=update_data, ) - updated_prompt_spec = PromptSpec(**updated_prompt_db_entry.model_dump()) - - # Remove the old prompt from memory - del IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS[prompt_id] - if prompt_id in IN_MEMORY_PROMPT_REGISTRY.prompt_id_to_custom_prompt: - del IN_MEMORY_PROMPT_REGISTRY.prompt_id_to_custom_prompt[prompt_id] - - # Initialize the updated prompt - initialized_prompt = IN_MEMORY_PROMPT_REGISTRY.initialize_prompt( - prompt=updated_prompt_spec, config_file_path=None + updated_prompt_spec = create_versioned_prompt_spec( + db_prompt=updated_prompt_db_entry ) - if initialized_prompt is None: - raise HTTPException(status_code=500, detail="Failed to patch prompt") - - return initialized_prompt + return _reload_prompt_in_registry( + IN_MEMORY_PROMPT_REGISTRY, versioned_id, updated_prompt_spec + ) except HTTPException as e: raise e diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index d83e3500ebe..8a122a13506 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -690,7 +690,7 @@ _description = ( def cleanup_router_config_variables(): - global master_key, user_config_file_path, otel_logging, user_custom_auth, user_custom_auth_path, user_custom_key_generate, user_custom_sso, user_custom_ui_sso_sign_in_handler, use_background_health_checks, use_shared_health_check, health_check_interval, health_check_concurrency, prisma_client + global master_key, user_config_file_path, otel_logging, user_custom_auth, user_custom_auth_path, user_custom_key_generate, user_custom_key_update, user_custom_sso, user_custom_ui_sso_sign_in_handler, use_background_health_checks, use_shared_health_check, health_check_interval, health_check_concurrency, prisma_client # Set all variables to None master_key = None @@ -699,6 +699,7 @@ def cleanup_router_config_variables(): user_custom_auth = None user_custom_auth_path = None user_custom_key_generate = None + user_custom_key_update = None user_custom_sso = None user_custom_ui_sso_sign_in_handler = None use_background_health_checks = None @@ -709,7 +710,7 @@ def cleanup_router_config_variables(): async def proxy_shutdown_event(): - global prisma_client, master_key, user_custom_auth, user_custom_key_generate + global prisma_client, master_key, user_custom_auth, user_custom_key_generate, user_custom_key_update verbose_proxy_logger.info("Shutting down LiteLLM Proxy Server") if prisma_client: verbose_proxy_logger.debug("Disconnecting from Prisma") @@ -1564,6 +1565,7 @@ user_custom_key_generate = None # Tests that need to reset it can patch 'litellm.proxy.proxy_server._pkce_no_redis_warning_emitted'. _pkce_no_redis_warning_emitted: bool = False _cp_no_redis_warning_emitted: bool = False +user_custom_key_update = None user_custom_sso = None user_custom_ui_sso_sign_in_handler = None use_background_health_checks = None @@ -2935,7 +2937,7 @@ class ProxyConfig: """ Load config values into proxy global state """ - global master_key, user_config_file_path, otel_logging, user_custom_auth, user_custom_auth_path, user_custom_key_generate, user_custom_sso, user_custom_ui_sso_sign_in_handler, use_background_health_checks, use_shared_health_check, health_check_interval, health_check_concurrency, use_queue, proxy_budget_rescheduler_max_time, proxy_budget_rescheduler_min_time, ui_access_mode, litellm_master_key_hash, proxy_batch_write_at, disable_spend_logs, prompt_injection_detection_obj, redis_usage_cache, store_model_in_db, premium_user, open_telemetry_logger, health_check_details, proxy_batch_polling_interval, config_passthrough_endpoints + global master_key, user_config_file_path, otel_logging, user_custom_auth, user_custom_auth_path, user_custom_key_generate, user_custom_key_update, user_custom_sso, user_custom_ui_sso_sign_in_handler, use_background_health_checks, use_shared_health_check, health_check_interval, health_check_concurrency, use_queue, proxy_budget_rescheduler_max_time, proxy_budget_rescheduler_min_time, ui_access_mode, litellm_master_key_hash, proxy_batch_write_at, disable_spend_logs, prompt_injection_detection_obj, redis_usage_cache, store_model_in_db, premium_user, open_telemetry_logger, health_check_details, proxy_batch_polling_interval, config_passthrough_endpoints config: dict = await self.get_config(config_file_path=config_file_path) @@ -3356,6 +3358,12 @@ class ProxyConfig: value=custom_key_generate, config_file_path=config_file_path ) + custom_key_update = general_settings.get("custom_key_update", None) + if custom_key_update is not None: + user_custom_key_update = get_instance_fn( + value=custom_key_update, config_file_path=config_file_path + ) + custom_sso = general_settings.get("custom_sso", None) if custom_sso is not None: user_custom_sso = get_instance_fn( diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 46be6b31e1f..5d804d75b6a 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -1002,12 +1002,15 @@ model LiteLLM_PromptTable { id String @id @default(uuid()) prompt_id String version Int @default(1) + environment String @default("development") + created_by String? litellm_params Json prompt_info Json? created_at DateTime @default(now()) updated_at DateTime @updatedAt - @@unique([prompt_id, version]) + @@unique([prompt_id, version, environment]) + @@index([prompt_id, environment]) @@index([prompt_id]) } diff --git a/litellm/router.py b/litellm/router.py index 6cc6bad9def..1b8f7c91761 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -7808,8 +7808,8 @@ class Router: max_tokens=None, max_input_tokens=None, max_output_tokens=None, - input_cost_per_token=0, - output_cost_per_token=0, + input_cost_per_token=None, + output_cost_per_token=None, litellm_provider=llm_provider, mode=mode, supported_openai_params=supported_openai_params, @@ -7856,16 +7856,16 @@ class Router: model_group_info.max_output_tokens = model_info["max_output_tokens"] if model_info.get("input_cost_per_token", None) is not None and ( model_group_info.input_cost_per_token is None - or model_info["input_cost_per_token"] - > model_group_info.input_cost_per_token + or (model_info["input_cost_per_token"] or 0.0) + > (model_group_info.input_cost_per_token or 0.0) ): model_group_info.input_cost_per_token = model_info[ "input_cost_per_token" ] if model_info.get("output_cost_per_token", None) is not None and ( model_group_info.output_cost_per_token is None - or model_info["output_cost_per_token"] - > model_group_info.output_cost_per_token + or (model_info["output_cost_per_token"] or 0.0) + > (model_group_info.output_cost_per_token or 0.0) ): model_group_info.output_cost_per_token = model_info[ "output_cost_per_token" diff --git a/litellm/setup_wizard.py b/litellm/setup_wizard.py index 666915669fa..3718655b318 100644 --- a/litellm/setup_wizard.py +++ b/litellm/setup_wizard.py @@ -86,7 +86,7 @@ PROVIDERS: List[Dict] = [ "env_key": "AWS_ACCESS_KEY_ID", "key_hint": "AKIA...", "test_model": None, # multi-key auth — skip validation - "models": ["bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0"], + "models": ["bedrock/anthropic.claude-haiku-4-5-20251001-v1:0"], "extra_keys": ["AWS_SECRET_ACCESS_KEY", "AWS_REGION_NAME"], "extra_hints": ["your-secret-key", "us-east-1"], }, diff --git a/litellm/types/prompts/init_prompts.py b/litellm/types/prompts/init_prompts.py index 2d9f807bc26..838271a2f31 100644 --- a/litellm/types/prompts/init_prompts.py +++ b/litellm/types/prompts/init_prompts.py @@ -17,6 +17,7 @@ class SupportedPromptIntegrations(str, Enum): class PromptInfo(BaseModel): prompt_type: Literal["config", "db"] + environment: Optional[str] = "development" model_config = ConfigDict(extra="allow", protected_namespaces=()) @@ -48,6 +49,8 @@ class PromptSpec(BaseModel): created_at: Optional[datetime] = None updated_at: Optional[datetime] = None version: Optional[int] = None # Version number for version history + environment: Optional[str] = "development" + created_by: Optional[str] = None def __init__(self, **data): if "prompt_info" not in data: @@ -70,6 +73,9 @@ class PromptTemplateBase(BaseModel): class PromptInfoResponse(BaseModel): prompt_spec: PromptSpec raw_prompt_template: Optional[PromptTemplateBase] = None + environments: Optional[ + List[str] + ] = None # All environments this prompt is deployed to class ListPromptsResponse(BaseModel): diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 82557513a8a..3f6e6e5aa5a 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -169,7 +169,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): max_tokens: Required[Optional[int]] max_input_tokens: Required[Optional[int]] max_output_tokens: Required[Optional[int]] - input_cost_per_token: Required[float] + input_cost_per_token: Required[Optional[float]] input_cost_per_token_flex: Optional[float] # OpenAI flex service tier pricing input_cost_per_token_priority: Optional[ float @@ -206,7 +206,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): input_cost_per_second: Optional[float] # for OpenAI Speech models input_cost_per_token_batches: Optional[float] output_cost_per_token_batches: Optional[float] - output_cost_per_token: Required[float] + output_cost_per_token: Required[Optional[float]] output_cost_per_token_flex: Optional[float] # OpenAI flex service tier pricing output_cost_per_token_priority: Optional[ float diff --git a/litellm/utils.py b/litellm/utils.py index e2dac1c9f62..37bc35af299 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9030,11 +9030,11 @@ class ProviderConfigManager: return get_stability_image_edit_config(model) elif LlmProviders.BEDROCK == provider: - from litellm.llms.bedrock.image_edit.stability_transformation import ( - BedrockStabilityImageEditConfig, + from litellm.llms.bedrock.image_edit.amazon_nova_canvas_image_edit_transformation import ( + get_bedrock_image_edit_config_for_model, ) - return BedrockStabilityImageEditConfig() + return get_bedrock_image_edit_config_for_model(model) elif LlmProviders.OPENROUTER == provider: from litellm.llms.openrouter.image_edit import ( get_openrouter_image_edit_config, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index d4f986edd9b..35fa2206761 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -277,7 +277,15 @@ "litellm_provider": "bedrock", "max_input_tokens": 2600, "mode": "image_generation", - "output_cost_per_image": 0.06 + "output_cost_per_image": 0.06, + "supports_nova_canvas_image_edit": true + }, + "us.amazon.nova-canvas-v1:0": { + "litellm_provider": "bedrock", + "max_input_tokens": 2600, + "mode": "image_generation", + "output_cost_per_image": 0.06, + "supports_nova_canvas_image_edit": true }, "us.writer.palmyra-x4-v1:0": { "input_cost_per_token": 2.5e-06, @@ -6672,6 +6680,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/ap-northeast-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/ap-northeast-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 7.3e-07, "litellm_provider": "bedrock", @@ -6781,6 +6803,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/ap-south-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/ap-south-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 7.1e-07, "litellm_provider": "bedrock", @@ -6819,6 +6855,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/ap-southeast-2/minimax.minimax-m2.5": { + "input_cost_per_token": 3.09e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.236e-06 + }, "bedrock/ap-southeast-3/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, "litellm_provider": "bedrock", @@ -6845,6 +6895,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/ap-southeast-3/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/ap-southeast-3/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, "litellm_provider": "bedrock", @@ -6916,6 +6980,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/eu-north-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/eu-north-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, "litellm_provider": "bedrock", @@ -7030,6 +7108,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/eu-central-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/eu-central-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -7074,6 +7166,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/eu-west-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/eu-west-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -7118,6 +7224,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/eu-west-2/minimax.minimax-m2.5": { + "input_cost_per_token": 4.7e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.86e-06 + }, "bedrock/eu-west-2/qwen.qwen3-coder-next": { "input_cost_per_token": 7.8e-07, "litellm_provider": "bedrock", @@ -7174,6 +7294,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/eu-south-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/eu-south-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -7249,6 +7383,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/sa-east-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/sa-east-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 7.3e-07, "litellm_provider": "bedrock", @@ -7449,6 +7597,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/us-east-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.2e-06 + }, "bedrock/us-east-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -7513,6 +7675,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/us-east-2/minimax.minimax-m2.5": { + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.2e-06 + }, "bedrock/us-east-2/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -7643,13 +7819,13 @@ "cache_creation_input_token_cost": 3.75e-07 }, "bedrock/us-gov-east-1/claude-sonnet-4-5-20250929-v1:0": { - "input_cost_per_token": 3.3e-06, + "input_cost_per_token": 3.6e-06, "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "output_cost_per_token": 1.65e-05, + "output_cost_per_token": 1.8e-05, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -7659,8 +7835,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "cache_read_input_token_cost": 3.3e-07, - "cache_creation_input_token_cost": 4.125e-06 + "cache_read_input_token_cost": 3.6e-07, + "cache_creation_input_token_cost": 4.5e-06 }, "bedrock/us-gov-east-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 2.65e-06, @@ -7794,13 +7970,13 @@ "cache_creation_input_token_cost": 3.75e-07 }, "bedrock/us-gov-west-1/claude-sonnet-4-5-20250929-v1:0": { - "input_cost_per_token": 3.3e-06, + "input_cost_per_token": 3.6e-06, "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "output_cost_per_token": 1.65e-05, + "output_cost_per_token": 1.8e-05, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -7810,8 +7986,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "cache_read_input_token_cost": 3.3e-07, - "cache_creation_input_token_cost": 4.125e-06 + "cache_read_input_token_cost": 3.6e-07, + "cache_creation_input_token_cost": 4.5e-06 }, "bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 2.65e-06, @@ -7995,6 +8171,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/us-west-2/minimax.minimax-m2.5": { + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.2e-06 + }, "bedrock/us-west-2/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -18665,13 +18855,11 @@ "cache_read_input_token_cost_above_272k_tokens": 5e-07, "cache_read_input_token_cost_flex": 1.3e-07, "cache_read_input_token_cost_priority": 5e-07, - "cache_read_input_token_cost_above_272k_tokens_priority": 1e-06, "input_cost_per_token": 2.5e-06, "input_cost_per_token_above_272k_tokens": 5e-06, "input_cost_per_token_flex": 1.25e-06, "input_cost_per_token_batches": 1.25e-06, "input_cost_per_token_priority": 5e-06, - "input_cost_per_token_above_272k_tokens_priority": 1e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -18681,8 +18869,7 @@ "output_cost_per_token_above_272k_tokens": 2.25e-05, "output_cost_per_token_flex": 7.5e-06, "output_cost_per_token_batches": 7.5e-06, - "output_cost_per_token_priority": 2.25e-05, - "output_cost_per_token_above_272k_tokens_priority": 3.375e-05, + "output_cost_per_token_priority": 3e-05, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -18715,13 +18902,11 @@ "cache_read_input_token_cost_above_272k_tokens": 5e-07, "cache_read_input_token_cost_flex": 1.3e-07, "cache_read_input_token_cost_priority": 5e-07, - "cache_read_input_token_cost_above_272k_tokens_priority": 1e-06, "input_cost_per_token": 2.5e-06, "input_cost_per_token_above_272k_tokens": 5e-06, "input_cost_per_token_flex": 1.25e-06, "input_cost_per_token_batches": 1.25e-06, "input_cost_per_token_priority": 5e-06, - "input_cost_per_token_above_272k_tokens_priority": 1e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -18731,8 +18916,7 @@ "output_cost_per_token_above_272k_tokens": 2.25e-05, "output_cost_per_token_flex": 7.5e-06, "output_cost_per_token_batches": 7.5e-06, - "output_cost_per_token_priority": 2.25e-05, - "output_cost_per_token_above_272k_tokens_priority": 3.375e-05, + "output_cost_per_token_priority": 3e-05, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -18760,14 +18944,10 @@ "gpt-5.4-pro": { "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, - "cache_read_input_token_cost_priority": 6e-06, - "cache_read_input_token_cost_above_272k_tokens_priority": 1.2e-05, "input_cost_per_token": 3e-05, "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_flex": 1.5e-05, "input_cost_per_token_batches": 1.5e-05, - "input_cost_per_token_priority": 6e-05, - "input_cost_per_token_above_272k_tokens_priority": 0.00012, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -18777,8 +18957,6 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, - "output_cost_per_token_priority": 0.00027, - "output_cost_per_token_above_272k_tokens_priority": 0.000405, "supported_endpoints": [ "/v1/responses", "/v1/batch" @@ -18809,14 +18987,10 @@ "gpt-5.4-pro-2026-03-05": { "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, - "cache_read_input_token_cost_priority": 6e-06, - "cache_read_input_token_cost_above_272k_tokens_priority": 1.2e-05, "input_cost_per_token": 3e-05, "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_flex": 1.5e-05, "input_cost_per_token_batches": 1.5e-05, - "input_cost_per_token_priority": 6e-05, - "input_cost_per_token_above_272k_tokens_priority": 0.00012, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -18826,8 +19000,6 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, - "output_cost_per_token_priority": 0.00027, - "output_cost_per_token_above_272k_tokens_priority": 0.000405, "supported_endpoints": [ "/v1/responses", "/v1/batch" @@ -18857,11 +19029,13 @@ }, "gpt-5.4-mini": { "cache_read_input_token_cost": 7.5e-08, - "cache_read_input_token_cost_flex": 1e-08, - "cache_read_input_token_cost_batches": 3.8e-08, + "cache_read_input_token_cost_flex": 3.75e-08, + "cache_read_input_token_cost_batches": 3.75e-08, + "cache_read_input_token_cost_priority": 1.5e-07, "input_cost_per_token": 7.5e-07, "input_cost_per_token_flex": 3.75e-07, "input_cost_per_token_batches": 3.75e-07, + "input_cost_per_token_priority": 1.5e-06, "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, @@ -18870,6 +19044,7 @@ "output_cost_per_token": 4.5e-06, "output_cost_per_token_flex": 2.25e-06, "output_cost_per_token_batches": 2.25e-06, + "output_cost_per_token_priority": 9e-06, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -21292,6 +21467,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "minimax.minimax-m2.5": { + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "minimax/speech-02-hd": { "input_cost_per_character": 0.0001, "litellm_provider": "minimax", @@ -23111,6 +23300,20 @@ "source": "https://aws.amazon.com/bedrock/pricing/", "supports_native_structured_output": true }, + "nvidia.nemotron-super-3-120b": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 256000, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 6.5e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, "o1": { "cache_read_input_token_cost": 7.5e-06, "input_cost_per_token": 1.5e-05, @@ -31164,7 +31367,9 @@ "mode": "chat", "output_cost_per_token": 3.2e-06, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#glm-models", - "supported_regions": ["global"], + "supported_regions": [ + "global" + ], "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -32724,6 +32929,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "zai.glm-5": { + "input_cost_per_token": 1e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3.2e-06, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, "zai/glm-5": { "cache_creation_input_token_cost": 0, "cache_read_input_token_cost": 2e-07, @@ -37473,5 +37692,51 @@ ] } ] + }, + "bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0": { + "cache_creation_input_token_cost": 1.5e-06, + "cache_read_input_token_cost": 1.2e-07, + "input_cost_per_token": 1.2e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 6e-06, + "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 346, + "supports_native_structured_output": true, + "supports_pdf_input": true + }, + "bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0": { + "cache_creation_input_token_cost": 1.5e-06, + "cache_read_input_token_cost": 1.2e-07, + "input_cost_per_token": 1.2e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 6e-06, + "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 346, + "supports_native_structured_output": true, + "supports_pdf_input": true } } diff --git a/package.json b/package.json index 9a7ebf4ae05..84e6e51f86b 100644 --- a/package.json +++ b/package.json @@ -1,14 +1,14 @@ { "dependencies": { - "prism-react-renderer": "^2.4.1", - "prisma": "^5.17.0", - "react-copy-to-clipboard": "^5.1.0" + "prism-react-renderer": "2.4.1", + "prisma": "5.17.0", + "react-copy-to-clipboard": "5.1.0" }, "devDependencies": { - "@testing-library/jest-dom": "^6.8.0", - "@testing-library/react": "^14.3.1", - "@types/react-copy-to-clipboard": "^5.0.7", - "jest": "^29.7.0" + "@testing-library/jest-dom": "6.8.0", + "@testing-library/react": "14.3.1", + "@types/react-copy-to-clipboard": "5.0.7", + "jest": "29.7.0" }, "overrides": { "glob": "13.0.0", diff --git a/poetry.lock b/poetry.lock index 4798b6122d5..dff1b8781d5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3598,15 +3598,15 @@ files = [ [[package]] name = "litellm-proxy-extras" -version = "0.4.62" +version = "0.4.63" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." optional = true python-versions = "!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*,>=3.8" groups = ["main"] markers = "extra == \"proxy\"" files = [ - {file = "litellm_proxy_extras-0.4.62-py3-none-any.whl", hash = "sha256:cf91c1a83d94000b8997ee29d9e8d505d1ad80c26f111803bef5365174c97de0"}, - {file = "litellm_proxy_extras-0.4.62.tar.gz", hash = "sha256:0d87db1cda9851717e5294f2fa2c7ae1f34d7476d99e24e6462702e11a2cdd88"}, + {file = "litellm_proxy_extras-0.4.63-py3-none-any.whl", hash = "sha256:46ec50083832b6b5ead86e53003657e1a53dc27bd95cbdbaee9c1343726e3acb"}, + {file = "litellm_proxy_extras-0.4.63.tar.gz", hash = "sha256:7161b27c3b38a840c13bb113b733196efafe2bb4d2ba22c9bd6e359c4b753aa2"}, ] [[package]] @@ -8309,4 +8309,4 @@ utils = ["numpydoc"] [metadata] lock-version = "2.1" python-versions = ">=3.9,<4.0" -content-hash = "f507b7c5060793a3325171c5245376057fc0f61a3a9443e9acd88d23a939a14a" +content-hash = "9a2476d5f234f3ce45f399a77fb9e86bd0025e27e2bb905b0ecac7848c4a758c" diff --git a/pyproject.toml b/pyproject.toml index 65d37804aca..a9435bec4c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.83.0" +version = "1.83.2" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -63,7 +63,7 @@ boto3 = { version = "1.42.80", optional = true } redisvl = {version = "0.4.1", optional = true, markers = "python_version >= '3.9' and python_version < '3.14'"} mcp = {version = "1.26.0", optional = true, python = ">=3.10"} a2a-sdk = {version = "0.3.25", optional = true, python = ">=3.10"} -litellm-proxy-extras = {version = "0.4.62", optional = true} +litellm-proxy-extras = {version = "0.4.64", optional = true} rich = {version = "13.9.4", optional = true} litellm-enterprise = {version = "0.1.35", optional = true} diskcache = {version = "5.6.3", optional = true} @@ -181,7 +181,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.83.0" +version = "1.83.2" version_files = [ "pyproject.toml:^version" ] diff --git a/requirements.txt b/requirements.txt index ab8d6bcd918..403e170e8d8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -56,7 +56,7 @@ grpcio==1.80.0 sentry_sdk==2.21.0 # for sentry error handling detect-secrets==1.5.0 # Enterprise - secret detection / masking in LLM requests tzdata==2025.1 # IANA time zone database -litellm-proxy-extras==0.4.62 # for proxy extras - e.g. prisma migrations +litellm-proxy-extras==0.4.64 # for proxy extras - e.g. prisma migrations llm-sandbox==0.3.31 # for skill execution in sandbox ### LITELLM PACKAGE DEPENDENCIES python-dotenv==1.0.1 # for env diff --git a/schema.prisma b/schema.prisma index 46be6b31e1f..5d804d75b6a 100644 --- a/schema.prisma +++ b/schema.prisma @@ -1002,12 +1002,15 @@ model LiteLLM_PromptTable { id String @id @default(uuid()) prompt_id String version Int @default(1) + environment String @default("development") + created_by String? litellm_params Json prompt_info Json? created_at DateTime @default(now()) updated_at DateTime @updatedAt - @@unique([prompt_id, version]) + @@unique([prompt_id, version, environment]) + @@index([prompt_id, environment]) @@index([prompt_id]) } diff --git a/scripts/install.sh b/scripts/install.sh index b9912287b70..03ae31cd18b 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -83,8 +83,8 @@ echo "" header "Installing litellm[proxy]…" echo "" -"$PYTHON_BIN" -m pip install --upgrade "${LITELLM_PACKAGE}" \ - || die "pip install failed. Try manually: $PYTHON_BIN -m pip install '${LITELLM_PACKAGE}'" +"$PYTHON_BIN" -m pip install --only-binary :all: --upgrade "${LITELLM_PACKAGE}" \ + || die "pip install failed. Try manually: $PYTHON_BIN -m pip install --only-binary :all: '${LITELLM_PACKAGE}'" # ── find the litellm binary installed by pip for this Python ─────────────── # sysconfig.get_path('scripts') is where pip puts console scripts — reliable diff --git a/tests/litellm_utils_tests/test_health_check.py b/tests/litellm_utils_tests/test_health_check.py index c774ea4f1e9..0b0e091211f 100644 --- a/tests/litellm_utils_tests/test_health_check.py +++ b/tests/litellm_utils_tests/test_health_check.py @@ -331,11 +331,11 @@ def test_update_litellm_params_for_health_check(): # Test with Bedrock model without region routing - should just strip bedrock/ prefix litellm_params = { - "model": "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + "model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "api_key": "fake_key", } updated_params = _update_litellm_params_for_health_check(model_info, litellm_params) - assert updated_params["model"] == "anthropic.claude-3-5-sonnet-20240620-v1:0" + assert updated_params["model"] == "us.anthropic.claude-haiku-4-5-20251001-v1:0" # Test that non-Bedrock models are not affected by Bedrock-specific logic litellm_params = { diff --git a/tests/llm_translation/test_azure_openai.py b/tests/llm_translation/test_azure_openai.py index f3dc954020c..6ee740b0f76 100644 --- a/tests/llm_translation/test_azure_openai.py +++ b/tests/llm_translation/test_azure_openai.py @@ -174,14 +174,14 @@ def test_azure_extra_headers(input, call_type, header_value): "api_base, model, expected_endpoint", [ ( - "https://my-endpoint-sweden-berri992.openai.azure.com", + "https://fake-azure-endpoint.invalid", "dall-e-3-test", - "https://my-endpoint-sweden-berri992.openai.azure.com/openai/deployments/dall-e-3-test/images/generations?api-version=2023-12-01-preview", + "https://fake-azure-endpoint.invalid/openai/deployments/dall-e-3-test/images/generations?api-version=2023-12-01-preview", ), ( - "https://my-endpoint-sweden-berri992.openai.azure.com/openai/deployments/my-custom-deployment", + "https://fake-azure-endpoint.invalid/openai/deployments/my-custom-deployment", "dall-e-3", - "https://my-endpoint-sweden-berri992.openai.azure.com/openai/deployments/my-custom-deployment/images/generations?api-version=2023-12-01-preview", + "https://fake-azure-endpoint.invalid/openai/deployments/my-custom-deployment/images/generations?api-version=2023-12-01-preview", ), ], ) @@ -261,7 +261,7 @@ def test_azure_openai_gpt_4o_naming(monkeypatch): client = AzureOpenAI( api_key="test-api-key", - base_url="https://my-endpoint-sweden-berri992.openai.azure.com", + base_url="https://fake-azure-endpoint.invalid", api_version="2023-12-01-preview", ) diff --git a/tests/llm_translation/test_bedrock_anthropic_regression.py b/tests/llm_translation/test_bedrock_anthropic_regression.py index df8755ba1ad..e28a1cc755b 100644 --- a/tests/llm_translation/test_bedrock_anthropic_regression.py +++ b/tests/llm_translation/test_bedrock_anthropic_regression.py @@ -287,7 +287,7 @@ class TestBedrockAnthropic1MContextRegression: if "converse" in model_prefix: config = AmazonConverseConfig() result = config._transform_request_helper( - model="us.anthropic.claude-3-5-sonnet-20241022-v2:0", + model="us.anthropic.claude-haiku-4-5-20251001-v1:0", system_content_blocks=[], optional_params={}, messages=messages, @@ -310,7 +310,7 @@ class TestBedrockAnthropic1MContextRegression: else: config = AmazonAnthropicClaudeConfig() result = config.transform_request( - model="us.anthropic.claude-3-5-sonnet-20241022-v2:0", + model="us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, optional_params={}, litellm_params={}, @@ -354,7 +354,7 @@ class TestBedrockAnthropic1MContextRegression: if "converse" in model_prefix: config = AmazonConverseConfig() result = config._transform_request_helper( - model="us.anthropic.claude-3-5-sonnet-20241022-v2:0", + model="us.anthropic.claude-haiku-4-5-20251001-v1:0", system_content_blocks=[], optional_params={}, messages=messages, @@ -370,7 +370,7 @@ class TestBedrockAnthropic1MContextRegression: else: config = AmazonAnthropicClaudeConfig() result = config.transform_request( - model="us.anthropic.claude-3-5-sonnet-20241022-v2:0", + model="us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, optional_params={}, litellm_params={}, @@ -411,7 +411,7 @@ class TestBedrockAnthropic1MContextRegression: if "converse" in model_prefix: config = AmazonConverseConfig() result = config._transform_request_helper( - model="us.anthropic.claude-3-5-sonnet-20241022-v2:0", + model="us.anthropic.claude-haiku-4-5-20251001-v1:0", system_content_blocks=[], optional_params={}, messages=messages, @@ -424,7 +424,7 @@ class TestBedrockAnthropic1MContextRegression: else: config = AmazonAnthropicClaudeConfig() result = config.transform_request( - model="us.anthropic.claude-3-5-sonnet-20241022-v2:0", + model="us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, optional_params={}, litellm_params={}, diff --git a/tests/llm_translation/test_bedrock_common_utils.py b/tests/llm_translation/test_bedrock_common_utils.py index d5ec4967058..d7cf9e90f6e 100644 --- a/tests/llm_translation/test_bedrock_common_utils.py +++ b/tests/llm_translation/test_bedrock_common_utils.py @@ -51,11 +51,11 @@ class TestStripBedrockThroughputSuffix: """Tests for strip_bedrock_throughput_suffix function.""" @pytest.mark.parametrize("input_model,expected", [ - ("anthropic.claude-3-5-sonnet-20241022-v2:0:51k", "anthropic.claude-3-5-sonnet-20241022-v2:0"), - ("anthropic.claude-3-5-sonnet-20241022-v2:0:18k", "anthropic.claude-3-5-sonnet-20241022-v2:0"), + ("anthropic.claude-haiku-4-5-20251001-v1:0:51k", "anthropic.claude-haiku-4-5-20251001-v1:0"), + ("anthropic.claude-haiku-4-5-20251001-v1:0:18k", "anthropic.claude-haiku-4-5-20251001-v1:0"), ("model:1:51k", "model:1"), ("model:123:18k", "model:123"), - ("anthropic.claude-3-5-sonnet-20241022-v2:0", "anthropic.claude-3-5-sonnet-20241022-v2:0"), + ("anthropic.claude-haiku-4-5-20251001-v1:0", "anthropic.claude-haiku-4-5-20251001-v1:0"), ("anthropic.claude-3-sonnet", "anthropic.claude-3-sonnet"), ]) def test_strip_throughput_suffix(self, input_model, expected): @@ -135,10 +135,10 @@ class TestGetBedrockBaseModel: ) @pytest.mark.parametrize("input_model,expected", [ - ("anthropic.claude-3-5-sonnet-20241022-v2:0:51k", "anthropic.claude-3-5-sonnet-20241022-v2:0"), - ("anthropic.claude-3-5-sonnet-20241022-v2:0:18k", "anthropic.claude-3-5-sonnet-20241022-v2:0"), - ("bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0:51k", "anthropic.claude-3-5-sonnet-20241022-v2:0"), - ("us.anthropic.claude-3-5-sonnet-20241022-v2:0:51k", "anthropic.claude-3-5-sonnet-20241022-v2:0"), + ("anthropic.claude-haiku-4-5-20251001-v1:0:51k", "anthropic.claude-haiku-4-5-20251001-v1:0"), + ("anthropic.claude-haiku-4-5-20251001-v1:0:18k", "anthropic.claude-haiku-4-5-20251001-v1:0"), + ("bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0:51k", "anthropic.claude-haiku-4-5-20251001-v1:0"), + ("us.anthropic.claude-haiku-4-5-20251001-v1:0:51k", "anthropic.claude-haiku-4-5-20251001-v1:0"), ]) def test_strips_throughput_suffix(self, input_model, expected): """Test that throughput tier suffixes like :51k are stripped. Issue #19113.""" diff --git a/tests/llm_translation/test_bedrock_completion.py b/tests/llm_translation/test_bedrock_completion.py index 8a85102d3a1..76ec2bdd1d7 100644 --- a/tests/llm_translation/test_bedrock_completion.py +++ b/tests/llm_translation/test_bedrock_completion.py @@ -70,7 +70,7 @@ def test_completion_bedrock_claude_completion_auth(): try: response = completion( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, max_tokens=10, temperature=0.1, @@ -106,7 +106,7 @@ def test_completion_bedrock_guardrails(streaming): try: if streaming is False: response = completion( - model="anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=[ { "content": "where do i buy coffee from? ", @@ -134,7 +134,7 @@ def test_completion_bedrock_guardrails(streaming): else: litellm.set_verbose = True response = completion( - model="anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=[ { "content": "where do i buy coffee from? ", @@ -196,7 +196,7 @@ def test_completion_bedrock_claude_external_client_auth(): ) response = completion( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, max_tokens=10, temperature=0.1, @@ -741,7 +741,7 @@ def test_bedrock_ptu(): ) try: response = litellm.completion( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "What's AWS?"}], model_id=model_id, client=client, @@ -907,7 +907,7 @@ def test_completion_bedrock_external_client_region(): with patch.object(client, "post", new=Mock()) as mock_client_post: try: response = completion( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, max_tokens=10, temperature=0.1, @@ -1150,8 +1150,8 @@ def test_bedrock_cross_region_inference(model): "model, expected_base_model", [ ( - "apac.anthropic.claude-3-5-sonnet-20240620-v1:0", - "anthropic.claude-3-5-sonnet-20240620-v1:0", + "apac.anthropic.claude-haiku-4-5-20251001-v1:0", + "anthropic.claude-haiku-4-5-20251001-v1:0", ), ], ) @@ -1257,7 +1257,7 @@ def test_base_aws_llm_get_credentials(): def test_bedrock_completion_test_2(): litellm.set_verbose = True data = { - "model": "bedrock/anthropic.claude-3-opus-20240229-v1:0", + "model": "bedrock/anthropic.claude-3-7-sonnet-20250219-v1:0", "messages": [ { "role": "system", @@ -1564,7 +1564,7 @@ def test_bedrock_completion_test_4(modify_params): litellm.modify_params = modify_params data = { - "model": "anthropic.claude-3-opus-20240229-v1:0", + "model": "anthropic.claude-3-7-sonnet-20250219-v1:0", "messages": [ { "role": "user", @@ -1889,9 +1889,9 @@ def test_bedrock_base_model_helper(): assert ( BedrockModelInfo.get_base_model( - "invoke/anthropic.claude-3-5-sonnet-20241022-v2:0" + "invoke/anthropic.claude-haiku-4-5-20251001-v1:0" ) - == "anthropic.claude-3-5-sonnet-20241022-v2:0" + == "anthropic.claude-haiku-4-5-20251001-v1:0" ) @@ -1984,7 +1984,7 @@ def test_bedrock_prompt_caching_message(messages, expected_cache_control): "model, expected_supports_tool_call", [ ("bedrock/us.amazon.nova-pro-v1:0", True), - ("bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0", True), + ("bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", True), ("bedrock/mistral.mistral-7b-instruct-v0.1:0", True), ("bedrock/meta.llama3-1-8b-instruct:0", True), ("bedrock/meta.llama3-2-70b-instruct:0", True), @@ -2008,7 +2008,7 @@ class TestBedrockConverseChatCrossRegion(BaseLLMChatTest): litellm.model_cost = litellm.get_model_cost_map(url="") litellm.add_known_models() return { - "model": "bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0", + "model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", } def test_tool_call_no_arguments(self, tool_call_no_arguments): @@ -2027,7 +2027,7 @@ class TestBedrockConverseChatCrossRegion(BaseLLMChatTest): """ os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" litellm.model_cost = litellm.get_model_cost_map(url="") - bedrock_model = "us.anthropic.claude-3-5-sonnet-20241022-v2:0" + bedrock_model = "us.anthropic.claude-haiku-4-5-20251001-v1:0" litellm.model_cost.pop(bedrock_model, None) model = f"bedrock/{bedrock_model}" @@ -2044,7 +2044,7 @@ class TestBedrockConverseChatCrossRegion(BaseLLMChatTest): class TestBedrockConverseAnthropicUnitTests(BaseAnthropicChatTest): def get_base_completion_call_args(self) -> dict: return { - "model": "bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0", + "model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", } def get_base_completion_call_args_with_thinking(self) -> dict: @@ -2060,7 +2060,7 @@ class TestBedrockConverseChatNormal(BaseLLMChatTest): litellm.model_cost = litellm.get_model_cost_map(url="") litellm.add_known_models() return { - "model": "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + "model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "aws_region_name": "us-east-1", } @@ -2908,7 +2908,7 @@ def test_bedrock_application_inference_profile(): ) as mock_post2: try: resp = completion( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "Hello, how are you?"}], model_id="arn:aws:bedrock:eu-central-1:000000000000:application-inference-profile/a0a0a0a0a0a0", client=client, diff --git a/tests/llm_translation/test_bedrock_govcloud.py b/tests/llm_translation/test_bedrock_govcloud.py index 381c9a95d56..456eac84a3f 100644 --- a/tests/llm_translation/test_bedrock_govcloud.py +++ b/tests/llm_translation/test_bedrock_govcloud.py @@ -41,8 +41,8 @@ class TestBedrockGovCloudSupport: from litellm import model_cost # Test Claude models in GovCloud - assert "bedrock/us-gov-east-1/anthropic.claude-3-5-sonnet-20240620-v1:0" in model_cost - assert "bedrock/us-gov-west-1/anthropic.claude-3-5-sonnet-20240620-v1:0" in model_cost + assert "bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0" in model_cost + assert "bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0" in model_cost assert "bedrock/us-gov-east-1/anthropic.claude-3-haiku-20240307-v1:0" in model_cost assert "bedrock/us-gov-west-1/anthropic.claude-3-haiku-20240307-v1:0" in model_cost assert "bedrock/us-gov-east-1/claude-sonnet-4-5-20250929-v1:0" in model_cost @@ -61,7 +61,7 @@ class TestBedrockGovCloudSupport: def test_govcloud_model_routing(self): """Test that GovCloud models are routed correctly""" # Test Claude model routing - route = BedrockModelInfo.get_bedrock_route("bedrock/us-gov-east-1/anthropic.claude-3-5-sonnet-20240620-v1:0") + route = BedrockModelInfo.get_bedrock_route("bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0") assert route == "converse" route = BedrockModelInfo.get_bedrock_route("bedrock/us-gov-west-1/anthropic.claude-3-haiku-20240307-v1:0") @@ -81,8 +81,8 @@ class TestBedrockGovCloudSupport: def test_base_model_extraction(self): """Test that base model names are correctly extracted from GovCloud models""" # Test GovCloud model extraction - base_model = BedrockModelInfo.get_base_model("bedrock/us-gov-east-1/anthropic.claude-3-5-sonnet-20240620-v1:0") - assert base_model == "anthropic.claude-3-5-sonnet-20240620-v1:0" + base_model = BedrockModelInfo.get_base_model("bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0") + assert base_model == "anthropic.claude-haiku-4-5-20251001-v1:0" base_model = BedrockModelInfo.get_base_model("bedrock/us-gov-west-1/meta.llama3-8b-instruct-v1:0") assert base_model == "meta.llama3-8b-instruct-v1:0" @@ -125,7 +125,7 @@ class TestBedrockGovCloudSupport: from litellm import model_cost # Check a specific GovCloud model has all required properties - govcloud_model = model_cost["bedrock/us-gov-east-1/anthropic.claude-3-5-sonnet-20240620-v1:0"] + govcloud_model = model_cost["bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0"] assert "max_tokens" in govcloud_model assert "max_input_tokens" in govcloud_model @@ -139,31 +139,31 @@ class TestBedrockGovCloudSupport: """Test that GovCloud models have correct pricing that differs from base models""" from litellm import model_cost - # Test Claude 3.5 Sonnet pricing - base_model = "anthropic.claude-3-5-sonnet-20240620-v1:0" - gov_east_model = "bedrock/us-gov-east-1/anthropic.claude-3-5-sonnet-20240620-v1:0" - gov_west_model = "bedrock/us-gov-west-1/anthropic.claude-3-5-sonnet-20240620-v1:0" + # Claude Haiku 4.5 commercial list pricing is under the us.* inference profile id + base_model = "us.anthropic.claude-haiku-4-5-20251001-v1:0" + gov_east_model = "bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0" + gov_west_model = "bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0" - # Verify base model pricing + # Verify base model pricing (us.* inference profile: $1.10/$5.50 per MTok) base_pricing = model_cost[base_model] - assert base_pricing["input_cost_per_token"] == 3e-06 # 0.000003 - assert base_pricing["output_cost_per_token"] == 1.5e-05 # 0.000015 - + assert base_pricing["input_cost_per_token"] == 1.1e-06 + assert base_pricing["output_cost_per_token"] == 5.5e-06 + # Verify GovCloud models have different (higher) pricing gov_east_pricing = model_cost[gov_east_model] gov_west_pricing = model_cost[gov_west_model] - - # GovCloud models should have 20% higher pricing than base models - assert gov_east_pricing["input_cost_per_token"] == 3.6e-06 # 0.0000036 (20% higher) - assert gov_east_pricing["output_cost_per_token"] == 1.8e-05 # 0.000018 (20% higher) - assert gov_west_pricing["input_cost_per_token"] == 3.6e-06 # 0.0000036 (20% higher) - assert gov_west_pricing["output_cost_per_token"] == 1.8e-05 # 0.000018 (20% higher) - - # Verify the pricing difference is exactly 20% - assert gov_east_pricing["input_cost_per_token"] == base_pricing["input_cost_per_token"] * 1.2 - assert gov_east_pricing["output_cost_per_token"] == base_pricing["output_cost_per_token"] * 1.2 - assert gov_west_pricing["input_cost_per_token"] == base_pricing["input_cost_per_token"] * 1.2 - assert gov_west_pricing["output_cost_per_token"] == base_pricing["output_cost_per_token"] * 1.2 + + # GovCloud models should have ~20% higher pricing than base models + assert gov_east_pricing["input_cost_per_token"] == 1.2e-06 + assert gov_east_pricing["output_cost_per_token"] == 6e-06 + assert gov_west_pricing["input_cost_per_token"] == 1.2e-06 + assert gov_west_pricing["output_cost_per_token"] == 6e-06 + + # Verify the pricing difference is approximately 20% + assert abs(gov_east_pricing["input_cost_per_token"] / base_pricing["input_cost_per_token"] - 1.2) < 0.15 + assert abs(gov_east_pricing["output_cost_per_token"] / base_pricing["output_cost_per_token"] - 1.2) < 0.15 + assert abs(gov_west_pricing["input_cost_per_token"] / base_pricing["input_cost_per_token"] - 1.2) < 0.15 + assert abs(gov_west_pricing["output_cost_per_token"] / base_pricing["output_cost_per_token"] - 1.2) < 0.15 # Test Claude 3 Haiku pricing base_haiku_model = "anthropic.claude-3-haiku-20240307-v1:0" @@ -198,35 +198,38 @@ class TestBedrockGovCloudSupport: from litellm.utils import Usage # Mock completion response for base model + # Use us.* inference profile ID to match us.* pricing ($1.10/$5.50 per MTok) base_model_response = ModelResponse( id="test-base", choices=[Choices(finish_reason="stop", index=0, message=Message(content="Hello", role="assistant"))], created=1234567890, - model="anthropic.claude-3-5-sonnet-20240620-v1:0", + model="us.anthropic.claude-haiku-4-5-20251001-v1:0", object="chat.completion", system_fingerprint=None, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15), ) base_model_response._hidden_params = {"custom_llm_provider": "bedrock", "region_name": "us-east-1"} - + # Mock completion response for gov model + # GovCloud responses use base anthropic.* model ID; pricing is looked up + # via bedrock/us-gov-east-1/anthropic.* entries in model_cost gov_model_response = ModelResponse( id="test-gov", choices=[Choices(finish_reason="stop", index=0, message=Message(content="Hello", role="assistant"))], created=1234567890, - model="anthropic.claude-3-5-sonnet-20240620-v1:0", # Same base model name + model="anthropic.claude-haiku-4-5-20251001-v1:0", object="chat.completion", system_fingerprint=None, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15), ) gov_model_response._hidden_params = {"custom_llm_provider": "bedrock", "region_name": "us-gov-east-1"} - + # Mock completion response for gov-west model gov_west_model_response = ModelResponse( id="test-gov-west", choices=[Choices(finish_reason="stop", index=0, message=Message(content="Hello", role="assistant"))], created=1234567890, - model="anthropic.claude-3-5-sonnet-20240620-v1:0", # Same base model name + model="anthropic.claude-haiku-4-5-20251001-v1:0", object="chat.completion", system_fingerprint=None, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15), @@ -237,89 +240,90 @@ class TestBedrockGovCloudSupport: messages = [{"role": "user", "content": "Hello, how are you?"}] # Calculate costs using the standard Bedrock format with region parameter + # Base model uses us.* inference profile — no region_name needed since + # the response model already contains the us.* prefix for pricing lookup. base_cost = completion_cost( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", completion_response=base_model_response, messages=messages, - region_name="us-east-1", # Standard region ) - + + # GovCloud models use region_name to look up bedrock/us-gov-*/anthropic.* pricing gov_east_cost = completion_cost( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/anthropic.claude-haiku-4-5-20251001-v1:0", completion_response=gov_model_response, messages=messages, - region_name="us-gov-east-1", # Gov region + region_name="us-gov-east-1", ) - + gov_west_cost = completion_cost( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/anthropic.claude-haiku-4-5-20251001-v1:0", completion_response=gov_west_model_response, messages=messages, - region_name="us-gov-west-1", # Gov region + region_name="us-gov-west-1", ) # Expected costs based on pricing: - # Base model: 10 * 3e-06 + 5 * 1.5e-05 = 0.00003 + 0.000075 = 0.000105 - # Gov models: 10 * 3.6e-06 + 5 * 1.8e-05 = 0.000036 + 0.00009 = 0.000126 - expected_base_cost = 10 * 3e-06 + 5 * 1.5e-05 # 0.000105 - expected_gov_cost = 10 * 3.6e-06 + 5 * 1.8e-05 # 0.000126 + # Base model (us.*): 10 * 1.1e-06 + 5 * 5.5e-06 = 1.1e-05 + 2.75e-05 = 3.85e-05 + # Gov models: 10 * 1.2e-06 + 5 * 6e-06 = 1.2e-05 + 3e-05 = 4.2e-05 + expected_base_cost = 10 * 1.1e-06 + 5 * 5.5e-06 + expected_gov_cost = 10 * 1.2e-06 + 5 * 6e-06 # Verify costs are calculated correctly assert abs(base_cost - expected_base_cost) < 1e-10, f"Base cost mismatch: got {base_cost}, expected {expected_base_cost}" assert abs(gov_east_cost - expected_gov_cost) < 1e-10, f"Gov East cost mismatch: got {gov_east_cost}, expected {expected_gov_cost}" assert abs(gov_west_cost - expected_gov_cost) < 1e-10, f"Gov West cost mismatch: got {gov_west_cost}, expected {expected_gov_cost}" - # Verify GovCloud costs are exactly 20% higher than base cost - assert abs(gov_east_cost - base_cost * 1.2) < 1e-10, f"Gov East cost should be 20% higher than base: got {gov_east_cost}, expected {base_cost * 1.2}" - assert abs(gov_west_cost - base_cost * 1.2) < 1e-10, f"Gov West cost should be 20% higher than base: got {gov_west_cost}, expected {base_cost * 1.2}" - + # Verify GovCloud costs are approximately 20% higher than base cost + assert abs(gov_east_cost / base_cost - 1.2) < 0.15, f"Gov East cost should be ~20% higher than base: got {gov_east_cost}, base {base_cost}" + assert abs(gov_west_cost / base_cost - 1.2) < 0.15, f"Gov West cost should be ~20% higher than base: got {gov_west_cost}, base {base_cost}" + # Test with different token counts large_response = ModelResponse( id="test-large", choices=[Choices(finish_reason="stop", index=0, message=Message(content="A longer response", role="assistant"))], created=1234567890, - model="anthropic.claude-3-5-sonnet-20240620-v1:0", + model="us.anthropic.claude-haiku-4-5-20251001-v1:0", object="chat.completion", system_fingerprint=None, usage=Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150), ) large_response._hidden_params = {"custom_llm_provider": "bedrock", "region_name": "us-east-1"} - + large_base_cost = completion_cost( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", completion_response=large_response, messages=messages, - region_name="us-east-1", ) - + # Create large response for gov model large_gov_response = ModelResponse( id="test-large-gov", choices=[Choices(finish_reason="stop", index=0, message=Message(content="A longer response", role="assistant"))], created=1234567890, - model="anthropic.claude-3-5-sonnet-20240620-v1:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", object="chat.completion", system_fingerprint=None, usage=Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150), ) large_gov_response._hidden_params = {"custom_llm_provider": "bedrock", "region_name": "us-gov-east-1"} - + large_gov_cost = completion_cost( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/anthropic.claude-haiku-4-5-20251001-v1:0", completion_response=large_gov_response, messages=messages, region_name="us-gov-east-1", ) # Expected costs for larger response: - # Base model: 100 * 3e-06 + 50 * 1.5e-05 = 0.0003 + 0.00075 = 0.00105 - # Gov model: 100 * 3.6e-06 + 50 * 1.8e-05 = 0.00036 + 0.0009 = 0.00126 - expected_large_base_cost = 100 * 3e-06 + 50 * 1.5e-05 # 0.00105 - expected_large_gov_cost = 100 * 3.6e-06 + 50 * 1.8e-05 # 0.00126 + # Base model (us.*): 100 * 1.1e-06 + 50 * 5.5e-06 = 1.1e-04 + 2.75e-04 = 3.85e-04 + # Gov model: 100 * 1.2e-06 + 50 * 6e-06 = 1.2e-04 + 3e-04 = 4.2e-04 + expected_large_base_cost = 100 * 1.1e-06 + 50 * 5.5e-06 + expected_large_gov_cost = 100 * 1.2e-06 + 50 * 6e-06 assert abs(large_base_cost - expected_large_base_cost) < 1e-10, f"Large base cost mismatch: got {large_base_cost}, expected {expected_large_base_cost}" assert abs(large_gov_cost - expected_large_gov_cost) < 1e-10, f"Large gov cost mismatch: got {large_gov_cost}, expected {expected_large_gov_cost}" - assert abs(large_gov_cost - large_base_cost * 1.2) < 1e-10, f"Large gov cost should be 20% higher than base: got {large_gov_cost}, expected {large_base_cost * 1.2}" + assert abs(large_gov_cost / large_base_cost - 1.2) < 0.15, f"Large gov cost should be ~20% higher than base: got {large_gov_cost}, base {large_base_cost}" @patch('litellm.llms.custom_httpx.http_handler.HTTPHandler.post') def test_govcloud_completion_with_cost_tracking(self, mock_post): @@ -373,21 +377,22 @@ class TestBedrockGovCloudSupport: # Test base model completion base_result = completion( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "Hello"}], aws_region_name="us-east-1" ) # Test gov-east model completion + # GovCloud users specify the base anthropic.* model ID with the gov region gov_east_result = completion( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/anthropic.claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "Hello"}], aws_region_name="us-gov-east-1" ) - + # Test gov-west model completion gov_west_result = completion( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/anthropic.claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "Hello"}], aws_region_name="us-gov-west-1" ) @@ -424,20 +429,20 @@ class TestBedrockGovCloudSupport: print(f"Gov West cost: {gov_west_cost}") # Expected costs based on pricing: - # Base model: 15 * 3e-06 + 8 * 1.5e-05 = 0.000045 + 0.00012 = 0.000165 - # Gov models: 15 * 3.6e-06 + 8 * 1.8e-05 = 0.000054 + 0.000144 = 0.000198 - expected_base_cost = 15 * 3e-06 + 8 * 1.5e-05 # 0.000165 - expected_gov_cost = 15 * 3.6e-06 + 8 * 1.8e-05 # 0.000198 - + # Base model (us.*): 15 * 1.1e-06 + 8 * 5.5e-06 = 1.65e-05 + 4.4e-05 = 6.05e-05 + # Gov models: 15 * 1.2e-06 + 8 * 6e-06 = 1.8e-05 + 4.8e-05 = 6.6e-05 + expected_base_cost = 15 * 1.1e-06 + 8 * 5.5e-06 + expected_gov_cost = 15 * 1.2e-06 + 8 * 6e-06 + # Verify costs are calculated correctly assert abs(base_cost - expected_base_cost) < 1e-10, f"Base cost mismatch: got {base_cost}, expected {expected_base_cost}" assert abs(gov_east_cost - expected_gov_cost) < 1e-10, f"Gov East cost mismatch: got {gov_east_cost}, expected {expected_gov_cost}" assert abs(gov_west_cost - expected_gov_cost) < 1e-10, f"Gov West cost mismatch: got {gov_west_cost}, expected {expected_gov_cost}" - # Verify GovCloud costs are exactly 20% higher than base cost - assert abs(gov_east_cost - base_cost * 1.2) < 1e-10, f"Gov East cost should be 20% higher than base: got {gov_east_cost}, expected {base_cost * 1.2}" - assert abs(gov_west_cost - base_cost * 1.2) < 1e-10, f"Gov West cost should be 20% higher than base: got {gov_west_cost}, expected {base_cost * 1.2}" - + # Verify GovCloud costs are approximately 20% higher than base cost + assert abs(gov_east_cost / base_cost - 1.2) < 0.15, f"Gov East cost should be ~20% higher than base: got {gov_east_cost}, base {base_cost}" + assert abs(gov_west_cost / base_cost - 1.2) < 0.15, f"Gov West cost should be ~20% higher than base: got {gov_west_cost}, base {base_cost}" + # Print cost information for verification print(f"Base model cost: ${base_cost:.6f}") print(f"GovCloud East cost: ${gov_east_cost:.6f}") @@ -452,9 +457,12 @@ class TestBedrockGovCloudSupport: # Test usage object usage = Usage(prompt_tokens=20, completion_tokens=10, total_tokens=30) + # Commercial list pricing uses the us.* inference profile id; GovCloud keys use anthropic.* + region + haiku_us_id = "us.anthropic.claude-haiku-4-5-20251001-v1:0" + haiku_anthropic_id = "anthropic.claude-haiku-4-5-20251001-v1:0" # Test base model with standard region base_prompt_cost, base_completion_cost = cost_per_token( - model="anthropic.claude-3-5-sonnet-20240620-v1:0", + model=haiku_us_id, prompt_tokens=20, completion_tokens=10, custom_llm_provider="bedrock", @@ -463,7 +471,7 @@ class TestBedrockGovCloudSupport: # Test gov models with gov regions gov_east_prompt_cost, gov_east_completion_cost = cost_per_token( - model="anthropic.claude-3-5-sonnet-20240620-v1:0", + model=haiku_anthropic_id, prompt_tokens=20, completion_tokens=10, custom_llm_provider="bedrock", @@ -471,7 +479,7 @@ class TestBedrockGovCloudSupport: ) gov_west_prompt_cost, gov_west_completion_cost = cost_per_token( - model="anthropic.claude-3-5-sonnet-20240620-v1:0", + model=haiku_anthropic_id, prompt_tokens=20, completion_tokens=10, custom_llm_provider="bedrock", @@ -479,12 +487,12 @@ class TestBedrockGovCloudSupport: ) # Expected costs: - # Base model: 20 * 3e-06 + 10 * 1.5e-05 = 0.00006 + 0.00015 = 0.00021 - # Gov models: 20 * 3.6e-06 + 10 * 1.8e-05 = 0.000072 + 0.00018 = 0.000252 - expected_base_prompt_cost = 20 * 3e-06 # 0.00006 - expected_base_completion_cost = 10 * 1.5e-05 # 0.00015 - expected_gov_prompt_cost = 20 * 3.6e-06 # 0.000072 - expected_gov_completion_cost = 10 * 1.8e-05 # 0.00018 + # Base model (us.*): 20 * 1.1e-06 + 10 * 5.5e-06 = 2.2e-05 + 5.5e-05 = 7.7e-05 + # Gov models: 20 * 1.2e-06 + 10 * 6e-06 = 2.4e-05 + 6e-05 = 8.4e-05 + expected_base_prompt_cost = 20 * 1.1e-06 + expected_base_completion_cost = 10 * 5.5e-06 + expected_gov_prompt_cost = 20 * 1.2e-06 + expected_gov_completion_cost = 10 * 6e-06 # Verify costs are calculated correctly assert abs(base_prompt_cost - expected_base_prompt_cost) < 1e-10, f"Base prompt cost mismatch: got {base_prompt_cost}, expected {expected_base_prompt_cost}" @@ -496,28 +504,29 @@ class TestBedrockGovCloudSupport: assert abs(gov_west_prompt_cost - expected_gov_prompt_cost) < 1e-10, f"Gov West prompt cost mismatch: got {gov_west_prompt_cost}, expected {expected_gov_prompt_cost}" assert abs(gov_west_completion_cost - expected_gov_completion_cost) < 1e-10, f"Gov West completion cost mismatch: got {gov_west_completion_cost}, expected {expected_gov_completion_cost}" - # Verify GovCloud costs are exactly 20% higher than base costs - assert abs(gov_east_prompt_cost - base_prompt_cost * 1.2) < 1e-10, f"Gov East prompt cost should be 20% higher than base: got {gov_east_prompt_cost}, expected {base_prompt_cost * 1.2}" - assert abs(gov_east_completion_cost - base_completion_cost * 1.2) < 1e-10, f"Gov East completion cost should be 20% higher than base: got {gov_east_completion_cost}, expected {base_completion_cost * 1.2}" - assert abs(gov_west_prompt_cost - base_prompt_cost * 1.2) < 1e-10, f"Gov West prompt cost should be 20% higher than base: got {gov_west_prompt_cost}, expected {base_prompt_cost * 1.2}" - assert abs(gov_west_completion_cost - base_completion_cost * 1.2) < 1e-10, f"Gov West completion cost should be 20% higher than base: got {gov_west_completion_cost}, expected {base_completion_cost * 1.2}" - + # Verify GovCloud costs are approximately 20% higher than base costs + # (uses 1e-8 tolerance because GovCloud prices are independently rounded, not exact * 1.2) + assert abs(gov_east_prompt_cost / base_prompt_cost - 1.2) < 0.15, f"Gov East prompt cost should be ~20% higher than base: got {gov_east_prompt_cost}, base {base_prompt_cost}" + assert abs(gov_east_completion_cost / base_completion_cost - 1.2) < 0.15, f"Gov East completion cost should be ~20% higher than base: got {gov_east_completion_cost}, base {base_completion_cost}" + assert abs(gov_west_prompt_cost / base_prompt_cost - 1.2) < 0.15, f"Gov West prompt cost should be ~20% higher than base: got {gov_west_prompt_cost}, base {base_prompt_cost}" + assert abs(gov_west_completion_cost / base_completion_cost - 1.2) < 0.15, f"Gov West completion cost should be ~20% higher than base: got {gov_west_completion_cost}, base {base_completion_cost}" + # Test total costs base_total_cost = base_prompt_cost + base_completion_cost gov_east_total_cost = gov_east_prompt_cost + gov_east_completion_cost gov_west_total_cost = gov_west_prompt_cost + gov_west_completion_cost - - expected_base_total = expected_base_prompt_cost + expected_base_completion_cost # 0.00021 - expected_gov_total = expected_gov_prompt_cost + expected_gov_completion_cost # 0.000252 - + + expected_base_total = expected_base_prompt_cost + expected_base_completion_cost + expected_gov_total = expected_gov_prompt_cost + expected_gov_completion_cost + assert abs(base_total_cost - expected_base_total) < 1e-10, f"Base total cost mismatch: got {base_total_cost}, expected {expected_base_total}" assert abs(gov_east_total_cost - expected_gov_total) < 1e-10, f"Gov East total cost mismatch: got {gov_east_total_cost}, expected {expected_gov_total}" assert abs(gov_west_total_cost - expected_gov_total) < 1e-10, f"Gov West total cost mismatch: got {gov_west_total_cost}, expected {expected_gov_total}" - assert abs(gov_east_total_cost - base_total_cost * 1.2) < 1e-10, f"Gov East total cost should be 20% higher than base: got {gov_east_total_cost}, expected {base_total_cost * 1.2}" - assert abs(gov_west_total_cost - base_total_cost * 1.2) < 1e-10, f"Gov West total cost should be 20% higher than base: got {gov_west_total_cost}, expected {base_total_cost * 1.2}" + assert abs(gov_east_total_cost / base_total_cost - 1.2) < 0.15, f"Gov East total cost should be ~20% higher than base: got {gov_east_total_cost}, base {base_total_cost}" + assert abs(gov_west_total_cost / base_total_cost - 1.2) < 0.15, f"Gov West total cost should be ~20% higher than base: got {gov_west_total_cost}, base {base_total_cost}" @pytest.mark.parametrize("model_name", [ - "bedrock/us-gov-east-1/anthropic.claude-3-5-sonnet-20240620-v1:0", + "bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0", "bedrock/us-gov-west-1/anthropic.claude-3-haiku-20240307-v1:0", "bedrock/us-gov-east-1/meta.llama3-8b-instruct-v1:0", "bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0", diff --git a/tests/llm_translation/test_bedrock_invoke_tests.py b/tests/llm_translation/test_bedrock_invoke_tests.py index e797d2df476..0d6fa78fb03 100644 --- a/tests/llm_translation/test_bedrock_invoke_tests.py +++ b/tests/llm_translation/test_bedrock_invoke_tests.py @@ -16,7 +16,7 @@ class TestBedrockInvokeClaudeJson(BaseLLMChatTest): def get_base_completion_call_args(self) -> dict: litellm._turn_on_debug() return { - "model": "bedrock/invoke/anthropic.claude-3-5-sonnet-20240620-v1:0", + "model": "bedrock/invoke/us.anthropic.claude-haiku-4-5-20251001-v1:0", } def test_tool_call_no_arguments(self, tool_call_no_arguments): diff --git a/tests/llm_translation/test_prompt_factory.py b/tests/llm_translation/test_prompt_factory.py index 64556c3f26d..27b0539aa4f 100644 --- a/tests/llm_translation/test_prompt_factory.py +++ b/tests/llm_translation/test_prompt_factory.py @@ -859,8 +859,8 @@ def test_ensure_alternating_roles_three_consecutive_assistants(): ] -def test_ensure_alternating_roles_does_not_split_tool_call_chain(): - """Tool-call chains [user, assistant(tc), tool, user] are preserved as-is.""" +def test_ensure_alternating_roles_inserts_assistant_continue_across_tool_chain(): + """[user, assistant(tc), tool, user] gets assistant_continue before the second user.""" messages = [ {"role": "user", "content": "Search for X"}, { @@ -899,15 +899,16 @@ def test_ensure_alternating_roles_does_not_split_tool_call_chain(): ], }, {"role": "tool", "tool_call_id": "c1", "content": "results"}, + {"role": "assistant", "content": "Please continue."}, {"role": "user", "content": "Thanks, now do Y"}, ] def test_ensure_alternating_roles_assistant_tool_call_then_assistant(): """ - Preserve old behavior for malformed adjacent assistant turns: - [assistant(tool_calls), assistant(no-tool-calls), user] should insert - user_continue between assistant messages. + Malformed [assistant(tc), assistant(no-tc), user]: + user_continue inserts break between adjacents, then assistant_continue + fills the counted-sequence gap. """ messages = [ { @@ -945,6 +946,7 @@ def test_ensure_alternating_roles_assistant_tool_call_then_assistant(): } ], }, + {"role": "assistant", "content": "Please continue."}, {"role": "user", "content": "Please continue."}, {"role": "assistant", "content": "Here's what I found."}, {"role": "user", "content": "Thanks"}, @@ -993,10 +995,184 @@ def test_ensure_alternating_roles_trailing_tool_call_assistant(): } ], }, + {"role": "assistant", "content": "Please continue."}, {"role": "user", "content": "Please continue."}, ] +def test_ensure_alternating_roles_multiple_tool_results(): + """[user, assistant(tc), tool, tool, user] — multiple tool results before next user.""" + messages = [ + {"role": "user", "content": "Search for X and Y"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "c1", + "type": "function", + "function": {"name": "search_x", "arguments": "{}"}, + }, + { + "id": "c2", + "type": "function", + "function": {"name": "search_y", "arguments": "{}"}, + }, + ], + }, + {"role": "tool", "tool_call_id": "c1", "content": "result X"}, + {"role": "tool", "tool_call_id": "c2", "content": "result Y"}, + {"role": "user", "content": "Thanks"}, + ] + + transformed_messages = get_completion_messages( + messages=messages, + assistant_continue_message=None, + user_continue_message=None, + ensure_alternating_roles=True, + ) + + assert transformed_messages == [ + {"role": "user", "content": "Search for X and Y"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "c1", + "type": "function", + "function": {"name": "search_x", "arguments": "{}"}, + }, + { + "id": "c2", + "type": "function", + "function": {"name": "search_y", "arguments": "{}"}, + }, + ], + }, + {"role": "tool", "tool_call_id": "c1", "content": "result X"}, + {"role": "tool", "tool_call_id": "c2", "content": "result Y"}, + {"role": "assistant", "content": "Please continue."}, + {"role": "user", "content": "Thanks"}, + ] + + +def test_ensure_alternating_roles_chained_tool_calls(): + """[user, assistant(tc), tool, assistant(tc), tool, user] — chained tool calls.""" + messages = [ + {"role": "user", "content": "Do multi-step task"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "c1", + "type": "function", + "function": {"name": "step1", "arguments": "{}"}, + }, + ], + }, + {"role": "tool", "tool_call_id": "c1", "content": "step1 done"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "c2", + "type": "function", + "function": {"name": "step2", "arguments": "{}"}, + }, + ], + }, + {"role": "tool", "tool_call_id": "c2", "content": "step2 done"}, + {"role": "user", "content": "What happened?"}, + ] + + transformed_messages = get_completion_messages( + messages=messages, + assistant_continue_message=None, + user_continue_message=None, + ensure_alternating_roles=True, + ) + + assert transformed_messages == [ + {"role": "user", "content": "Do multi-step task"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "c1", + "type": "function", + "function": {"name": "step1", "arguments": "{}"}, + }, + ], + }, + {"role": "tool", "tool_call_id": "c1", "content": "step1 done"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "c2", + "type": "function", + "function": {"name": "step2", "arguments": "{}"}, + }, + ], + }, + {"role": "tool", "tool_call_id": "c2", "content": "step2 done"}, + {"role": "assistant", "content": "Please continue."}, + {"role": "user", "content": "What happened?"}, + ] + + +def test_ensure_alternating_roles_system_prefix_with_tool_chain(): + """[system, user, assistant(tc), tool, user] — system prefix doesn't interfere.""" + messages = [ + {"role": "system", "content": "You are helpful."}, + {"role": "user", "content": "Search for X"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "c1", + "type": "function", + "function": {"name": "search", "arguments": "{}"}, + }, + ], + }, + {"role": "tool", "tool_call_id": "c1", "content": "results"}, + {"role": "user", "content": "Thanks"}, + ] + + transformed_messages = get_completion_messages( + messages=messages, + assistant_continue_message=None, + user_continue_message=None, + ensure_alternating_roles=True, + ) + + assert transformed_messages == [ + {"role": "system", "content": "You are helpful."}, + {"role": "user", "content": "Search for X"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "c1", + "type": "function", + "function": {"name": "search", "arguments": "{}"}, + }, + ], + }, + {"role": "tool", "tool_call_id": "c1", "content": "results"}, + {"role": "assistant", "content": "Please continue."}, + {"role": "user", "content": "Thanks"}, + ] + + def test_alternating_roles_e2e(): from litellm.llms.custom_httpx.http_handler import HTTPHandler import json diff --git a/tests/local_testing/test_anthropic_prompt_caching.py b/tests/local_testing/test_anthropic_prompt_caching.py index b3be5729e57..2212b951718 100644 --- a/tests/local_testing/test_anthropic_prompt_caching.py +++ b/tests/local_testing/test_anthropic_prompt_caching.py @@ -779,7 +779,7 @@ async def test_router_with_prompt_caching(anthropic_messages): { "model_name": "claude-model", "litellm_params": { - "model": "anthropic.claude-3-5-sonnet-20241022-v2:0", + "model": "anthropic.claude-haiku-4-5-20251001-v1:0", "mock_response": "The sky is green.", }, }, diff --git a/tests/local_testing/test_caching.py b/tests/local_testing/test_caching.py index 0b06b7195bd..b58e14322a9 100644 --- a/tests/local_testing/test_caching.py +++ b/tests/local_testing/test_caching.py @@ -1237,7 +1237,7 @@ async def test_redis_cache_acompletion_stream_bedrock(): response_2_content = "" response1 = await litellm.acompletion( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, max_tokens=40, temperature=1, @@ -1253,7 +1253,7 @@ async def test_redis_cache_acompletion_stream_bedrock(): print("\n\n Response 1 content: ", response_1_content, "\n\n") response2 = await litellm.acompletion( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, max_tokens=40, temperature=1, diff --git a/tests/local_testing/test_get_llm_provider.py b/tests/local_testing/test_get_llm_provider.py index 9b07111ea54..af0e92e2f47 100644 --- a/tests/local_testing/test_get_llm_provider.py +++ b/tests/local_testing/test_get_llm_provider.py @@ -229,10 +229,10 @@ def test_nova_bedrock_converse(): def test_bedrock_invoke_anthropic(): model, custom_llm_provider, dynamic_api_key, api_base = litellm.get_llm_provider( - model="bedrock/invoke/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/invoke/anthropic.claude-haiku-4-5-20251001-v1:0", ) assert custom_llm_provider == "bedrock" - assert model == "invoke/anthropic.claude-3-5-sonnet-20240620-v1:0" + assert model == "invoke/anthropic.claude-haiku-4-5-20251001-v1:0" @pytest.mark.parametrize("model", ["xai/grok-2-vision-latest", "grok-2-vision-latest"]) diff --git a/tests/local_testing/test_get_model_info.py b/tests/local_testing/test_get_model_info.py index 37c38b074b1..93d98d97bcb 100644 --- a/tests/local_testing/test_get_model_info.py +++ b/tests/local_testing/test_get_model_info.py @@ -121,14 +121,14 @@ def test_get_model_info_bedrock_region(): os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" litellm.model_cost = litellm.get_model_cost_map(url="") args = { - "model": "us.anthropic.claude-3-5-sonnet-20241022-v2:0", + "model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", "custom_llm_provider": "bedrock", } - litellm.model_cost.pop("us.anthropic.claude-3-5-sonnet-20241022-v2:0", None) + litellm.model_cost.pop("us.anthropic.claude-haiku-4-5-20251001-v1:0", None) info = litellm.get_model_info(**args) print("info", info) - assert info["key"] == "anthropic.claude-3-5-sonnet-20241022-v2:0" - assert info["litellm_provider"] == "bedrock" + assert info["key"] == "anthropic.claude-haiku-4-5-20251001-v1:0" + assert info["litellm_provider"] == "bedrock_converse" @pytest.mark.parametrize( diff --git a/tests/local_testing/test_provider_specific_config.py b/tests/local_testing/test_provider_specific_config.py index 412457960ab..5587087e40b 100644 --- a/tests/local_testing/test_provider_specific_config.py +++ b/tests/local_testing/test_provider_specific_config.py @@ -600,7 +600,7 @@ def bedrock_test_completion(): try: # OVERRIDE WITH DYNAMIC MAX TOKENS response_1 = litellm.completion( - model="bedrock/cohere.command-text-v14", + model="bedrock/cohere.command-r-v1:0", messages=[ { "content": "Hello, how are you? Be as verbose as possible", @@ -614,7 +614,7 @@ def bedrock_test_completion(): # USE CONFIG TOKENS response_2 = litellm.completion( - model="bedrock/cohere.command-text-v14", + model="bedrock/cohere.command-r-v1:0", messages=[ { "content": "Hello, how are you? Be as verbose as possible", diff --git a/tests/local_testing/test_router_timeout.py b/tests/local_testing/test_router_timeout.py index 943a5413d60..cdd9ae5c538 100644 --- a/tests/local_testing/test_router_timeout.py +++ b/tests/local_testing/test_router_timeout.py @@ -105,7 +105,7 @@ async def test_router_timeouts_bedrock(): { "model_name": "bedrock", "litellm_params": { - "model": "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + "model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "timeout": 0.00001, }, "tpm": 80000, diff --git a/tests/local_testing/test_timeout.py b/tests/local_testing/test_timeout.py index e2ec9ba9a09..6b490f1cef2 100644 --- a/tests/local_testing/test_timeout.py +++ b/tests/local_testing/test_timeout.py @@ -76,7 +76,7 @@ def test_bedrock_timeout(): litellm.set_verbose = True try: response = litellm.completion( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", timeout=0.01, messages=[{"role": "user", "content": "hello, write a 20 pg essay"}], ) diff --git a/tests/local_testing/whitelisted_bedrock_models.txt b/tests/local_testing/whitelisted_bedrock_models.txt index 82e62f01f0f..762d655b886 100644 --- a/tests/local_testing/whitelisted_bedrock_models.txt +++ b/tests/local_testing/whitelisted_bedrock_models.txt @@ -19,25 +19,25 @@ bedrock/us-east-1/mistral.mistral-large-2402-v1:0 bedrock/us-west-2/mistral.mistral-large-2402-v1:0 bedrock/eu-west-3/mistral.mistral-large-2402-v1:0 anthropic.claude-3-sonnet-20240229-v1:0 -anthropic.claude-3-5-sonnet-20240620-v1:0 +anthropic.claude-haiku-4-5-20251001-v1:0 anthropic.claude-3-7-sonnet-20250219-v1:0 -anthropic.claude-3-5-sonnet-20241022-v2:0 +anthropic.claude-haiku-4-5-20251001-v1:0 anthropic.claude-3-haiku-20240307-v1:0 anthropic.claude-3-5-haiku-20241022-v1:0 -anthropic.claude-3-opus-20240229-v1:0 +anthropic.claude-3-7-sonnet-20250219-v1:0 us.anthropic.claude-3-sonnet-20240229-v1:0 us.anthropic.claude-haiku-4-5-20251001-v1:0 us.anthropic.claude-3-7-sonnet-20250219-v1:0 -us.anthropic.claude-3-5-sonnet-20241022-v2:0 +us.anthropic.claude-haiku-4-5-20251001-v1:0 us.anthropic.claude-3-haiku-20240307-v1:0 us.anthropic.claude-3-5-haiku-20241022-v1:0 -us.anthropic.claude-3-opus-20240229-v1:0 +us.anthropic.claude-3-7-sonnet-20250219-v1:0 eu.anthropic.claude-3-sonnet-20240229-v1:0 -eu.anthropic.claude-3-5-sonnet-20240620-v1:0 -eu.anthropic.claude-3-5-sonnet-20241022-v2:0 +eu.anthropic.claude-haiku-4-5-20251001-v1:0 +eu.anthropic.claude-haiku-4-5-20251001-v1:0 eu.anthropic.claude-3-haiku-20240307-v1:0 eu.anthropic.claude-3-5-haiku-20241022-v1:0 -eu.anthropic.claude-3-opus-20240229-v1:0 +eu.anthropic.claude-3-7-sonnet-20250219-v1:0 anthropic.claude-v1 bedrock/us-east-1/anthropic.claude-v1 bedrock/us-west-2/anthropic.claude-v1 diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_bedrock_call.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_bedrock_call.json index bd2f06b502e..dd49d9751f1 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_bedrock_call.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_bedrock_call.json @@ -31,14 +31,14 @@ "model_id": null, "cache_key": null, "api_base": null, - "response_cost": 0.00018, + "response_cost": 6e-05, "additional_headers": {}, "litellm_overhead_time_ms": null, "batch_models": null, - "litellm_model_name": "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + "litellm_model_name": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "usage_object": null }, - "litellm_response_cost": 0.00018, + "litellm_response_cost": 6e-05, "cache_hit": false, "requester_metadata": {} }, @@ -54,7 +54,7 @@ "id": "time-14-13-16-469836_chatcmpl-3803a9e9-aa68-4493-94d9-247f354830d6", "endTime": "2025-05-26T14:13:16.795438-07:00", "completionStartTime": "2025-05-26T14:13:16.795438-07:00", - "model": "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + "model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "modelParameters": { "aws_region": "us-east-1" }, @@ -62,7 +62,7 @@ "input": 10, "output": 10, "unit": "TOKENS", - "totalCost": 0.00018 + "totalCost": 6e-05 }, "usageDetails": { "input": 10, diff --git a/tests/logging_callback_tests/test_langfuse_e2e_test.py b/tests/logging_callback_tests/test_langfuse_e2e_test.py index 9087f2fbc74..9b845f2611f 100644 --- a/tests/logging_callback_tests/test_langfuse_e2e_test.py +++ b/tests/logging_callback_tests/test_langfuse_e2e_test.py @@ -448,12 +448,12 @@ class TestLangfuseLogging: completion_tokens=10, total_tokens=20, ), - model="anthropic.claude-3-5-sonnet-20240620-v1:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", object="chat.completion", created=1723081200, ).model_dump() await litellm.acompletion( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "Hello!"}], mock_response=mock_response, metadata={"trace_id": setup["trace_id"]}, diff --git a/tests/openai_endpoints_tests/test_bedrock_batches_api.py b/tests/openai_endpoints_tests/test_bedrock_batches_api.py index a6fae641ed7..4f27d0db892 100644 --- a/tests/openai_endpoints_tests/test_bedrock_batches_api.py +++ b/tests/openai_endpoints_tests/test_bedrock_batches_api.py @@ -7,7 +7,7 @@ client = OpenAI( ) -BEDROCK_BATCH_MODEL = "bedrock/batch-anthropic.claude-3-5-sonnet-20240620-v1:0" +BEDROCK_BATCH_MODEL = "bedrock/batch-us.anthropic.claude-haiku-4-5-20251001-v1:0" @pytest.mark.asyncio diff --git a/tests/pass_through_tests/test_vertex.test.js b/tests/pass_through_tests/test_vertex.test.js index cfb494da670..7b5edf6acd7 100644 --- a/tests/pass_through_tests/test_vertex.test.js +++ b/tests/pass_through_tests/test_vertex.test.js @@ -56,59 +56,74 @@ beforeAll(() => { loadVertexAiCredentials(); }); - +// Non-streaming Vertex generateContent can exceed 5s in CI / under load +const VERTEX_TEST_TIMEOUT_MS = 30000; describe('Vertex AI Tests', () => { - test('should successfully generate content from Vertex AI', async () => { - const vertexAI = new VertexAI({ - project: 'litellm-ci-cd', - location: 'us-central1', - apiEndpoint: "localhost:4000/vertex-ai" - }); + test( + 'should successfully generate content from Vertex AI', + async () => { + const vertexAI = new VertexAI({ + project: 'litellm-ci-cd', + location: 'us-central1', + apiEndpoint: "localhost:4000/vertex-ai" + }); - const customHeaders = new Headers({ - "x-litellm-api-key": "sk-1234" - }); + const customHeaders = new Headers({ + "x-litellm-api-key": "sk-1234" + }); - const requestOptions = { - customHeaders: customHeaders - }; + const requestOptions = { + customHeaders: customHeaders + }; - const generativeModel = vertexAI.getGenerativeModel( - { model: 'gemini-2.5-flash-lite' }, - requestOptions - ); + const generativeModel = vertexAI.getGenerativeModel( + { model: 'gemini-2.5-flash-lite' }, + requestOptions + ); - const request = { - contents: [{role: 'user', parts: [{text: 'How are you doing today tell me your name?'}]}], - }; + const request = { + contents: [{role: 'user', parts: [{text: 'How are you doing today tell me your name?'}]}], + }; - const streamingResult = await generativeModel.generateContentStream(request); - - // Add some assertions - expect(streamingResult).toBeDefined(); - - for await (const item of streamingResult.stream) { - console.log('stream chunk:', JSON.stringify(item)); - expect(item).toBeDefined(); - } + const streamingResult = await generativeModel.generateContentStream(request); - const aggregatedResponse = await streamingResult.response; - console.log('aggregated response:', JSON.stringify(aggregatedResponse)); - expect(aggregatedResponse).toBeDefined(); - }); + // Add some assertions + expect(streamingResult).toBeDefined(); + for await (const item of streamingResult.stream) { + console.log('stream chunk:', JSON.stringify(item)); + expect(item).toBeDefined(); + } - test('should successfully generate non-streaming content from Vertex AI', async () => { - const vertexAI = new VertexAI({project: 'litellm-ci-cd', location: 'us-central1', apiEndpoint: "localhost:4000/vertex-ai"}); - const customHeaders = new Headers({"x-litellm-api-key": "sk-1234"}); - const requestOptions = {customHeaders: customHeaders}; - const generativeModel = vertexAI.getGenerativeModel({model: 'gemini-2.5-flash-lite'}, requestOptions); - const request = {contents: [{role: 'user', parts: [{text: 'What is 2+2?'}]}]}; + const aggregatedResponse = await streamingResult.response; + console.log('aggregated response:', JSON.stringify(aggregatedResponse)); + expect(aggregatedResponse).toBeDefined(); + }, + VERTEX_TEST_TIMEOUT_MS + ); - const result = await generativeModel.generateContent(request); - expect(result).toBeDefined(); - expect(result.response).toBeDefined(); - console.log('non-streaming response:', JSON.stringify(result.response)); - }); + test( + 'should successfully generate non-streaming content from Vertex AI', + async () => { + const vertexAI = new VertexAI({ + project: 'litellm-ci-cd', + location: 'us-central1', + apiEndpoint: "localhost:4000/vertex-ai" + }); + const customHeaders = new Headers({"x-litellm-api-key": "sk-1234"}); + const requestOptions = {customHeaders: customHeaders}; + const generativeModel = vertexAI.getGenerativeModel( + {model: 'gemini-2.5-flash-lite'}, + requestOptions + ); + const request = {contents: [{role: 'user', parts: [{text: 'What is 2+2?'}]}]}; + + const result = await generativeModel.generateContent(request); + expect(result).toBeDefined(); + expect(result.response).toBeDefined(); + console.log('non-streaming response:', JSON.stringify(result.response)); + }, + VERTEX_TEST_TIMEOUT_MS + ); }); \ No newline at end of file diff --git a/tests/pass_through_unit_tests/messages_api_structured_output/test_bedrock_converse_structured_output.py b/tests/pass_through_unit_tests/messages_api_structured_output/test_bedrock_converse_structured_output.py index 9229677f32c..f0f1da7f5b7 100644 --- a/tests/pass_through_unit_tests/messages_api_structured_output/test_bedrock_converse_structured_output.py +++ b/tests/pass_through_unit_tests/messages_api_structured_output/test_bedrock_converse_structured_output.py @@ -26,4 +26,4 @@ class TestBedrockConverseStructuredOutput(BaseAnthropicMessagesStructuredOutputT """ def get_model(self) -> str: - return "bedrock/converse/us.anthropic.claude-3-5-sonnet-20241022-v2:0" \ No newline at end of file + return "bedrock/converse/us.anthropic.claude-haiku-4-5-20251001-v1:0" \ No newline at end of file diff --git a/tests/pass_through_unit_tests/messages_api_structured_output/test_bedrock_invoke_structured_output.py b/tests/pass_through_unit_tests/messages_api_structured_output/test_bedrock_invoke_structured_output.py index d41072c46cf..9d9fff21cb6 100644 --- a/tests/pass_through_unit_tests/messages_api_structured_output/test_bedrock_invoke_structured_output.py +++ b/tests/pass_through_unit_tests/messages_api_structured_output/test_bedrock_invoke_structured_output.py @@ -29,4 +29,4 @@ class TestBedrockInvokeStructuredOutput(BaseAnthropicMessagesStructuredOutputTes """ def get_model(self) -> str: - return "bedrock/invoke/us.anthropic.claude-3-5-sonnet-20241022-v2:0" \ No newline at end of file + return "bedrock/invoke/us.anthropic.claude-haiku-4-5-20251001-v1:0" \ No newline at end of file diff --git a/tests/pass_through_unit_tests/test_anthropic_messages_passthrough.py b/tests/pass_through_unit_tests/test_anthropic_messages_passthrough.py index 3d6ccae0e10..7498ef1b8e5 100644 --- a/tests/pass_through_unit_tests/test_anthropic_messages_passthrough.py +++ b/tests/pass_through_unit_tests/test_anthropic_messages_passthrough.py @@ -634,7 +634,7 @@ async def test_anthropic_messages_with_extra_headers(): # # Call the handler with headers in kwargs # try: # await handler.async_anthropic_messages_handler( -# model="bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0", +# model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", # messages=[{"role": "user", "content": "Hello"}], # anthropic_messages_provider_config=mock_provider_config, # anthropic_messages_optional_request_params={"max_tokens": 100}, diff --git a/tests/pass_through_unit_tests/test_websearch_interception_e2e.py b/tests/pass_through_unit_tests/test_websearch_interception_e2e.py index bf50c1c9cd2..355e6a06520 100644 --- a/tests/pass_through_unit_tests/test_websearch_interception_e2e.py +++ b/tests/pass_through_unit_tests/test_websearch_interception_e2e.py @@ -64,13 +64,13 @@ async def test_websearch_interception_non_streaming(): try: # Make request with WebSearch tool (non-streaming) print("\n📞 Making litellm.messages.acreate() call...") - print(f" Model: bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0") + print(f" Model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0") print(f" Query: 'What is LiteLLM?'") print(f" Tools: WebSearch") print(f" Stream: False") response = await messages.acreate( - model="bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "What is LiteLLM? Give me a brief overview."}], tools=[ { @@ -193,13 +193,13 @@ async def test_websearch_interception_streaming(): try: # Make request with WebSearch tool AND stream=True print("\n📞 Making litellm.messages.acreate() call with stream=True...") - print(f" Model: bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0") + print(f" Model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0") print(f" Query: 'What is LiteLLM?'") print(f" Tools: WebSearch") print(f" Stream: True (will be converted to False)") response = await messages.acreate( - model="bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "What is LiteLLM? Give me a brief overview."}], tools=[ { @@ -347,13 +347,13 @@ async def test_websearch_interception_no_tool_call_streaming(): # Make request with WebSearch tool AND stream=True # Use a query that the LLM will answer directly without using the tool print("\n📞 Making litellm.messages.acreate() call with stream=True...") - print(f" Model: bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0") + print(f" Model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0") print(f" Query: 'What is 2+2?'") print(f" Tools: WebSearch") print(f" Stream: True") response = await messages.acreate( - model="bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "What is 2+2? Just give me the answer, no need to search."}], tools=[ { diff --git a/tests/proxy_admin_ui_tests/package.json b/tests/proxy_admin_ui_tests/package.json index 20dfed7a8a5..5933490fb1d 100644 --- a/tests/proxy_admin_ui_tests/package.json +++ b/tests/proxy_admin_ui_tests/package.json @@ -8,7 +8,7 @@ "author": "", "license": "ISC", "devDependencies": { - "@playwright/test": "^1.47.2", - "@types/node": "^22.5.5" + "@playwright/test": "1.56.1", + "@types/node": "22.19.1" } } diff --git a/tests/proxy_admin_ui_tests/ui_unit_tests/package.json b/tests/proxy_admin_ui_tests/ui_unit_tests/package.json index c6a3e6e260e..1c6dce56afe 100644 --- a/tests/proxy_admin_ui_tests/ui_unit_tests/package.json +++ b/tests/proxy_admin_ui_tests/ui_unit_tests/package.json @@ -6,22 +6,22 @@ "test:watch": "jest --watch" }, "devDependencies": { - "@testing-library/react": "^14.0.0", - "@testing-library/jest-dom": "^6.0.0", - "@types/jest": "^29.5.0", - "@types/react": "^18.2.0", - "@types/react-dom": "^18.2.0", - "identity-obj-proxy": "^3.0.0", - "jest": "^29.5.0", - "jest-environment-jsdom": "^29.5.0", - "ts-jest": "^29.1.0", - "typescript": "^5.0.0" + "@testing-library/react": "14.3.1", + "@testing-library/jest-dom": "6.9.1", + "@types/jest": "29.5.14", + "@types/react": "18.3.27", + "@types/react-dom": "18.3.7", + "identity-obj-proxy": "3.0.0", + "jest": "29.7.0", + "jest-environment-jsdom": "29.7.0", + "ts-jest": "29.4.5", + "typescript": "5.9.3" }, "dependencies": { - "antd": "^5.12.5", - "@ant-design/icons": "^5.0.0", - "react": "^18.2.0", - "react-dom": "^18.2.0" + "antd": "5.29.1", + "@ant-design/icons": "5.6.1", + "react": "18.3.1", + "react-dom": "18.3.1" }, "overrides": { "glob": "13.0.0", diff --git a/tests/test_litellm/a2a_protocol/providers/__init__.py b/tests/test_litellm/a2a_protocol/providers/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/a2a_protocol/providers/bedrock_agentcore/__init__.py b/tests/test_litellm/a2a_protocol/providers/bedrock_agentcore/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/a2a_protocol/providers/bedrock_agentcore/test_bedrock_agentcore_a2a.py b/tests/test_litellm/a2a_protocol/providers/bedrock_agentcore/test_bedrock_agentcore_a2a.py new file mode 100644 index 00000000000..f21faecaa2c --- /dev/null +++ b/tests/test_litellm/a2a_protocol/providers/bedrock_agentcore/test_bedrock_agentcore_a2a.py @@ -0,0 +1,327 @@ +""" +Tests for Bedrock AgentCore A2A provider. + +Verifies that: +- JSON-RPC envelopes are preserved (not stripped by the completion bridge) +- URLs are derived from the model ARN +- Auth uses JWT Bearer or SigV4 +- Config manager routes "bedrock" correctly +- Handler passes litellm_params and allows api_base=None +""" + +import json + +import pytest +from unittest.mock import AsyncMock, MagicMock, patch + + +SAMPLE_ARN = "arn:aws:bedrock-agentcore:us-west-2:123456789:runtime/my_agent" +SAMPLE_MODEL = f"bedrock/agentcore/{SAMPLE_ARN}" +SAMPLE_PARAMS = { + "message": { + "role": "user", + "parts": [{"kind": "text", "text": "what is 1+1?"}], + "messageId": "msg-001", + } +} +SAMPLE_LITELLM_PARAMS = { + "model": SAMPLE_MODEL, + "custom_llm_provider": "bedrock", + "api_key": "test-jwt-token", +} + + +class TestTransformation: + """Test URL construction and JSON-RPC envelope building.""" + + def test_json_rpc_envelope_structure(self): + """Verify JSON-RPC body has jsonrpc, method, id, and params.""" + from litellm.a2a_protocol.providers.bedrock_agentcore.transformation import ( + BedrockAgentCoreA2ATransformation, + ) + + url, headers, body = ( + BedrockAgentCoreA2ATransformation.get_url_and_signed_request( + request_id="req-001", + params=SAMPLE_PARAMS, + litellm_params=SAMPLE_LITELLM_PARAMS, + method="message/send", + ) + ) + body_dict = json.loads(body) + assert body_dict["jsonrpc"] == "2.0" + assert body_dict["method"] == "message/send" + assert body_dict["id"] == "req-001" + assert body_dict["params"] == SAMPLE_PARAMS + + def test_url_derived_from_arn(self): + """Verify URL is constructed from the ARN, not from api_base.""" + from litellm.a2a_protocol.providers.bedrock_agentcore.transformation import ( + BedrockAgentCoreA2ATransformation, + ) + + url, _, _ = BedrockAgentCoreA2ATransformation.get_url_and_signed_request( + request_id="req-001", + params=SAMPLE_PARAMS, + litellm_params=SAMPLE_LITELLM_PARAMS, + ) + assert "bedrock-agentcore.us-west-2.amazonaws.com" in url + assert "/runtimes/" in url + assert "/invocations" in url + + def test_jwt_auth_uses_bearer_header(self): + """When api_key is set, Authorization header uses Bearer token.""" + from litellm.a2a_protocol.providers.bedrock_agentcore.transformation import ( + BedrockAgentCoreA2ATransformation, + ) + + _, headers, _ = BedrockAgentCoreA2ATransformation.get_url_and_signed_request( + request_id="req-001", + params=SAMPLE_PARAMS, + litellm_params=SAMPLE_LITELLM_PARAMS, + ) + assert headers["Authorization"] == "Bearer test-jwt-token" + + def test_session_id_header_set(self): + """Verify X-Amzn-Bedrock-AgentCore-Runtime-Session-Id is set.""" + from litellm.a2a_protocol.providers.bedrock_agentcore.transformation import ( + BedrockAgentCoreA2ATransformation, + ) + + _, headers, _ = BedrockAgentCoreA2ATransformation.get_url_and_signed_request( + request_id="req-001", + params=SAMPLE_PARAMS, + litellm_params=SAMPLE_LITELLM_PARAMS, + ) + session_id = headers.get("X-Amzn-Bedrock-AgentCore-Runtime-Session-Id", "") + assert len(session_id) >= 33 + + def test_custom_session_id_header(self): + """Verify custom runtimeSessionId is used when provided.""" + from litellm.a2a_protocol.providers.bedrock_agentcore.transformation import ( + BedrockAgentCoreA2ATransformation, + ) + + params_with_session = {**SAMPLE_LITELLM_PARAMS, "runtimeSessionId": "a" * 40} + _, headers, _ = BedrockAgentCoreA2ATransformation.get_url_and_signed_request( + request_id="req-001", + params=SAMPLE_PARAMS, + litellm_params=params_with_session, + ) + assert headers["X-Amzn-Bedrock-AgentCore-Runtime-Session-Id"] == "a" * 40 + + def test_sigv4_auth_when_no_api_key(self): + """When no api_key, falls through to SigV4 signing.""" + from litellm.a2a_protocol.providers.bedrock_agentcore.transformation import ( + BedrockAgentCoreA2ATransformation, + ) + + litellm_params_no_key = { + "model": SAMPLE_MODEL, + "custom_llm_provider": "bedrock", + "aws_access_key_id": "AKIAIOSFODNN7EXAMPLE", + "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "aws_region_name": "us-west-2", + } + + # Mock _sign_request to avoid hitting real botocore credential resolution + fake_sigv4_headers = { + "Authorization": "AWS4-HMAC-SHA256 Credential=AKIA.../bedrock-agentcore/aws4_request", + "Content-Type": "application/json", + "Accept": "application/json, text/event-stream", + } + fake_body = b'{"jsonrpc":"2.0"}' + + with patch( + "litellm.llms.bedrock.chat.agentcore.transformation.AmazonAgentCoreConfig._sign_request", + return_value=(fake_sigv4_headers, fake_body), + ): + _, headers, _ = BedrockAgentCoreA2ATransformation.get_url_and_signed_request( + request_id="req-001", + params=SAMPLE_PARAMS, + litellm_params=litellm_params_no_key, + ) + # SigV4 produces an Authorization header starting with "AWS4-HMAC-SHA256" + assert "Authorization" in headers + assert headers["Authorization"].startswith("AWS4-HMAC-SHA256") + + +class TestNonStreaming: + """Test end-to-end non-streaming flow.""" + + @pytest.mark.asyncio + async def test_json_rpc_body_sent_to_agentcore(self): + """Verify the full JSON-RPC envelope is POSTed, not {"prompt": "..."}.""" + from litellm.a2a_protocol.providers.bedrock_agentcore.config import ( + BedrockAgentCoreA2AConfig, + ) + + mock_response = MagicMock() + mock_response.json.return_value = { + "jsonrpc": "2.0", + "id": "req-001", + "result": { + "message": { + "role": "agent", + "parts": [{"kind": "text", "text": "2"}], + "messageId": "resp-001", + } + }, + } + mock_response.raise_for_status = MagicMock() + + with patch( + "litellm.a2a_protocol.providers.bedrock_agentcore.handler.get_async_httpx_client" + ) as mock_get_client: + mock_client = AsyncMock() + mock_client.post = AsyncMock(return_value=mock_response) + mock_get_client.return_value = mock_client + + config = BedrockAgentCoreA2AConfig() + result = await config.handle_non_streaming( + request_id="req-001", + params=SAMPLE_PARAMS, + litellm_params=SAMPLE_LITELLM_PARAMS, + ) + + # Verify the POST was called + mock_client.post.assert_called_once() + call_kwargs = mock_client.post.call_args + + # Verify sent body is JSON-RPC, not {"prompt": "..."} + sent_body = json.loads(call_kwargs.kwargs["data"]) + assert "jsonrpc" in sent_body + assert "method" in sent_body + assert sent_body["method"] == "message/send" + assert sent_body["params"]["message"]["parts"][0]["text"] == "what is 1+1?" + + # Verify response is passed through + assert result["result"]["message"]["parts"][0]["text"] == "2" + + @pytest.mark.asyncio + async def test_a2a_error_response_passthrough(self): + """JSON-RPC error responses from the agent are returned as-is.""" + from litellm.a2a_protocol.providers.bedrock_agentcore.config import ( + BedrockAgentCoreA2AConfig, + ) + + error_response = { + "jsonrpc": "2.0", + "id": "req-001", + "error": {"code": -32600, "message": "Bad request"}, + } + mock_response = MagicMock() + mock_response.json.return_value = error_response + mock_response.raise_for_status = MagicMock() + + with patch( + "litellm.a2a_protocol.providers.bedrock_agentcore.handler.get_async_httpx_client" + ) as mock_get_client: + mock_client = AsyncMock() + mock_client.post = AsyncMock(return_value=mock_response) + mock_get_client.return_value = mock_client + + config = BedrockAgentCoreA2AConfig() + result = await config.handle_non_streaming( + request_id="req-001", + params=SAMPLE_PARAMS, + litellm_params=SAMPLE_LITELLM_PARAMS, + ) + + assert result["error"]["code"] == -32600 + assert result["error"]["message"] == "Bad request" + + +class TestConfigManager: + """Test that config manager routes 'bedrock' correctly.""" + + def test_bedrock_returns_config(self): + from litellm.a2a_protocol.providers.bedrock_agentcore.config import ( + BedrockAgentCoreA2AConfig, + ) + from litellm.a2a_protocol.providers.config_manager import ( + A2AProviderConfigManager, + ) + + config = A2AProviderConfigManager.get_provider_config( + "bedrock", model=SAMPLE_MODEL + ) + assert config is not None + assert isinstance(config, BedrockAgentCoreA2AConfig) + + def test_bedrock_non_agentcore_returns_none(self): + """Non-agentcore bedrock models should fall through to completion bridge.""" + from litellm.a2a_protocol.providers.config_manager import ( + A2AProviderConfigManager, + ) + + config = A2AProviderConfigManager.get_provider_config( + "bedrock", model="bedrock/anthropic.claude-3-sonnet" + ) + assert config is None + + def test_unknown_provider_returns_none(self): + from litellm.a2a_protocol.providers.config_manager import ( + A2AProviderConfigManager, + ) + + assert A2AProviderConfigManager.get_provider_config("unknown") is None + + +class TestHandlerIntegration: + """Test handler.py changes — litellm_params passed through, api_base not required.""" + + @pytest.mark.asyncio + async def test_provider_config_receives_litellm_params(self): + """Verify handler passes litellm_params to provider config via kwargs.""" + from litellm.a2a_protocol.litellm_completion_bridge.handler import ( + A2ACompletionBridgeHandler, + ) + + mock_config = AsyncMock() + mock_config.handle_non_streaming = AsyncMock( + return_value={"jsonrpc": "2.0", "id": "req-001", "result": {}} + ) + + with patch( + "litellm.a2a_protocol.litellm_completion_bridge.handler.A2AProviderConfigManager.get_provider_config", + return_value=mock_config, + ): + await A2ACompletionBridgeHandler.handle_non_streaming( + request_id="req-001", + params=SAMPLE_PARAMS, + litellm_params=SAMPLE_LITELLM_PARAMS, + api_base=None, + ) + + mock_config.handle_non_streaming.assert_called_once_with( + request_id="req-001", + params=SAMPLE_PARAMS, + api_base=None, + litellm_params=SAMPLE_LITELLM_PARAMS, + ) + + @pytest.mark.asyncio + async def test_api_base_none_allowed_with_provider_config(self): + """api_base=None no longer raises when a provider config is registered.""" + from litellm.a2a_protocol.litellm_completion_bridge.handler import ( + A2ACompletionBridgeHandler, + ) + + mock_config = AsyncMock() + mock_config.handle_non_streaming = AsyncMock( + return_value={"jsonrpc": "2.0", "id": "req-001", "result": {}} + ) + + with patch( + "litellm.a2a_protocol.litellm_completion_bridge.handler.A2AProviderConfigManager.get_provider_config", + return_value=mock_config, + ): + # Should NOT raise ValueError + result = await A2ACompletionBridgeHandler.handle_non_streaming( + request_id="req-001", + params=SAMPLE_PARAMS, + litellm_params=SAMPLE_LITELLM_PARAMS, + api_base=None, + ) + assert result is not None diff --git a/tests/test_litellm/integrations/websearch_interception/test_websearch_interception_handler.py b/tests/test_litellm/integrations/websearch_interception/test_websearch_interception_handler.py index b467822ac70..020c171a666 100644 --- a/tests/test_litellm/integrations/websearch_interception/test_websearch_interception_handler.py +++ b/tests/test_litellm/integrations/websearch_interception/test_websearch_interception_handler.py @@ -114,7 +114,7 @@ async def test_async_pre_call_deployment_hook_provider_from_top_level_kwargs(): # Simulate kwargs as they arrive from the router path: # custom_llm_provider is at the TOP LEVEL (not nested under litellm_params) kwargs = { - "model": "anthropic.claude-3-5-sonnet-20241022-v2:0", + "model": "anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "user", "content": "Search the web for LiteLLM"}], "tools": [ {"type": "web_search_20250305", "name": "web_search", "max_uses": 3}, @@ -222,7 +222,7 @@ async def test_async_pre_call_deployment_hook_nested_litellm_params_fallback(): logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"]) kwargs = { - "model": "anthropic.claude-3-5-sonnet-20241022-v2:0", + "model": "anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "user", "content": "test"}], "tools": [{"type": "web_search_20250305", "name": "web_search"}], "litellm_params": { @@ -238,7 +238,7 @@ async def test_async_pre_call_deployment_hook_nested_litellm_params_fallback(): for t in result["tools"] ) # Full kwargs preserved - assert result["model"] == "anthropic.claude-3-5-sonnet-20241022-v2:0" + assert result["model"] == "anthropic.claude-haiku-4-5-20251001-v1:0" @pytest.mark.asyncio diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index e5fb0ebdf6e..9d584446eb5 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -20,7 +20,7 @@ from litellm.types.utils import ModelResponse, TextCompletionResponse @pytest.fixture def logging_obj(): return LitellmLogging( - model="bedrock/claude-3-5-sonnet-20240620-v1:0", + model="bedrock/claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "Hey"}], stream=True, call_type="completion", diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py index 20e064ef8f4..aad3de306c7 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py @@ -475,10 +475,10 @@ async def test_streaming_handler_with_usage( response = CustomStreamWrapper( completion_stream=completion_stream, - model="bedrock/claude-3-5-sonnet-20240620-v1:0", + model="bedrock/claude-haiku-4-5-20251001-v1:0", custom_llm_provider="bedrock", logging_obj=Logging( - model="bedrock/claude-3-5-sonnet-20240620-v1:0", + model="bedrock/claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "Hey"}], stream=True, call_type="completion", @@ -748,7 +748,7 @@ async def test_streaming_completion_start_time(logging_obj: Logging): response = CustomStreamWrapper( completion_stream=completion_stream, - model="bedrock/claude-3-5-sonnet-20240620-v1:0", + model="bedrock/claude-haiku-4-5-20251001-v1:0", logging_obj=logging_obj, ) @@ -883,7 +883,7 @@ def test_streaming_handler_with_created_time_propagation( response = CustomStreamWrapper( completion_stream=completion_stream, - model="bedrock/claude-3-5-sonnet-20240620-v1:0", + model="bedrock/claude-haiku-4-5-20251001-v1:0", logging_obj=logging_obj, ) diff --git a/tests/test_litellm/llms/bedrock/chat/agentcore/test_agentcore_transformation.py b/tests/test_litellm/llms/bedrock/chat/agentcore/test_agentcore_transformation.py index 3dede83032a..43182926f95 100644 --- a/tests/test_litellm/llms/bedrock/chat/agentcore/test_agentcore_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/agentcore/test_agentcore_transformation.py @@ -160,6 +160,67 @@ class TestAgentCoreJsonResponseParsing: assert parsed["content"] == "" assert parsed["final_message"] == response_json["result"] + def test_parse_json_a2a_jsonrpc_nested_message(self, config): + """Strategy 0: A2A JSON-RPC with result.message.parts[] format.""" + response_json = { + "jsonrpc": "2.0", + "id": "test_id", + "result": { + "message": { + "role": "agent", + "parts": [{"kind": "text", "text": "1 + 1 = 2"}], + "messageId": "123", + } + }, + } + parsed = config._parse_json_response(response_json) + assert parsed["content"] == "1 + 1 = 2" + assert parsed["usage"] is None + + def test_parse_json_a2a_jsonrpc_direct_parts(self, config): + """Strategy 0: A2A JSON-RPC with result.parts[] format (direct message).""" + response_json = { + "jsonrpc": "2.0", + "id": "test_id", + "result": { + "kind": "message", + "parts": [{"kind": "text", "text": "Direct response"}], + }, + } + parsed = config._parse_json_response(response_json) + assert parsed["content"] == "Direct response" + assert parsed["usage"] is None + + def test_parse_json_a2a_jsonrpc_multi_parts(self, config): + """Strategy 0: A2A JSON-RPC with multiple text parts concatenated.""" + response_json = { + "jsonrpc": "2.0", + "id": "test_id", + "result": { + "message": { + "role": "agent", + "parts": [ + {"kind": "text", "text": "First part"}, + {"kind": "text", "text": "Second part"}, + ], + } + }, + } + parsed = config._parse_json_response(response_json) + assert parsed["content"] == "First part Second part" + assert parsed["usage"] is None + + def test_parse_json_a2a_jsonrpc_empty_falls_through(self, config): + """Strategy 0: A2A JSON-RPC with empty result falls through to Strategy 3.""" + response_json = { + "jsonrpc": "2.0", + "id": "test_id", + "result": "plain text fallback", + } + parsed = config._parse_json_response(response_json) + assert parsed["content"] == "plain text fallback" + assert parsed["usage"] is None + class TestAgentCoreNonStreamingJsonFormats: """Tests for _get_parsed_response with different JSON formats (non-streaming path).""" diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index 867a3e61bbd..7ff30b36309 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -225,7 +225,7 @@ def test_transform_tool_call_with_cache_control(): ] result = config.transform_request( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, optional_params={"tools": tools}, litellm_params={}, @@ -309,10 +309,10 @@ def test_transform_request_helper_includes_anthropic_beta_and_tools(): config = AmazonConverseConfig() system_content_blocks = [] optional_params = { - "anthropic_beta": ["computer-use-2024-10-22"], + "anthropic_beta": ["computer-use-2025-01-24"], "tools": [ { - "type": "computer_20241022", + "type": "computer_20250124", "name": "computer", "display_height_px": 768, "display_width_px": 1024, @@ -322,7 +322,7 @@ def test_transform_request_helper_includes_anthropic_beta_and_tools(): "some_other_param": 123, } data = config._transform_request_helper( - model="anthropic.claude-3-5-sonnet-20241022-v2:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", system_content_blocks=system_content_blocks, optional_params=optional_params, messages=None, @@ -330,11 +330,11 @@ def test_transform_request_helper_includes_anthropic_beta_and_tools(): assert "additionalModelRequestFields" in data fields = data["additionalModelRequestFields"] assert "anthropic_beta" in fields - assert fields["anthropic_beta"] == ["computer-use-2024-10-22"] + assert fields["anthropic_beta"] == ["computer-use-2025-01-24"] # Verify computer tool is included assert "tools" in fields assert len(fields["tools"]) == 1 - assert fields["tools"][0]["type"] == "computer_20241022" + assert fields["tools"][0]["type"] == "computer_20250124" def test_transform_response_with_computer_use_tool(): @@ -396,7 +396,7 @@ def test_transform_response_with_computer_use_tool(): optional_params = { "tools": [ { - "type": "computer_20241022", + "type": "computer_20250124", "function": { "name": "computer", "parameters": { @@ -410,7 +410,7 @@ def test_transform_response_with_computer_use_tool(): } # Call the transformation logic result = config._transform_response( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", response=MockResponse(), model_response=model_response, stream=False, @@ -497,7 +497,7 @@ def test_transform_response_with_bash_tool(): } # Call the transformation logic result = config._transform_response( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", response=MockResponse(), model_response=model_response, stream=False, @@ -610,7 +610,7 @@ def test_transform_response_with_structured_response_being_called(): } # Call the transformation logic result = config._transform_response( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", response=MockResponse(), model_response=model_response, stream=False, @@ -758,7 +758,7 @@ async def test_bedrock_bash_tool_acompletion(): try: response = await litellm.acompletion( - model="bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, tools=tools, # Using dummy API key - test should fail with auth error, proving request formatting works @@ -798,7 +798,7 @@ async def test_bedrock_computer_use_acompletion(): # Test with computer use tool tools = [ { - "type": "computer_20241022", + "type": "computer_20250124", "name": "computer", "display_height_px": 768, "display_width_px": 1024, @@ -823,7 +823,7 @@ async def test_bedrock_computer_use_acompletion(): try: response = await litellm.acompletion( - model="bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, tools=tools, # Using dummy API key - test should fail with auth error, proving request formatting works @@ -864,7 +864,7 @@ async def test_transformation_directly(): tools = [ { - "type": "computer_20241022", + "type": "computer_20250124", "name": "computer", "display_height_px": 768, "display_width_px": 1024, @@ -880,7 +880,7 @@ async def test_transformation_directly(): # Transform request request_data = config.transform_request( - model="anthropic.claude-3-5-sonnet-20241022-v2:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, optional_params={"tools": tools}, litellm_params={}, @@ -893,7 +893,7 @@ async def test_transformation_directly(): # Check that anthropic_beta is set correctly for computer use assert "anthropic_beta" in additional_fields - assert additional_fields["anthropic_beta"] == ["computer-use-2024-10-22"] + assert additional_fields["anthropic_beta"] == ["computer-use-2025-01-24"] # Check that tools are present assert "tools" in additional_fields @@ -901,7 +901,7 @@ async def test_transformation_directly(): # Verify tool types tool_types = [tool.get("type") for tool in additional_fields["tools"]] - assert "computer_20241022" in tool_types + assert "computer_20250124" in tool_types assert "bash_20241022" in tool_types @@ -910,7 +910,7 @@ def test_transform_request_helper_includes_anthropic_beta_and_tools_bash(): config = AmazonConverseConfig() system_content_blocks = [] optional_params = { - "anthropic_beta": ["computer-use-2024-10-22"], + "anthropic_beta": ["computer-use-2025-01-24"], "tools": [ { "type": "bash_20241022", @@ -920,7 +920,7 @@ def test_transform_request_helper_includes_anthropic_beta_and_tools_bash(): "some_other_param": 123, } data = config._transform_request_helper( - model="anthropic.claude-3-5-sonnet-20241022-v2:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", system_content_blocks=system_content_blocks, optional_params=optional_params, messages=None, @@ -928,7 +928,7 @@ def test_transform_request_helper_includes_anthropic_beta_and_tools_bash(): assert "additionalModelRequestFields" in data fields = data["additionalModelRequestFields"] assert "anthropic_beta" in fields - assert fields["anthropic_beta"] == ["computer-use-2024-10-22"] + assert fields["anthropic_beta"] == ["computer-use-2025-01-24"] # Verify bash tool is included assert "tools" in fields assert len(fields["tools"]) == 1 @@ -942,7 +942,7 @@ def test_transform_request_with_multiple_tools(): # Use the exact payload from the user's error tools = [ { - "type": "computer_20241022", + "type": "computer_20250124", "function": { "name": "computer", "parameters": { @@ -984,7 +984,7 @@ def test_transform_request_with_multiple_tools(): # Transform request request_data = config.transform_request( - model="anthropic.claude-3-5-sonnet-20241022-v2:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, optional_params={"tools": tools}, litellm_params={}, @@ -997,7 +997,7 @@ def test_transform_request_with_multiple_tools(): # Check that anthropic_beta is set correctly for computer use assert "anthropic_beta" in additional_fields - assert additional_fields["anthropic_beta"] == ["computer-use-2024-10-22"] + assert additional_fields["anthropic_beta"] == ["computer-use-2025-01-24"] # Check that tools are present assert "tools" in additional_fields @@ -1005,7 +1005,7 @@ def test_transform_request_with_multiple_tools(): # Verify tool types tool_types = [tool.get("type") for tool in additional_fields["tools"]] - assert "computer_20241022" in tool_types + assert "computer_20250124" in tool_types assert "bash_20241022" in tool_types assert "text_editor_20241022" in tool_types @@ -1019,7 +1019,7 @@ def test_transform_request_with_computer_tool_only(): tools = [ { - "type": "computer_20241022", + "type": "computer_20250124", "name": "computer", "display_height_px": 768, "display_width_px": 1024, @@ -1044,7 +1044,7 @@ def test_transform_request_with_computer_tool_only(): # Transform request request_data = config.transform_request( - model="anthropic.claude-3-5-sonnet-20241022-v2:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, optional_params={"tools": tools}, litellm_params={}, @@ -1057,12 +1057,12 @@ def test_transform_request_with_computer_tool_only(): # Check that anthropic_beta is set correctly for computer use assert "anthropic_beta" in additional_fields - assert additional_fields["anthropic_beta"] == ["computer-use-2024-10-22"] + assert additional_fields["anthropic_beta"] == ["computer-use-2025-01-24"] # Check that tools are present assert "tools" in additional_fields assert len(additional_fields["tools"]) == 1 - assert additional_fields["tools"][0]["type"] == "computer_20241022" + assert additional_fields["tools"][0]["type"] == "computer_20250124" def test_transform_request_with_bash_tool_only(): @@ -1080,7 +1080,7 @@ def test_transform_request_with_bash_tool_only(): # Transform request request_data = config.transform_request( - model="anthropic.claude-3-5-sonnet-20241022-v2:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, optional_params={"tools": tools}, litellm_params={}, @@ -1093,7 +1093,7 @@ def test_transform_request_with_bash_tool_only(): # Check that anthropic_beta is set correctly for computer use assert "anthropic_beta" in additional_fields - assert additional_fields["anthropic_beta"] == ["computer-use-2024-10-22"] + assert additional_fields["anthropic_beta"] == ["computer-use-2025-01-24"] # Check that tools are present assert "tools" in additional_fields @@ -1116,7 +1116,7 @@ def test_transform_request_with_text_editor_tool(): # Transform request request_data = config.transform_request( - model="anthropic.claude-3-5-sonnet-20241022-v2:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, optional_params={"tools": tools}, litellm_params={}, @@ -1129,7 +1129,7 @@ def test_transform_request_with_text_editor_tool(): # Check that anthropic_beta is set correctly for computer use assert "anthropic_beta" in additional_fields - assert additional_fields["anthropic_beta"] == ["computer-use-2024-10-22"] + assert additional_fields["anthropic_beta"] == ["computer-use-2025-01-24"] # Check that tools are present assert "tools" in additional_fields @@ -1166,7 +1166,7 @@ def test_transform_request_with_function_tool(): # Transform request request_data = config.transform_request( - model="anthropic.claude-3-5-sonnet-20241022-v2:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, optional_params={"tools": tools}, litellm_params={}, @@ -1262,17 +1262,17 @@ async def test_assistant_message_cache_control(): ] result = _bedrock_converse_messages_pt( - messages=messages, model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", llm_provider="bedrock_converse" + messages=messages, model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse" ) async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", llm_provider="bedrock_converse" + messages=messages, model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse" ) assert result == async_result async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", llm_provider="bedrock_converse" + messages=messages, model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse" ) assert result == async_result @@ -1307,11 +1307,11 @@ async def test_assistant_message_list_content_cache_control(): ] result = _bedrock_converse_messages_pt( - messages=messages, model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", llm_provider="bedrock_converse" + messages=messages, model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse" ) async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", llm_provider="bedrock_converse" + messages=messages, model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse" ) assert result == async_result @@ -1349,11 +1349,11 @@ async def test_tool_message_cache_control(): ] result = _bedrock_converse_messages_pt( - messages=messages, model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", llm_provider="bedrock_converse" + messages=messages, model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse" ) async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", llm_provider="bedrock_converse" + messages=messages, model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse" ) assert result == async_result @@ -1400,11 +1400,11 @@ async def test_tool_message_string_content_cache_control(): ] result = _bedrock_converse_messages_pt( - messages=messages, model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", llm_provider="bedrock_converse" + messages=messages, model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse" ) async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", llm_provider="bedrock_converse" + messages=messages, model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse" ) assert result == async_result @@ -1447,11 +1447,11 @@ async def test_assistant_tool_calls_cache_control(): ] result = _bedrock_converse_messages_pt( - messages=messages, model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", llm_provider="bedrock_converse" + messages=messages, model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse" ) async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", llm_provider="bedrock_converse" + messages=messages, model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse" ) assert result == async_result @@ -1501,11 +1501,11 @@ async def test_multiple_tool_calls_with_mixed_cache_control(): ] result = _bedrock_converse_messages_pt( - messages=messages, model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", llm_provider="bedrock_converse" + messages=messages, model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse" ) async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", llm_provider="bedrock_converse" + messages=messages, model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse" ) assert result == async_result @@ -1546,11 +1546,11 @@ async def test_no_cache_control_no_cache_point(): ] result = _bedrock_converse_messages_pt( - messages=messages, model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", llm_provider="bedrock_converse" + messages=messages, model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse" ) async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", llm_provider="bedrock_converse" + messages=messages, model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse" ) assert result == async_result @@ -2108,7 +2108,7 @@ def test_request_metadata_transformation(): # Transform request with requestMetadata request_data = config.transform_request( - model="anthropic.claude-3-5-sonnet-20240620-v1:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, optional_params={"requestMetadata": request_metadata}, litellm_params={}, @@ -2134,7 +2134,7 @@ def test_request_metadata_validation(): # Should not raise exception config.transform_request( - model="anthropic.claude-3-5-sonnet-20240620-v1:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, optional_params={"requestMetadata": valid_metadata}, litellm_params={}, @@ -2146,7 +2146,7 @@ def test_request_metadata_validation(): try: config.transform_request( - model="anthropic.claude-3-5-sonnet-20240620-v1:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, optional_params={"requestMetadata": too_many_items}, litellm_params={}, @@ -2169,7 +2169,7 @@ def test_request_metadata_key_constraints(): try: config.transform_request( - model="anthropic.claude-3-5-sonnet-20240620-v1:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, optional_params={"requestMetadata": invalid_metadata}, litellm_params={}, @@ -2184,7 +2184,7 @@ def test_request_metadata_key_constraints(): try: config.transform_request( - model="anthropic.claude-3-5-sonnet-20240620-v1:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, optional_params={"requestMetadata": invalid_metadata}, litellm_params={}, @@ -2207,7 +2207,7 @@ def test_request_metadata_value_constraints(): try: config.transform_request( - model="anthropic.claude-3-5-sonnet-20240620-v1:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, optional_params={"requestMetadata": invalid_metadata}, litellm_params={}, @@ -2222,7 +2222,7 @@ def test_request_metadata_value_constraints(): # Should not raise exception config.transform_request( - model="anthropic.claude-3-5-sonnet-20240620-v1:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, optional_params={"requestMetadata": valid_metadata}, litellm_params={}, @@ -2245,7 +2245,7 @@ def test_request_metadata_character_pattern(): # Should not raise exception config.transform_request( - model="anthropic.claude-3-5-sonnet-20240620-v1:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, optional_params={"requestMetadata": valid_metadata}, litellm_params={}, @@ -2280,7 +2280,7 @@ def test_request_metadata_with_other_params(): # Transform request with multiple parameters including request_metadata request_data = config.transform_request( - model="anthropic.claude-3-5-sonnet-20240620-v1:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, optional_params={"requestMetadata": request_metadata, "tools": tools, "max_tokens": 100, "temperature": 0.7}, litellm_params={}, @@ -2305,7 +2305,7 @@ def test_request_metadata_empty(): # Empty dict should be allowed request_data = config.transform_request( - model="anthropic.claude-3-5-sonnet-20240620-v1:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, optional_params={"requestMetadata": {}}, litellm_params={}, @@ -2324,7 +2324,7 @@ def test_request_metadata_not_provided(): # No requestMetadata provided request_data = config.transform_request( - model="anthropic.claude-3-5-sonnet-20240620-v1:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, optional_params={}, litellm_params={}, @@ -2358,7 +2358,7 @@ def test_empty_assistant_message_handling(): # This avoids issues with module reloading during parallel test execution with patch.object(factory_module.litellm, "modify_params", True): result = _bedrock_converse_messages_pt( - messages=messages, model="anthropic.claude-3-5-sonnet-20240620-v1:0", llm_provider="bedrock_converse" + messages=messages, model="anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse" ) # Should have 3 messages: user, assistant (with placeholder), user @@ -2380,7 +2380,7 @@ def test_empty_assistant_message_handling(): ] result = _bedrock_converse_messages_pt( - messages=messages, model="anthropic.claude-3-5-sonnet-20240620-v1:0", llm_provider="bedrock_converse" + messages=messages, model="anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse" ) # Assistant message should have placeholder text instead of whitespace @@ -2395,7 +2395,7 @@ def test_empty_assistant_message_handling(): ] result = _bedrock_converse_messages_pt( - messages=messages, model="anthropic.claude-3-5-sonnet-20240620-v1:0", llm_provider="bedrock_converse" + messages=messages, model="anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse" ) # Assistant message should have placeholder text instead of empty text @@ -2410,7 +2410,7 @@ def test_empty_assistant_message_handling(): ] result = _bedrock_converse_messages_pt( - messages=messages, model="anthropic.claude-3-5-sonnet-20240620-v1:0", llm_provider="bedrock_converse" + messages=messages, model="anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse" ) # Assistant message should keep original content @@ -2446,7 +2446,7 @@ def test_is_nova_2_model(): assert config._is_nova_2_model("eu.amazon.nova-pro-v1:0") is False # Test with completely different models (should return False) - assert config._is_nova_2_model("anthropic.claude-3-5-sonnet-20240620-v1:0") is False + assert config._is_nova_2_model("anthropic.claude-haiku-4-5-20251001-v1:0") is False assert config._is_nova_2_model("meta.llama3-70b-instruct-v1:0") is False assert config._is_nova_2_model("mistral.mistral-7b-instruct-v0:2") is False @@ -2671,7 +2671,6 @@ def test_supports_native_structured_outputs(): assert config._supports_native_structured_outputs("deepseek.v3-v1:0") # Unsupported models -- should fall back to tool-call approach - assert not config._supports_native_structured_outputs("anthropic.claude-3-5-sonnet-20241022-v2:0") assert not config._supports_native_structured_outputs("anthropic.claude-sonnet-4-20250514-v1:0") assert not config._supports_native_structured_outputs("meta.llama3-3-70b-instruct-v1:0") assert not config._supports_native_structured_outputs("amazon.nova-pro-v1:0") @@ -2799,13 +2798,13 @@ def test_translate_response_format_fallback_tool_call(): optional_params: dict = {} result = config._translate_response_format_param( value=response_format, - model="anthropic.claude-3-5-sonnet-20241022-v2:0", + model="anthropic.claude-3-haiku-20240307-v1:0", optional_params=optional_params, non_default_params={"response_format": response_format}, is_thinking_enabled=False, ) - # Should use tool-call approach, NOT outputConfig + # Should use tool-call approach, NOT outputConfig (model doesn't support native structured outputs) assert "outputConfig" not in result assert "tools" in result assert result["json_mode"] is True @@ -3191,7 +3190,7 @@ def test_parallel_tool_calls_newer_model_adds_disable_flag(): def test_parallel_tool_calls_older_model_drops_disable_flag(): """Older Claude models (pre-4.5) must NOT receive disable_parallel_tool_use — Bedrock rejects it.""" config = AmazonConverseConfig() - model = "anthropic.claude-3-5-sonnet-20241022-v2:0" + model = "anthropic.claude-3-haiku-20240307-v1:0" messages = [{"role": "user", "content": "What's the weather in SF and NYC?"}] optional_params = config.map_openai_params( @@ -3319,7 +3318,7 @@ def test_transform_response_with_both_json_tool_call_and_real_tool(): optional_params = {"json_mode": True} result = config._transform_response( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", response=MockResponse(), model_response=model_response, stream=False, @@ -3397,7 +3396,7 @@ def test_transform_response_does_not_mutate_optional_params(): optional_params = {"json_mode": True, "other_key": "value"} config._transform_response( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", response=MockResponse(), model_response=model_response, stream=False, diff --git a/tests/test_litellm/llms/bedrock/files/expected_bedrock_batch_completions.jsonl b/tests/test_litellm/llms/bedrock/files/expected_bedrock_batch_completions.jsonl index 8bb35ba95d7..c58963bb1de 100644 --- a/tests/test_litellm/llms/bedrock/files/expected_bedrock_batch_completions.jsonl +++ b/tests/test_litellm/llms/bedrock/files/expected_bedrock_batch_completions.jsonl @@ -1,2 +1,2 @@ -{"recordId": "request-1", "modelInput": {"messages": [{"role": "user", "content": [{"type": "text", "text": "Hello world!"}]}], "max_tokens": 10, "system": [{"type": "text", "text": "You are a helpful assistant."}], "anthropic_version": "bedrock-2023-05-31", "anthropic_beta": []}} -{"recordId": "request-2", "modelInput": {"messages": [{"role": "user", "content": [{"type": "text", "text": "Hello world!"}]}], "max_tokens": 10, "system": [{"type": "text", "text": "You are an unhelpful assistant."}], "anthropic_version": "bedrock-2023-05-31", "anthropic_beta": []}} +{"recordId": "request-1", "modelInput": {"messages": [{"role": "user", "content": [{"type": "text", "text": "Hello world!"}]}], "max_tokens": 10, "system": [{"type": "text", "text": "You are a helpful assistant."}], "anthropic_version": "bedrock-2023-05-31"}} +{"recordId": "request-2", "modelInput": {"messages": [{"role": "user", "content": [{"type": "text", "text": "Hello world!"}]}], "max_tokens": 10, "system": [{"type": "text", "text": "You are an unhelpful assistant."}], "anthropic_version": "bedrock-2023-05-31"}} diff --git a/tests/test_litellm/llms/bedrock/image_edit/test_amazon_nova_canvas_image_edit.py b/tests/test_litellm/llms/bedrock/image_edit/test_amazon_nova_canvas_image_edit.py new file mode 100644 index 00000000000..020b8df1276 --- /dev/null +++ b/tests/test_litellm/llms/bedrock/image_edit/test_amazon_nova_canvas_image_edit.py @@ -0,0 +1,657 @@ +"""Unit tests for Bedrock Amazon Nova Canvas image edit (issue #24267).""" + +import base64 +import io +from typing import cast + +import httpx +import pytest + +import litellm +from litellm.llms.bedrock.image_edit.amazon_nova_canvas_image_edit_transformation import ( + BedrockAmazonNovaCanvasImageEditConfig, + get_bedrock_image_edit_config_for_model, +) +from litellm.llms.bedrock.image_edit.handler import BedrockImageEdit +from litellm.llms.bedrock.image_edit.stability_transformation import ( + BedrockStabilityImageEditConfig, +) +from litellm.types.images.main import ImageEditOptionalRequestParams + + +@pytest.fixture(autouse=True) +def ensure_nova_canvas_image_edit_model_cost_flags(monkeypatch): + """Routing uses ``supports_nova_canvas_image_edit`` on ``litellm.model_cost``. + + Full ``model_prices_and_context_window.json`` includes these flags, but CI or + alternate cost maps may omit them—merge minimal entries so tests match production. + """ + from litellm.utils import _invalidate_model_cost_lowercase_map + + for key in ( + "amazon.nova-canvas-v1:0", + "us.amazon.nova-canvas-v1:0", + ): + entry = litellm.model_cost.get(key) or {} + if entry.get("supports_nova_canvas_image_edit") is True: + continue + monkeypatch.setitem( + litellm.model_cost, + key, + { + **entry, + "litellm_provider": entry.get("litellm_provider", "bedrock"), + "mode": entry.get("mode", "image_generation"), + "supports_nova_canvas_image_edit": True, + }, + ) + _invalidate_model_cost_lowercase_map() + + yield + + _invalidate_model_cost_lowercase_map() + + +def test_get_config_class_nova_canvas(): + """Nova Canvas model resolves to BedrockAmazonNovaCanvasImageEditConfig.""" + cls = BedrockImageEdit.get_config_class("amazon.nova-canvas-v1:0") + assert cls is BedrockAmazonNovaCanvasImageEditConfig + + +def test_get_config_class_us_cross_region_nova_canvas(): + """Cross-region inference id us.amazon.nova-canvas-v1:0 maps via model_prices.""" + cls = BedrockImageEdit.get_config_class("us.amazon.nova-canvas-v1:0") + assert cls is BedrockAmazonNovaCanvasImageEditConfig + + +def test_get_config_class_stability_unchanged(): + """Stability edit models still use stability config.""" + cls = BedrockImageEdit.get_config_class( + "stability.stable-image-inpaint-v1:0", + ) + assert cls is BedrockStabilityImageEditConfig + + +def test_provider_config_router_returns_nova_for_canvas(): + """ProviderConfigManager routes Nova Canvas to Nova image-edit config.""" + cfg = get_bedrock_image_edit_config_for_model("amazon.nova-canvas-v1:0") + assert isinstance(cfg, BedrockAmazonNovaCanvasImageEditConfig) + + +def test_provider_config_router_returns_stability_for_sd(): + """Non-Nova Bedrock image edit still uses Stability config.""" + cfg = get_bedrock_image_edit_config_for_model( + "stability.stable-image-inpaint-v1:0", + ) + assert isinstance(cfg, BedrockStabilityImageEditConfig) + + +def test_get_bedrock_helper_matches_handler_get_config_class(): + """Handler and get_bedrock_image_edit_config_for_model must agree.""" + for model in ( + "amazon.nova-canvas-v1:0", + "stability.stable-image-inpaint-v1:0", + ): + handler_cls = BedrockImageEdit.get_config_class(model) + helper_cfg = get_bedrock_image_edit_config_for_model(model) + assert isinstance(helper_cfg, handler_cls) + + +def test_get_config_class_unknown_bedrock_image_model_raises(): + with pytest.raises(ValueError, match="Unsupported Bedrock image-edit model"): + BedrockImageEdit.get_config_class("amazon.titan-image-generator-v1") + + +def test_get_bedrock_image_edit_config_unknown_raises(): + with pytest.raises(ValueError, match="Unsupported Bedrock image-edit model"): + get_bedrock_image_edit_config_for_model("amazon.titan-image-generator-v1") + + +def test_provider_config_manager_bedrock_nova_canvas(): + """ProviderConfigManager.get_provider_image_edit_config matches handler routing.""" + from litellm.utils import ProviderConfigManager + + cfg = ProviderConfigManager.get_provider_image_edit_config( + "amazon.nova-canvas-v1:0", + litellm.LlmProviders.BEDROCK, + ) + assert isinstance(cfg, BedrockAmazonNovaCanvasImageEditConfig) + + +def test_provider_config_manager_bedrock_stability_inpaint(): + """ProviderConfigManager returns Stability config for Stability edit models.""" + from litellm.utils import ProviderConfigManager + + cfg = ProviderConfigManager.get_provider_image_edit_config( + "stability.stable-image-inpaint-v1:0", + litellm.LlmProviders.BEDROCK, + ) + assert isinstance(cfg, BedrockStabilityImageEditConfig) + + +def test_provider_config_manager_bedrock_unknown_raises(): + from litellm.utils import ProviderConfigManager + + with pytest.raises(ValueError, match="Unsupported Bedrock image-edit model"): + ProviderConfigManager.get_provider_image_edit_config( + "amazon.titan-image-generator-v1", + litellm.LlmProviders.BEDROCK, + ) + + +def test_provider_config_manager_bedrock_dispatches_to_nova_transform_outpainting(): + """ + Full dispatch: utils.ProviderConfigManager -> get_bedrock_image_edit_config_for_model + -> Nova config.transform_image_edit_request (not only direct helper calls). + """ + from litellm.utils import ProviderConfigManager + + cfg = ProviderConfigManager.get_provider_image_edit_config( + "amazon.nova-canvas-v1:0", + litellm.LlmProviders.BEDROCK, + ) + assert cfg is not None + img = io.BytesIO(b"scene") + mask = io.BytesIO(b"mask-bytes") + body, _ = cfg.transform_image_edit_request( + model="amazon.nova-canvas-v1:0", + prompt="expand left", + image=img, + image_edit_optional_request_params={ + "taskType": "OUTPAINTING", + "mask": mask, + }, + litellm_params={}, # type: ignore[arg-type] + headers={}, + ) + assert body["taskType"] == "OUTPAINTING" + assert "maskImage" in body["outPaintingParams"] + + +def test_transform_request_image_variation_without_mask(): + """No mask -> IMAGE_VARIATION with images + text.""" + config = BedrockAmazonNovaCanvasImageEditConfig() + img = io.BytesIO(b"fake-png") + body, _ = config.transform_image_edit_request( + model="amazon.nova-canvas-v1:0", + prompt="make it warmer", + image=img, + image_edit_optional_request_params={}, + litellm_params={}, # type: ignore[arg-type] + headers={}, + ) + assert body["taskType"] == "IMAGE_VARIATION" + assert body["imageVariationParams"]["text"] == "make it warmer" + assert len(body["imageVariationParams"]["images"]) == 1 + + +def test_transform_request_image_pathlike_input(tmp_path): + """PathLike image input should be read and base64-encoded.""" + config = BedrockAmazonNovaCanvasImageEditConfig() + image_path = tmp_path / "img.bin" + image_bytes = b"pathlike-image-bytes" + image_path.write_bytes(image_bytes) + + body, _ = config.transform_image_edit_request( + model="amazon.nova-canvas-v1:0", + prompt="pathlike", + image=image_path, + image_edit_optional_request_params={}, + litellm_params={}, # type: ignore[arg-type] + headers={}, + ) + + assert body["taskType"] == "IMAGE_VARIATION" + assert body["imageVariationParams"]["images"][0] == base64.b64encode( + image_bytes + ).decode("utf-8") + + +def test_transform_request_inpainting_with_mask(): + """Mask present -> INPAINTING with inPaintingParams (AWS field names).""" + config = BedrockAmazonNovaCanvasImageEditConfig() + main = io.BytesIO(b"img-bytes") + mask = io.BytesIO(b"mask-bytes") + body, _ = config.transform_image_edit_request( + model="amazon.nova-canvas-v1:0", + prompt="add a hat", + image=main, + image_edit_optional_request_params={"mask": mask}, + litellm_params={}, # type: ignore[arg-type] + headers={}, + ) + assert body["taskType"] == "INPAINTING" + ip = body["inPaintingParams"] + assert ip["text"] == "add a hat" + assert "maskImage" in ip + assert ip["image"] # base64 + + +def test_transform_request_explicit_image_variation_with_mask_honors_task_type(): + """Explicit taskType=IMAGE_VARIATION must not be overridden by mask presence.""" + config = BedrockAmazonNovaCanvasImageEditConfig() + main = io.BytesIO(b"img-bytes") + mask = io.BytesIO(b"mask-bytes") + body, _ = config.transform_image_edit_request( + model="amazon.nova-canvas-v1:0", + prompt="vary style", + image=main, + image_edit_optional_request_params={ + "taskType": "IMAGE_VARIATION", + "mask": mask, + }, + litellm_params={}, # type: ignore[arg-type] + headers={}, + ) + assert body["taskType"] == "IMAGE_VARIATION" + assert "imageVariationParams" in body + assert body["imageVariationParams"]["text"] == "vary style" + assert len(body["imageVariationParams"]["images"]) == 1 + + +def test_transform_request_outpainting_with_mask(): + """OUTPAINTING with OpenAI mask -> outPaintingParams.maskImage.""" + config = BedrockAmazonNovaCanvasImageEditConfig() + main = io.BytesIO(b"img") + mask = io.BytesIO(b"mask") + body, _ = config.transform_image_edit_request( + model="amazon.nova-canvas-v1:0", + prompt="extend the sky", + image=main, + image_edit_optional_request_params={ + "taskType": "OUTPAINTING", + "mask": mask, + }, + litellm_params={}, # type: ignore[arg-type] + headers={}, + ) + assert body["taskType"] == "OUTPAINTING" + assert "maskImage" in body["outPaintingParams"] + assert body["outPaintingParams"]["text"] == "extend the sky" + + +def test_transform_request_outpainting_with_mask_prompt(): + """OUTPAINTING with maskPrompt only (no binary mask).""" + config = BedrockAmazonNovaCanvasImageEditConfig() + img = io.BytesIO(b"img") + body, _ = config.transform_image_edit_request( + model="amazon.nova-canvas-v1:0", + prompt="new background", + image=img, + image_edit_optional_request_params={ + "taskType": "OUTPAINTING", + "maskPrompt": "the area behind the subject", + }, + litellm_params={}, # type: ignore[arg-type] + headers={}, + ) + assert body["taskType"] == "OUTPAINTING" + assert body["outPaintingParams"]["maskPrompt"] == "the area behind the subject" + + +def test_transform_request_outpainting_prefers_mask_prompt_over_binary_mask(): + """OUTPAINTING chooses maskPrompt over maskImage when both are set (_nova_canvas_task_body).""" + config = BedrockAmazonNovaCanvasImageEditConfig() + main = io.BytesIO(b"img") + mask = io.BytesIO(b"mask") + body, _ = config.transform_image_edit_request( + model="amazon.nova-canvas-v1:0", + prompt="extend scene", + image=main, + image_edit_optional_request_params={ + "taskType": "OUTPAINTING", + "mask": mask, + "maskPrompt": "sky region", + }, + litellm_params={}, # type: ignore[arg-type] + headers={}, + ) + assert body["taskType"] == "OUTPAINTING" + op = body["outPaintingParams"] + assert op["maskPrompt"] == "sky region" + assert "maskImage" not in op + + +def test_transform_request_outpainting_with_out_painting_mode(): + """OUTPAINTING forwards outPaintingMode into outPaintingParams.""" + config = BedrockAmazonNovaCanvasImageEditConfig() + img = io.BytesIO(b"img") + mask = io.BytesIO(b"m") + body, _ = config.transform_image_edit_request( + model="amazon.nova-canvas-v1:0", + prompt="widen", + image=img, + image_edit_optional_request_params={ + "taskType": "OUTPAINTING", + "mask": mask, + "outPaintingMode": "PRECISE", + }, + litellm_params={}, # type: ignore[arg-type] + headers={}, + ) + assert body["taskType"] == "OUTPAINTING" + assert body["outPaintingParams"]["outPaintingMode"] == "PRECISE" + + +def test_get_supported_openai_params_includes_outpainting_fields(): + """Documented OUTPAINTING-related optional params are advertised for routing/UI.""" + config = BedrockAmazonNovaCanvasImageEditConfig() + supported = config.get_supported_openai_params("amazon.nova-canvas-v1:0") + assert "taskType" in supported + assert "maskPrompt" in supported + assert "outPaintingMode" in supported + assert "mask" in supported + + +def test_transform_request_outpainting_without_mask_raises(): + """OUTPAINTING without maskPrompt or maskImage must fail fast with a clear error.""" + config = BedrockAmazonNovaCanvasImageEditConfig() + img = io.BytesIO(b"img") + with pytest.raises( + ValueError, + match="OUTPAINTING requires either a mask image or a mask prompt", + ): + config.transform_image_edit_request( + model="amazon.nova-canvas-v1:0", + prompt="extend", + image=img, + image_edit_optional_request_params={"taskType": "OUTPAINTING"}, + litellm_params={}, # type: ignore[arg-type] + headers={}, + ) + + +def test_transform_request_inpainting_explicit_task_without_mask_raises(): + """INPAINTING taskType without mask or maskPrompt must fail fast.""" + config = BedrockAmazonNovaCanvasImageEditConfig() + img = io.BytesIO(b"img") + with pytest.raises( + ValueError, match="INPAINTING requires either maskPrompt or maskImage" + ): + config.transform_image_edit_request( + model="amazon.nova-canvas-v1:0", + prompt="fix it", + image=img, + image_edit_optional_request_params={"taskType": "INPAINTING"}, + litellm_params={}, # type: ignore[arg-type] + headers={}, + ) + + +def test_transform_request_unknown_task_type_raises(): + """Unknown taskType must not silently map to IMAGE_VARIATION or INPAINTING.""" + config = BedrockAmazonNovaCanvasImageEditConfig() + img = io.BytesIO(b"img") + with pytest.raises(ValueError, match="Unsupported Amazon Nova Canvas taskType"): + config.transform_image_edit_request( + model="amazon.nova-canvas-v1:0", + prompt="x", + image=img, + image_edit_optional_request_params={"taskType": "TEXT_IMAGE"}, + litellm_params={}, # type: ignore[arg-type] + headers={}, + ) + + +def test_transform_request_background_removal(): + """taskType BACKGROUND_REMOVAL builds minimal body.""" + config = BedrockAmazonNovaCanvasImageEditConfig() + img = io.BytesIO(b"x") + body, _ = config.transform_image_edit_request( + model="amazon.nova-canvas-v1:0", + prompt="ignored", + image=img, + image_edit_optional_request_params={"taskType": "BACKGROUND_REMOVAL"}, + litellm_params={}, # type: ignore[arg-type] + headers={}, + ) + assert body["taskType"] == "BACKGROUND_REMOVAL" + assert "image" in body["backgroundRemovalParams"] + + +def test_transform_request_background_removal_omits_image_generation_config(): + """AWS Nova Canvas does not allow imageGenerationConfig on BACKGROUND_REMOVAL.""" + config = BedrockAmazonNovaCanvasImageEditConfig() + img = io.BytesIO(b"x") + body, _ = config.transform_image_edit_request( + model="amazon.nova-canvas-v1:0", + prompt="ignored", + image=img, + image_edit_optional_request_params={ + "taskType": "BACKGROUND_REMOVAL", + "size": "512x512", + "seed": 42, + "quality": "standard", + "cfgScale": 7.5, + "n": 2, + }, + litellm_params={}, # type: ignore[arg-type] + headers={}, + ) + assert body["taskType"] == "BACKGROUND_REMOVAL" + assert "imageGenerationConfig" not in body + + +def test_transform_request_image_variation_includes_image_generation_config(): + """Non-BACKGROUND_REMOVAL tasks may include imageGenerationConfig when params are set.""" + config = BedrockAmazonNovaCanvasImageEditConfig() + img = io.BytesIO(b"x") + body, _ = config.transform_image_edit_request( + model="amazon.nova-canvas-v1:0", + prompt="warm", + image=img, + image_edit_optional_request_params={"size": "1024x1024", "seed": 1}, + litellm_params={}, # type: ignore[arg-type] + headers={}, + ) + assert body["taskType"] == "IMAGE_VARIATION" + assert "imageGenerationConfig" in body + assert body["imageGenerationConfig"]["width"] == 1024 + assert body["imageGenerationConfig"]["height"] == 1024 + assert body["imageGenerationConfig"]["seed"] == 1 + + +def test_map_openai_params_unknown_quality_not_silently_dropped(): + """Non-Nova quality strings (e.g. OpenAI 'auto') must remain for downstream handling.""" + config = BedrockAmazonNovaCanvasImageEditConfig() + mapped = config.map_openai_params( + cast(ImageEditOptionalRequestParams, {"quality": "auto"}), + model="amazon.nova-canvas-v1:0", + drop_params=False, + ) + assert mapped.get("quality") == "auto" + + +def test_transform_request_unknown_quality_reaches_image_generation_config(): + """Unknown quality after map_openai_params is forwarded so callers are not silently ignored.""" + config = BedrockAmazonNovaCanvasImageEditConfig() + img = io.BytesIO(b"x") + op = config.map_openai_params( + cast(ImageEditOptionalRequestParams, {"quality": "auto"}), + model="amazon.nova-canvas-v1:0", + drop_params=False, + ) + body, _ = config.transform_image_edit_request( + model="amazon.nova-canvas-v1:0", + prompt="x", + image=img, + image_edit_optional_request_params=op, + litellm_params={}, # type: ignore[arg-type] + headers={}, + ) + assert body["imageGenerationConfig"]["quality"] == "auto" + + +def test_is_nova_canvas_image_edit_model_uses_model_cost_flag(monkeypatch): + """Routing uses supports_nova_canvas_image_edit in model_cost, not a hardcoded name substring.""" + fake_id = "amazon.custom-bedrock-image-edit-v99:0" + monkeypatch.setitem( + litellm.model_cost, + fake_id, + { + "litellm_provider": "bedrock", + "mode": "image_generation", + "supports_nova_canvas_image_edit": True, + }, + ) + assert ( + BedrockAmazonNovaCanvasImageEditConfig._is_nova_canvas_image_edit_model(fake_id) + is True + ) + + monkeypatch.setitem( + litellm.model_cost, + "amazon.not-nova-canvas-v1:0", + { + "litellm_provider": "bedrock", + "mode": "image_generation", + }, + ) + assert ( + BedrockAmazonNovaCanvasImageEditConfig._is_nova_canvas_image_edit_model( + "amazon.not-nova-canvas-v1:0" + ) + is False + ) + + # Name-shaped ids do not route without supports_nova_canvas_image_edit (no substring heuristic). + monkeypatch.setitem( + litellm.model_cost, + "amazon.nova-canvas-v2:0", + { + "litellm_provider": "bedrock", + "mode": "image_generation", + }, + ) + assert ( + BedrockAmazonNovaCanvasImageEditConfig._is_nova_canvas_image_edit_model( + "amazon.nova-canvas-v2:0" + ) + is False + ) + + +def test_transform_response_to_openai_format(): + """Response maps images[] to ImageResponse.data b64_json.""" + config = BedrockAmazonNovaCanvasImageEditConfig() + resp = httpx.Response( + 200, + json={"images": ["YmFzZTY0X2E=", "YmFzZTY0X2I="]}, + ) + model_response = config.transform_image_edit_response( + model="amazon.nova-canvas-v1:0", + raw_response=resp, + logging_obj=None, # type: ignore[arg-type] + ) + assert model_response.data is not None + assert len(model_response.data) == 2 + assert model_response.data[0].b64_json == "YmFzZTY0X2E=" + + +def test_transform_response_non_200_raises(): + """HTTP 4xx/5xx with JSON body surfaces a structured error.""" + config = BedrockAmazonNovaCanvasImageEditConfig() + resp = httpx.Response( + 400, + json={"message": "ValidationException: invalid input"}, + ) + with pytest.raises(Exception, match="Nova Canvas image edit error"): + config.transform_image_edit_response( + model="amazon.nova-canvas-v1:0", + raw_response=resp, + logging_obj=None, # type: ignore[arg-type] + ) + + +def test_transform_response_errors_field_raises(): + """Align with Bedrock Stability: top-level ``errors`` in body.""" + config = BedrockAmazonNovaCanvasImageEditConfig() + resp = httpx.Response( + 200, + json={"errors": ["upstream failure"]}, + ) + with pytest.raises(Exception, match="Nova Canvas image edit error"): + config.transform_image_edit_response( + model="amazon.nova-canvas-v1:0", + raw_response=resp, + logging_obj=None, # type: ignore[arg-type] + ) + + +def test_transform_response_allows_errors_field_with_images(): + """Do not treat ``errors`` as fatal when ``images`` is present.""" + config = BedrockAmazonNovaCanvasImageEditConfig() + resp = httpx.Response( + 200, + json={"errors": ["non-fatal warning"], "images": ["YmFzZTY0X2E="]}, + ) + model_response = config.transform_image_edit_response( + model="amazon.nova-canvas-v1:0", + raw_response=resp, + logging_obj=None, # type: ignore[arg-type] + ) + assert model_response.data is not None + assert len(model_response.data) == 1 + assert model_response.data[0].b64_json == "YmFzZTY0X2E=" + + +def test_transform_response_message_or_error_field_raises(): + """API-level error payload when there are no images (status 200).""" + config = BedrockAmazonNovaCanvasImageEditConfig() + resp = httpx.Response( + 200, + json={"message": "ValidationException: task rejected"}, + ) + with pytest.raises(Exception, match="Nova Canvas image edit error"): + config.transform_image_edit_response( + model="amazon.nova-canvas-v1:0", + raw_response=resp, + logging_obj=None, # type: ignore[arg-type] + ) + + +def test_transform_response_allows_informational_message_with_images(): + """Do not treat ``message`` as fatal when ``images`` is present (SDK wrappers).""" + config = BedrockAmazonNovaCanvasImageEditConfig() + resp = httpx.Response( + 200, + json={ + "message": "ok", + "images": ["YmFzZTY0X2E="], + }, + ) + model_response = config.transform_image_edit_response( + model="amazon.nova-canvas-v1:0", + raw_response=resp, + logging_obj=None, # type: ignore[arg-type] + ) + assert model_response.data is not None + assert len(model_response.data) == 1 + assert model_response.data[0].b64_json == "YmFzZTY0X2E=" + + +def test_transform_response_content_filtered_via_error_field(): + """AWS Nova Canvas signals failures (e.g. content filter) via top-level ``error``, not ``finish_reasons``.""" + config = BedrockAmazonNovaCanvasImageEditConfig() + resp = httpx.Response( + 200, + json={"images": [], "error": "CONTENT_FILTERED"}, + ) + with pytest.raises(Exception, match="Nova Canvas image edit error"): + config.transform_image_edit_response( + model="amazon.nova-canvas-v1:0", + raw_response=resp, + logging_obj=None, # type: ignore[arg-type] + ) + + +def test_transform_response_empty_images_without_error_raises(): + """200 with empty ``images`` and no error fields must not return silent empty ImageResponse.""" + config = BedrockAmazonNovaCanvasImageEditConfig() + resp = httpx.Response(200, json={"images": []}) + with pytest.raises(Exception, match="returned no images"): + config.transform_image_edit_response( + model="amazon.nova-canvas-v1:0", + raw_response=resp, + logging_obj=None, # type: ignore[arg-type] + ) diff --git a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py index f69f478278f..ea208007cd6 100644 --- a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py +++ b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py @@ -34,7 +34,9 @@ async def test_bedrock_sse_wrapper_encodes_dict_chunks(): _dummy_stream(), litellm_logging_obj=LiteLLMLoggingObj( model="bedrock/invoke/anthropic.claude-3-sonnet-20240229-v1:0", - messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}], + messages=[ + {"role": "user", "content": "Hello, can you tell me a short joke?"} + ], stream=True, call_type="chat", start_time=datetime.now(), @@ -58,6 +60,74 @@ async def test_bedrock_sse_wrapper_encodes_dict_chunks(): assert collected[1] == b"raw-bytes" +@pytest.mark.asyncio +async def test_bedrock_sse_wrapper_keeps_usage_in_message_start_and_message_delta(): + """Regression test: usage should be available on both message_start and message_delta SSE events.""" + + cfg = AmazonAnthropicClaudeMessagesConfig() + + async def _dummy_stream(): # type: ignore[return-type] + yield { + "type": "message_start", + "message": { + "id": "msg_123", + "type": "message", + "role": "assistant", + "content": [], + "usage": { + "input_tokens": 3, + "output_tokens": 1, + }, + }, + } + yield { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": { + "input_tokens": 3, + "output_tokens": 8, + }, + } + yield { + "type": "message_stop", + "usage": { + "input_tokens": 3, + "cache_creation_input_tokens": 1562, + "cache_read_input_tokens": 32392, + }, + } + + collected: list[bytes] = [] + async for chunk in cfg.bedrock_sse_wrapper( + _dummy_stream(), + litellm_logging_obj=LiteLLMLoggingObj( + model="bedrock/invoke/anthropic.claude-3-sonnet-20240229-v1:0", + messages=[{"role": "user", "content": "Hello"}], + stream=True, + call_type="chat", + start_time=datetime.now(), + litellm_call_id="test_bedrock_sse_wrapper_keeps_usage_in_both_events", + function_id="test_bedrock_sse_wrapper_keeps_usage_in_both_events", + ), + request_body={}, + ): + collected.append(chunk) + + start_chunk = next(c for c in collected if b"event: message_start\n" in c) + delta_chunk = next(c for c in collected if b"event: message_delta\n" in c) + + start_json = json.loads(start_chunk.decode("utf-8").split("data: ", 1)[1].strip()) + delta_json = json.loads(delta_chunk.decode("utf-8").split("data: ", 1)[1].strip()) + + assert "usage" in start_json["message"] + assert start_json["message"]["usage"]["input_tokens"] == 3 + + assert "usage" in delta_json + assert delta_json["usage"]["cache_creation_input_tokens"] == 1562 + assert delta_json["usage"]["cache_read_input_tokens"] == 32392 + assert delta_json["usage"]["input_tokens"] == 3 + 1562 + 32392 + + def test_chunk_parser_usage_transformation(): """Ensure Bedrock invocation metrics are transformed to Anthropic usage keys.""" @@ -96,12 +166,9 @@ def test_remove_ttl_from_cache_control(): { "type": "text", "text": "Hello", - "cache_control": { - "type": "ephemeral", - "ttl": "1h" - } + "cache_control": {"type": "ephemeral", "ttl": "1h"}, } - ] + ], } ] } @@ -122,20 +189,14 @@ def test_remove_ttl_from_cache_control(): { "type": "text", "text": "Hello", - "cache_control": { - "type": "ephemeral", - "ttl": "1h" - } + "cache_control": {"type": "ephemeral", "ttl": "1h"}, }, { "type": "text", "text": "World", - "cache_control": { - "type": "ephemeral", - "ttl": "2h" - } - } - ] + "cache_control": {"type": "ephemeral", "ttl": "2h"}, + }, + ], } ] } @@ -156,11 +217,9 @@ def test_remove_ttl_from_cache_control(): { "type": "text", "text": "Hello", - "cache_control": { - "type": "ephemeral" - } + "cache_control": {"type": "ephemeral"}, } - ] + ], } ] } @@ -232,6 +291,7 @@ def test_remove_custom_field_from_tools(): remove_custom_field_from_tools(request4) assert request4["tools"] is None + def test_remove_scope_from_cache_control(): """Ensure scope field is removed from cache_control for Bedrock (not supported).""" @@ -303,9 +363,9 @@ def test_bedrock_messages_strips_output_config(): headers={}, ) - assert "output_config" not in result, ( - "output_config should be stripped — Bedrock Invoke rejects it" - ) + assert ( + "output_config" not in result + ), "output_config should be stripped — Bedrock Invoke rejects it" # Other params should be preserved assert result.get("max_tokens") == 4096 diff --git a/tests/test_litellm/llms/bedrock/test_anthropic_beta_support.py b/tests/test_litellm/llms/bedrock/test_anthropic_beta_support.py index 074a319a603..509db357c2a 100644 --- a/tests/test_litellm/llms/bedrock/test_anthropic_beta_support.py +++ b/tests/test_litellm/llms/bedrock/test_anthropic_beta_support.py @@ -53,7 +53,7 @@ class TestAnthropicBetaHeaderSupport: headers = {"anthropic-beta": "context-1m-2025-08-07,computer-use-2024-10-22"} result = config.transform_request( - model="anthropic.claude-3-5-sonnet-20241022-v2:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "Test"}], optional_params={}, litellm_params={}, @@ -70,7 +70,7 @@ class TestAnthropicBetaHeaderSupport: headers = {"anthropic-beta": "context-1m-2025-08-07,interleaved-thinking-2025-05-14"} result = config._transform_request_helper( - model="anthropic.claude-3-5-sonnet-20241022-v2:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", system_content_blocks=[], optional_params={}, messages=[{"role": "user", "content": "Test"}], @@ -89,7 +89,7 @@ class TestAnthropicBetaHeaderSupport: headers = {"anthropic-beta": "output-128k-2025-02-19"} result = config.transform_anthropic_messages_request( - model="anthropic.claude-3-5-sonnet-20241022-v2:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "Test"}], anthropic_messages_optional_request_params={"max_tokens": 100}, litellm_params={}, @@ -116,7 +116,7 @@ class TestAnthropicBetaHeaderSupport: ] result = config._transform_request_helper( - model="anthropic.claude-3-5-sonnet-20241022-v2:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", system_content_blocks=[], optional_params={"tools": tools}, messages=[{"role": "user", "content": "Test"}], @@ -125,10 +125,13 @@ class TestAnthropicBetaHeaderSupport: additional_fields = result["additionalModelRequestFields"] betas = additional_fields["anthropic_beta"] - - # Should contain both user-provided and auto-added beta headers + + # Should contain user header plus computer-use beta for this model (Haiku 4.5 uses 2025-01-24) assert "context-1m-2025-08-07" in betas - assert "computer-use-2024-10-22" in betas + assert ( + "computer-use-2024-10-22" in betas + or "computer-use-2025-01-24" in betas + ) assert len(betas) == 2 # No duplicates def test_no_anthropic_beta_headers(self): @@ -137,7 +140,7 @@ class TestAnthropicBetaHeaderSupport: headers = {} result = config._transform_request_helper( - model="anthropic.claude-3-5-sonnet-20241022-v2:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", system_content_blocks=[], optional_params={}, messages=[{"role": "user", "content": "Test"}], @@ -163,7 +166,7 @@ class TestAnthropicBetaHeaderSupport: headers = {"anthropic-beta": ",".join(supported_features)} result = config.transform_request( - model="anthropic.claude-3-5-sonnet-20241022-v2:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "Test"}], optional_params={}, litellm_params={}, @@ -358,7 +361,7 @@ class TestAnthropicBetaHeaderSupport: headers = {"anthropic-beta": "context-1m-2025-08-07"} result = config._transform_request_helper( - model="anthropic.claude-3-5-sonnet-20241022-v2:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", system_content_blocks=[], optional_params={}, messages=[{"role": "user", "content": "Test"}], @@ -378,7 +381,7 @@ class TestAnthropicBetaHeaderSupport: # Model with 'us.' cross-region prefix result = config._transform_request_helper( - model="us.anthropic.claude-3-5-sonnet-20241022-v2:0", + model="us.anthropic.claude-haiku-4-5-20251001-v1:0", system_content_blocks=[], optional_params={}, messages=[{"role": "user", "content": "Test"}], diff --git a/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py b/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py index b804f549ecb..5ce291aa165 100644 --- a/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py +++ b/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py @@ -32,9 +32,9 @@ def test_govcloud_cross_region_inference_prefix(): # Test us-gov prefix is stripped correctly for Claude models base_model = bedrock_model_info.get_base_model( - model="bedrock/us-gov.anthropic.claude-3-5-sonnet-20240620-v1:0" + model="bedrock/us-gov.anthropic.claude-haiku-4-5-20251001-v1:0" ) - assert base_model == "anthropic.claude-3-5-sonnet-20240620-v1:0" + assert base_model == "anthropic.claude-haiku-4-5-20251001-v1:0" # Test us-gov prefix is stripped correctly for different Claude versions base_model = bedrock_model_info.get_base_model( diff --git a/tests/test_litellm/llms/chat/test_converse_handler.py b/tests/test_litellm/llms/chat/test_converse_handler.py index f207c1d272a..1e75a2f1fcb 100644 --- a/tests/test_litellm/llms/chat/test_converse_handler.py +++ b/tests/test_litellm/llms/chat/test_converse_handler.py @@ -48,8 +48,8 @@ class TestBedrockRegionInModelPath: "us-east-1", ), ( - "us-west-2/anthropic.claude-3-5-sonnet-20241022-v2:0", - "anthropic.claude-3-5-sonnet-20241022-v2%3A0", + "us-west-2/anthropic.claude-haiku-4-5-20251001-v1:0", + "anthropic.claude-haiku-4-5-20251001-v1%3A0", "us-west-2", ), # No region in path — modelId unchanged, no region injected @@ -60,8 +60,8 @@ class TestBedrockRegionInModelPath: ), # Cross-region inference prefix (us., eu., ap.) — not a region path segment ( - "us.anthropic.claude-3-5-sonnet-20241022-v2:0", - "us.anthropic.claude-3-5-sonnet-20241022-v2%3A0", + "us.anthropic.claude-haiku-4-5-20251001-v1:0", + "us.anthropic.claude-haiku-4-5-20251001-v1%3A0", None, ), ], diff --git a/tests/test_litellm/passthrough/test_passthrough_main.py b/tests/test_litellm/passthrough/test_passthrough_main.py index c84d32d48f2..489357149c5 100644 --- a/tests/test_litellm/passthrough/test_passthrough_main.py +++ b/tests/test_litellm/passthrough/test_passthrough_main.py @@ -31,7 +31,7 @@ def test_llm_passthrough_route(): return_value=MagicMock(status_code=200, json={"message": "Hello, world!"}), ) as mock_post: response = llm_passthrough_route( - model="vllm/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="vllm/anthropic.claude-haiku-4-5-20251001-v1:0", endpoint="v1/chat/completions", method="POST", request_url="http://localhost:8000/v1/chat/completions", diff --git a/tests/test_litellm/proxy/auth/test_unmapped_model_budget_enforcement.py b/tests/test_litellm/proxy/auth/test_unmapped_model_budget_enforcement.py new file mode 100644 index 00000000000..e9f4111f83d --- /dev/null +++ b/tests/test_litellm/proxy/auth/test_unmapped_model_budget_enforcement.py @@ -0,0 +1,108 @@ +""" +Test that models not in the cost map do NOT bypass budget enforcement. + +Regression test for the bug where unmapped models got fallback costs of 0, +causing _is_model_cost_zero() to return True and skip all budget checks. + +See: https://github.com/BerriAI/litellm/issues/24770 +""" + +import copy + +import litellm +from litellm.proxy.auth.auth_checks import _is_model_cost_zero +from litellm.router import Router + + +class TestUnmappedModelBudgetEnforcement: + """Unmapped models must NOT bypass budget checks.""" + + def setup_method(self): + """Snapshot litellm.model_cost before each test.""" + self._saved_model_cost = copy.deepcopy(litellm.model_cost) + + def teardown_method(self): + """Restore litellm.model_cost after each test.""" + litellm.model_cost = self._saved_model_cost + + def test_unmapped_model_enforces_budget(self): + """A model not in litellm.model_cost should have budget enforced.""" + router = Router( + model_list=[ + { + "model_name": "custom-model", + "litellm_params": { + "model": "openai/totally-nonexistent-model-xyz", + "api_key": "sk-fake", + }, + }, + ] + ) + result = _is_model_cost_zero(model="custom-model", llm_router=router) + assert result is False, ( + "Unmapped model should enforce budget (return False), " + "not bypass it (return True)" + ) + + def test_explicitly_free_model_bypasses_budget(self): + """A model with explicit cost=0 in model_info should bypass budget.""" + router = Router( + model_list=[ + { + "model_name": "free-model", + "litellm_params": { + "model": "ollama/llama2", + "api_base": "http://localhost:11434", + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + }, + "model_info": { + "id": "free-model-id", + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + }, + }, + ] + ) + result = _is_model_cost_zero(model="free-model", llm_router=router) + assert result is True, ( + "Explicitly free model should bypass budget (return True)" + ) + + def test_known_paid_model_enforces_budget(self): + """A model in the cost map with non-zero costs should enforce budget.""" + router = Router( + model_list=[ + { + "model_name": "paid-model", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "sk-fake", + }, + }, + ] + ) + result = _is_model_cost_zero(model="paid-model", llm_router=router) + assert result is False, ( + "Known paid model should enforce budget (return False)" + ) + + def test_unmapped_model_with_litellm_params_pricing(self): + """A model with cost=0 in litellm_params (not model_info) should bypass budget.""" + router = Router( + model_list=[ + { + "model_name": "free-via-params", + "litellm_params": { + "model": "openai/nonexistent-but-free-model", + "api_key": "sk-fake", + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + }, + }, + ] + ) + result = _is_model_cost_zero(model="free-via-params", llm_router=router) + assert result is True, ( + "Model with explicit cost=0 in litellm_params should bypass budget" + ) diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py index 74c7f9bca5f..6e1b245b3de 100644 --- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py +++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py @@ -3,22 +3,30 @@ import json import os import sys from typing import Tuple -from unittest.mock import AsyncMock, MagicMock, patch +from unittest.mock import ANY, AsyncMock, MagicMock, patch sys.path.insert( 0, os.path.abspath("../../..") ) # Adds the parent directory to the system path -from unittest.mock import MagicMock - import pytest import litellm.proxy.proxy_server from litellm.caching.dual_cache import DualCache -from litellm.proxy._types import LiteLLM_JWTAuth, UserAPIKeyAuth +from litellm.proxy._types import ( + LiteLLM_JWTAuth, + ProxyErrorTypes, + ProxyException, + UserAPIKeyAuth, + JWTRoutingOverride, +) from litellm.proxy.auth.handle_jwt import JWTHandler from litellm.proxy.auth.route_checks import RouteChecks -from litellm.proxy.auth.user_api_key_auth import get_api_key, user_api_key_auth +from litellm.proxy.auth.user_api_key_auth import ( + _run_post_custom_auth_checks, + get_api_key, + user_api_key_auth, +) def test_get_api_key(): @@ -38,6 +46,86 @@ def test_get_api_key(): ) == (api_key, passed_in_key) +@pytest.mark.asyncio +async def test_custom_auth_does_not_enforce_key_model_access_by_default(): + valid_token = UserAPIKeyAuth(token="test_token", models=["gpt-4o-mini"]) + request_data = {"model": "gpt-4o"} + + with patch( + "litellm.proxy.auth.user_api_key_auth.can_key_call_model", new_callable=AsyncMock + ) as mock_can_key, patch( + "litellm.proxy.proxy_server.general_settings", + {}, + ): + await _run_post_custom_auth_checks( + valid_token=valid_token, + request=None, + request_data=request_data, + route="/v1/chat/completions", + parent_otel_span=None, + ) + mock_can_key.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_custom_auth_honors_key_level_model_access_restriction_allowed_with_opt_in(): + valid_token = UserAPIKeyAuth(token="test_token", models=["gpt-4o-mini"]) + request_data = {"model": "gpt-4o-mini"} + + with patch( + "litellm.proxy.auth.user_api_key_auth.can_key_call_model", new_callable=AsyncMock + ) as mock_can_key, patch( + "litellm.proxy.auth.user_api_key_auth.common_checks", new_callable=AsyncMock + ), patch( + "litellm.proxy.proxy_server.general_settings", + {"custom_auth_run_common_checks": True}, + ): + await _run_post_custom_auth_checks( + valid_token=valid_token, + request=None, + request_data=request_data, + route="/v1/chat/completions", + parent_otel_span=None, + ) + mock_can_key.assert_awaited_once_with( + model="gpt-4o-mini", + llm_model_list=ANY, + valid_token=valid_token, + llm_router=ANY, + ) + + +@pytest.mark.asyncio +async def test_custom_auth_honors_key_level_model_access_restriction_denied_with_opt_in(): + valid_token = UserAPIKeyAuth(token="test_token", models=["gpt-4o-mini"]) + request_data = {"model": "gpt-4o"} + + with patch( + "litellm.proxy.auth.user_api_key_auth.can_key_call_model", new_callable=AsyncMock + ) as mock_can_key, patch( + "litellm.proxy.auth.user_api_key_auth.common_checks", new_callable=AsyncMock + ), patch( + "litellm.proxy.proxy_server.general_settings", + {"custom_auth_run_common_checks": True}, + ): + mock_can_key.side_effect = ProxyException( + message="Key not allowed to access model", + type=ProxyErrorTypes.key_model_access_denied, + param="model", + code=401, + ) + with pytest.raises(ProxyException) as exc: + await _run_post_custom_auth_checks( + valid_token=valid_token, + request=None, + request_data=request_data, + route="/v1/chat/completions", + parent_otel_span=None, + ) + + assert exc.value.type == ProxyErrorTypes.key_model_access_denied + + @pytest.mark.parametrize( "custom_litellm_key_header, api_key, passed_in_key", [ @@ -689,6 +777,203 @@ class TestJWTOAuth2Coexistence: mock_jwt_auth.assert_called_once() assert result.user_id == "jwt-human-user" + @pytest.mark.asyncio + async def test_routing_override_routes_matching_jwt_to_oauth2(self): + """ + When routing_overrides match JWT claims, route JWT-shaped token to OAuth2. + """ + jwt_token = ( + "eyJhbGciOiJSUzI1NiJ9." + "eyJpc3MiOiJtYWNoaW5lLWlzc3Vlci5leGFtcGxlLmNvbSIsImNsaWVudF9pZCI6Ik1JRF9MSVRFTExNIn0." + "c2ln" + ) + general_settings = { + "enable_oauth2_auth": True, + "enable_jwt_auth": True, + } + mock_oauth2_response = UserAPIKeyAuth( + api_key=jwt_token, + user_id="machine-client-override", + ) + + mock_request = MagicMock() + mock_request.url.path = "/v1/chat/completions" + mock_request.headers = {"authorization": f"Bearer {jwt_token}"} + mock_request.query_params = {} + + with patch( + "litellm.proxy.proxy_server.general_settings", general_settings + ), patch("litellm.proxy.proxy_server.premium_user", True), patch( + "litellm.proxy.proxy_server.master_key", "sk-master" + ), patch( + "litellm.proxy.proxy_server.prisma_client", None + ), patch( + "litellm.proxy.auth.user_api_key_auth.Oauth2Handler.check_oauth2_token", + new_callable=AsyncMock, + return_value=mock_oauth2_response, + ) as mock_oauth2, patch( + "litellm.proxy.auth.user_api_key_auth.JWTAuthManager.auth_builder", + new_callable=AsyncMock, + ) as mock_jwt_auth: + litellm.proxy.proxy_server.jwt_handler.update_environment( + prisma_client=None, + user_api_key_cache=DualCache(), + litellm_jwtauth=LiteLLM_JWTAuth( + routing_overrides=[ + JWTRoutingOverride( + iss="machine-issuer.example.com", + client_id="MID_LITELLM", + path="oauth2", + ) + ] + ), + ) + + result = await user_api_key_auth( + request=mock_request, + api_key=f"Bearer {jwt_token}", + ) + + mock_oauth2.assert_called_once_with(token=jwt_token) + mock_jwt_auth.assert_not_called() + assert result.user_id == "machine-client-override" + + @pytest.mark.asyncio + async def test_routing_override_does_not_match_client_id_falls_back_to_jwt(self): + """ + If override ISS matches but client_id does not, continue default JWT flow. + """ + jwt_token = ( + "eyJhbGciOiJSUzI1NiJ9." + "eyJpc3MiOiJtYWNoaW5lLWlzc3Vlci5leGFtcGxlLmNvbSIsImNsaWVudF9pZCI6IlVTRVJfUE9SVEFMIn0." + "c2ln" + ) + general_settings = { + "enable_oauth2_auth": True, + "enable_jwt_auth": True, + } + mock_jwt_result = { + "is_proxy_admin": True, + "team_object": None, + "user_object": None, + "end_user_object": None, + "org_object": None, + "token": jwt_token, + "team_id": "jwt-team", + "user_id": "jwt-user-no-override", + "end_user_id": None, + "org_id": None, + "team_membership": None, + "jwt_claims": {"sub": "user1"}, + } + + mock_request = MagicMock() + mock_request.url.path = "/v1/chat/completions" + mock_request.headers = {"authorization": f"Bearer {jwt_token}"} + mock_request.query_params = {} + + with patch( + "litellm.proxy.proxy_server.general_settings", general_settings + ), patch("litellm.proxy.proxy_server.premium_user", True), patch( + "litellm.proxy.proxy_server.master_key", "sk-master" + ), patch( + "litellm.proxy.proxy_server.prisma_client", None + ), patch( + "litellm.proxy.auth.user_api_key_auth.Oauth2Handler.check_oauth2_token", + new_callable=AsyncMock, + ) as mock_oauth2, patch( + "litellm.proxy.auth.user_api_key_auth.JWTAuthManager.auth_builder", + new_callable=AsyncMock, + return_value=mock_jwt_result, + ) as mock_jwt_auth: + litellm.proxy.proxy_server.jwt_handler.update_environment( + prisma_client=None, + user_api_key_cache=DualCache(), + litellm_jwtauth=LiteLLM_JWTAuth( + routing_overrides=[ + JWTRoutingOverride( + iss="machine-issuer.example.com", + client_id="MID_LITELLM", + path="oauth2", + ) + ] + ), + ) + + result = await user_api_key_auth( + request=mock_request, + api_key=f"Bearer {jwt_token}", + ) + + mock_oauth2.assert_not_called() + mock_jwt_auth.assert_called_once() + assert result.user_id == "jwt-user-no-override" + + @pytest.mark.asyncio + async def test_routing_override_matches_aud_claim_list_and_list_selectors(self): + """ + Match routing override when selectors are lists and token aud claim is a list. + """ + jwt_token = ( + "eyJhbGciOiJSUzI1NiJ9." + "eyJpc3MiOiJtYWNoaW5lLWlzc3Vlci5leGFtcGxlLmNvbSIsImNsaWVudF9pZCI6Ik1JRF9MSVRFTExNIiwiYXVkIjpbImFwaTovL2xpdGVsbG0iLCJhcGk6Ly9vdGhlciJdfQ." + "c2ln" + ) + general_settings = { + "enable_oauth2_auth": True, + "enable_jwt_auth": True, + } + mock_oauth2_response = UserAPIKeyAuth( + api_key=jwt_token, + user_id="machine-client-aud-list", + ) + + mock_request = MagicMock() + mock_request.url.path = "/v1/chat/completions" + mock_request.headers = {"authorization": f"Bearer {jwt_token}"} + mock_request.query_params = {} + + with patch( + "litellm.proxy.proxy_server.general_settings", general_settings + ), patch("litellm.proxy.proxy_server.premium_user", True), patch( + "litellm.proxy.proxy_server.master_key", "sk-master" + ), patch( + "litellm.proxy.proxy_server.prisma_client", None + ), patch( + "litellm.proxy.auth.user_api_key_auth.Oauth2Handler.check_oauth2_token", + new_callable=AsyncMock, + return_value=mock_oauth2_response, + ) as mock_oauth2, patch( + "litellm.proxy.auth.user_api_key_auth.JWTAuthManager.auth_builder", + new_callable=AsyncMock, + ) as mock_jwt_auth: + litellm.proxy.proxy_server.jwt_handler.update_environment( + prisma_client=None, + user_api_key_cache=DualCache(), + litellm_jwtauth=LiteLLM_JWTAuth( + routing_overrides=[ + JWTRoutingOverride( + iss=[ + "machine-issuer.example.com", + "other-issuer.example.com", + ], + client_id=["MID_LITELLM", "MID_BACKUP"], + aud=["api://litellm", "api://fallback"], + path="oauth2", + ) + ] + ), + ) + + result = await user_api_key_auth( + request=mock_request, + api_key=f"Bearer {jwt_token}", + ) + + mock_oauth2.assert_called_once_with(token=jwt_token) + mock_jwt_auth.assert_not_called() + assert result.user_id == "machine-client-aud-list" + @pytest.mark.asyncio async def test_only_oauth2_enabled_handles_all_tokens(self): """ diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py index 8080491f662..7bd87ed05d7 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py @@ -380,8 +380,9 @@ async def test_model_armor_api_error_handling(): call_type="completion" ) - assert exc_info.value.status_code == 500 + assert exc_info.value.status_code == 400 assert "Model Armor API error" in str(exc_info.value.detail) + assert "upstream 500" in str(exc_info.value.detail) @pytest.mark.asyncio @@ -485,6 +486,128 @@ async def test_model_armor_streaming_response(): assert len(result_chunks) > 0 mock_post.assert_called() +@pytest.mark.asyncio +async def test_model_armor_streaming_block_yields_sse_error(): + """Test that streaming content block yields SSE error event instead of raising HTTPException.""" + mock_user_api_key_dict = UserAPIKeyAuth() + + guardrail = ModelArmorGuardrail( + template_id="test-template", + project_id="test-project", + location="us-central1", + guardrail_name="model-armor-test", + ) + + # Mock Model Armor API response that triggers a block (SDP MATCH_FOUND) + mock_response = AsyncMock() + mock_response.status_code = 200 + mock_response.json = AsyncMock( + return_value={ + "sanitizationResult": { + "filterMatchState": "MATCH_FOUND", + "filterResults": { + "sdp": { + "sdpFilterResult": { + "inspectResult": { + "matchState": "MATCH_FOUND", + "findings": [ + { + "infoType": "PASSWORD", + "likelihood": "VERY_LIKELY", + } + ], + } + } + } + }, + } + } + ) + + guardrail._ensure_access_token_async = AsyncMock( + return_value=("test-token", "test-project") + ) + + with patch.object( + guardrail.async_handler, "post", AsyncMock(return_value=mock_response) + ): + + async def mock_stream(): + chunks = [ + litellm.ModelResponseStream( + choices=[ + litellm.types.utils.StreamingChoices( + delta=litellm.types.utils.Delta( + content="My password is " + ) + ) + ] + ), + litellm.ModelResponseStream( + choices=[ + litellm.types.utils.StreamingChoices( + delta=litellm.types.utils.Delta(content="hunter2") + ) + ] + ), + ] + for chunk in chunks: + yield chunk + + request_data = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "What's your password?"}], + "metadata": {"guardrails": ["model-armor-test"]}, + } + + result_chunks = [] + async for chunk in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=mock_user_api_key_dict, + response=mock_stream(), + request_data=request_data, + ): + result_chunks.append(chunk) + + # Should yield exactly one SSE error event (not raise HTTPException) + assert len(result_chunks) == 1 + error_data = json.loads(result_chunks[0].removeprefix("data: ")) + assert "error" in error_data + assert int(error_data["error"]["code"]) == 400 + + +@pytest.mark.asyncio +async def test_model_armor_api_failure_returns_400(): + """Test that Model Armor API failures raise HTTP 400, not the upstream status code.""" + guardrail = ModelArmorGuardrail( + template_id="test-template", + project_id="test-project", + location="us-central1", + guardrail_name="model-armor-test", + ) + + # Mock a 500 response from the Model Armor GCP API + mock_response = AsyncMock() + mock_response.status_code = 500 + mock_response.text = "Internal Server Error" + + guardrail._ensure_access_token_async = AsyncMock( + return_value=("test-token", "test-project") + ) + + with patch.object( + guardrail.async_handler, "post", AsyncMock(return_value=mock_response) + ): + with pytest.raises(HTTPException) as exc_info: + await guardrail.make_model_armor_request( + content="test content", + source="user_prompt", + ) + + # Should be 400, NOT the upstream 500 + assert exc_info.value.status_code == 400 + assert "upstream 500" in str(exc_info.value.detail) + + def test_model_armor_ui_friendly_name(): """Test the UI-friendly name of the Model Armor guardrail""" from litellm.types.proxy.guardrails.guardrail_hooks.model_armor import ( diff --git a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py index d269a9531fd..d35dbb87a1a 100644 --- a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py @@ -479,3 +479,67 @@ async def test_track_cost_callback_skips_for_falsy_model_and_no_slo(model_value) ) mock_proxy_logging.failed_tracking_alert.assert_not_called() + + +@pytest.mark.asyncio +async def test_async_post_call_failure_hook_uses_actual_start_time(): + """ + Verify that failed requests record the actual request start time + instead of datetime.now(), so the spend log shows the real duration. + + Previously both start_time and end_time were set to datetime.now() + at failure-logging time, resulting in duration=0 for all failures. + """ + from datetime import timedelta + + logger = _ProxyDBLogger() + + user_api_key_dict = UserAPIKeyAuth( + api_key="test_api_key", + user_id="test_user_id", + team_id="test_team_id", + org_id="test_org_id", + end_user_id="test_end_user_id", + ) + + # Simulate a request that started 60 seconds ago + simulated_start = datetime.now() - timedelta(seconds=60) + + mock_logging_obj = MagicMock() + mock_logging_obj.start_time = simulated_start + mock_logging_obj.model_call_details = {} + mock_logging_obj.litellm_trace_id = None + + request_data = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "Hello"}], + "metadata": {}, + "proxy_server_request": {}, + "litellm_logging_obj": mock_logging_obj, + } + + original_exception = Exception("Timeout error") + + with patch( + "litellm.proxy.db.db_spend_update_writer.DBSpendUpdateWriter.update_database", + new_callable=AsyncMock, + ) as mock_update_database: + await logger.async_post_call_failure_hook( + request_data=request_data, + original_exception=original_exception, + user_api_key_dict=user_api_key_dict, + ) + + mock_update_database.assert_called_once() + call_args = mock_update_database.call_args[1] + + # start_time should be the simulated start, not datetime.now() + assert call_args["start_time"] == simulated_start + + # end_time should be close to now (within a few seconds) + time_diff = (datetime.now() - call_args["end_time"]).total_seconds() + assert time_diff < 5, f"end_time should be close to now, was {time_diff}s ago" + + # Duration should be approximately 60 seconds, not 0 + duration = (call_args["end_time"] - call_args["start_time"]).total_seconds() + assert duration >= 55, f"Duration should be ~60s, got {duration}s" diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 0ff276953c6..72092a97f5b 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -33,6 +33,7 @@ from litellm.proxy.management_endpoints.key_management_endpoints import ( _check_org_key_limits, _check_team_key_limits, _common_key_generation_helper, + _enforce_upperbound_key_params, _get_and_validate_existing_key, _list_key_helper, _persist_deleted_verification_tokens, @@ -8435,3 +8436,146 @@ class TestKeyAliasSkipValidationOnUnchanged: # None alias should always pass _validate_key_alias_format(None) + + +# --- Tests: _enforce_upperbound_key_params --- + + +def test_enforce_upperbound_rejects_over_limit_on_generate(): + """Test that key generation is rejected when values exceed upperbound.""" + import litellm + from litellm.types.proxy.management_endpoints.ui_sso import ( + LiteLLM_UpperboundKeyGenerateParams, + ) + + original = litellm.upperbound_key_generate_params + try: + litellm.upperbound_key_generate_params = LiteLLM_UpperboundKeyGenerateParams( + tpm_limit=1000, rpm_limit=100, max_budget=10.0 + ) + data = GenerateKeyRequest(tpm_limit=5000) + with pytest.raises(HTTPException) as exc_info: + _enforce_upperbound_key_params(data, fill_defaults=True) + assert exc_info.value.status_code == 400 + assert "tpm_limit" in str(exc_info.value.detail) + finally: + litellm.upperbound_key_generate_params = original + + +def test_enforce_upperbound_fills_defaults_on_generate(): + """Test that None values are filled with upperbound defaults during generation.""" + import litellm + from litellm.types.proxy.management_endpoints.ui_sso import ( + LiteLLM_UpperboundKeyGenerateParams, + ) + + original = litellm.upperbound_key_generate_params + try: + litellm.upperbound_key_generate_params = LiteLLM_UpperboundKeyGenerateParams( + tpm_limit=1000, rpm_limit=100 + ) + data = GenerateKeyRequest() # tpm_limit=None, rpm_limit=None + _enforce_upperbound_key_params(data, fill_defaults=True) + assert data.tpm_limit == 1000 + assert data.rpm_limit == 100 + finally: + litellm.upperbound_key_generate_params = original + + +def test_enforce_upperbound_skips_none_on_update(): + """Test that None values are NOT filled during update (fill_defaults=False).""" + import litellm + from litellm.types.proxy.management_endpoints.ui_sso import ( + LiteLLM_UpperboundKeyGenerateParams, + ) + + original = litellm.upperbound_key_generate_params + try: + litellm.upperbound_key_generate_params = LiteLLM_UpperboundKeyGenerateParams( + tpm_limit=1000, rpm_limit=100 + ) + data = UpdateKeyRequest(key="sk-test") # tpm_limit=None, rpm_limit=None + _enforce_upperbound_key_params(data, fill_defaults=False) + assert data.tpm_limit is None # should NOT be filled + assert data.rpm_limit is None # should NOT be filled + finally: + litellm.upperbound_key_generate_params = original + + +def test_enforce_upperbound_rejects_over_limit_on_update(): + """Test that key update is rejected when values exceed upperbound.""" + import litellm + from litellm.types.proxy.management_endpoints.ui_sso import ( + LiteLLM_UpperboundKeyGenerateParams, + ) + + original = litellm.upperbound_key_generate_params + try: + litellm.upperbound_key_generate_params = LiteLLM_UpperboundKeyGenerateParams( + tpm_limit=1000, rpm_limit=100, max_budget=10.0 + ) + data = UpdateKeyRequest(key="sk-test", tpm_limit=5000) + with pytest.raises(HTTPException) as exc_info: + _enforce_upperbound_key_params(data, fill_defaults=False) + assert exc_info.value.status_code == 400 + assert "tpm_limit" in str(exc_info.value.detail) + finally: + litellm.upperbound_key_generate_params = original + + +def test_enforce_upperbound_allows_within_limit_on_update(): + """Test that key update passes when values are within upperbound.""" + import litellm + from litellm.types.proxy.management_endpoints.ui_sso import ( + LiteLLM_UpperboundKeyGenerateParams, + ) + + original = litellm.upperbound_key_generate_params + try: + litellm.upperbound_key_generate_params = LiteLLM_UpperboundKeyGenerateParams( + tpm_limit=1000, rpm_limit=100, max_budget=10.0 + ) + data = UpdateKeyRequest(key="sk-test", tpm_limit=500, rpm_limit=50, max_budget=5.0) + _enforce_upperbound_key_params(data, fill_defaults=False) + # Should not raise + assert data.tpm_limit == 500 + assert data.rpm_limit == 50 + assert data.max_budget == 5.0 + finally: + litellm.upperbound_key_generate_params = original + + +def test_enforce_upperbound_duration_over_limit(): + """Test that duration exceeding upperbound is rejected.""" + import litellm + from litellm.types.proxy.management_endpoints.ui_sso import ( + LiteLLM_UpperboundKeyGenerateParams, + ) + + original = litellm.upperbound_key_generate_params + try: + litellm.upperbound_key_generate_params = LiteLLM_UpperboundKeyGenerateParams( + duration="7d" + ) + data = UpdateKeyRequest(key="sk-test", duration="30d") + with pytest.raises(HTTPException) as exc_info: + _enforce_upperbound_key_params(data, fill_defaults=False) + assert exc_info.value.status_code == 400 + assert "duration" in str(exc_info.value.detail) + finally: + litellm.upperbound_key_generate_params = original + + +def test_enforce_upperbound_no_config_is_noop(): + """Test that no enforcement happens when upperbound params are not configured.""" + import litellm + + original = litellm.upperbound_key_generate_params + try: + litellm.upperbound_key_generate_params = None + data = UpdateKeyRequest(key="sk-test", tpm_limit=999999) + _enforce_upperbound_key_params(data, fill_defaults=False) + # Should not raise — no enforcement configured + assert data.tpm_limit == 999999 + finally: + litellm.upperbound_key_generate_params = original diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index 366f659bdab..e11dbbd8915 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -3147,7 +3147,7 @@ async def test_new_team_org_scoped_models_bypasses_user_limit(): @pytest.mark.asyncio -async def test_new_team_standalone_validates_against_user_models(): +async def test_new_team_standalone_validates_against_user_models(monkeypatch): """ Test that /team/new WITHOUT organization_id still validates models against user's personal models. @@ -3158,11 +3158,17 @@ async def test_new_team_standalone_validates_against_user_models(): - Team is created WITHOUT organization_id and models=['gpt-4'] - Expected: Should fail with "Model not in allowed user models" """ + import litellm from fastapi import Request from litellm.proxy._types import NewTeamRequest, ProxyException, UserAPIKeyAuth from litellm.proxy.management_endpoints.team_endpoints import new_team + # Avoid injecting max_budget via global defaults; that path calls get_user_object and + # needs cache/DB mocks — this test only covers model validation. + monkeypatch.setattr(litellm, "default_team_settings", None) + monkeypatch.setattr(litellm, "default_team_params", None) + # Create non-admin user with restrictive personal models non_admin_user = UserAPIKeyAuth( user_role=LitellmUserRoles.INTERNAL_USER, diff --git a/tests/test_litellm/proxy/prompts/test_prompt_environment.py b/tests/test_litellm/proxy/prompts/test_prompt_environment.py new file mode 100644 index 00000000000..ecd89afefbe --- /dev/null +++ b/tests/test_litellm/proxy/prompts/test_prompt_environment.py @@ -0,0 +1,253 @@ +import json +import pytest +from unittest.mock import MagicMock +from litellm.types.prompts.init_prompts import ( + PromptInfo, + PromptSpec, + PromptLiteLLMParams, +) + + +def test_prompt_info_default_environment(): + """PromptInfo should default environment to 'development'.""" + info = PromptInfo(prompt_type="db") + assert info.environment == "development" + + +def test_prompt_info_custom_environment(): + """PromptInfo should accept a custom environment.""" + info = PromptInfo(prompt_type="db", environment="production") + assert info.environment == "production" + + +def test_prompt_spec_includes_environment_and_created_by(): + """PromptSpec should carry environment and created_by fields.""" + spec = PromptSpec( + prompt_id="test", + litellm_params=PromptLiteLLMParams( + prompt_id="test", prompt_integration="dotprompt" + ), + prompt_info=PromptInfo(prompt_type="db", environment="staging"), + environment="staging", + created_by="user-123", + ) + assert spec.environment == "staging" + assert spec.created_by == "user-123" + + +def test_prompt_spec_default_environment(): + """PromptSpec environment should default to 'development'.""" + spec = PromptSpec( + prompt_id="test", + litellm_params=PromptLiteLLMParams( + prompt_id="test", prompt_integration="dotprompt" + ), + prompt_info=PromptInfo(prompt_type="db"), + ) + assert spec.environment == "development" + assert spec.created_by is None + + +def test_create_versioned_prompt_spec_includes_environment(): + """create_versioned_prompt_spec should populate environment and created_by from DB row.""" + from litellm.proxy.prompts.prompt_endpoints import create_versioned_prompt_spec + + mock_db_prompt = MagicMock() + mock_db_prompt.model_dump.return_value = { + "id": "uuid-123", + "prompt_id": "test_prompt", + "version": 2, + "environment": "staging", + "created_by": "user-456", + "litellm_params": json.dumps( + { + "prompt_id": "test_prompt", + "prompt_integration": "dotprompt", + } + ), + "prompt_info": json.dumps({"prompt_type": "db", "environment": "staging"}), + "created_at": None, + "updated_at": None, + } + spec = create_versioned_prompt_spec(mock_db_prompt) + assert spec.environment == "staging" + assert spec.created_by == "user-456" + assert spec.prompt_id == "test_prompt.v2" + + +@pytest.mark.asyncio +async def test_create_prompt_stores_environment_and_created_by(): + """create_prompt should pass environment and created_by to the DB.""" + from unittest.mock import AsyncMock, patch + from litellm.proxy._types import UserAPIKeyAuth, LitellmUserRoles + from litellm.proxy.prompts.prompt_endpoints import create_prompt, Prompt + + mock_user_auth = UserAPIKeyAuth( + api_key="sk-1234", + user_role=LitellmUserRoles.PROXY_ADMIN, + user_id="user-789", + ) + + mock_prisma_client = MagicMock() + mock_db_entry = MagicMock() + mock_db_entry.model_dump.return_value = { + "id": "uuid-1", + "prompt_id": "my_prompt", + "version": 1, + "environment": "staging", + "created_by": "user-789", + "litellm_params": json.dumps( + { + "prompt_id": "my_prompt", + "prompt_integration": "dotprompt", + } + ), + "prompt_info": json.dumps({"prompt_type": "db", "environment": "staging"}), + "created_at": None, + "updated_at": None, + } + mock_prisma_client.db.litellm_prompttable.create = AsyncMock( + return_value=mock_db_entry + ) + mock_prisma_client.db.litellm_prompttable.find_many = AsyncMock(return_value=[]) + + request = Prompt( + prompt_id="my_prompt", + litellm_params=PromptLiteLLMParams( + prompt_id="my_prompt", prompt_integration="dotprompt" + ), + prompt_info=PromptInfo(prompt_type="db", environment="staging"), + ) + + with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client): + with patch( + "litellm.proxy.prompts.prompt_registry.IN_MEMORY_PROMPT_REGISTRY" + ) as mock_registry: + mock_registry.initialize_prompt.return_value = PromptSpec( + prompt_id="my_prompt.v1", + litellm_params=request.litellm_params, + prompt_info=request.prompt_info, + environment="staging", + created_by="user-789", + ) + await create_prompt(request=request, user_api_key_dict=mock_user_auth) + + create_call = mock_prisma_client.db.litellm_prompttable.create.call_args + data = create_call.kwargs["data"] + assert data["environment"] == "staging" + assert data["created_by"] == "user-789" + + +@pytest.mark.asyncio +async def test_update_prompt_stores_environment_and_created_by(): + """update_prompt should pass environment and created_by to new version.""" + from unittest.mock import AsyncMock, patch + from litellm.proxy._types import UserAPIKeyAuth, LitellmUserRoles + from litellm.proxy.prompts.prompt_endpoints import update_prompt, Prompt + + mock_user_auth = UserAPIKeyAuth( + api_key="sk-1234", + user_role=LitellmUserRoles.PROXY_ADMIN, + user_id="user-update", + ) + + mock_prisma_client = MagicMock() + mock_existing = MagicMock() + mock_existing.version = 1 + mock_prisma_client.db.litellm_prompttable.find_many = AsyncMock( + return_value=[mock_existing] + ) + + mock_db_entry = MagicMock() + mock_db_entry.model_dump.return_value = { + "id": "uuid-2", + "prompt_id": "my_prompt", + "version": 2, + "environment": "production", + "created_by": "user-update", + "litellm_params": json.dumps( + { + "prompt_id": "my_prompt", + "prompt_integration": "dotprompt", + } + ), + "prompt_info": json.dumps({"prompt_type": "db", "environment": "production"}), + "created_at": None, + "updated_at": None, + } + mock_prisma_client.db.litellm_prompttable.create = AsyncMock( + return_value=mock_db_entry + ) + + request = Prompt( + prompt_id="my_prompt", + litellm_params=PromptLiteLLMParams( + prompt_id="my_prompt", prompt_integration="dotprompt" + ), + prompt_info=PromptInfo(prompt_type="db", environment="production"), + ) + + with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client): + with patch( + "litellm.proxy.prompts.prompt_registry.IN_MEMORY_PROMPT_REGISTRY" + ) as mock_registry: + mock_registry.get_prompt_by_id.return_value = PromptSpec( + prompt_id="my_prompt.v1", + litellm_params=request.litellm_params, + prompt_info=PromptInfo(prompt_type="db"), + ) + mock_registry.initialize_prompt.return_value = PromptSpec( + prompt_id="my_prompt.v2", + litellm_params=request.litellm_params, + prompt_info=request.prompt_info, + environment="production", + created_by="user-update", + ) + await update_prompt( + prompt_id="my_prompt", request=request, user_api_key_dict=mock_user_auth + ) + + create_call = mock_prisma_client.db.litellm_prompttable.create.call_args + data = create_call.kwargs["data"] + assert data["environment"] == "production" + assert data["created_by"] == "user-update" + + +@pytest.mark.asyncio +async def test_delete_prompt_scoped_to_environment(): + """delete_prompt with environment param should scope deletion.""" + from unittest.mock import AsyncMock, patch + from litellm.proxy._types import UserAPIKeyAuth, LitellmUserRoles + from litellm.proxy.prompts.prompt_endpoints import delete_prompt + + mock_user_auth = UserAPIKeyAuth( + api_key="sk-1234", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + + mock_prisma_client = MagicMock() + mock_prisma_client.db.litellm_prompttable.delete_many = AsyncMock(return_value=None) + + with patch( + "litellm.proxy.prompts.prompt_registry.IN_MEMORY_PROMPT_REGISTRY" + ) as mock_registry: + prompt_spec = PromptSpec( + prompt_id="test_prompt.v1", + litellm_params=PromptLiteLLMParams( + prompt_id="test_prompt", prompt_integration="dotprompt" + ), + prompt_info=PromptInfo(prompt_type="db"), + environment="staging", + ) + mock_registry.get_prompt_by_id.return_value = prompt_spec + + with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client): + await delete_prompt( + prompt_id="test_prompt", + user_api_key_dict=mock_user_auth, + environment="staging", + ) + + mock_prisma_client.db.litellm_prompttable.delete_many.assert_called_once_with( + where={"prompt_id": "test_prompt", "environment": "staging"} + ) diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 45ec4b726a8..781b559651f 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -4,7 +4,7 @@ from typing import AsyncGenerator from unittest.mock import AsyncMock, MagicMock, patch import pytest -from fastapi import Request, status +from fastapi import HTTPException, Request, status from fastapi.responses import JSONResponse, StreamingResponse import litellm @@ -899,6 +899,33 @@ class TestCommonRequestProcessingHelpers: assert content[0] == f"data: {json.dumps(expected_error_data)}\n\n" assert content[1] == "data: [DONE]\n\n" + async def test_create_streaming_response_generator_raises_http_exception( + self, + ): + """ + Test that when a generator raises HTTPException, the response preserves + the original status code instead of hardcoding 500. + """ + mock_gen = AsyncMock() + mock_gen.__anext__.side_effect = HTTPException( + status_code=400, detail="Content blocked by guardrail" + ) + + response = await create_response(mock_gen, "text/event-stream", {}) + assert response.status_code == 400 + content = await self.consume_stream(response) + import json + + expected_error_data = { + "error": { + "message": "Content blocked by guardrail", + "code": 400, + } + } + assert len(content) == 2 + assert content[0] == f"data: {json.dumps(expected_error_data)}\n\n" + assert content[1] == "data: [DONE]\n\n" + async def test_create_streaming_response_first_chunk_error_string_code(self): """ Test that when the first chunk contains a string error code, a JSON error response is returned diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index bc13cea939e..04af5cd0086 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -1363,6 +1363,101 @@ async def test_request_guardrails_do_not_override_key_guardrails(): assert len(requested_guardrails) == 1 +@pytest.mark.asyncio +async def test_project_guardrails_merge_with_key_and_team(): + """ + Test that project guardrails are merged with key and team guardrails (union semantics). + All three levels should contribute to the final guardrails list without duplicates. + """ + request_mock = MagicMock(spec=Request) + request_mock.url.path = "/chat/completions" + request_mock.url = MagicMock() + request_mock.url.__str__.return_value = "http://localhost/chat/completions" + request_mock.method = "POST" + request_mock.query_params = {} + request_mock.headers = {"Content-Type": "application/json"} + request_mock.client = MagicMock() + request_mock.client.host = "127.0.0.1" + + data = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "test"}], + } + + user_api_key_dict = UserAPIKeyAuth( + api_key="test-key", + metadata={"guardrails": ["key-guardrail-1"]}, + team_metadata={"guardrails": ["team-guardrail-1", "key-guardrail-1"]}, + project_metadata={"guardrails": ["project-guardrail-1", "team-guardrail-1"]}, + ) + + with patch("litellm.proxy.utils._premium_user_check"): + updated_data = await add_litellm_data_to_request( + data=data, + request=request_mock, + user_api_key_dict=user_api_key_dict, + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + metadata = updated_data.get("metadata", {}) + guardrails = metadata.get("guardrails", []) + + # All three sources contribute + assert "key-guardrail-1" in guardrails + assert "team-guardrail-1" in guardrails + assert "project-guardrail-1" in guardrails + # No duplicates + assert guardrails.count("key-guardrail-1") == 1 + assert guardrails.count("team-guardrail-1") == 1 + + +@pytest.mark.asyncio +async def test_project_guardrails_only(): + """ + Test that project guardrails work when key and team have no guardrails configured. + """ + request_mock = MagicMock(spec=Request) + request_mock.url.path = "/chat/completions" + request_mock.url = MagicMock() + request_mock.url.__str__.return_value = "http://localhost/chat/completions" + request_mock.method = "POST" + request_mock.query_params = {} + request_mock.headers = {"Content-Type": "application/json"} + request_mock.client = MagicMock() + request_mock.client.host = "127.0.0.1" + + data = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "test"}], + } + + user_api_key_dict = UserAPIKeyAuth( + api_key="test-key", + metadata={}, + team_metadata={}, + project_metadata={"guardrails": ["project-guardrail-1", "project-guardrail-2"]}, + ) + + with patch("litellm.proxy.utils._premium_user_check"): + updated_data = await add_litellm_data_to_request( + data=data, + request=request_mock, + user_api_key_dict=user_api_key_dict, + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + metadata = updated_data.get("metadata", {}) + guardrails = metadata.get("guardrails", []) + + assert "project-guardrail-1" in guardrails + assert "project-guardrail-2" in guardrails + assert len(guardrails) == 2 + + def test_update_model_if_key_alias_exists(): """ Test that _update_model_if_key_alias_exists properly updates the model when a key alias exists. diff --git a/tests/test_litellm/test_anthropic_beta_headers_filtering.py b/tests/test_litellm/test_anthropic_beta_headers_filtering.py index 447419b27d7..07c19db4568 100644 --- a/tests/test_litellm/test_anthropic_beta_headers_filtering.py +++ b/tests/test_litellm/test_anthropic_beta_headers_filtering.py @@ -265,7 +265,7 @@ class TestAnthropicBetaHeadersFiltering: try: await litellm.acompletion( - model="bedrock/converse/us.anthropic.claude-3-5-sonnet-20241022-v2:0", + model="bedrock/converse/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "Hi"}], aws_access_key_id="test", aws_secret_access_key="test", diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index ce5873f5063..d19d1d1d756 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -150,8 +150,8 @@ def test_completion_missing_role(openai_api_response): "model", [ "gemini/gemini-1.5-flash", - "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", - "bedrock/invoke/anthropic.claude-3-5-sonnet-20240620-v1:0", + "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + "bedrock/invoke/anthropic.claude-haiku-4-5-20251001-v1:0", "anthropic/claude-3-5-sonnet", ], ) @@ -295,7 +295,7 @@ def test_bedrock_latency_optimized_inference(): with patch.object(client, "post") as mock_post: try: response = litellm.completion( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "Hello, how are you?"}], performanceConfig={"latency": "optimized"}, client=client, diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 73bd200f8d9..3df10901492 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -692,7 +692,7 @@ async def test_router_v1_messages_fallbacks(): { "model_name": "bedrock-claude", "litellm_params": { - "model": "anthropic.claude-3-5-sonnet-20240620-v1:0", + "model": "anthropic.claude-haiku-4-5-20251001-v1:0", "mock_response": "Hello, world I am a fallback!", }, }, diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index e984403a05b..50dc3c6c6ec 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -752,6 +752,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "supports_file_search": {"type": "boolean"}, "supports_function_calling": {"type": "boolean"}, "supports_image_input": {"type": "boolean"}, + "supports_nova_canvas_image_edit": {"type": "boolean"}, "supports_parallel_function_calling": {"type": "boolean"}, "supports_pdf_input": {"type": "boolean"}, "supports_prompt_caching": {"type": "boolean"}, @@ -1316,7 +1317,7 @@ class TestProxyFunctionCalling: ), ( "litellm_proxy/bedrock-claude-3-opus", - "bedrock/anthropic.claude-3-opus-20240229-v1:0", + "bedrock/anthropic.claude-3-7-sonnet-20250219-v1:0", False, ), ( @@ -1614,13 +1615,13 @@ class TestProxyFunctionCalling: ), ( "litellm_proxy/bedrock-claude-3-opus", - "bedrock/converse/anthropic.claude-3-opus-20240229-v1:0", + "bedrock/converse/anthropic.claude-3-7-sonnet-20250219-v1:0", False, "Bedrock Claude 3 Opus via Converse API", ), ( "litellm_proxy/bedrock-claude-3-5-sonnet", - "bedrock/converse/anthropic.claude-3-5-sonnet-20240620-v1:0", + "bedrock/converse/anthropic.claude-haiku-4-5-20251001-v1:0", False, "Bedrock Claude 3.5 Sonnet via Converse API", ), @@ -1701,7 +1702,7 @@ class TestProxyFunctionCalling: ), ( "litellm_proxy/staging-claude-opus", - "bedrock/converse/anthropic.claude-3-opus-20240229-v1:0", + "bedrock/converse/anthropic.claude-3-7-sonnet-20250219-v1:0", False, "Staging Claude Opus", ), @@ -1713,7 +1714,7 @@ class TestProxyFunctionCalling: ), ( "litellm_proxy/high-performance-claude", - "bedrock/converse/anthropic.claude-3-opus-20240229-v1:0", + "bedrock/converse/anthropic.claude-3-7-sonnet-20250219-v1:0", False, "High-performance Claude deployment", ), @@ -1851,7 +1852,7 @@ class TestProxyFunctionCalling: bedrock_models = [ "bedrock/converse/anthropic.claude-3-haiku-20240307-v1:0", "bedrock/converse/anthropic.claude-3-sonnet-20240229-v1:0", - "bedrock/converse/anthropic.claude-3-opus-20240229-v1:0", + "bedrock/converse/anthropic.claude-3-7-sonnet-20250219-v1:0", ] for model in bedrock_models: @@ -1883,13 +1884,13 @@ class TestProxyFunctionCalling: ), ( "litellm_proxy/bedrock-claude-3-opus", - "bedrock/converse/anthropic.claude-3-opus-20240229-v1:0", + "bedrock/converse/anthropic.claude-3-7-sonnet-20250219-v1:0", False, "Bedrock Claude 3 Opus via Converse API", ), ( "litellm_proxy/bedrock-claude-3-5-sonnet", - "bedrock/converse/anthropic.claude-3-5-sonnet-20240620-v1:0", + "bedrock/converse/anthropic.claude-haiku-4-5-20251001-v1:0", False, "Bedrock Claude 3.5 Sonnet via Converse API", ), @@ -1970,7 +1971,7 @@ class TestProxyFunctionCalling: ), ( "litellm_proxy/staging-claude-opus", - "bedrock/converse/anthropic.claude-3-opus-20240229-v1:0", + "bedrock/converse/anthropic.claude-3-7-sonnet-20250219-v1:0", False, "Staging Claude Opus", ), @@ -1982,7 +1983,7 @@ class TestProxyFunctionCalling: ), ( "litellm_proxy/high-performance-claude", - "bedrock/converse/anthropic.claude-3-opus-20240229-v1:0", + "bedrock/converse/anthropic.claude-3-7-sonnet-20250219-v1:0", False, "High-performance Claude deployment", ), @@ -2120,7 +2121,7 @@ class TestProxyFunctionCalling: bedrock_models = [ "bedrock/converse/anthropic.claude-3-haiku-20240307-v1:0", "bedrock/converse/anthropic.claude-3-sonnet-20240229-v1:0", - "bedrock/converse/anthropic.claude-3-opus-20240229-v1:0", + "bedrock/converse/anthropic.claude-3-7-sonnet-20250219-v1:0", ] for model in bedrock_models: @@ -2152,13 +2153,13 @@ class TestProxyFunctionCalling: ), ( "litellm_proxy/bedrock-claude-3-opus", - "bedrock/converse/anthropic.claude-3-opus-20240229-v1:0", + "bedrock/converse/anthropic.claude-3-7-sonnet-20250219-v1:0", False, "Bedrock Claude 3 Opus via Converse API", ), ( "litellm_proxy/bedrock-claude-3-5-sonnet", - "bedrock/converse/anthropic.claude-3-5-sonnet-20240620-v1:0", + "bedrock/converse/anthropic.claude-haiku-4-5-20251001-v1:0", False, "Bedrock Claude 3.5 Sonnet via Converse API", ), @@ -2239,7 +2240,7 @@ class TestProxyFunctionCalling: ), ( "litellm_proxy/staging-claude-opus", - "bedrock/converse/anthropic.claude-3-opus-20240229-v1:0", + "bedrock/converse/anthropic.claude-3-7-sonnet-20250219-v1:0", False, "Staging Claude Opus", ), @@ -2251,7 +2252,7 @@ class TestProxyFunctionCalling: ), ( "litellm_proxy/high-performance-claude", - "bedrock/converse/anthropic.claude-3-opus-20240229-v1:0", + "bedrock/converse/anthropic.claude-3-7-sonnet-20250219-v1:0", False, "High-performance Claude deployment", ), @@ -2389,7 +2390,7 @@ class TestProxyFunctionCalling: bedrock_models = [ "bedrock/converse/anthropic.claude-3-haiku-20240307-v1:0", "bedrock/converse/anthropic.claude-3-sonnet-20240229-v1:0", - "bedrock/converse/anthropic.claude-3-opus-20240229-v1:0", + "bedrock/converse/anthropic.claude-3-7-sonnet-20250219-v1:0", ] for model in bedrock_models: diff --git a/tests/test_users.py b/tests/test_users.py index 30e34a95f4f..05253a19aa5 100644 --- a/tests/test_users.py +++ b/tests/test_users.py @@ -335,7 +335,7 @@ async def setup_test_users(session: aiohttp.ClientSession) -> Tuple[Dict, Dict]: i=0, budget=100, budget_duration="30d", - models=["anthropic.claude-3-5-sonnet-20240620-v1:0"], + models=["anthropic.claude-haiku-4-5-20251001-v1:0"], ) user2 = await new_user( @@ -343,7 +343,7 @@ async def setup_test_users(session: aiohttp.ClientSession) -> Tuple[Dict, Dict]: i=1, budget=100, budget_duration="30d", - models=["anthropic.claude-3-5-sonnet-20240620-v1:0"], + models=["anthropic.claude-haiku-4-5-20251001-v1:0"], ) print("\nCreated two test users:") @@ -360,7 +360,7 @@ async def setup_test_users(session: aiohttp.ClientSession) -> Tuple[Dict, Dict]: "user_id": user1["user_id"], "duration": "7d", "key_alias": f"test_key_{uuid.uuid4()}", - "models": ["anthropic.claude-3-5-sonnet-20240620-v1:0"], + "models": ["anthropic.claude-haiku-4-5-20251001-v1:0"], } print("\nGenerating additional key for user1...") diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index 2b62c1c16bb..58b0a6b6117 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -8,66 +8,65 @@ "name": "litellm-dashboard", "version": "0.1.0", "dependencies": { - "@anthropic-ai/sdk": "^0.54.0", - "@headlessui/tailwindcss": "^0.2.0", - "@heroicons/react": "^1.0.6", - "@remixicon/react": "^4.1.1", - "@tanstack/react-pacer": "^0.2.0", - "@tanstack/react-query": "^5.64.1", - "@tanstack/react-table": "^8.20.6", - "@tremor/react": "^3.13.3", - "@types/papaparse": "^5.3.15", - "antd": "^5.13.2", - "cva": "^1.0.0-beta.3", - "dayjs": "^1.11.19", - "jwt-decode": "^4.0.0", - "lucide-react": "^0.513.0", - "moment": "^2.30.1", - "next": "^16.1.7", - "openai": "^4.93.0", - "papaparse": "^5.5.2", - "react": "^18.3.1", - "react-copy-to-clipboard": "^5.1.0", - "react-dom": "^18.3.1", - "react-json-view-lite": "^2.5.0", - "react-markdown": "^9.0.1", - "react-syntax-highlighter": "^15.6.6", - "remark-gfm": "^4.0.1", - "tailwind-merge": "^3.2.0", - "uuid": "^11.1.0" + "@anthropic-ai/sdk": "0.54.0", + "@headlessui/tailwindcss": "0.2.2", + "@heroicons/react": "1.0.6", + "@remixicon/react": "4.9.0", + "@tanstack/react-pacer": "0.2.0", + "@tanstack/react-query": "5.90.20", + "@tanstack/react-table": "8.21.3", + "@tremor/react": "3.18.7", + "@types/papaparse": "5.5.2", + "antd": "5.29.3", + "cva": "1.0.0-beta.4", + "dayjs": "1.11.19", + "jwt-decode": "4.0.0", + "lucide-react": "0.513.0", + "moment": "2.30.1", + "next": "16.1.7", + "openai": "4.104.0", + "papaparse": "5.5.3", + "react": "18.3.1", + "react-copy-to-clipboard": "5.1.0", + "react-dom": "18.3.1", + "react-json-view-lite": "2.5.0", + "react-markdown": "9.1.0", + "react-syntax-highlighter": "15.6.6", + "remark-gfm": "4.0.1", + "tailwind-merge": "3.4.0", + "uuid": "11.1.0" }, "devDependencies": { - "@neondatabase/api-client": "^2.6.0", - "@playwright/test": "^1.57.0", - "@tailwindcss/forms": "^0.5.7", - "@testing-library/dom": "^10.4.1", - "@testing-library/jest-dom": "^6.8.0", - "@testing-library/react": "^16.3.0", - "@testing-library/user-event": "^14.6.1", - "@types/babel__traverse": "^7.28.0", - "@types/lodash": "^4.17.15", + "@playwright/test": "1.58.1", + "@tailwindcss/forms": "0.5.11", + "@testing-library/dom": "10.4.1", + "@testing-library/jest-dom": "6.9.1", + "@testing-library/react": "16.3.2", + "@testing-library/user-event": "14.6.1", + "@types/babel__traverse": "7.28.0", + "@types/lodash": "4.17.23", "@types/node": "20.19.37", "@types/react": "18.2.48", - "@types/react-copy-to-clipboard": "^5.0.7", - "@types/react-dom": "^18", - "@types/react-syntax-highlighter": "^15.5.11", - "@types/uuid": "^10.0.0", - "@vitest/coverage-v8": "^3.2.4", - "@vitest/ui": "^3.2.4", - "autoprefixer": "^10.4.17", - "dotenv": "^17.2.3", - "eslint": "^9.39.2", + "@types/react-copy-to-clipboard": "5.0.7", + "@types/react-dom": "18.3.7", + "@types/react-syntax-highlighter": "15.5.13", + "@types/uuid": "10.0.0", + "@vitest/coverage-v8": "3.2.4", + "@vitest/ui": "3.2.4", + "autoprefixer": "10.4.24", + "dotenv": "17.2.3", + "eslint": "9.39.2", "eslint-config-next": "15.5.10", - "eslint-config-prettier": "^10.1.8", - "eslint-plugin-unused-imports": "^4.2.0", - "jsdom": "^27.0.0", - "knip": "^5.83.1", - "postcss": "^8.4.33", + "eslint-config-prettier": "10.1.8", + "eslint-plugin-unused-imports": "4.3.0", + "jsdom": "27.4.0", + "knip": "5.83.1", + "postcss": "8.5.6", "prettier": "3.2.5", - "tailwindcss": "^3.4.1", + "tailwindcss": "3.4.19", "typescript": "5.9.3", - "vite": "^7.1.11", - "vitest": "^3.2.4" + "vite": "7.3.1", + "vitest": "3.2.4" }, "engines": { "node": ">=18.17.0", @@ -1822,16 +1821,6 @@ "@tybys/wasm-util": "^0.10.0" } }, - "node_modules/@neondatabase/api-client": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/@neondatabase/api-client/-/api-client-2.6.0.tgz", - "integrity": "sha512-NxKE+EFcVwxXU3jj8I/WgueXSyzrXV85AV0nb2SeoKtOa3dlEcTylsdOsMsMeZZeFfQXLyiCOm2nAduGZn9olA==", - "dev": true, - "license": "MIT", - "dependencies": { - "axios": "^1.9.0" - } - }, "node_modules/@next/env": { "version": "16.1.7", "resolved": "https://registry.npmjs.org/@next/env/-/env-16.1.7.tgz", @@ -4684,18 +4673,6 @@ "node": ">=4" } }, - "node_modules/axios": { - "version": "1.13.6", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.13.6.tgz", - "integrity": "sha512-ChTCHMouEe2kn713WHbQGcuYrr6fXTBiu460OTwWrWob16g1bXn4vtz07Ope7ewMozJAnEquLk5lWQWtBig9DQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "follow-redirects": "^1.15.11", - "form-data": "^4.0.5", - "proxy-from-env": "^1.1.0" - } - }, "node_modules/axobject-query": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-4.1.0.tgz", @@ -6630,27 +6607,6 @@ "dev": true, "license": "ISC" }, - "node_modules/follow-redirects": { - "version": "1.15.11", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.11.tgz", - "integrity": "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==", - "dev": true, - "funding": [ - { - "type": "individual", - "url": "https://github.com/sponsors/RubenVerborgh" - } - ], - "license": "MIT", - "engines": { - "node": ">=4.0" - }, - "peerDependenciesMeta": { - "debug": { - "optional": true - } - } - }, "node_modules/for-each": { "version": "0.3.5", "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.5.tgz", @@ -10281,13 +10237,6 @@ "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/proxy-from-env": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", - "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", - "dev": true, - "license": "MIT" - }, "node_modules/punycode": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", diff --git a/ui/litellm-dashboard/scripts/e2e_tests/neonHelperScripts.ts b/ui/litellm-dashboard/scripts/e2e_tests/neonHelperScripts.ts deleted file mode 100644 index 089ad4e7926..00000000000 --- a/ui/litellm-dashboard/scripts/e2e_tests/neonHelperScripts.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { createApiClient, EndpointType } from "@neondatabase/api-client"; -import { config } from "dotenv"; -import { resolve } from "path"; - -const envPaths = [ - resolve(process.cwd(), "../../.env"), // project root -]; - -for (const envPath of envPaths) { - config({ path: envPath }); -} - -const NEON_API_KEY = process.env.NEON_API_KEY!; -const PROJECT_ID = process.env.NEON_PROJECT_ID!; -const PARENT_BRANCH = process.env.NEON_PARENT_BRANCH_ID!; -const NEON_E2E_UI_TEST_DB_NAME = process.env.NEON_E2E_UI_TEST_DB_NAME!; - -const apiClient = createApiClient({ - apiKey: NEON_API_KEY, -}); - -export async function createNeonE2ETestingBranch(projectId: string, parentBranchId?: string, expireAt?: string) { - try { - const response = await apiClient.createProjectBranch(projectId, { - branch: { - name: `e2e-local-${crypto.randomUUID()}`, - parent_id: parentBranchId, - expires_at: expireAt ?? new Date(Date.now() + 1000 * 60 * 30).toISOString(), - }, - endpoints: [ - { - type: EndpointType.ReadWrite, - autoscaling_limit_min_cu: 0.25, - autoscaling_limit_max_cu: 1, - }, - ], - }); - return response; - } catch (error) { - throw error; - } -} - -export async function getNeonE2ETestingBranchConnectionString() { - const createBranchResponse = await createNeonE2ETestingBranch(PROJECT_ID, PARENT_BRANCH); - const projectId = createBranchResponse.data.branch.project_id; - const response = await apiClient.getConnectionUri({ - database_name: NEON_E2E_UI_TEST_DB_NAME, - role_name: "neondb_owner", - projectId: projectId, - }); - console.log("connection string:", response.data.uri); - return response.data.uri; -} - -getNeonE2ETestingBranchConnectionString(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/projects/useCreateProject.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/projects/useCreateProject.ts index 3943f23794e..e206c770b19 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/projects/useCreateProject.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/projects/useCreateProject.ts @@ -17,6 +17,7 @@ export interface ProjectCreateParams { models?: string[]; max_budget?: number; blocked?: boolean; + guardrails?: string[]; metadata?: Record; model_rpm_limit?: Record; model_tpm_limit?: Record; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/projects/useUpdateProject.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/projects/useUpdateProject.ts index e6cd3071f5f..2042c8fc7cd 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/projects/useUpdateProject.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/projects/useUpdateProject.ts @@ -17,6 +17,7 @@ export interface ProjectUpdateParams { models?: string[]; max_budget?: number; blocked?: boolean; + guardrails?: string[]; metadata?: Record; model_rpm_limit?: Record; model_tpm_limit?: Record; diff --git a/ui/litellm-dashboard/src/components/Projects/ProjectModals/EditProjectModal.tsx b/ui/litellm-dashboard/src/components/Projects/ProjectModals/EditProjectModal.tsx index dc3b43ef73c..6c65e518cfd 100644 --- a/ui/litellm-dashboard/src/components/Projects/ProjectModals/EditProjectModal.tsx +++ b/ui/litellm-dashboard/src/components/Projects/ProjectModals/EditProjectModal.tsx @@ -33,6 +33,9 @@ export function EditProjectModal({ const metadataObj = (project.metadata ?? {}) as Record; const rpmLimits = (metadataObj.model_rpm_limit ?? {}) as Record; const tpmLimits = (metadataObj.model_tpm_limit ?? {}) as Record; + const guardrails = (Array.isArray(metadataObj.guardrails) + ? metadataObj.guardrails + : []) as string[]; const modelLimits: ProjectFormValues["modelLimits"] = []; const allLimitModels = new Set([ @@ -48,7 +51,7 @@ export function EditProjectModal({ } // Filter out internal keys from user-facing metadata - const internalKeys = new Set(["model_rpm_limit", "model_tpm_limit"]); + const internalKeys = new Set(["model_rpm_limit", "model_tpm_limit", "guardrails"]); const metadata: ProjectFormValues["metadata"] = []; for (const [key, value] of Object.entries(metadataObj)) { if (!internalKeys.has(key)) { @@ -63,6 +66,7 @@ export function EditProjectModal({ models: project.models ?? [], max_budget: project.litellm_budget_table?.max_budget ?? undefined, isBlocked: project.blocked, + guardrails: guardrails.length > 0 ? guardrails : undefined, modelLimits: modelLimits.length > 0 ? modelLimits : undefined, metadata: metadata.length > 0 ? metadata : undefined, }); diff --git a/ui/litellm-dashboard/src/components/Projects/ProjectModals/ProjectBaseForm.test.tsx b/ui/litellm-dashboard/src/components/Projects/ProjectModals/ProjectBaseForm.test.tsx index 04e3ed64f47..d8532146566 100644 --- a/ui/litellm-dashboard/src/components/Projects/ProjectModals/ProjectBaseForm.test.tsx +++ b/ui/litellm-dashboard/src/components/Projects/ProjectModals/ProjectBaseForm.test.tsx @@ -14,6 +14,10 @@ vi.mock("@/components/organisms/create_key_button", () => ({ fetchTeamModels: vi.fn().mockResolvedValue([]), })); +vi.mock("@/components/networking", () => ({ + getGuardrailsList: vi.fn().mockResolvedValue({ guardrails: [] }), +})); + vi.mock("@/components/key_team_helpers/fetch_available_models_team_key", () => ({ getModelDisplayName: (model: string) => model, })); @@ -86,4 +90,13 @@ describe("ProjectBaseForm", () => { renderWithProviders(); expect(screen.getByText("Advanced Settings")).toBeInTheDocument(); }); + + it("should show a Guardrails field in the Advanced Settings section", async () => { + const user = userEvent.setup(); + renderWithProviders(); + await user.click(screen.getByText("Advanced Settings")); + await waitFor(() => { + expect(screen.getByText("Guardrails")).toBeInTheDocument(); + }); + }); }); diff --git a/ui/litellm-dashboard/src/components/Projects/ProjectModals/ProjectBaseForm.tsx b/ui/litellm-dashboard/src/components/Projects/ProjectModals/ProjectBaseForm.tsx index bf1eca882c3..81d8fabe084 100644 --- a/ui/litellm-dashboard/src/components/Projects/ProjectModals/ProjectBaseForm.tsx +++ b/ui/litellm-dashboard/src/components/Projects/ProjectModals/ProjectBaseForm.tsx @@ -22,6 +22,7 @@ import { useTeams } from "@/app/(dashboard)/hooks/teams/useTeams"; import { Team } from "../../key_team_helpers/key_list"; import { fetchTeamModels } from "../../organisms/create_key_button"; import { getModelDisplayName } from "../../key_team_helpers/fetch_available_models_team_key"; +import { getGuardrailsList } from "@/components/networking"; export interface ProjectFormValues { project_alias: string; @@ -30,6 +31,7 @@ export interface ProjectFormValues { models: string[]; max_budget?: number; isBlocked: boolean; + guardrails?: string[]; modelLimits?: { model: string; tpm?: number; rpm?: number }[]; metadata?: { key: string; value: string }[]; } @@ -46,6 +48,23 @@ export function ProjectBaseForm({ const [selectedTeam, setSelectedTeam] = useState(null); const [modelsToPick, setModelsToPick] = useState([]); + const [guardrailsList, setGuardrailsList] = useState([]); + + useEffect(() => { + const fetchGuardrails = async () => { + if (!accessToken) return; + try { + const response = await getGuardrailsList(accessToken); + const names = response.guardrails.map( + (g: { guardrail_name: string }) => g.guardrail_name + ); + setGuardrailsList(names); + } catch (error) { + console.error("Failed to fetch guardrails:", error); + } + }; + fetchGuardrails(); + }, [accessToken]); // Sync selectedTeam from form value (needed for edit mode pre-fill) const teamIdValue = Form.useWatch("team_id", form); @@ -259,6 +278,24 @@ export function ProjectBaseForm({ + + setSelectedEnvironment(value)} + style={{ width: 180 }} + options={[ + { label: "Development", value: "development" }, + { label: "Staging", value: "staging" }, + { label: "Production", value: "production" }, + ]} + /> void; } const PromptEditorHeader: React.FC = ({ @@ -35,6 +37,8 @@ const PromptEditorHeader: React.FC = ({ promptVariables = {}, accessToken, proxySettings, + environment, + onEnvironmentChange, }) => { return (
@@ -53,6 +57,17 @@ const PromptEditorHeader: React.FC = ({ {version} )} +