From 006d48102563de962f62a8a118a4aa3919858f90 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 1 Apr 2026 16:15:32 -0700 Subject: [PATCH 01/25] [Fix] Remove neon CLI dependency and pin all JS dependencies Remove @neondatabase/api-client and neonctl to address CVE-2026-25639 (axios supply chain vulnerability). Pin all JS dependencies to exact versions across all package.json files to prevent future supply chain attacks via semver range resolution. Co-Authored-By: Claude Opus 4.6 (1M context) --- .circleci/config.yml | 24 +-- ci_cd/security_scans.sh | 1 - docs/my-website/package-lock.json | 25 +-- docs/my-website/package.json | 18 +- litellm-js/proxy/package.json | 8 +- litellm-js/spend-logs/package-lock.json | 8 +- litellm-js/spend-logs/package.json | 8 +- package.json | 14 +- tests/proxy_admin_ui_tests/package.json | 4 +- .../ui_unit_tests/package.json | 28 ++-- ui/litellm-dashboard/package-lock.json | 155 ++++++------------ ui/litellm-dashboard/package.json | 105 ++++++------ .../scripts/e2e_tests/neonHelperScripts.ts | 56 ------- 13 files changed, 158 insertions(+), 296 deletions(-) delete mode 100644 ui/litellm-dashboard/scripts/e2e_tests/neonHelperScripts.ts diff --git a/.circleci/config.yml b/.circleci/config.yml index 5bbabad9107..307247651e7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3233,31 +3233,9 @@ jobs: name: Install Playwright Browsers command: | npx playwright install - - run: - name: Install Neon CLI - command: | - npm i -g neonctl - - run: - name: Create Neon branch - command: | - export EXPIRES_AT=$(date -u -d "+3 hours" +"%Y-%m-%dT%H:%M:%SZ") - echo "Expires at: $EXPIRES_AT" - neon branches create \ - --project-id $NEON_PROJECT_ID \ - --name preview/commit-${CIRCLE_SHA1:0:7}-<< parameters.browser >> \ - --expires-at $EXPIRES_AT \ - --parent br-fancy-paper-ad1olsb3 \ - --api-key $NEON_API_KEY || true - run: name: Run Docker container command: | - E2E_UI_TEST_DATABASE_URL=$(neon connection-string \ - --project-id $NEON_PROJECT_ID \ - --api-key $NEON_API_KEY \ - --branch preview/commit-${CIRCLE_SHA1:0:7}-<< parameters.browser >> \ - --database-name yuneng-trial-db \ - --role neondb_owner) - echo $E2E_UI_TEST_DATABASE_URL docker run -d \ -p 4000:4000 \ -e DATABASE_URL=$E2E_UI_TEST_DATABASE_URL \ @@ -3531,7 +3509,7 @@ workflows: only: - main - /litellm_.*/ - # - e2e_ui_testing: # migrate to dynamic db - currently requires neon cli + # - e2e_ui_testing: # name: e2e_ui_testing_chromium # browser: chromium # context: e2e_ui_tests diff --git a/ci_cd/security_scans.sh b/ci_cd/security_scans.sh index ee33393c8b1..2138fca6cd5 100755 --- a/ci_cd/security_scans.sh +++ b/ci_cd/security_scans.sh @@ -160,7 +160,6 @@ run_grype_scans() { "CVE-2026-0775" # npm cli incorrect permission assignment - no fix available yet, npm is only used at build/prisma-generate time "GHSA-3ppc-4f35-3m26" # minimatch ReDoS via repeated wildcards - from nodejs_wheel bundled npm, not used in application runtime code "GHSA-83g3-92jg-28cx" # tar arbitrary file read/write via hardlink - from nodejs_wheel bundled npm, not used in application runtime code - "CVE-2026-25639" # axios DoS via __proto__ in mergeConfig - transitive dev dep via @neondatabase/api-client, not imported in application code "CVE-2026-2297" # Python 3.13 SourcelessFileLoader audit hook bypass - no fix available in base image "GHSA-qffp-2rhf-9h96" # tar hardlink path traversal - from nodejs_wheel bundled npm, not used in application runtime code "CVE-2026-2673" # OpenSSL 3.6.1 TLS 1.3 key exchange group negotiation issue - no fix available yet diff --git a/docs/my-website/package-lock.json b/docs/my-website/package-lock.json index a3e9cb61428..56684b737de 100644 --- a/docs/my-website/package-lock.json +++ b/docs/my-website/package-lock.json @@ -13,18 +13,18 @@ "@docusaurus/plugin-ideal-image": "3.8.1", "@docusaurus/preset-classic": "3.8.1", "@docusaurus/theme-mermaid": "3.8.1", - "@inkeep/cxkit-docusaurus": "^0.5.89", - "@mdx-js/react": "^3.0.0", - "clsx": "^1.2.1", - "prism-react-renderer": "^1.3.5", - "react": "^18.0.0 || ^19.0.0", - "react-dom": "^18.0.0 || ^19.0.0", - "sharp": "^0.32.6", - "uuid": "^9.0.1" + "@inkeep/cxkit-docusaurus": "0.5.107", + "@mdx-js/react": "3.1.1", + "clsx": "1.2.1", + "prism-react-renderer": "1.3.5", + "react": "18.3.1", + "react-dom": "18.3.1", + "sharp": "0.32.6", + "uuid": "9.0.1" }, "devDependencies": { "@docusaurus/module-type-aliases": "3.8.1", - "dotenv": "^16.4.5" + "dotenv": "16.6.1" }, "engines": { "node": ">=16.14", @@ -20403,13 +20403,6 @@ "url": "https://opencollective.com/webpack" } }, - "node_modules/search-insights": { - "version": "2.17.3", - "resolved": "https://registry.npmjs.org/search-insights/-/search-insights-2.17.3.tgz", - "integrity": "sha512-RQPdCYTa8A68uM2jwxoY842xDhvx3E5LFL1LxvxCNMev4o5mLuokczhzjAgGwUZBAmOKZknArSxLKmXtIi2AxQ==", - "license": "MIT", - "peer": true - }, "node_modules/section-matter": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/section-matter/-/section-matter-1.0.0.tgz", diff --git a/docs/my-website/package.json b/docs/my-website/package.json index 2802d24b57e..73ff62dcb43 100644 --- a/docs/my-website/package.json +++ b/docs/my-website/package.json @@ -19,18 +19,18 @@ "@docusaurus/plugin-ideal-image": "3.8.1", "@docusaurus/preset-classic": "3.8.1", "@docusaurus/theme-mermaid": "3.8.1", - "@inkeep/cxkit-docusaurus": "^0.5.89", - "@mdx-js/react": "^3.0.0", - "clsx": "^1.2.1", - "prism-react-renderer": "^1.3.5", - "react": "^18.0.0 || ^19.0.0", - "react-dom": "^18.0.0 || ^19.0.0", - "sharp": "^0.32.6", - "uuid": "^9.0.1" + "@inkeep/cxkit-docusaurus": "0.5.107", + "@mdx-js/react": "3.1.1", + "clsx": "1.2.1", + "prism-react-renderer": "1.3.5", + "react": "18.3.1", + "react-dom": "18.3.1", + "sharp": "0.32.6", + "uuid": "9.0.1" }, "devDependencies": { "@docusaurus/module-type-aliases": "3.8.1", - "dotenv": "^16.4.5" + "dotenv": "16.6.1" }, "browserslist": { "production": [ diff --git a/litellm-js/proxy/package.json b/litellm-js/proxy/package.json index f63cf36d2ed..7195c6b3601 100644 --- a/litellm-js/proxy/package.json +++ b/litellm-js/proxy/package.json @@ -4,11 +4,11 @@ "deploy": "wrangler deploy --minify src/index.ts" }, "dependencies": { - "hono": "^4.1.4", - "openai": "^4.29.2" + "hono": "4.1.4", + "openai": "4.29.2" }, "devDependencies": { - "@cloudflare/workers-types": "^4.20240208.0", - "wrangler": "^3.32.0" + "@cloudflare/workers-types": "4.20240208.0", + "wrangler": "3.32.0" } } diff --git a/litellm-js/spend-logs/package-lock.json b/litellm-js/spend-logs/package-lock.json index b24ff0a4940..3784fb78658 100644 --- a/litellm-js/spend-logs/package-lock.json +++ b/litellm-js/spend-logs/package-lock.json @@ -5,12 +5,12 @@ "packages": { "": { "dependencies": { - "@hono/node-server": "^1.10.1", - "hono": "^4.12.7" + "@hono/node-server": "1.19.6", + "hono": "4.12.7" }, "devDependencies": { - "@types/node": "^20.11.17", - "tsx": "^4.7.1" + "@types/node": "20.19.25", + "tsx": "4.20.6" } }, "node_modules/@esbuild/aix-ppc64": { diff --git a/litellm-js/spend-logs/package.json b/litellm-js/spend-logs/package.json index 8bccbdba27b..742dafe2b8e 100644 --- a/litellm-js/spend-logs/package.json +++ b/litellm-js/spend-logs/package.json @@ -3,11 +3,11 @@ "dev": "tsx watch src/index.ts" }, "dependencies": { - "@hono/node-server": "^1.10.1", - "hono": "^4.12.7" + "@hono/node-server": "1.19.6", + "hono": "4.12.7" }, "devDependencies": { - "@types/node": "^20.11.17", - "tsx": "^4.7.1" + "@types/node": "20.19.25", + "tsx": "4.20.6" } } diff --git a/package.json b/package.json index 9a7ebf4ae05..84e6e51f86b 100644 --- a/package.json +++ b/package.json @@ -1,14 +1,14 @@ { "dependencies": { - "prism-react-renderer": "^2.4.1", - "prisma": "^5.17.0", - "react-copy-to-clipboard": "^5.1.0" + "prism-react-renderer": "2.4.1", + "prisma": "5.17.0", + "react-copy-to-clipboard": "5.1.0" }, "devDependencies": { - "@testing-library/jest-dom": "^6.8.0", - "@testing-library/react": "^14.3.1", - "@types/react-copy-to-clipboard": "^5.0.7", - "jest": "^29.7.0" + "@testing-library/jest-dom": "6.8.0", + "@testing-library/react": "14.3.1", + "@types/react-copy-to-clipboard": "5.0.7", + "jest": "29.7.0" }, "overrides": { "glob": "13.0.0", diff --git a/tests/proxy_admin_ui_tests/package.json b/tests/proxy_admin_ui_tests/package.json index 20dfed7a8a5..5933490fb1d 100644 --- a/tests/proxy_admin_ui_tests/package.json +++ b/tests/proxy_admin_ui_tests/package.json @@ -8,7 +8,7 @@ "author": "", "license": "ISC", "devDependencies": { - "@playwright/test": "^1.47.2", - "@types/node": "^22.5.5" + "@playwright/test": "1.56.1", + "@types/node": "22.19.1" } } diff --git a/tests/proxy_admin_ui_tests/ui_unit_tests/package.json b/tests/proxy_admin_ui_tests/ui_unit_tests/package.json index c6a3e6e260e..1c6dce56afe 100644 --- a/tests/proxy_admin_ui_tests/ui_unit_tests/package.json +++ b/tests/proxy_admin_ui_tests/ui_unit_tests/package.json @@ -6,22 +6,22 @@ "test:watch": "jest --watch" }, "devDependencies": { - "@testing-library/react": "^14.0.0", - "@testing-library/jest-dom": "^6.0.0", - "@types/jest": "^29.5.0", - "@types/react": "^18.2.0", - "@types/react-dom": "^18.2.0", - "identity-obj-proxy": "^3.0.0", - "jest": "^29.5.0", - "jest-environment-jsdom": "^29.5.0", - "ts-jest": "^29.1.0", - "typescript": "^5.0.0" + "@testing-library/react": "14.3.1", + "@testing-library/jest-dom": "6.9.1", + "@types/jest": "29.5.14", + "@types/react": "18.3.27", + "@types/react-dom": "18.3.7", + "identity-obj-proxy": "3.0.0", + "jest": "29.7.0", + "jest-environment-jsdom": "29.7.0", + "ts-jest": "29.4.5", + "typescript": "5.9.3" }, "dependencies": { - "antd": "^5.12.5", - "@ant-design/icons": "^5.0.0", - "react": "^18.2.0", - "react-dom": "^18.2.0" + "antd": "5.29.1", + "@ant-design/icons": "5.6.1", + "react": "18.3.1", + "react-dom": "18.3.1" }, "overrides": { "glob": "13.0.0", diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index 2b62c1c16bb..58b0a6b6117 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -8,66 +8,65 @@ "name": "litellm-dashboard", "version": "0.1.0", "dependencies": { - "@anthropic-ai/sdk": "^0.54.0", - "@headlessui/tailwindcss": "^0.2.0", - "@heroicons/react": "^1.0.6", - "@remixicon/react": "^4.1.1", - "@tanstack/react-pacer": "^0.2.0", - "@tanstack/react-query": "^5.64.1", - "@tanstack/react-table": "^8.20.6", - "@tremor/react": "^3.13.3", - "@types/papaparse": "^5.3.15", - "antd": "^5.13.2", - "cva": "^1.0.0-beta.3", - "dayjs": "^1.11.19", - "jwt-decode": "^4.0.0", - "lucide-react": "^0.513.0", - "moment": "^2.30.1", - "next": "^16.1.7", - "openai": "^4.93.0", - "papaparse": "^5.5.2", - "react": "^18.3.1", - "react-copy-to-clipboard": "^5.1.0", - "react-dom": "^18.3.1", - "react-json-view-lite": "^2.5.0", - "react-markdown": "^9.0.1", - "react-syntax-highlighter": "^15.6.6", - "remark-gfm": "^4.0.1", - "tailwind-merge": "^3.2.0", - "uuid": "^11.1.0" + "@anthropic-ai/sdk": "0.54.0", + "@headlessui/tailwindcss": "0.2.2", + "@heroicons/react": "1.0.6", + "@remixicon/react": "4.9.0", + "@tanstack/react-pacer": "0.2.0", + "@tanstack/react-query": "5.90.20", + "@tanstack/react-table": "8.21.3", + "@tremor/react": "3.18.7", + "@types/papaparse": "5.5.2", + "antd": "5.29.3", + "cva": "1.0.0-beta.4", + "dayjs": "1.11.19", + "jwt-decode": "4.0.0", + "lucide-react": "0.513.0", + "moment": "2.30.1", + "next": "16.1.7", + "openai": "4.104.0", + "papaparse": "5.5.3", + "react": "18.3.1", + "react-copy-to-clipboard": "5.1.0", + "react-dom": "18.3.1", + "react-json-view-lite": "2.5.0", + "react-markdown": "9.1.0", + "react-syntax-highlighter": "15.6.6", + "remark-gfm": "4.0.1", + "tailwind-merge": "3.4.0", + "uuid": "11.1.0" }, "devDependencies": { - "@neondatabase/api-client": "^2.6.0", - "@playwright/test": "^1.57.0", - "@tailwindcss/forms": "^0.5.7", - "@testing-library/dom": "^10.4.1", - "@testing-library/jest-dom": "^6.8.0", - "@testing-library/react": "^16.3.0", - "@testing-library/user-event": "^14.6.1", - "@types/babel__traverse": "^7.28.0", - "@types/lodash": "^4.17.15", + "@playwright/test": "1.58.1", + "@tailwindcss/forms": "0.5.11", + "@testing-library/dom": "10.4.1", + "@testing-library/jest-dom": "6.9.1", + "@testing-library/react": "16.3.2", + "@testing-library/user-event": "14.6.1", + "@types/babel__traverse": "7.28.0", + "@types/lodash": "4.17.23", "@types/node": "20.19.37", "@types/react": "18.2.48", - "@types/react-copy-to-clipboard": "^5.0.7", - "@types/react-dom": "^18", - "@types/react-syntax-highlighter": "^15.5.11", - "@types/uuid": "^10.0.0", - "@vitest/coverage-v8": "^3.2.4", - "@vitest/ui": "^3.2.4", - "autoprefixer": "^10.4.17", - "dotenv": "^17.2.3", - "eslint": "^9.39.2", + "@types/react-copy-to-clipboard": "5.0.7", + "@types/react-dom": "18.3.7", + "@types/react-syntax-highlighter": "15.5.13", + "@types/uuid": "10.0.0", + "@vitest/coverage-v8": "3.2.4", + "@vitest/ui": "3.2.4", + "autoprefixer": "10.4.24", + "dotenv": "17.2.3", + "eslint": "9.39.2", "eslint-config-next": "15.5.10", - "eslint-config-prettier": "^10.1.8", - "eslint-plugin-unused-imports": "^4.2.0", - "jsdom": "^27.0.0", - "knip": "^5.83.1", - "postcss": "^8.4.33", + "eslint-config-prettier": "10.1.8", + "eslint-plugin-unused-imports": "4.3.0", + "jsdom": "27.4.0", + "knip": "5.83.1", + "postcss": "8.5.6", "prettier": "3.2.5", - "tailwindcss": "^3.4.1", + "tailwindcss": "3.4.19", "typescript": "5.9.3", - "vite": "^7.1.11", - "vitest": "^3.2.4" + "vite": "7.3.1", + "vitest": "3.2.4" }, "engines": { "node": ">=18.17.0", @@ -1822,16 +1821,6 @@ "@tybys/wasm-util": "^0.10.0" } }, - "node_modules/@neondatabase/api-client": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/@neondatabase/api-client/-/api-client-2.6.0.tgz", - "integrity": "sha512-NxKE+EFcVwxXU3jj8I/WgueXSyzrXV85AV0nb2SeoKtOa3dlEcTylsdOsMsMeZZeFfQXLyiCOm2nAduGZn9olA==", - "dev": true, - "license": "MIT", - "dependencies": { - "axios": "^1.9.0" - } - }, "node_modules/@next/env": { "version": "16.1.7", "resolved": "https://registry.npmjs.org/@next/env/-/env-16.1.7.tgz", @@ -4684,18 +4673,6 @@ "node": ">=4" } }, - "node_modules/axios": { - "version": "1.13.6", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.13.6.tgz", - "integrity": "sha512-ChTCHMouEe2kn713WHbQGcuYrr6fXTBiu460OTwWrWob16g1bXn4vtz07Ope7ewMozJAnEquLk5lWQWtBig9DQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "follow-redirects": "^1.15.11", - "form-data": "^4.0.5", - "proxy-from-env": "^1.1.0" - } - }, "node_modules/axobject-query": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-4.1.0.tgz", @@ -6630,27 +6607,6 @@ "dev": true, "license": "ISC" }, - "node_modules/follow-redirects": { - "version": "1.15.11", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.11.tgz", - "integrity": "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==", - "dev": true, - "funding": [ - { - "type": "individual", - "url": "https://github.com/sponsors/RubenVerborgh" - } - ], - "license": "MIT", - "engines": { - "node": ">=4.0" - }, - "peerDependenciesMeta": { - "debug": { - "optional": true - } - } - }, "node_modules/for-each": { "version": "0.3.5", "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.5.tgz", @@ -10281,13 +10237,6 @@ "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/proxy-from-env": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", - "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", - "dev": true, - "license": "MIT" - }, "node_modules/punycode": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index 7b763eaa66a..dfad2a1e795 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -20,66 +20,65 @@ "knip:fix": "knip --fix" }, "dependencies": { - "@anthropic-ai/sdk": "^0.54.0", - "@headlessui/tailwindcss": "^0.2.0", - "@heroicons/react": "^1.0.6", - "@remixicon/react": "^4.1.1", - "@tanstack/react-pacer": "^0.2.0", - "@tanstack/react-query": "^5.64.1", - "@tanstack/react-table": "^8.20.6", - "@tremor/react": "^3.13.3", - "@types/papaparse": "^5.3.15", - "antd": "^5.13.2", - "cva": "^1.0.0-beta.3", - "dayjs": "^1.11.19", - "jwt-decode": "^4.0.0", - "lucide-react": "^0.513.0", - "moment": "^2.30.1", - "next": "^16.1.7", - "openai": "^4.93.0", - "papaparse": "^5.5.2", - "react": "^18.3.1", - "react-copy-to-clipboard": "^5.1.0", - "react-dom": "^18.3.1", - "react-json-view-lite": "^2.5.0", - "react-markdown": "^9.0.1", - "react-syntax-highlighter": "^15.6.6", - "remark-gfm": "^4.0.1", - "tailwind-merge": "^3.2.0", - "uuid": "^11.1.0" + "@anthropic-ai/sdk": "0.54.0", + "@headlessui/tailwindcss": "0.2.2", + "@heroicons/react": "1.0.6", + "@remixicon/react": "4.9.0", + "@tanstack/react-pacer": "0.2.0", + "@tanstack/react-query": "5.90.20", + "@tanstack/react-table": "8.21.3", + "@tremor/react": "3.18.7", + "@types/papaparse": "5.5.2", + "antd": "5.29.3", + "cva": "1.0.0-beta.4", + "dayjs": "1.11.19", + "jwt-decode": "4.0.0", + "lucide-react": "0.513.0", + "moment": "2.30.1", + "next": "16.1.7", + "openai": "4.104.0", + "papaparse": "5.5.3", + "react": "18.3.1", + "react-copy-to-clipboard": "5.1.0", + "react-dom": "18.3.1", + "react-json-view-lite": "2.5.0", + "react-markdown": "9.1.0", + "react-syntax-highlighter": "15.6.6", + "remark-gfm": "4.0.1", + "tailwind-merge": "3.4.0", + "uuid": "11.1.0" }, "devDependencies": { - "@neondatabase/api-client": "^2.6.0", - "@playwright/test": "^1.57.0", - "@tailwindcss/forms": "^0.5.7", - "@testing-library/dom": "^10.4.1", - "@testing-library/jest-dom": "^6.8.0", - "@testing-library/react": "^16.3.0", - "@testing-library/user-event": "^14.6.1", - "@types/babel__traverse": "^7.28.0", - "@types/lodash": "^4.17.15", + "@playwright/test": "1.58.1", + "@tailwindcss/forms": "0.5.11", + "@testing-library/dom": "10.4.1", + "@testing-library/jest-dom": "6.9.1", + "@testing-library/react": "16.3.2", + "@testing-library/user-event": "14.6.1", + "@types/babel__traverse": "7.28.0", + "@types/lodash": "4.17.23", "@types/node": "20.19.37", "@types/react": "18.2.48", - "@types/react-copy-to-clipboard": "^5.0.7", - "@types/react-dom": "^18", - "@types/react-syntax-highlighter": "^15.5.11", - "@types/uuid": "^10.0.0", - "@vitest/coverage-v8": "^3.2.4", - "@vitest/ui": "^3.2.4", - "autoprefixer": "^10.4.17", - "dotenv": "^17.2.3", - "eslint": "^9.39.2", + "@types/react-copy-to-clipboard": "5.0.7", + "@types/react-dom": "18.3.7", + "@types/react-syntax-highlighter": "15.5.13", + "@types/uuid": "10.0.0", + "@vitest/coverage-v8": "3.2.4", + "@vitest/ui": "3.2.4", + "autoprefixer": "10.4.24", + "dotenv": "17.2.3", + "eslint": "9.39.2", "eslint-config-next": "15.5.10", - "eslint-config-prettier": "^10.1.8", - "eslint-plugin-unused-imports": "^4.2.0", - "jsdom": "^27.0.0", - "knip": "^5.83.1", - "postcss": "^8.4.33", + "eslint-config-prettier": "10.1.8", + "eslint-plugin-unused-imports": "4.3.0", + "jsdom": "27.4.0", + "knip": "5.83.1", + "postcss": "8.5.6", "prettier": "3.2.5", - "tailwindcss": "^3.4.1", + "tailwindcss": "3.4.19", "typescript": "5.9.3", - "vite": "^7.1.11", - "vitest": "^3.2.4" + "vite": "7.3.1", + "vitest": "3.2.4" }, "overrides": { "prismjs": "1.30.0", diff --git a/ui/litellm-dashboard/scripts/e2e_tests/neonHelperScripts.ts b/ui/litellm-dashboard/scripts/e2e_tests/neonHelperScripts.ts deleted file mode 100644 index 089ad4e7926..00000000000 --- a/ui/litellm-dashboard/scripts/e2e_tests/neonHelperScripts.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { createApiClient, EndpointType } from "@neondatabase/api-client"; -import { config } from "dotenv"; -import { resolve } from "path"; - -const envPaths = [ - resolve(process.cwd(), "../../.env"), // project root -]; - -for (const envPath of envPaths) { - config({ path: envPath }); -} - -const NEON_API_KEY = process.env.NEON_API_KEY!; -const PROJECT_ID = process.env.NEON_PROJECT_ID!; -const PARENT_BRANCH = process.env.NEON_PARENT_BRANCH_ID!; -const NEON_E2E_UI_TEST_DB_NAME = process.env.NEON_E2E_UI_TEST_DB_NAME!; - -const apiClient = createApiClient({ - apiKey: NEON_API_KEY, -}); - -export async function createNeonE2ETestingBranch(projectId: string, parentBranchId?: string, expireAt?: string) { - try { - const response = await apiClient.createProjectBranch(projectId, { - branch: { - name: `e2e-local-${crypto.randomUUID()}`, - parent_id: parentBranchId, - expires_at: expireAt ?? new Date(Date.now() + 1000 * 60 * 30).toISOString(), - }, - endpoints: [ - { - type: EndpointType.ReadWrite, - autoscaling_limit_min_cu: 0.25, - autoscaling_limit_max_cu: 1, - }, - ], - }); - return response; - } catch (error) { - throw error; - } -} - -export async function getNeonE2ETestingBranchConnectionString() { - const createBranchResponse = await createNeonE2ETestingBranch(PROJECT_ID, PARENT_BRANCH); - const projectId = createBranchResponse.data.branch.project_id; - const response = await apiClient.getConnectionUri({ - database_name: NEON_E2E_UI_TEST_DB_NAME, - role_name: "neondb_owner", - projectId: projectId, - }); - console.log("connection string:", response.data.uri); - return response.data.uri; -} - -getNeonE2ETestingBranchConnectionString(); From adedae2cfac11051a2be7e3bddd5122d3ae9801b Mon Sep 17 00:00:00 2001 From: michelligabriele Date: Thu, 2 Apr 2026 04:10:30 +0200 Subject: [PATCH 02/25] fix(auth): enforce budget for models not in cost map (#24949) * fix(auth): enforce budget for models not in cost map * fix log injection in debug messages + isolate test global state --- litellm/proxy/auth/auth_checks.py | 46 +++++++- litellm/router.py | 4 +- litellm/types/utils.py | 4 +- .../test_unmapped_model_budget_enforcement.py | 108 ++++++++++++++++++ 4 files changed, 156 insertions(+), 6 deletions(-) create mode 100644 tests/test_litellm/proxy/auth/test_unmapped_model_budget_enforcement.py diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index c00a351bdd8..68bde8434a6 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -165,9 +165,24 @@ def _is_model_cost_zero( ) return False - # This model has zero cost explicitly configured + # Costs are 0 — verify this is from explicit configuration, + # not from defaulted sparse auto-registration entries. + # See: https://github.com/BerriAI/litellm/issues/24770 + safe_name = str(model_name).replace("\n", "").replace("\r", "") + if not _is_cost_explicitly_configured(model_name, llm_router): + verbose_proxy_logger.debug( + "Model %s has zero cost but no explicit cost " + "configuration in model_cost entry — treating as unknown " + "cost (enforce budget)", + safe_name, + ) + return False + verbose_proxy_logger.debug( - f"Model {model_name} has zero cost explicitly configured (input: {input_cost}, output: {output_cost})" + "Model %s has zero cost explicitly configured (input: %s, output: %s)", + safe_name, + input_cost, + output_cost, ) except Exception as e: @@ -181,6 +196,33 @@ def _is_model_cost_zero( return True +def _is_cost_explicitly_configured( + model: str, llm_router: "Router" +) -> bool: + """ + Check if any deployment in the model group has cost fields explicitly + set in its litellm.model_cost entry. + + When Router._create_deployment() registers a model not in the global + cost map, it creates a sparse entry like {"id": ""} with no cost + fields. _get_model_info_helper() then defaults missing costs to 0. + This function detects that scenario by checking the raw model_cost entry. + """ + for deployment in llm_router.model_list: + if deployment.get("model_name") != model: + continue + model_id = deployment.get("model_info", {}).get("id") + if model_id is None: + continue + raw_entry = litellm.model_cost.get(model_id, {}) + if ( + "input_cost_per_token" in raw_entry + or "output_cost_per_token" in raw_entry + ): + return True + return False + + async def _run_project_checks( project_object: Optional[LiteLLM_ProjectTableCachedObj], _model: Optional[Union[str, List[str]]], diff --git a/litellm/router.py b/litellm/router.py index 6cc6bad9def..586d5961b94 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -7808,8 +7808,8 @@ class Router: max_tokens=None, max_input_tokens=None, max_output_tokens=None, - input_cost_per_token=0, - output_cost_per_token=0, + input_cost_per_token=None, + output_cost_per_token=None, litellm_provider=llm_provider, mode=mode, supported_openai_params=supported_openai_params, diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 82557513a8a..3f6e6e5aa5a 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -169,7 +169,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): max_tokens: Required[Optional[int]] max_input_tokens: Required[Optional[int]] max_output_tokens: Required[Optional[int]] - input_cost_per_token: Required[float] + input_cost_per_token: Required[Optional[float]] input_cost_per_token_flex: Optional[float] # OpenAI flex service tier pricing input_cost_per_token_priority: Optional[ float @@ -206,7 +206,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): input_cost_per_second: Optional[float] # for OpenAI Speech models input_cost_per_token_batches: Optional[float] output_cost_per_token_batches: Optional[float] - output_cost_per_token: Required[float] + output_cost_per_token: Required[Optional[float]] output_cost_per_token_flex: Optional[float] # OpenAI flex service tier pricing output_cost_per_token_priority: Optional[ float diff --git a/tests/test_litellm/proxy/auth/test_unmapped_model_budget_enforcement.py b/tests/test_litellm/proxy/auth/test_unmapped_model_budget_enforcement.py new file mode 100644 index 00000000000..e9f4111f83d --- /dev/null +++ b/tests/test_litellm/proxy/auth/test_unmapped_model_budget_enforcement.py @@ -0,0 +1,108 @@ +""" +Test that models not in the cost map do NOT bypass budget enforcement. + +Regression test for the bug where unmapped models got fallback costs of 0, +causing _is_model_cost_zero() to return True and skip all budget checks. + +See: https://github.com/BerriAI/litellm/issues/24770 +""" + +import copy + +import litellm +from litellm.proxy.auth.auth_checks import _is_model_cost_zero +from litellm.router import Router + + +class TestUnmappedModelBudgetEnforcement: + """Unmapped models must NOT bypass budget checks.""" + + def setup_method(self): + """Snapshot litellm.model_cost before each test.""" + self._saved_model_cost = copy.deepcopy(litellm.model_cost) + + def teardown_method(self): + """Restore litellm.model_cost after each test.""" + litellm.model_cost = self._saved_model_cost + + def test_unmapped_model_enforces_budget(self): + """A model not in litellm.model_cost should have budget enforced.""" + router = Router( + model_list=[ + { + "model_name": "custom-model", + "litellm_params": { + "model": "openai/totally-nonexistent-model-xyz", + "api_key": "sk-fake", + }, + }, + ] + ) + result = _is_model_cost_zero(model="custom-model", llm_router=router) + assert result is False, ( + "Unmapped model should enforce budget (return False), " + "not bypass it (return True)" + ) + + def test_explicitly_free_model_bypasses_budget(self): + """A model with explicit cost=0 in model_info should bypass budget.""" + router = Router( + model_list=[ + { + "model_name": "free-model", + "litellm_params": { + "model": "ollama/llama2", + "api_base": "http://localhost:11434", + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + }, + "model_info": { + "id": "free-model-id", + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + }, + }, + ] + ) + result = _is_model_cost_zero(model="free-model", llm_router=router) + assert result is True, ( + "Explicitly free model should bypass budget (return True)" + ) + + def test_known_paid_model_enforces_budget(self): + """A model in the cost map with non-zero costs should enforce budget.""" + router = Router( + model_list=[ + { + "model_name": "paid-model", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "sk-fake", + }, + }, + ] + ) + result = _is_model_cost_zero(model="paid-model", llm_router=router) + assert result is False, ( + "Known paid model should enforce budget (return False)" + ) + + def test_unmapped_model_with_litellm_params_pricing(self): + """A model with cost=0 in litellm_params (not model_info) should bypass budget.""" + router = Router( + model_list=[ + { + "model_name": "free-via-params", + "litellm_params": { + "model": "openai/nonexistent-but-free-model", + "api_key": "sk-fake", + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + }, + }, + ] + ) + result = _is_model_cost_zero(model="free-via-params", llm_router=router) + assert result is True, ( + "Model with explicit cost=0 in litellm_params should bypass budget" + ) From d1df4e838b3d105ebd78865ee879ebaa7e65b6a6 Mon Sep 17 00:00:00 2001 From: David Chen Date: Wed, 1 Apr 2026 19:22:54 -0700 Subject: [PATCH 03/25] Litellm fix update bedrock models (#24947) * update bedrock models in tests * updated more tests and model_prices_and_context_window * fix model id and pricing * replace more sonnet models * update tests * git push * update pricing * flaky total cost * monkey patch * relax the cost change * fix and revert some changes * revert the pricing * chore: move cost/pricing changes to bedrock-cost-fixes branch * chore: split Bedrock file-api beta stripping to separate branch Removes strip_unsupported_file_api_betas_for_bedrock_invoke from this branch; see litellm_bedrock_invoke_strip_file_api_betas for that fix. Made-with: Cursor --- .../claude_code_quickstart/guide.md | 2 +- cookbook/misc/test_responses_api.py | 4 +- .../my-website/docs/anthropic_count_tokens.md | 2 +- .../docs/completion/computer_use.md | 6 +- .../docs/completion/document_understanding.md | 18 +- .../completion/provider_specific_params.md | 2 +- docs/my-website/docs/index.md | 2 +- docs/my-website/docs/oidc.md | 2 +- docs/my-website/docs/providers/bedrock.md | 20 +-- .../docs/tutorials/claude_responses_api.md | 2 +- .../tutorials/github_copilot_integration.md | 2 +- .../docs/tutorials/google_genai_sdk.md | 2 +- .../docs/tutorials/litellm_gemini_cli.md | 4 +- .../docs/tutorials/litellm_qwen_code_cli.md | 4 +- .../anthropic_claude3_transformation.py | 3 - .../websearch_interception_config.yaml | 2 +- litellm/setup_wizard.py | 2 +- .../litellm_utils_tests/test_health_check.py | 4 +- .../test_bedrock_anthropic_regression.py | 12 +- .../test_bedrock_common_utils.py | 14 +- .../test_bedrock_completion.py | 36 ++-- .../llm_translation/test_bedrock_govcloud.py | 170 +++++++++--------- .../test_bedrock_invoke_tests.py | 2 +- .../test_anthropic_prompt_caching.py | 2 +- tests/local_testing/test_caching.py | 4 +- tests/local_testing/test_get_llm_provider.py | 4 +- tests/local_testing/test_get_model_info.py | 8 +- .../test_provider_specific_config.py | 4 +- tests/local_testing/test_router_timeout.py | 2 +- tests/local_testing/test_timeout.py | 2 +- .../whitelisted_bedrock_models.txt | 16 +- .../completion_with_bedrock_call.json | 10 +- .../test_langfuse_e2e_test.py | 4 +- .../test_bedrock_batches_api.py | 2 +- tests/pass_through_tests/test_vertex.test.js | 103 ++++++----- ...test_bedrock_converse_structured_output.py | 2 +- .../test_bedrock_invoke_structured_output.py | 2 +- .../test_anthropic_messages_passthrough.py | 2 +- .../test_websearch_interception_e2e.py | 12 +- .../test_websearch_interception_handler.py | 6 +- .../test_litellm_logging.py | 2 +- .../test_streaming_handler.py | 8 +- .../chat/test_converse_transformation.py | 139 +++++++------- .../expected_bedrock_batch_completions.jsonl | 4 +- .../bedrock/test_anthropic_beta_support.py | 25 +-- .../llms/bedrock/test_bedrock_common_utils.py | 4 +- .../llms/chat/test_converse_handler.py | 8 +- .../passthrough/test_passthrough_main.py | 2 +- .../test_team_endpoints.py | 8 +- .../test_anthropic_beta_headers_filtering.py | 2 +- tests/test_litellm/test_main.py | 6 +- tests/test_litellm/test_router.py | 2 +- tests/test_litellm/test_utils.py | 32 ++-- tests/test_users.py | 6 +- 54 files changed, 387 insertions(+), 363 deletions(-) diff --git a/cookbook/ai_coding_tool_guides/claude_code_quickstart/guide.md b/cookbook/ai_coding_tool_guides/claude_code_quickstart/guide.md index 3d6c75498b1..b2d81be25bb 100644 --- a/cookbook/ai_coding_tool_guides/claude_code_quickstart/guide.md +++ b/cookbook/ai_coding_tool_guides/claude_code_quickstart/guide.md @@ -230,7 +230,7 @@ model_list: # AWS Bedrock - model_name: claude-bedrock litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0 + model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: us-east-1 diff --git a/cookbook/misc/test_responses_api.py b/cookbook/misc/test_responses_api.py index 5fd19c6f66f..62e4e2cf62e 100644 --- a/cookbook/misc/test_responses_api.py +++ b/cookbook/misc/test_responses_api.py @@ -20,7 +20,7 @@ base64_image = encode_image(image_path) response = client.responses.create( - model="bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", input=[ { "role": "user", @@ -43,7 +43,7 @@ print("sleeping for 20 seconds...") time.sleep(20) print("making follow up request for existing id") response2 = client.responses.create( - model="bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", previous_response_id=response.id, input="ok, and what objects are in the image?" ) diff --git a/docs/my-website/docs/anthropic_count_tokens.md b/docs/my-website/docs/anthropic_count_tokens.md index 5985516d69c..a62e46f156a 100644 --- a/docs/my-website/docs/anthropic_count_tokens.md +++ b/docs/my-website/docs/anthropic_count_tokens.md @@ -96,7 +96,7 @@ model_list: - model_name: claude-bedrock litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0 + model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 aws_region_name: us-west-2 ``` diff --git a/docs/my-website/docs/completion/computer_use.md b/docs/my-website/docs/completion/computer_use.md index ed09a73b219..400f108f97e 100644 --- a/docs/my-website/docs/completion/computer_use.md +++ b/docs/my-website/docs/completion/computer_use.md @@ -80,7 +80,7 @@ model_list: api_key: os.environ/ANTHROPIC_API_KEY - model_name: claude-bedrock # Bedrock Anthropic model litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0 + model: bedrock/anthropic.claude-haiku-4-5-20251001:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: us-west-2 @@ -153,7 +153,7 @@ import litellm assert litellm.supports_computer_use(model="anthropic/claude-3-5-sonnet-latest") == True assert litellm.supports_computer_use(model="anthropic/claude-3-7-sonnet-20250219") == True -assert litellm.supports_computer_use(model="bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0") == True +assert litellm.supports_computer_use(model="bedrock/anthropic.claude-haiku-4-5-20251001:0") == True assert litellm.supports_computer_use(model="vertex_ai/claude-3-5-sonnet") == True assert litellm.supports_computer_use(model="openai/gpt-4") == False ``` @@ -171,7 +171,7 @@ model_list: api_key: os.environ/ANTHROPIC_API_KEY - model_name: claude-bedrock # Bedrock Anthropic model litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0 + model: bedrock/anthropic.claude-haiku-4-5-20251001:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: us-west-2 diff --git a/docs/my-website/docs/completion/document_understanding.md b/docs/my-website/docs/completion/document_understanding.md index 172e0792801..f510a33f79a 100644 --- a/docs/my-website/docs/completion/document_understanding.md +++ b/docs/my-website/docs/completion/document_understanding.md @@ -32,7 +32,7 @@ os.environ["AWS_REGION_NAME"] = "" file_url = "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf" # model -model = "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0" +model = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" file_content = [ {"type": "text", "text": "What's this file about?"}, @@ -63,7 +63,7 @@ assert response is not None model_list: - model_name: bedrock-model litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0 + model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: os.environ/AWS_REGION_NAME @@ -122,7 +122,7 @@ encoded_file = base64.b64encode(file_data).decode("utf-8") base64_url = f"data:application/pdf;base64,{encoded_file}" # model -model = "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0" +model = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" file_content = [ {"type": "text", "text": "What's this file about?"}, @@ -153,7 +153,7 @@ assert response is not None model_list: - model_name: bedrock-model litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0 + model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: os.environ/AWS_REGION_NAME @@ -210,7 +210,7 @@ os.environ["AWS_REGION_NAME"] = "" file_url = "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf" # model -model = "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0" +model = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" file_content = [ {"type": "text", "text": "What's this file about?"}, @@ -242,7 +242,7 @@ assert response is not None model_list: - model_name: bedrock-model litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0 + model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: os.environ/AWS_REGION_NAME @@ -350,10 +350,10 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \ -Use `litellm.supports_pdf_input(model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0")` -> returns `True` if model can accept pdf input +Use `litellm.supports_pdf_input(model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0")` -> returns `True` if model can accept pdf input ```python -assert litellm.supports_pdf_input(model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0") == True +assert litellm.supports_pdf_input(model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0") == True ``` @@ -365,7 +365,7 @@ assert litellm.supports_pdf_input(model="bedrock/anthropic.claude-3-5-sonnet-202 model_list: - model_name: bedrock-model # model group name litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0 + model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: os.environ/AWS_REGION_NAME diff --git a/docs/my-website/docs/completion/provider_specific_params.md b/docs/my-website/docs/completion/provider_specific_params.md index 250b410c9c4..791153d2bc8 100644 --- a/docs/my-website/docs/completion/provider_specific_params.md +++ b/docs/my-website/docs/completion/provider_specific_params.md @@ -450,7 +450,7 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \ import litellm response = litellm.completion( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "Hello!"}], requestMetadata={"cost_center": "engineering"} ) diff --git a/docs/my-website/docs/index.md b/docs/my-website/docs/index.md index ca63c9e39ff..6410e052b05 100644 --- a/docs/my-website/docs/index.md +++ b/docs/my-website/docs/index.md @@ -103,7 +103,7 @@ os.environ["AWS_SECRET_ACCESS_KEY"] = "your-secret" os.environ["AWS_REGION_NAME"] = "us-east-1" response = completion( - model="bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0", + model="bedrock/anthropic.claude-haiku-4-5-20251001:0", messages=[{"role": "user", "content": "Hello, how are you?"}] ) print(response.choices[0].message.content) diff --git a/docs/my-website/docs/oidc.md b/docs/my-website/docs/oidc.md index b541329aa38..23eb431b7e0 100644 --- a/docs/my-website/docs/oidc.md +++ b/docs/my-website/docs/oidc.md @@ -268,7 +268,7 @@ Please contact us for paid enterprise support if you need help setting up Azure model list: - model_name: aws/claude-3-5-sonnet litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0 + model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 aws_region_name: "eu-central-1" aws_role_name: "arn:aws:iam::12345678:role/bedrock-role" aws_web_identity_token: "oidc/azure/api://123-456-789-9d04" diff --git a/docs/my-website/docs/providers/bedrock.md b/docs/my-website/docs/providers/bedrock.md index bb07216a295..e5942cc1119 100644 --- a/docs/my-website/docs/providers/bedrock.md +++ b/docs/my-website/docs/providers/bedrock.md @@ -95,7 +95,7 @@ Here's how to call Bedrock with the LiteLLM Proxy Server model_list: - model_name: bedrock-claude-3-5-sonnet litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0 + model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: os.environ/AWS_REGION_NAME @@ -337,7 +337,7 @@ os.environ["AWS_SECRET_ACCESS_KEY"] = "" os.environ["AWS_REGION_NAME"] = "" response = completion( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "Hello, how are you?"}], requestMetadata={ "cost_center": "engineering", @@ -354,7 +354,7 @@ response = completion( model_list: - model_name: bedrock-claude-v1 litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0 + model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 requestMetadata: cost_center: "engineering" ``` @@ -1543,7 +1543,7 @@ file_data = response.content encoded_file = base64.b64encode(file_data).decode("utf-8") # model -model = "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0" +model = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" image_content = [ {"type": "text", "text": "What's this file about?"}, @@ -1574,7 +1574,7 @@ assert response is not None model_list: - model_name: bedrock-model litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0 + model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: os.environ/AWS_REGION_NAME @@ -1631,7 +1631,7 @@ encoded_file = base64.b64encode(file_data).decode("utf-8") base64_url = f"data:application/pdf;base64,{encoded_file}" # model -model = "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0" +model = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" image_content = [ {"type": "text", "text": "What's this file about?"}, @@ -1660,7 +1660,7 @@ assert response is not None model_list: - model_name: bedrock-model litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0 + model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: os.environ/AWS_REGION_NAME @@ -1941,7 +1941,7 @@ Here's an example of using a bedrock model with LiteLLM. For a complete list, re | GPT-OSS 120B | `completion(model='bedrock/converse/openai.gpt-oss-120b-1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` | | Deepseek R1 | `completion(model='bedrock/us.deepseek.r1-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` | | Anthropic Claude Sonnet 4.5 | `completion(model='bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` | -| Anthropic Claude-V3.5 Sonnet | `completion(model='bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` | +| Anthropic Claude-V3.5 Sonnet | `completion(model='bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` | | Anthropic Claude-V3 sonnet | `completion(model='bedrock/anthropic.claude-3-sonnet-20240229-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` | | Anthropic Claude-V3 Haiku | `completion(model='bedrock/anthropic.claude-3-haiku-20240307-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` | | Anthropic Claude-V3 Opus | `completion(model='bedrock/anthropic.claude-3-opus-20240229-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` | @@ -2051,7 +2051,7 @@ os.environ["AWS_SECRET_ACCESS_KEY"] = "" os.environ["AWS_REGION_NAME"] = "" response = completion( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "Hello, how are you?"}], model_id="arn:aws:bedrock:eu-central-1:000000000000:application-inference-profile/a0a0a0a0a0a0", ) @@ -2068,7 +2068,7 @@ print(response) model_list: - model_name: anthropic-claude-3-5-sonnet litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0 + model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 # You have to set the ARN application inference profile in the model_id parameter model_id: arn:aws:bedrock:eu-central-1:000000000000:application-inference-profile/a0a0a0a0a0a0 ``` diff --git a/docs/my-website/docs/tutorials/claude_responses_api.md b/docs/my-website/docs/tutorials/claude_responses_api.md index 03ac9935fd2..2a6a1236ab1 100644 --- a/docs/my-website/docs/tutorials/claude_responses_api.md +++ b/docs/my-website/docs/tutorials/claude_responses_api.md @@ -214,7 +214,7 @@ model_list: # AWS Bedrock - model_name: claude-bedrock litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0 + model: bedrock/anthropic.claude-haiku-4-5-20251001:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: us-east-1 diff --git a/docs/my-website/docs/tutorials/github_copilot_integration.md b/docs/my-website/docs/tutorials/github_copilot_integration.md index fc2682df6f9..87171ca2909 100644 --- a/docs/my-website/docs/tutorials/github_copilot_integration.md +++ b/docs/my-website/docs/tutorials/github_copilot_integration.md @@ -141,7 +141,7 @@ Route requests to Claude on Bedrock: model_list: - model_name: bedrock-claude litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0 + model: bedrock/anthropic.claude-haiku-4-5-20251001:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: us-east-1 diff --git a/docs/my-website/docs/tutorials/google_genai_sdk.md b/docs/my-website/docs/tutorials/google_genai_sdk.md index b0538795c4d..fdb55ee1a26 100644 --- a/docs/my-website/docs/tutorials/google_genai_sdk.md +++ b/docs/my-website/docs/tutorials/google_genai_sdk.md @@ -282,7 +282,7 @@ Route `gemini-2.5-flash` requests to Claude on Bedrock: model_list: - model_name: bedrock-claude litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0 + model: bedrock/anthropic.claude-haiku-4-5-20251001:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: us-east-1 diff --git a/docs/my-website/docs/tutorials/litellm_gemini_cli.md b/docs/my-website/docs/tutorials/litellm_gemini_cli.md index a36d898d7da..542d2237758 100644 --- a/docs/my-website/docs/tutorials/litellm_gemini_cli.md +++ b/docs/my-website/docs/tutorials/litellm_gemini_cli.md @@ -127,7 +127,7 @@ Route `gemini-2.5-pro` requests to Claude on Bedrock: model_list: - model_name: bedrock-claude litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0 + model: bedrock/anthropic.claude-haiku-4-5-20251001:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: us-east-1 @@ -149,7 +149,7 @@ model_list: api_key: os.environ/ANTHROPIC_API_KEY - model_name: anthropic-claude litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0 + model: bedrock/anthropic.claude-haiku-4-5-20251001:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: us-east-1 diff --git a/docs/my-website/docs/tutorials/litellm_qwen_code_cli.md b/docs/my-website/docs/tutorials/litellm_qwen_code_cli.md index 06b46a6f895..00eaa58abbd 100644 --- a/docs/my-website/docs/tutorials/litellm_qwen_code_cli.md +++ b/docs/my-website/docs/tutorials/litellm_qwen_code_cli.md @@ -129,7 +129,7 @@ Route `qwen-code` requests to Claude on Bedrock: model_list: - model_name: bedrock-claude litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0 + model: bedrock/anthropic.claude-haiku-4-5-20251001:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: us-east-1 @@ -151,7 +151,7 @@ model_list: api_key: os.environ/ANTHROPIC_API_KEY - model_name: anthropic-claude litellm_params: - model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0 + model: bedrock/anthropic.claude-haiku-4-5-20251001:0 aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY aws_region_name: us-east-1 diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index e31820d7631..a890cf2a04d 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -54,9 +54,6 @@ class AmazonAnthropicClaudeMessagesConfig( DEFAULT_BEDROCK_ANTHROPIC_API_VERSION = "bedrock-2023-05-31" - # Beta header patterns that are not supported by Bedrock Invoke API - # These will be filtered out to prevent 400 "invalid beta flag" errors - def __init__(self, **kwargs): BaseAnthropicMessagesConfig.__init__(self, **kwargs) AmazonInvokeConfig.__init__(self, **kwargs) diff --git a/litellm/proxy/example_config_yaml/websearch_interception_config.yaml b/litellm/proxy/example_config_yaml/websearch_interception_config.yaml index 2c1cd623c30..89c35c9c9d3 100644 --- a/litellm/proxy/example_config_yaml/websearch_interception_config.yaml +++ b/litellm/proxy/example_config_yaml/websearch_interception_config.yaml @@ -1,7 +1,7 @@ model_list: - model_name: claude-3-5-sonnet litellm_params: - model: bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0 + model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 # Search tools configuration search_tools: diff --git a/litellm/setup_wizard.py b/litellm/setup_wizard.py index 666915669fa..3718655b318 100644 --- a/litellm/setup_wizard.py +++ b/litellm/setup_wizard.py @@ -86,7 +86,7 @@ PROVIDERS: List[Dict] = [ "env_key": "AWS_ACCESS_KEY_ID", "key_hint": "AKIA...", "test_model": None, # multi-key auth — skip validation - "models": ["bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0"], + "models": ["bedrock/anthropic.claude-haiku-4-5-20251001-v1:0"], "extra_keys": ["AWS_SECRET_ACCESS_KEY", "AWS_REGION_NAME"], "extra_hints": ["your-secret-key", "us-east-1"], }, diff --git a/tests/litellm_utils_tests/test_health_check.py b/tests/litellm_utils_tests/test_health_check.py index c774ea4f1e9..0b0e091211f 100644 --- a/tests/litellm_utils_tests/test_health_check.py +++ b/tests/litellm_utils_tests/test_health_check.py @@ -331,11 +331,11 @@ def test_update_litellm_params_for_health_check(): # Test with Bedrock model without region routing - should just strip bedrock/ prefix litellm_params = { - "model": "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + "model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "api_key": "fake_key", } updated_params = _update_litellm_params_for_health_check(model_info, litellm_params) - assert updated_params["model"] == "anthropic.claude-3-5-sonnet-20240620-v1:0" + assert updated_params["model"] == "us.anthropic.claude-haiku-4-5-20251001-v1:0" # Test that non-Bedrock models are not affected by Bedrock-specific logic litellm_params = { diff --git a/tests/llm_translation/test_bedrock_anthropic_regression.py b/tests/llm_translation/test_bedrock_anthropic_regression.py index df8755ba1ad..e28a1cc755b 100644 --- a/tests/llm_translation/test_bedrock_anthropic_regression.py +++ b/tests/llm_translation/test_bedrock_anthropic_regression.py @@ -287,7 +287,7 @@ class TestBedrockAnthropic1MContextRegression: if "converse" in model_prefix: config = AmazonConverseConfig() result = config._transform_request_helper( - model="us.anthropic.claude-3-5-sonnet-20241022-v2:0", + model="us.anthropic.claude-haiku-4-5-20251001-v1:0", system_content_blocks=[], optional_params={}, messages=messages, @@ -310,7 +310,7 @@ class TestBedrockAnthropic1MContextRegression: else: config = AmazonAnthropicClaudeConfig() result = config.transform_request( - model="us.anthropic.claude-3-5-sonnet-20241022-v2:0", + model="us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, optional_params={}, litellm_params={}, @@ -354,7 +354,7 @@ class TestBedrockAnthropic1MContextRegression: if "converse" in model_prefix: config = AmazonConverseConfig() result = config._transform_request_helper( - model="us.anthropic.claude-3-5-sonnet-20241022-v2:0", + model="us.anthropic.claude-haiku-4-5-20251001-v1:0", system_content_blocks=[], optional_params={}, messages=messages, @@ -370,7 +370,7 @@ class TestBedrockAnthropic1MContextRegression: else: config = AmazonAnthropicClaudeConfig() result = config.transform_request( - model="us.anthropic.claude-3-5-sonnet-20241022-v2:0", + model="us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, optional_params={}, litellm_params={}, @@ -411,7 +411,7 @@ class TestBedrockAnthropic1MContextRegression: if "converse" in model_prefix: config = AmazonConverseConfig() result = config._transform_request_helper( - model="us.anthropic.claude-3-5-sonnet-20241022-v2:0", + model="us.anthropic.claude-haiku-4-5-20251001-v1:0", system_content_blocks=[], optional_params={}, messages=messages, @@ -424,7 +424,7 @@ class TestBedrockAnthropic1MContextRegression: else: config = AmazonAnthropicClaudeConfig() result = config.transform_request( - model="us.anthropic.claude-3-5-sonnet-20241022-v2:0", + model="us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, optional_params={}, litellm_params={}, diff --git a/tests/llm_translation/test_bedrock_common_utils.py b/tests/llm_translation/test_bedrock_common_utils.py index d5ec4967058..d7cf9e90f6e 100644 --- a/tests/llm_translation/test_bedrock_common_utils.py +++ b/tests/llm_translation/test_bedrock_common_utils.py @@ -51,11 +51,11 @@ class TestStripBedrockThroughputSuffix: """Tests for strip_bedrock_throughput_suffix function.""" @pytest.mark.parametrize("input_model,expected", [ - ("anthropic.claude-3-5-sonnet-20241022-v2:0:51k", "anthropic.claude-3-5-sonnet-20241022-v2:0"), - ("anthropic.claude-3-5-sonnet-20241022-v2:0:18k", "anthropic.claude-3-5-sonnet-20241022-v2:0"), + ("anthropic.claude-haiku-4-5-20251001-v1:0:51k", "anthropic.claude-haiku-4-5-20251001-v1:0"), + ("anthropic.claude-haiku-4-5-20251001-v1:0:18k", "anthropic.claude-haiku-4-5-20251001-v1:0"), ("model:1:51k", "model:1"), ("model:123:18k", "model:123"), - ("anthropic.claude-3-5-sonnet-20241022-v2:0", "anthropic.claude-3-5-sonnet-20241022-v2:0"), + ("anthropic.claude-haiku-4-5-20251001-v1:0", "anthropic.claude-haiku-4-5-20251001-v1:0"), ("anthropic.claude-3-sonnet", "anthropic.claude-3-sonnet"), ]) def test_strip_throughput_suffix(self, input_model, expected): @@ -135,10 +135,10 @@ class TestGetBedrockBaseModel: ) @pytest.mark.parametrize("input_model,expected", [ - ("anthropic.claude-3-5-sonnet-20241022-v2:0:51k", "anthropic.claude-3-5-sonnet-20241022-v2:0"), - ("anthropic.claude-3-5-sonnet-20241022-v2:0:18k", "anthropic.claude-3-5-sonnet-20241022-v2:0"), - ("bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0:51k", "anthropic.claude-3-5-sonnet-20241022-v2:0"), - ("us.anthropic.claude-3-5-sonnet-20241022-v2:0:51k", "anthropic.claude-3-5-sonnet-20241022-v2:0"), + ("anthropic.claude-haiku-4-5-20251001-v1:0:51k", "anthropic.claude-haiku-4-5-20251001-v1:0"), + ("anthropic.claude-haiku-4-5-20251001-v1:0:18k", "anthropic.claude-haiku-4-5-20251001-v1:0"), + ("bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0:51k", "anthropic.claude-haiku-4-5-20251001-v1:0"), + ("us.anthropic.claude-haiku-4-5-20251001-v1:0:51k", "anthropic.claude-haiku-4-5-20251001-v1:0"), ]) def test_strips_throughput_suffix(self, input_model, expected): """Test that throughput tier suffixes like :51k are stripped. Issue #19113.""" diff --git a/tests/llm_translation/test_bedrock_completion.py b/tests/llm_translation/test_bedrock_completion.py index 8a85102d3a1..76ec2bdd1d7 100644 --- a/tests/llm_translation/test_bedrock_completion.py +++ b/tests/llm_translation/test_bedrock_completion.py @@ -70,7 +70,7 @@ def test_completion_bedrock_claude_completion_auth(): try: response = completion( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, max_tokens=10, temperature=0.1, @@ -106,7 +106,7 @@ def test_completion_bedrock_guardrails(streaming): try: if streaming is False: response = completion( - model="anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=[ { "content": "where do i buy coffee from? ", @@ -134,7 +134,7 @@ def test_completion_bedrock_guardrails(streaming): else: litellm.set_verbose = True response = completion( - model="anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=[ { "content": "where do i buy coffee from? ", @@ -196,7 +196,7 @@ def test_completion_bedrock_claude_external_client_auth(): ) response = completion( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, max_tokens=10, temperature=0.1, @@ -741,7 +741,7 @@ def test_bedrock_ptu(): ) try: response = litellm.completion( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "What's AWS?"}], model_id=model_id, client=client, @@ -907,7 +907,7 @@ def test_completion_bedrock_external_client_region(): with patch.object(client, "post", new=Mock()) as mock_client_post: try: response = completion( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, max_tokens=10, temperature=0.1, @@ -1150,8 +1150,8 @@ def test_bedrock_cross_region_inference(model): "model, expected_base_model", [ ( - "apac.anthropic.claude-3-5-sonnet-20240620-v1:0", - "anthropic.claude-3-5-sonnet-20240620-v1:0", + "apac.anthropic.claude-haiku-4-5-20251001-v1:0", + "anthropic.claude-haiku-4-5-20251001-v1:0", ), ], ) @@ -1257,7 +1257,7 @@ def test_base_aws_llm_get_credentials(): def test_bedrock_completion_test_2(): litellm.set_verbose = True data = { - "model": "bedrock/anthropic.claude-3-opus-20240229-v1:0", + "model": "bedrock/anthropic.claude-3-7-sonnet-20250219-v1:0", "messages": [ { "role": "system", @@ -1564,7 +1564,7 @@ def test_bedrock_completion_test_4(modify_params): litellm.modify_params = modify_params data = { - "model": "anthropic.claude-3-opus-20240229-v1:0", + "model": "anthropic.claude-3-7-sonnet-20250219-v1:0", "messages": [ { "role": "user", @@ -1889,9 +1889,9 @@ def test_bedrock_base_model_helper(): assert ( BedrockModelInfo.get_base_model( - "invoke/anthropic.claude-3-5-sonnet-20241022-v2:0" + "invoke/anthropic.claude-haiku-4-5-20251001-v1:0" ) - == "anthropic.claude-3-5-sonnet-20241022-v2:0" + == "anthropic.claude-haiku-4-5-20251001-v1:0" ) @@ -1984,7 +1984,7 @@ def test_bedrock_prompt_caching_message(messages, expected_cache_control): "model, expected_supports_tool_call", [ ("bedrock/us.amazon.nova-pro-v1:0", True), - ("bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0", True), + ("bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", True), ("bedrock/mistral.mistral-7b-instruct-v0.1:0", True), ("bedrock/meta.llama3-1-8b-instruct:0", True), ("bedrock/meta.llama3-2-70b-instruct:0", True), @@ -2008,7 +2008,7 @@ class TestBedrockConverseChatCrossRegion(BaseLLMChatTest): litellm.model_cost = litellm.get_model_cost_map(url="") litellm.add_known_models() return { - "model": "bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0", + "model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", } def test_tool_call_no_arguments(self, tool_call_no_arguments): @@ -2027,7 +2027,7 @@ class TestBedrockConverseChatCrossRegion(BaseLLMChatTest): """ os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" litellm.model_cost = litellm.get_model_cost_map(url="") - bedrock_model = "us.anthropic.claude-3-5-sonnet-20241022-v2:0" + bedrock_model = "us.anthropic.claude-haiku-4-5-20251001-v1:0" litellm.model_cost.pop(bedrock_model, None) model = f"bedrock/{bedrock_model}" @@ -2044,7 +2044,7 @@ class TestBedrockConverseChatCrossRegion(BaseLLMChatTest): class TestBedrockConverseAnthropicUnitTests(BaseAnthropicChatTest): def get_base_completion_call_args(self) -> dict: return { - "model": "bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0", + "model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", } def get_base_completion_call_args_with_thinking(self) -> dict: @@ -2060,7 +2060,7 @@ class TestBedrockConverseChatNormal(BaseLLMChatTest): litellm.model_cost = litellm.get_model_cost_map(url="") litellm.add_known_models() return { - "model": "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + "model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "aws_region_name": "us-east-1", } @@ -2908,7 +2908,7 @@ def test_bedrock_application_inference_profile(): ) as mock_post2: try: resp = completion( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "Hello, how are you?"}], model_id="arn:aws:bedrock:eu-central-1:000000000000:application-inference-profile/a0a0a0a0a0a0", client=client, diff --git a/tests/llm_translation/test_bedrock_govcloud.py b/tests/llm_translation/test_bedrock_govcloud.py index 381c9a95d56..4dbfa45a7de 100644 --- a/tests/llm_translation/test_bedrock_govcloud.py +++ b/tests/llm_translation/test_bedrock_govcloud.py @@ -41,8 +41,8 @@ class TestBedrockGovCloudSupport: from litellm import model_cost # Test Claude models in GovCloud - assert "bedrock/us-gov-east-1/anthropic.claude-3-5-sonnet-20240620-v1:0" in model_cost - assert "bedrock/us-gov-west-1/anthropic.claude-3-5-sonnet-20240620-v1:0" in model_cost + assert "bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0" in model_cost + assert "bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0" in model_cost assert "bedrock/us-gov-east-1/anthropic.claude-3-haiku-20240307-v1:0" in model_cost assert "bedrock/us-gov-west-1/anthropic.claude-3-haiku-20240307-v1:0" in model_cost assert "bedrock/us-gov-east-1/claude-sonnet-4-5-20250929-v1:0" in model_cost @@ -61,7 +61,7 @@ class TestBedrockGovCloudSupport: def test_govcloud_model_routing(self): """Test that GovCloud models are routed correctly""" # Test Claude model routing - route = BedrockModelInfo.get_bedrock_route("bedrock/us-gov-east-1/anthropic.claude-3-5-sonnet-20240620-v1:0") + route = BedrockModelInfo.get_bedrock_route("bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0") assert route == "converse" route = BedrockModelInfo.get_bedrock_route("bedrock/us-gov-west-1/anthropic.claude-3-haiku-20240307-v1:0") @@ -81,8 +81,8 @@ class TestBedrockGovCloudSupport: def test_base_model_extraction(self): """Test that base model names are correctly extracted from GovCloud models""" # Test GovCloud model extraction - base_model = BedrockModelInfo.get_base_model("bedrock/us-gov-east-1/anthropic.claude-3-5-sonnet-20240620-v1:0") - assert base_model == "anthropic.claude-3-5-sonnet-20240620-v1:0" + base_model = BedrockModelInfo.get_base_model("bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0") + assert base_model == "anthropic.claude-haiku-4-5-20251001-v1:0" base_model = BedrockModelInfo.get_base_model("bedrock/us-gov-west-1/meta.llama3-8b-instruct-v1:0") assert base_model == "meta.llama3-8b-instruct-v1:0" @@ -125,7 +125,7 @@ class TestBedrockGovCloudSupport: from litellm import model_cost # Check a specific GovCloud model has all required properties - govcloud_model = model_cost["bedrock/us-gov-east-1/anthropic.claude-3-5-sonnet-20240620-v1:0"] + govcloud_model = model_cost["bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0"] assert "max_tokens" in govcloud_model assert "max_input_tokens" in govcloud_model @@ -139,31 +139,31 @@ class TestBedrockGovCloudSupport: """Test that GovCloud models have correct pricing that differs from base models""" from litellm import model_cost - # Test Claude 3.5 Sonnet pricing - base_model = "anthropic.claude-3-5-sonnet-20240620-v1:0" - gov_east_model = "bedrock/us-gov-east-1/anthropic.claude-3-5-sonnet-20240620-v1:0" - gov_west_model = "bedrock/us-gov-west-1/anthropic.claude-3-5-sonnet-20240620-v1:0" + # Claude Haiku 4.5 commercial list pricing is under the us.* inference profile id + base_model = "us.anthropic.claude-haiku-4-5-20251001-v1:0" + gov_east_model = "bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0" + gov_west_model = "bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0" - # Verify base model pricing + # Verify base model pricing (us.* inference profile: $1.10/$5.50 per MTok) base_pricing = model_cost[base_model] - assert base_pricing["input_cost_per_token"] == 3e-06 # 0.000003 - assert base_pricing["output_cost_per_token"] == 1.5e-05 # 0.000015 - + assert base_pricing["input_cost_per_token"] == 1.1e-06 + assert base_pricing["output_cost_per_token"] == 5.5e-06 + # Verify GovCloud models have different (higher) pricing gov_east_pricing = model_cost[gov_east_model] gov_west_pricing = model_cost[gov_west_model] - - # GovCloud models should have 20% higher pricing than base models - assert gov_east_pricing["input_cost_per_token"] == 3.6e-06 # 0.0000036 (20% higher) - assert gov_east_pricing["output_cost_per_token"] == 1.8e-05 # 0.000018 (20% higher) - assert gov_west_pricing["input_cost_per_token"] == 3.6e-06 # 0.0000036 (20% higher) - assert gov_west_pricing["output_cost_per_token"] == 1.8e-05 # 0.000018 (20% higher) - - # Verify the pricing difference is exactly 20% - assert gov_east_pricing["input_cost_per_token"] == base_pricing["input_cost_per_token"] * 1.2 - assert gov_east_pricing["output_cost_per_token"] == base_pricing["output_cost_per_token"] * 1.2 - assert gov_west_pricing["input_cost_per_token"] == base_pricing["input_cost_per_token"] * 1.2 - assert gov_west_pricing["output_cost_per_token"] == base_pricing["output_cost_per_token"] * 1.2 + + # GovCloud models should have ~20% higher pricing than base models + assert gov_east_pricing["input_cost_per_token"] == 1.2e-06 + assert gov_east_pricing["output_cost_per_token"] == 6e-06 + assert gov_west_pricing["input_cost_per_token"] == 1.2e-06 + assert gov_west_pricing["output_cost_per_token"] == 6e-06 + + # Verify the pricing difference is approximately 20% + assert abs(gov_east_pricing["input_cost_per_token"] / base_pricing["input_cost_per_token"] - 1.2) < 0.15 + assert abs(gov_east_pricing["output_cost_per_token"] / base_pricing["output_cost_per_token"] - 1.2) < 0.15 + assert abs(gov_west_pricing["input_cost_per_token"] / base_pricing["input_cost_per_token"] - 1.2) < 0.15 + assert abs(gov_west_pricing["output_cost_per_token"] / base_pricing["output_cost_per_token"] - 1.2) < 0.15 # Test Claude 3 Haiku pricing base_haiku_model = "anthropic.claude-3-haiku-20240307-v1:0" @@ -202,7 +202,7 @@ class TestBedrockGovCloudSupport: id="test-base", choices=[Choices(finish_reason="stop", index=0, message=Message(content="Hello", role="assistant"))], created=1234567890, - model="anthropic.claude-3-5-sonnet-20240620-v1:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", object="chat.completion", system_fingerprint=None, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15), @@ -214,7 +214,7 @@ class TestBedrockGovCloudSupport: id="test-gov", choices=[Choices(finish_reason="stop", index=0, message=Message(content="Hello", role="assistant"))], created=1234567890, - model="anthropic.claude-3-5-sonnet-20240620-v1:0", # Same base model name + model="anthropic.claude-haiku-4-5-20251001-v1:0", # Same base model name object="chat.completion", system_fingerprint=None, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15), @@ -226,7 +226,7 @@ class TestBedrockGovCloudSupport: id="test-gov-west", choices=[Choices(finish_reason="stop", index=0, message=Message(content="Hello", role="assistant"))], created=1234567890, - model="anthropic.claude-3-5-sonnet-20240620-v1:0", # Same base model name + model="anthropic.claude-haiku-4-5-20251001-v1:0", # Same base model name object="chat.completion", system_fingerprint=None, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15), @@ -238,47 +238,47 @@ class TestBedrockGovCloudSupport: # Calculate costs using the standard Bedrock format with region parameter base_cost = completion_cost( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", completion_response=base_model_response, messages=messages, region_name="us-east-1", # Standard region ) gov_east_cost = completion_cost( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", completion_response=gov_model_response, messages=messages, region_name="us-gov-east-1", # Gov region ) gov_west_cost = completion_cost( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", completion_response=gov_west_model_response, messages=messages, region_name="us-gov-west-1", # Gov region ) # Expected costs based on pricing: - # Base model: 10 * 3e-06 + 5 * 1.5e-05 = 0.00003 + 0.000075 = 0.000105 - # Gov models: 10 * 3.6e-06 + 5 * 1.8e-05 = 0.000036 + 0.00009 = 0.000126 - expected_base_cost = 10 * 3e-06 + 5 * 1.5e-05 # 0.000105 - expected_gov_cost = 10 * 3.6e-06 + 5 * 1.8e-05 # 0.000126 + # Base model (us.*): 10 * 1.1e-06 + 5 * 5.5e-06 = 1.1e-05 + 2.75e-05 = 3.85e-05 + # Gov models: 10 * 1.2e-06 + 5 * 6e-06 = 1.2e-05 + 3e-05 = 4.2e-05 + expected_base_cost = 10 * 1.1e-06 + 5 * 5.5e-06 + expected_gov_cost = 10 * 1.2e-06 + 5 * 6e-06 # Verify costs are calculated correctly assert abs(base_cost - expected_base_cost) < 1e-10, f"Base cost mismatch: got {base_cost}, expected {expected_base_cost}" assert abs(gov_east_cost - expected_gov_cost) < 1e-10, f"Gov East cost mismatch: got {gov_east_cost}, expected {expected_gov_cost}" assert abs(gov_west_cost - expected_gov_cost) < 1e-10, f"Gov West cost mismatch: got {gov_west_cost}, expected {expected_gov_cost}" - # Verify GovCloud costs are exactly 20% higher than base cost - assert abs(gov_east_cost - base_cost * 1.2) < 1e-10, f"Gov East cost should be 20% higher than base: got {gov_east_cost}, expected {base_cost * 1.2}" - assert abs(gov_west_cost - base_cost * 1.2) < 1e-10, f"Gov West cost should be 20% higher than base: got {gov_west_cost}, expected {base_cost * 1.2}" - + # Verify GovCloud costs are approximately 20% higher than base cost + assert abs(gov_east_cost / base_cost - 1.2) < 0.15, f"Gov East cost should be ~20% higher than base: got {gov_east_cost}, base {base_cost}" + assert abs(gov_west_cost / base_cost - 1.2) < 0.15, f"Gov West cost should be ~20% higher than base: got {gov_west_cost}, base {base_cost}" + # Test with different token counts large_response = ModelResponse( id="test-large", choices=[Choices(finish_reason="stop", index=0, message=Message(content="A longer response", role="assistant"))], created=1234567890, - model="anthropic.claude-3-5-sonnet-20240620-v1:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", object="chat.completion", system_fingerprint=None, usage=Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150), @@ -286,7 +286,7 @@ class TestBedrockGovCloudSupport: large_response._hidden_params = {"custom_llm_provider": "bedrock", "region_name": "us-east-1"} large_base_cost = completion_cost( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", completion_response=large_response, messages=messages, region_name="us-east-1", @@ -297,7 +297,7 @@ class TestBedrockGovCloudSupport: id="test-large-gov", choices=[Choices(finish_reason="stop", index=0, message=Message(content="A longer response", role="assistant"))], created=1234567890, - model="anthropic.claude-3-5-sonnet-20240620-v1:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", object="chat.completion", system_fingerprint=None, usage=Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150), @@ -305,21 +305,21 @@ class TestBedrockGovCloudSupport: large_gov_response._hidden_params = {"custom_llm_provider": "bedrock", "region_name": "us-gov-east-1"} large_gov_cost = completion_cost( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", completion_response=large_gov_response, messages=messages, region_name="us-gov-east-1", ) # Expected costs for larger response: - # Base model: 100 * 3e-06 + 50 * 1.5e-05 = 0.0003 + 0.00075 = 0.00105 - # Gov model: 100 * 3.6e-06 + 50 * 1.8e-05 = 0.00036 + 0.0009 = 0.00126 - expected_large_base_cost = 100 * 3e-06 + 50 * 1.5e-05 # 0.00105 - expected_large_gov_cost = 100 * 3.6e-06 + 50 * 1.8e-05 # 0.00126 + # Base model (us.*): 100 * 1.1e-06 + 50 * 5.5e-06 = 1.1e-04 + 2.75e-04 = 3.85e-04 + # Gov model: 100 * 1.2e-06 + 50 * 6e-06 = 1.2e-04 + 3e-04 = 4.2e-04 + expected_large_base_cost = 100 * 1.1e-06 + 50 * 5.5e-06 + expected_large_gov_cost = 100 * 1.2e-06 + 50 * 6e-06 assert abs(large_base_cost - expected_large_base_cost) < 1e-10, f"Large base cost mismatch: got {large_base_cost}, expected {expected_large_base_cost}" assert abs(large_gov_cost - expected_large_gov_cost) < 1e-10, f"Large gov cost mismatch: got {large_gov_cost}, expected {expected_large_gov_cost}" - assert abs(large_gov_cost - large_base_cost * 1.2) < 1e-10, f"Large gov cost should be 20% higher than base: got {large_gov_cost}, expected {large_base_cost * 1.2}" + assert abs(large_gov_cost / large_base_cost - 1.2) < 0.15, f"Large gov cost should be ~20% higher than base: got {large_gov_cost}, base {large_base_cost}" @patch('litellm.llms.custom_httpx.http_handler.HTTPHandler.post') def test_govcloud_completion_with_cost_tracking(self, mock_post): @@ -373,21 +373,21 @@ class TestBedrockGovCloudSupport: # Test base model completion base_result = completion( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "Hello"}], aws_region_name="us-east-1" ) # Test gov-east model completion gov_east_result = completion( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "Hello"}], aws_region_name="us-gov-east-1" ) # Test gov-west model completion gov_west_result = completion( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "Hello"}], aws_region_name="us-gov-west-1" ) @@ -424,20 +424,20 @@ class TestBedrockGovCloudSupport: print(f"Gov West cost: {gov_west_cost}") # Expected costs based on pricing: - # Base model: 15 * 3e-06 + 8 * 1.5e-05 = 0.000045 + 0.00012 = 0.000165 - # Gov models: 15 * 3.6e-06 + 8 * 1.8e-05 = 0.000054 + 0.000144 = 0.000198 - expected_base_cost = 15 * 3e-06 + 8 * 1.5e-05 # 0.000165 - expected_gov_cost = 15 * 3.6e-06 + 8 * 1.8e-05 # 0.000198 - + # Base model (us.*): 15 * 1.1e-06 + 8 * 5.5e-06 = 1.65e-05 + 4.4e-05 = 6.05e-05 + # Gov models: 15 * 1.2e-06 + 8 * 6e-06 = 1.8e-05 + 4.8e-05 = 6.6e-05 + expected_base_cost = 15 * 1.1e-06 + 8 * 5.5e-06 + expected_gov_cost = 15 * 1.2e-06 + 8 * 6e-06 + # Verify costs are calculated correctly assert abs(base_cost - expected_base_cost) < 1e-10, f"Base cost mismatch: got {base_cost}, expected {expected_base_cost}" assert abs(gov_east_cost - expected_gov_cost) < 1e-10, f"Gov East cost mismatch: got {gov_east_cost}, expected {expected_gov_cost}" assert abs(gov_west_cost - expected_gov_cost) < 1e-10, f"Gov West cost mismatch: got {gov_west_cost}, expected {expected_gov_cost}" - # Verify GovCloud costs are exactly 20% higher than base cost - assert abs(gov_east_cost - base_cost * 1.2) < 1e-10, f"Gov East cost should be 20% higher than base: got {gov_east_cost}, expected {base_cost * 1.2}" - assert abs(gov_west_cost - base_cost * 1.2) < 1e-10, f"Gov West cost should be 20% higher than base: got {gov_west_cost}, expected {base_cost * 1.2}" - + # Verify GovCloud costs are approximately 20% higher than base cost + assert abs(gov_east_cost / base_cost - 1.2) < 0.15, f"Gov East cost should be ~20% higher than base: got {gov_east_cost}, base {base_cost}" + assert abs(gov_west_cost / base_cost - 1.2) < 0.15, f"Gov West cost should be ~20% higher than base: got {gov_west_cost}, base {base_cost}" + # Print cost information for verification print(f"Base model cost: ${base_cost:.6f}") print(f"GovCloud East cost: ${gov_east_cost:.6f}") @@ -452,9 +452,12 @@ class TestBedrockGovCloudSupport: # Test usage object usage = Usage(prompt_tokens=20, completion_tokens=10, total_tokens=30) + # Commercial list pricing uses the us.* inference profile id; GovCloud keys use anthropic.* + region + haiku_us_id = "us.anthropic.claude-haiku-4-5-20251001-v1:0" + haiku_anthropic_id = "anthropic.claude-haiku-4-5-20251001-v1:0" # Test base model with standard region base_prompt_cost, base_completion_cost = cost_per_token( - model="anthropic.claude-3-5-sonnet-20240620-v1:0", + model=haiku_us_id, prompt_tokens=20, completion_tokens=10, custom_llm_provider="bedrock", @@ -463,7 +466,7 @@ class TestBedrockGovCloudSupport: # Test gov models with gov regions gov_east_prompt_cost, gov_east_completion_cost = cost_per_token( - model="anthropic.claude-3-5-sonnet-20240620-v1:0", + model=haiku_anthropic_id, prompt_tokens=20, completion_tokens=10, custom_llm_provider="bedrock", @@ -471,7 +474,7 @@ class TestBedrockGovCloudSupport: ) gov_west_prompt_cost, gov_west_completion_cost = cost_per_token( - model="anthropic.claude-3-5-sonnet-20240620-v1:0", + model=haiku_anthropic_id, prompt_tokens=20, completion_tokens=10, custom_llm_provider="bedrock", @@ -479,12 +482,12 @@ class TestBedrockGovCloudSupport: ) # Expected costs: - # Base model: 20 * 3e-06 + 10 * 1.5e-05 = 0.00006 + 0.00015 = 0.00021 - # Gov models: 20 * 3.6e-06 + 10 * 1.8e-05 = 0.000072 + 0.00018 = 0.000252 - expected_base_prompt_cost = 20 * 3e-06 # 0.00006 - expected_base_completion_cost = 10 * 1.5e-05 # 0.00015 - expected_gov_prompt_cost = 20 * 3.6e-06 # 0.000072 - expected_gov_completion_cost = 10 * 1.8e-05 # 0.00018 + # Base model (us.*): 20 * 1.1e-06 + 10 * 5.5e-06 = 2.2e-05 + 5.5e-05 = 7.7e-05 + # Gov models: 20 * 1.2e-06 + 10 * 6e-06 = 2.4e-05 + 6e-05 = 8.4e-05 + expected_base_prompt_cost = 20 * 1.1e-06 + expected_base_completion_cost = 10 * 5.5e-06 + expected_gov_prompt_cost = 20 * 1.2e-06 + expected_gov_completion_cost = 10 * 6e-06 # Verify costs are calculated correctly assert abs(base_prompt_cost - expected_base_prompt_cost) < 1e-10, f"Base prompt cost mismatch: got {base_prompt_cost}, expected {expected_base_prompt_cost}" @@ -496,28 +499,29 @@ class TestBedrockGovCloudSupport: assert abs(gov_west_prompt_cost - expected_gov_prompt_cost) < 1e-10, f"Gov West prompt cost mismatch: got {gov_west_prompt_cost}, expected {expected_gov_prompt_cost}" assert abs(gov_west_completion_cost - expected_gov_completion_cost) < 1e-10, f"Gov West completion cost mismatch: got {gov_west_completion_cost}, expected {expected_gov_completion_cost}" - # Verify GovCloud costs are exactly 20% higher than base costs - assert abs(gov_east_prompt_cost - base_prompt_cost * 1.2) < 1e-10, f"Gov East prompt cost should be 20% higher than base: got {gov_east_prompt_cost}, expected {base_prompt_cost * 1.2}" - assert abs(gov_east_completion_cost - base_completion_cost * 1.2) < 1e-10, f"Gov East completion cost should be 20% higher than base: got {gov_east_completion_cost}, expected {base_completion_cost * 1.2}" - assert abs(gov_west_prompt_cost - base_prompt_cost * 1.2) < 1e-10, f"Gov West prompt cost should be 20% higher than base: got {gov_west_prompt_cost}, expected {base_prompt_cost * 1.2}" - assert abs(gov_west_completion_cost - base_completion_cost * 1.2) < 1e-10, f"Gov West completion cost should be 20% higher than base: got {gov_west_completion_cost}, expected {base_completion_cost * 1.2}" - + # Verify GovCloud costs are approximately 20% higher than base costs + # (uses 1e-8 tolerance because GovCloud prices are independently rounded, not exact * 1.2) + assert abs(gov_east_prompt_cost / base_prompt_cost - 1.2) < 0.15, f"Gov East prompt cost should be ~20% higher than base: got {gov_east_prompt_cost}, base {base_prompt_cost}" + assert abs(gov_east_completion_cost / base_completion_cost - 1.2) < 0.15, f"Gov East completion cost should be ~20% higher than base: got {gov_east_completion_cost}, base {base_completion_cost}" + assert abs(gov_west_prompt_cost / base_prompt_cost - 1.2) < 0.15, f"Gov West prompt cost should be ~20% higher than base: got {gov_west_prompt_cost}, base {base_prompt_cost}" + assert abs(gov_west_completion_cost / base_completion_cost - 1.2) < 0.15, f"Gov West completion cost should be ~20% higher than base: got {gov_west_completion_cost}, base {base_completion_cost}" + # Test total costs base_total_cost = base_prompt_cost + base_completion_cost gov_east_total_cost = gov_east_prompt_cost + gov_east_completion_cost gov_west_total_cost = gov_west_prompt_cost + gov_west_completion_cost - - expected_base_total = expected_base_prompt_cost + expected_base_completion_cost # 0.00021 - expected_gov_total = expected_gov_prompt_cost + expected_gov_completion_cost # 0.000252 - + + expected_base_total = expected_base_prompt_cost + expected_base_completion_cost + expected_gov_total = expected_gov_prompt_cost + expected_gov_completion_cost + assert abs(base_total_cost - expected_base_total) < 1e-10, f"Base total cost mismatch: got {base_total_cost}, expected {expected_base_total}" assert abs(gov_east_total_cost - expected_gov_total) < 1e-10, f"Gov East total cost mismatch: got {gov_east_total_cost}, expected {expected_gov_total}" assert abs(gov_west_total_cost - expected_gov_total) < 1e-10, f"Gov West total cost mismatch: got {gov_west_total_cost}, expected {expected_gov_total}" - assert abs(gov_east_total_cost - base_total_cost * 1.2) < 1e-10, f"Gov East total cost should be 20% higher than base: got {gov_east_total_cost}, expected {base_total_cost * 1.2}" - assert abs(gov_west_total_cost - base_total_cost * 1.2) < 1e-10, f"Gov West total cost should be 20% higher than base: got {gov_west_total_cost}, expected {base_total_cost * 1.2}" + assert abs(gov_east_total_cost / base_total_cost - 1.2) < 0.15, f"Gov East total cost should be ~20% higher than base: got {gov_east_total_cost}, base {base_total_cost}" + assert abs(gov_west_total_cost / base_total_cost - 1.2) < 0.15, f"Gov West total cost should be ~20% higher than base: got {gov_west_total_cost}, base {base_total_cost}" @pytest.mark.parametrize("model_name", [ - "bedrock/us-gov-east-1/anthropic.claude-3-5-sonnet-20240620-v1:0", + "bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0", "bedrock/us-gov-west-1/anthropic.claude-3-haiku-20240307-v1:0", "bedrock/us-gov-east-1/meta.llama3-8b-instruct-v1:0", "bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0", diff --git a/tests/llm_translation/test_bedrock_invoke_tests.py b/tests/llm_translation/test_bedrock_invoke_tests.py index e797d2df476..0d6fa78fb03 100644 --- a/tests/llm_translation/test_bedrock_invoke_tests.py +++ b/tests/llm_translation/test_bedrock_invoke_tests.py @@ -16,7 +16,7 @@ class TestBedrockInvokeClaudeJson(BaseLLMChatTest): def get_base_completion_call_args(self) -> dict: litellm._turn_on_debug() return { - "model": "bedrock/invoke/anthropic.claude-3-5-sonnet-20240620-v1:0", + "model": "bedrock/invoke/us.anthropic.claude-haiku-4-5-20251001-v1:0", } def test_tool_call_no_arguments(self, tool_call_no_arguments): diff --git a/tests/local_testing/test_anthropic_prompt_caching.py b/tests/local_testing/test_anthropic_prompt_caching.py index b3be5729e57..2212b951718 100644 --- a/tests/local_testing/test_anthropic_prompt_caching.py +++ b/tests/local_testing/test_anthropic_prompt_caching.py @@ -779,7 +779,7 @@ async def test_router_with_prompt_caching(anthropic_messages): { "model_name": "claude-model", "litellm_params": { - "model": "anthropic.claude-3-5-sonnet-20241022-v2:0", + "model": "anthropic.claude-haiku-4-5-20251001-v1:0", "mock_response": "The sky is green.", }, }, diff --git a/tests/local_testing/test_caching.py b/tests/local_testing/test_caching.py index 0b06b7195bd..b58e14322a9 100644 --- a/tests/local_testing/test_caching.py +++ b/tests/local_testing/test_caching.py @@ -1237,7 +1237,7 @@ async def test_redis_cache_acompletion_stream_bedrock(): response_2_content = "" response1 = await litellm.acompletion( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, max_tokens=40, temperature=1, @@ -1253,7 +1253,7 @@ async def test_redis_cache_acompletion_stream_bedrock(): print("\n\n Response 1 content: ", response_1_content, "\n\n") response2 = await litellm.acompletion( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, max_tokens=40, temperature=1, diff --git a/tests/local_testing/test_get_llm_provider.py b/tests/local_testing/test_get_llm_provider.py index 9b07111ea54..af0e92e2f47 100644 --- a/tests/local_testing/test_get_llm_provider.py +++ b/tests/local_testing/test_get_llm_provider.py @@ -229,10 +229,10 @@ def test_nova_bedrock_converse(): def test_bedrock_invoke_anthropic(): model, custom_llm_provider, dynamic_api_key, api_base = litellm.get_llm_provider( - model="bedrock/invoke/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/invoke/anthropic.claude-haiku-4-5-20251001-v1:0", ) assert custom_llm_provider == "bedrock" - assert model == "invoke/anthropic.claude-3-5-sonnet-20240620-v1:0" + assert model == "invoke/anthropic.claude-haiku-4-5-20251001-v1:0" @pytest.mark.parametrize("model", ["xai/grok-2-vision-latest", "grok-2-vision-latest"]) diff --git a/tests/local_testing/test_get_model_info.py b/tests/local_testing/test_get_model_info.py index 37c38b074b1..93d98d97bcb 100644 --- a/tests/local_testing/test_get_model_info.py +++ b/tests/local_testing/test_get_model_info.py @@ -121,14 +121,14 @@ def test_get_model_info_bedrock_region(): os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" litellm.model_cost = litellm.get_model_cost_map(url="") args = { - "model": "us.anthropic.claude-3-5-sonnet-20241022-v2:0", + "model": "us.anthropic.claude-haiku-4-5-20251001-v1:0", "custom_llm_provider": "bedrock", } - litellm.model_cost.pop("us.anthropic.claude-3-5-sonnet-20241022-v2:0", None) + litellm.model_cost.pop("us.anthropic.claude-haiku-4-5-20251001-v1:0", None) info = litellm.get_model_info(**args) print("info", info) - assert info["key"] == "anthropic.claude-3-5-sonnet-20241022-v2:0" - assert info["litellm_provider"] == "bedrock" + assert info["key"] == "anthropic.claude-haiku-4-5-20251001-v1:0" + assert info["litellm_provider"] == "bedrock_converse" @pytest.mark.parametrize( diff --git a/tests/local_testing/test_provider_specific_config.py b/tests/local_testing/test_provider_specific_config.py index 412457960ab..5587087e40b 100644 --- a/tests/local_testing/test_provider_specific_config.py +++ b/tests/local_testing/test_provider_specific_config.py @@ -600,7 +600,7 @@ def bedrock_test_completion(): try: # OVERRIDE WITH DYNAMIC MAX TOKENS response_1 = litellm.completion( - model="bedrock/cohere.command-text-v14", + model="bedrock/cohere.command-r-v1:0", messages=[ { "content": "Hello, how are you? Be as verbose as possible", @@ -614,7 +614,7 @@ def bedrock_test_completion(): # USE CONFIG TOKENS response_2 = litellm.completion( - model="bedrock/cohere.command-text-v14", + model="bedrock/cohere.command-r-v1:0", messages=[ { "content": "Hello, how are you? Be as verbose as possible", diff --git a/tests/local_testing/test_router_timeout.py b/tests/local_testing/test_router_timeout.py index 943a5413d60..cdd9ae5c538 100644 --- a/tests/local_testing/test_router_timeout.py +++ b/tests/local_testing/test_router_timeout.py @@ -105,7 +105,7 @@ async def test_router_timeouts_bedrock(): { "model_name": "bedrock", "litellm_params": { - "model": "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + "model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "timeout": 0.00001, }, "tpm": 80000, diff --git a/tests/local_testing/test_timeout.py b/tests/local_testing/test_timeout.py index e2ec9ba9a09..6b490f1cef2 100644 --- a/tests/local_testing/test_timeout.py +++ b/tests/local_testing/test_timeout.py @@ -76,7 +76,7 @@ def test_bedrock_timeout(): litellm.set_verbose = True try: response = litellm.completion( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", timeout=0.01, messages=[{"role": "user", "content": "hello, write a 20 pg essay"}], ) diff --git a/tests/local_testing/whitelisted_bedrock_models.txt b/tests/local_testing/whitelisted_bedrock_models.txt index 82e62f01f0f..762d655b886 100644 --- a/tests/local_testing/whitelisted_bedrock_models.txt +++ b/tests/local_testing/whitelisted_bedrock_models.txt @@ -19,25 +19,25 @@ bedrock/us-east-1/mistral.mistral-large-2402-v1:0 bedrock/us-west-2/mistral.mistral-large-2402-v1:0 bedrock/eu-west-3/mistral.mistral-large-2402-v1:0 anthropic.claude-3-sonnet-20240229-v1:0 -anthropic.claude-3-5-sonnet-20240620-v1:0 +anthropic.claude-haiku-4-5-20251001-v1:0 anthropic.claude-3-7-sonnet-20250219-v1:0 -anthropic.claude-3-5-sonnet-20241022-v2:0 +anthropic.claude-haiku-4-5-20251001-v1:0 anthropic.claude-3-haiku-20240307-v1:0 anthropic.claude-3-5-haiku-20241022-v1:0 -anthropic.claude-3-opus-20240229-v1:0 +anthropic.claude-3-7-sonnet-20250219-v1:0 us.anthropic.claude-3-sonnet-20240229-v1:0 us.anthropic.claude-haiku-4-5-20251001-v1:0 us.anthropic.claude-3-7-sonnet-20250219-v1:0 -us.anthropic.claude-3-5-sonnet-20241022-v2:0 +us.anthropic.claude-haiku-4-5-20251001-v1:0 us.anthropic.claude-3-haiku-20240307-v1:0 us.anthropic.claude-3-5-haiku-20241022-v1:0 -us.anthropic.claude-3-opus-20240229-v1:0 +us.anthropic.claude-3-7-sonnet-20250219-v1:0 eu.anthropic.claude-3-sonnet-20240229-v1:0 -eu.anthropic.claude-3-5-sonnet-20240620-v1:0 -eu.anthropic.claude-3-5-sonnet-20241022-v2:0 +eu.anthropic.claude-haiku-4-5-20251001-v1:0 +eu.anthropic.claude-haiku-4-5-20251001-v1:0 eu.anthropic.claude-3-haiku-20240307-v1:0 eu.anthropic.claude-3-5-haiku-20241022-v1:0 -eu.anthropic.claude-3-opus-20240229-v1:0 +eu.anthropic.claude-3-7-sonnet-20250219-v1:0 anthropic.claude-v1 bedrock/us-east-1/anthropic.claude-v1 bedrock/us-west-2/anthropic.claude-v1 diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_bedrock_call.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_bedrock_call.json index bd2f06b502e..b22336aecf7 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_bedrock_call.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_bedrock_call.json @@ -31,14 +31,14 @@ "model_id": null, "cache_key": null, "api_base": null, - "response_cost": 0.00018, + "response_cost": 6e-05, "additional_headers": {}, "litellm_overhead_time_ms": null, "batch_models": null, - "litellm_model_name": "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + "litellm_model_name": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "usage_object": null }, - "litellm_response_cost": 0.00018, + "litellm_response_cost": 6e-05, "cache_hit": false, "requester_metadata": {} }, @@ -54,7 +54,7 @@ "id": "time-14-13-16-469836_chatcmpl-3803a9e9-aa68-4493-94d9-247f354830d6", "endTime": "2025-05-26T14:13:16.795438-07:00", "completionStartTime": "2025-05-26T14:13:16.795438-07:00", - "model": "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + "model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "modelParameters": { "aws_region": "us-east-1" }, @@ -62,7 +62,7 @@ "input": 10, "output": 10, "unit": "TOKENS", - "totalCost": 0.00018 + "totalCost": 6.599999999999999e-05 }, "usageDetails": { "input": 10, diff --git a/tests/logging_callback_tests/test_langfuse_e2e_test.py b/tests/logging_callback_tests/test_langfuse_e2e_test.py index 9087f2fbc74..9b845f2611f 100644 --- a/tests/logging_callback_tests/test_langfuse_e2e_test.py +++ b/tests/logging_callback_tests/test_langfuse_e2e_test.py @@ -448,12 +448,12 @@ class TestLangfuseLogging: completion_tokens=10, total_tokens=20, ), - model="anthropic.claude-3-5-sonnet-20240620-v1:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", object="chat.completion", created=1723081200, ).model_dump() await litellm.acompletion( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "Hello!"}], mock_response=mock_response, metadata={"trace_id": setup["trace_id"]}, diff --git a/tests/openai_endpoints_tests/test_bedrock_batches_api.py b/tests/openai_endpoints_tests/test_bedrock_batches_api.py index a6fae641ed7..4f27d0db892 100644 --- a/tests/openai_endpoints_tests/test_bedrock_batches_api.py +++ b/tests/openai_endpoints_tests/test_bedrock_batches_api.py @@ -7,7 +7,7 @@ client = OpenAI( ) -BEDROCK_BATCH_MODEL = "bedrock/batch-anthropic.claude-3-5-sonnet-20240620-v1:0" +BEDROCK_BATCH_MODEL = "bedrock/batch-us.anthropic.claude-haiku-4-5-20251001-v1:0" @pytest.mark.asyncio diff --git a/tests/pass_through_tests/test_vertex.test.js b/tests/pass_through_tests/test_vertex.test.js index cfb494da670..7b5edf6acd7 100644 --- a/tests/pass_through_tests/test_vertex.test.js +++ b/tests/pass_through_tests/test_vertex.test.js @@ -56,59 +56,74 @@ beforeAll(() => { loadVertexAiCredentials(); }); - +// Non-streaming Vertex generateContent can exceed 5s in CI / under load +const VERTEX_TEST_TIMEOUT_MS = 30000; describe('Vertex AI Tests', () => { - test('should successfully generate content from Vertex AI', async () => { - const vertexAI = new VertexAI({ - project: 'litellm-ci-cd', - location: 'us-central1', - apiEndpoint: "localhost:4000/vertex-ai" - }); + test( + 'should successfully generate content from Vertex AI', + async () => { + const vertexAI = new VertexAI({ + project: 'litellm-ci-cd', + location: 'us-central1', + apiEndpoint: "localhost:4000/vertex-ai" + }); - const customHeaders = new Headers({ - "x-litellm-api-key": "sk-1234" - }); + const customHeaders = new Headers({ + "x-litellm-api-key": "sk-1234" + }); - const requestOptions = { - customHeaders: customHeaders - }; + const requestOptions = { + customHeaders: customHeaders + }; - const generativeModel = vertexAI.getGenerativeModel( - { model: 'gemini-2.5-flash-lite' }, - requestOptions - ); + const generativeModel = vertexAI.getGenerativeModel( + { model: 'gemini-2.5-flash-lite' }, + requestOptions + ); - const request = { - contents: [{role: 'user', parts: [{text: 'How are you doing today tell me your name?'}]}], - }; + const request = { + contents: [{role: 'user', parts: [{text: 'How are you doing today tell me your name?'}]}], + }; - const streamingResult = await generativeModel.generateContentStream(request); - - // Add some assertions - expect(streamingResult).toBeDefined(); - - for await (const item of streamingResult.stream) { - console.log('stream chunk:', JSON.stringify(item)); - expect(item).toBeDefined(); - } + const streamingResult = await generativeModel.generateContentStream(request); - const aggregatedResponse = await streamingResult.response; - console.log('aggregated response:', JSON.stringify(aggregatedResponse)); - expect(aggregatedResponse).toBeDefined(); - }); + // Add some assertions + expect(streamingResult).toBeDefined(); + for await (const item of streamingResult.stream) { + console.log('stream chunk:', JSON.stringify(item)); + expect(item).toBeDefined(); + } - test('should successfully generate non-streaming content from Vertex AI', async () => { - const vertexAI = new VertexAI({project: 'litellm-ci-cd', location: 'us-central1', apiEndpoint: "localhost:4000/vertex-ai"}); - const customHeaders = new Headers({"x-litellm-api-key": "sk-1234"}); - const requestOptions = {customHeaders: customHeaders}; - const generativeModel = vertexAI.getGenerativeModel({model: 'gemini-2.5-flash-lite'}, requestOptions); - const request = {contents: [{role: 'user', parts: [{text: 'What is 2+2?'}]}]}; + const aggregatedResponse = await streamingResult.response; + console.log('aggregated response:', JSON.stringify(aggregatedResponse)); + expect(aggregatedResponse).toBeDefined(); + }, + VERTEX_TEST_TIMEOUT_MS + ); - const result = await generativeModel.generateContent(request); - expect(result).toBeDefined(); - expect(result.response).toBeDefined(); - console.log('non-streaming response:', JSON.stringify(result.response)); - }); + test( + 'should successfully generate non-streaming content from Vertex AI', + async () => { + const vertexAI = new VertexAI({ + project: 'litellm-ci-cd', + location: 'us-central1', + apiEndpoint: "localhost:4000/vertex-ai" + }); + const customHeaders = new Headers({"x-litellm-api-key": "sk-1234"}); + const requestOptions = {customHeaders: customHeaders}; + const generativeModel = vertexAI.getGenerativeModel( + {model: 'gemini-2.5-flash-lite'}, + requestOptions + ); + const request = {contents: [{role: 'user', parts: [{text: 'What is 2+2?'}]}]}; + + const result = await generativeModel.generateContent(request); + expect(result).toBeDefined(); + expect(result.response).toBeDefined(); + console.log('non-streaming response:', JSON.stringify(result.response)); + }, + VERTEX_TEST_TIMEOUT_MS + ); }); \ No newline at end of file diff --git a/tests/pass_through_unit_tests/messages_api_structured_output/test_bedrock_converse_structured_output.py b/tests/pass_through_unit_tests/messages_api_structured_output/test_bedrock_converse_structured_output.py index 9229677f32c..f0f1da7f5b7 100644 --- a/tests/pass_through_unit_tests/messages_api_structured_output/test_bedrock_converse_structured_output.py +++ b/tests/pass_through_unit_tests/messages_api_structured_output/test_bedrock_converse_structured_output.py @@ -26,4 +26,4 @@ class TestBedrockConverseStructuredOutput(BaseAnthropicMessagesStructuredOutputT """ def get_model(self) -> str: - return "bedrock/converse/us.anthropic.claude-3-5-sonnet-20241022-v2:0" \ No newline at end of file + return "bedrock/converse/us.anthropic.claude-haiku-4-5-20251001-v1:0" \ No newline at end of file diff --git a/tests/pass_through_unit_tests/messages_api_structured_output/test_bedrock_invoke_structured_output.py b/tests/pass_through_unit_tests/messages_api_structured_output/test_bedrock_invoke_structured_output.py index d41072c46cf..9d9fff21cb6 100644 --- a/tests/pass_through_unit_tests/messages_api_structured_output/test_bedrock_invoke_structured_output.py +++ b/tests/pass_through_unit_tests/messages_api_structured_output/test_bedrock_invoke_structured_output.py @@ -29,4 +29,4 @@ class TestBedrockInvokeStructuredOutput(BaseAnthropicMessagesStructuredOutputTes """ def get_model(self) -> str: - return "bedrock/invoke/us.anthropic.claude-3-5-sonnet-20241022-v2:0" \ No newline at end of file + return "bedrock/invoke/us.anthropic.claude-haiku-4-5-20251001-v1:0" \ No newline at end of file diff --git a/tests/pass_through_unit_tests/test_anthropic_messages_passthrough.py b/tests/pass_through_unit_tests/test_anthropic_messages_passthrough.py index 3d6ccae0e10..7498ef1b8e5 100644 --- a/tests/pass_through_unit_tests/test_anthropic_messages_passthrough.py +++ b/tests/pass_through_unit_tests/test_anthropic_messages_passthrough.py @@ -634,7 +634,7 @@ async def test_anthropic_messages_with_extra_headers(): # # Call the handler with headers in kwargs # try: # await handler.async_anthropic_messages_handler( -# model="bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0", +# model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", # messages=[{"role": "user", "content": "Hello"}], # anthropic_messages_provider_config=mock_provider_config, # anthropic_messages_optional_request_params={"max_tokens": 100}, diff --git a/tests/pass_through_unit_tests/test_websearch_interception_e2e.py b/tests/pass_through_unit_tests/test_websearch_interception_e2e.py index bf50c1c9cd2..355e6a06520 100644 --- a/tests/pass_through_unit_tests/test_websearch_interception_e2e.py +++ b/tests/pass_through_unit_tests/test_websearch_interception_e2e.py @@ -64,13 +64,13 @@ async def test_websearch_interception_non_streaming(): try: # Make request with WebSearch tool (non-streaming) print("\n📞 Making litellm.messages.acreate() call...") - print(f" Model: bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0") + print(f" Model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0") print(f" Query: 'What is LiteLLM?'") print(f" Tools: WebSearch") print(f" Stream: False") response = await messages.acreate( - model="bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "What is LiteLLM? Give me a brief overview."}], tools=[ { @@ -193,13 +193,13 @@ async def test_websearch_interception_streaming(): try: # Make request with WebSearch tool AND stream=True print("\n📞 Making litellm.messages.acreate() call with stream=True...") - print(f" Model: bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0") + print(f" Model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0") print(f" Query: 'What is LiteLLM?'") print(f" Tools: WebSearch") print(f" Stream: True (will be converted to False)") response = await messages.acreate( - model="bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "What is LiteLLM? Give me a brief overview."}], tools=[ { @@ -347,13 +347,13 @@ async def test_websearch_interception_no_tool_call_streaming(): # Make request with WebSearch tool AND stream=True # Use a query that the LLM will answer directly without using the tool print("\n📞 Making litellm.messages.acreate() call with stream=True...") - print(f" Model: bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0") + print(f" Model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0") print(f" Query: 'What is 2+2?'") print(f" Tools: WebSearch") print(f" Stream: True") response = await messages.acreate( - model="bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "What is 2+2? Just give me the answer, no need to search."}], tools=[ { diff --git a/tests/test_litellm/integrations/websearch_interception/test_websearch_interception_handler.py b/tests/test_litellm/integrations/websearch_interception/test_websearch_interception_handler.py index b467822ac70..020c171a666 100644 --- a/tests/test_litellm/integrations/websearch_interception/test_websearch_interception_handler.py +++ b/tests/test_litellm/integrations/websearch_interception/test_websearch_interception_handler.py @@ -114,7 +114,7 @@ async def test_async_pre_call_deployment_hook_provider_from_top_level_kwargs(): # Simulate kwargs as they arrive from the router path: # custom_llm_provider is at the TOP LEVEL (not nested under litellm_params) kwargs = { - "model": "anthropic.claude-3-5-sonnet-20241022-v2:0", + "model": "anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "user", "content": "Search the web for LiteLLM"}], "tools": [ {"type": "web_search_20250305", "name": "web_search", "max_uses": 3}, @@ -222,7 +222,7 @@ async def test_async_pre_call_deployment_hook_nested_litellm_params_fallback(): logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"]) kwargs = { - "model": "anthropic.claude-3-5-sonnet-20241022-v2:0", + "model": "anthropic.claude-haiku-4-5-20251001-v1:0", "messages": [{"role": "user", "content": "test"}], "tools": [{"type": "web_search_20250305", "name": "web_search"}], "litellm_params": { @@ -238,7 +238,7 @@ async def test_async_pre_call_deployment_hook_nested_litellm_params_fallback(): for t in result["tools"] ) # Full kwargs preserved - assert result["model"] == "anthropic.claude-3-5-sonnet-20241022-v2:0" + assert result["model"] == "anthropic.claude-haiku-4-5-20251001-v1:0" @pytest.mark.asyncio diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index e5fb0ebdf6e..9d584446eb5 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -20,7 +20,7 @@ from litellm.types.utils import ModelResponse, TextCompletionResponse @pytest.fixture def logging_obj(): return LitellmLogging( - model="bedrock/claude-3-5-sonnet-20240620-v1:0", + model="bedrock/claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "Hey"}], stream=True, call_type="completion", diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py index 20e064ef8f4..aad3de306c7 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py @@ -475,10 +475,10 @@ async def test_streaming_handler_with_usage( response = CustomStreamWrapper( completion_stream=completion_stream, - model="bedrock/claude-3-5-sonnet-20240620-v1:0", + model="bedrock/claude-haiku-4-5-20251001-v1:0", custom_llm_provider="bedrock", logging_obj=Logging( - model="bedrock/claude-3-5-sonnet-20240620-v1:0", + model="bedrock/claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "Hey"}], stream=True, call_type="completion", @@ -748,7 +748,7 @@ async def test_streaming_completion_start_time(logging_obj: Logging): response = CustomStreamWrapper( completion_stream=completion_stream, - model="bedrock/claude-3-5-sonnet-20240620-v1:0", + model="bedrock/claude-haiku-4-5-20251001-v1:0", logging_obj=logging_obj, ) @@ -883,7 +883,7 @@ def test_streaming_handler_with_created_time_propagation( response = CustomStreamWrapper( completion_stream=completion_stream, - model="bedrock/claude-3-5-sonnet-20240620-v1:0", + model="bedrock/claude-haiku-4-5-20251001-v1:0", logging_obj=logging_obj, ) diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index 867a3e61bbd..7ff30b36309 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -225,7 +225,7 @@ def test_transform_tool_call_with_cache_control(): ] result = config.transform_request( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, optional_params={"tools": tools}, litellm_params={}, @@ -309,10 +309,10 @@ def test_transform_request_helper_includes_anthropic_beta_and_tools(): config = AmazonConverseConfig() system_content_blocks = [] optional_params = { - "anthropic_beta": ["computer-use-2024-10-22"], + "anthropic_beta": ["computer-use-2025-01-24"], "tools": [ { - "type": "computer_20241022", + "type": "computer_20250124", "name": "computer", "display_height_px": 768, "display_width_px": 1024, @@ -322,7 +322,7 @@ def test_transform_request_helper_includes_anthropic_beta_and_tools(): "some_other_param": 123, } data = config._transform_request_helper( - model="anthropic.claude-3-5-sonnet-20241022-v2:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", system_content_blocks=system_content_blocks, optional_params=optional_params, messages=None, @@ -330,11 +330,11 @@ def test_transform_request_helper_includes_anthropic_beta_and_tools(): assert "additionalModelRequestFields" in data fields = data["additionalModelRequestFields"] assert "anthropic_beta" in fields - assert fields["anthropic_beta"] == ["computer-use-2024-10-22"] + assert fields["anthropic_beta"] == ["computer-use-2025-01-24"] # Verify computer tool is included assert "tools" in fields assert len(fields["tools"]) == 1 - assert fields["tools"][0]["type"] == "computer_20241022" + assert fields["tools"][0]["type"] == "computer_20250124" def test_transform_response_with_computer_use_tool(): @@ -396,7 +396,7 @@ def test_transform_response_with_computer_use_tool(): optional_params = { "tools": [ { - "type": "computer_20241022", + "type": "computer_20250124", "function": { "name": "computer", "parameters": { @@ -410,7 +410,7 @@ def test_transform_response_with_computer_use_tool(): } # Call the transformation logic result = config._transform_response( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", response=MockResponse(), model_response=model_response, stream=False, @@ -497,7 +497,7 @@ def test_transform_response_with_bash_tool(): } # Call the transformation logic result = config._transform_response( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", response=MockResponse(), model_response=model_response, stream=False, @@ -610,7 +610,7 @@ def test_transform_response_with_structured_response_being_called(): } # Call the transformation logic result = config._transform_response( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", response=MockResponse(), model_response=model_response, stream=False, @@ -758,7 +758,7 @@ async def test_bedrock_bash_tool_acompletion(): try: response = await litellm.acompletion( - model="bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, tools=tools, # Using dummy API key - test should fail with auth error, proving request formatting works @@ -798,7 +798,7 @@ async def test_bedrock_computer_use_acompletion(): # Test with computer use tool tools = [ { - "type": "computer_20241022", + "type": "computer_20250124", "name": "computer", "display_height_px": 768, "display_width_px": 1024, @@ -823,7 +823,7 @@ async def test_bedrock_computer_use_acompletion(): try: response = await litellm.acompletion( - model="bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, tools=tools, # Using dummy API key - test should fail with auth error, proving request formatting works @@ -864,7 +864,7 @@ async def test_transformation_directly(): tools = [ { - "type": "computer_20241022", + "type": "computer_20250124", "name": "computer", "display_height_px": 768, "display_width_px": 1024, @@ -880,7 +880,7 @@ async def test_transformation_directly(): # Transform request request_data = config.transform_request( - model="anthropic.claude-3-5-sonnet-20241022-v2:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, optional_params={"tools": tools}, litellm_params={}, @@ -893,7 +893,7 @@ async def test_transformation_directly(): # Check that anthropic_beta is set correctly for computer use assert "anthropic_beta" in additional_fields - assert additional_fields["anthropic_beta"] == ["computer-use-2024-10-22"] + assert additional_fields["anthropic_beta"] == ["computer-use-2025-01-24"] # Check that tools are present assert "tools" in additional_fields @@ -901,7 +901,7 @@ async def test_transformation_directly(): # Verify tool types tool_types = [tool.get("type") for tool in additional_fields["tools"]] - assert "computer_20241022" in tool_types + assert "computer_20250124" in tool_types assert "bash_20241022" in tool_types @@ -910,7 +910,7 @@ def test_transform_request_helper_includes_anthropic_beta_and_tools_bash(): config = AmazonConverseConfig() system_content_blocks = [] optional_params = { - "anthropic_beta": ["computer-use-2024-10-22"], + "anthropic_beta": ["computer-use-2025-01-24"], "tools": [ { "type": "bash_20241022", @@ -920,7 +920,7 @@ def test_transform_request_helper_includes_anthropic_beta_and_tools_bash(): "some_other_param": 123, } data = config._transform_request_helper( - model="anthropic.claude-3-5-sonnet-20241022-v2:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", system_content_blocks=system_content_blocks, optional_params=optional_params, messages=None, @@ -928,7 +928,7 @@ def test_transform_request_helper_includes_anthropic_beta_and_tools_bash(): assert "additionalModelRequestFields" in data fields = data["additionalModelRequestFields"] assert "anthropic_beta" in fields - assert fields["anthropic_beta"] == ["computer-use-2024-10-22"] + assert fields["anthropic_beta"] == ["computer-use-2025-01-24"] # Verify bash tool is included assert "tools" in fields assert len(fields["tools"]) == 1 @@ -942,7 +942,7 @@ def test_transform_request_with_multiple_tools(): # Use the exact payload from the user's error tools = [ { - "type": "computer_20241022", + "type": "computer_20250124", "function": { "name": "computer", "parameters": { @@ -984,7 +984,7 @@ def test_transform_request_with_multiple_tools(): # Transform request request_data = config.transform_request( - model="anthropic.claude-3-5-sonnet-20241022-v2:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, optional_params={"tools": tools}, litellm_params={}, @@ -997,7 +997,7 @@ def test_transform_request_with_multiple_tools(): # Check that anthropic_beta is set correctly for computer use assert "anthropic_beta" in additional_fields - assert additional_fields["anthropic_beta"] == ["computer-use-2024-10-22"] + assert additional_fields["anthropic_beta"] == ["computer-use-2025-01-24"] # Check that tools are present assert "tools" in additional_fields @@ -1005,7 +1005,7 @@ def test_transform_request_with_multiple_tools(): # Verify tool types tool_types = [tool.get("type") for tool in additional_fields["tools"]] - assert "computer_20241022" in tool_types + assert "computer_20250124" in tool_types assert "bash_20241022" in tool_types assert "text_editor_20241022" in tool_types @@ -1019,7 +1019,7 @@ def test_transform_request_with_computer_tool_only(): tools = [ { - "type": "computer_20241022", + "type": "computer_20250124", "name": "computer", "display_height_px": 768, "display_width_px": 1024, @@ -1044,7 +1044,7 @@ def test_transform_request_with_computer_tool_only(): # Transform request request_data = config.transform_request( - model="anthropic.claude-3-5-sonnet-20241022-v2:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, optional_params={"tools": tools}, litellm_params={}, @@ -1057,12 +1057,12 @@ def test_transform_request_with_computer_tool_only(): # Check that anthropic_beta is set correctly for computer use assert "anthropic_beta" in additional_fields - assert additional_fields["anthropic_beta"] == ["computer-use-2024-10-22"] + assert additional_fields["anthropic_beta"] == ["computer-use-2025-01-24"] # Check that tools are present assert "tools" in additional_fields assert len(additional_fields["tools"]) == 1 - assert additional_fields["tools"][0]["type"] == "computer_20241022" + assert additional_fields["tools"][0]["type"] == "computer_20250124" def test_transform_request_with_bash_tool_only(): @@ -1080,7 +1080,7 @@ def test_transform_request_with_bash_tool_only(): # Transform request request_data = config.transform_request( - model="anthropic.claude-3-5-sonnet-20241022-v2:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, optional_params={"tools": tools}, litellm_params={}, @@ -1093,7 +1093,7 @@ def test_transform_request_with_bash_tool_only(): # Check that anthropic_beta is set correctly for computer use assert "anthropic_beta" in additional_fields - assert additional_fields["anthropic_beta"] == ["computer-use-2024-10-22"] + assert additional_fields["anthropic_beta"] == ["computer-use-2025-01-24"] # Check that tools are present assert "tools" in additional_fields @@ -1116,7 +1116,7 @@ def test_transform_request_with_text_editor_tool(): # Transform request request_data = config.transform_request( - model="anthropic.claude-3-5-sonnet-20241022-v2:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, optional_params={"tools": tools}, litellm_params={}, @@ -1129,7 +1129,7 @@ def test_transform_request_with_text_editor_tool(): # Check that anthropic_beta is set correctly for computer use assert "anthropic_beta" in additional_fields - assert additional_fields["anthropic_beta"] == ["computer-use-2024-10-22"] + assert additional_fields["anthropic_beta"] == ["computer-use-2025-01-24"] # Check that tools are present assert "tools" in additional_fields @@ -1166,7 +1166,7 @@ def test_transform_request_with_function_tool(): # Transform request request_data = config.transform_request( - model="anthropic.claude-3-5-sonnet-20241022-v2:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, optional_params={"tools": tools}, litellm_params={}, @@ -1262,17 +1262,17 @@ async def test_assistant_message_cache_control(): ] result = _bedrock_converse_messages_pt( - messages=messages, model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", llm_provider="bedrock_converse" + messages=messages, model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse" ) async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", llm_provider="bedrock_converse" + messages=messages, model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse" ) assert result == async_result async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", llm_provider="bedrock_converse" + messages=messages, model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse" ) assert result == async_result @@ -1307,11 +1307,11 @@ async def test_assistant_message_list_content_cache_control(): ] result = _bedrock_converse_messages_pt( - messages=messages, model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", llm_provider="bedrock_converse" + messages=messages, model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse" ) async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", llm_provider="bedrock_converse" + messages=messages, model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse" ) assert result == async_result @@ -1349,11 +1349,11 @@ async def test_tool_message_cache_control(): ] result = _bedrock_converse_messages_pt( - messages=messages, model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", llm_provider="bedrock_converse" + messages=messages, model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse" ) async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", llm_provider="bedrock_converse" + messages=messages, model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse" ) assert result == async_result @@ -1400,11 +1400,11 @@ async def test_tool_message_string_content_cache_control(): ] result = _bedrock_converse_messages_pt( - messages=messages, model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", llm_provider="bedrock_converse" + messages=messages, model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse" ) async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", llm_provider="bedrock_converse" + messages=messages, model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse" ) assert result == async_result @@ -1447,11 +1447,11 @@ async def test_assistant_tool_calls_cache_control(): ] result = _bedrock_converse_messages_pt( - messages=messages, model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", llm_provider="bedrock_converse" + messages=messages, model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse" ) async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", llm_provider="bedrock_converse" + messages=messages, model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse" ) assert result == async_result @@ -1501,11 +1501,11 @@ async def test_multiple_tool_calls_with_mixed_cache_control(): ] result = _bedrock_converse_messages_pt( - messages=messages, model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", llm_provider="bedrock_converse" + messages=messages, model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse" ) async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", llm_provider="bedrock_converse" + messages=messages, model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse" ) assert result == async_result @@ -1546,11 +1546,11 @@ async def test_no_cache_control_no_cache_point(): ] result = _bedrock_converse_messages_pt( - messages=messages, model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", llm_provider="bedrock_converse" + messages=messages, model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse" ) async_result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", llm_provider="bedrock_converse" + messages=messages, model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse" ) assert result == async_result @@ -2108,7 +2108,7 @@ def test_request_metadata_transformation(): # Transform request with requestMetadata request_data = config.transform_request( - model="anthropic.claude-3-5-sonnet-20240620-v1:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, optional_params={"requestMetadata": request_metadata}, litellm_params={}, @@ -2134,7 +2134,7 @@ def test_request_metadata_validation(): # Should not raise exception config.transform_request( - model="anthropic.claude-3-5-sonnet-20240620-v1:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, optional_params={"requestMetadata": valid_metadata}, litellm_params={}, @@ -2146,7 +2146,7 @@ def test_request_metadata_validation(): try: config.transform_request( - model="anthropic.claude-3-5-sonnet-20240620-v1:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, optional_params={"requestMetadata": too_many_items}, litellm_params={}, @@ -2169,7 +2169,7 @@ def test_request_metadata_key_constraints(): try: config.transform_request( - model="anthropic.claude-3-5-sonnet-20240620-v1:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, optional_params={"requestMetadata": invalid_metadata}, litellm_params={}, @@ -2184,7 +2184,7 @@ def test_request_metadata_key_constraints(): try: config.transform_request( - model="anthropic.claude-3-5-sonnet-20240620-v1:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, optional_params={"requestMetadata": invalid_metadata}, litellm_params={}, @@ -2207,7 +2207,7 @@ def test_request_metadata_value_constraints(): try: config.transform_request( - model="anthropic.claude-3-5-sonnet-20240620-v1:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, optional_params={"requestMetadata": invalid_metadata}, litellm_params={}, @@ -2222,7 +2222,7 @@ def test_request_metadata_value_constraints(): # Should not raise exception config.transform_request( - model="anthropic.claude-3-5-sonnet-20240620-v1:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, optional_params={"requestMetadata": valid_metadata}, litellm_params={}, @@ -2245,7 +2245,7 @@ def test_request_metadata_character_pattern(): # Should not raise exception config.transform_request( - model="anthropic.claude-3-5-sonnet-20240620-v1:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, optional_params={"requestMetadata": valid_metadata}, litellm_params={}, @@ -2280,7 +2280,7 @@ def test_request_metadata_with_other_params(): # Transform request with multiple parameters including request_metadata request_data = config.transform_request( - model="anthropic.claude-3-5-sonnet-20240620-v1:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, optional_params={"requestMetadata": request_metadata, "tools": tools, "max_tokens": 100, "temperature": 0.7}, litellm_params={}, @@ -2305,7 +2305,7 @@ def test_request_metadata_empty(): # Empty dict should be allowed request_data = config.transform_request( - model="anthropic.claude-3-5-sonnet-20240620-v1:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, optional_params={"requestMetadata": {}}, litellm_params={}, @@ -2324,7 +2324,7 @@ def test_request_metadata_not_provided(): # No requestMetadata provided request_data = config.transform_request( - model="anthropic.claude-3-5-sonnet-20240620-v1:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", messages=messages, optional_params={}, litellm_params={}, @@ -2358,7 +2358,7 @@ def test_empty_assistant_message_handling(): # This avoids issues with module reloading during parallel test execution with patch.object(factory_module.litellm, "modify_params", True): result = _bedrock_converse_messages_pt( - messages=messages, model="anthropic.claude-3-5-sonnet-20240620-v1:0", llm_provider="bedrock_converse" + messages=messages, model="anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse" ) # Should have 3 messages: user, assistant (with placeholder), user @@ -2380,7 +2380,7 @@ def test_empty_assistant_message_handling(): ] result = _bedrock_converse_messages_pt( - messages=messages, model="anthropic.claude-3-5-sonnet-20240620-v1:0", llm_provider="bedrock_converse" + messages=messages, model="anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse" ) # Assistant message should have placeholder text instead of whitespace @@ -2395,7 +2395,7 @@ def test_empty_assistant_message_handling(): ] result = _bedrock_converse_messages_pt( - messages=messages, model="anthropic.claude-3-5-sonnet-20240620-v1:0", llm_provider="bedrock_converse" + messages=messages, model="anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse" ) # Assistant message should have placeholder text instead of empty text @@ -2410,7 +2410,7 @@ def test_empty_assistant_message_handling(): ] result = _bedrock_converse_messages_pt( - messages=messages, model="anthropic.claude-3-5-sonnet-20240620-v1:0", llm_provider="bedrock_converse" + messages=messages, model="anthropic.claude-haiku-4-5-20251001-v1:0", llm_provider="bedrock_converse" ) # Assistant message should keep original content @@ -2446,7 +2446,7 @@ def test_is_nova_2_model(): assert config._is_nova_2_model("eu.amazon.nova-pro-v1:0") is False # Test with completely different models (should return False) - assert config._is_nova_2_model("anthropic.claude-3-5-sonnet-20240620-v1:0") is False + assert config._is_nova_2_model("anthropic.claude-haiku-4-5-20251001-v1:0") is False assert config._is_nova_2_model("meta.llama3-70b-instruct-v1:0") is False assert config._is_nova_2_model("mistral.mistral-7b-instruct-v0:2") is False @@ -2671,7 +2671,6 @@ def test_supports_native_structured_outputs(): assert config._supports_native_structured_outputs("deepseek.v3-v1:0") # Unsupported models -- should fall back to tool-call approach - assert not config._supports_native_structured_outputs("anthropic.claude-3-5-sonnet-20241022-v2:0") assert not config._supports_native_structured_outputs("anthropic.claude-sonnet-4-20250514-v1:0") assert not config._supports_native_structured_outputs("meta.llama3-3-70b-instruct-v1:0") assert not config._supports_native_structured_outputs("amazon.nova-pro-v1:0") @@ -2799,13 +2798,13 @@ def test_translate_response_format_fallback_tool_call(): optional_params: dict = {} result = config._translate_response_format_param( value=response_format, - model="anthropic.claude-3-5-sonnet-20241022-v2:0", + model="anthropic.claude-3-haiku-20240307-v1:0", optional_params=optional_params, non_default_params={"response_format": response_format}, is_thinking_enabled=False, ) - # Should use tool-call approach, NOT outputConfig + # Should use tool-call approach, NOT outputConfig (model doesn't support native structured outputs) assert "outputConfig" not in result assert "tools" in result assert result["json_mode"] is True @@ -3191,7 +3190,7 @@ def test_parallel_tool_calls_newer_model_adds_disable_flag(): def test_parallel_tool_calls_older_model_drops_disable_flag(): """Older Claude models (pre-4.5) must NOT receive disable_parallel_tool_use — Bedrock rejects it.""" config = AmazonConverseConfig() - model = "anthropic.claude-3-5-sonnet-20241022-v2:0" + model = "anthropic.claude-3-haiku-20240307-v1:0" messages = [{"role": "user", "content": "What's the weather in SF and NYC?"}] optional_params = config.map_openai_params( @@ -3319,7 +3318,7 @@ def test_transform_response_with_both_json_tool_call_and_real_tool(): optional_params = {"json_mode": True} result = config._transform_response( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", response=MockResponse(), model_response=model_response, stream=False, @@ -3397,7 +3396,7 @@ def test_transform_response_does_not_mutate_optional_params(): optional_params = {"json_mode": True, "other_key": "value"} config._transform_response( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", response=MockResponse(), model_response=model_response, stream=False, diff --git a/tests/test_litellm/llms/bedrock/files/expected_bedrock_batch_completions.jsonl b/tests/test_litellm/llms/bedrock/files/expected_bedrock_batch_completions.jsonl index 8bb35ba95d7..c58963bb1de 100644 --- a/tests/test_litellm/llms/bedrock/files/expected_bedrock_batch_completions.jsonl +++ b/tests/test_litellm/llms/bedrock/files/expected_bedrock_batch_completions.jsonl @@ -1,2 +1,2 @@ -{"recordId": "request-1", "modelInput": {"messages": [{"role": "user", "content": [{"type": "text", "text": "Hello world!"}]}], "max_tokens": 10, "system": [{"type": "text", "text": "You are a helpful assistant."}], "anthropic_version": "bedrock-2023-05-31", "anthropic_beta": []}} -{"recordId": "request-2", "modelInput": {"messages": [{"role": "user", "content": [{"type": "text", "text": "Hello world!"}]}], "max_tokens": 10, "system": [{"type": "text", "text": "You are an unhelpful assistant."}], "anthropic_version": "bedrock-2023-05-31", "anthropic_beta": []}} +{"recordId": "request-1", "modelInput": {"messages": [{"role": "user", "content": [{"type": "text", "text": "Hello world!"}]}], "max_tokens": 10, "system": [{"type": "text", "text": "You are a helpful assistant."}], "anthropic_version": "bedrock-2023-05-31"}} +{"recordId": "request-2", "modelInput": {"messages": [{"role": "user", "content": [{"type": "text", "text": "Hello world!"}]}], "max_tokens": 10, "system": [{"type": "text", "text": "You are an unhelpful assistant."}], "anthropic_version": "bedrock-2023-05-31"}} diff --git a/tests/test_litellm/llms/bedrock/test_anthropic_beta_support.py b/tests/test_litellm/llms/bedrock/test_anthropic_beta_support.py index 074a319a603..509db357c2a 100644 --- a/tests/test_litellm/llms/bedrock/test_anthropic_beta_support.py +++ b/tests/test_litellm/llms/bedrock/test_anthropic_beta_support.py @@ -53,7 +53,7 @@ class TestAnthropicBetaHeaderSupport: headers = {"anthropic-beta": "context-1m-2025-08-07,computer-use-2024-10-22"} result = config.transform_request( - model="anthropic.claude-3-5-sonnet-20241022-v2:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "Test"}], optional_params={}, litellm_params={}, @@ -70,7 +70,7 @@ class TestAnthropicBetaHeaderSupport: headers = {"anthropic-beta": "context-1m-2025-08-07,interleaved-thinking-2025-05-14"} result = config._transform_request_helper( - model="anthropic.claude-3-5-sonnet-20241022-v2:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", system_content_blocks=[], optional_params={}, messages=[{"role": "user", "content": "Test"}], @@ -89,7 +89,7 @@ class TestAnthropicBetaHeaderSupport: headers = {"anthropic-beta": "output-128k-2025-02-19"} result = config.transform_anthropic_messages_request( - model="anthropic.claude-3-5-sonnet-20241022-v2:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "Test"}], anthropic_messages_optional_request_params={"max_tokens": 100}, litellm_params={}, @@ -116,7 +116,7 @@ class TestAnthropicBetaHeaderSupport: ] result = config._transform_request_helper( - model="anthropic.claude-3-5-sonnet-20241022-v2:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", system_content_blocks=[], optional_params={"tools": tools}, messages=[{"role": "user", "content": "Test"}], @@ -125,10 +125,13 @@ class TestAnthropicBetaHeaderSupport: additional_fields = result["additionalModelRequestFields"] betas = additional_fields["anthropic_beta"] - - # Should contain both user-provided and auto-added beta headers + + # Should contain user header plus computer-use beta for this model (Haiku 4.5 uses 2025-01-24) assert "context-1m-2025-08-07" in betas - assert "computer-use-2024-10-22" in betas + assert ( + "computer-use-2024-10-22" in betas + or "computer-use-2025-01-24" in betas + ) assert len(betas) == 2 # No duplicates def test_no_anthropic_beta_headers(self): @@ -137,7 +140,7 @@ class TestAnthropicBetaHeaderSupport: headers = {} result = config._transform_request_helper( - model="anthropic.claude-3-5-sonnet-20241022-v2:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", system_content_blocks=[], optional_params={}, messages=[{"role": "user", "content": "Test"}], @@ -163,7 +166,7 @@ class TestAnthropicBetaHeaderSupport: headers = {"anthropic-beta": ",".join(supported_features)} result = config.transform_request( - model="anthropic.claude-3-5-sonnet-20241022-v2:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "Test"}], optional_params={}, litellm_params={}, @@ -358,7 +361,7 @@ class TestAnthropicBetaHeaderSupport: headers = {"anthropic-beta": "context-1m-2025-08-07"} result = config._transform_request_helper( - model="anthropic.claude-3-5-sonnet-20241022-v2:0", + model="anthropic.claude-haiku-4-5-20251001-v1:0", system_content_blocks=[], optional_params={}, messages=[{"role": "user", "content": "Test"}], @@ -378,7 +381,7 @@ class TestAnthropicBetaHeaderSupport: # Model with 'us.' cross-region prefix result = config._transform_request_helper( - model="us.anthropic.claude-3-5-sonnet-20241022-v2:0", + model="us.anthropic.claude-haiku-4-5-20251001-v1:0", system_content_blocks=[], optional_params={}, messages=[{"role": "user", "content": "Test"}], diff --git a/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py b/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py index b804f549ecb..5ce291aa165 100644 --- a/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py +++ b/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py @@ -32,9 +32,9 @@ def test_govcloud_cross_region_inference_prefix(): # Test us-gov prefix is stripped correctly for Claude models base_model = bedrock_model_info.get_base_model( - model="bedrock/us-gov.anthropic.claude-3-5-sonnet-20240620-v1:0" + model="bedrock/us-gov.anthropic.claude-haiku-4-5-20251001-v1:0" ) - assert base_model == "anthropic.claude-3-5-sonnet-20240620-v1:0" + assert base_model == "anthropic.claude-haiku-4-5-20251001-v1:0" # Test us-gov prefix is stripped correctly for different Claude versions base_model = bedrock_model_info.get_base_model( diff --git a/tests/test_litellm/llms/chat/test_converse_handler.py b/tests/test_litellm/llms/chat/test_converse_handler.py index f207c1d272a..1e75a2f1fcb 100644 --- a/tests/test_litellm/llms/chat/test_converse_handler.py +++ b/tests/test_litellm/llms/chat/test_converse_handler.py @@ -48,8 +48,8 @@ class TestBedrockRegionInModelPath: "us-east-1", ), ( - "us-west-2/anthropic.claude-3-5-sonnet-20241022-v2:0", - "anthropic.claude-3-5-sonnet-20241022-v2%3A0", + "us-west-2/anthropic.claude-haiku-4-5-20251001-v1:0", + "anthropic.claude-haiku-4-5-20251001-v1%3A0", "us-west-2", ), # No region in path — modelId unchanged, no region injected @@ -60,8 +60,8 @@ class TestBedrockRegionInModelPath: ), # Cross-region inference prefix (us., eu., ap.) — not a region path segment ( - "us.anthropic.claude-3-5-sonnet-20241022-v2:0", - "us.anthropic.claude-3-5-sonnet-20241022-v2%3A0", + "us.anthropic.claude-haiku-4-5-20251001-v1:0", + "us.anthropic.claude-haiku-4-5-20251001-v1%3A0", None, ), ], diff --git a/tests/test_litellm/passthrough/test_passthrough_main.py b/tests/test_litellm/passthrough/test_passthrough_main.py index c84d32d48f2..489357149c5 100644 --- a/tests/test_litellm/passthrough/test_passthrough_main.py +++ b/tests/test_litellm/passthrough/test_passthrough_main.py @@ -31,7 +31,7 @@ def test_llm_passthrough_route(): return_value=MagicMock(status_code=200, json={"message": "Hello, world!"}), ) as mock_post: response = llm_passthrough_route( - model="vllm/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="vllm/anthropic.claude-haiku-4-5-20251001-v1:0", endpoint="v1/chat/completions", method="POST", request_url="http://localhost:8000/v1/chat/completions", diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index 366f659bdab..e11dbbd8915 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -3147,7 +3147,7 @@ async def test_new_team_org_scoped_models_bypasses_user_limit(): @pytest.mark.asyncio -async def test_new_team_standalone_validates_against_user_models(): +async def test_new_team_standalone_validates_against_user_models(monkeypatch): """ Test that /team/new WITHOUT organization_id still validates models against user's personal models. @@ -3158,11 +3158,17 @@ async def test_new_team_standalone_validates_against_user_models(): - Team is created WITHOUT organization_id and models=['gpt-4'] - Expected: Should fail with "Model not in allowed user models" """ + import litellm from fastapi import Request from litellm.proxy._types import NewTeamRequest, ProxyException, UserAPIKeyAuth from litellm.proxy.management_endpoints.team_endpoints import new_team + # Avoid injecting max_budget via global defaults; that path calls get_user_object and + # needs cache/DB mocks — this test only covers model validation. + monkeypatch.setattr(litellm, "default_team_settings", None) + monkeypatch.setattr(litellm, "default_team_params", None) + # Create non-admin user with restrictive personal models non_admin_user = UserAPIKeyAuth( user_role=LitellmUserRoles.INTERNAL_USER, diff --git a/tests/test_litellm/test_anthropic_beta_headers_filtering.py b/tests/test_litellm/test_anthropic_beta_headers_filtering.py index 447419b27d7..07c19db4568 100644 --- a/tests/test_litellm/test_anthropic_beta_headers_filtering.py +++ b/tests/test_litellm/test_anthropic_beta_headers_filtering.py @@ -265,7 +265,7 @@ class TestAnthropicBetaHeadersFiltering: try: await litellm.acompletion( - model="bedrock/converse/us.anthropic.claude-3-5-sonnet-20241022-v2:0", + model="bedrock/converse/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "Hi"}], aws_access_key_id="test", aws_secret_access_key="test", diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index ce5873f5063..d19d1d1d756 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -150,8 +150,8 @@ def test_completion_missing_role(openai_api_response): "model", [ "gemini/gemini-1.5-flash", - "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", - "bedrock/invoke/anthropic.claude-3-5-sonnet-20240620-v1:0", + "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + "bedrock/invoke/anthropic.claude-haiku-4-5-20251001-v1:0", "anthropic/claude-3-5-sonnet", ], ) @@ -295,7 +295,7 @@ def test_bedrock_latency_optimized_inference(): with patch.object(client, "post") as mock_post: try: response = litellm.completion( - model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "Hello, how are you?"}], performanceConfig={"latency": "optimized"}, client=client, diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 73bd200f8d9..3df10901492 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -692,7 +692,7 @@ async def test_router_v1_messages_fallbacks(): { "model_name": "bedrock-claude", "litellm_params": { - "model": "anthropic.claude-3-5-sonnet-20240620-v1:0", + "model": "anthropic.claude-haiku-4-5-20251001-v1:0", "mock_response": "Hello, world I am a fallback!", }, }, diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index e984403a05b..bdb38ab65f3 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -1316,7 +1316,7 @@ class TestProxyFunctionCalling: ), ( "litellm_proxy/bedrock-claude-3-opus", - "bedrock/anthropic.claude-3-opus-20240229-v1:0", + "bedrock/anthropic.claude-3-7-sonnet-20250219-v1:0", False, ), ( @@ -1614,13 +1614,13 @@ class TestProxyFunctionCalling: ), ( "litellm_proxy/bedrock-claude-3-opus", - "bedrock/converse/anthropic.claude-3-opus-20240229-v1:0", + "bedrock/converse/anthropic.claude-3-7-sonnet-20250219-v1:0", False, "Bedrock Claude 3 Opus via Converse API", ), ( "litellm_proxy/bedrock-claude-3-5-sonnet", - "bedrock/converse/anthropic.claude-3-5-sonnet-20240620-v1:0", + "bedrock/converse/anthropic.claude-haiku-4-5-20251001-v1:0", False, "Bedrock Claude 3.5 Sonnet via Converse API", ), @@ -1701,7 +1701,7 @@ class TestProxyFunctionCalling: ), ( "litellm_proxy/staging-claude-opus", - "bedrock/converse/anthropic.claude-3-opus-20240229-v1:0", + "bedrock/converse/anthropic.claude-3-7-sonnet-20250219-v1:0", False, "Staging Claude Opus", ), @@ -1713,7 +1713,7 @@ class TestProxyFunctionCalling: ), ( "litellm_proxy/high-performance-claude", - "bedrock/converse/anthropic.claude-3-opus-20240229-v1:0", + "bedrock/converse/anthropic.claude-3-7-sonnet-20250219-v1:0", False, "High-performance Claude deployment", ), @@ -1851,7 +1851,7 @@ class TestProxyFunctionCalling: bedrock_models = [ "bedrock/converse/anthropic.claude-3-haiku-20240307-v1:0", "bedrock/converse/anthropic.claude-3-sonnet-20240229-v1:0", - "bedrock/converse/anthropic.claude-3-opus-20240229-v1:0", + "bedrock/converse/anthropic.claude-3-7-sonnet-20250219-v1:0", ] for model in bedrock_models: @@ -1883,13 +1883,13 @@ class TestProxyFunctionCalling: ), ( "litellm_proxy/bedrock-claude-3-opus", - "bedrock/converse/anthropic.claude-3-opus-20240229-v1:0", + "bedrock/converse/anthropic.claude-3-7-sonnet-20250219-v1:0", False, "Bedrock Claude 3 Opus via Converse API", ), ( "litellm_proxy/bedrock-claude-3-5-sonnet", - "bedrock/converse/anthropic.claude-3-5-sonnet-20240620-v1:0", + "bedrock/converse/anthropic.claude-haiku-4-5-20251001-v1:0", False, "Bedrock Claude 3.5 Sonnet via Converse API", ), @@ -1970,7 +1970,7 @@ class TestProxyFunctionCalling: ), ( "litellm_proxy/staging-claude-opus", - "bedrock/converse/anthropic.claude-3-opus-20240229-v1:0", + "bedrock/converse/anthropic.claude-3-7-sonnet-20250219-v1:0", False, "Staging Claude Opus", ), @@ -1982,7 +1982,7 @@ class TestProxyFunctionCalling: ), ( "litellm_proxy/high-performance-claude", - "bedrock/converse/anthropic.claude-3-opus-20240229-v1:0", + "bedrock/converse/anthropic.claude-3-7-sonnet-20250219-v1:0", False, "High-performance Claude deployment", ), @@ -2120,7 +2120,7 @@ class TestProxyFunctionCalling: bedrock_models = [ "bedrock/converse/anthropic.claude-3-haiku-20240307-v1:0", "bedrock/converse/anthropic.claude-3-sonnet-20240229-v1:0", - "bedrock/converse/anthropic.claude-3-opus-20240229-v1:0", + "bedrock/converse/anthropic.claude-3-7-sonnet-20250219-v1:0", ] for model in bedrock_models: @@ -2152,13 +2152,13 @@ class TestProxyFunctionCalling: ), ( "litellm_proxy/bedrock-claude-3-opus", - "bedrock/converse/anthropic.claude-3-opus-20240229-v1:0", + "bedrock/converse/anthropic.claude-3-7-sonnet-20250219-v1:0", False, "Bedrock Claude 3 Opus via Converse API", ), ( "litellm_proxy/bedrock-claude-3-5-sonnet", - "bedrock/converse/anthropic.claude-3-5-sonnet-20240620-v1:0", + "bedrock/converse/anthropic.claude-haiku-4-5-20251001-v1:0", False, "Bedrock Claude 3.5 Sonnet via Converse API", ), @@ -2239,7 +2239,7 @@ class TestProxyFunctionCalling: ), ( "litellm_proxy/staging-claude-opus", - "bedrock/converse/anthropic.claude-3-opus-20240229-v1:0", + "bedrock/converse/anthropic.claude-3-7-sonnet-20250219-v1:0", False, "Staging Claude Opus", ), @@ -2251,7 +2251,7 @@ class TestProxyFunctionCalling: ), ( "litellm_proxy/high-performance-claude", - "bedrock/converse/anthropic.claude-3-opus-20240229-v1:0", + "bedrock/converse/anthropic.claude-3-7-sonnet-20250219-v1:0", False, "High-performance Claude deployment", ), @@ -2389,7 +2389,7 @@ class TestProxyFunctionCalling: bedrock_models = [ "bedrock/converse/anthropic.claude-3-haiku-20240307-v1:0", "bedrock/converse/anthropic.claude-3-sonnet-20240229-v1:0", - "bedrock/converse/anthropic.claude-3-opus-20240229-v1:0", + "bedrock/converse/anthropic.claude-3-7-sonnet-20250219-v1:0", ] for model in bedrock_models: diff --git a/tests/test_users.py b/tests/test_users.py index 30e34a95f4f..05253a19aa5 100644 --- a/tests/test_users.py +++ b/tests/test_users.py @@ -335,7 +335,7 @@ async def setup_test_users(session: aiohttp.ClientSession) -> Tuple[Dict, Dict]: i=0, budget=100, budget_duration="30d", - models=["anthropic.claude-3-5-sonnet-20240620-v1:0"], + models=["anthropic.claude-haiku-4-5-20251001-v1:0"], ) user2 = await new_user( @@ -343,7 +343,7 @@ async def setup_test_users(session: aiohttp.ClientSession) -> Tuple[Dict, Dict]: i=1, budget=100, budget_duration="30d", - models=["anthropic.claude-3-5-sonnet-20240620-v1:0"], + models=["anthropic.claude-haiku-4-5-20251001-v1:0"], ) print("\nCreated two test users:") @@ -360,7 +360,7 @@ async def setup_test_users(session: aiohttp.ClientSession) -> Tuple[Dict, Dict]: "user_id": user1["user_id"], "duration": "7d", "key_alias": f"test_key_{uuid.uuid4()}", - "models": ["anthropic.claude-3-5-sonnet-20240620-v1:0"], + "models": ["anthropic.claude-haiku-4-5-20251001-v1:0"], } print("\nGenerating additional key for user1...") From cae86136608d6a160cdbc46e42aa3d190a13b427 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 2 Apr 2026 14:10:49 -0700 Subject: [PATCH 04/25] Announce April Townhall (#25021) * fix: replace hardcoded url * docs: announce april townhall --- .../blog/april_townhall_announcement/index.md | 35 ++++++++++++++++++ .../docs/providers/azure/azure_responses.md | 4 +- docs/my-website/docs/proxy/team_model_add.md | 2 +- docs/my-website/docs/realtime.md | 2 +- docs/my-website/img/april_townhall_banner.png | Bin 0 -> 50618 bytes tests/llm_translation/test_azure_openai.py | 10 ++--- 6 files changed, 44 insertions(+), 9 deletions(-) create mode 100644 docs/my-website/blog/april_townhall_announcement/index.md create mode 100644 docs/my-website/img/april_townhall_banner.png diff --git a/docs/my-website/blog/april_townhall_announcement/index.md b/docs/my-website/blog/april_townhall_announcement/index.md new file mode 100644 index 00000000000..b81f61bae62 --- /dev/null +++ b/docs/my-website/blog/april_townhall_announcement/index.md @@ -0,0 +1,35 @@ +--- +slug: april-townhall-announcement +title: "April Townhall: Friday, 10 April at 7:30 AM" +date: 2026-04-02T07:30:00 +authors: + - krrish +description: "Join the LiteLLM April townhall on Friday, 10 April at 7:30 AM." +tags: [announcement, townhall] +hide_table_of_contents: true +--- + +import Image from '@theme/IdealImage'; + +We are hosting our April townhall on **Friday, 10 April at 7:30 AM PST**. + + + +{/* truncate */} + +## Agenda + +- Product updates and roadmap progress +- Reliability and security updates +- Open Q&A with the team + +## Register + +Register here: [LiteLLM April Townhall Form](https://forms.gle/hvyVXwbFjzJQE7dEA) + +We will hold the townhall from **7:30 AM to 8:30 AM PST on Zoom**. + +For security, attendance is restricted to corporate emails. If you register with a non-corporate email, we will share the townhall slides and accompanying blog post after the event. diff --git a/docs/my-website/docs/providers/azure/azure_responses.md b/docs/my-website/docs/providers/azure/azure_responses.md index 34ec0e194f7..de085001ba1 100644 --- a/docs/my-website/docs/providers/azure/azure_responses.md +++ b/docs/my-website/docs/providers/azure/azure_responses.md @@ -246,7 +246,7 @@ You can also call the Azure Responses API via the `/chat/completions` endpoint. from litellm import completion import os -os.environ["AZURE_API_BASE"] = "https://my-endpoint-sweden-berri992.openai.azure.com/" +os.environ["AZURE_API_BASE"] = "https://my-azure-endpoint.openai.azure.com/" os.environ["AZURE_API_VERSION"] = "2023-03-15-preview" os.environ["AZURE_API_KEY"] = "my-api-key" @@ -268,7 +268,7 @@ model_list: litellm_params: model: azure/responses/my-custom-o1-pro api_key: os.environ/AZURE_API_KEY - api_base: https://my-endpoint-sweden-berri992.openai.azure.com/ + api_base: https://my-azure-endpoint.openai.azure.com/ api_version: 2023-03-15-preview ``` diff --git a/docs/my-website/docs/proxy/team_model_add.md b/docs/my-website/docs/proxy/team_model_add.md index bb4238055b5..4aa286f3e5f 100644 --- a/docs/my-website/docs/proxy/team_model_add.md +++ b/docs/my-website/docs/proxy/team_model_add.md @@ -26,7 +26,7 @@ curl -L -X POST 'http://0.0.0.0:4000/model/new' \ "model": "openai/gpt-4o", "custom_llm_provider": "openai", "api_key": "******ccb07", - "api_base": "https://my-endpoint-sweden-berri992.openai.azure.com", + "api_base": "https://my-azure-endpoint.openai.azure.com", "api_version": "2023-12-01-preview" }, "model_info": { diff --git a/docs/my-website/docs/realtime.md b/docs/my-website/docs/realtime.md index 15a838bb7d7..08f1e47fa73 100644 --- a/docs/my-website/docs/realtime.md +++ b/docs/my-website/docs/realtime.md @@ -82,7 +82,7 @@ Run this script using node - `node test.js` const WebSocket = require("ws"); const url = "ws://0.0.0.0:4000/v1/realtime?model=openai-gpt-4o-realtime-audio"; -// const url = "wss://my-endpoint-sweden-berri992.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview"; +// const url = "wss://my-azure-endpoint.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview"; const ws = new WebSocket(url, { headers: { "api-key": `sk-1234`, diff --git a/docs/my-website/img/april_townhall_banner.png b/docs/my-website/img/april_townhall_banner.png new file mode 100644 index 0000000000000000000000000000000000000000..e589101f2fcf17133c49c368355ff96bcf463b74 GIT binary patch literal 50618 zcmb4qWmH^Cw{2s=-QC^Y-5r|73GVK}-Dwj0tc4>!Gjar6Wrx-?)T%~_v5}X z-r7=g*P2x|Yme^UyH<6*FTei=U?|Bc$^oFD005|u4e-7LwX7&DZK45Gms3=c{f}b| z@L{kV0DzOTr#nzyib7A{fCB#ae@gu8Gqdn;{Wtwz?4#bhrGIM&0Onc$7ta4H8`09r z!{UQ*@o~_&e>DCf7Uu)w*!%}G{EN;1gGK(uUp!qsKR6oyVs{Ww`U6{jU zu#faV1_k{g@e@1%5drDLZ-oJXf`)~LMTA5C(2D|t_;CpXfQ7^SLPUBMsCFE1D&s;gyj!cc%qur#1C&TX=*Qw_eck>w0-dq-Qj@v|S3w z%`xg&dVYy49hm=Uh8XHY*8e5r-**1h@!{mh1VDd?fP?%zL&+|Z*fQN zFJZ`E7XeaYD#UdNf;`QAY6x_T7GC{dZGDqpt4#&5P`6*oU|9{@_c5lXnT`*K;&I_B zC=R~F+v~voau2dVEo(*u%Fg%5`kp8|WrUS(UJ5X=iC_r)Q`<4*@wc+OL1KicIf+h7 z^NdR+obWRhf|XTU5aJA!(LbTKNigsx7Uoh<@LeX(^ciEhR`oV9qTuB@|3y$0`vOQ+LyrGt* zw~}gT8Ln&bWSmJm8h_o6$I7bB^LfKGgTigitGXmzHBd~9XP=cMfq)L#9l3BzxeXIe^nnCy~TsMtuwMc}*xcHUD&^hBP{;m`mBb zXT$7kivr+I%0nO$o+2V;1&{Y=+7eoctL-i(@r!(#;7RGj@F-zaW~Hh^0nv3}XO9p$|XPk=Ga*BHM5KFsd}95lCZWhZ$F0WWtvMsLt^UVlgIfK$xO2 z>bM_39`rz@u#dkRscxg6lUYd+Dg9&+Le>PeF9p{o?^HB!gv}^e>N6l0&lxPoF3P{2 z+lsM>Cqv3BTN=bc70Kyvl0@*AY7!k}*iL@2Y;8(3Sh#Pyuan0iCKK>fGJ;SoJose` z8Z;;pa@EqYYI;=Lo0dBEHzI-ZPw1wpNY?LFd9bh+b{Y-s={$lbsh3GR%_oZNC8l3G zfo=$kg4UyYK11nkgriU*ri~juBT8^1Dn6XpN3v~;b@K@97%KIS&$Z2f-Em;1V z+}M9@{tK_qNXHMk8+TbV_JCVx=_Hh(*$uJTh{6-WdlNoOx3m>)QEL332{i{_(FyQI z0N?J*kY5w{5`J5uhoV+rM5|^LgUZ!Z#(urGz+QOL+RGzL;&03jN6};4Rvvd1p@=_4 z+%R^v_K(v1+@bsI)QIG&A*e-Y2tI}Sk2E5OZC#fp{uG2I#yr`xtCmyPtu6DqF2wUdKdbV7+=n*rBEs#m! z6k}~O1Y|3)_)lE;c zqQ(2=XMF>~Ulx@li1s)0gUtk+^~niV>Uwg{5_t&{%aYPtIrq_Riw&GA)pRJMT*dKW ziJY5R!a8Hl@)7{Ye87oA=>XGgzvo65ZK2nQNvxTF!!l2~^?02s`e$R#krpo1+YhbP zvxcW4OJ4p8Vazff@}}yoRDGEJf5Mf@M|}5v>!|g?vXjzEK8k;)ONJhH5wRqM^jKnR)kgKhXy9zB)PVj(AX^g!=`E3Z} zAM9=`CR~+%y)Yk`i@0g5tDC<|3?t=rM0{JPsTJiXc#-=Tw4!ApyI1k#qsW&mR7NH4 zt#nV>5<}PpD*YqW5cPTs3pwPX(v4xItwJ`uYh9Ewk!|+1Is);y?Y_03Vb`nm5v|z6 zw5CB@{ygqJRh=Wu=A2L15{I<8UIZK^rvI>&cI;w4l#{<^o)KYPai+x4E8c*`(udQ| zQ>N*=d9CJMfs%^|lE*zn2Fi#d>ubh-dYcK48XeOR?F2X{UpoM2;P~S$>n>hQA;P4lN4wjzOtJ zk}Y4KEX5fWCT7uX0V}xmh8QPK|3uMs#9#z*j{UOAw=9GkhptSvmn+~t9;cCq-_fYJ z)-E_p^TK3yppm26O``3 zJMQgjXNz!RQ0iut&sy>2W*`Wvk0`+6^A|asTS(PEft8X|nB$^H6J_Mi{7EAmfxpg_ z2Nu^Saz{L5{U&q*n^@@+wBaqUTmgQWU3JpIy~=1!y7~7oDx< zJMWzQR;yd(9#6%BJY!b2W@$wmy=ImIk)p&u7yw+EgMe{6Ze* zOvgrfX$lf`Jss$!Z;~pU$I$)kIfRPNwY^3P5=`%chAtMraHS-k6z)C!)=pyD+k4v8 zyKgLN6}+Z4K;iM1mY6ncrg(8cUS(hI#j8J1JUq8_4E%B&#!RM*!$L=r5IzYOnR6MX z$sbdmibK*-aN{Fxv(S67KzovXM%=DuI}qdC4={4CC`1!$V76Q<>E$icu%(nc!FwZc zz9T9u-_zGU!rVZQ(JYX$JW~9^HJIcuWSG&l3wOQQ)sfz*?r57vV4oa5sra){+J#~s zsG3WYw2?Tr!`K_sKcL(tL$K2wtgjcM!&A>Zm+QVE$376R46p5o6&!^U=fs+JC2IMn zGr`NOu~~HT#e!)^PRki{4NixvuJWa9i(ppeJ0mizVkxm>D`APM>^p$d@h+xL=J}I+ zS0c3BgFbAfr5RV&-d=rW#pJ*l*G$d%1R4|`e(2(_lf*@QOn%a3S(;O)aLNqQ`_K-7 zLnK-y)gnF)tcj5$_cf)?u+ofNt6j8gMZQU&5#LpEppt?9luRYFZaY$yHkKlPv?@(i zx;3PGn`^VW*+B8A$93ANMH{mQjDu=QWe~~I?*R_LbV?yB6>0`(i0VeiV!N+aFB0WJ zbLKi3?8|!4ZMkcgj{#1N3Chj#ISr|R;5*_ROQM<<)ULjGL4-CX;EnljMeho}d`P0( z!GwK*-uOE}#rTgO-zJCwIKizCLLSOrOGbu&?&BlXx20ndN3xnq}hq zDY&2=up8$SQ=fi#W?#jBVtgR-a)PHC0#l9c?AG(w-NG~d>#PJ#2WM^R`7c2N{e`2t zVK~}@Rcb15P?}Gc#jX~lN;UY=^k_n?^c~QZE+_>3m{5wg>^iK+BRB@| z42RfO59F{SH4UNP!yt4;-RR8qsjKAgqV86SsinUIevi#(rrvXpo{YKaqhllH#I~gZ zg+k-chUsMyb&q4)?q~+=(`Kc(R9>#SuM?=_%YQL(zXRgwRqs>p8IEfho2DreL(Fh> zakQs|`s~@KDr6ay+RAM*7a^ot2DDDX^yz+o@rQ=RZ|L6v*P82T#FWrL&oz=q*0hUJb8%p8O$TR?WYFhO0~Av5#*a3VtO;nnoLF0$>KhTq?Zk< z?|^INmEbgb_ye%J1@Qusd#*=`Bd0)jahe;?{RR@f{!BJlU=(A4cO{6w~-GcP5cR-?#VK}n6B90mFcKV zMzXXZgafKIC3;zBkqUi#yytH!B*B`w^?d?Gxj#fN!m6*C_;>Ml@gE@;iUtxk$V!Dl z7j#tg4yvl_#VITU!Csud2uBAtN1OtcimN^74{>+lv=y(ft4S|e+I>~eBIqlT4TzZ1 zrCQobe5MOLt>+1tieGyNAmhzu|KBoK({pCCsS?YSGV^n$9#I1ayfUlz*hon6$2U4f zTj)7uaE2h`SB$;c^i8YqEkeOT^HfXyS{2e7IwYnflv}W+BGmd zD~IVX%rSvb4LW8w;D)Fd=-SS~yp-6FC`t2)bV)U>wFCJ=8`~DT(RecTsUKBl(frPL{!rLozClrIr@N z)2|(4lhJnTGibqZ)WId7*d3_g=lQD)>CEKEe8pk9|L^KL)w~9_jmAtmFW3aVZH9xJ z*O{yN73@h|9qU9R_ZKr>?edroq8jghaOx99b-LaLPKIdMnXESa-<-u=xcz$Y?PAa} zXoBf6W}8l8g=iDG@pH)f^RI_tpTQOT^ndhPYDh1_-DwS}l;l#|YApk(5jzcwly;bF zCL>Zt*{N^9^!vvPXIUProJ_BX5+q3x{5Z0hM}f0p8O$EizER^ti?HBwBXZk;Df6dX zWAIw|Ie~@-WWAtUcucw1P7EV=u(t|Tu*m!JyS8BLufWQHrBr6tQZ$`rp-DO4-nTu5 z#4)N$4@Pmkk_|#3J$1eTfbRPJnF-@tSS~d}@K3Y?+K} zr|ywtmFN$sxk%I8>lb6Nr92uJwA5Ac8*Pv;`@{S$S;)%Ju_!si1vxCDmygPO5F&Ixjd#D zT<5ViwP@SIpPbs}g1hZj^LJpH>YMFXv!AyA5J?a1t-era$OoV31Wno&DVuw!$WH9d zKHiI#Ao^NczXM46u#p56vigPu&j#c&xx~zVTJvdEG(t)dYbI;BPJXT?Ue&w<^!n_P z7`g57vHoDb(7|d}ppDz0uW{1#x_;ykBA-fgfubyp<13BVkB8Sz$PA z`J~{;Jf^ZhD%PD8z_4)Lag(tNuSQZ;a`=^bsi%FGDphltAjEO3gq`2uK|A>JnQsCs3mG`eUu5WWy>Q|Qt)4j; z2eql2^?xD^Z&!*{wlSuSjrjvW45nS1}H!A^>kmN?`0`FbSD-7oj8W=1(w)e9J4f zOAW29No+wEc?SN4Km-?i{EsBnLSxL?5w z{_vJeDA7nm(jLjLF-!SBD!{Lp>EnTWQ?N z3M-{V)sI^PglxvVV+GO1YQ_e^HaqwV14b=}V$$VLRxK&wo`memd~D?d-P`n4>fz43 zWD9h5=p^YU==Q}opT9*}iy)`cFz~<|A<29n7YZTG#3b5h-OGhf7F+OWRu>4#5Q$hb z>s@w0rJvjDT29qSSJJMlE+b8z8lKqTQ}X?$5H*snB}IJhX7PPNM&!`6lUB$ z87;9CG-+mzaNSHw zg-}5&r6?WZJvbk+Kc7b{Z0v-{fKEzry{ zCC-(&jE?dhqBLuE2P`SS5ClPK$V|%2eU3(DH&$A1?Q`XZ@b?Kt@Hb{!%BwxfVhg6H zGcHwrTFdpbhqM_SI=jpaM?K2Zi~DJ8bvg^-Covsbp86t_%~+)z8O%YgD0i_o1sZT8 z*e;uicIJ&8hcxz;+8s33QH6ssLSsTR1x%R~d-X>%Ml&}%*q3u}3(!2`B0E7`Cadu~ zjpG4h2Dsh~(9R$lm^$d2o9fySoWAV@1Y3ikU))$@Crq(O)(J0=Z}}?EdDiscK`$WD z;KCp9Q-Z3q`-{2Gn(nIia8_|7GXqQ}aKz9BPrsI&>jXh*Klem2rl}+|Pp3K0l6+G~ zv!BHaDRCtMmvFR6J2y8)Qv||5Ax&?Ru8D`lbjn$jfX6rmtXS@NLkIvOh0^A;AO=U%PhUV z?dlrj+%$9XuZQA#-#1c>)Xyq@BdRDRXv|R+*2AHs@iZo-j8p1LRHoZt+Nx z{i8i=Y=#I>j{JC8WgZ5ew*qPAgK~!>PgpS*DkE6&j-rkSLEJgT$R-IY+?@EH-5^l{c6O6`5i=Zrvzo-!dLcX9zt~i2x)V|gM{xyZa_*OLI;FD z`U|}9B=HnPOgG58!O?D5VVRicxZdgE8kzQWSo2t5{DzaPq@SK{p(&q=VPZQI@;UvgVbHtvd- z=n{vHn+rKDb`OZetx?>-%gaBc&xSjM9AbT{@sEgDDk|{R zkY9*>-_G2ITl`mTU$Z)E&=l?W2F2E~r+j%2^z1h%T{|Zbcq~C^EIHIWIi#@>86SZ{ z4=E9Cr4G~t2}hRq~9dy?Z<<0B(y(oYd_ z1`dCcG(|7OT)V}b#4tUD#LH-njvurkP}0-|8dUhC(L{se>FMS+sbY-q-DsTBy9c?V zv)*b+M>GZda0a*pFz99cpr;c4q`|S6ZPb>*N*lb7K^zDK{m zi0n{iGw4@gMJz$YHfKW@R}5qxB5IE9L6ct~)34ITlb?TuGMHZ5#A0OkFX+4j^noQ- ziG#hd&s;v7e@qAGsvnDAyL#d^Pna1yk2QM2L@EvBH9v7$atT zy}D;wMXVG2`57oNKzY;9>(#P=uLTG_<>67lctY7QzP!LjP5D7ZaT?t=A~%9B$9VDU zrC>KL#E02!sKf7?(!25z#sNuDS}9+}WQPlJfD8kR~TRuIG`U+<84ew zRnG>7E2eLQt6{7gBEIBj5bve?b9!9cN5Bn4w?S_Zg{=I{Ll>oA=E>g|6*wt|v>Dz} zYzBj(hT6_#hjF?;HGPw~%hJ$lJQ6vOB)*Q$`PyH8&sa5ws!dJttBK@0<>J`t@|ld$ zJbUCo%-+!bKq!F`vb&^n3#t+s_4q9V@*JFr`<&1%C1)wp*kk`0!gsMhgf2O;ur>7; zA^vDa?*RIE?~(D*A+>^qa#V^X`RUL*iogqId5eDnv$~321RPO$n}u8;EL9LD2{IMH zO(X%g{E$H~FqtRTQ2pmSpoa8swBgzW>NC|znVA*c4kSQ@M&y^y3LMOW*+EiFhIp06 z{a`&KAA71W<)Dycec|5aMLb_UAsM(j)l6raZ0r*0B4XX(NJqWuvyj{n5*{{Ehep(> zfgV?yhUi2zDUPGF^Sa8`-zdvE#SQWP!RM_dwmVuo|LY3e#qRvv#0JN(Q1#rc(^o zAM^c8e##uv0Yj2m-Z|QMo(fGAVi!R2$sAh`*@?3e}+Y3 zO*A#f9{$gqalGAMc>72|?{;)gC|R1y0Ho-Za263+n=T!Kx6b%gv;KhgEcSxMZ(He| z{*#6zva~Qe=|wNXlsx5$aw<&kfI;lkf|#>=W~Mn*FGFj5n7;|1+al&h!3I7jNVryt z4EK_t19H0K6piz6waic*|BTR5q5JC!O5WXxourBNOHTXUBg9upn??!5YVtZ|rn3)P zOzl0>_^5WF{GpmjZIi-P%&mdVIs zTcokmc2FdNCFf_;q>P@R*#u0j8+fnL)MD|3lG9YiXx5os1jBt|b@$9eRnvI)iq{gj z>(qvT@`5|Cal{nKc^RYxyw6}AXCJ*1r}?#XMK3*6`5ZOgzT#5_i5fWy)iY;sxchc~SSt0-Zg{5N@Rs*aW4$sq)dusmy(iDrwoQH0K;r)W*=Wmh zxPwi;bu>*G51iIwrHS;CUvLii%F5NpUy($aI4j@`l%|^TsB+E{oU*I6##?09$WqO| zUA69q;R@~YuUI*OH|l5@n}LK%@*sN7Qi>Co;m^`mlkN>s=Ggne9yW2Mvt`N(nuZ!m z6$OUT_h6MD3QGL`BKzHw7SxJb3_zTD)G}$`X;M^1^)6R0AUX)^!J74{cSo;U^C+Qd z)rl6Uz&_Qu2-5$=!$o83&RQ7RX%0+7v(|9s5SMSa5@4 zN+2Zl>!Q=1T=Ij++&*1j=cX8YwcO;)U~TCO)sotPj;VBOS@n33gzQh+Ax2a6 zoJ9LJrZ!5TPM#)0pi1c<6&Lm2875PF)|VIO*nxR+AFp%O_P#6(eq}$h8h#$bz-s}e zOuU`?yM(D>vU0O>HMM@$4Fsh`cnOUsDJAtYN<$7-21Psz6iOX<>sTkm!q&kJ7^$oy z4#bI_#Xogs#hxI3PWz`pmvN8KcdbxrZfA1?^QCpzaoT14lV7#wh<$!39C5BYhTcLh zW0EQQsf0k7U(G+WYej8%Y{86b@wRtTk%K(tNxvfYpyjM{V_Tc&I@|x+h1<1TgM5r_ zv4;L6QVm+l{p*VLhR8V!po91g=ln*shgYc-L|x;;qJ9iBDZ5UqW?CtX?;|LkVX2eJ zNRtMo!7b@+nlF-9wxoSA&2`(88nZVFecUMi=H;2i*F0KlI`&sj1*7tTsOT85?u+`01RX7rV z)B*Tl;~!b-2wQbg9W&q4fm&JF`p+6Qa>=DHRdzp*<$_4{H%+A{h$of%J5J5&Zg?t6 z?W`IN^19v_On-m1(!mBKE|(PDv!_myR4w|APzmspwR9P5N$pgWMU9634et>9;r$>X zvEfsO#%-#3nYiR%IxIpJXsOoZxvwHrtn|;{&|TWlK9aQ6%F@#w{w$0RAr%s;E4(Hz z;J$9#UgCFK?XOmuPXmBAhGIrF)WZ?@L@#QmZ@+!yoq>!xPeh2R=Ibl1hQ$}DWvP%^ zuE_i_^Ef(Vn0Yad5)W*q8?VJ>6nRA26J~u!69cOIHQ)EX-XvFC9X?rr=F z0#!Ck!CtUK;CkGlmWtnY9u9p{DC+REr)15`*)qkU4G2I{`etWfE_8D|+AVkhd|vpD+d-Q)OVg@h;LmJu zME_)6U$=mkP8)0{R_@;ku}YiJ4Jf5%Hz|^{r57s2dbrkT?5VoTkXKU2-Xmc3Zr_W( z5|+)_A5C)GwFIfuFY!{$DQKU4Wr>IiB- zcUPQ=fFBYRbd}9v`9N?J)PH1LDF$Nnl@1}ms5M%RzgIl{?qh(?lbBT(A{JmECA%5J z_n+aP50Yd_XO=~j>MGGqlO(Ks7)`$}u%wGFs1}>`q7slvLMfLw*ME0-2bd*B>NiMR z?!PKrS8=mpR}<1V=)k&{Ii>@ju`K4IJW)%It&@wC%#?-W)OWGwx@rHpQasQog^1E3 zy;ywqW#4Z$NJE1gg69wVu-$hW(N_je{C7ysFgJj`0*aHc&DQ z^6Yd$Ia*E~l1bg1pQ2kMfs%$?3*+8PtWKbK?_naFFgA|dT;yCoxl^GbN;_y#gr%hF zAJ56V!Re1Yy5;F=watj4h~aS=8c$qI{71Hl%f}$j`B2B2Z(1J{y-fI@A>Mdyqv; zZxihse^&T{qy>%8mO6$PQ!83*kcZ{wfiS>Rfuq{5Wx?J{;)Yu9_8i+!KOsM0Xg3dcIi{^`5xtv@h~Y3TWXbt20%!EdfJcXBYwv(y1ryHJE&N3|3T2X-RIKo;WRg^7R1>kUud(dskRXia#y4s>`XBCwe5fV^ zJPTDjefM|BT30stiv0$X8&Y}VM<_U^DnZbhdxSIakJx0fZPA=aI(QDvQqQsK+wjJC z4RuVt6v>P_+b#q224!#B_wN9x7jZh@Yyq$`+s7EpJC9@SS9@_b#nu3Q*#)0?jvX3* z^dU|dmx$!F*lpe{&g9EGfWf^&g>Ca`Lj6JJ$#vK*Ii24)15p97k!rAoVR0cNqnM{? zQ$D|Pwl`%-u+#g?QyO$0p&g}kZgfR(73a~HT(8Rl2(Y9{`XZEPVzVxh>g%75Z9@CS zOB1)%uV>ebLJV_;A3tH?*LmmCUvGpVIGqr(I z88Ni)fUR!I#k$~(z`+M&mFrsr;<;GWVI-HXPDLTYxzzEtPaZ72Y@!HUUWUs%a`Ml2 zQkA6(&`41)4IVY&E4W zL0QS#JD`<$uj_WNR&B1kZ(Vw^W~X?NCA7BTkqC3V6vpf^>oK{d~x}c$qKSgN8EGxoN@U_o0w%@P9}}j&EZIa+t-RRUL5Ujc=mHqHluJYQmZ5 z4&7u7eO?Ukn;$Jjyi|r#=1a=!NCr$L-6P-P*tJmDz=uuVH^W};;w&f1< z7Xn2tI}M@bSXp^HL_Y9_(i`(iT2A8G11kv;PxIwO8i(m%_@}Rk(-|DXyML`|Fg|AS zwwXfLl@u^^X*KISV&lHCW7ov2_;Y+G{V9lBG*P6$SUJZ0DiEPvbTUhST}1`|=s8Re zEaVe8Y&BiYpv)xO?~55`STLD19wFRra=p*1BTX8wY1+%$E?OPA-H!q&^p8d-gqO1-9RaMDA^>kBb&&q5zB3F(Bf+tXlvy2O*K%;{ux3zKb4gi$TF64IDt@y$Q?4ORKyrbe zwUX5G8X-A}yE!LP)3hwNL#Hp}{Jd+8c1>cE{iNAyH|~`&xxO^iZCTdfpXP&2$(B;9 zjj>~H={F*ruENafGb{T?&8mruh8D}Y1Re`0D=jR8PxSgKaQ(tV=Jz$qcoH5@!YpRog5b`?ksD9_(iQ7V6l*i&Jw z4%#u}C`86LZXoJHCDziZx9-!YZ{{?5vD4 zf1Wo)+B!FWh$|yc?V1n;t03eJvyJv+OqULsCw}X*D}!?);u3IDhA!uFy#awgHIG=T zhbovo`HwP=<~{DH8&6t(cQFI-v$*mAnr(x@oo_|;XU(gR?~QNT*=2aGWd{4}>k|i31;rWjUySUEBK|33 z%;F#4Fs#^2j{135=&cIz>C)p1^R-6PE zKSq0^ey-)CqmTFWWi2~ysc~DN(%pOq$Uis@}x`Pj+!s@NY`{jj=& z4Sj9w=~meppw)9*B&ki`ye#LB^R$u{&)#mEW^iP$|NZcf%JD!3O`d@c`PbAVO2!|5 z;yfYU6sljC7S2h`k-k<(3Oe%D#e4q~cxjV7BXcv5^5c}}7;o`r9UoJ-=Gd=D%OF0H zUDsWz7FOH(l4oC5U*I(QUXef4to(VId)S-bjupXFpE#~p=qr*!m$Kl;vzx9uDB6}8 z2!Ui*;p`!~Slr8TD-U@2ZKG8oWtu+yF|o8wXC*chGSB_2+!U6w@Kd>USm72l+B>;#j0ugmjEG1<`5EXnj^ux01)T)uEuoaP+$n{j zP~tzrO{Ppk%OA0GYQsF8Nd5Zl6>xTGXG^F3S)2)aaaU4I+L;S|1kg&`+kp~%*UpMs zAt2v*iQj-5j1>d+2BxT2b>i-|apkA-$=oC&4#XmXcG633y%T(dT;^hZVySjH2jGoQ zu72xpLWB8;tc+eK=oW78KWn10OoJH-1*vwH_)ZOeMNiC?LznRorj@p?sMiB%Zl$&h zzhTsL%HQX*@DOzl4#v2(%k|Y>wH?88{_359@Y-Fb$zpL5aW8ie zM)p(~c7Pw{k-UYf9rY2Tb>prj;<|=vdl>4sVts)=c^;6ywyY|PlVIp;%9YVlCZls@ z$Y5N1z(c|pMNn7xRI=N>5No>hpk!1}Qgh%N2Tq*xoB<8${R?GD53}ofrGY*w1$iM# zfX@LVclht4#8humtq189B+Fa5X+ktw@t&B2W>#RBL!cU$T7q}$jqa}&#)4;UV96{A!Yx#(HN$@MGe%8o>v6z^0bsW{WpnL??_@BwyC}Y$h;L(;L6a&Pr5lS^ zVVY4jU_%}5Yf|mVEM zmDG+en>I>$_Le!8jVeu}84;+d3#KCDoVH+EvQ%)8F?VBWPI#n#Bi}|6jk8KkkC;&L z+Q%;=#;qK+8^IkeNd8#P;iZfJR2juE>=?|GbrJcA+0ij2cz|AsY9aY_TIiB$E^MYI zE3P-4{+iR)+|5;(hFo-pVfbS>{_({;8ZrQT_sD+Md?W}Fh{m7^p_TYV00EXU?MQxf zRiB`+GJK5u2ZMb&#-f~%KpG~EVH%CGfH{l&tzv|B>}^b&=+VBpSJKMg2ifC6b5c8# z!aZ{mg&;c)H~pasF-Ge8OEOOC`sy9YQ^}N=O-1Tt5p&N%kt@rP+8uU1`VoXz7Ysf& zXcn^yNK)n6c!Q7p$HE#~p{g>WruG9=hEI0L7Tmo0LkVTXcq$<(7s3wL3In(7EvC#3+&?4RYzfN%6vRh9#Bp%IaS&82sFp zNp%ExRTP>o>-uMfW1Mo2;R~h3Sr;)HW;HjBtFfP$PzBDUtJntjRd?X%hHutt`=#6V z(UV+82y-^nT~dFGw1LE%i|pjh>=hm{m6d+-mVL}Z86)Dc*51q5b7n|3S)QObQ@r_u zShH+jQ^@HWuSM5nyekYe5Bd5KsKPBh3^o5m*$-b&+!~NuO4;D4!?cl28Wd3)P`8C~ z@BG?EbRd30zKFn$r(bNdyTHh4qB)7OS9ksRe+IUtfT)PUUuYxUSr89Fn8z_Xf`;xv zI-1=@bm^$2fRvHu0i{i&F%HWVjrDx{8-Zfn%@wBix)QoZ;uoAnG&%DfJV=}((hG0o znjR@$2(xEG{8=ln4qfHuy6r`XKu#f=gW{j);^A~pyq$zL-0e?5oFFIZZo&$zpD=-z z4-yb%jkOW_0lf5Y$ZR_kc0XPWqCH(R-HeNSDA@&1;D0Ip38%+hkQ*);KoyJvG0++j z8|p^5m*N#7IN5X~)an9-bqb)MIlUAG;W4+7X(c8Q%X^%xb^J@xlmh8@@V7q^OjJ3z z#a@Il<1gY~H(r6rK;dx?v3x8I12|Z06&wREl+8_6D)DYJFu02nGbbxdb|GFKO<4k$26W#ukmqnU%7D9I!3Eox>MhU5|lmvE7 zha;}f$u|eW{k3-b=&H`M*e#j{(3^9k-|Q#ztRizW|3;@0T3 zTuP)7 zZklwq$XlrICAfT5j{v0Go@KyvbCV0%(>t!dmcYv~RiUOS-xa-)>i)P#nO&ZfX|a5f zKoma5s?5wlrMfz_p2^+hirDGS2;oMeZYL$t)zUH(DU%d!0>6=KuU(@m2@`QQ&jnb4 zoBAwqIN1vhj!6OM;48H6&`j>;-1ca;=gN318mq?IW@nG@!U_PaSD2F zQt1(D=|avB502tA-4$D^&#D#I)8lIfSbVhPd5xU7qN+S`zk9SPH2zI0J@Uhu9SJX$ zIbf*c)PhJk??#lyClv7u4#O*m@LAcV*3aC0k+UygKFZz$Wa_R;5Xvzl8`otZY@KXm zJ|bF$^+X!}9*$}Rg;woO^wjadW(rbhFV#2$O#XuBv0Xz39t4D`Lv2|o63rl9->sNy zjk$W{W0@Y7yv8S9^;aUX)Nv{h$pd7F^7wL6E)bcH$3|zjgNTmb?C0HseyqJffOSDJ zZZS8Jah!2&*5zbgHP?pAuo7(98t(i5VE5%ap-qDdAPlWta>0HK0yJ;#4YNbhraS!az17FC(GHV)7+tiw2LO z^x!_TZ?Dm9gN^L%h=@H0w&N@4kmmP4(YN1|1E z()~!Z>_EB@G$5^_8RwJ37f8t#cEap|b%qxpe6G@c2}cQmTYamv!qSTg(OZx<#-)b3 z%ik7H85M&IabYREjVkl(t0lIT(*(U1yJ|%#Y{)kk5KL!EEJ(jFJ%y-JPE_G1qbE^Q z1jMf&SVsw8Mlo0v%9wC)MFiu67_(CA*c@(yASy>u-h{Ka&FGTgTA)oa!ln87Q%feF zU8g~|yX7iymhRVVF001Vzev&uK{23T8rSK&EG*nB4XHH!K(<*faP80!H{#o&3 zge@Xws7SOwd}G3(x(LU>Kv7X4Fvpg9hFANXevFE~tphFHEaDOFD~JHe=)Yld4|fEX z^{+So9{`g;Y`=(*3oBZPIS3Ny2Ux_n7ttEV67Iq`O_bKvOtUCR+z|&{w4y;uWzm-V zK{)Fs@FD9Un_QTT=>|~gm69#1j0*oMLPs>cUNE$T8W08#S0Q; zWTsiM({UVHmD$8K?80UAq*#ky5-R|~sErjN&nA-@8DuN2&{Z)gG7`ug(2!Y4fG>^l zI*6S%ES9A}HK{-6AHsr|ggt*MI zf|m#&-X`PIV#<`4*7-Uh!dh6S=2-ytLL7QY^uG8LFaWSOiEhHtV#;P+ApUNT2p1Zc zN>lI%&-Y=sD7t{Xp-xhjWv{X>GqFmOLds%T+w*jIK=E?9Yf7abSAisE@pW2u zYIto}vqn{2BpleV+m>*VtDu~?p9tZFWGt1qSFEo@1GXI_3e%FD z^}*q;hui zHnkuQE;tYSY~Db&s%1BL5%xMA;$^5ldTq}k{YBua7!V=l(Llpz5!u|(nw_) zjJnT6WzxELr%l!xrzT&Vr%Ppb+ZUPI@+q|0sp8KVmQq#|eyhZ5Bt0gHr3Mn9o+J~i zat7!GI*+JId!qS`bw|%Dn!h$BARMedhfm7J7J z(_Hyin&U>c;SE(x)zN60Vr^?mhZvNs8x3rdd*hJ%vkq#7Uci+q_5$07OlGxhFH1{e zNh)NaYIaDI##rW)6-tK)8JTp|0vSrVGuo`A8)bM-vJDn)jZLpnDo|QAuWQFOI^u$? z6no<`!kS{1uDsGF<`U%m!U9`mD!4(5t92x@ILBT}8&=XyfQ71vblN40$_Xk;ij{j> z``{HYyWA+{lC_PjdLtUf+EbdLV|i&AX{Otzmi2?7R~p^19L}02yzrBhMB^)waM{;Z zatEdsvMWWtX5?Qw=}MLgkI^@RDq(3)J1B%_5lhZ3#G9})~6v-9ZjgQEp0f2;_Z6C@u*L4@H4 zZKLA2nwuX}rxGO?4MkQ|(o#B2;7$VPAI}l@#K)AC&<(m^$f>C$9aQHM@m01BI1onx zuJ+{#fmQ)nNj!MUCrtg-azN_50;+4zS7GQdmM*r%>A-_)0x3!Zlz|T$(aBPX z>8TrnQ?eIsyy8@#FBQdAeTj777EeWC#`Tpv2aFl*6gHgsZTq5Dn_hKIC==ryF^F9a zkA$%%IbJfg?2)RcPQ((lUoZUp_!tyux>%o=6WAh5;wJ#PT40LAK#Qq&FqSJ64m5_N zMH0kcma5&5Cq)v916H7K(GGmi(}cET&f$k*0Ft<`uqj9V63A;yOoQb83or>MYt z&awskqLT#kn0;x#S&s9qO`Ge3Zn#?b&BtmR3MWJipV|TrjSovHRy7bI6=RjBEmZiy zJn~wF?2>z9H&fydxPJp;L}+GB2_)2k>Ji@al>JbnP*;R7MlU3Sq6N2J9=DBIAO%~- zF?Bv5yc|SBWXQ%fb-6pl^sBw_Y)XA zN5nUN$h90e&AkoN%B*Ua?uV$*y(FgQqM~D)#nL3-T`4DZ;b!rCm=@#H;~C1o#5|v| zNn!RAZ$mU&^QYEFK!Y6&(C_lctqu{3qz^ZaNGE;cjx3YCgu*?j%U|Lb`QNc=Vb@`@ zv^|H@YtBEdz7ump_;?G6*Zd*b#j=3gk3j>j8q8w}Yuq}Y7`WE>gMTys0N>b8huCg( zQHM}-nV3D$Md;5AAX}8pYj#2W@RmDXNbiC@j3u1a9AXF&(5mD6$^QWF>`(AE8Lvk) z1T6k=`Ajxe1)!%cHTafpzb!obOt66S?kFJ*whEM`6JP)vNil!MdK1TbI-xUK#ds>S zhv@GpAv3eGi$Vg9z@kSqoxxU()of+1?q=glwTzK?GKzze-cW=N9Hca&IJcGC78fUJ z$stK}D#v6U%DpiN*5>P5dcyuAv0F_Nk$;MF#sPGi8U}2+e%O_mtw@qJRFQs& zX6*Eu&GnHT5vyCh%-kTl3`B(p zGs=jAAs`RL8+O5kK9-<$no>_uiP?9lqcs!})?r0CNpsNvb;9CixmlHwZdVmbrIC?P zgqKREApZoH42Ft!AL7tE?!pCYlQ6*j>D#jD7nei9G!OdF}tT;qWr4F=A|CI z&7zY;!kRacbyWIdDN>+aH5ZNCP#v+D=?{h4q>{M|KBs5jNYxV$<`dZ%4kyO=yGe5@ zNor}QJ`i)ukCGAom8n-x9Hj=Vktyt%WEwk3AthIp45)$wG~rmK}F zUkW7gTKOlsAyHOHP(+(cb$cuXY&4DzQmmm4RO)5NjfrgoF37#ma1I~CstRINwK;2R zr~@9{FtX`u9$fKpT@Rn>IzGg>n$B??npm9F;kBakLC}>wg$F7Qxja*}H*lNuM!fNn zWx${TDcqBkB;|fu`K2HYz*;+Fg^KFVDdMF1A}Wo8r75ibVXW1%QPSr|Fgmo`ebB_}Olt0?Wf8e+7} zor7Tsx2@3#%4qu#)Gfx8%cPs?lb8`dC!#rYl$wO`>qyS=N^abcWQ*q}ROc~I78LqKc5 zAt~B8p~?iQ!0XE!RHKoJlu@K)$H2-k5B&fH&w2tk)dJdD+#y?Ig3FO z_Ob{nR&d?c+;3}XZ}*h6{xI3u)|(ep?6Y>AIWoHGN_I0|98KP*Q z%bKP_M%W;F`NbpOLkX2wR~hk#2-1!a0>-%4j8MM!B=2m*hW zS<9{i-)jj4i`H6FPTcbmE}>ajZw@NiK?|zCOjH-8>>-}5VZ6G3_Jod?@R8YAY!TZF zP_&tI)>4jQ3^ltVioGx4L71!{_kV;3zYY@zI=V^hgHV?iU#_Ay!AjI3i|@&V$L(aE z;~nqIgoo`^p5qHSo>pFirrE)QtU-zy(@qgj?KG3ve+VJdMiV4jq%GKg2s4+J*tJ*P zDyGa0*NTRUf|R<6N%R-!eJBdPm@tCr&Y?9eb*nG{{{RTDX!}XHVv?D)zfj_yTJDWu zm7^I=+UANkicGg@Ur6p~{zZW#+FYM{+NoH$K;Ic6^#1^g^++X>;mj*gWe#-Nj%YZp zr6g%nCTELHon5!vbY{-F(N&TSj?!a>rEXSe?S(6HM#-b?HdSlYWK-&;4u=hrKm$0} zYBLKgDJwQoMqQEewh5Z2(lJzw?AyFtvuXuLqd4BG`e4EirOjH9Ac0^8adaxX@yN{%3SY?c+(aS)%_#^zTjZ98f3x?3ka#|l^dG~mpib_Bu7bTK0;Ee zneF)uvM*@gxuioIe*eg@r$9;rV?9)N}f9{YahY^dUL`S7M(eDAHor7GMJtY zGfF~=s) zr_3pQl;Q4&s4R@X-AlEIgC#5KjP&XE3Us(TpbrQfERmME?l83nl17dJ2Xrpd@>a14 z1vw~f1~Sva4+|X19)dZ-Ck(cN5X-8+PUyV%Rwco-S~2Zbb;hNs@)YqsHt7ftI5JK5 z#z}@WZ#O-TYqaGlwM-#}T|TK%5pz!oOgPv{GK*5jt-cyd@PopVGs>G{MZ`Skrn?KO zGUe2?q?M8*W;8{Km4}<7U5-!3>+4@J)f+tqlH`>Mi3KedX|Te{^@XE0949dHSAI(I zAlyMu3Hk=5mn7qv-=fg-F2vfV!a7W*QgA*p!;Z?xt6Z+mp}h6(N8=P6OD@#zH>os< zb1K4aw6+G^qWLkxOA5`*Gcf%my8A}xTDjKohHvTb1kKI6H2niwVa=-s6KR?(`v~xV zp;IakRHvpHS1Vy)GJb*2IN2)HhU=8|)7@69E_%S{87(t}E)bzM;%U@%F~%Nw7Ux6X z%6|``#ZYTYFL3V%1pT3x#XM^4dC_17QST{2`?$P53g5vZ+c6(7&gx&hd(EF(o8vI7dg4 zcsM-Flg>ULPO^)~(Tf^o!L^*@0LxpZGQ+}Zg~g{*obIxERkjFtOi;=F9JKWXL(GZE zCTips@&*fSZKg7#!d{81VNY2-YJ`=-R*n50uWAdR$s^~(R(Sg4KzUi=>zLM$2f(jJ zq>ljv{GM0+DA4YYj9(Y{cc0mw1MnHDfv5P`dzO9 z2jd?OUca;d005{}eT;nxkMG9hdzm(eQ)*PR<>bHoQbH#(o{=_G8ek zv5txNPp+>RQ56S%k@1at@^o~5(rd&=Cxf%Q>MeMPUyS@Y`u@XMde5S z5@Mk8fV$12WtZUUa@MIfJH#vbJTt0w>TURl=YJdca(_SBmax_KFylP$(l8=*-=s`# zACvT&GF3$$(198VrzGc5K>I=;@FUmhVqvV^8g-kIp-V7bRkDT=TwpH2nRGX zLJj9qLVnPN`0fj}{h0V#h74vDb0;_hFs_pugPIqiTK-mScBn&TXl{ZDy=k%bglfJu zgM0fj9v-4GQu#aJyn4Qoje+eHVfKzyme1M{)$JMK)`iV!8r>HWynZc(eJ||DbY_gF zj#gab1IH}yF`};mP?D<(vO98wSA{4Nx4lsd>=D%*Z7KYz`x&}z6h1Y7NWcZTy5=s1 zpFwD!&ccUKf)hh#_IFtJL*iFxKVoe6DdA6T=Rn>4yy&=!ogJZM*y*H+iV6^M z675VmZdCZkhth)auis*#(!YC){>+u zVYI0yVFg0sKSUprriXvz&kO$m-Hm<`;Oz?*(U)70{Kc?L!+-dMevRYM!Bll~XNFr? z%BO2H^MS%vn8gEx4v}fJ$yVfP@|D@fW#yPiLv0~0s3j=_AOPJNpmFQ^(e#H2s=X9& zgwiu^G>$bj>A<6H{A!~xROKPSOKr7&xZ##9sWj=cM3uUhz6JV3!#XYDHGy$9HfN2) zk~3@L+)_2fy42Fp)8#wEZYc}iMyQ+Pb$@i&WT7^GF{GZL~?nsnn+ zZp}R7Wi5?Fl2QVSNl~_~ue4e`vfz=Tyf3G25ZF@#*V6JS6;4;jDn-EC-vuc1$<dzxGk@D7$}jOi|{x}$i53}R+pkwm1*5iV566#IJVC?t)2wj(Zx^(FwkCwt6D+B z?$;cyDyG6pCub1j9}MZ&=7z!FmT~eg1`Y~1uZ2D_a+il#MLv@z0_ruH#E`_QPQU=o zYgLVsJ4pZ&7|A%xE>Wp>x%2RK(oWL?9dz`^wRj`kkI=;eoP=TtismVwx57N+-z94Q z0Jw&LAaxZhwj(rw?n7w0w37Z5tJZMz$k{D0@@A!vR%r_zy2r9L(v`1aXFO>RIy!IB z9@p>#;QOPDX~7dm)r?1hC{=32(^G}{$=L?nb<0k^lQLuDd}YZv`t%4V{(#3?vu1fqTCx^lz&2)WpXkN(&aTD4azhumcV_%mu;`#1r*$_ z<77jsr1GcC^Sj+4;L%H^X*&on17P}fE}AJTRLd0Ubh3vW(exV-HNN`}$Jk2gl&1mlq zOgdRcy4O!_MM~8dIf(M!u{R+pFbXSE6U){CSJ}2bm!zB0r2hcSqyGR1jjCbQ1d>5R z5bL*o$BX1c{O)^OVrkEU3IjKm<3*7g`KAC{9Z4oIf(vdK| zw|JBN@uhIitF4|BIQgWmz)vJv+5A!?B9U?Fge!<;_leBiJNl#0g*atgjtZ3~k(!@FPRlKzq_~?Lh&yA{73e!h&I>CBQB*n%l0OLY>w<0Km?OuBbyO;NhQeZ!lfv?^E}u0LbN?8qbx^4#JGM%ntckY z(Uf*pQJ{n~Fy42t$ZMqdp;7V!P1L7Vb|sy3otBSiI6>&jS4z4rt5TkqOY>|emPuS( zhuZ~71Nep0*r@Y7OqnN28JA4Dfv^ZhV8-++DMHeH4qQbi8lRPQG!6lr9N))y{(lkT z*#;)??J8WLNldmDhvpo~Af$^Bl!Yku>h#CnpOdzXFMlww{r>>w_Wu9~XTie&Iiu|q z(q*QOwN80Fx^YBUg}4C-`zh5=Xfdbljw-brrrLauMwK-fB-b;`?LXkWD~9yliR9#{ z(z0^Q%p;3ohQb?cacDUtcEvN5vdRh>QV_`4l%~+{hHjTI&Jf`bjb*0iRH^2s8w+6w zB#(hcTRP+B+Mf!VTWDKSR5eU1bB?T!xJO?VXsDcgHRRd!37;rAD^Xnp-Zvfyx;|$} zT25Q_X%tUVX_Bjb*57YgwXIylB!iLKPuY(+svIb*^9j_Hl9!enj;UAnnDw*V#iLdn z;OLB9ULVRBnsv#BuAN&;HcL(A&`5DZ_@k0loVgt^pp&`uN4(Ai{{ZfdAO6OV!ahsk z-=o|=qq$5wiR5LaXf(H0sR@oDu-j$Z6{6s8DGqxvzT++t(pXQ01_aS6(8`sFsZ*1# zyrOm5bp<9}BRg3^6Bib}^(}1f*M{zjPMG*Gx@f@;=0y`Iws4X2APxLmf>w4Bd;Rw{fXaGa?jh9+i+-a!0g<&qLX1l-5i z-sd+2Obx=SqmEg|Z`IjF6{$tbc`9WLl3Xr@Yil7_^aZwWgHf& zMIKa5@QlWjfblZ{q5F-9h?~;pI-CTmr!WGu=72xCKI`Fk?rjYT;Q8EVqxsp2&Ksu! z^+KII5~-`Z*$!D9kZH07uKCxR=2Wtufj$B_EuTuW|%mR7N< zq7(*c8>1&Te5}z$TILrqi?M9!_)47Oue*#e#nsnlPr}YVZ}glo>;9B|%RLm~%^l)e zy%J*;%~qtT@4qE3x%QTqA4&;AN<$!0V0wuSx4-5*kLbfcO`n5g0$Q83b1G}NTw zVpdUPT%o-{6uT2*r4kLY1ATsBaz0G=E3}b?bW5hH;&+UFktinMB7W2scxOXe1`-B{sdz zzeVHa?g$(da6X>6KxtbBVg=QusQ6{q#EFoaOKw>cYH1;AQWbt^8-{VQaKzv(qpl9R z0IOpBDCKr)uPHMw(@oST8+E%az?EDE(smx0&B}Rhi%K`M82gmk1;QA+3TXu6JS|GZ za}w1FWXro!l}p96mefc|P{MP0;pbPHel?L}{zxaH?fApar?2{3sTX0R2C23gk z)o(a*4lMV_-JazJh-jKZ(*7m$!w$(;=}4m$GR;kdyxLsjTjv0ww%5)+5x+XoyJ*u( zHQFSLLWfbAq)fJsDQl9LTPk%u1f+v#;;&uNN=aPal_gV^?0bj#C+KVbOfa|o*4N=3 z`6p;+{!B2p{nppv9x>N+!HpP^FQXsgB)qMuO=e>+_2bI>y`9FtVm2svY;c28gbuSrLDUvMA%Z8;TDvvbN zV9K&isuG|D$tNMS$Ita$A>%w-DdftgDy`O~o?2}uIZ}FI3tjoV#VCSrvnXz|O{)^2 zerA0kg3&s)Bur`%(|9Fo1-VNXLxtXwFO+&y!L!_{oWxXV3`eBu)a3w8AUcW-H7OHF zNz8H1EGQt6>V%RnnFio}EmQY0bRScuIVh@qWtSG6UR2dld}!+cf}8!DM5R((`xNpc z!R0ih9bID`nDvdL?|v8l<)w2%cxrbT;2NyfFvD_pa&|^#peLmo7bV_c33UkroIpt% zZEM>x_dK{h=<|fel&jb`9l2Lw)Y8pDob4`Y4Bbt*l(>a!AzH+xz@f6*is_A_pL0H4 zQS!ETwaGB~sx0)wVM&*BRQfgs<~?E4-?^Kj{SD}fyNd9Vt5a2qXl>w{yyI@NTh~Zk z!dp(`eY>L~;6I>@Il>ngRb(oYZ?`8aDC6@Ar?`|j(bh^3loFKza-w!@NsDB*k3Jux zPfNBOc3sAixk^a@!%aipBicPE`<2=?a{%He==FS9^=XL*-H~o)pwiker3xqtNMu|U z0hPyWd0oNlLyl>WPxvzc%ojs7T9O%^RI-3^66w0%QcZ{iHieLEMZ_yKQn8g2ZKgPO z$F&t-a)X2R4WZQQbMsi7$y4QBn|;Zen_8%Cv=ymF%#45p$Il!NI3H+7g-iI)jdrq@ zXq=5r2??aMQXVrIFIlx;CPvF|hhcI&-3rX-91ZpyN&l2m4aayPrw-z(*-x>xU_a-zKAxsTcB`Wo~y($JXO!fW^C8xjQ!3(;N=`<^0y`t=&KLgX zW**T7iYu7AM}mH)%{d%=#FX;GuQ1idve2ZYYb7Hx2)G&C*u@vzq`?&QX~K~D5RsPNi2WOIwWdv0xH0KQpCZEMr)IG1>kKf`b(G@N(t=V3@<{?dLC`O_^?~WQ zW=?-fRVtNszcA8P^){U`$)zP!KqVT$Ypp|Owz5Xtx5H}Sk0WH3q&P?yiFPLe)z(CM zucwc44+*Od&eb$aKTxLESZz)>MXAh(<)(-P@o8~DDYYGdsYN|D+a6a`i?nKt%|3Qj za+8vdsh8MwI;(BOk`$E<#1KWgqt2~O_eWhhplXv1IU%T&>YLwWS4D^yCOr?}o$g@I z>CXpKv7V4B^Eh7))KDLhr&G|LTQb41OQlN%Eu}*=# z)we5EUCuK7HWp3(&7)jBDMCERsWjQ)vCL%}#{~KC0Hm(F~tLLfd&Tf!oX_4+R@s= zJ6J##0Ep{#M-@k0b-)%!TdF%&`=g442C#rEcCGh8b#1x~RyrUGx}duYAf-J};zn67 zEbHus>Rs{>;!W~^K!}~vE-4ai(&Sw>Q-ufafbvA54U2_CqzC}ErW*;>I{i?R99t?O zNKp~OV{Z}Owc7!V3p_55R)?8 zoz)~n?BzwLlnZPW6W0*4t5JB|S~}>;Pv$a|P~TTAmV*F0Ah&?l&O1(b)KcAX*6f1o zM{7go1)(}Xt`FKTdvGU6Rpa|X?`kUCrJZl9>Lm!~khs5FAqaA@I~Z z7qDtia>g<3bdSp4En1E_s)f|Dm9deTd9&Mv89}b_&y^&g#HzbveDTl z+c6kro!V2B2$XYjV_h06(!GgOksxtJOzoU|+b~7qI*NvKZlAVcry_RO)v)P;88-|y zPL9ZV}+0ei))w;F?aNg zITZH}$T+3CM_fGc7crK){*-pQ&iHxc$8Wj-(_Kl~26%2RV^UYs9jvdUVb7D?9Z+6G zz2*$jA2u&RQoOrO%>1Rl{6araG5Lj|Nvv0CrvCt}xBmc$N9pQ1sWj8DnfbxtTH?nE zjVM@inCt^$B)pk}QB$)KH;E)>D)LEAt1q%& zX)kMCH_fm?>W^_F$}JP=8y8hH%YbEGt*enQ5L3|OAyx}PnQRemv3T`%G}3gHnUpFj zmuT}R97y7PSu0hcV(ep6#Mb?7L|VOiInN z(~}D#>QaeVPym|&mdo#p7Y2msZkokTo21EQI>G{7k)L(uoObuJn^8iEBVh!Lv&`je zPlX2#e0iV^EYvW(z7wNN(-PB9$xY6eg5PaP1e>Km8!K34(G;u??u2QtNpH=wg)~`9 z#4gGqE0|3=nWwm*qC!fntgmBkn;7U+WtR5pORohxW_Q}(^l856{U&CSf^haPr#!YM zrRO}IFqC8*ZNw>QC-_v^1Nh8p{0eFlHC_x@CTekI$(Io|tfRiuZvpoK9|&-|)0Ug` zt)nI|{xs2(_$HQWEk25pWG0%{btzt}VNM-f!BMu}Y<<1JBA+u#;FEwQXB4GOA*sD$ zy}8c5N8ChQ;^RdpxT(0xvy#~Pi`_%S*rJJ~JRZ#ouH`mabd{;pp?bQ?C+?{e=sd`e|Eh6?;!zkW9W2NXapGaj6vo!k8 z@1T-C=OStZx{7`+n&W9rk*a%r`=ondl80B0Wl(a@w)BvP+PFo1h? zM0oz3DD;fOX%&fv&`7dFWj2pssbbX0L3{iPGy5Vru`Pa7j=U0==5qQTPw=eK6P0cf zSn{l@sBAwvm7$UkH{g&h_eBaz2-T#uQ7=5g6>ODpdqty$mLj82_(kJ`HC&(N9IYi% za;zyq9|VlAX!(Y(rP-E)h@;8RYaNchgMYnLRW% zu3?Y?Qo7WVg?*^_-oK#?M(?^SDS{=G`N?XB$!&cshnUEtx>qGt!wo0N$3&l(y4N4sUo6f7&QbksdUYj)5fKx zC@Dw_wZchCNf))iJEJR_**>N}!y8R!gZBqKUyCJNLp5YdnJQkjtR~&&G;P0BoqKNx zF?Z0c!?SuGz|Xw3I`fn{C9P^VDp6qsl6zy!Z+2r%&sFiZ7SUvc*%CCCe6Lbr2PI6n z3Rnv3Lwe+Yp6riJbQ}Kw64(p=VUhm;XGBRyhyLhXgKzN5(pr|~uR7LW^>(A@lPuE~ z4FIKdnEDsp53%8qq`(D_Eo(peyHWFyLiZ{%ZI+Q`z_=cFA7eTe9+OtI#>EaOv_VS3%XXsqQVHj(Qu{3AM>U?(8JZ)Oi@6l)+8JcNYo{pnqk4~ zHnfE#_z@JP%xIO>scUxyt!g(?l&L~V=`nG`)ru`vOHzkBAvq~8X0>VVp>4I6N#+|2 zw?&~d^VP!_MF&D$B(csGWu8q>W_?pqEri)3S>PRPC#WTQzh*ww_Y%o9jVs*W;3@wA z_@9S9?G}j8m#{99<)!SUh0tf6N_N{><}cUr!sn}^8~cf3)r~9v0GOx$0OEcpl4IV@kbq^RzX8?*tVO177@%Y?9<8eP|>suc!MwYHK&EQNyOs<6o^ zNK%RFiJNKiDmLzZlBVTZo|Sr9Wu!kY@(XS>;zE+Nkz$e#L=bIg@q2=vkEC>M?vGP2 z?5x6+id}A{D${B=DMDI&Bp$(N_L@9n&>enT5__vI)MZt!#KtD8DGD16n|xpHh^o2A z`i>nL>|P?SO5uz@Q>@OYrb?eG?Az){xed00rCYcd@sCLU>rR=m4lVVeZZHd7CdnNw>o|q!PhhaDQj9=Vbd2ir7 zC6%nK`ASzZf5|E)z1na79dB<19>svKP=|4o@@mGU$zw$Jn{z4=3MLjxZ zT~g4VEwXMK8w7rrG)eye6*x2gWs?5@$V6{OIc{fF-0;uc8GNILwv&e#QUuhlK>q;v zHX_sRU+AV=O8hFRPgFpdr(uaFYpycxX>r6U4YE5!j-h{G#XsF27vZg>?7v!-{{UlR zHP3O|7Rll66P0R2GRqeEI7#u(Xc zq2*C2uFW>no{NrwX3~0!%13;7rG)*_tu|3Kl`D#8@x1j?P)!GBsL$&1H(J)`w3dLf z2Ccv;u^Ai3uv{|mEdG*sE@>kVQkh)EN@|j~CD?3TPOYUS(116S#oN9=JhVJNKZz@{ zG)W0pCTerD2$f=82_A8(2o08@z+Q;d>k5ZJySawRF(o*z|N>vY`Di;cE z#03C*LZit%34W#!v#%MrXdl@939 znV!dKTL+^^Iy1!>ZN!!NPMrL*K>BJy2|m@Maj`E(AsLe4T-nXb* zL2+Zn)JN7@6vwS;M?_dV7Bb**HHuuk+LR6X>Oj{Wp;sNz8=)-~Q?wU=XDG}mrlr+W zs8wEcWRM=1an#)Y<7Ei!L{N+TwDh$4rV}pvZtMF!pWiXDNbcduy)Y_iKQgqpEKsn8sWV)4~x31r@2xedIl4E&{ z8^ISDGbFVlcD9!2?dpq?hg3Mx-q3J&$IG7LWj5WgerqZ!S)z$=F+`9h&+Y9@! zT78_twfqBtsY^j9W$MK87M);9h-rb;KuiA6*}hZUa?%Au9Is1ac`TtaiKGV7$~8wj zmfrrzgSa1J@Uhdl;OU`yo-a#DdTc(-l&D&Dke6Ilf|NP{3O&F_$66QR>5Xc7WWiMV z`lEQ03}QLdQ|6jnst~zlxsn`hNCMVft_{H#*lb3t?zl`UUxEzLPpw;$N$k4+00{FR zfzp^|}_bxGn(_AlTbR?ky zS@!BHS=HxVGVYO_HZ$3rd5b0R76!yvo^7=|D@CVF)Shh~l90F@+46=Ad0gyuSt~@ zrR{dxb;J|6D5*mIaqb?0f8sw!-}HV%{hblyUkN=Kv!-u#BBO$4oLf?rip+}>3vbd4 ziB^>5y7dyMQddKZvApQ>-Coj0kP>BZzXzxB5}E;^nl$U~A+v0O371eATPujputpBy z=Z|!yC5WAW70{V>c@MmbrAZQW_8Qk3$ug=~1ppgrrLsuf9;3@sFqVLHT{J0Fsd^R_ z!?F&{nMnpf2x+txsDLw4jfo`YNx3=4oA|%onZ!7Pw8bv~XzoTxOiU(SnX25<=A2kk zf(Dg;FekmMgsdH3oYV5xIM}$~Xoh?ZBPvvB#8cPz)X(>&% z+C_p=CyjuQc4SuTvNG#@YP_b>RxN8P+5Pe1jTJxrsfqsp^)-L(#v2kn)sF`K&0qdK zG=KhftN#F}O~<^E;NQ8ef6LcL{Tyvq{{T*#f%QO-X!t_^0K_(tf9Ul80Q)-Q^Ivfv zr4A6^@e8DH`aM7X&ba*7TH8bnXfPrM;t7{T7#^sCkRXT~hypG$5C9uMFxLh;;s^z! zjn2pbS6MdM1W|xI)Rcal9jAIY>mRJ6a8^ z0mYj~N-zUj1_M`ga7De$xaDZMnt1oNUJ%Te>O3B|a z?H!4cNM3&eusa~Dqq(#;uLy=X+N-N>Fk7%fXBSk6_|LdT%MVv^SrdYg@fQsr2m?uc z8y1vLWI>EwQ%JCP?hqb!)V{ku!4HS2xj9ihDDSA-iC2UQsE+Steh~sOeNb$B?hrnA z)ep}b_6U4EUB{IZN~E}i{9AzZS_mYZSIxLc>WGnxDun!Z_D3fbRMI{>Z%~7Wsuo1D zfHk%Cm8!D1;^IST9GYU* zH?@p?(WNX^M;bH5+LdV)FUb>nsZ7cVCy8Bv&O9bXK~9A!DLS{YSUJOYrqreLR#ntv zN#UZ}ceRYE<0ll>Tim_j%^*!-d{aoS%DJsIOr2(Rl&dgS+QQR*^oaI%M4ElUI%mTY zwR}HalW}e!7vvmyDVUaBvqI2pMUMMe?F5MM)}KSFRoqV_lv|x-McRp>y1VlYOg}Zq z0m$kq%JC6ZXzMOrh@psd9iMa1FLZxL*h$R>yLyMB1>!j~8SrEi|V&3yM7SC=B?rZmWQ7ZIUoG|3IF`6b2qqlb!hDsUNRkoj5YTcd5 z*&b!4w>HMIn|a2;$m%Sn{t(6bqt68uIHyV4mshN>6C7MRe3ZEzYZET6D(3ORb5BxQ zUaHB-(~#hxQ&v{T9*Y;0@;Xf(RKb_NmtZ(s;Am*#InO3baRF-6P72L!#+ET&vp<`HVI5)P9VI0p1Zn8Gx z7igR3x_^%G9UDlvK2ruswKl`FvbW}?QXJyd$EaJIkjpsqQjLhIzDAHPRHXv=hV`!c z2703VrC zI9BYM`HY`l5?89{Mi^Ay0NWQMUT>k*JGvWUL zQNI%H@|#Tl9-K$I%++C;V2UC!OD+kt02v745-+W?*N;_niC(5mX#W5cG~G%~ZaGYv zN>Z25NhPHK265rs#|c^qO_NW5yZ-=qK=72Jl-tR!+qM@B{E)brc$uz_Br&FpG>M0C zbuyVW-1Qcq%=<+sIhNKHs(Oe75xf&0psIeGuypnam&Cqbry^b2dTIrQO|~6MOMGUb zMeY;^R@;1|#3XQ*q6FB}o*jle@|vThr>s0A258!!s>Jp7S)ecDuLZ3(}P+nnr{xE;aXAf)=vmOyVET5?Y#Xl3!{{V8wM-nw4u(?`@k)3~;k6>lmU{mH!vZ)-VRK-qSG7k%bAyL=wa3K%ljlWf z{{RwSDn!ly0L>~s{r4cT-9tyZPQ`dOywd6Py7Y*gmUd&9fi%JNZgS@~wV(82Ft<2J z;bkiPmGC-|{xDI(FA4tuG=JFt02pfx`iI2dvFm3Hyb(szZw9P$h7maQ60?srW-_%1?v-7o%b- zr9JwE7GkW;I)9TOt=1WF3;0lx>1wGdByShCc8QH2$Y|BJT*NpoOAJd2l0&Z4Xv|Br zsf*`ztpE~p&s#Xf{n3nHPr*DNTd35KrzO0vF5QqWZc^pHvOLI^;VXru?i)uRlx-%& zi$|J%=anL>P2rD#Diuk0rKQsR%AJ{+1t|o}JDE}GC+~+z;T?m5%DFq2JNUoxg%QFx z2IdP6`w^nSxjVTcRx9+g*v&cDY13L2$E?fLWgUIW4DuU7A4yV_SXclH=5HUJS1_I> zd(x%Np4OH4N7Jrwj=|OmOAir$p#>ZxaA_l6UF-ECDd4ZxNERyIlWO-IWTvprl`;FZ zmi(I2lva}3=DrLl@!;4MTKPxTKIzosHQoq#l1`U0#Ps5)B+6KqaVuE?_77pH|X~f^sHTObr zy{zRF6_uq7A<#AI^>cSbp*gya1eW2zP|!}RHyYQ-XH6H?A8 zl@ygE`YIdJMZp`vF|-~MxJ^dWo9z{-|>mz7kMI9^utA|W~9J`66I!{U4{jfp8eu^)vPjp}X z`eL|sh;K$`dMz(b4i5m`Ij3jta(ymXFoXcj&Tsra|x6rWrp-!qrBbZ4acCD9Snj0{N0met9Icb%<(8S0<-C;7;9C9HVQT;W{ig$Y^H*OFvMRY4&JREx3g~kW!$+ z0R&&BJf`6#JzvrHxk;$Cb3nWqxFNqYq=|ebB;9&XS=w}(W?Y+iymkq8R<#71pHAG(w-M$0TqW;dR8KRC<)vEM+bwMG8Jxcf>S`K5 zRD+pxS`~AJH8>??)7GMP8x64dcKt%~cp9rz zmlBlrU8$EJbn2x#)cK^Q zLDQhDV6BoQ_QsF-Q?+l=rtl+$OJHYBCLavEhr~-`I2s4W1pI~s=o=SbCs@MKZ5L3da?xl|y zmxld8uRjuoodDwHeykx+$?F%WUy~-?v2+|N+#4xl#su)Kbc-5VObZOV$Tj8SQokph zT9I=spK1VbdBv+ACr^xB6~e9M0sNGVpcTTlV2{yF4#=(+K0&W7ig)DWj5djMdr%{T zo-fr0h)-RNTmJwm*}H1_F|Z(`g=F9s6xPMILg8WFLBv$pseVqlu;;}N^8$D{V$t&` zBc}0f{HteeZzncKTo~}MpHR1xn;n7|3lAZ26*ejZgN7|XFp=K|{G2go)T72Nf0cDW ze$Q<6K~D#<*`~E9I;s*f46iP<3Egd8+{g|9t`zk5nh zY>?c+Fx=?=m8tW4R$dvy8d~F|y*W2!hbYM|ag{eGV zv6py2qYYAmZkAnFY+Pn0SPHO`Qj^gLs(lC{DW)nnSo+O$GjyVg{W}lS3+W1yTB2F} zCrEH886_Z$&T%OtKW9=AO;zs?ou69J@{EdtgmcdyUpFt@+7uyfWlcCXguh9b1Ru;w z8@V7UC#=N7m0$wDlgh2v2sK^RNhzgVo$4{HF<6-p;YktR_mrdzfmeJpe*#D*_G}w< z#Ix9$?Kb?x~?v4Efz)MIfwEz)(3ndYA$7Y zDRiku+~c-7AzFRRTw&O(D+Htt_Hq=q*ImNE4q*~S=kyMpFi6qifv4-1DvU=~fGdk9iag5uF3YA7z z=G<6EtxFG5NjY*nnz)$8>L$v~dk$c!R7v6LJf%1?ol@!&3fI?rMMn^64q~WCn0&ZW zd8V_r)+?kyFBh{r!FkKHSb*Db&fl|#5)g2j<>~q>rxhT1U<*%&(sJc-D4xMsBQA z{HWDYovk)on^O{vB`r!e!2-!cDbCQ7(+gS^td+E}otKnT92mjmD5hrP)poF%k-aSf zZxrO{MtBmb+2b^$w5h{rkd@Ivlt>Aakz1RTv3QM1wQCy25MH7-t-?--%Zf!hk2N+& zqN|WUvMcj?f5kN*MKcPm2XlqBEBm8Fm%-JkOCb$NGM&W?i5Pv*Isscx1&c$eVYjw&j=!ioXfTv0J5#|1jlkHgQQzU#*B-WJ7+ z+g;U#JkP|J=T^z@i@hc3V$_rFj9OQMa+IJRtPB3@f8!621sMj-msoV3rpMzN^LRTD zjKz0W6PUsHg*@Y3z2dV|YU%{uP5j14I4)A#JnD<;>_CU|oERu7_{pi2b=U~doWacrsyxTj3`2+07AuRuQb~{3PFij{KPL&F#lc4&fRA!SzLuqh49R(*FS3 z46GSax9Qbiwhc)5j?(fo^TC4*0rH8V+#xf;gAHE7YGb?z(1r}C-}Kwxd)Gv^wkvuMR`BB9X*FQ zO;op2`)XkL#JaTKCj?TZQ9G%#SJSqTvA+o15}YTDYjRG~Q#9P>tdxRTUl@R(o_4Xz zoN|wze7}aPSD6CYo`DHhrnx90w5m6EGPI5f z6lE&jQ+GV}C-H-$f;|mn0abeUwT`I%_-R(gf|$2bDzj4!vTR)Fh{fqnO~kEAsm?gG z+Q=C~c&cmATcbYGEDrEQp|`3HSl&tUx5J^uyw*_zOaQMAzhzLJe`gSy60lI>`KBj&M-J+|VmZ z%xetOnDK~O&LgGL9w-%QwYjVjstL3qhx~<#KXZ2QJ0ZEI4j|> zKTNFkT|bNHJ_nCW;8#Stqck@r(GtRPFN<9@&B%3rj(DzhDy2BqMqGf#sOR# z@X$F{$scF&i_pOo#QM*lwqSh8rhk|3S%X!0pv(0rgM*eC-&!NRKa2?ASw39b+LMo9{CV!On$HY}9b{qTc22*Pti zS(?I@-_iv+{qcCOV2UF&upJI53OSu~lr2labO$97$`Ep3svRB)Xc6xYFNMCu!J zjAhy{qEeQNI5FdweQNom2OfcbR&YCO_f?v0Bsh9lc4+$s-&v2 zy=!H&BNrM*8KbUT4oXw2ys9Nm6{HbE}KEUe9RFCFEz`&|ou`VlfH%^4`L0JZ>ty}bn<>+SvC3>Q$vN~=f zN%L-)22x|j0AaddImr(gt#6_yFVxOcp?nN~X|KdVm7#13pV~|D5u=PhOCPBZ7!^=y zWPFi6Flzk~8D8daG&O*~q?|p3M|;iSB>vEF_lVIEK&66zG*65uC=`?WqI_ZVI0TQDI!|bgAIMa}woNwefD}q+t0F*GK zdVL;*{HfAYMnc>k!kCFwSaa}hssx(`p%Y0(bd}tz* zOh~bmMsNm|O;75Gg{<~M$WT4(G<&$L_s0zxX!IPy&wO9gOQ&1ph=U@VA{EV(O}e1g zdm-c~KK8mg*=61iG;O1#i`hg)_)}zg*OMLM3@LQ&L3viK`!jg6{F1_<~WBtpy|=xk3uqKe`rRH$fYmxR=b28%1Ubd znH~_GR{J8XjL*ysAA|B`tFk!YO&^SmqV3aY(gduaKPE?nah8+^qc<26 zN2m2w_&|&YqyGTruG^?ajAi9_N~6L!OH142?T+Y|RgFa%(7@V12PrpI)vE>=CjbxG z?|U(znR#^l!aIZ+EXwa;9uYdSFDgDi^SXGO@yaF_YZy+X8$P7>#0irr-aKH2SdISx ze0;8j7ug=pq_-^1;>ojeX4C=5f?>y<(;A^XaZ-9NVw_u0{{Y?qIHZiWM7$*@*rirW zxqO$!E-Zt}6oNY9URq^U6zeXcL5#T*3J0nrWh!cAK&5)U@yk~Nt$1=ajd$GAouO*8 z#tN#h(Rhw_USSL7)D(N-yXOM6N5bEzm~6W1g;}|N$m50nnBHR=lqyD7x7cc`wXR1wDK<*2 zHlg;6GjdjGAbxhEZ{CT!%Q{Kp+5rCa85HRfEV-8Tk3yq$TQ61iGig5MRZUHkoWhUV zgp|z6ZVbYQ+KZ|%?J3oMkZnI|7Df%Fx-zM>J5?HS)Z^FoBTchEDR>W*FUAusICJF+ zzwt(hGOaQJXVz)0fU+elS$Dva521U)8za z8gg_`Afh#EvbprC7*zCqEka3|a;Ylv5tvF zUT~dOPPWosn4gzzvPHs{1bqZC3f4!7JT#lQl(jL}?uL-1Ec(`o`c94Jr>vKc6ocz3 zEmX>`9j7FJC`#lu>j-r2I-rL2O-+KnrGfqNvaK6)7KFPIvRXK$($Y(-GcFClgq;LmXQ}y@g^}kK!MUcb!y4W#xHe_J`vasddekp=D%=HN2%LlC>Mc4i1H7 z(F~o%)C7W`E3vFP2>f9KjIi2L3aadtc8|s;>GfcvBuSO?KqP<-C%(&cc=C)E=rwGaYFh=3Bu=m4&pS|E;?aI04WqS3ch zG=EIkT<7HkcL+{#+)fgGi&|;RIjcU?3#LU1wz07hMOKqTWnu+#I3iPK(SM3FRCL!V@v7(@DD66CuW{YpRj+m>mTCs7kdt((kOrezLP;j24 z7$UTfLL;3v;q6I`9d3MsN13fZTxA?Se+!$if&$aFWfLH(q12IUB{jT<}89@br@+H zQt?VP?W#xvC`EXqop9MU*%`Z9O2h7`s5ttqvdx$kX)=g~SP6sn3`3~O(_0!7xUK&H zRka9g<|dn7U|Y5*p82#}`ZFHEfL@Ylsy>6~wFVQL8iHMm!t2wdSP%)`Ar~sM?h6-H z6yv0-DFAVAcObzIlIX+c=1dvGE(ORg=GGuV zt_JfIlcefkSIFkQg9GoKP5xyy>_Kp`buM%$U#!8--u>wYOxiD{i zDtT3nK5&kWSY)g0IIL*eTD3H;Q8MBsN!HYC0h}~*LzIV-RNKxa-GJq?7i&r~Lg&e| zhp46%MwDdlaY~!k=+%YDtFgNDAxL-vX5LDa08zMw>6#j($WJO^PrB-@Ym*AANz$kR zLX|kI^X9@n`%;Til4$e2$%rRBcj5v_YB#uSv%9VVV!Qz}}Ua1JwE z=4}KTP++3X(q|FWMehb)lUh(VqRo;;i6SSEb0w)*F^Xj4uBGK6g>!-d-xIBqy+qq& zQjU_)LHiSo*GUHe2A&s;*h(}$IpNf9B;ys2XOF3?iD>|NrNoN0;dZ9o8Z z+6r-&DD`L{cZhug&6k9@pgnHr9yMRSJ7S1&ZY*%$$cXPSBNO7|vWC}rhj7YWiu2ok z=sFZm`yPu$5m<*9IBn#4boauDtYZ)_FgN!TqE(Kn6` zVTn}78Ju!RHkC<$lHWAZn3QoQ!D<#6MIMWz`MPxTvr^U7^^h0^N1npJyx5ft9N2XG9QHru@xZv}AvcMfWrzj|)V>N3J zF0?;};Fzn&VJq%j*rlRN;L)t4h&L4BQL9p=U@--uENd_!dy zo+W9_l_#z^C0|IM(-d@JOi)P&68j*>93hCkY)0YiFuI7m^O3*rfK}`WIq$Sm4iK&@ zZY6i`j`K3IEv<=Ox9;8xFfPQ{Z7@<#LcZvwoEudmJWDb6!43$iA4?HFuJZ+lbY0P` zNz3mLWg&LhJJceKj0ZS>Z%huH%pBl152st2S7M9;KW+fMi+3K{KUibh5y_ir zIr64c*s&EeqsmggdTZN&8absC@sFr}^2{*3%W0pM8)q8tSc00Il0}s}E(h^NU})}& zpGZ1S@t?*xUs(fj0HNO|UAIAB_`%bF zXeVr<)!M)Dg7ywYgyITXKqjdxex<=?tsD$$df|P%G1SDFn!S9HbU`#p{en^C%?<&as@KJ;C)vk#) zRtN8Yj29J=A?L0iN{CfetMx#6>xb+~t2YgN%){;~$fvSS@dB(HLPpASId;LAt07fm ztUjMcs*k_$f}DMa(le=~_9XrgRN*=<>RSd5oHa;FY$=k$+lpYl!pR9QNLYCz^oTu) zKZF!%7Y~0tQohOjA!CQ>BQGV(cP0!dRHS~(mlyrm1<3hXbmVSIk8~5sTG@m-8U(#k zfT7Mnygfx%4x?yE5%BLFQa{UQw4)c?&Dz7I?V9Q9FyR<6=M~L4MwE)i7$~s`5VKY6 zG}3Rz4$PyTUlGK5X}@DtP8*rKcnRaS_J_@A*r@=Q2r;0si!kC(#zTeT>@uWSyKc)z zG`=B>Z2Lm&Gx! z`EoW?DET8T8ak6rii@J~=xE+WBPF_f<3wIE!%pR>k>LgyyAC&y2|eO6e-OsMOp4K? zx+o3c2?p#IUd|skqv}e4x*t99ZF?Hw{D1=X*_1PL9?{u6*0rKIoNW3=L&-&VL?F@C z8B0xz;C8_(`ZuGQ6^4}QUf`pgO${VdHLh3?Md<>X^0Ht9{m?3y^(6lQFxsPQaxrR# z7iwK-tl6;Gv0>QM5LqG87T3Jre6}C32K8`TcO_GTvlqX>;$i@YM@e*mlNh(k| z<|A36<4~%4QWt9-y4lCMtkk(P4J6TZI3ErHZbSaGH-7_RkUl_3KfMrZ#m$lqxg&o% z5?(Gs$1zv$&N_8&b2@el+!Cfa`EUS5Rp=s%?OJc)|F^W+B0jjvK?{F-{uaysV&$ z!&rS^vs?C|xvI$o-BlTIdP{@xfHjI@GgVex^#zZ{9&&OHJk_9D;+)sD2`>xK_fs1k z@SmJCRQgqEan-CNhAW^qa;mc@_)Hnck&Cy6D0}IRc*2Rpv^1a5%#qs>=Nr%vuhM8R zMPs}=6@Mrvt{!paWWp#EC1vE+L>(A*hkwz{L?^x;9Bo(Sz>c8;G?Rx$$>ZS$VPq%# z+M?g|b1-iKjA3aC@teJaf; zZ8^e&OE^v9l$n*>Z?qm`NLVVMr1F)a^#NdANH&8q9#&A5r93&upZGkW&g6j>oH1Uf z;ukVvv7T_jG>F=XlFl_uInudJc8 z!pw%E%XSLIG)mf3nR;Ry3eTP|c)B85OoXX2Y;4ac$?zRh3gJ0q2de6I?Ot&R$R2?g zQKeiW-H4o8t~1IE$TF5rM^V&66E}qj|hVrO~WF@&I@mYCrLP6S3I}~S&*`_bW6*R)mZR_62z$e zRg>5v0UAfcOBUtBw&D~TLBl?^K_k3DnFdx)2H6+1qjv0oOKhC^Gh@0Slcbz4a-Lp~ z7*RSw!v`Tz1F(oQA!KCHX{-F5zX(vY)raQo_(Te{gN6s~A>KrA(k>TTa;czOt{u>{ z*qC8<;21Z!a z>D-UHIWrCrZUI)+*!fnpK z2soT|Jw6duR+KQX**K^v(vBG$XBGEBoH8=o%bZryd>}bfr`A3ZQuooq+c|0Pf%nnF z2p=rCg`Je5%W0Wi@=@@Cr@1ceO3gyP#5N~~4E^Ybg@QTidm6pRW=hIh& z4<5dg6+xw3EI(Z^2TOQWC*>r;of6{9YFt?t=NJO(fcn5iPv2_`os&dB9V}sjN9RV+ zXGN&83a;{C)EaYPDWig#USigd>FW+W0z!7 z#+LBXjrl^LQ>FYWqnqp29Z<9EEQQ6%g>jjU(cEp$V<@IMrVK7r4;|;JAr(c*3+SWja*jEu?To&7i zkbTmpHmPx&moe$`H3n)etN@f`VO1)^;W0%{mT;;HKw7Rgl7xy$Rw{A~ zbjgNExl@+tg`6@n?n;ith$7UnYo@{}qfA&|={W$cG+~6_Hz8*XjJfNb;sGSzIHrv< z;g=KXx(pZJj}4z)L%Iy?c^P$ZePrn`;*4m}QsM8i4zDiIKXBI|W!os$Q zVrELR$YUMJ9B{7P4bFJ(*_*XBo+2Sk)~`mtiBN>L7aIRcMCSz0F|4ZqL=T>hepD( zAg*bj4xkGuaA#qWmu0NqWCt3d6rrYkIHY<@yb$S=4S{l}J0Q-*m6v`nC|tpiq{SQG zj}45{w=LHIG}DJtK)ao>S;vu=)TtrWT>Kh^uN7I`5aY8htWGH<({W?6BhHzy;cHV_ zXSNmh@zceDm9A zjKZ{wlVBw|z^!Suh3^#6raU-Pa#frc-**oj!qg$DBpGDOqIGp|dmfnWeQ#f6QWvI7 zJfB%+Iju2a;txGyD>@8}`NNG^&e6qgD>!>*)GSDj)M#a_Bg4$LI~B$n?q1o)Ypis` z!h{9npkPa=K{*2kZIN8YNpV-s;uc6M-v=sefv`cf+De6#A-D7wFgT?gwX=fpPG4`| z4mjq%lnP#M-jIy*J%gAGFy^vscJ;)=GLssYxVLOavoM4xC@5S+x+fb53fCIPbaGX4 z?2Th_w!><1lwOYFk$=rXWtD~ zGLfGfoDp+E*Dl+z#pRIhKLN{&sNOlnyY%6Rnb)vNl%*JXFG`!4u{Q}j%u7l<}NNQG50HNW2$mN|#zm(lU_ZO#9U7BzLu$^OT z-YluW@)Jl;D%Dp?fzB(e)Id&7fOkXmR9)i04*l`v*4A$Em5#V5E~?(crE05O$FMf|*q zdhzVqE)u}+gjJ+kn|U^rH}-E9*XUIF^`JP|1zXz|v{Las8eWk)rdo74ZSnLtd}J!T zsdG3dO~rEx{*Y^r8yTXJ`=AJH<}$1=Y~!9W7PbnBv?ivMnD!I7&tgG%o~YomryT6O zH&tSFM>(bVm}I1oV2TDSku6wqs+7q&%q;i7a_7`Db!WaAn~l-h#6-Csu6$aO|0v!e4d>dF=8HWrH8)z&s>PQfAXpQA0Y>p4O zcf^jwmo(*^8&Nq#V13BrFy~AGWJ&6Q1}F!7MG2=mU<|{AoZzro5`d&^$_H2z5dr1& z#|eer*UBp>695gjFtT=JVjZX4Rzl_+rN=dQh>NS(EIe?Lz7bJLRnFLC>#hy)&8co= zHp3N%#G^cgpg~7QR)*z=LT`|SR#uzb#N7;h76!;S7;Th9@vG^94=-inz?Ci%EqLxS z!y^%VYWiTo^^Nd^N$Xr~F~s%#@f;=(Oc=}(AOY~pI5xQvAIF}U?mqkAuo4=@M>8F3 zZ-xc#xDG$A9e{suAZ{Uun9FQ;9!~f{$a~)$RcL3%+k6#Izyq}|@KAHSGB4)^LY-U4{U#`8EW>isC#zZ?*@I z&ujobDm~BvP|J@!G2DM#06x}^))2#w>w)8sOdzoQC<6dpvE2-K{Pw|1moUP^@}PIY zf`mthPfQOVlmp~H3)RjzDL20Ok1rmW4n6@gM_nYq6(pEoAG8N3fJBD82~aT~$`4Ej zD1yvm@Ssd66gTE#FidL4a;}I05@l%{Ace&5h~obMTz4JU0LeZuZjR$3F)tgz`11

tsKoy8#$G{h%fh3ir2p%+bL~k8 zd=DOPwgC?pGB2|@;W^c4WS6bkT0Z+;K_Xn8ANv)APbNMi2*Q@ZFn(x z3Lcb3YxO{DKo%tU^ntd}DVa&%5t{eg0+nrraIq#_Ne4Ta;Oan#!0n)T*e)VZ$y2~v zi&O!pO{CdLA`>{GeEQ}SRwA5M(|FHJ*2q-KOeIF(-aC@a9mKfpD`(aem-aFzm%CiK*sH;rwHi6S&$dzuH zG^e_H%vpiU&8J6&1t zjAufmxhG>t$}6fh=}aprCI!kb;|om4f-Knt4uKfzaWz31CDo_C6izRyBV+6Fk2YTn zoTYa&rB{kpv8Ovq<4-w70Q3km6ggD-PTcjzVsVX8TV7d$99K{|mlRAt!>V*sBU(Ee z?~zyonxv37L#Al$DI0`i0~ORq&M0@mhA^p(k1Th_Z{bxRtfh#K#>nO!Ly5UCEMfU{ z8>xD|agBN9RQ_Pge{_23GnU~OOLT2*L%I=ip0uAt_(H42RR!%>JX=N+5$c#St1pw9L?)`6iuFWUEaJiFfGz654C2+c3(6FCLngwSkX=#? z7mrs&2Pm=G0IVNY+hC+$ngJyAM+uDm9HB<%%U9KG5OrxM(9kq4(OmbkpkLFs{lbtheRyQ*A#9N zA--UXH5!7K8ncT+3EpC9j2$STc**2kbQq+fruoZvY%Gd{{U^pa7Q7*%gjj z_5COwTKCd0MwFLuqdI}-@&k!;}M^`vD7P!-~w`Z3&E z#^@(2aEo3$l)2Liz$hADL=O_ai$Z**V|;fiS%54bi&sPyOs{at3(8d;%GNS~0;!iT zvI<#P-5h5Z1?6fxVul-uSrB5v;G$(f?11Hpi6~QxP~9D+YcB+S~1m$b!ffI`xgewl=!W4Z|0mUx+3>TLyZV+x< z<+cbdVRZ^xSUCs~($3=p%a_m~d3uMcAPWWJ*Yr$x6uyWzFJI9S+`WH9L13_6DSZ*# zTK-8eR54@k)bvMSny`8k+uyg1{%LzK^;)ido(FLHwk@pdfst zzoH>og@OiFeNzL;oC5l$59L=z;cmzuEILzdQ3aT+CB!-OObM4?(J-K^yYxhHtGazr z2Qk=7h+4KJ!18sUP{M+&;PpgstGGQ;2QgSnlej%o1`Mv=husG?XF3E2m7N=hgdE0_ z3T52-Fa}l6qG0B%>D)Xcjalx21(@uD@^=oz7~v}K(Sr)Rsn`T@t2}lA1(>W9%Q!RD zG0B#3N6|2EE5B=4M<%TLM(8ZUz;ZW!fq^n^ftm~mtiK#yf+9FSbX$B%q$SFv6iWW31yE|m=7wtcMk|RmEG7tg3NXbSa2Ue$8vWM zs$hJq=Roj*6=w$F;R?(^8Fv8mOmLQ6y%PrVyN75I-d1y6ISqC#`(7wo_qeBabB*^j9OVv^|A z>1JbaIrM1N)#Gw0zQa{b!D*CDDHg1x?J_o=FShYV4|Bk zMpDjuG4!1EWS2E~w!(UKL@;uo1Avy$Q*Fr+qMgj#51!RdoknI^W4a1t=J&F%2>Jzc z4LA<66WM4)mQL2u;3BgTlKspG%FPd&Uxarhn);=9M)I*Maqwv;_(UdADt5M?WI>oT z{-zRT=10w|!T_-T_Lboq$0n0s%_rG}os^MEtXWOJghNscrWwRP^-A!7EI;*X@Quhx zQ&+VK3fwhB0g#YM_@?5s22&0)?{93!33szGYVeK4GBo5U zkkfy3DAN)K}6j#l9&~y164Hl^$z$?t&S5ee6SVN@0}3RVM!c z)hYnWK0aw)5w*WBAcyV)WI+m~fU!eyS&UNcOif3$?lD@CYBCi2f7ReC1ipSoVMBw;grO=3)c9f03=`f zrAFM5Nkzrbe#|9mW_C&pEaS>z{E6?XRB@4ib7nSeYI0SQ5<3JK`7J{#0MBe@j8`L%dP)2~b$5SS|8rz2_*ZxwVrrZ31JKQ?2$p8o*V9|+jXw9;}y1QAck zBxHun7Zy}x2Aux@)fnW{WdYX zlPW#YsrgD#3m0hwM3Yr{B{p?hLVIAitYb^0&4GR)eh@`cChhSG@Qt0ytf)@JS{kaS zD6Nhs!V8NjkqSiE`IUG;iZtqmOept8wwjcxMr%sDV})K=Wvq}OxMebLAh{Q3P|UJyP&sDC$Kvl_xG>;Zl;Aa=oA ztV^4GJQob8$~=Kk{%*fyz#0_tax5r!MziH66fT*I9@r5$+1mOPJ7HW|Oq0k{x3>_l z2oEDp4XmhlM!U+ICnpkn;D%ueR%$^Vuv{{kCz0m9rCt%jku|-gct*CjTM7hmeS!rJ zw4=^SA>1;VAChKA&8QB@1523tC&D%osZuOdfz<#SleVxelM3OK$Gn{O^q&YSsqXX` z*c@t5jg5WK36zm>7gAcwq6>+{#}!U=?l2xsccL}ZCrxoC$wlEN>Gaho>m@Mr0!B_` z9UGuI1HBQh8bvKADzZ*{vz1Fq1nI~f_1aBhO~M>->86nZj+__PnT0pwaT zwl;zGptxAac_W<>##O=U#>M9CA$yQM)D+>GKK)+hL6|a__a<}bz!_zqRK~h+?Hwx2 zi*~^IuSQkQzTUWbgDH^mWsg<>u;BG$TvimN9g03M3Bq)%X1}~cy~8UqK+B&-IJCc^ z8qvZuqH=dG*x`jK1JZNv5M2gVWV~93s$;mdeG?kk!ZdO+bgRA-Ru!a-tfAi%aIur| zYUqFnRrW@L_l3)F*UmrwLL)%qab#0x+!@ZA!hU zGYcy+k`Bul01rWpTg@s$mt@>_ss-E~NkIbS@5qQ(9gLuZ(Hsfc{n4oTt4ai$D&3+x z%}PlZxgFvuFtL|d-(fM?-(ihCI7*bU=^pYSRN$Ioy?}nqr78O^E!}0 zA$vOEFEr_DNg$tU9%L?NVRP5v1GjOFFU=}yjfMT80IVfUM%A14!_0%3ni16H& z!qmOG#yic5QnCsnCopp}ZJVREosp!BHA*>O?}1hqr6gvf;|5`wt-6eG?2RcnZjrp0 zD)8kf9Flxs%pA<$x(3}*q>L*}AR80k2N+(IouKw>@n6L?2Q%5JryG$qI+R8N2G!|sw1`qAT#ij(Ew6<<3sW_AzKnb?Sb+Y zFL5Wd!@2`I)jb5j>XYi2(2W+F6Jxdq$du~~0|mg(H5JfftVc+V6i{j8W|AO^pwssz z{qVC4&oxzj@zzl3jT6o3BV(&!zc?tv@>kyrFmp5O9=`}19=OpC8Ix9N*#qXxo(HRHcheF#Whp<%f8W$7^bX0S$Enn&08QhL1ZS__~M$W z+cM|0V5bw*&Rn6-Xu~)USixi~vFOD0M4yy7;$cCG>fg>+?L>SEU}*)}Dx6nV)yieh zY8)J2SW*DxZu>?d$D<3UV2QFRNsb-GHFYY+Y0^K4!F=bd6xhuxL;B0U(T31&n5!VW7W0m!f0R4oV}>ND{{T35#KU;#EHTfV)j|5h zzR`ds#=|ByULb$J# z=LgQHcgG5p-`1 zit&Q-`JwZw@s3;&?28q{a`}_B#UA+N&&&hyit&ObIB$Sr=%`G6HF(0pp)`JRUNGGr zj4GXw3gEdvHTA0Tgv|8F`PD}`w06N&00tBDJKR@%cP^Pex$%I4z)*2LF@KC#jB%fs z`egUPQvi0r4Tt5kJ9W>D0iT#ZIIkEXfJ_96ndy(FPkeWjCNJ@d@q$4|r2q?gMr3Sp zUN9o5G(L4+CJ|Q15$M1I$*N6X<5lAWutSG5R7Ko=MCq|rYbuNdK0njbi?7(zRA z0ddEb=8w)R#tY?XvGc3Xt`UJd1OOdXNu%?M@q&t+0f+*3{m|Ir{9zO6z`|fH93<_Yd3ks~&t;Q?F$9Ym{eB!)e0qI8_u$e=TD$Q?oUM4%r(|=g6 z69^^@Ho^f@Rhv0+it#btRhu6=zZgkdV2auyfal87gXfpy0&5eWTCW&s1sh=K1J5hX z{c8MUystO)it&cSYzf#PxG})1&JB*NcgJ~VbL-dbLs%)>3gA5ziOsxn{7e|ur+&G9 z(S}20bTA#7#NxKPquyW*UTl2gykVktL5A1_IP%2T)++t5VOX65@^{VkOVOE+SIXSCaKrQu3$NP=W36P9b4cI=wfLEIDouT2ykG@NWFJjlFlguiHE5-}H0Z@i2D^rNe zjaQ6tu1$}fSBxZf!4BA09fuxWoch&x$9ZySAB|P}VYCU@Ah?+5*JjVotHuD Date: Thu, 2 Apr 2026 14:13:57 -0700 Subject: [PATCH 05/25] [Infra] Harden supply chain: remove unused scripts, add pip binary-only install Remove ci_cd/publish-proxy-extras.sh (dead, unreferenced PyPI publish script) and .pre-commit-config.yaml (pulls external repos from GitHub on git commit). Add --only-binary :all: to scripts/install.sh to prevent execution of malicious setup.py during pip install. Co-Authored-By: Claude Opus 4.6 (1M context) --- .pre-commit-config.yaml | 40 ----------------------------------- ci_cd/publish-proxy-extras.sh | 19 ----------------- scripts/install.sh | 4 ++-- 3 files changed, 2 insertions(+), 61 deletions(-) delete mode 100644 .pre-commit-config.yaml delete mode 100644 ci_cd/publish-proxy-extras.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml deleted file mode 100644 index 2bc361bc48f..00000000000 --- a/.pre-commit-config.yaml +++ /dev/null @@ -1,40 +0,0 @@ -repos: -- repo: local - hooks: - - id: pyright - name: pyright - entry: pyright - language: system - types: [python] - files: ^(litellm/|litellm_proxy_extras/|enterprise/) - - id: isort - name: isort - entry: isort - language: system - types: [python] - files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py - exclude: ^litellm/__init__.py$ - - id: black - name: black - entry: poetry run black - language: system - types: [python] - files: (litellm/|litellm_proxy_extras/).*\.py -- repo: https://github.com/pycqa/flake8 - rev: 7.0.0 # The version of flake8 to use - hooks: - - id: flake8 - exclude: ^litellm/tests/|^litellm/proxy/tests/|^litellm/tests/test_litellm/|^tests/test_litellm/|^tests/enterprise/ - additional_dependencies: [flake8-print] - files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py -- repo: https://github.com/python-poetry/poetry - rev: 1.8.0 - hooks: - - id: poetry-check - files: ^(pyproject.toml|litellm-proxy-extras/pyproject.toml)$ -- repo: local - hooks: - - id: check-files-match - name: Check if files match - entry: python3 ci_cd/check_files_match.py - language: system \ No newline at end of file diff --git a/ci_cd/publish-proxy-extras.sh b/ci_cd/publish-proxy-extras.sh deleted file mode 100644 index 6c83d1f9212..00000000000 --- a/ci_cd/publish-proxy-extras.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -# Exit on error -set -e - -echo "🚀 Building and publishing litellm-proxy-extras" - -# Navigate to litellm-proxy-extras directory -cd "$(dirname "$0")/../litellm-proxy-extras" - -# Build the package -echo "📦 Building package..." -poetry build - -# Publish to PyPI -echo "🌎 Publishing to PyPI..." -poetry publish - -echo "✅ Done! Package published successfully" \ No newline at end of file diff --git a/scripts/install.sh b/scripts/install.sh index b9912287b70..03ae31cd18b 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -83,8 +83,8 @@ echo "" header "Installing litellm[proxy]…" echo "" -"$PYTHON_BIN" -m pip install --upgrade "${LITELLM_PACKAGE}" \ - || die "pip install failed. Try manually: $PYTHON_BIN -m pip install '${LITELLM_PACKAGE}'" +"$PYTHON_BIN" -m pip install --only-binary :all: --upgrade "${LITELLM_PACKAGE}" \ + || die "pip install failed. Try manually: $PYTHON_BIN -m pip install --only-binary :all: '${LITELLM_PACKAGE}'" # ── find the litellm binary installed by pip for this Python ─────────────── # sysconfig.get_path('scripts') is where pip puts console scripts — reliable From 06df8edf922b6b08c70afbc4aa1cfa1f1ac46192 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 2 Apr 2026 15:18:24 -0700 Subject: [PATCH 06/25] docs: cleanup (#25026) --- docs/my-website/blog/april_townhall_announcement/index.md | 8 ++++++-- docs/my-website/docusaurus.config.js | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/my-website/blog/april_townhall_announcement/index.md b/docs/my-website/blog/april_townhall_announcement/index.md index b81f61bae62..466d9e845f0 100644 --- a/docs/my-website/blog/april_townhall_announcement/index.md +++ b/docs/my-website/blog/april_townhall_announcement/index.md @@ -1,10 +1,10 @@ --- slug: april-townhall-announcement -title: "April Townhall: Friday, 10 April at 7:30 AM" +title: "April Townhall: Security + Product Roadmap" date: 2026-04-02T07:30:00 authors: - krrish -description: "Join the LiteLLM April townhall on Friday, 10 April at 7:30 AM." +description: "Join the LiteLLM April townhall on Friday, 10 April at 7:30 AM to learn about LiteLLM's security and product roadmap." tags: [announcement, townhall] hide_table_of_contents: true --- @@ -26,6 +26,10 @@ We are hosting our April townhall on **Friday, 10 April at 7:30 AM PST**. - Reliability and security updates - Open Q&A with the team +## How to contribute + +Add your thoughts to this [ticket](https://github.com/BerriAI/litellm/issues/24825) to help us shape the agenda. + ## Register Register here: [LiteLLM April Townhall Form](https://forms.gle/hvyVXwbFjzJQE7dEA) diff --git a/docs/my-website/docusaurus.config.js b/docs/my-website/docusaurus.config.js index 81c08609e5d..0102d96ee6f 100644 --- a/docs/my-website/docusaurus.config.js +++ b/docs/my-website/docusaurus.config.js @@ -284,8 +284,8 @@ const config = { label: 'Enterprise', to: "docs/enterprise" }, + { to: '/release_notes', label: 'Changelog', position: 'left' }, { to: '/blog', label: 'Blog', position: 'left' }, - { to: '/release_notes', label: 'Release Notes', position: 'left' }, { href: 'https://github.com/BerriAI/litellm', position: 'right', From b7ccc5b691d90ecca83cf1b8ce12140022247634 Mon Sep 17 00:00:00 2001 From: David Chen Date: Thu, 2 Apr 2026 15:55:55 -0700 Subject: [PATCH 07/25] [Test Fix] fix gov pricing tests (#25022) * fix pricing tests * fix mypy * fix cost expectation since us based model is used now. * fix test get model info --- litellm/cost_calculator.py | 4 +- .../litellm_core_utils/llm_request_utils.py | 5 +- litellm/llms/azure_ai/cost_calculator.py | 2 +- litellm/llms/vertex_ai/cost_calculator.py | 4 +- ...odel_prices_and_context_window_backup.json | 66 ++++++++++++++++--- litellm/router.py | 8 +-- model_prices_and_context_window.json | 66 ++++++++++++++++--- .../llm_translation/test_bedrock_govcloud.py | 65 +++++++++--------- .../completion_with_bedrock_call.json | 2 +- 9 files changed, 161 insertions(+), 61 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 29d28b8c896..3b73b853eca 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -545,8 +545,8 @@ def cost_per_token( # noqa: PLR0915 ) if ( - model_info.get("input_cost_per_token", 0) > 0 - or model_info.get("output_cost_per_token", 0) > 0 + (model_info.get("input_cost_per_token") or 0.0) > 0 + or (model_info.get("output_cost_per_token") or 0.0) > 0 ): return generic_cost_per_token( model=model, diff --git a/litellm/litellm_core_utils/llm_request_utils.py b/litellm/litellm_core_utils/llm_request_utils.py index a2292d6e00f..dc70069ac5a 100644 --- a/litellm/litellm_core_utils/llm_request_utils.py +++ b/litellm/litellm_core_utils/llm_request_utils.py @@ -56,9 +56,8 @@ def pick_cheapest_chat_models_from_llm_provider(custom_llm_provider: str, n=1): continue if model_info.get("mode") != "chat": continue - _cost = model_info.get("input_cost_per_token", 0) + model_info.get( - "output_cost_per_token", 0 - ) + _cost = (model_info.get("input_cost_per_token") or 0.0) + (model_info.get( + "output_cost_per_token") or 0.0) model_costs.append((model, _cost)) # Sort by cost (ascending) diff --git a/litellm/llms/azure_ai/cost_calculator.py b/litellm/llms/azure_ai/cost_calculator.py index 3cca61b2186..067181b946a 100644 --- a/litellm/llms/azure_ai/cost_calculator.py +++ b/litellm/llms/azure_ai/cost_calculator.py @@ -54,7 +54,7 @@ def calculate_azure_model_router_flat_cost(model: str, prompt_tokens: int) -> fl model_info = get_model_info(model="model_router", custom_llm_provider="azure_ai") router_flat_cost_per_token = model_info.get("input_cost_per_token", 0) - if router_flat_cost_per_token > 0: + if router_flat_cost_per_token and router_flat_cost_per_token > 0: return prompt_tokens * router_flat_cost_per_token return 0.0 diff --git a/litellm/llms/vertex_ai/cost_calculator.py b/litellm/llms/vertex_ai/cost_calculator.py index e7ac453e949..4ca3d29e7d2 100644 --- a/litellm/llms/vertex_ai/cost_calculator.py +++ b/litellm/llms/vertex_ai/cost_calculator.py @@ -203,7 +203,7 @@ def _handle_128k_pricing( ): prompt_cost = prompt_tokens * input_cost_per_token_above_128k_tokens else: - prompt_cost = prompt_tokens * model_info["input_cost_per_token"] + prompt_cost = prompt_tokens * (model_info["input_cost_per_token"] or 0.0) ## CALCULATE OUTPUT COST output_cost_per_token_above_128k_tokens = model_info.get( @@ -215,7 +215,7 @@ def _handle_128k_pricing( ): completion_cost = completion_tokens * output_cost_per_token_above_128k_tokens else: - completion_cost = completion_tokens * model_info["output_cost_per_token"] + completion_cost = completion_tokens * (model_info["output_cost_per_token"] or 0.0) return prompt_cost, completion_cost diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index d4f986edd9b..51b7b56a285 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -7643,13 +7643,13 @@ "cache_creation_input_token_cost": 3.75e-07 }, "bedrock/us-gov-east-1/claude-sonnet-4-5-20250929-v1:0": { - "input_cost_per_token": 3.3e-06, + "input_cost_per_token": 3.6e-06, "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "output_cost_per_token": 1.65e-05, + "output_cost_per_token": 1.8e-05, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -7659,8 +7659,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "cache_read_input_token_cost": 3.3e-07, - "cache_creation_input_token_cost": 4.125e-06 + "cache_read_input_token_cost": 3.6e-07, + "cache_creation_input_token_cost": 4.5e-06 }, "bedrock/us-gov-east-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 2.65e-06, @@ -7794,13 +7794,13 @@ "cache_creation_input_token_cost": 3.75e-07 }, "bedrock/us-gov-west-1/claude-sonnet-4-5-20250929-v1:0": { - "input_cost_per_token": 3.3e-06, + "input_cost_per_token": 3.6e-06, "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "output_cost_per_token": 1.65e-05, + "output_cost_per_token": 1.8e-05, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -7810,8 +7810,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "cache_read_input_token_cost": 3.3e-07, - "cache_creation_input_token_cost": 4.125e-06 + "cache_read_input_token_cost": 3.6e-07, + "cache_creation_input_token_cost": 4.5e-06 }, "bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 2.65e-06, @@ -31164,7 +31164,9 @@ "mode": "chat", "output_cost_per_token": 3.2e-06, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#glm-models", - "supported_regions": ["global"], + "supported_regions": [ + "global" + ], "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -37473,5 +37475,51 @@ ] } ] + }, + "bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0": { + "cache_creation_input_token_cost": 1.5e-06, + "cache_read_input_token_cost": 1.2e-07, + "input_cost_per_token": 1.2e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 6e-06, + "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 346, + "supports_native_structured_output": true, + "supports_pdf_input": true + }, + "bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0": { + "cache_creation_input_token_cost": 1.5e-06, + "cache_read_input_token_cost": 1.2e-07, + "input_cost_per_token": 1.2e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 6e-06, + "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 346, + "supports_native_structured_output": true, + "supports_pdf_input": true } } diff --git a/litellm/router.py b/litellm/router.py index 586d5961b94..1b8f7c91761 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -7856,16 +7856,16 @@ class Router: model_group_info.max_output_tokens = model_info["max_output_tokens"] if model_info.get("input_cost_per_token", None) is not None and ( model_group_info.input_cost_per_token is None - or model_info["input_cost_per_token"] - > model_group_info.input_cost_per_token + or (model_info["input_cost_per_token"] or 0.0) + > (model_group_info.input_cost_per_token or 0.0) ): model_group_info.input_cost_per_token = model_info[ "input_cost_per_token" ] if model_info.get("output_cost_per_token", None) is not None and ( model_group_info.output_cost_per_token is None - or model_info["output_cost_per_token"] - > model_group_info.output_cost_per_token + or (model_info["output_cost_per_token"] or 0.0) + > (model_group_info.output_cost_per_token or 0.0) ): model_group_info.output_cost_per_token = model_info[ "output_cost_per_token" diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index d4f986edd9b..51b7b56a285 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -7643,13 +7643,13 @@ "cache_creation_input_token_cost": 3.75e-07 }, "bedrock/us-gov-east-1/claude-sonnet-4-5-20250929-v1:0": { - "input_cost_per_token": 3.3e-06, + "input_cost_per_token": 3.6e-06, "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "output_cost_per_token": 1.65e-05, + "output_cost_per_token": 1.8e-05, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -7659,8 +7659,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "cache_read_input_token_cost": 3.3e-07, - "cache_creation_input_token_cost": 4.125e-06 + "cache_read_input_token_cost": 3.6e-07, + "cache_creation_input_token_cost": 4.5e-06 }, "bedrock/us-gov-east-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 2.65e-06, @@ -7794,13 +7794,13 @@ "cache_creation_input_token_cost": 3.75e-07 }, "bedrock/us-gov-west-1/claude-sonnet-4-5-20250929-v1:0": { - "input_cost_per_token": 3.3e-06, + "input_cost_per_token": 3.6e-06, "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "output_cost_per_token": 1.65e-05, + "output_cost_per_token": 1.8e-05, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -7810,8 +7810,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "cache_read_input_token_cost": 3.3e-07, - "cache_creation_input_token_cost": 4.125e-06 + "cache_read_input_token_cost": 3.6e-07, + "cache_creation_input_token_cost": 4.5e-06 }, "bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 2.65e-06, @@ -31164,7 +31164,9 @@ "mode": "chat", "output_cost_per_token": 3.2e-06, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#glm-models", - "supported_regions": ["global"], + "supported_regions": [ + "global" + ], "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -37473,5 +37475,51 @@ ] } ] + }, + "bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0": { + "cache_creation_input_token_cost": 1.5e-06, + "cache_read_input_token_cost": 1.2e-07, + "input_cost_per_token": 1.2e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 6e-06, + "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 346, + "supports_native_structured_output": true, + "supports_pdf_input": true + }, + "bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0": { + "cache_creation_input_token_cost": 1.5e-06, + "cache_read_input_token_cost": 1.2e-07, + "input_cost_per_token": 1.2e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 6e-06, + "source": "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock", + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 346, + "supports_native_structured_output": true, + "supports_pdf_input": true } } diff --git a/tests/llm_translation/test_bedrock_govcloud.py b/tests/llm_translation/test_bedrock_govcloud.py index 4dbfa45a7de..456eac84a3f 100644 --- a/tests/llm_translation/test_bedrock_govcloud.py +++ b/tests/llm_translation/test_bedrock_govcloud.py @@ -198,35 +198,38 @@ class TestBedrockGovCloudSupport: from litellm.utils import Usage # Mock completion response for base model + # Use us.* inference profile ID to match us.* pricing ($1.10/$5.50 per MTok) base_model_response = ModelResponse( id="test-base", choices=[Choices(finish_reason="stop", index=0, message=Message(content="Hello", role="assistant"))], created=1234567890, + model="us.anthropic.claude-haiku-4-5-20251001-v1:0", + object="chat.completion", + system_fingerprint=None, + usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15), + ) + base_model_response._hidden_params = {"custom_llm_provider": "bedrock", "region_name": "us-east-1"} + + # Mock completion response for gov model + # GovCloud responses use base anthropic.* model ID; pricing is looked up + # via bedrock/us-gov-east-1/anthropic.* entries in model_cost + gov_model_response = ModelResponse( + id="test-gov", + choices=[Choices(finish_reason="stop", index=0, message=Message(content="Hello", role="assistant"))], + created=1234567890, model="anthropic.claude-haiku-4-5-20251001-v1:0", object="chat.completion", system_fingerprint=None, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15), ) - base_model_response._hidden_params = {"custom_llm_provider": "bedrock", "region_name": "us-east-1"} - - # Mock completion response for gov model - gov_model_response = ModelResponse( - id="test-gov", - choices=[Choices(finish_reason="stop", index=0, message=Message(content="Hello", role="assistant"))], - created=1234567890, - model="anthropic.claude-haiku-4-5-20251001-v1:0", # Same base model name - object="chat.completion", - system_fingerprint=None, - usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15), - ) gov_model_response._hidden_params = {"custom_llm_provider": "bedrock", "region_name": "us-gov-east-1"} - + # Mock completion response for gov-west model gov_west_model_response = ModelResponse( id="test-gov-west", choices=[Choices(finish_reason="stop", index=0, message=Message(content="Hello", role="assistant"))], created=1234567890, - model="anthropic.claude-haiku-4-5-20251001-v1:0", # Same base model name + model="anthropic.claude-haiku-4-5-20251001-v1:0", object="chat.completion", system_fingerprint=None, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15), @@ -237,25 +240,27 @@ class TestBedrockGovCloudSupport: messages = [{"role": "user", "content": "Hello, how are you?"}] # Calculate costs using the standard Bedrock format with region parameter + # Base model uses us.* inference profile — no region_name needed since + # the response model already contains the us.* prefix for pricing lookup. base_cost = completion_cost( model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", completion_response=base_model_response, messages=messages, - region_name="us-east-1", # Standard region ) - + + # GovCloud models use region_name to look up bedrock/us-gov-*/anthropic.* pricing gov_east_cost = completion_cost( - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + model="bedrock/anthropic.claude-haiku-4-5-20251001-v1:0", completion_response=gov_model_response, messages=messages, - region_name="us-gov-east-1", # Gov region + region_name="us-gov-east-1", ) - + gov_west_cost = completion_cost( - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + model="bedrock/anthropic.claude-haiku-4-5-20251001-v1:0", completion_response=gov_west_model_response, messages=messages, - region_name="us-gov-west-1", # Gov region + region_name="us-gov-west-1", ) # Expected costs based on pricing: @@ -278,20 +283,19 @@ class TestBedrockGovCloudSupport: id="test-large", choices=[Choices(finish_reason="stop", index=0, message=Message(content="A longer response", role="assistant"))], created=1234567890, - model="anthropic.claude-haiku-4-5-20251001-v1:0", + model="us.anthropic.claude-haiku-4-5-20251001-v1:0", object="chat.completion", system_fingerprint=None, usage=Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150), ) large_response._hidden_params = {"custom_llm_provider": "bedrock", "region_name": "us-east-1"} - + large_base_cost = completion_cost( model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", completion_response=large_response, messages=messages, - region_name="us-east-1", ) - + # Create large response for gov model large_gov_response = ModelResponse( id="test-large-gov", @@ -303,9 +307,9 @@ class TestBedrockGovCloudSupport: usage=Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150), ) large_gov_response._hidden_params = {"custom_llm_provider": "bedrock", "region_name": "us-gov-east-1"} - + large_gov_cost = completion_cost( - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + model="bedrock/anthropic.claude-haiku-4-5-20251001-v1:0", completion_response=large_gov_response, messages=messages, region_name="us-gov-east-1", @@ -379,15 +383,16 @@ class TestBedrockGovCloudSupport: ) # Test gov-east model completion + # GovCloud users specify the base anthropic.* model ID with the gov region gov_east_result = completion( - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + model="bedrock/anthropic.claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "Hello"}], aws_region_name="us-gov-east-1" ) - + # Test gov-west model completion gov_west_result = completion( - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + model="bedrock/anthropic.claude-haiku-4-5-20251001-v1:0", messages=[{"role": "user", "content": "Hello"}], aws_region_name="us-gov-west-1" ) diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_bedrock_call.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_bedrock_call.json index b22336aecf7..dd49d9751f1 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_bedrock_call.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_bedrock_call.json @@ -62,7 +62,7 @@ "input": 10, "output": 10, "unit": "TOKENS", - "totalCost": 6.599999999999999e-05 + "totalCost": 6e-05 }, "usageDetails": { "input": 10, From 3ad953e75cc49535932695321cdf963f59e118cc Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 2 Apr 2026 16:10:32 -0700 Subject: [PATCH 08/25] =?UTF-8?q?bump:=20version=200.4.62=20=E2=86=92=200.?= =?UTF-8?q?4.63?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- litellm-proxy-extras/pyproject.toml | 4 ++-- pyproject.toml | 2 +- requirements.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml index 55c964557fd..9946ac0af8c 100644 --- a/litellm-proxy-extras/pyproject.toml +++ b/litellm-proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm-proxy-extras" -version = "0.4.62" +version = "0.4.63" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." authors = ["BerriAI"] readme = "README.md" @@ -22,7 +22,7 @@ requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "0.4.62" +version = "0.4.63" version_files = [ "pyproject.toml:version", "../requirements.txt:litellm-proxy-extras==", diff --git a/pyproject.toml b/pyproject.toml index 65d37804aca..ed9b0b8c57f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,7 +63,7 @@ boto3 = { version = "1.42.80", optional = true } redisvl = {version = "0.4.1", optional = true, markers = "python_version >= '3.9' and python_version < '3.14'"} mcp = {version = "1.26.0", optional = true, python = ">=3.10"} a2a-sdk = {version = "0.3.25", optional = true, python = ">=3.10"} -litellm-proxy-extras = {version = "0.4.62", optional = true} +litellm-proxy-extras = {version = "0.4.63", optional = true} rich = {version = "13.9.4", optional = true} litellm-enterprise = {version = "0.1.35", optional = true} diskcache = {version = "5.6.3", optional = true} diff --git a/requirements.txt b/requirements.txt index ab8d6bcd918..e42d4d54484 100644 --- a/requirements.txt +++ b/requirements.txt @@ -56,7 +56,7 @@ grpcio==1.80.0 sentry_sdk==2.21.0 # for sentry error handling detect-secrets==1.5.0 # Enterprise - secret detection / masking in LLM requests tzdata==2025.1 # IANA time zone database -litellm-proxy-extras==0.4.62 # for proxy extras - e.g. prisma migrations +litellm-proxy-extras==0.4.63 # for proxy extras - e.g. prisma migrations llm-sandbox==0.3.31 # for skill execution in sandbox ### LITELLM PACKAGE DEPENDENCIES python-dotenv==1.0.1 # for env From 3604b600d3fa6b0217c751ba9166f3fcbc5a24c7 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 2 Apr 2026 16:38:01 -0700 Subject: [PATCH 09/25] [Infra] Merge internal dev branch with main (#25036) * fix(proxy): enforce key-level model allowlist for custom auth custom_auth_run_common_checks only runs common_checks (team/user/project model checks). Custom auth now also enforces key-level model restrictions via can_key_call_model. Move the custom-auth key-access regression tests to test_user_api_key_auth.py and keep test_custom_auth_end_user_budget.py focused on end-user budget behavior. Made-with: Cursor * fix(proxy): gate custom-auth key model checks behind opt-in Keep key-level model allowlist enforcement in custom auth behind `custom_auth_run_common_checks` to preserve backwards compatibility, and update tests to verify default non-enforcement and opt-in enforcement behavior. Made-with: Cursor * test(proxy): isolate custom auth default check from shared settings state Patch `proxy_server.general_settings` to an empty dict in the default custom-auth key-access test so it remains deterministic under shared module state. Made-with: Cursor * test(proxy): strengthen custom auth post-check assertions Tighten custom auth regression tests by asserting exact can_key_call_model args and remove an unused common_checks mock from the default behavior path. Made-with: Cursor * fix(agentcore): parse A2A JSON-RPC responses in AgentCore provider * fix(prompt-templates): ensure_alternating_roles handles tool-call chains * feat(auth): add JWT claim routing overrides for OAuth2 validation Made-with: Cursor * docs(auth): document JWT-to-OAuth2 routing overrides Add generic docs for running JWT and OAuth2 together, including routing_overrides YAML examples and list-based selector behavior for iss/client_id/aud. Made-with: Cursor --------- Co-authored-by: Milan Co-authored-by: michelligabriele --- docs/my-website/docs/proxy/oauth2.md | 21 ++ docs/my-website/docs/proxy/token_auth.md | 41 +++ .../prompt_templates/common_utils.py | 29 +- .../bedrock/chat/agentcore/transformation.py | 14 + litellm/proxy/_types.py | 22 ++ litellm/proxy/auth/handle_jwt.py | 64 ++-- litellm/proxy/auth/user_api_key_auth.py | 183 ++++++++--- tests/llm_translation/test_prompt_factory.py | 186 ++++++++++- .../test_agentcore_transformation.py | 61 ++++ .../proxy/auth/test_user_api_key_auth.py | 295 +++++++++++++++++- 10 files changed, 829 insertions(+), 87 deletions(-) diff --git a/docs/my-website/docs/proxy/oauth2.md b/docs/my-website/docs/proxy/oauth2.md index 204a01538cc..c0597058cfd 100644 --- a/docs/my-website/docs/proxy/oauth2.md +++ b/docs/my-website/docs/proxy/oauth2.md @@ -61,3 +61,24 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ Start the LiteLLM Proxy with [`--detailed_debug` mode and you should see more verbose logs](cli.md#detailed_debug) +## Using OAuth2 + JWT Together + +If both `enable_oauth2_auth` and `enable_jwt_auth` are enabled, LiteLLM can split auth paths: +- JWT validation for user tokens +- OAuth2 introspection for machine tokens + +For JWT-shaped machine tokens, configure `litellm_jwtauth.routing_overrides`: + +```yaml title="config.yaml" +general_settings: + enable_jwt_auth: true + enable_oauth2_auth: true + litellm_jwtauth: + routing_overrides: + - iss: "machine-issuer.example.com" + client_id: "MID_LITELLM" + path: "oauth2" +``` + +For full `routing_overrides` behavior and list-based selectors, see [`/proxy/token_auth`](./token_auth.md#route-jwt-shaped-machine-tokens-to-oauth2). + diff --git a/docs/my-website/docs/proxy/token_auth.md b/docs/my-website/docs/proxy/token_auth.md index c287ab03644..d37b05391b6 100644 --- a/docs/my-website/docs/proxy/token_auth.md +++ b/docs/my-website/docs/proxy/token_auth.md @@ -790,6 +790,47 @@ litellm_jwtauth: user_roles_jwt_field: "resource_access.your-client.roles" ``` +## Route JWT-Shaped Machine Tokens to OAuth2 + +Use this when both are enabled: +- `enable_jwt_auth: true` for standard JWT validation +- `enable_oauth2_auth: true` for OAuth2 introspection + +If some machine tokens are also JWT-shaped, configure `routing_overrides` to route matching tokens to OAuth2. + +```yaml title="config.yaml" +general_settings: + enable_jwt_auth: true + enable_oauth2_auth: true + litellm_jwtauth: + user_id_jwt_field: "sub" + routing_overrides: + - iss: "machine-issuer.example.com" + client_id: "MID_LITELLM" + path: "oauth2" +``` + +### Matching behavior + +- A rule matches when all configured selectors match token claims +- Supported selectors: `iss` (required), `client_id` (optional), `aud` (optional) +- Selector values support both string and list forms +- If no rule matches, LiteLLM continues with standard JWT validation + +### List-based override example + +```yaml title="config.yaml" +general_settings: + enable_jwt_auth: true + enable_oauth2_auth: true + litellm_jwtauth: + routing_overrides: + - iss: ["machine-issuer.example.com", "backup-issuer.example.com"] + client_id: ["MID_LITELLM", "MID_BACKUP"] + aud: ["api://litellm", "api://fallback"] + path: "oauth2" +``` + ## [BETA] Control Access with OIDC Roles Allow JWT tokens with supported roles to access the proxy. diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index cec61405ebb..46e60c24d39 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -337,24 +337,35 @@ def _insert_assistant_continue_message( """ Add assistant continuation messages between consecutive user messages. - Only checks directly adjacent messages to preserve backward compatibility. + Skips tool messages and assistant messages with tool calls in the + alternation check, matching strict templates like llama.cpp. """ if not ensure_alternating_roles or len(messages) <= 1: return messages continue_message = assistant_continue_message or DEFAULT_ASSISTANT_CONTINUE_MESSAGE + # Find indexes where assistant_continue should be inserted (before that index) + insert_before_indexes: set = set() + + for i in range(len(messages)): + curr = messages[i] + if _counts_for_alternation(curr) and curr["role"] == "user": + # Look backwards for the previous counted message + j = i - 1 + while j >= 0: + if _counts_for_alternation(messages[j]): + if messages[j]["role"] == "user": + insert_before_indexes.add(i) + break + j -= 1 + + # Build the result with assistant_continue inserted at the right positions modified_messages: List[AllMessageValues] = [] for i, message in enumerate(messages): - if ( - i < len(messages) - 1 - and message.get("role") == "user" - and messages[i + 1].get("role") == "user" - ): - modified_messages.append(message) + if i in insert_before_indexes: modified_messages.append(continue_message) - else: - modified_messages.append(message) + modified_messages.append(message) return modified_messages diff --git a/litellm/llms/bedrock/chat/agentcore/transformation.py b/litellm/llms/bedrock/chat/agentcore/transformation.py index d6eb5a734c4..f066322b814 100644 --- a/litellm/llms/bedrock/chat/agentcore/transformation.py +++ b/litellm/llms/bedrock/chat/agentcore/transformation.py @@ -19,6 +19,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM +from litellm.llms.a2a.common_utils import extract_text_from_a2a_response from litellm.llms.bedrock.common_utils import BedrockError from litellm.types.llms.bedrock_agentcore import ( AgentCoreMessage, @@ -343,6 +344,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): Parse direct JSON response (non-streaming). Supports multiple agent response schemas: + 0. {"jsonrpc": "2.0", "result": {"message": {"parts": [...]}}} - A2A JSON-RPC 1. {"result": {"role": "assistant", "content": [{"text": "..."}]}} - standard AgentCore 2. {"response": [{"text": "..."}]} - Strands agent format 3. {"result": "plain text"} or {"response": "plain text"} - simple string @@ -361,6 +363,18 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): final_message=None, ) + # Strategy 0: A2A JSON-RPC format + # {"jsonrpc": "2.0", "result": {"message": {"parts": [{"kind": "text", "text": "..."}]}}} + if "jsonrpc" in response_json: + content = extract_text_from_a2a_response(response_json) + if content: + return AgentCoreParsedResponse( + content=content, + usage=None, + final_message=None, + ) + # Fall through to other strategies if A2A extraction returned empty + # Strategy 1: {"result": {"content": [{"text": "..."}]}} - standard AgentCore format if "result" in response_json and isinstance(response_json["result"], dict): result = response_json["result"] diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 8faf36df4c6..e4702364a3e 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -4098,6 +4098,24 @@ class ScopeMapping(OIDCPermissions): } +class JWTRoutingOverride(BaseModel): + """ + Override default auth routing for JWT-shaped bearer tokens. + + A rule matches when all provided selectors match token claims. + If matched, request is routed to the configured auth path. + """ + + iss: Union[str, List[str]] + client_id: Optional[Union[str, List[str]]] = None + aud: Optional[Union[str, List[str]]] = None + path: Literal["oauth2"] = "oauth2" + + model_config = { + "extra": "forbid", + } + + class LiteLLM_JWTAuth(LiteLLMPydanticObjectBase): """ A class to define the roles and permissions for a LiteLLM Proxy w/ JWT Auth. @@ -4198,6 +4216,10 @@ class LiteLLM_JWTAuth(LiteLLMPydanticObjectBase): default=300, description="TTL (seconds) for caching JWT-to-virtual-key mapping lookups.", ) + routing_overrides: Optional[List[JWTRoutingOverride]] = Field( + default=None, + description="Optional claim-based routing overrides for JWT-shaped tokens. Matching rules route requests to oauth2 before default JWT flow.", + ) ######################################################### def __init__(self, **kwargs: Any) -> None: diff --git a/litellm/proxy/auth/handle_jwt.py b/litellm/proxy/auth/handle_jwt.py index 6ad372f3b5a..4a6856b6d14 100644 --- a/litellm/proxy/auth/handle_jwt.py +++ b/litellm/proxy/auth/handle_jwt.py @@ -16,6 +16,8 @@ from cryptography import x509 from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization from fastapi import HTTPException +import jwt +from jwt.api_jwk import PyJWK from litellm._logging import verbose_proxy_logger from litellm.caching.caching import DualCache @@ -71,6 +73,21 @@ class JWTHandler: prisma_client: Optional[PrismaClient] user_api_key_cache: DualCache + # Supported algos: https://pyjwt.readthedocs.io/en/stable/algorithms.html + # "Warning: Make sure not to mix symmetric and asymmetric algorithms that interpret + # the key in different ways (e.g. HS* and RS*)." + SUPPORTED_JWT_ALGORITHMS = [ + "RS256", + "RS384", + "RS512", + "PS256", + "PS384", + "PS512", + "ES256", + "ES384", + "ES512", + "EdDSA", + ] def __init__( self, @@ -97,6 +114,30 @@ class JWTHandler: parts = token.split(".") return len(parts) == 3 + @staticmethod + def get_unverified_claims(token: str) -> Optional[dict]: + """ + Decode JWT claims without signature verification. + Used for routing decisions before selecting validation path. + """ + if not JWTHandler.is_jwt(token): + return None + + try: + claims = jwt.decode( + token, + options={"verify_signature": False, "verify_aud": False}, + algorithms=JWTHandler.SUPPORTED_JWT_ALGORITHMS, + ) + if isinstance(claims, dict): + return claims + return None + except Exception as e: + verbose_proxy_logger.debug( + "Failed to decode unverified JWT claims for routing: %s", e + ) + return None + def _rbac_role_from_role_mapping(self, token: dict) -> Optional[RBAC_ROLES]: """ Returns the RBAC role the token 'belongs' to based on role mappings. @@ -664,30 +705,11 @@ class JWTHandler: raise Exception(f"Failed to fetch OIDC UserInfo: {str(e)}") async def auth_jwt(self, token: str) -> dict: - # Supported algos: https://pyjwt.readthedocs.io/en/stable/algorithms.html - # "Warning: Make sure not to mix symmetric and asymmetric algorithms that interpret - # the key in different ways (e.g. HS* and RS*)." - algorithms = [ - "RS256", - "RS384", - "RS512", - "PS256", - "PS384", - "PS512", - "ES256", - "ES384", - "ES512", - "EdDSA", - ] - audience = os.getenv("JWT_AUDIENCE") decode_options = None if audience is None: decode_options = {"verify_aud": False} - import jwt - from jwt.api_jwk import PyJWK - header = jwt.get_unverified_header(token) verbose_proxy_logger.debug("header: %s", header) @@ -721,7 +743,7 @@ class JWTHandler: payload = jwt.decode( token, public_key_obj, # type: ignore - algorithms=algorithms, + algorithms=self.SUPPORTED_JWT_ALGORITHMS, options=decode_options, # type: ignore[arg-type] audience=audience, leeway=self.leeway, # allow testing of expired tokens @@ -749,7 +771,7 @@ class JWTHandler: payload = jwt.decode( token, key, - algorithms=algorithms, + algorithms=self.SUPPORTED_JWT_ALGORITHMS, audience=audience, options=decode_options, ) diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 53ae08aefb1..046c39a9101 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -11,7 +11,7 @@ import asyncio import re import secrets from datetime import datetime, timezone -from typing import List, Optional, Tuple, cast +from typing import Any, List, Optional, Tuple, cast import fastapi from fastapi import HTTPException, Request, WebSocket, status @@ -139,6 +139,58 @@ def _get_bearer_token_or_received_api_key(api_key: str) -> str: return api_key +def _routing_selector_matches_claim( + selector_value: Optional[Any], claim_value: Optional[Any] +) -> bool: + if selector_value is None: + return True + + selector_list = ( + [str(v) for v in selector_value] + if isinstance(selector_value, list) + else [str(selector_value)] + ) + + if isinstance(claim_value, list): + claim_list = [str(v) for v in claim_value] + return any(v in claim_list for v in selector_list) + + return str(claim_value) in selector_list if claim_value is not None else False + + +def _matches_routing_override( + token_claims: dict, override: "JWTRoutingOverride" +) -> bool: + return ( + _routing_selector_matches_claim(override.iss, token_claims.get("iss")) + and _routing_selector_matches_claim( + override.client_id, token_claims.get("client_id") + ) + and _routing_selector_matches_claim(override.aud, token_claims.get("aud")) + ) + + +def _should_route_jwt_to_oauth2_override(token: str, jwt_handler: JWTHandler) -> bool: + routing_overrides = jwt_handler.litellm_jwtauth.routing_overrides + if not routing_overrides: + return False + + token_claims = jwt_handler.get_unverified_claims(token=token) + if token_claims is None: + return False + + for override in routing_overrides: + if override.path == "oauth2" and _matches_routing_override( + token_claims=token_claims, override=override + ): + verbose_proxy_logger.debug( + "JWT routing override matched. Routing token to OAuth2 introspection." + ) + return True + + return False + + def _get_bearer_token( api_key: str, ): @@ -649,12 +701,20 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 # - JWT tokens (3 dot-separated parts) -> skip OAuth2, fall through to JWT handler # - Opaque tokens -> use OAuth2 handler # This allows JWT for users and OAuth2 for M2M on the same instance - is_jwt_token = ( + is_jwt = ( jwt_handler.is_jwt(token=api_key) if general_settings.get("enable_jwt_auth", False) is True else False ) - if not is_jwt_token: + # Routing uses unverified JWT claims only to choose auth path. + # Final authentication is enforced by the selected validator. + route_jwt_to_oauth2 = ( + is_jwt + and _should_route_jwt_to_oauth2_override( + token=api_key, jwt_handler=jwt_handler + ) + ) + if not is_jwt or route_jwt_to_oauth2: # return UserAPIKeyAuth object # helper to check if the api_key is a valid oauth2 token from litellm.proxy.proxy_server import premium_user @@ -688,7 +748,7 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 jwt_claims: Optional[dict] if ( jwt_handler.litellm_jwtauth.oidc_userinfo_enabled - and not jwt_handler.is_jwt(token=api_key) + and not is_jwt ): jwt_claims = await jwt_handler.get_oidc_userinfo(token=api_key) else: @@ -1193,49 +1253,13 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 raise Exception( "Key is blocked. Update via `/key/unblock` if you're an admin." ) - config = valid_token.config - - if config != {}: - model_list = config.get("model_list", []) - new_model_list = model_list - verbose_proxy_logger.debug( - f"\n new llm router model list {new_model_list}" - ) - elif ( - isinstance(valid_token.models, list) - and "all-team-models" in valid_token.models - ): - # Do not do any validation at this step - # the validation will occur when checking the team has access to this model - pass - else: - model = get_model_from_request(request_data, route) - fallback_models = cast( - Optional[List[ALL_FALLBACK_MODEL_VALUES]], - request_data.get("fallbacks", None), - ) - - if model is not None: - await can_key_call_model( - model=model, - llm_model_list=llm_model_list, - valid_token=valid_token, - llm_router=llm_router, - ) - - if fallback_models is not None: - for m in fallback_models: - await can_key_call_model( - model=m["model"] if isinstance(m, dict) else m, - llm_model_list=llm_model_list, - valid_token=valid_token, - llm_router=llm_router, - ) - await is_valid_fallback_model( - model=m["model"] if isinstance(m, dict) else m, - llm_router=llm_router, - user_model=None, - ) + await _enforce_key_and_fallback_model_access( + valid_token=valid_token, + request_data=request_data, + route=route, + llm_model_list=llm_model_list, + llm_router=llm_router, + ) # Check 2. If user_id for this token is in budget - done in common_checks() if valid_token.user_id is not None: @@ -1764,6 +1788,61 @@ async def _lookup_end_user_and_apply_budget( return valid_token, end_user_object +async def _enforce_key_and_fallback_model_access( + *, + valid_token: UserAPIKeyAuth, + request_data: dict, + route: str, + llm_model_list: Optional[list], + llm_router: Optional[Any], +) -> None: + """ + Key-level model allowlist and client fallbacks (same as standard auth). + Not included in common_checks — common_checks enforces team/user/project model access only. + """ + config = valid_token.config + + if config != {}: + model_list = config.get("model_list", []) + new_model_list = model_list + verbose_proxy_logger.debug( + f"\n new llm router model list {new_model_list}" + ) + elif ( + isinstance(valid_token.models, list) + and "all-team-models" in valid_token.models + ): + pass + else: + model = get_model_from_request(request_data, route) + fallback_models = cast( + Optional[List[ALL_FALLBACK_MODEL_VALUES]], + request_data.get("fallbacks", None), + ) + + if model is not None: + await can_key_call_model( + model=model, + llm_model_list=llm_model_list, + valid_token=valid_token, + llm_router=llm_router, + ) + + if fallback_models is not None: + for m in fallback_models: + await can_key_call_model( + model=m["model"] if isinstance(m, dict) else m, + llm_model_list=llm_model_list, + valid_token=valid_token, + llm_router=llm_router, + ) + await is_valid_fallback_model( + model=m["model"] if isinstance(m, dict) else m, + llm_router=llm_router, + user_model=None, + ) + + async def _run_post_custom_auth_checks( valid_token: UserAPIKeyAuth, request: Request, @@ -1773,6 +1852,7 @@ async def _run_post_custom_auth_checks( ) -> UserAPIKeyAuth: from litellm.proxy.proxy_server import ( general_settings, + llm_model_list, llm_router, model_max_budget_limiter, prisma_client, @@ -1816,6 +1896,15 @@ async def _run_post_custom_auth_checks( ), ) + if general_settings.get("custom_auth_run_common_checks", False): + await _enforce_key_and_fallback_model_access( + valid_token=valid_token, + request_data=request_data, + route=route, + llm_model_list=llm_model_list, + llm_router=llm_router, + ) + current_model = request_data.get("model", None) # 3. Check key-level model_max_budget diff --git a/tests/llm_translation/test_prompt_factory.py b/tests/llm_translation/test_prompt_factory.py index 64556c3f26d..27b0539aa4f 100644 --- a/tests/llm_translation/test_prompt_factory.py +++ b/tests/llm_translation/test_prompt_factory.py @@ -859,8 +859,8 @@ def test_ensure_alternating_roles_three_consecutive_assistants(): ] -def test_ensure_alternating_roles_does_not_split_tool_call_chain(): - """Tool-call chains [user, assistant(tc), tool, user] are preserved as-is.""" +def test_ensure_alternating_roles_inserts_assistant_continue_across_tool_chain(): + """[user, assistant(tc), tool, user] gets assistant_continue before the second user.""" messages = [ {"role": "user", "content": "Search for X"}, { @@ -899,15 +899,16 @@ def test_ensure_alternating_roles_does_not_split_tool_call_chain(): ], }, {"role": "tool", "tool_call_id": "c1", "content": "results"}, + {"role": "assistant", "content": "Please continue."}, {"role": "user", "content": "Thanks, now do Y"}, ] def test_ensure_alternating_roles_assistant_tool_call_then_assistant(): """ - Preserve old behavior for malformed adjacent assistant turns: - [assistant(tool_calls), assistant(no-tool-calls), user] should insert - user_continue between assistant messages. + Malformed [assistant(tc), assistant(no-tc), user]: + user_continue inserts break between adjacents, then assistant_continue + fills the counted-sequence gap. """ messages = [ { @@ -945,6 +946,7 @@ def test_ensure_alternating_roles_assistant_tool_call_then_assistant(): } ], }, + {"role": "assistant", "content": "Please continue."}, {"role": "user", "content": "Please continue."}, {"role": "assistant", "content": "Here's what I found."}, {"role": "user", "content": "Thanks"}, @@ -993,10 +995,184 @@ def test_ensure_alternating_roles_trailing_tool_call_assistant(): } ], }, + {"role": "assistant", "content": "Please continue."}, {"role": "user", "content": "Please continue."}, ] +def test_ensure_alternating_roles_multiple_tool_results(): + """[user, assistant(tc), tool, tool, user] — multiple tool results before next user.""" + messages = [ + {"role": "user", "content": "Search for X and Y"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "c1", + "type": "function", + "function": {"name": "search_x", "arguments": "{}"}, + }, + { + "id": "c2", + "type": "function", + "function": {"name": "search_y", "arguments": "{}"}, + }, + ], + }, + {"role": "tool", "tool_call_id": "c1", "content": "result X"}, + {"role": "tool", "tool_call_id": "c2", "content": "result Y"}, + {"role": "user", "content": "Thanks"}, + ] + + transformed_messages = get_completion_messages( + messages=messages, + assistant_continue_message=None, + user_continue_message=None, + ensure_alternating_roles=True, + ) + + assert transformed_messages == [ + {"role": "user", "content": "Search for X and Y"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "c1", + "type": "function", + "function": {"name": "search_x", "arguments": "{}"}, + }, + { + "id": "c2", + "type": "function", + "function": {"name": "search_y", "arguments": "{}"}, + }, + ], + }, + {"role": "tool", "tool_call_id": "c1", "content": "result X"}, + {"role": "tool", "tool_call_id": "c2", "content": "result Y"}, + {"role": "assistant", "content": "Please continue."}, + {"role": "user", "content": "Thanks"}, + ] + + +def test_ensure_alternating_roles_chained_tool_calls(): + """[user, assistant(tc), tool, assistant(tc), tool, user] — chained tool calls.""" + messages = [ + {"role": "user", "content": "Do multi-step task"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "c1", + "type": "function", + "function": {"name": "step1", "arguments": "{}"}, + }, + ], + }, + {"role": "tool", "tool_call_id": "c1", "content": "step1 done"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "c2", + "type": "function", + "function": {"name": "step2", "arguments": "{}"}, + }, + ], + }, + {"role": "tool", "tool_call_id": "c2", "content": "step2 done"}, + {"role": "user", "content": "What happened?"}, + ] + + transformed_messages = get_completion_messages( + messages=messages, + assistant_continue_message=None, + user_continue_message=None, + ensure_alternating_roles=True, + ) + + assert transformed_messages == [ + {"role": "user", "content": "Do multi-step task"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "c1", + "type": "function", + "function": {"name": "step1", "arguments": "{}"}, + }, + ], + }, + {"role": "tool", "tool_call_id": "c1", "content": "step1 done"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "c2", + "type": "function", + "function": {"name": "step2", "arguments": "{}"}, + }, + ], + }, + {"role": "tool", "tool_call_id": "c2", "content": "step2 done"}, + {"role": "assistant", "content": "Please continue."}, + {"role": "user", "content": "What happened?"}, + ] + + +def test_ensure_alternating_roles_system_prefix_with_tool_chain(): + """[system, user, assistant(tc), tool, user] — system prefix doesn't interfere.""" + messages = [ + {"role": "system", "content": "You are helpful."}, + {"role": "user", "content": "Search for X"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "c1", + "type": "function", + "function": {"name": "search", "arguments": "{}"}, + }, + ], + }, + {"role": "tool", "tool_call_id": "c1", "content": "results"}, + {"role": "user", "content": "Thanks"}, + ] + + transformed_messages = get_completion_messages( + messages=messages, + assistant_continue_message=None, + user_continue_message=None, + ensure_alternating_roles=True, + ) + + assert transformed_messages == [ + {"role": "system", "content": "You are helpful."}, + {"role": "user", "content": "Search for X"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "c1", + "type": "function", + "function": {"name": "search", "arguments": "{}"}, + }, + ], + }, + {"role": "tool", "tool_call_id": "c1", "content": "results"}, + {"role": "assistant", "content": "Please continue."}, + {"role": "user", "content": "Thanks"}, + ] + + def test_alternating_roles_e2e(): from litellm.llms.custom_httpx.http_handler import HTTPHandler import json diff --git a/tests/test_litellm/llms/bedrock/chat/agentcore/test_agentcore_transformation.py b/tests/test_litellm/llms/bedrock/chat/agentcore/test_agentcore_transformation.py index 3dede83032a..43182926f95 100644 --- a/tests/test_litellm/llms/bedrock/chat/agentcore/test_agentcore_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/agentcore/test_agentcore_transformation.py @@ -160,6 +160,67 @@ class TestAgentCoreJsonResponseParsing: assert parsed["content"] == "" assert parsed["final_message"] == response_json["result"] + def test_parse_json_a2a_jsonrpc_nested_message(self, config): + """Strategy 0: A2A JSON-RPC with result.message.parts[] format.""" + response_json = { + "jsonrpc": "2.0", + "id": "test_id", + "result": { + "message": { + "role": "agent", + "parts": [{"kind": "text", "text": "1 + 1 = 2"}], + "messageId": "123", + } + }, + } + parsed = config._parse_json_response(response_json) + assert parsed["content"] == "1 + 1 = 2" + assert parsed["usage"] is None + + def test_parse_json_a2a_jsonrpc_direct_parts(self, config): + """Strategy 0: A2A JSON-RPC with result.parts[] format (direct message).""" + response_json = { + "jsonrpc": "2.0", + "id": "test_id", + "result": { + "kind": "message", + "parts": [{"kind": "text", "text": "Direct response"}], + }, + } + parsed = config._parse_json_response(response_json) + assert parsed["content"] == "Direct response" + assert parsed["usage"] is None + + def test_parse_json_a2a_jsonrpc_multi_parts(self, config): + """Strategy 0: A2A JSON-RPC with multiple text parts concatenated.""" + response_json = { + "jsonrpc": "2.0", + "id": "test_id", + "result": { + "message": { + "role": "agent", + "parts": [ + {"kind": "text", "text": "First part"}, + {"kind": "text", "text": "Second part"}, + ], + } + }, + } + parsed = config._parse_json_response(response_json) + assert parsed["content"] == "First part Second part" + assert parsed["usage"] is None + + def test_parse_json_a2a_jsonrpc_empty_falls_through(self, config): + """Strategy 0: A2A JSON-RPC with empty result falls through to Strategy 3.""" + response_json = { + "jsonrpc": "2.0", + "id": "test_id", + "result": "plain text fallback", + } + parsed = config._parse_json_response(response_json) + assert parsed["content"] == "plain text fallback" + assert parsed["usage"] is None + class TestAgentCoreNonStreamingJsonFormats: """Tests for _get_parsed_response with different JSON formats (non-streaming path).""" diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py index 74c7f9bca5f..6e1b245b3de 100644 --- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py +++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py @@ -3,22 +3,30 @@ import json import os import sys from typing import Tuple -from unittest.mock import AsyncMock, MagicMock, patch +from unittest.mock import ANY, AsyncMock, MagicMock, patch sys.path.insert( 0, os.path.abspath("../../..") ) # Adds the parent directory to the system path -from unittest.mock import MagicMock - import pytest import litellm.proxy.proxy_server from litellm.caching.dual_cache import DualCache -from litellm.proxy._types import LiteLLM_JWTAuth, UserAPIKeyAuth +from litellm.proxy._types import ( + LiteLLM_JWTAuth, + ProxyErrorTypes, + ProxyException, + UserAPIKeyAuth, + JWTRoutingOverride, +) from litellm.proxy.auth.handle_jwt import JWTHandler from litellm.proxy.auth.route_checks import RouteChecks -from litellm.proxy.auth.user_api_key_auth import get_api_key, user_api_key_auth +from litellm.proxy.auth.user_api_key_auth import ( + _run_post_custom_auth_checks, + get_api_key, + user_api_key_auth, +) def test_get_api_key(): @@ -38,6 +46,86 @@ def test_get_api_key(): ) == (api_key, passed_in_key) +@pytest.mark.asyncio +async def test_custom_auth_does_not_enforce_key_model_access_by_default(): + valid_token = UserAPIKeyAuth(token="test_token", models=["gpt-4o-mini"]) + request_data = {"model": "gpt-4o"} + + with patch( + "litellm.proxy.auth.user_api_key_auth.can_key_call_model", new_callable=AsyncMock + ) as mock_can_key, patch( + "litellm.proxy.proxy_server.general_settings", + {}, + ): + await _run_post_custom_auth_checks( + valid_token=valid_token, + request=None, + request_data=request_data, + route="/v1/chat/completions", + parent_otel_span=None, + ) + mock_can_key.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_custom_auth_honors_key_level_model_access_restriction_allowed_with_opt_in(): + valid_token = UserAPIKeyAuth(token="test_token", models=["gpt-4o-mini"]) + request_data = {"model": "gpt-4o-mini"} + + with patch( + "litellm.proxy.auth.user_api_key_auth.can_key_call_model", new_callable=AsyncMock + ) as mock_can_key, patch( + "litellm.proxy.auth.user_api_key_auth.common_checks", new_callable=AsyncMock + ), patch( + "litellm.proxy.proxy_server.general_settings", + {"custom_auth_run_common_checks": True}, + ): + await _run_post_custom_auth_checks( + valid_token=valid_token, + request=None, + request_data=request_data, + route="/v1/chat/completions", + parent_otel_span=None, + ) + mock_can_key.assert_awaited_once_with( + model="gpt-4o-mini", + llm_model_list=ANY, + valid_token=valid_token, + llm_router=ANY, + ) + + +@pytest.mark.asyncio +async def test_custom_auth_honors_key_level_model_access_restriction_denied_with_opt_in(): + valid_token = UserAPIKeyAuth(token="test_token", models=["gpt-4o-mini"]) + request_data = {"model": "gpt-4o"} + + with patch( + "litellm.proxy.auth.user_api_key_auth.can_key_call_model", new_callable=AsyncMock + ) as mock_can_key, patch( + "litellm.proxy.auth.user_api_key_auth.common_checks", new_callable=AsyncMock + ), patch( + "litellm.proxy.proxy_server.general_settings", + {"custom_auth_run_common_checks": True}, + ): + mock_can_key.side_effect = ProxyException( + message="Key not allowed to access model", + type=ProxyErrorTypes.key_model_access_denied, + param="model", + code=401, + ) + with pytest.raises(ProxyException) as exc: + await _run_post_custom_auth_checks( + valid_token=valid_token, + request=None, + request_data=request_data, + route="/v1/chat/completions", + parent_otel_span=None, + ) + + assert exc.value.type == ProxyErrorTypes.key_model_access_denied + + @pytest.mark.parametrize( "custom_litellm_key_header, api_key, passed_in_key", [ @@ -689,6 +777,203 @@ class TestJWTOAuth2Coexistence: mock_jwt_auth.assert_called_once() assert result.user_id == "jwt-human-user" + @pytest.mark.asyncio + async def test_routing_override_routes_matching_jwt_to_oauth2(self): + """ + When routing_overrides match JWT claims, route JWT-shaped token to OAuth2. + """ + jwt_token = ( + "eyJhbGciOiJSUzI1NiJ9." + "eyJpc3MiOiJtYWNoaW5lLWlzc3Vlci5leGFtcGxlLmNvbSIsImNsaWVudF9pZCI6Ik1JRF9MSVRFTExNIn0." + "c2ln" + ) + general_settings = { + "enable_oauth2_auth": True, + "enable_jwt_auth": True, + } + mock_oauth2_response = UserAPIKeyAuth( + api_key=jwt_token, + user_id="machine-client-override", + ) + + mock_request = MagicMock() + mock_request.url.path = "/v1/chat/completions" + mock_request.headers = {"authorization": f"Bearer {jwt_token}"} + mock_request.query_params = {} + + with patch( + "litellm.proxy.proxy_server.general_settings", general_settings + ), patch("litellm.proxy.proxy_server.premium_user", True), patch( + "litellm.proxy.proxy_server.master_key", "sk-master" + ), patch( + "litellm.proxy.proxy_server.prisma_client", None + ), patch( + "litellm.proxy.auth.user_api_key_auth.Oauth2Handler.check_oauth2_token", + new_callable=AsyncMock, + return_value=mock_oauth2_response, + ) as mock_oauth2, patch( + "litellm.proxy.auth.user_api_key_auth.JWTAuthManager.auth_builder", + new_callable=AsyncMock, + ) as mock_jwt_auth: + litellm.proxy.proxy_server.jwt_handler.update_environment( + prisma_client=None, + user_api_key_cache=DualCache(), + litellm_jwtauth=LiteLLM_JWTAuth( + routing_overrides=[ + JWTRoutingOverride( + iss="machine-issuer.example.com", + client_id="MID_LITELLM", + path="oauth2", + ) + ] + ), + ) + + result = await user_api_key_auth( + request=mock_request, + api_key=f"Bearer {jwt_token}", + ) + + mock_oauth2.assert_called_once_with(token=jwt_token) + mock_jwt_auth.assert_not_called() + assert result.user_id == "machine-client-override" + + @pytest.mark.asyncio + async def test_routing_override_does_not_match_client_id_falls_back_to_jwt(self): + """ + If override ISS matches but client_id does not, continue default JWT flow. + """ + jwt_token = ( + "eyJhbGciOiJSUzI1NiJ9." + "eyJpc3MiOiJtYWNoaW5lLWlzc3Vlci5leGFtcGxlLmNvbSIsImNsaWVudF9pZCI6IlVTRVJfUE9SVEFMIn0." + "c2ln" + ) + general_settings = { + "enable_oauth2_auth": True, + "enable_jwt_auth": True, + } + mock_jwt_result = { + "is_proxy_admin": True, + "team_object": None, + "user_object": None, + "end_user_object": None, + "org_object": None, + "token": jwt_token, + "team_id": "jwt-team", + "user_id": "jwt-user-no-override", + "end_user_id": None, + "org_id": None, + "team_membership": None, + "jwt_claims": {"sub": "user1"}, + } + + mock_request = MagicMock() + mock_request.url.path = "/v1/chat/completions" + mock_request.headers = {"authorization": f"Bearer {jwt_token}"} + mock_request.query_params = {} + + with patch( + "litellm.proxy.proxy_server.general_settings", general_settings + ), patch("litellm.proxy.proxy_server.premium_user", True), patch( + "litellm.proxy.proxy_server.master_key", "sk-master" + ), patch( + "litellm.proxy.proxy_server.prisma_client", None + ), patch( + "litellm.proxy.auth.user_api_key_auth.Oauth2Handler.check_oauth2_token", + new_callable=AsyncMock, + ) as mock_oauth2, patch( + "litellm.proxy.auth.user_api_key_auth.JWTAuthManager.auth_builder", + new_callable=AsyncMock, + return_value=mock_jwt_result, + ) as mock_jwt_auth: + litellm.proxy.proxy_server.jwt_handler.update_environment( + prisma_client=None, + user_api_key_cache=DualCache(), + litellm_jwtauth=LiteLLM_JWTAuth( + routing_overrides=[ + JWTRoutingOverride( + iss="machine-issuer.example.com", + client_id="MID_LITELLM", + path="oauth2", + ) + ] + ), + ) + + result = await user_api_key_auth( + request=mock_request, + api_key=f"Bearer {jwt_token}", + ) + + mock_oauth2.assert_not_called() + mock_jwt_auth.assert_called_once() + assert result.user_id == "jwt-user-no-override" + + @pytest.mark.asyncio + async def test_routing_override_matches_aud_claim_list_and_list_selectors(self): + """ + Match routing override when selectors are lists and token aud claim is a list. + """ + jwt_token = ( + "eyJhbGciOiJSUzI1NiJ9." + "eyJpc3MiOiJtYWNoaW5lLWlzc3Vlci5leGFtcGxlLmNvbSIsImNsaWVudF9pZCI6Ik1JRF9MSVRFTExNIiwiYXVkIjpbImFwaTovL2xpdGVsbG0iLCJhcGk6Ly9vdGhlciJdfQ." + "c2ln" + ) + general_settings = { + "enable_oauth2_auth": True, + "enable_jwt_auth": True, + } + mock_oauth2_response = UserAPIKeyAuth( + api_key=jwt_token, + user_id="machine-client-aud-list", + ) + + mock_request = MagicMock() + mock_request.url.path = "/v1/chat/completions" + mock_request.headers = {"authorization": f"Bearer {jwt_token}"} + mock_request.query_params = {} + + with patch( + "litellm.proxy.proxy_server.general_settings", general_settings + ), patch("litellm.proxy.proxy_server.premium_user", True), patch( + "litellm.proxy.proxy_server.master_key", "sk-master" + ), patch( + "litellm.proxy.proxy_server.prisma_client", None + ), patch( + "litellm.proxy.auth.user_api_key_auth.Oauth2Handler.check_oauth2_token", + new_callable=AsyncMock, + return_value=mock_oauth2_response, + ) as mock_oauth2, patch( + "litellm.proxy.auth.user_api_key_auth.JWTAuthManager.auth_builder", + new_callable=AsyncMock, + ) as mock_jwt_auth: + litellm.proxy.proxy_server.jwt_handler.update_environment( + prisma_client=None, + user_api_key_cache=DualCache(), + litellm_jwtauth=LiteLLM_JWTAuth( + routing_overrides=[ + JWTRoutingOverride( + iss=[ + "machine-issuer.example.com", + "other-issuer.example.com", + ], + client_id=["MID_LITELLM", "MID_BACKUP"], + aud=["api://litellm", "api://fallback"], + path="oauth2", + ) + ] + ), + ) + + result = await user_api_key_auth( + request=mock_request, + api_key=f"Bearer {jwt_token}", + ) + + mock_oauth2.assert_called_once_with(token=jwt_token) + mock_jwt_auth.assert_not_called() + assert result.user_id == "machine-client-aud-list" + @pytest.mark.asyncio async def test_only_oauth2_enabled_handles_all_tokens(self): """ From fa629c307c31d7b9793b9b2137a67ca71190ce8a Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 2 Apr 2026 17:29:47 -0700 Subject: [PATCH 10/25] [Fix] Dockerfile.non_root: install node-gyp for npm ci The wolfi-base npm@11.12.1 package does not bundle node-gyp, causing `npm ci` to fail with `Cannot find module 'node-gyp/bin/node-gyp.js'` when building the Admin UI in the non-root Docker image. Install node-gyp@12.2.0 globally and symlink it into npm's internal node_modules where @npmcli/run-script expects to find it. Co-Authored-By: Claude Opus 4.6 (1M context) --- docker/Dockerfile.non_root | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index 1cf6c0dfd15..bf1af9ed756 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -46,7 +46,10 @@ ENV LITELLM_NON_ROOT=true # package-lock.json with pinned versions + integrity hashes. RUN mkdir -p /var/lib/litellm/ui && \ mv /app/.npmrc /app/.npmrc.bak && \ - npm install -g npm@11.12.1 && npm cache clean --force && \ + npm install -g npm@11.12.1 && \ + npm install -g node-gyp@12.2.0 && \ + ln -sf /usr/local/lib/node_modules/node-gyp /usr/lib/node_modules/npm/node_modules/node-gyp && \ + npm cache clean --force && \ cd /app/ui/litellm-dashboard && \ if [ -f "/app/enterprise/enterprise_ui/enterprise_colors.json" ]; then \ cp /app/enterprise/enterprise_ui/enterprise_colors.json ./ui_colors.json; \ From d4813a2a0f01c0bc4f48959d8d90d3e54d66ffa9 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 2 Apr 2026 17:31:16 -0700 Subject: [PATCH 11/25] adding poetry lock --- poetry.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index 4798b6122d5..dff1b8781d5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3598,15 +3598,15 @@ files = [ [[package]] name = "litellm-proxy-extras" -version = "0.4.62" +version = "0.4.63" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." optional = true python-versions = "!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*,>=3.8" groups = ["main"] markers = "extra == \"proxy\"" files = [ - {file = "litellm_proxy_extras-0.4.62-py3-none-any.whl", hash = "sha256:cf91c1a83d94000b8997ee29d9e8d505d1ad80c26f111803bef5365174c97de0"}, - {file = "litellm_proxy_extras-0.4.62.tar.gz", hash = "sha256:0d87db1cda9851717e5294f2fa2c7ae1f34d7476d99e24e6462702e11a2cdd88"}, + {file = "litellm_proxy_extras-0.4.63-py3-none-any.whl", hash = "sha256:46ec50083832b6b5ead86e53003657e1a53dc27bd95cbdbaee9c1343726e3acb"}, + {file = "litellm_proxy_extras-0.4.63.tar.gz", hash = "sha256:7161b27c3b38a840c13bb113b733196efafe2bb4d2ba22c9bd6e359c4b753aa2"}, ] [[package]] @@ -8309,4 +8309,4 @@ utils = ["numpydoc"] [metadata] lock-version = "2.1" python-versions = ">=3.9,<4.0" -content-hash = "f507b7c5060793a3325171c5245376057fc0f61a3a9443e9acd88d23a939a14a" +content-hash = "9a2476d5f234f3ce45f399a77fb9e86bd0025e27e2bb905b0ecac7848c4a758c" From ee3e848deddd275cca1ad0a0abb20b01ab8b1135 Mon Sep 17 00:00:00 2001 From: Joe Reyna Date: Thu, 2 Apr 2026 21:13:33 -0700 Subject: [PATCH 12/25] allow hashed token_id in /key/update endpoint (#24969) --- .../management_endpoints/key_management_endpoints.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index c140de98191..e105d9b8586 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -942,9 +942,9 @@ async def _check_team_key_limits( where={"team_id": team_table.team_id}, ) # Exclude the key being updated to avoid double-counting its limits. - # key.token is the SHA-256 hash stored in DB; data.key is the raw key string. + # data.key may be a raw key (sk-...) or a pre-hashed token_id. if isinstance(data, UpdateKeyRequest): - hashed_key = hash_token(data.key) + hashed_key = _hash_token_if_needed(data.key) keys = [key for key in keys if key.token != hashed_key] check_team_key_model_specific_limits( keys=keys, @@ -1101,9 +1101,9 @@ async def _check_org_key_limits( where={"organization_id": org_table.organization_id}, ) # Exclude the key being updated to avoid double-counting its limits. - # key.token is the SHA-256 hash stored in DB; data.key is the raw key string. + # data.key may be a raw key (sk-...) or a pre-hashed token_id. if isinstance(data, UpdateKeyRequest): - hashed_key = hash_token(data.key) + hashed_key = _hash_token_if_needed(data.key) keys = [key for key in keys if key.token != hashed_key] check_org_key_model_specific_limits( keys=keys, @@ -2157,7 +2157,7 @@ async def update_key_fn( # Delete - key from cache, since it's been updated! # key updated - a new model could have been added to this key. it should not block requests after this is done await _delete_cache_key_object( - hashed_token=hash_token(key), + hashed_token=_hash_token_if_needed(key), user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, ) From 1e5b79d887ca01c3ac477db837e9d830697d12b4 Mon Sep 17 00:00:00 2001 From: ishaan-berri <155045088+ishaan-berri@users.noreply.github.com> Date: Thu, 2 Apr 2026 21:13:56 -0700 Subject: [PATCH 13/25] =?UTF-8?q?[Bedrock]=20Fix=20Anthropic=20file=5Fid?= =?UTF-8?q?=20support=20-=20async=20path=20+=20document=20URL=E2=86=92base?= =?UTF-8?q?64=20+=20beta=20header=20filtering=20(#25047)=20(#25050)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Ishaan Jaffer --- litellm/llms/bedrock/chat/invoke_handler.py | 115 +++++++++++ .../anthropic_claude3_transformation.py | 188 ++++++++++++++++-- .../anthropic_claude3_transformation.py | 13 +- 3 files changed, 291 insertions(+), 25 deletions(-) diff --git a/litellm/llms/bedrock/chat/invoke_handler.py b/litellm/llms/bedrock/chat/invoke_handler.py index 1077731779d..67bba28e4c5 100644 --- a/litellm/llms/bedrock/chat/invoke_handler.py +++ b/litellm/llms/bedrock/chat/invoke_handler.py @@ -855,6 +855,32 @@ class BedrockLLM(BaseAWSLLM): endpoint_url = f"{endpoint_url}/model/{modelId}/invoke" proxy_endpoint_url = f"{proxy_endpoint_url}/model/{modelId}/invoke" + if acompletion and provider == "anthropic" and self.is_claude_messages_api_model( + model + ): + if isinstance(client, HTTPHandler): + client = None + return self._async_anthropic_messages_completion( + model=model, + messages=messages, + endpoint_url=endpoint_url, + proxy_endpoint_url=proxy_endpoint_url, + credentials=credentials, + aws_region_name=aws_region_name, + model_response=model_response, + print_verbose=print_verbose, + encoding=encoding, + logging_obj=logging_obj, + optional_params=optional_params, + stream=stream, + litellm_params=litellm_params, + logger_fn=logger_fn, + extra_headers=extra_headers, + timeout=timeout, + client=client, + stream_chunk_size=stream_chunk_size, + ) # type: ignore[return-value] + prompt, chat_history = self.convert_messages_to_prompt( model, messages, provider, custom_prompt_dict ) @@ -1148,6 +1174,95 @@ class BedrockLLM(BaseAWSLLM): encoding=encoding, ) + async def _async_anthropic_messages_completion( + self, + model: str, + messages: list, + endpoint_url: str, + proxy_endpoint_url: str, + credentials, + aws_region_name: str, + model_response: ModelResponse, + print_verbose: Callable, + encoding, + logging_obj: Logging, + optional_params: dict, + stream, + litellm_params=None, + logger_fn=None, + extra_headers: Optional[dict] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + client: Optional[AsyncHTTPHandler] = None, + stream_chunk_size: int = 1024, + ) -> Union[ModelResponse, CustomStreamWrapper]: + transformed_request = await litellm.AmazonAnthropicClaudeConfig().async_transform_request( + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params or {}, + headers=extra_headers or {}, + ) + data = json.dumps(transformed_request) + + headers = {"Content-Type": "application/json"} + if extra_headers is not None: + headers = {"Content-Type": "application/json", **extra_headers} + prepped = self.get_request_headers( + credentials=credentials, + aws_region_name=aws_region_name, + extra_headers=extra_headers, + endpoint_url=endpoint_url, + data=data, + headers=headers, + ) + + logging_obj.pre_call( + input=messages, + api_key="", + additional_args={ + "complete_input_dict": data, + "api_base": proxy_endpoint_url, + "headers": prepped.headers, + }, + ) + + if stream is True: + return await self.async_streaming( + model=model, + messages=messages, + data=data, + api_base=proxy_endpoint_url, + model_response=model_response, + print_verbose=print_verbose, + encoding=encoding, + logging_obj=logging_obj, + optional_params=optional_params, + stream=True, + litellm_params=litellm_params, + logger_fn=logger_fn, + headers=prepped.headers, + timeout=timeout, + client=client, + stream_chunk_size=stream_chunk_size, + ) + return await self.async_completion( + model=model, + messages=messages, + data=data, + api_base=proxy_endpoint_url, + model_response=model_response, + print_verbose=print_verbose, + encoding=encoding, + logging_obj=logging_obj, + optional_params=optional_params, + stream=stream, # type: ignore + litellm_params=litellm_params, + logger_fn=logger_fn, + headers=prepped.headers, + timeout=timeout, + client=client, + ) + async def async_completion( self, model: str, diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py index 7936b6ea644..2b28473ad3e 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py @@ -2,6 +2,14 @@ from typing import TYPE_CHECKING, Any, List, Optional import httpx +from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers +from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_anthropic_image_obj, +) +from litellm.litellm_core_utils.prompt_templates.image_handling import ( + async_convert_url_to_base64, + convert_url_to_base64, +) from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( AmazonInvokeConfig, @@ -85,8 +93,62 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): litellm_params: dict, headers: dict, ) -> dict: - # Filter out AWS authentication parameters before passing to Anthropic transformation - # AWS params should only be used for signing requests, not included in request body + _anthropic_request = self._build_bedrock_anthropic_request_base( + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers=headers, + ) + + self._convert_document_url_sources_to_base64(_anthropic_request) + beta_list = self._compute_bedrock_invoke_beta_headers( + model=model, + messages=messages, + optional_params=optional_params, + headers=headers, + ) + if beta_list: + _anthropic_request["anthropic_beta"] = beta_list + + return _anthropic_request + + async def async_transform_request( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + _anthropic_request = self._build_bedrock_anthropic_request_base( + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers=headers, + ) + + await self._async_convert_document_url_sources_to_base64(_anthropic_request) + beta_list = self._compute_bedrock_invoke_beta_headers( + model=model, + messages=messages, + optional_params=optional_params, + headers=headers, + ) + if beta_list: + _anthropic_request["anthropic_beta"] = beta_list + + return _anthropic_request + + def _build_bedrock_anthropic_request_base( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: filtered_params = { k: v for k, v in optional_params.items() @@ -94,7 +156,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): } filtered_params = self._normalize_bedrock_tool_search_tools(filtered_params) - _anthropic_request = AnthropicConfig.transform_request( + anthropic_request = AnthropicConfig.transform_request( self, model=model, messages=messages, @@ -103,28 +165,31 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): headers=headers, ) - _anthropic_request.pop("model", None) - _anthropic_request.pop("stream", None) - # Bedrock Invoke doesn't support output_format parameter - _anthropic_request.pop("output_format", None) - # Bedrock Invoke doesn't support output_config parameter - # Fixes: https://github.com/BerriAI/litellm/issues/22797 - _anthropic_request.pop("output_config", None) - if "anthropic_version" not in _anthropic_request: - _anthropic_request["anthropic_version"] = self.anthropic_version + anthropic_request.pop("model", None) + anthropic_request.pop("stream", None) + anthropic_request.pop("output_format", None) + anthropic_request.pop("output_config", None) + if "anthropic_version" not in anthropic_request: + anthropic_request["anthropic_version"] = self.anthropic_version # Remove `custom` field from tools (Bedrock doesn't support it) - # Claude Code sends `custom: {defer_loading: true}` on tool definitions, - # which causes Bedrock to reject the request with "Extra inputs are not permitted" - # Ref: https://github.com/BerriAI/litellm/issues/22847 - remove_custom_field_from_tools(_anthropic_request) + remove_custom_field_from_tools(anthropic_request) + return anthropic_request + def _compute_bedrock_invoke_beta_headers( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + headers: dict, + ) -> List[str]: tools = optional_params.get("tools") tool_search_used = self.is_tool_search_used(tools) programmatic_tool_calling_used = self.is_programmatic_tool_calling_used(tools) input_examples_used = self.is_input_examples_used(tools) - beta_set = set(get_anthropic_beta_from_headers(headers)) + user_beta_set = set(get_anthropic_beta_from_headers(headers)) + beta_set = set(user_beta_set) auto_betas = self.get_anthropic_beta_list( model=model, optional_params=optional_params, @@ -142,12 +207,91 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): if "opus-4" in model.lower() or "opus_4" in model.lower(): beta_set.add("tool-search-tool-2025-10-19") - # Filter out beta headers that Bedrock Invoke doesn't support - # Uses centralized configuration from anthropic_beta_headers_config.json - beta_list = list(beta_set) - _anthropic_request["anthropic_beta"] = beta_list + auto_beta_list = filter_and_transform_beta_headers( + beta_headers=list(beta_set - user_beta_set), + provider="bedrock", + ) + return sorted(user_beta_set.union(set(auto_beta_list))) - return _anthropic_request + def _convert_document_url_sources_to_base64(self, anthropic_request: dict) -> None: + """ + Bedrock Invoke does not accept document URL sources. Convert to base64 payloads. + """ + messages = anthropic_request.get("messages") + if not isinstance(messages, list): + return + + for message in messages: + if not isinstance(message, dict): + continue + content = message.get("content") + if not isinstance(content, list): + continue + + for block in content: + if not isinstance(block, dict) or block.get("type") != "document": + continue + source = block.get("source") + if not isinstance(source, dict) or source.get("type") != "url": + continue + source_url = source.get("url") + if not isinstance(source_url, str): + continue + + inferred_format: Optional[str] = None + if source_url.lower().endswith(".pdf"): + inferred_format = "application/pdf" + base64_url = convert_url_to_base64(url=source_url) + image_chunk = convert_to_anthropic_image_obj( + openai_image_url=base64_url, + format=inferred_format, + ) + block["source"] = { + "type": "base64", + "media_type": image_chunk["media_type"], + "data": image_chunk["data"], + } + + async def _async_convert_document_url_sources_to_base64( + self, anthropic_request: dict + ) -> None: + """ + Async version of document URL conversion for async completion paths. + """ + messages = anthropic_request.get("messages") + if not isinstance(messages, list): + return + + for message in messages: + if not isinstance(message, dict): + continue + content = message.get("content") + if not isinstance(content, list): + continue + + for block in content: + if not isinstance(block, dict) or block.get("type") != "document": + continue + source = block.get("source") + if not isinstance(source, dict) or source.get("type") != "url": + continue + source_url = source.get("url") + if not isinstance(source_url, str): + continue + + inferred_format: Optional[str] = None + if source_url.lower().endswith(".pdf"): + inferred_format = "application/pdf" + base64_url = await async_convert_url_to_base64(url=source_url) + image_chunk = convert_to_anthropic_image_obj( + openai_image_url=base64_url, + format=inferred_format, + ) + block["source"] = { + "type": "base64", + "media_type": image_chunk["media_type"], + "data": image_chunk["data"], + } def _normalize_bedrock_tool_search_tools(self, optional_params: dict) -> dict: """ diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index a890cf2a04d..784c6a50cfd 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -12,6 +12,7 @@ from typing import ( import httpx +from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers from litellm.llms.anthropic.common_utils import AnthropicModelInfo from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( AnthropicMessagesConfig, @@ -436,7 +437,8 @@ class AmazonAnthropicClaudeMessagesConfig( ) input_examples_used = anthropic_model_info.is_input_examples_used(tools) - beta_set = set(get_anthropic_beta_from_headers(headers)) + user_beta_set = set(get_anthropic_beta_from_headers(headers)) + beta_set = set(user_beta_set) auto_betas = anthropic_model_info.get_anthropic_beta_list( model=model, optional_params=anthropic_messages_optional_request_params, @@ -460,8 +462,13 @@ class AmazonAnthropicClaudeMessagesConfig( if "tool-search-tool-2025-10-19" in beta_set: beta_set.add("tool-examples-2025-10-29") - if beta_set: - anthropic_messages_request["anthropic_beta"] = list(beta_set) + filtered_auto_betas = filter_and_transform_beta_headers( + beta_headers=list(beta_set - user_beta_set), + provider="bedrock", + ) + filtered_betas = sorted(user_beta_set.union(set(filtered_auto_betas))) + if filtered_betas: + anthropic_messages_request["anthropic_beta"] = filtered_betas return anthropic_messages_request From 52a596d2a4402cef5990de848a6b354497a34eb0 Mon Sep 17 00:00:00 2001 From: Marty Sullivan Date: Fri, 3 Apr 2026 00:26:04 -0400 Subject: [PATCH 14/25] Bedrock Model Updates 2026-03-26 (#24645) * add new bedrock models & remove duplicate vertexai entry * adding non-regional entry for minimax-2.5 --- ...odel_prices_and_context_window_backup.json | 224 ++++++++++++++++++ model_prices_and_context_window.json | 224 ++++++++++++++++++ 2 files changed, 448 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 51b7b56a285..4072aab6544 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -6672,6 +6672,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/ap-northeast-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/ap-northeast-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 7.3e-07, "litellm_provider": "bedrock", @@ -6781,6 +6795,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/ap-south-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/ap-south-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 7.1e-07, "litellm_provider": "bedrock", @@ -6819,6 +6847,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/ap-southeast-2/minimax.minimax-m2.5": { + "input_cost_per_token": 3.09e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.236e-06 + }, "bedrock/ap-southeast-3/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, "litellm_provider": "bedrock", @@ -6845,6 +6887,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/ap-southeast-3/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/ap-southeast-3/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, "litellm_provider": "bedrock", @@ -6916,6 +6972,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/eu-north-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/eu-north-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, "litellm_provider": "bedrock", @@ -7030,6 +7100,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/eu-central-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/eu-central-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -7074,6 +7158,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/eu-west-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/eu-west-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -7118,6 +7216,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/eu-west-2/minimax.minimax-m2.5": { + "input_cost_per_token": 4.7e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.86e-06 + }, "bedrock/eu-west-2/qwen.qwen3-coder-next": { "input_cost_per_token": 7.8e-07, "litellm_provider": "bedrock", @@ -7174,6 +7286,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/eu-south-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/eu-south-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -7249,6 +7375,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/sa-east-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/sa-east-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 7.3e-07, "litellm_provider": "bedrock", @@ -7449,6 +7589,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/us-east-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.2e-06 + }, "bedrock/us-east-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -7513,6 +7667,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/us-east-2/minimax.minimax-m2.5": { + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.2e-06 + }, "bedrock/us-east-2/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -7995,6 +8163,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/us-west-2/minimax.minimax-m2.5": { + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.2e-06 + }, "bedrock/us-west-2/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -21292,6 +21474,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "minimax.minimax-m2.5": { + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "minimax/speech-02-hd": { "input_cost_per_character": 0.0001, "litellm_provider": "minimax", @@ -23111,6 +23307,20 @@ "source": "https://aws.amazon.com/bedrock/pricing/", "supports_native_structured_output": true }, + "nvidia.nemotron-super-3-120b": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 256000, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 6.5e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, "o1": { "cache_read_input_token_cost": 7.5e-06, "input_cost_per_token": 1.5e-05, @@ -32726,6 +32936,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "zai.glm-5": { + "input_cost_per_token": 1e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3.2e-06, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, "zai/glm-5": { "cache_creation_input_token_cost": 0, "cache_read_input_token_cost": 2e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 51b7b56a285..4072aab6544 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -6672,6 +6672,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/ap-northeast-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/ap-northeast-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 7.3e-07, "litellm_provider": "bedrock", @@ -6781,6 +6795,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/ap-south-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/ap-south-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 7.1e-07, "litellm_provider": "bedrock", @@ -6819,6 +6847,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/ap-southeast-2/minimax.minimax-m2.5": { + "input_cost_per_token": 3.09e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.236e-06 + }, "bedrock/ap-southeast-3/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, "litellm_provider": "bedrock", @@ -6845,6 +6887,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/ap-southeast-3/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/ap-southeast-3/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, "litellm_provider": "bedrock", @@ -6916,6 +6972,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/eu-north-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/eu-north-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, "litellm_provider": "bedrock", @@ -7030,6 +7100,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/eu-central-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/eu-central-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -7074,6 +7158,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/eu-west-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/eu-west-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -7118,6 +7216,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/eu-west-2/minimax.minimax-m2.5": { + "input_cost_per_token": 4.7e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.86e-06 + }, "bedrock/eu-west-2/qwen.qwen3-coder-next": { "input_cost_per_token": 7.8e-07, "litellm_provider": "bedrock", @@ -7174,6 +7286,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/eu-south-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/eu-south-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -7249,6 +7375,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/sa-east-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3.6e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.44e-06 + }, "bedrock/sa-east-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 7.3e-07, "litellm_provider": "bedrock", @@ -7449,6 +7589,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/us-east-1/minimax.minimax-m2.5": { + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.2e-06 + }, "bedrock/us-east-1/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -7513,6 +7667,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/us-east-2/minimax.minimax-m2.5": { + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.2e-06 + }, "bedrock/us-east-2/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -7995,6 +8163,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/us-west-2/minimax.minimax-m2.5": { + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "output_cost_per_token": 1.2e-06 + }, "bedrock/us-west-2/moonshotai.kimi-k2-thinking": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", @@ -21292,6 +21474,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "minimax.minimax-m2.5": { + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "minimax/speech-02-hd": { "input_cost_per_character": 0.0001, "litellm_provider": "minimax", @@ -23111,6 +23307,20 @@ "source": "https://aws.amazon.com/bedrock/pricing/", "supports_native_structured_output": true }, + "nvidia.nemotron-super-3-120b": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 256000, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 6.5e-07, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, "o1": { "cache_read_input_token_cost": 7.5e-06, "input_cost_per_token": 1.5e-05, @@ -32726,6 +32936,20 @@ "supports_tool_choice": true, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "zai.glm-5": { + "input_cost_per_token": 1e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3.2e-06, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, "zai/glm-5": { "cache_creation_input_token_cost": 0, "cache_read_input_token_cost": 2e-07, From a6dfd02610bf3e4a5b94ab1a96d0be37b09d4203 Mon Sep 17 00:00:00 2001 From: michelligabriele Date: Fri, 3 Apr 2026 06:28:52 +0200 Subject: [PATCH 15/25] fix(guardrails): return HTTP 400 instead of 500 for Model Armor streaming blocks (#24693) When Model Armor blocks a streaming response, it correctly raises HTTPException(status_code=400) but create_response() catches it with a bare except Exception and hardcodes a 500 response, discarding the original status code. Fix create_response() to preserve status_code from HTTPException instead of hardcoding 500. Also update Model Armor's streaming hook to yield an SSE error event instead of raising (matching the Prisma Airs pattern), and fix make_model_armor_request() to return 400 for upstream API failures instead of passing through the upstream status code. --- litellm/proxy/common_request_processing.py | 11 +- .../model_armor/model_armor.py | 23 +++- .../guardrail_hooks/test_model_armor.py | 125 +++++++++++++++++- .../proxy/test_common_request_processing.py | 29 +++- 4 files changed, 179 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 386b9d27723..09200c96841 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -221,16 +221,21 @@ async def create_response( f"Error consuming first chunk from generator: {e}" ) - # Fallback to a generic error stream + # Preserve status code from HTTPException (e.g., guardrail blocks) + error_status = getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR) + error_detail = getattr(e, "detail", "Error processing stream start") + if not isinstance(error_detail, str): + error_detail = str(error_detail) + async def error_gen_message() -> AsyncGenerator[str, None]: - yield f"data: {json.dumps({'error': {'message': 'Error processing stream start', 'code': status.HTTP_500_INTERNAL_SERVER_ERROR}})}\n\n" + yield f"data: {json.dumps({'error': {'message': error_detail, 'code': error_status}})}\n\n" yield "data: [DONE]\n\n" return StreamingResponse( error_gen_message(), media_type=media_type, headers=headers, - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + status_code=error_status, ) async def combined_generator() -> AsyncGenerator[str, None]: diff --git a/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py b/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py index 2d3f048f81b..3ee8e6a27bd 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py +++ b/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py @@ -14,6 +14,8 @@ from fastapi import HTTPException if TYPE_CHECKING: from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel +import json + import litellm from litellm._logging import verbose_proxy_logger from litellm.caching import DualCache @@ -203,8 +205,8 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): response.text, ) raise HTTPException( - status_code=response.status_code, - detail=f"Model Armor API error: {response.text}", + status_code=400, + detail=f"Model Armor API error (upstream {response.status_code}): {response.text}", ) json_response = response.json() @@ -746,8 +748,21 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): yield chunk return - except HTTPException: - raise + except HTTPException as e: + # Yield error as SSE event so create_response() detects it and + # returns a proper JSON error response with the correct status code. + # (Raising from a generator hits create_response's generic except → 500.) + detail = ( + e.detail if isinstance(e.detail, dict) else {"message": str(e.detail)} + ) + error_value = detail.get("error", detail) + if isinstance(error_value, dict): + error_obj = dict(error_value) + else: + error_obj = {"message": str(error_value)} + error_obj["code"] = e.status_code + yield f"data: {json.dumps({'error': error_obj})}\n\n" + return except Exception as e: verbose_proxy_logger.error( "Model Armor streaming error: %s", str(e), exc_info=True diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py index 8080491f662..97ca55073df 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py @@ -380,8 +380,9 @@ async def test_model_armor_api_error_handling(): call_type="completion" ) - assert exc_info.value.status_code == 500 + assert exc_info.value.status_code == 400 assert "Model Armor API error" in str(exc_info.value.detail) + assert "upstream 500" in str(exc_info.value.detail) @pytest.mark.asyncio @@ -485,6 +486,128 @@ async def test_model_armor_streaming_response(): assert len(result_chunks) > 0 mock_post.assert_called() +@pytest.mark.asyncio +async def test_model_armor_streaming_block_yields_sse_error(): + """Test that streaming content block yields SSE error event instead of raising HTTPException.""" + mock_user_api_key_dict = UserAPIKeyAuth() + + guardrail = ModelArmorGuardrail( + template_id="test-template", + project_id="test-project", + location="us-central1", + guardrail_name="model-armor-test", + ) + + # Mock Model Armor API response that triggers a block (SDP MATCH_FOUND) + mock_response = AsyncMock() + mock_response.status_code = 200 + mock_response.json = AsyncMock( + return_value={ + "sanitizationResult": { + "filterMatchState": "MATCH_FOUND", + "filterResults": { + "sdp": { + "sdpFilterResult": { + "inspectResult": { + "matchState": "MATCH_FOUND", + "findings": [ + { + "infoType": "PASSWORD", + "likelihood": "VERY_LIKELY", + } + ], + } + } + } + }, + } + } + ) + + guardrail._ensure_access_token_async = AsyncMock( + return_value=("test-token", "test-project") + ) + + with patch.object( + guardrail.async_handler, "post", AsyncMock(return_value=mock_response) + ): + + async def mock_stream(): + chunks = [ + litellm.ModelResponseStream( + choices=[ + litellm.types.utils.StreamingChoices( + delta=litellm.types.utils.Delta( + content="My password is " + ) + ) + ] + ), + litellm.ModelResponseStream( + choices=[ + litellm.types.utils.StreamingChoices( + delta=litellm.types.utils.Delta(content="hunter2") + ) + ] + ), + ] + for chunk in chunks: + yield chunk + + request_data = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "What's your password?"}], + "metadata": {"guardrails": ["model-armor-test"]}, + } + + result_chunks = [] + async for chunk in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=mock_user_api_key_dict, + response=mock_stream(), + request_data=request_data, + ): + result_chunks.append(chunk) + + # Should yield exactly one SSE error event (not raise HTTPException) + assert len(result_chunks) == 1 + error_data = json.loads(result_chunks[0].removeprefix("data: ")) + assert "error" in error_data + assert error_data["error"]["code"] == 400 + + +@pytest.mark.asyncio +async def test_model_armor_api_failure_returns_400(): + """Test that Model Armor API failures raise HTTP 400, not the upstream status code.""" + guardrail = ModelArmorGuardrail( + template_id="test-template", + project_id="test-project", + location="us-central1", + guardrail_name="model-armor-test", + ) + + # Mock a 500 response from the Model Armor GCP API + mock_response = AsyncMock() + mock_response.status_code = 500 + mock_response.text = "Internal Server Error" + + guardrail._ensure_access_token_async = AsyncMock( + return_value=("test-token", "test-project") + ) + + with patch.object( + guardrail.async_handler, "post", AsyncMock(return_value=mock_response) + ): + with pytest.raises(HTTPException) as exc_info: + await guardrail.make_model_armor_request( + content="test content", + source="user_prompt", + ) + + # Should be 400, NOT the upstream 500 + assert exc_info.value.status_code == 400 + assert "upstream 500" in str(exc_info.value.detail) + + def test_model_armor_ui_friendly_name(): """Test the UI-friendly name of the Model Armor guardrail""" from litellm.types.proxy.guardrails.guardrail_hooks.model_armor import ( diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 45ec4b726a8..781b559651f 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -4,7 +4,7 @@ from typing import AsyncGenerator from unittest.mock import AsyncMock, MagicMock, patch import pytest -from fastapi import Request, status +from fastapi import HTTPException, Request, status from fastapi.responses import JSONResponse, StreamingResponse import litellm @@ -899,6 +899,33 @@ class TestCommonRequestProcessingHelpers: assert content[0] == f"data: {json.dumps(expected_error_data)}\n\n" assert content[1] == "data: [DONE]\n\n" + async def test_create_streaming_response_generator_raises_http_exception( + self, + ): + """ + Test that when a generator raises HTTPException, the response preserves + the original status code instead of hardcoding 500. + """ + mock_gen = AsyncMock() + mock_gen.__anext__.side_effect = HTTPException( + status_code=400, detail="Content blocked by guardrail" + ) + + response = await create_response(mock_gen, "text/event-stream", {}) + assert response.status_code == 400 + content = await self.consume_stream(response) + import json + + expected_error_data = { + "error": { + "message": "Content blocked by guardrail", + "code": 400, + } + } + assert len(content) == 2 + assert content[0] == f"data: {json.dumps(expected_error_data)}\n\n" + assert content[1] == "data: [DONE]\n\n" + async def test_create_streaming_response_first_chunk_error_string_code(self): """ Test that when the first chunk contains a string error code, a JSON error response is returned From 50f4fdea3e4a0d2666c7100e1ff5ebb70e0d0c7a Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 2 Apr 2026 21:43:42 -0700 Subject: [PATCH 16/25] =?UTF-8?q?bump:=20version=201.83.0=20=E2=86=92=201.?= =?UTF-8?q?83.1=20(#25054)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ed9b0b8c57f..90b88165e86 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.83.0" +version = "1.83.1" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -181,7 +181,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.83.0" +version = "1.83.1" version_files = [ "pyproject.toml:^version" ] From d4a3a5e530422d6ccd4dff82637ca275aa4e507d Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Fri, 3 Apr 2026 00:51:21 -0400 Subject: [PATCH 17/25] fix gpt-5.4 pricing (#24748) --- model_prices_and_context_window.json | 29 +++++++--------------------- 1 file changed, 7 insertions(+), 22 deletions(-) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 4072aab6544..71918ed049b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -18847,13 +18847,11 @@ "cache_read_input_token_cost_above_272k_tokens": 5e-07, "cache_read_input_token_cost_flex": 1.3e-07, "cache_read_input_token_cost_priority": 5e-07, - "cache_read_input_token_cost_above_272k_tokens_priority": 1e-06, "input_cost_per_token": 2.5e-06, "input_cost_per_token_above_272k_tokens": 5e-06, "input_cost_per_token_flex": 1.25e-06, "input_cost_per_token_batches": 1.25e-06, "input_cost_per_token_priority": 5e-06, - "input_cost_per_token_above_272k_tokens_priority": 1e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -18863,8 +18861,7 @@ "output_cost_per_token_above_272k_tokens": 2.25e-05, "output_cost_per_token_flex": 7.5e-06, "output_cost_per_token_batches": 7.5e-06, - "output_cost_per_token_priority": 2.25e-05, - "output_cost_per_token_above_272k_tokens_priority": 3.375e-05, + "output_cost_per_token_priority": 3e-05, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -18897,13 +18894,11 @@ "cache_read_input_token_cost_above_272k_tokens": 5e-07, "cache_read_input_token_cost_flex": 1.3e-07, "cache_read_input_token_cost_priority": 5e-07, - "cache_read_input_token_cost_above_272k_tokens_priority": 1e-06, "input_cost_per_token": 2.5e-06, "input_cost_per_token_above_272k_tokens": 5e-06, "input_cost_per_token_flex": 1.25e-06, "input_cost_per_token_batches": 1.25e-06, "input_cost_per_token_priority": 5e-06, - "input_cost_per_token_above_272k_tokens_priority": 1e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -18913,8 +18908,7 @@ "output_cost_per_token_above_272k_tokens": 2.25e-05, "output_cost_per_token_flex": 7.5e-06, "output_cost_per_token_batches": 7.5e-06, - "output_cost_per_token_priority": 2.25e-05, - "output_cost_per_token_above_272k_tokens_priority": 3.375e-05, + "output_cost_per_token_priority": 3e-05, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -18942,14 +18936,10 @@ "gpt-5.4-pro": { "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, - "cache_read_input_token_cost_priority": 6e-06, - "cache_read_input_token_cost_above_272k_tokens_priority": 1.2e-05, "input_cost_per_token": 3e-05, "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_flex": 1.5e-05, "input_cost_per_token_batches": 1.5e-05, - "input_cost_per_token_priority": 6e-05, - "input_cost_per_token_above_272k_tokens_priority": 0.00012, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -18959,8 +18949,6 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, - "output_cost_per_token_priority": 0.00027, - "output_cost_per_token_above_272k_tokens_priority": 0.000405, "supported_endpoints": [ "/v1/responses", "/v1/batch" @@ -18991,14 +18979,10 @@ "gpt-5.4-pro-2026-03-05": { "cache_read_input_token_cost": 3e-06, "cache_read_input_token_cost_above_272k_tokens": 6e-06, - "cache_read_input_token_cost_priority": 6e-06, - "cache_read_input_token_cost_above_272k_tokens_priority": 1.2e-05, "input_cost_per_token": 3e-05, "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_flex": 1.5e-05, "input_cost_per_token_batches": 1.5e-05, - "input_cost_per_token_priority": 6e-05, - "input_cost_per_token_above_272k_tokens_priority": 0.00012, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -19008,8 +18992,6 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, - "output_cost_per_token_priority": 0.00027, - "output_cost_per_token_above_272k_tokens_priority": 0.000405, "supported_endpoints": [ "/v1/responses", "/v1/batch" @@ -19039,11 +19021,13 @@ }, "gpt-5.4-mini": { "cache_read_input_token_cost": 7.5e-08, - "cache_read_input_token_cost_flex": 1e-08, - "cache_read_input_token_cost_batches": 3.8e-08, + "cache_read_input_token_cost_flex": 3.75e-08, + "cache_read_input_token_cost_batches": 3.75e-08, + "cache_read_input_token_cost_priority": 1.5e-07, "input_cost_per_token": 7.5e-07, "input_cost_per_token_flex": 3.75e-07, "input_cost_per_token_batches": 3.75e-07, + "input_cost_per_token_priority": 1.5e-06, "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, @@ -19052,6 +19036,7 @@ "output_cost_per_token": 4.5e-06, "output_cost_per_token_flex": 2.25e-06, "output_cost_per_token_batches": 2.25e-06, + "output_cost_per_token_priority": 9e-06, "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", From 6cc56f58fd273541d7c73d6809c9f401e95da6be Mon Sep 17 00:00:00 2001 From: joereyna Date: Fri, 3 Apr 2026 11:36:02 -0700 Subject: [PATCH 18/25] Fix broken codeql-action SHA in scorecard workflow --- .github/workflows/scorecard.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 7cd12bb219c..3a00064c3bd 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -42,6 +42,6 @@ jobs: retention-days: 5 - name: Upload to code scanning - uses: github/codeql-action/upload-sarif@c10b806170c8ee63ea24152429041b5624f0baf5 # v4.35.1 + uses: github/codeql-action/upload-sarif@c10b8064de6f491fea524254123dbe5e09572f13 # v4.35.1 with: sarif_file: results.sarif From fc885af99441f5e1e344942895063a1188c1c5e1 Mon Sep 17 00:00:00 2001 From: ishaan-berri <155045088+ishaan-berri@users.noreply.github.com> Date: Fri, 3 Apr 2026 13:06:14 -0700 Subject: [PATCH 19/25] docs(blog): add security hardening April 2026 post (#25101) (#25102) --- .../security_hardening_april_2026/index.md | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 docs/my-website/blog/security_hardening_april_2026/index.md diff --git a/docs/my-website/blog/security_hardening_april_2026/index.md b/docs/my-website/blog/security_hardening_april_2026/index.md new file mode 100644 index 00000000000..1af4caa3e1f --- /dev/null +++ b/docs/my-website/blog/security_hardening_april_2026/index.md @@ -0,0 +1,66 @@ +--- +slug: security-hardening-april-2026 +title: "Security Update: Vulnerability Disclosures and Ongoing Hardening" +date: 2026-04-03T12:00:00 +authors: + - krrish + - ishaan-alt +description: "Disclosure of security vulnerabilities fixed in LiteLLM v1.83.0, and the launch of our bug bounty program." +tags: [security] +hide_table_of_contents: false +--- + +After the [supply chain incident](https://docs.litellm.ai/blog/security-update-march-2026) in March, we brought in [Veria Labs](https://verialabs.com/) to audit the LiteLLM proxy and fixed a number of vulnerability reports from independent researchers. All issues below are fixed in v1.83.0. If you are affected, particularly if you have JWT auth enabled, we recommend upgrading. + +We've also launched a [bug bounty program](#bug-bounty-program) and Veria Labs is continuing to audit the proxy. More fixes will ship in upcoming versions. + +The two high-severity issues ([CVE-2026-35029](https://github.com/BerriAI/litellm/security/advisories/GHSA-53mr-6c8q-9789) and [GHSA-69x8-hrgq-fjj8](https://github.com/BerriAI/litellm/security/advisories/GHSA-69x8-hrgq-fjj8)) **both require the attacker to already have a valid API key for the proxy**. These are not exploitable by unauthenticated users. + +The critical-severity issue ([CVE-2026-35030](https://github.com/BerriAI/litellm/security/advisories/GHSA-jjhc-v7c2-5hh6)) is an authentication bypass, but only affects deployments with `enable_jwt_auth` explicitly enabled, which is off by default. **The default LiteLLM configuration is not affected, and no LiteLLM Cloud customers had this feature enabled.** + +{/* truncate */} + +## Vulnerabilities + +### CVE-2026-35030: Authentication bypass via OIDC cache collision (Critical) + +Found by Veria Labs. + +When `enable_jwt_auth` is enabled, LiteLLM cached OIDC userinfo using `token[:20]` as the cache key. JWTs from the same signing algorithm share the same header prefix, so an attacker could forge a token that hits another user's cache entry and inherit their session. We fixed this by keying the cache on `sha256(token)` instead. + +**Most deployments are not affected.** This requires `enable_jwt_auth: true`, which is off by default. If you can't upgrade, disable JWT auth as a workaround. + +Full advisory: [GHSA-jjhc-v7c2-5hh6](https://github.com/BerriAI/litellm/security/advisories/GHSA-jjhc-v7c2-5hh6) + +### CVE-2026-35029: Privilege escalation via `/config/update` (High) + +Found by Lakera. + +`/config/update` didn't check the caller's role. Any authenticated user could modify the proxy's runtime configuration, which could lead to arbitrary file read, admin account takeover, or remote code execution. We now require the `proxy_admin` role on this endpoint. + +Full advisory: [GHSA-53mr-6c8q-9789](https://github.com/BerriAI/litellm/security/advisories/GHSA-53mr-6c8q-9789) + +### Password hash exposure and pass-the-hash login (High) + +Weak hashing originally reported by GitHub user [hamzayevmaqsud](https://github.com/hamzayevmaqsud) ([#15484](https://github.com/BerriAI/litellm/issues/15484)). The full chain was identified by Luca Vandenweghe and Maarten De Rammelaere of [iO Digital](https://www.iodigital.com/). + +Passwords were stored as unsalted SHA-256 hashes, and in some cases plaintext. Several API endpoints returned the hash to any authenticated user, and `/v2/login` accepted the raw hash as a credential without re-hashing it, so a stolen hash was as good as the password itself. We've moved to scrypt with random salts and stripped hashes from all API responses. + +Full advisory: [GHSA-69x8-hrgq-fjj8](https://github.com/BerriAI/litellm/security/advisories/GHSA-69x8-hrgq-fjj8) + +## Bug bounty program + +After the supply chain incident and these disclosures it was clear we needed more external eyes on the project. We've set up a bug bounty program so researchers have a way to report issues. + +Bounties are currently paid for P0 (supply chain) and P1 (unauthenticated proxy access) vulnerabilities: + +| Severity | Bounty | Example | +|----------|--------|---------| +| Critical | $1,500 – $3,000 | Supply chain compromise | +| High | $500 – $1,500 | Unauthenticated access to protected data | + +We plan on expanding the program further in the coming months. More info about the bug bounty program is available [here](https://github.com/BerriAI/litellm/security). + +## What's next + +Veria Labs is continuing to work with us on a broader audit of the proxy. Security advisories sent through Github will be responded to within five business days. We'll publish advisories as issues are confirmed and fixed. From c6aa3ea45226db5912170722a524bd5c89657c5b Mon Sep 17 00:00:00 2001 From: ishaan-berri <155045088+ishaan-berri@users.noreply.github.com> Date: Fri, 3 Apr 2026 14:57:44 -0700 Subject: [PATCH 20/25] Litellm ishaan april1 try2 (#25110) * Litellm ishaan april1 (#25103) * fix(proxy): enforce upperbound key params on key/update and add custom_key_update hook The /key/update endpoint did not enforce upperbound_key_generate_params, allowing users to bypass configured limits (tpm_limit, rpm_limit, max_budget, duration, budget_duration) by updating an existing key instead of generating a new one. Extract the upperbound enforcement logic from _common_key_generation_helper() into a standalone _enforce_upperbound_key_params() function and call it from both the generate and update paths. For updates, None values are skipped (not filled with defaults) since they mean "don't change this field". Also adds a custom_key_update config option and user_custom_key_update global, mirroring the existing custom_key_generate pattern, so custom key validation logic can fire during key updates as well. * fix(proxy): invoke custom_key_update hook in bulk update path The user_custom_key_update hook was only called in update_key_fn (single key update) but not in _process_single_key_update (bulk update path), allowing custom validation to be bypassed via the /key/update/bulk endpoint. Mirror the hook invocation in both paths. * fix(proxy): pass UpdateKeyRequest to hook in bulk path, not BulkUpdateKeyRequestItem Move the custom_key_update hook invocation to after UpdateKeyRequest is constructed so the hook receives the same type in both single and bulk update paths. Previously the bulk path passed BulkUpdateKeyRequestItem (5 fields only), which would cause AttributeError for hooks accessing fields like tpm_limit or models. * fix(bedrock): promote cache usage to message_delta for Claude Code (#24850) Ensure Bedrock/Anthropic-compatible streaming exposes cache usage where Claude Code reads it by promoting message_stop usage onto message_delta and preserving usage fields in fake-streamed message_delta events. Made-with: Cursor * fix(search): Support self-hosted Firecrawl response format in search transform (#24866) The `transform_search_response` method only handled Firecrawl Cloud (v2) response format where `data` is a dict with `web`/`news` keys. Self-hosted Firecrawl (v1) returns `data` as a flat list of result objects, causing an `AttributeError: 'list' object has no attribute 'get'`. Detect the response format by checking if `data` is a list (self-hosted) or dict (cloud) and handle both cases. Cloud format: {"data": {"web": [...], "news": [...]}} Self-hosted: {"success": true, "data": [{"url": "...", "title": "...", ...}]} Co-authored-by: Synergy * feat: add environment and user tracking to prompt management (#24855) * feat: add environment and user tracking to prompt management - Add environment (development/staging/production) and created_by columns to LiteLLM_PromptTable - Update unique constraint to [prompt_id, version, environment] - All CRUD endpoints support environment filtering and user tracking - Redesigned prompt detail page with environment tabs and version history - UI: environment filter on list page, environment selector in editor - 8 new tests for environment and user tracking Co-Authored-By: Claude Opus 4.6 (1M context) * fix: Black formatting and add environments to PromptInfoResponse TypeScript type Co-Authored-By: Claude Opus 4.6 (1M context) * fix: address Greptile review findings - P1: delete_prompt scopes in-memory cleanup to environment when provided - P2: dotprompt_content parsed directly regardless of environment flag - P2: use distinct for environments query - P2: fix double-fetch on initial mount in prompt_info.tsx - fix: remove unsupported select kwarg from find_many Co-Authored-By: Claude Opus 4.6 (1M context) * fix: address remaining Greptile review comments - Remove unused useCallback import (index.tsx) - Remove unused ENV_COLORS variable (prompt_info.tsx) - P1: in-memory fallback in get_prompt_versions now respects environment filter - P1: reset selectedEnv when promptId changes to avoid stale state - Cyclic imports are pre-existing pattern, not introduced by this PR Co-Authored-By: Claude Opus 4.6 (1M context) * fix: scope patch_prompt to environment using primary key - Add environment query param to patch_prompt endpoint - Look up target row by composite key (prompt_id + version + environment) - Update by primary key (id) to target exactly one row - Fixes Greptile finding: patch with multiple environments no longer ambiguous Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Claude Opus 4.6 (1M context) * fix: use actual start_time for failed request spend logs (#24906) async_post_call_failure_hook set both start_time and end_time to datetime.now(), making all failed requests show duration=0. Use the actual start_time from litellm_logging_obj instead, so spend logs reflect the real request duration on timeout and other failures. Fixes #24888 * feat(bedrock): add nova canvas image edit support (#24869) * feat(bedrock): add nova canvas image edit support * fix(bedrock): support PathLike inputs for nova image edit * chore: sync schema.prisma copies from root * fix(mypy): correct type-ignore code for delta_usage arg-type * fix(mypy): cast status_code to str, suppress intentional str yield * fix(lint): extract _create_content_block_chunks to fix PLR0915 * fix(lint): extract helpers to fix PLR0915 in prompt endpoints --------- Co-authored-by: michelligabriele Co-authored-by: Sameer Kankute Co-authored-by: redhelix Co-authored-by: Synergy Co-authored-by: Talha Anwar <37379131+talhaanwarch@users.noreply.github.com> Co-authored-by: Claude Opus 4.6 (1M context) Co-authored-by: madhu19991 Co-authored-by: Srikanth @adobe Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * fix(test): update model armor streaming test to handle string or int error code --------- Co-authored-by: michelligabriele Co-authored-by: Sameer Kankute Co-authored-by: redhelix Co-authored-by: Synergy Co-authored-by: Talha Anwar <37379131+talhaanwarch@users.noreply.github.com> Co-authored-by: Claude Opus 4.6 (1M context) Co-authored-by: madhu19991 Co-authored-by: Srikanth @adobe Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .../docs/providers/bedrock_image_gen.md | 24 +- .../migration.sql | 12 + .../litellm_proxy_extras/schema.prisma | 16 +- .../messages/fake_stream_iterator.py | 254 ++++--- ...n_nova_canvas_image_edit_transformation.py | 515 ++++++++++++++ litellm/llms/bedrock/image_edit/handler.py | 14 +- .../anthropic_claude3_transformation.py | 80 ++- .../llms/firecrawl/search/transformation.py | 80 ++- ...odel_prices_and_context_window_backup.json | 10 +- .../model_armor/model_armor.py | 4 +- .../proxy/hooks/proxy_track_cost_callback.py | 10 +- .../key_management_endpoints.py | 137 ++-- litellm/proxy/prompts/prompt_endpoints.py | 447 ++++++++---- litellm/proxy/proxy_server.py | 14 +- litellm/proxy/schema.prisma | 5 +- litellm/types/prompts/init_prompts.py | 6 + litellm/utils.py | 6 +- model_prices_and_context_window.json | 10 +- schema.prisma | 5 +- .../test_amazon_nova_canvas_image_edit.py | 657 ++++++++++++++++++ .../test_anthropic_claude3_transformation.py | 106 ++- .../guardrail_hooks/test_model_armor.py | 2 +- .../hooks/test_proxy_track_cost_callback.py | 64 ++ .../test_key_management_endpoints.py | 144 ++++ .../proxy/prompts/test_prompt_environment.py | 253 +++++++ tests/test_litellm/test_utils.py | 1 + .../src/components/networking.tsx | 28 +- .../src/components/prompts.tsx | 25 +- .../prompt_editor_view/PromptEditorHeader.tsx | 17 +- .../prompts/prompt_editor_view/index.tsx | 21 +- .../prompts/prompt_editor_view/types.ts | 1 + .../prompts/prompt_editor_view/utils.ts | 1 + .../src/components/prompts/prompt_info.tsx | 317 ++++++--- .../src/components/prompts/prompt_table.tsx | 30 + 34 files changed, 2816 insertions(+), 500 deletions(-) create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260331000000_add_prompt_environment_and_created_by/migration.sql create mode 100644 litellm/llms/bedrock/image_edit/amazon_nova_canvas_image_edit_transformation.py create mode 100644 tests/test_litellm/llms/bedrock/image_edit/test_amazon_nova_canvas_image_edit.py create mode 100644 tests/test_litellm/proxy/prompts/test_prompt_environment.py diff --git a/docs/my-website/docs/providers/bedrock_image_gen.md b/docs/my-website/docs/providers/bedrock_image_gen.md index 799c6d46437..e6e8429817d 100644 --- a/docs/my-website/docs/providers/bedrock_image_gen.md +++ b/docs/my-website/docs/providers/bedrock_image_gen.md @@ -111,6 +111,29 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/images/generations' \ +## Amazon Nova Canvas - Image Edit + +Use OpenAI-compatible `image_edit()` with Bedrock Nova Canvas (`amazon.nova-canvas-v1:0`). Requests use the same `InvokeModel` API as generation; LiteLLM maps inputs to [Nova Canvas task types](https://docs.aws.amazon.com/nova/latest/userguide/image-gen-access.html): + +| Scenario | `taskType` sent to Bedrock | +|----------|----------------------------| +| Image + prompt (no mask) | `IMAGE_VARIATION` | +| Image + prompt + mask | `INPAINTING` (`inPaintingParams.image`, `maskImage` or `maskPrompt`) | +| `taskType: OUTPAINTING` + `mask` or `maskPrompt` | `OUTPAINTING` (Bedrock requires one; LiteLLM raises a clear error if both are missing) | +| `taskType: BACKGROUND_REMOVAL` | `BACKGROUND_REMOVAL` | + +```python +from litellm import image_edit + +response = image_edit( + image=open("photo.png", "rb"), + prompt="Add soft sunset lighting", + model="bedrock/amazon.nova-canvas-v1:0", +) +``` + +For **`BACKGROUND_REMOVAL`**, the AWS request must not include `imageGenerationConfig`; LiteLLM omits it for that task even if you pass `size`, `n`, `seed`, etc. Additional Nova Canvas inference IDs for image edit should set **`supports_nova_canvas_image_edit`: true** in `model_prices_and_context_window.json` (see `amazon.nova-canvas-v1:0`). + ## Using Inference Profiles with Image Generation For AWS Bedrock Application Inference Profiles with image generation, use the `model_id` parameter to specify the inference profile ARN: @@ -147,4 +170,3 @@ model_list: ## Authentication All standard Bedrock authentication methods are supported for image generation. See [Bedrock Authentication](./bedrock#boto3---authentication) for details. - diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260331000000_add_prompt_environment_and_created_by/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260331000000_add_prompt_environment_and_created_by/migration.sql new file mode 100644 index 00000000000..74357814d8d --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260331000000_add_prompt_environment_and_created_by/migration.sql @@ -0,0 +1,12 @@ +-- AlterTable +ALTER TABLE "LiteLLM_PromptTable" ADD COLUMN "environment" TEXT NOT NULL DEFAULT 'development'; +ALTER TABLE "LiteLLM_PromptTable" ADD COLUMN "created_by" TEXT; + +-- DropIndex (old unique constraint) +DROP INDEX IF EXISTS "LiteLLM_PromptTable_prompt_id_version_key"; + +-- CreateIndex (new unique constraint) +CREATE UNIQUE INDEX "LiteLLM_PromptTable_prompt_id_version_environment_key" ON "LiteLLM_PromptTable"("prompt_id", "version", "environment"); + +-- CreateIndex (new composite index) +CREATE INDEX "LiteLLM_PromptTable_prompt_id_environment_idx" ON "LiteLLM_PromptTable"("prompt_id", "environment"); diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index d8d6015ce87..5d804d75b6a 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -321,11 +321,12 @@ model LiteLLM_MCPServerTable { byok_description String[] @default([]) byok_api_key_help_url String? source_url String? - approval_status String? @default("active") - submitted_by String? - submitted_at DateTime? - reviewed_at DateTime? - review_notes String? + // BYOM submission lifecycle + approval_status String? @default("active") + submitted_by String? + submitted_at DateTime? + reviewed_at DateTime? + review_notes String? @@index([approval_status]) } @@ -1001,12 +1002,15 @@ model LiteLLM_PromptTable { id String @id @default(uuid()) prompt_id String version Int @default(1) + environment String @default("development") + created_by String? litellm_params Json prompt_info Json? created_at DateTime @default(now()) updated_at DateTime @updatedAt - @@unique([prompt_id, version]) + @@unique([prompt_id, version, environment]) + @@index([prompt_id, environment]) @@index([prompt_id]) } diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/fake_stream_iterator.py b/litellm/llms/anthropic/experimental_pass_through/messages/fake_stream_iterator.py index 80afea78504..7fc9b00f2c7 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/fake_stream_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/fake_stream_iterator.py @@ -38,6 +38,102 @@ class FakeAnthropicMessagesStreamIterator: self.chunks = self._create_streaming_chunks() self.current_index = 0 + def _create_content_block_chunks( + self, block_dict: Dict[str, Any], index: int + ) -> List[bytes]: + """Build SSE chunks for a single content block.""" + chunks = [] + block_type = block_dict.get("type") + + if block_type == "text": + content_block_start = { + "type": "content_block_start", + "index": index, + "content_block": {"type": "text", "text": ""}, + } + chunks.append( + f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode() + ) + text = block_dict.get("text", "") + content_block_delta = { + "type": "content_block_delta", + "index": index, + "delta": {"type": "text_delta", "text": text}, + } + chunks.append( + f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode() + ) + + elif block_type == "thinking": + content_block_start = { + "type": "content_block_start", + "index": index, + "content_block": {"type": "thinking", "thinking": "", "signature": ""}, + } + chunks.append( + f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode() + ) + thinking_text = block_dict.get("thinking", "") + if thinking_text: + content_block_delta = { + "type": "content_block_delta", + "index": index, + "delta": {"type": "thinking_delta", "thinking": thinking_text}, + } + chunks.append( + f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode() + ) + signature = block_dict.get("signature", "") + if signature: + signature_delta = { + "type": "content_block_delta", + "index": index, + "delta": {"type": "signature_delta", "signature": signature}, + } + chunks.append( + f"event: content_block_delta\ndata: {json.dumps(signature_delta)}\n\n".encode() + ) + + elif block_type == "redacted_thinking": + content_block_start = { + "type": "content_block_start", + "index": index, + "content_block": {"type": "redacted_thinking"}, + } + chunks.append( + f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode() + ) + + elif block_type == "tool_use": + content_block_start = { + "type": "content_block_start", + "index": index, + "content_block": { + "type": "tool_use", + "id": block_dict.get("id"), + "name": block_dict.get("name"), + "input": {}, + }, + } + chunks.append( + f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode() + ) + input_data = block_dict.get("input", {}) + content_block_delta = { + "type": "content_block_delta", + "index": index, + "delta": {"type": "input_json_delta", "partial_json": json.dumps(input_data)}, + } + chunks.append( + f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode() + ) + + content_block_stop = {"type": "content_block_stop", "index": index} + chunks.append( + f"event: content_block_stop\ndata: {json.dumps(content_block_stop)}\n\n".encode() + ) + return chunks + def _create_streaming_chunks(self) -> List[bytes]: """Convert the non-streaming response to streaming chunks""" chunks = [] @@ -69,152 +165,34 @@ class FakeAnthropicMessagesStreamIterator: # 2-4. For each content block, send start/delta/stop events content_blocks = response_dict.get("content", []) - if content_blocks: - for index, block in enumerate(content_blocks): - # Cast block to dict for easier access - block_dict = cast(Dict[str, Any], block) - block_type = block_dict.get("type") - - if block_type == "text": - # content_block_start - content_block_start = { - "type": "content_block_start", - "index": index, - "content_block": {"type": "text", "text": ""}, - } - chunks.append( - f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode() - ) - - # content_block_delta (send full text as one delta for simplicity) - text = block_dict.get("text", "") - content_block_delta = { - "type": "content_block_delta", - "index": index, - "delta": {"type": "text_delta", "text": text}, - } - chunks.append( - f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode() - ) - - # content_block_stop - content_block_stop = {"type": "content_block_stop", "index": index} - chunks.append( - f"event: content_block_stop\ndata: {json.dumps(content_block_stop)}\n\n".encode() - ) - - elif block_type == "thinking": - # content_block_start for thinking - content_block_start = { - "type": "content_block_start", - "index": index, - "content_block": { - "type": "thinking", - "thinking": "", - "signature": "", - }, - } - chunks.append( - f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode() - ) - - # content_block_delta for thinking text - thinking_text = block_dict.get("thinking", "") - if thinking_text: - content_block_delta = { - "type": "content_block_delta", - "index": index, - "delta": { - "type": "thinking_delta", - "thinking": thinking_text, - }, - } - chunks.append( - f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode() - ) - - # content_block_delta for signature (if present) - signature = block_dict.get("signature", "") - if signature: - signature_delta = { - "type": "content_block_delta", - "index": index, - "delta": { - "type": "signature_delta", - "signature": signature, - }, - } - chunks.append( - f"event: content_block_delta\ndata: {json.dumps(signature_delta)}\n\n".encode() - ) - - # content_block_stop - content_block_stop = {"type": "content_block_stop", "index": index} - chunks.append( - f"event: content_block_stop\ndata: {json.dumps(content_block_stop)}\n\n".encode() - ) - - elif block_type == "redacted_thinking": - # content_block_start for redacted_thinking - content_block_start = { - "type": "content_block_start", - "index": index, - "content_block": {"type": "redacted_thinking"}, - } - chunks.append( - f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode() - ) - - # content_block_stop (no delta for redacted thinking) - content_block_stop = {"type": "content_block_stop", "index": index} - chunks.append( - f"event: content_block_stop\ndata: {json.dumps(content_block_stop)}\n\n".encode() - ) - - elif block_type == "tool_use": - # content_block_start - content_block_start = { - "type": "content_block_start", - "index": index, - "content_block": { - "type": "tool_use", - "id": block_dict.get("id"), - "name": block_dict.get("name"), - "input": {}, - }, - } - chunks.append( - f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode() - ) - - # content_block_delta (send input as JSON delta) - input_data = block_dict.get("input", {}) - content_block_delta = { - "type": "content_block_delta", - "index": index, - "delta": { - "type": "input_json_delta", - "partial_json": json.dumps(input_data), - }, - } - chunks.append( - f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode() - ) - - # content_block_stop - content_block_stop = {"type": "content_block_stop", "index": index} - chunks.append( - f"event: content_block_stop\ndata: {json.dumps(content_block_stop)}\n\n".encode() - ) + for index, block in enumerate(content_blocks): + block_dict = cast(Dict[str, Any], block) + chunks.extend(self._create_content_block_chunks(block_dict, index)) # 5. message_delta event (with final usage and stop_reason) + # Include cache usage fields so clients that only read message_delta + # (like Claude Code's SDK) see the full input token breakdown. + delta_usage: Dict[str, Any] = { + "output_tokens": usage.get("output_tokens", 0) if usage else 0, + } + if usage: + if usage.get("input_tokens") is not None: + delta_usage["input_tokens"] = usage["input_tokens"] + if usage.get("cache_creation_input_tokens") is not None: + delta_usage["cache_creation_input_tokens"] = usage[ + "cache_creation_input_tokens" + ] + if usage.get("cache_read_input_tokens") is not None: + delta_usage["cache_read_input_tokens"] = usage[ + "cache_read_input_tokens" + ] message_delta = { "type": "message_delta", "delta": { "stop_reason": response_dict.get("stop_reason"), "stop_sequence": response_dict.get("stop_sequence"), }, - "usage": {"output_tokens": usage.get("output_tokens", 0) if usage else 0}, + "usage": delta_usage, } chunks.append( f"event: message_delta\ndata: {json.dumps(message_delta)}\n\n".encode() diff --git a/litellm/llms/bedrock/image_edit/amazon_nova_canvas_image_edit_transformation.py b/litellm/llms/bedrock/image_edit/amazon_nova_canvas_image_edit_transformation.py new file mode 100644 index 00000000000..f806cd2a81a --- /dev/null +++ b/litellm/llms/bedrock/image_edit/amazon_nova_canvas_image_edit_transformation.py @@ -0,0 +1,515 @@ +""" +Amazon Nova Canvas image edit on Bedrock (InvokeModel). + +Maps OpenAI-style image edit (image + prompt, optional mask) to Nova Canvas task types: +- With mask: INPAINTING (inPaintingParams per AWS docs) +- Without mask: IMAGE_VARIATION (imageVariationParams) + +Refs: +- https://docs.aws.amazon.com/nova/latest/userguide/image-gen-access.html +- https://docs.aws.amazon.com/nova/latest/userguide/image-gen-req-resp-structure.html +""" + +from __future__ import annotations + +import base64 +import os +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple + +import httpx + +from litellm._logging import verbose_logger +from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig +from litellm.types.images.main import ImageEditOptionalRequestParams +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import FileTypes, ImageObject, ImageResponse +from litellm.utils import ( + _get_model_cost_key, + _get_potential_model_names, + get_model_info, +) + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +def _nova_canvas_task_body( + *, + image_b64: str, + mask_b64: Optional[str], + text: str, + negative_text: Optional[str], + similarity_strength: Optional[float], + task_type: Optional[str], + mask_prompt: Optional[str], + out_painting_mode: Optional[str], +) -> Dict[str, Any]: + """Build InvokeModel body task section (without imageGenerationConfig).""" + if task_type == "BACKGROUND_REMOVAL": + return { + "taskType": "BACKGROUND_REMOVAL", + "backgroundRemovalParams": {"image": image_b64}, + } + if task_type == "OUTPAINTING": + if mask_prompt is None and mask_b64 is None: + raise ValueError( + "OUTPAINTING requires either a mask image or a mask prompt. " + "Pass mask= or maskPrompt= in the request." + ) + out_params: Dict[str, Any] = { + "image": image_b64, + "text": text, + } + if mask_prompt is not None: + out_params["maskPrompt"] = mask_prompt + elif mask_b64 is not None: + out_params["maskImage"] = mask_b64 + if negative_text is not None: + out_params["negativeText"] = negative_text + if out_painting_mode is not None: + out_params["outPaintingMode"] = out_painting_mode + return { + "taskType": "OUTPAINTING", + "outPaintingParams": out_params, + } + # Honour explicit IMAGE_VARIATION even when a mask is present (mask is ignored + # for this task type; callers use INPAINTING when they want mask semantics). + if task_type == "IMAGE_VARIATION": + var_params_explicit: Dict[str, Any] = { + "images": [image_b64], + "text": text, + } + if negative_text is not None: + var_params_explicit["negativeText"] = negative_text + if similarity_strength is not None: + var_params_explicit["similarityStrength"] = similarity_strength + return { + "taskType": "IMAGE_VARIATION", + "imageVariationParams": var_params_explicit, + } + # Explicit taskType must be INPAINTING or omitted from here on; anything else is invalid. + if task_type is not None and str(task_type).strip() != "": + if task_type != "INPAINTING": + raise ValueError( + f"Unsupported Amazon Nova Canvas taskType: {task_type!r}. " + "Use BACKGROUND_REMOVAL, OUTPAINTING, IMAGE_VARIATION, INPAINTING, " + "or omit taskType for automatic routing (mask → INPAINTING, else IMAGE_VARIATION)." + ) + if mask_b64 is not None or mask_prompt is not None or task_type == "INPAINTING": + in_params: Dict[str, Any] = {"image": image_b64, "text": text} + if mask_prompt is not None: + in_params["maskPrompt"] = mask_prompt + elif mask_b64 is not None: + in_params["maskImage"] = mask_b64 + if negative_text is not None: + in_params["negativeText"] = negative_text + if "maskPrompt" not in in_params and "maskImage" not in in_params: + raise ValueError( + "Amazon Nova Canvas INPAINTING requires either maskPrompt or maskImage " + "(use OpenAI mask= for maskImage, or pass maskPrompt in optional params). " + "See https://docs.aws.amazon.com/nova/latest/userguide/image-gen-req-resp-structure.html" + ) + return {"taskType": "INPAINTING", "inPaintingParams": in_params} + var_params: Dict[str, Any] = { + "images": [image_b64], + "text": text, + } + if negative_text is not None: + var_params["negativeText"] = negative_text + if similarity_strength is not None: + var_params["similarityStrength"] = similarity_strength + return { + "taskType": "IMAGE_VARIATION", + "imageVariationParams": var_params, + } + + +def _file_types_to_b64(image: Optional[FileTypes]) -> str: + """Encode OpenAI image input to base64 string for Nova Canvas.""" + if image is None: + raise ValueError("Nova Canvas image edit requires an image input") + if hasattr(image, "read") and callable(getattr(image, "read", None)): + if hasattr(image, "seek"): + image.seek(0) # type: ignore[union-attr] + image_bytes = image.read() # type: ignore[union-attr] + return base64.b64encode(image_bytes).decode("utf-8") + if isinstance(image, bytes): + return base64.b64encode(image).decode("utf-8") + if isinstance(image, str): + return image + if isinstance(image, os.PathLike): + with open(image, "rb") as f: + return base64.b64encode(f.read()).decode("utf-8") + if isinstance(image, tuple): + raise ValueError( + "Nova Canvas image edit does not support tuple FileTypes. " + "Pass a file-like object, bytes, or a base64-encoded string." + ) + return base64.b64encode(bytes(image)).decode("utf-8") # type: ignore[arg-type] + + +def _supports_nova_canvas_image_edit_from_model_cost(model: str) -> bool: + """ + True when model_cost has supports_nova_canvas_image_edit for a resolved catalog key. + + get_model_info / ModelInfoBase omit arbitrary JSON keys, so we read model_cost + directly (same idea as supports_* bare_entry fallback). + """ + import litellm as _litellm + + if not model: + return False + + seen: set[str] = set() + candidates: List[str] = [] + + def _add(name: Optional[str]) -> None: + if name and name not in seen: + seen.add(name) + candidates.append(name) + + _add(model) + if "/" in model: + suffix = model.split("/")[-1] + _add(suffix) + _add(f"bedrock/{suffix}") + + # Cross-region inference ids (e.g. us.amazon.nova-canvas-v1:0) share pricing with + # the base model id (amazon.nova-canvas-v1:0) in model_cost. + try: + from litellm.llms.bedrock.common_utils import BedrockModelInfo + + base_model = BedrockModelInfo.get_base_model(model) + if base_model and base_model != model: + _add(base_model) + _add(f"bedrock/{base_model}") + except Exception: + pass + + try: + potential = _get_potential_model_names(model=model, custom_llm_provider=None) + for field in ( + "combined_model_name", + "combined_stripped_model_name", + "stripped_model_name", + "split_model", + ): + raw = potential.get(field) + if isinstance(raw, str): + _add(raw) + except Exception: + pass + + for name in candidates: + key = _get_model_cost_key(name) + if key is None: + continue + entry = _litellm.model_cost.get(key) or {} + if entry.get("supports_nova_canvas_image_edit") is True: + return True + return False + + +class BedrockAmazonNovaCanvasImageEditConfig(BaseImageEditConfig): + """ + Bedrock InvokeModel image edit for amazon.nova-canvas-v1:0 and regional variants. + """ + + @classmethod + def _is_nova_canvas_image_edit_model(cls, model: Optional[str] = None) -> bool: + """ + Use model_cost.supports_nova_canvas_image_edit so new Nova Canvas inference IDs + are added via model_prices_and_context_window.json only (not get_model_info, which + drops keys not on ModelInfoBase). + """ + return _supports_nova_canvas_image_edit_from_model_cost(model or "") + + def get_supported_openai_params(self, model: str) -> list: + return [ + "n", + "size", + "response_format", + "mask", + "negativeText", + "similarityStrength", + "cfgScale", + "seed", + "quality", + "taskType", + "maskPrompt", + "outPaintingMode", + "imageGenerationConfig", + ] + + def map_openai_params( + self, + image_edit_optional_params: ImageEditOptionalRequestParams, + model: str, + drop_params: bool, + ) -> Dict[str, Any]: + supported = set(self.get_supported_openai_params(model)) + mapped: Dict[str, Any] = dict(image_edit_optional_params) + _size = mapped.pop("size", None) + if _size is not None and isinstance(_size, str) and "x" in _size: + w, h = _size.split("x", 1) + try: + mapped["width"], mapped["height"] = int(w), int(h) + except ValueError: + pass + _n = mapped.pop("n", None) + if _n is not None: + mapped["numberOfImages"] = _n + _quality = mapped.pop("quality", None) + if _quality is not None: + if _quality in ("hd", "premium"): + mapped["quality"] = "premium" + elif _quality == "standard": + mapped["quality"] = "standard" + else: + # Re-emit unknown values (e.g. OpenAI "auto") so transform_image_edit_request + # forwards them and the API can reject, or drop_params can still apply upstream. + mapped["quality"] = _quality + # Accepted for OpenAI compatibility but ignored for Nova Canvas image edit; + # Bedrock returns base64 images only (no URL mode). + response_format = mapped.pop("response_format", None) + if response_format not in (None, "b64_json"): + verbose_logger.debug( + "Nova Canvas image edit ignores response_format=%s and returns base64 images", + response_format, + ) + # Drop unknown keys if drop_params + if drop_params: + for k in list(mapped.keys()): + if k.startswith("_"): + continue + if k not in supported and k not in ( + "width", + "height", + "numberOfImages", + "mask", + ): + mapped.pop(k, None) + return mapped + + def transform_image_edit_request( + self, + model: str, + prompt: Optional[str], + image: Optional[FileTypes], + image_edit_optional_request_params: Dict, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[Dict, Any]: + op = dict(image_edit_optional_request_params) + image_b64 = _file_types_to_b64(image) + + mask_raw = op.pop("mask", None) + mask_b64: Optional[str] = None + if mask_raw is not None: + mask_b64 = _file_types_to_b64(mask_raw) # type: ignore[arg-type] + + _size = op.pop("size", None) + width = op.pop("width", None) + height = op.pop("height", None) + if ( + width is None + and height is None + and _size is not None + and isinstance(_size, str) + and "x" in _size + ): + w, h = _size.split("x", 1) + try: + width, height = int(w), int(h) + except ValueError: + pass + + number_of_images = op.pop("numberOfImages", None) + quality = op.pop("quality", None) + cfg_scale = op.pop("cfgScale", None) + seed = op.pop("seed", None) + + image_generation_config: Dict[str, Any] = {} + nested_igc = op.pop("imageGenerationConfig", None) + if isinstance(nested_igc, dict): + image_generation_config.update(nested_igc) + if width is not None: + image_generation_config["width"] = width + if height is not None: + image_generation_config["height"] = height + if number_of_images is not None: + image_generation_config["numberOfImages"] = number_of_images + if quality is not None: + image_generation_config["quality"] = quality + if cfg_scale is not None: + image_generation_config["cfgScale"] = cfg_scale + if seed is not None: + image_generation_config["seed"] = seed + + task_type = op.pop("taskType", None) + if (prompt is None or prompt == "") and task_type in ( + "INPAINTING", + "OUTPAINTING", + ): + raise ValueError( + f"Amazon Nova Canvas {task_type} requires a text prompt. " + "Pass a non-empty `prompt` in your request." + ) + text = prompt if prompt is not None and prompt != "" else " " + negative_text = op.pop("negativeText", None) + similarity_strength = op.pop("similarityStrength", None) + mask_prompt = op.pop("maskPrompt", None) + out_painting_mode = op.pop("outPaintingMode", None) + + body = _nova_canvas_task_body( + image_b64=image_b64, + mask_b64=mask_b64, + text=text, + negative_text=negative_text, + similarity_strength=similarity_strength, + task_type=task_type, + mask_prompt=mask_prompt, + out_painting_mode=out_painting_mode, + ) + + # BACKGROUND_REMOVAL InvokeModel body must not include imageGenerationConfig (AWS rejects it). + if image_generation_config and body.get("taskType") != "BACKGROUND_REMOVAL": + body["imageGenerationConfig"] = image_generation_config + + return body, {} + + def transform_image_edit_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> ImageResponse: + try: + response_data = raw_response.json() + except Exception as e: + raise self.get_error_class( + error_message=f"Error parsing Nova Canvas image edit response: {e}", + status_code=raw_response.status_code, + headers=raw_response.headers, + ) + + if raw_response.status_code not in (200,): + raise self.get_error_class( + error_message=f"Nova Canvas image edit error: {response_data}", + status_code=raw_response.status_code, + headers=raw_response.headers, + ) + + images: List[str] = response_data.get("images") or [] + + if "errors" in response_data and not images: + raise self.get_error_class( + error_message=f"Nova Canvas image edit error: {response_data['errors']}", + status_code=raw_response.status_code, + headers=raw_response.headers, + ) + + # Nova Canvas InvokeModel success body uses "images" and optional "error" (AWS docs); + # it does not use Stability-style "finish_reasons". + error_msg = response_data.get("message") or response_data.get("error") + if error_msg and not images: + if not isinstance(error_msg, str): + error_msg = str(error_msg) + raise self.get_error_class( + error_message=f"Nova Canvas image edit error: {error_msg}", + status_code=raw_response.status_code, + headers=raw_response.headers, + ) + + model_response = ImageResponse() + model_response.data = [] + for image_b64 in images: + if image_b64: + model_response.data.append( + ImageObject( + b64_json=image_b64, + url=None, + revised_prompt=None, + ) + ) + + if not model_response.data: + raise self.get_error_class( + error_message="Nova Canvas image edit returned no images", + status_code=raw_response.status_code, + headers=raw_response.headers, + ) + + if not hasattr(model_response, "_hidden_params"): + model_response._hidden_params = {} + if "additional_headers" not in model_response._hidden_params: + model_response._hidden_params["additional_headers"] = {} + + try: + model_info = get_model_info(model, custom_llm_provider="bedrock") + cost_per_image = model_info.get("output_cost_per_image", 0) + if cost_per_image is not None and model_response.data: + model_response._hidden_params["additional_headers"][ + "llm_provider-x-litellm-response-cost" + ] = float(cost_per_image) * len(model_response.data) + except Exception: + pass + + return model_response + + def use_multipart_form_data(self) -> bool: + return False + + def get_complete_url( + self, + model: str, + api_base: Optional[str], + litellm_params: dict, + ) -> str: + raise NotImplementedError( + "Nova Canvas image edit URLs are built in BedrockImageEdit._prepare_request " + "(AWS runtime endpoint + model invoke path). Do not use get_complete_url for " + "this config." + ) + + def validate_environment( + self, + headers: dict, + model: str, + api_key: Optional[str] = None, + ) -> dict: + if headers is None: + headers = {} + if "Content-Type" not in headers: + headers["Content-Type"] = "application/json" + return headers + + +def get_bedrock_image_edit_config_for_model( + model: str, +) -> BaseImageEditConfig: + """ + Return the correct Bedrock image-edit config for the model id. + + Same routing as ``BedrockImageEdit.get_config_class``: Stability edit models, + Nova Canvas when marked in model_cost; otherwise raises ``ValueError``. + """ + from litellm.llms.bedrock.image_edit.stability_transformation import ( + BedrockStabilityImageEditConfig, + ) + + if BedrockStabilityImageEditConfig._is_stability_edit_model(model): + return BedrockStabilityImageEditConfig() + if BedrockAmazonNovaCanvasImageEditConfig._is_nova_canvas_image_edit_model(model): + return BedrockAmazonNovaCanvasImageEditConfig() + raise ValueError( + f"Unsupported Bedrock image-edit model: {model!r}. " + "Use a stability.* image-edit model id or add supports_nova_canvas_image_edit " + "in model_prices for this id." + ) diff --git a/litellm/llms/bedrock/image_edit/handler.py b/litellm/llms/bedrock/image_edit/handler.py index 867944f8796..90344310746 100644 --- a/litellm/llms/bedrock/image_edit/handler.py +++ b/litellm/llms/bedrock/image_edit/handler.py @@ -15,6 +15,9 @@ from pydantic import BaseModel import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging +from litellm.llms.bedrock.image_edit.amazon_nova_canvas_image_edit_transformation import ( + BedrockAmazonNovaCanvasImageEditConfig, +) from litellm.llms.bedrock.image_edit.stability_transformation import ( BedrockStabilityImageEditConfig, ) @@ -55,8 +58,15 @@ class BedrockImageEdit(BaseAWSLLM): def get_config_class(cls, model: str | None): if BedrockStabilityImageEditConfig._is_stability_edit_model(model): return BedrockStabilityImageEditConfig - else: - raise ValueError(f"Unsupported model for bedrock image edit: {model}") + if BedrockAmazonNovaCanvasImageEditConfig._is_nova_canvas_image_edit_model( + model + ): + return BedrockAmazonNovaCanvasImageEditConfig + raise ValueError( + f"Unsupported Bedrock image-edit model: {model!r}. " + "Use a stability.* image-edit model id or add supports_nova_canvas_image_edit " + "in model_prices for this id." + ) def image_edit( self, diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index 784c6a50cfd..c1eccaebd04 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -502,6 +502,12 @@ class AmazonAnthropicClaudeMessagesConfig( ): """ Bedrock invoke does not return SSE formatted data. This function is a wrapper to ensure litellm chunks are SSE formatted. + + Bedrock's Anthropic-compatible streaming puts cache usage fields + (cache_creation_input_tokens, cache_read_input_tokens) only on + message_stop, not on message_start or message_delta. Claude Code's + SDK only merges usage from message_delta, so we promote those fields + from message_stop onto message_delta before yielding. """ from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import ( BaseAnthropicMessagesStreamingIterator, @@ -512,9 +518,81 @@ class AmazonAnthropicClaudeMessagesConfig( request_body=request_body, ) - async for chunk in handler.async_sse_wrapper(completion_stream): + patched_stream = self._promote_message_stop_usage(completion_stream) + + async for chunk in handler.async_sse_wrapper(patched_stream): yield chunk + @staticmethod + async def _promote_message_stop_usage( + completion_stream: AsyncIterator[ + Union[bytes, GenericStreamingChunk, ModelResponseStream, dict] + ], + ) -> AsyncIterator[Union[bytes, GenericStreamingChunk, ModelResponseStream, dict]]: + """ + Promote cache usage fields from message_stop onto message_delta. + + Bedrock reports input_tokens (uncached only) on message_start, and + the full breakdown (input_tokens, cache_creation_input_tokens, + cache_read_input_tokens) only on message_stop. Claude Code's SDK + merges usage from message_start and message_delta but ignores + message_stop. This method buffers message_delta and, when + message_stop arrives with cache usage, merges those fields into the + message_delta usage and also updates the input_tokens on + message_delta to include the full count (uncached + cache_creation + + cache_read). + """ + _CACHE_FIELDS = ("cache_creation_input_tokens", "cache_read_input_tokens") + pending_delta = None + + async for chunk in completion_stream: + if not isinstance(chunk, dict): + if pending_delta is not None: + yield pending_delta + pending_delta = None + yield chunk + continue + + chunk_type = chunk.get("type") + + if chunk_type == "message_delta": + pending_delta = chunk + continue + + if chunk_type == "message_stop" and pending_delta is not None: + stop_usage = dict(chunk.get("usage") or {}) + delta_usage = dict(pending_delta.get("usage") or {}) + + for field in _CACHE_FIELDS: + if field in stop_usage: + delta_usage[field] = stop_usage[field] + + raw_input = stop_usage.get("input_tokens") + if raw_input is not None: + uncached = raw_input if isinstance(raw_input, int) else 0 + raw_cc = delta_usage.get("cache_creation_input_tokens", 0) + cache_creation = raw_cc if isinstance(raw_cc, int) else 0 + raw_cr = delta_usage.get("cache_read_input_tokens", 0) + cache_read = raw_cr if isinstance(raw_cr, int) else 0 + delta_usage["input_tokens"] = uncached + cache_creation + cache_read + + if delta_usage: + pending_delta["usage"] = delta_usage # type: ignore[arg-type] + + yield pending_delta + pending_delta = None + yield chunk + continue + + if pending_delta is not None: + yield pending_delta + pending_delta = None + + yield chunk + + if pending_delta is not None: + yield pending_delta + class AmazonAnthropicClaudeMessagesStreamDecoder(AWSEventStreamDecoder): def __init__( diff --git a/litellm/llms/firecrawl/search/transformation.py b/litellm/llms/firecrawl/search/transformation.py index 61b589218cc..71136e1d3b3 100644 --- a/litellm/llms/firecrawl/search/transformation.py +++ b/litellm/llms/firecrawl/search/transformation.py @@ -159,15 +159,13 @@ class FirecrawlSearchConfig(BaseSearchConfig): """ Transform Firecrawl API response to LiteLLM unified SearchResponse format. - Firecrawl → LiteLLM mappings: - - data.web[].title → SearchResult.title - - data.web[].url → SearchResult.url - - data.web[].description OR data.web[].markdown → SearchResult.snippet - - No date field in web results (set to None) - - No last_updated field in Firecrawl response (set to None) + Supports both response formats: - Note: Firecrawl v2 returns results organized by source type (web, images, news). - We primarily use web results for the unified format. + Firecrawl Cloud (v2): + {"data": {"web": [...], "news": [...]}} + + Firecrawl Self-Hosted (v1): + {"success": true, "data": [{"url": "...", "title": "...", ...}, ...]} Args: raw_response: Raw httpx response from Firecrawl API @@ -181,36 +179,52 @@ class FirecrawlSearchConfig(BaseSearchConfig): # Transform results to SearchResult objects results = [] - # Process web results (primary source) data = response_json.get("data", {}) - web_results = data.get("web", []) - for result in web_results: - # Use markdown if available, otherwise fall back to description - snippet = result.get("markdown") or result.get("description", "") + if isinstance(data, list): + # Self-hosted Firecrawl (v1) format: data is a flat list of results + for result in data: + snippet = ( + result.get("markdown") or result.get("description", "") + ) + search_result = SearchResult( + title=result.get("title", ""), + url=result.get("url", ""), + snippet=snippet, + date=None, + last_updated=None, + ) + results.append(search_result) + elif isinstance(data, dict): + # Firecrawl Cloud (v2) format: data is a dict with web/news keys + web_results = data.get("web", []) - search_result = SearchResult( - title=result.get("title", ""), - url=result.get("url", ""), - snippet=snippet, - date=None, # Web results don't include date - last_updated=None, # Firecrawl doesn't provide last_updated in response - ) - results.append(search_result) + for result in web_results: + # Use markdown if available, otherwise fall back to description + snippet = result.get("markdown") or result.get("description", "") - # Process news results if available (they have date field) - news_results = data.get("news", []) - for result in news_results: - snippet = result.get("markdown") or result.get("snippet", "") + search_result = SearchResult( + title=result.get("title", ""), + url=result.get("url", ""), + snippet=snippet, + date=None, + last_updated=None, + ) + results.append(search_result) - search_result = SearchResult( - title=result.get("title", ""), - url=result.get("url", ""), - snippet=snippet, - date=result.get("date"), # News results include date - last_updated=None, - ) - results.append(search_result) + # Process news results if available (they have date field) + news_results = data.get("news", []) + for result in news_results: + snippet = result.get("markdown") or result.get("snippet", "") + + search_result = SearchResult( + title=result.get("title", ""), + url=result.get("url", ""), + snippet=snippet, + date=result.get("date"), # News results include date + last_updated=None, + ) + results.append(search_result) return SearchResponse( results=results, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 4072aab6544..4e1c7f4ac80 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -277,7 +277,15 @@ "litellm_provider": "bedrock", "max_input_tokens": 2600, "mode": "image_generation", - "output_cost_per_image": 0.06 + "output_cost_per_image": 0.06, + "supports_nova_canvas_image_edit": true + }, + "us.amazon.nova-canvas-v1:0": { + "litellm_provider": "bedrock", + "max_input_tokens": 2600, + "mode": "image_generation", + "output_cost_per_image": 0.06, + "supports_nova_canvas_image_edit": true }, "us.writer.palmyra-x4-v1:0": { "input_cost_per_token": 2.5e-06, diff --git a/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py b/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py index 3ee8e6a27bd..3250e0bb7cf 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py +++ b/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py @@ -760,8 +760,8 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): error_obj = dict(error_value) else: error_obj = {"message": str(error_value)} - error_obj["code"] = e.status_code - yield f"data: {json.dumps({'error': error_obj})}\n\n" + error_obj["code"] = str(e.status_code) + yield f"data: {json.dumps({'error': error_obj})}\n\n" # type: ignore[misc] return except Exception as e: verbose_proxy_logger.error( diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 948d6dd33af..46000f4fe6e 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -112,6 +112,14 @@ class _ProxyDBLogger(CustomLogger): _litellm_logging_obj, "litellm_trace_id", None ) + # Use the actual request start time from the logging object so that + # failed requests record the real duration instead of 0. + actual_start_time = datetime.now() + if _litellm_logging_obj is not None: + obj_start = getattr(_litellm_logging_obj, "start_time", None) + if obj_start is not None: + actual_start_time = obj_start + await proxy_logging_obj.db_spend_update_writer.update_database( token=user_api_key_dict.api_key, response_cost=0.0, @@ -120,7 +128,7 @@ class _ProxyDBLogger(CustomLogger): team_id=user_api_key_dict.team_id, kwargs=request_data, completion_response=original_exception, - start_time=datetime.now(), + start_time=actual_start_time, end_time=datetime.now(), org_id=user_api_key_dict.org_id, ) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index e105d9b8586..6f9166909c0 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -18,7 +18,7 @@ import re import secrets import traceback from datetime import datetime, timedelta, timezone -from typing import Any, Dict, List, Literal, Optional, Tuple, cast +from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, cast import fastapi import yaml @@ -487,6 +487,59 @@ async def validate_team_id_used_in_service_account_request( return True +def _enforce_upperbound_key_params( + data: Union[GenerateKeyRequest, UpdateKeyRequest], + fill_defaults: bool = True, +) -> None: + """ + Enforce upperbound limits on key parameters. + + For key generation (fill_defaults=True): fills None values with upperbound defaults. + For key update (fill_defaults=False): only validates explicitly provided values. + """ + if litellm.upperbound_key_generate_params is None: + return + + for elem in data: + key, value = elem + upperbound_value = getattr( + litellm.upperbound_key_generate_params, key, None + ) + if upperbound_value is not None: + if value is None: + if fill_defaults: + setattr(data, key, upperbound_value) + else: + if key in [ + "max_budget", + "max_parallel_requests", + "tpm_limit", + "rpm_limit", + ]: + if value > upperbound_value: + raise HTTPException( + status_code=400, + detail={ + "error": f"{key} is over max limit set in config - user_value={value}; max_value={upperbound_value}" + }, + ) + elif key in ["budget_duration", "duration"]: + upperbound_duration = duration_in_seconds( + duration=upperbound_value + ) + if value == "-1": + user_duration = float("inf") + else: + user_duration = duration_in_seconds(duration=value) + if user_duration > upperbound_duration: + raise HTTPException( + status_code=400, + detail={ + "error": f"{key} is over max limit set in config - user_value={value}; max_value={upperbound_value}" + }, + ) + + async def _common_key_generation_helper( # noqa: PLR0915 data: GenerateKeyRequest, user_api_key_dict: UserAPIKeyAuth, @@ -537,49 +590,8 @@ async def _common_key_generation_helper( # noqa: PLR0915 elif key == "metadata" and value == {}: setattr(data, key, litellm.default_key_generate_params.get(key, {})) - # check if user set default key/generate params on config.yaml - if litellm.upperbound_key_generate_params is not None: - for elem in data: - key, value = elem - upperbound_value = getattr( - litellm.upperbound_key_generate_params, key, None - ) - if upperbound_value is not None: - if value is None: - # Use the upperbound value if user didn't provide a value - setattr(data, key, upperbound_value) - else: - # Compare with upperbound for numeric fields - if key in [ - "max_budget", - "max_parallel_requests", - "tpm_limit", - "rpm_limit", - ]: - if value > upperbound_value: - raise HTTPException( - status_code=400, - detail={ - "error": f"{key} is over max limit set in config - user_value={value}; max_value={upperbound_value}" - }, - ) - # Compare durations - elif key in ["budget_duration", "duration"]: - upperbound_duration = duration_in_seconds( - duration=upperbound_value - ) - # Handle special case where duration is None or "-1" (never expires) - if value is None or value == "-1": - user_duration = float("inf") # Infinite duration - else: - user_duration = duration_in_seconds(duration=value) - if user_duration > upperbound_duration: - raise HTTPException( - status_code=400, - detail={ - "error": f"{key} is over max limit set in config - user_value={value}; max_value={upperbound_value}" - }, - ) + # check if user set upperbound key/generate params on config.yaml + _enforce_upperbound_key_params(data, fill_defaults=True) # APPLY ENTERPRISE KEY MANAGEMENT PARAMS try: @@ -1687,6 +1699,7 @@ async def _process_single_key_update( user_api_key_cache: DualCache, proxy_logging_obj: Any, llm_router: Optional[Router], + user_custom_key_update: Optional[Callable] = None, ) -> Dict[str, Any]: """ Process a single key update with all validations and checks. @@ -1737,6 +1750,22 @@ async def _process_single_key_update( tags=key_update_item.tags, ) + # Custom key update hook + if user_custom_key_update is not None: + if inspect.iscoroutinefunction(user_custom_key_update): + result = await user_custom_key_update(update_key_request) + else: + raise ValueError("user_custom_key_update must be a coroutine") + decision = result.get("decision", True) + message = result.get("message", "Authentication Failed - Custom Auth Rule") + if not decision: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, detail=message + ) + + # Enforce upperbound key params on update (don't fill defaults) + _enforce_upperbound_key_params(update_key_request, fill_defaults=False) + # Get team object and check team limits if team_id is provided team_obj: Optional[LiteLLM_TeamTableCachedObj] = None if update_key_request.team_id is not None: @@ -2014,7 +2043,7 @@ async def _validate_update_key_data( "/key/update", tags=["key management"], dependencies=[Depends(user_api_key_auth)] ) @management_endpoint_wrapper -async def update_key_fn( +async def update_key_fn( # noqa: PLR0915 request: Request, data: UpdateKeyRequest, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), @@ -2095,6 +2124,7 @@ async def update_key_fn( prisma_client, proxy_logging_obj, user_api_key_cache, + user_custom_key_update, ) try: @@ -2126,6 +2156,21 @@ async def update_key_fn( user_api_key_cache=user_api_key_cache, ) + # Custom key update hook + if user_custom_key_update is not None: + if inspect.iscoroutinefunction(user_custom_key_update): + result = await user_custom_key_update(data) + else: + raise ValueError("user_custom_key_update must be a coroutine") + decision = result.get("decision", True) + message = result.get("message", "Authentication Failed - Custom Auth Rule") + if not decision: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, detail=message + ) + + # Enforce upperbound key params on update (don't fill defaults) + _enforce_upperbound_key_params(data, fill_defaults=False) non_default_values = await prepare_key_update_data( data=data, existing_key_row=existing_key_row ) @@ -2261,6 +2306,7 @@ async def bulk_update_keys( prisma_client, proxy_logging_obj, user_api_key_cache, + user_custom_key_update, ) if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN.value: @@ -2304,6 +2350,7 @@ async def bulk_update_keys( user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, llm_router=llm_router, + user_custom_key_update=user_custom_key_update, ) successful_updates.append( diff --git a/litellm/proxy/prompts/prompt_endpoints.py b/litellm/proxy/prompts/prompt_endpoints.py index 73e0ece3e2c..483107a3759 100644 --- a/litellm/proxy/prompts/prompt_endpoints.py +++ b/litellm/proxy/prompts/prompt_endpoints.py @@ -192,19 +192,22 @@ def get_latest_prompt_versions(prompts: List[PromptSpec]) -> List[PromptSpec]: return list(latest_prompts.values()) -async def get_next_version_for_prompt(prisma_client, prompt_id: str) -> int: +async def get_next_version_for_prompt( + prisma_client, prompt_id: str, environment: str = "development" +) -> int: """ - Get the next version number for a prompt. + Get the next version number for a prompt in a specific environment. Args: prisma_client: Prisma database client prompt_id: Base prompt ID + environment: The environment to check versions for Returns: Next version number (1 if no versions exist, max_version + 1 otherwise) """ existing_prompts = await prisma_client.db.litellm_prompttable.find_many( - where={"prompt_id": prompt_id} + where={"prompt_id": prompt_id, "environment": environment} ) if existing_prompts: @@ -231,6 +234,8 @@ def create_versioned_prompt_spec(db_prompt) -> PromptSpec: prompt_dict = db_prompt.model_dump() base_prompt_id = prompt_dict["prompt_id"] version = prompt_dict.get("version", 1) + environment = prompt_dict.get("environment", "development") + created_by = prompt_dict.get("created_by") # Parse litellm_params litellm_params_data = prompt_dict.get("litellm_params") @@ -256,6 +261,8 @@ def create_versioned_prompt_spec(db_prompt) -> PromptSpec: prompt_info=prompt_info, created_at=prompt_dict.get("created_at"), updated_at=prompt_dict.get("updated_at"), + environment=environment, + created_by=created_by, ) @@ -277,6 +284,7 @@ class PatchPromptRequest(BaseModel): response_model=ListPromptsResponse, ) async def list_prompts( + environment: Optional[str] = None, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ @@ -318,23 +326,26 @@ async def list_prompts( if key_metadata is not None: prompts = cast(Optional[List[str]], key_metadata.get("prompts", None)) if prompts is not None: + all_prompts = [ + IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS[prompt_id] + for prompt_id in prompts + if prompt_id in IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS + ] + if environment: + all_prompts = [p for p in all_prompts if p.environment == environment] prompt_list = [] - for prompt_id in prompts: - if prompt_id in IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS: - original_prompt = IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS[ - prompt_id - ] - # Create a copy with base prompt_id (without version suffix) - prompt_copy = PromptSpec( - prompt_id=get_base_prompt_id( - prompt_id=original_prompt.prompt_id - ), - litellm_params=original_prompt.litellm_params, - prompt_info=original_prompt.prompt_info, - created_at=original_prompt.created_at, - updated_at=original_prompt.updated_at, - ) - prompt_list.append(prompt_copy) + for original_prompt in all_prompts: + # Create a copy with base prompt_id (without version suffix) + prompt_copy = PromptSpec( + prompt_id=get_base_prompt_id(prompt_id=original_prompt.prompt_id), + litellm_params=original_prompt.litellm_params, + prompt_info=original_prompt.prompt_info, + created_at=original_prompt.created_at, + updated_at=original_prompt.updated_at, + environment=original_prompt.environment, + created_by=original_prompt.created_by, + ) + prompt_list.append(prompt_copy) return ListPromptsResponse(prompts=prompt_list) # check if user is proxy admin - show all prompts if user_api_key_dict.user_role is not None and ( @@ -343,6 +354,8 @@ async def list_prompts( ): # Get all prompts and filter to show only the latest version of each all_prompts = list(IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS.values()) + if environment: + all_prompts = [p for p in all_prompts if p.environment == environment] latest_prompts = get_latest_prompt_versions(prompts=all_prompts) # Create copies with base prompt_id (without version suffix) for display prompts_for_display = [] @@ -353,6 +366,8 @@ async def list_prompts( prompt_info=original_prompt.prompt_info, created_at=original_prompt.created_at, updated_at=original_prompt.updated_at, + environment=original_prompt.environment, + created_by=original_prompt.created_by, ) prompts_for_display.append(prompt_copy) return ListPromptsResponse(prompts=prompts_for_display) @@ -368,6 +383,7 @@ async def list_prompts( ) async def get_prompt_versions( prompt_id: str, + environment: Optional[str] = None, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ @@ -404,6 +420,7 @@ async def get_prompt_versions( ``` """ from litellm.proxy.prompts.prompt_registry import IN_MEMORY_PROMPT_REGISTRY + from litellm.proxy.proxy_server import prisma_client # Only allow proxy admins to view version history if user_api_key_dict.user_role is None or ( @@ -414,49 +431,112 @@ async def get_prompt_versions( status_code=403, detail="Only proxy admins can view prompt versions" ) - # Strip version suffix if provided (e.g., "jack_success.v1" -> "jack_success") base_prompt_id = get_base_prompt_id(prompt_id=prompt_id) - # Get all prompts and filter by base_prompt_id - all_prompts = list(IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS.values()) - prompt_versions = [ - prompt - for prompt in all_prompts - if get_base_prompt_id(prompt_id=prompt.prompt_id) == base_prompt_id - ] + # Query DB for versions + versioned_prompts = [] + if prisma_client is not None: + where_clause: Dict[str, Any] = {"prompt_id": base_prompt_id} + if environment: + where_clause["environment"] = environment + db_prompts = await prisma_client.db.litellm_prompttable.find_many( + where=where_clause, + order={"version": "desc"}, + ) + for db_prompt in db_prompts: + spec = create_versioned_prompt_spec(db_prompt=db_prompt) + versioned_prompts.append( + PromptSpec( + prompt_id=base_prompt_id, + litellm_params=spec.litellm_params, + prompt_info=spec.prompt_info, + created_at=spec.created_at, + updated_at=spec.updated_at, + version=get_version_number(prompt_id=spec.prompt_id), + environment=spec.environment, + created_by=spec.created_by, + ) + ) + else: + # Fallback: in-memory registry (no DB) + all_prompts = list(IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS.values()) + prompt_versions = [ + prompt + for prompt in all_prompts + if get_base_prompt_id(prompt_id=prompt.prompt_id) == base_prompt_id + and (environment is None or prompt.environment == environment) + ] + for prompt in prompt_versions: + version_number = get_version_number(prompt_id=prompt.prompt_id) + versioned_prompts.append( + PromptSpec( + prompt_id=base_prompt_id, + litellm_params=prompt.litellm_params, + prompt_info=prompt.prompt_info, + created_at=prompt.created_at, + updated_at=prompt.updated_at, + version=version_number, + environment=prompt.environment, + created_by=prompt.created_by, + ) + ) + versioned_prompts.sort(key=lambda p: p.version or 1, reverse=True) - if not prompt_versions: + if not versioned_prompts: raise HTTPException( status_code=404, detail=f"No versions found for prompt ID {base_prompt_id}" ) - # Create response with explicit version field for each prompt - versioned_prompts = [] - for prompt in prompt_versions: - # Extract version number from the root prompt_id which has version suffix - # (e.g., "jack-sparrow.v3" -> 3) - version_number = get_version_number(prompt_id=prompt.prompt_id) - - # Strip version from prompt_id for clean display - base_prompt_id = get_base_prompt_id(prompt_id=prompt.prompt_id) - - # Create a copy with explicit version field and clean prompt_id - versioned_prompt = PromptSpec( - prompt_id=base_prompt_id, # Clean ID without version (e.g., "jack-sparrow") - litellm_params=prompt.litellm_params, - prompt_info=prompt.prompt_info, - created_at=prompt.created_at, - updated_at=prompt.updated_at, - version=version_number, # Explicit version field (e.g., 3) - ) - versioned_prompts.append(versioned_prompt) - - # Sort by version number (descending - newest first) - versioned_prompts.sort(key=lambda p: p.version or 1, reverse=True) - return ListPromptsResponse(prompts=versioned_prompts) +def _get_prompt_template( + prompt_spec: PromptSpec, base_prompt_id: str +) -> Optional[PromptTemplateBase]: + """Resolve the raw prompt template from dotprompt content or the in-memory registry.""" + from litellm.proxy.prompts.prompt_registry import IN_MEMORY_PROMPT_REGISTRY + + try: + dotprompt_content = prompt_spec.litellm_params.dotprompt_content + if dotprompt_content: + from litellm.integrations.dotprompt import ( + _get_prompt_data_from_dotprompt_content, + ) + + parsed = _get_prompt_data_from_dotprompt_content(dotprompt_content) + if parsed: + return PromptTemplateBase( + litellm_prompt_id=base_prompt_id, + content=parsed.get("content", ""), + metadata=parsed.get("metadata"), + ) + else: + prompt_callback = IN_MEMORY_PROMPT_REGISTRY.get_prompt_callback_by_id( + prompt_spec.prompt_id + ) + if prompt_callback is not None: + integration_name = prompt_callback.integration_name + if integration_name == "dotprompt": + from litellm.integrations.dotprompt.dotprompt_manager import ( + DotpromptManager, + ) + + if isinstance(prompt_callback, DotpromptManager): + template = ( + prompt_callback.prompt_manager.get_all_prompts_as_json() + ) + if template is not None and len(template) == 1: + template_id = list(template.keys())[0] + return PromptTemplateBase( + litellm_prompt_id=template_id, + content=template[template_id]["content"], + metadata=template[template_id]["metadata"], + ) + except Exception: + pass + return None + + @router.get( "/prompts/{prompt_id}", tags=["Prompt Management"], @@ -471,6 +551,7 @@ async def get_prompt_versions( ) async def get_prompt_info( prompt_id: str, + environment: Optional[str] = None, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ @@ -503,6 +584,7 @@ async def get_prompt_info( ``` """ from litellm.proxy.prompts.prompt_registry import IN_MEMORY_PROMPT_REGISTRY + from litellm.proxy.proxy_server import prisma_client ## CHECK IF USER HAS ACCESS TO PROMPT prompts: Optional[List[str]] = None @@ -523,68 +605,80 @@ async def get_prompt_info( detail=f"You are not authorized to access this prompt. Your role - {user_api_key_dict.user_role}, Your key's prompts - {prompts}", ) - # Try to get prompt directly first - prompt_spec = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(prompt_id) + base_prompt_id = get_base_prompt_id(prompt_id=prompt_id) - # If not found, try to find the latest version - if prompt_spec is None: - latest_prompt_id = get_latest_version_prompt_id( - prompt_id=prompt_id, - all_prompt_ids=IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS, + # Query all environments this prompt exists in (lightweight: distinct on environment) + all_environments: List[str] = [] + if prisma_client is not None: + all_prompt_rows = await prisma_client.db.litellm_prompttable.find_many( + where={"prompt_id": base_prompt_id}, + distinct=["environment"], ) - prompt_spec = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(latest_prompt_id) + all_environments = sorted( + set(row.environment for row in all_prompt_rows if row.environment) + ) + + # If environment is specified, find the version in that environment from DB + # If prompt_id has a version suffix (e.g., "testprompt.v2"), fetch that specific version + # Otherwise fetch the latest version in that environment + prompt_spec = None + requested_version = ( + get_version_number(prompt_id=prompt_id) if prompt_id != base_prompt_id else None + ) + if environment and prisma_client is not None: + where_clause: Dict[str, Any] = { + "prompt_id": base_prompt_id, + "environment": environment, + } + if requested_version is not None: + where_clause["version"] = requested_version + env_prompts = await prisma_client.db.litellm_prompttable.find_many( + where=where_clause, + order={"version": "desc"}, + take=1, + ) + if env_prompts: + prompt_spec = create_versioned_prompt_spec(db_prompt=env_prompts[0]) + + # Fallback: use in-memory registry (no environment filter) + if prompt_spec is None and environment is None: + prompt_spec = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(prompt_id) + if prompt_spec is None: + latest_prompt_id = get_latest_version_prompt_id( + prompt_id=prompt_id, + all_prompt_ids=IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS, + ) + prompt_spec = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(latest_prompt_id) if prompt_spec is None: - raise HTTPException(status_code=400, detail=f"Prompt {prompt_id} not found") + raise HTTPException( + status_code=400, + detail=f"Prompt {prompt_id} not found" + + (f" in environment {environment}" if environment else ""), + ) # Extract version number from the prompt_id version_number = get_version_number(prompt_id=prompt_spec.prompt_id) # Create a copy of the prompt spec with the base prompt ID (stripped of version) - # and explicit version field for consistency with list_prompts and versions endpoints prompt_spec_response = PromptSpec( prompt_id=get_base_prompt_id(prompt_id=prompt_spec.prompt_id), - litellm_params=prompt_spec.litellm_params, # This preserves the versioned ID + litellm_params=prompt_spec.litellm_params, prompt_info=prompt_spec.prompt_info, created_at=prompt_spec.created_at, updated_at=prompt_spec.updated_at, - version=version_number, # Explicit version field + version=version_number, + environment=prompt_spec.environment, + created_by=prompt_spec.created_by, ) - # Get prompt content from the callback - prompt_template: Optional[PromptTemplateBase] = None - try: - prompt_callback = IN_MEMORY_PROMPT_REGISTRY.get_prompt_callback_by_id( - prompt_spec.prompt_id - ) - if prompt_callback is not None: - # Extract content based on integration type - integration_name = prompt_callback.integration_name + # Get prompt content + prompt_template = _get_prompt_template(prompt_spec, base_prompt_id) - if integration_name == "dotprompt": - # For dotprompt integration, get content from the prompt manager - from litellm.integrations.dotprompt.dotprompt_manager import ( - DotpromptManager, - ) - - if isinstance(prompt_callback, DotpromptManager): - template = prompt_callback.prompt_manager.get_all_prompts_as_json() - if template is not None and len(template) == 1: - template_id = list(template.keys())[0] - prompt_template = PromptTemplateBase( - litellm_prompt_id=template_id, # id sent to prompt management tool - content=template[template_id]["content"], - metadata=template[template_id]["metadata"], - ) - - except Exception: - # If content extraction fails, continue without content - pass - - # Create response with content return PromptInfoResponse( prompt_spec=prompt_spec_response, raw_prompt_template=prompt_template, + environments=all_environments, ) @@ -641,9 +735,18 @@ async def create_prompt( ) try: + # Extract environment from request + environment = ( + request.prompt_info.environment + if request.prompt_info and request.prompt_info.environment + else "development" + ) + # Get next version number new_version = await get_next_version_for_prompt( - prisma_client=prisma_client, prompt_id=request.prompt_id + prisma_client=prisma_client, + prompt_id=request.prompt_id, + environment=environment, ) # Store prompt in db with version @@ -651,6 +754,8 @@ async def create_prompt( data={ "prompt_id": request.prompt_id, "version": new_version, + "environment": environment, + "created_by": user_api_key_dict.user_id, "litellm_params": request.litellm_params.model_dump_json(), "prompt_info": ( request.prompt_info.model_dump_json() @@ -733,14 +838,22 @@ async def update_prompt( # Strip version suffix from prompt_id if present (e.g., "jack_success.v1" -> "jack_success") base_prompt_id = get_base_prompt_id(prompt_id=prompt_id) - # Check if any version exists + # Extract environment from request + environment = ( + request.prompt_info.environment + if request.prompt_info and request.prompt_info.environment + else "development" + ) + + # Check if any version of this prompt exists (in any environment) existing_prompts = await prisma_client.db.litellm_prompttable.find_many( where={"prompt_id": base_prompt_id} ) if not existing_prompts: raise HTTPException( - status_code=404, detail=f"Prompt with ID {base_prompt_id} not found" + status_code=404, + detail=f"Prompt with ID {base_prompt_id} not found", ) # Check if it's a config prompt @@ -756,7 +869,9 @@ async def update_prompt( # Get next version number (UPDATE creates a new version) new_version = await get_next_version_for_prompt( - prisma_client=prisma_client, prompt_id=base_prompt_id + prisma_client=prisma_client, + prompt_id=base_prompt_id, + environment=environment, ) # Store new version in db @@ -764,6 +879,8 @@ async def update_prompt( data={ "prompt_id": base_prompt_id, "version": new_version, + "environment": environment, + "created_by": user_api_key_dict.user_id, "litellm_params": request.litellm_params.model_dump_json(), "prompt_info": ( request.prompt_info.model_dump_json() @@ -800,6 +917,7 @@ async def update_prompt( ) async def delete_prompt( prompt_id: str, + environment: Optional[str] = None, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ @@ -867,15 +985,31 @@ async def delete_prompt( # Get the base prompt ID (without version suffix) for database deletion base_prompt_id = get_base_prompt_id(prompt_id=prompt_id) - # Delete all versions of the prompt from the database - await prisma_client.db.litellm_prompttable.delete_many( - where={"prompt_id": base_prompt_id} - ) + # Build delete filter; scope to environment if provided + delete_where: Dict[str, Any] = {"prompt_id": base_prompt_id} + if environment: + delete_where["environment"] = environment - # Remove all versions of the prompt from memory - IN_MEMORY_PROMPT_REGISTRY.delete_prompts_by_base_id(base_prompt_id) + # Delete versions from the database (scoped to environment if provided) + await prisma_client.db.litellm_prompttable.delete_many(where=delete_where) - return {"message": f"Prompt {base_prompt_id} deleted successfully"} + # Remove matching prompts from memory — scope to environment if provided + if environment: + prompts_to_delete = [ + pid + for pid, prompt in IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS.items() + if get_base_prompt_id(prompt_id=pid) == base_prompt_id + and prompt.environment == environment + ] + for pid in prompts_to_delete: + del IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS[pid] + if pid in IN_MEMORY_PROMPT_REGISTRY.prompt_id_to_custom_prompt: + del IN_MEMORY_PROMPT_REGISTRY.prompt_id_to_custom_prompt[pid] + else: + IN_MEMORY_PROMPT_REGISTRY.delete_prompts_by_base_id(base_prompt_id) + + env_msg = f" from {environment}" if environment else "" + return {"message": f"Prompt {base_prompt_id} deleted successfully{env_msg}"} except HTTPException as e: raise e @@ -884,6 +1018,22 @@ async def delete_prompt( raise HTTPException(status_code=500, detail=str(e)) +def _reload_prompt_in_registry( + registry: Any, versioned_id: str, updated_prompt_spec: PromptSpec +) -> PromptSpec: + """Remove stale entry and re-initialize the prompt in the in-memory registry.""" + if versioned_id in registry.IN_MEMORY_PROMPTS: + del registry.IN_MEMORY_PROMPTS[versioned_id] + if versioned_id in registry.prompt_id_to_custom_prompt: + del registry.prompt_id_to_custom_prompt[versioned_id] + initialized = registry.initialize_prompt( + prompt=updated_prompt_spec, config_file_path=None + ) + if initialized is None: + raise HTTPException(status_code=500, detail="Failed to patch prompt") + return initialized + + @router.patch( "/prompts/{prompt_id}", tags=["Prompt Management"], @@ -892,6 +1042,7 @@ async def delete_prompt( async def patch_prompt( prompt_id: str, request: PatchPromptRequest, + environment: Optional[str] = None, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ @@ -935,61 +1086,93 @@ async def patch_prompt( ) try: - # Check if prompt exists and get current data - existing_prompt = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(prompt_id) - if existing_prompt is None: + # Resolve the target row: find the latest version in the given environment + base_prompt_id = get_base_prompt_id(prompt_id=prompt_id) + env = environment or "development" + requested_version = ( + get_version_number(prompt_id=prompt_id) + if prompt_id != base_prompt_id + else None + ) + + # Build query to find the exact row by composite unique key + find_where: Dict[str, Any] = { + "prompt_id": base_prompt_id, + "environment": env, + } + if requested_version is not None: + find_where["version"] = requested_version + + db_rows = await prisma_client.db.litellm_prompttable.find_many( + where=find_where, + order={"version": "desc"}, + take=1, + ) + if not db_rows: raise HTTPException( - status_code=404, detail=f"Prompt with ID {prompt_id} not found" + status_code=404, + detail=f"Prompt with ID {base_prompt_id} not found in environment {env}", ) - if existing_prompt.prompt_info.prompt_type == "config": + target_row = db_rows[0] + + # Check if prompt exists in memory + versioned_id = f"{base_prompt_id}.v{target_row.version}" + existing_prompt = IN_MEMORY_PROMPT_REGISTRY.get_prompt_by_id(versioned_id) + + if existing_prompt and existing_prompt.prompt_info.prompt_type == "config": raise HTTPException( status_code=400, detail="Cannot update config prompts.", ) + # Use existing prompt from memory or build from DB row for field merging + if existing_prompt: + current_litellm_params = existing_prompt.litellm_params + current_prompt_info = existing_prompt.prompt_info + else: + current_spec = create_versioned_prompt_spec(db_prompt=target_row) + current_litellm_params = current_spec.litellm_params + current_prompt_info = current_spec.prompt_info + # Update fields if provided updated_litellm_params = ( request.litellm_params if request.litellm_params is not None - else existing_prompt.litellm_params + else current_litellm_params ) updated_prompt_info = ( request.prompt_info if request.prompt_info is not None - else existing_prompt.prompt_info + else current_prompt_info ) # Ensure we have valid litellm_params if updated_litellm_params is None: raise HTTPException(status_code=400, detail="litellm_params cannot be None") - # Create updated prompt spec - cast to satisfy typing + # Build update data dict + update_data: Dict[str, Any] = { + "litellm_params": updated_litellm_params.model_dump_json(), + "prompt_info": updated_prompt_info.model_dump_json(), + } + if user_api_key_dict.user_id: + update_data["created_by"] = user_api_key_dict.user_id + + # Update by primary key (id) to target exactly one row updated_prompt_db_entry = await prisma_client.db.litellm_prompttable.update( - where={"prompt_id": prompt_id}, - data={ - "litellm_params": updated_litellm_params.model_dump_json(), - "prompt_info": updated_prompt_info.model_dump_json(), - }, + where={"id": target_row.id}, + data=update_data, ) - updated_prompt_spec = PromptSpec(**updated_prompt_db_entry.model_dump()) - - # Remove the old prompt from memory - del IN_MEMORY_PROMPT_REGISTRY.IN_MEMORY_PROMPTS[prompt_id] - if prompt_id in IN_MEMORY_PROMPT_REGISTRY.prompt_id_to_custom_prompt: - del IN_MEMORY_PROMPT_REGISTRY.prompt_id_to_custom_prompt[prompt_id] - - # Initialize the updated prompt - initialized_prompt = IN_MEMORY_PROMPT_REGISTRY.initialize_prompt( - prompt=updated_prompt_spec, config_file_path=None + updated_prompt_spec = create_versioned_prompt_spec( + db_prompt=updated_prompt_db_entry ) - if initialized_prompt is None: - raise HTTPException(status_code=500, detail="Failed to patch prompt") - - return initialized_prompt + return _reload_prompt_in_registry( + IN_MEMORY_PROMPT_REGISTRY, versioned_id, updated_prompt_spec + ) except HTTPException as e: raise e diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index d83e3500ebe..8a122a13506 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -690,7 +690,7 @@ _description = ( def cleanup_router_config_variables(): - global master_key, user_config_file_path, otel_logging, user_custom_auth, user_custom_auth_path, user_custom_key_generate, user_custom_sso, user_custom_ui_sso_sign_in_handler, use_background_health_checks, use_shared_health_check, health_check_interval, health_check_concurrency, prisma_client + global master_key, user_config_file_path, otel_logging, user_custom_auth, user_custom_auth_path, user_custom_key_generate, user_custom_key_update, user_custom_sso, user_custom_ui_sso_sign_in_handler, use_background_health_checks, use_shared_health_check, health_check_interval, health_check_concurrency, prisma_client # Set all variables to None master_key = None @@ -699,6 +699,7 @@ def cleanup_router_config_variables(): user_custom_auth = None user_custom_auth_path = None user_custom_key_generate = None + user_custom_key_update = None user_custom_sso = None user_custom_ui_sso_sign_in_handler = None use_background_health_checks = None @@ -709,7 +710,7 @@ def cleanup_router_config_variables(): async def proxy_shutdown_event(): - global prisma_client, master_key, user_custom_auth, user_custom_key_generate + global prisma_client, master_key, user_custom_auth, user_custom_key_generate, user_custom_key_update verbose_proxy_logger.info("Shutting down LiteLLM Proxy Server") if prisma_client: verbose_proxy_logger.debug("Disconnecting from Prisma") @@ -1564,6 +1565,7 @@ user_custom_key_generate = None # Tests that need to reset it can patch 'litellm.proxy.proxy_server._pkce_no_redis_warning_emitted'. _pkce_no_redis_warning_emitted: bool = False _cp_no_redis_warning_emitted: bool = False +user_custom_key_update = None user_custom_sso = None user_custom_ui_sso_sign_in_handler = None use_background_health_checks = None @@ -2935,7 +2937,7 @@ class ProxyConfig: """ Load config values into proxy global state """ - global master_key, user_config_file_path, otel_logging, user_custom_auth, user_custom_auth_path, user_custom_key_generate, user_custom_sso, user_custom_ui_sso_sign_in_handler, use_background_health_checks, use_shared_health_check, health_check_interval, health_check_concurrency, use_queue, proxy_budget_rescheduler_max_time, proxy_budget_rescheduler_min_time, ui_access_mode, litellm_master_key_hash, proxy_batch_write_at, disable_spend_logs, prompt_injection_detection_obj, redis_usage_cache, store_model_in_db, premium_user, open_telemetry_logger, health_check_details, proxy_batch_polling_interval, config_passthrough_endpoints + global master_key, user_config_file_path, otel_logging, user_custom_auth, user_custom_auth_path, user_custom_key_generate, user_custom_key_update, user_custom_sso, user_custom_ui_sso_sign_in_handler, use_background_health_checks, use_shared_health_check, health_check_interval, health_check_concurrency, use_queue, proxy_budget_rescheduler_max_time, proxy_budget_rescheduler_min_time, ui_access_mode, litellm_master_key_hash, proxy_batch_write_at, disable_spend_logs, prompt_injection_detection_obj, redis_usage_cache, store_model_in_db, premium_user, open_telemetry_logger, health_check_details, proxy_batch_polling_interval, config_passthrough_endpoints config: dict = await self.get_config(config_file_path=config_file_path) @@ -3356,6 +3358,12 @@ class ProxyConfig: value=custom_key_generate, config_file_path=config_file_path ) + custom_key_update = general_settings.get("custom_key_update", None) + if custom_key_update is not None: + user_custom_key_update = get_instance_fn( + value=custom_key_update, config_file_path=config_file_path + ) + custom_sso = general_settings.get("custom_sso", None) if custom_sso is not None: user_custom_sso = get_instance_fn( diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 46be6b31e1f..5d804d75b6a 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -1002,12 +1002,15 @@ model LiteLLM_PromptTable { id String @id @default(uuid()) prompt_id String version Int @default(1) + environment String @default("development") + created_by String? litellm_params Json prompt_info Json? created_at DateTime @default(now()) updated_at DateTime @updatedAt - @@unique([prompt_id, version]) + @@unique([prompt_id, version, environment]) + @@index([prompt_id, environment]) @@index([prompt_id]) } diff --git a/litellm/types/prompts/init_prompts.py b/litellm/types/prompts/init_prompts.py index 2d9f807bc26..838271a2f31 100644 --- a/litellm/types/prompts/init_prompts.py +++ b/litellm/types/prompts/init_prompts.py @@ -17,6 +17,7 @@ class SupportedPromptIntegrations(str, Enum): class PromptInfo(BaseModel): prompt_type: Literal["config", "db"] + environment: Optional[str] = "development" model_config = ConfigDict(extra="allow", protected_namespaces=()) @@ -48,6 +49,8 @@ class PromptSpec(BaseModel): created_at: Optional[datetime] = None updated_at: Optional[datetime] = None version: Optional[int] = None # Version number for version history + environment: Optional[str] = "development" + created_by: Optional[str] = None def __init__(self, **data): if "prompt_info" not in data: @@ -70,6 +73,9 @@ class PromptTemplateBase(BaseModel): class PromptInfoResponse(BaseModel): prompt_spec: PromptSpec raw_prompt_template: Optional[PromptTemplateBase] = None + environments: Optional[ + List[str] + ] = None # All environments this prompt is deployed to class ListPromptsResponse(BaseModel): diff --git a/litellm/utils.py b/litellm/utils.py index e2dac1c9f62..37bc35af299 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9030,11 +9030,11 @@ class ProviderConfigManager: return get_stability_image_edit_config(model) elif LlmProviders.BEDROCK == provider: - from litellm.llms.bedrock.image_edit.stability_transformation import ( - BedrockStabilityImageEditConfig, + from litellm.llms.bedrock.image_edit.amazon_nova_canvas_image_edit_transformation import ( + get_bedrock_image_edit_config_for_model, ) - return BedrockStabilityImageEditConfig() + return get_bedrock_image_edit_config_for_model(model) elif LlmProviders.OPENROUTER == provider: from litellm.llms.openrouter.image_edit import ( get_openrouter_image_edit_config, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 71918ed049b..35fa2206761 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -277,7 +277,15 @@ "litellm_provider": "bedrock", "max_input_tokens": 2600, "mode": "image_generation", - "output_cost_per_image": 0.06 + "output_cost_per_image": 0.06, + "supports_nova_canvas_image_edit": true + }, + "us.amazon.nova-canvas-v1:0": { + "litellm_provider": "bedrock", + "max_input_tokens": 2600, + "mode": "image_generation", + "output_cost_per_image": 0.06, + "supports_nova_canvas_image_edit": true }, "us.writer.palmyra-x4-v1:0": { "input_cost_per_token": 2.5e-06, diff --git a/schema.prisma b/schema.prisma index 46be6b31e1f..5d804d75b6a 100644 --- a/schema.prisma +++ b/schema.prisma @@ -1002,12 +1002,15 @@ model LiteLLM_PromptTable { id String @id @default(uuid()) prompt_id String version Int @default(1) + environment String @default("development") + created_by String? litellm_params Json prompt_info Json? created_at DateTime @default(now()) updated_at DateTime @updatedAt - @@unique([prompt_id, version]) + @@unique([prompt_id, version, environment]) + @@index([prompt_id, environment]) @@index([prompt_id]) } diff --git a/tests/test_litellm/llms/bedrock/image_edit/test_amazon_nova_canvas_image_edit.py b/tests/test_litellm/llms/bedrock/image_edit/test_amazon_nova_canvas_image_edit.py new file mode 100644 index 00000000000..020b8df1276 --- /dev/null +++ b/tests/test_litellm/llms/bedrock/image_edit/test_amazon_nova_canvas_image_edit.py @@ -0,0 +1,657 @@ +"""Unit tests for Bedrock Amazon Nova Canvas image edit (issue #24267).""" + +import base64 +import io +from typing import cast + +import httpx +import pytest + +import litellm +from litellm.llms.bedrock.image_edit.amazon_nova_canvas_image_edit_transformation import ( + BedrockAmazonNovaCanvasImageEditConfig, + get_bedrock_image_edit_config_for_model, +) +from litellm.llms.bedrock.image_edit.handler import BedrockImageEdit +from litellm.llms.bedrock.image_edit.stability_transformation import ( + BedrockStabilityImageEditConfig, +) +from litellm.types.images.main import ImageEditOptionalRequestParams + + +@pytest.fixture(autouse=True) +def ensure_nova_canvas_image_edit_model_cost_flags(monkeypatch): + """Routing uses ``supports_nova_canvas_image_edit`` on ``litellm.model_cost``. + + Full ``model_prices_and_context_window.json`` includes these flags, but CI or + alternate cost maps may omit them—merge minimal entries so tests match production. + """ + from litellm.utils import _invalidate_model_cost_lowercase_map + + for key in ( + "amazon.nova-canvas-v1:0", + "us.amazon.nova-canvas-v1:0", + ): + entry = litellm.model_cost.get(key) or {} + if entry.get("supports_nova_canvas_image_edit") is True: + continue + monkeypatch.setitem( + litellm.model_cost, + key, + { + **entry, + "litellm_provider": entry.get("litellm_provider", "bedrock"), + "mode": entry.get("mode", "image_generation"), + "supports_nova_canvas_image_edit": True, + }, + ) + _invalidate_model_cost_lowercase_map() + + yield + + _invalidate_model_cost_lowercase_map() + + +def test_get_config_class_nova_canvas(): + """Nova Canvas model resolves to BedrockAmazonNovaCanvasImageEditConfig.""" + cls = BedrockImageEdit.get_config_class("amazon.nova-canvas-v1:0") + assert cls is BedrockAmazonNovaCanvasImageEditConfig + + +def test_get_config_class_us_cross_region_nova_canvas(): + """Cross-region inference id us.amazon.nova-canvas-v1:0 maps via model_prices.""" + cls = BedrockImageEdit.get_config_class("us.amazon.nova-canvas-v1:0") + assert cls is BedrockAmazonNovaCanvasImageEditConfig + + +def test_get_config_class_stability_unchanged(): + """Stability edit models still use stability config.""" + cls = BedrockImageEdit.get_config_class( + "stability.stable-image-inpaint-v1:0", + ) + assert cls is BedrockStabilityImageEditConfig + + +def test_provider_config_router_returns_nova_for_canvas(): + """ProviderConfigManager routes Nova Canvas to Nova image-edit config.""" + cfg = get_bedrock_image_edit_config_for_model("amazon.nova-canvas-v1:0") + assert isinstance(cfg, BedrockAmazonNovaCanvasImageEditConfig) + + +def test_provider_config_router_returns_stability_for_sd(): + """Non-Nova Bedrock image edit still uses Stability config.""" + cfg = get_bedrock_image_edit_config_for_model( + "stability.stable-image-inpaint-v1:0", + ) + assert isinstance(cfg, BedrockStabilityImageEditConfig) + + +def test_get_bedrock_helper_matches_handler_get_config_class(): + """Handler and get_bedrock_image_edit_config_for_model must agree.""" + for model in ( + "amazon.nova-canvas-v1:0", + "stability.stable-image-inpaint-v1:0", + ): + handler_cls = BedrockImageEdit.get_config_class(model) + helper_cfg = get_bedrock_image_edit_config_for_model(model) + assert isinstance(helper_cfg, handler_cls) + + +def test_get_config_class_unknown_bedrock_image_model_raises(): + with pytest.raises(ValueError, match="Unsupported Bedrock image-edit model"): + BedrockImageEdit.get_config_class("amazon.titan-image-generator-v1") + + +def test_get_bedrock_image_edit_config_unknown_raises(): + with pytest.raises(ValueError, match="Unsupported Bedrock image-edit model"): + get_bedrock_image_edit_config_for_model("amazon.titan-image-generator-v1") + + +def test_provider_config_manager_bedrock_nova_canvas(): + """ProviderConfigManager.get_provider_image_edit_config matches handler routing.""" + from litellm.utils import ProviderConfigManager + + cfg = ProviderConfigManager.get_provider_image_edit_config( + "amazon.nova-canvas-v1:0", + litellm.LlmProviders.BEDROCK, + ) + assert isinstance(cfg, BedrockAmazonNovaCanvasImageEditConfig) + + +def test_provider_config_manager_bedrock_stability_inpaint(): + """ProviderConfigManager returns Stability config for Stability edit models.""" + from litellm.utils import ProviderConfigManager + + cfg = ProviderConfigManager.get_provider_image_edit_config( + "stability.stable-image-inpaint-v1:0", + litellm.LlmProviders.BEDROCK, + ) + assert isinstance(cfg, BedrockStabilityImageEditConfig) + + +def test_provider_config_manager_bedrock_unknown_raises(): + from litellm.utils import ProviderConfigManager + + with pytest.raises(ValueError, match="Unsupported Bedrock image-edit model"): + ProviderConfigManager.get_provider_image_edit_config( + "amazon.titan-image-generator-v1", + litellm.LlmProviders.BEDROCK, + ) + + +def test_provider_config_manager_bedrock_dispatches_to_nova_transform_outpainting(): + """ + Full dispatch: utils.ProviderConfigManager -> get_bedrock_image_edit_config_for_model + -> Nova config.transform_image_edit_request (not only direct helper calls). + """ + from litellm.utils import ProviderConfigManager + + cfg = ProviderConfigManager.get_provider_image_edit_config( + "amazon.nova-canvas-v1:0", + litellm.LlmProviders.BEDROCK, + ) + assert cfg is not None + img = io.BytesIO(b"scene") + mask = io.BytesIO(b"mask-bytes") + body, _ = cfg.transform_image_edit_request( + model="amazon.nova-canvas-v1:0", + prompt="expand left", + image=img, + image_edit_optional_request_params={ + "taskType": "OUTPAINTING", + "mask": mask, + }, + litellm_params={}, # type: ignore[arg-type] + headers={}, + ) + assert body["taskType"] == "OUTPAINTING" + assert "maskImage" in body["outPaintingParams"] + + +def test_transform_request_image_variation_without_mask(): + """No mask -> IMAGE_VARIATION with images + text.""" + config = BedrockAmazonNovaCanvasImageEditConfig() + img = io.BytesIO(b"fake-png") + body, _ = config.transform_image_edit_request( + model="amazon.nova-canvas-v1:0", + prompt="make it warmer", + image=img, + image_edit_optional_request_params={}, + litellm_params={}, # type: ignore[arg-type] + headers={}, + ) + assert body["taskType"] == "IMAGE_VARIATION" + assert body["imageVariationParams"]["text"] == "make it warmer" + assert len(body["imageVariationParams"]["images"]) == 1 + + +def test_transform_request_image_pathlike_input(tmp_path): + """PathLike image input should be read and base64-encoded.""" + config = BedrockAmazonNovaCanvasImageEditConfig() + image_path = tmp_path / "img.bin" + image_bytes = b"pathlike-image-bytes" + image_path.write_bytes(image_bytes) + + body, _ = config.transform_image_edit_request( + model="amazon.nova-canvas-v1:0", + prompt="pathlike", + image=image_path, + image_edit_optional_request_params={}, + litellm_params={}, # type: ignore[arg-type] + headers={}, + ) + + assert body["taskType"] == "IMAGE_VARIATION" + assert body["imageVariationParams"]["images"][0] == base64.b64encode( + image_bytes + ).decode("utf-8") + + +def test_transform_request_inpainting_with_mask(): + """Mask present -> INPAINTING with inPaintingParams (AWS field names).""" + config = BedrockAmazonNovaCanvasImageEditConfig() + main = io.BytesIO(b"img-bytes") + mask = io.BytesIO(b"mask-bytes") + body, _ = config.transform_image_edit_request( + model="amazon.nova-canvas-v1:0", + prompt="add a hat", + image=main, + image_edit_optional_request_params={"mask": mask}, + litellm_params={}, # type: ignore[arg-type] + headers={}, + ) + assert body["taskType"] == "INPAINTING" + ip = body["inPaintingParams"] + assert ip["text"] == "add a hat" + assert "maskImage" in ip + assert ip["image"] # base64 + + +def test_transform_request_explicit_image_variation_with_mask_honors_task_type(): + """Explicit taskType=IMAGE_VARIATION must not be overridden by mask presence.""" + config = BedrockAmazonNovaCanvasImageEditConfig() + main = io.BytesIO(b"img-bytes") + mask = io.BytesIO(b"mask-bytes") + body, _ = config.transform_image_edit_request( + model="amazon.nova-canvas-v1:0", + prompt="vary style", + image=main, + image_edit_optional_request_params={ + "taskType": "IMAGE_VARIATION", + "mask": mask, + }, + litellm_params={}, # type: ignore[arg-type] + headers={}, + ) + assert body["taskType"] == "IMAGE_VARIATION" + assert "imageVariationParams" in body + assert body["imageVariationParams"]["text"] == "vary style" + assert len(body["imageVariationParams"]["images"]) == 1 + + +def test_transform_request_outpainting_with_mask(): + """OUTPAINTING with OpenAI mask -> outPaintingParams.maskImage.""" + config = BedrockAmazonNovaCanvasImageEditConfig() + main = io.BytesIO(b"img") + mask = io.BytesIO(b"mask") + body, _ = config.transform_image_edit_request( + model="amazon.nova-canvas-v1:0", + prompt="extend the sky", + image=main, + image_edit_optional_request_params={ + "taskType": "OUTPAINTING", + "mask": mask, + }, + litellm_params={}, # type: ignore[arg-type] + headers={}, + ) + assert body["taskType"] == "OUTPAINTING" + assert "maskImage" in body["outPaintingParams"] + assert body["outPaintingParams"]["text"] == "extend the sky" + + +def test_transform_request_outpainting_with_mask_prompt(): + """OUTPAINTING with maskPrompt only (no binary mask).""" + config = BedrockAmazonNovaCanvasImageEditConfig() + img = io.BytesIO(b"img") + body, _ = config.transform_image_edit_request( + model="amazon.nova-canvas-v1:0", + prompt="new background", + image=img, + image_edit_optional_request_params={ + "taskType": "OUTPAINTING", + "maskPrompt": "the area behind the subject", + }, + litellm_params={}, # type: ignore[arg-type] + headers={}, + ) + assert body["taskType"] == "OUTPAINTING" + assert body["outPaintingParams"]["maskPrompt"] == "the area behind the subject" + + +def test_transform_request_outpainting_prefers_mask_prompt_over_binary_mask(): + """OUTPAINTING chooses maskPrompt over maskImage when both are set (_nova_canvas_task_body).""" + config = BedrockAmazonNovaCanvasImageEditConfig() + main = io.BytesIO(b"img") + mask = io.BytesIO(b"mask") + body, _ = config.transform_image_edit_request( + model="amazon.nova-canvas-v1:0", + prompt="extend scene", + image=main, + image_edit_optional_request_params={ + "taskType": "OUTPAINTING", + "mask": mask, + "maskPrompt": "sky region", + }, + litellm_params={}, # type: ignore[arg-type] + headers={}, + ) + assert body["taskType"] == "OUTPAINTING" + op = body["outPaintingParams"] + assert op["maskPrompt"] == "sky region" + assert "maskImage" not in op + + +def test_transform_request_outpainting_with_out_painting_mode(): + """OUTPAINTING forwards outPaintingMode into outPaintingParams.""" + config = BedrockAmazonNovaCanvasImageEditConfig() + img = io.BytesIO(b"img") + mask = io.BytesIO(b"m") + body, _ = config.transform_image_edit_request( + model="amazon.nova-canvas-v1:0", + prompt="widen", + image=img, + image_edit_optional_request_params={ + "taskType": "OUTPAINTING", + "mask": mask, + "outPaintingMode": "PRECISE", + }, + litellm_params={}, # type: ignore[arg-type] + headers={}, + ) + assert body["taskType"] == "OUTPAINTING" + assert body["outPaintingParams"]["outPaintingMode"] == "PRECISE" + + +def test_get_supported_openai_params_includes_outpainting_fields(): + """Documented OUTPAINTING-related optional params are advertised for routing/UI.""" + config = BedrockAmazonNovaCanvasImageEditConfig() + supported = config.get_supported_openai_params("amazon.nova-canvas-v1:0") + assert "taskType" in supported + assert "maskPrompt" in supported + assert "outPaintingMode" in supported + assert "mask" in supported + + +def test_transform_request_outpainting_without_mask_raises(): + """OUTPAINTING without maskPrompt or maskImage must fail fast with a clear error.""" + config = BedrockAmazonNovaCanvasImageEditConfig() + img = io.BytesIO(b"img") + with pytest.raises( + ValueError, + match="OUTPAINTING requires either a mask image or a mask prompt", + ): + config.transform_image_edit_request( + model="amazon.nova-canvas-v1:0", + prompt="extend", + image=img, + image_edit_optional_request_params={"taskType": "OUTPAINTING"}, + litellm_params={}, # type: ignore[arg-type] + headers={}, + ) + + +def test_transform_request_inpainting_explicit_task_without_mask_raises(): + """INPAINTING taskType without mask or maskPrompt must fail fast.""" + config = BedrockAmazonNovaCanvasImageEditConfig() + img = io.BytesIO(b"img") + with pytest.raises( + ValueError, match="INPAINTING requires either maskPrompt or maskImage" + ): + config.transform_image_edit_request( + model="amazon.nova-canvas-v1:0", + prompt="fix it", + image=img, + image_edit_optional_request_params={"taskType": "INPAINTING"}, + litellm_params={}, # type: ignore[arg-type] + headers={}, + ) + + +def test_transform_request_unknown_task_type_raises(): + """Unknown taskType must not silently map to IMAGE_VARIATION or INPAINTING.""" + config = BedrockAmazonNovaCanvasImageEditConfig() + img = io.BytesIO(b"img") + with pytest.raises(ValueError, match="Unsupported Amazon Nova Canvas taskType"): + config.transform_image_edit_request( + model="amazon.nova-canvas-v1:0", + prompt="x", + image=img, + image_edit_optional_request_params={"taskType": "TEXT_IMAGE"}, + litellm_params={}, # type: ignore[arg-type] + headers={}, + ) + + +def test_transform_request_background_removal(): + """taskType BACKGROUND_REMOVAL builds minimal body.""" + config = BedrockAmazonNovaCanvasImageEditConfig() + img = io.BytesIO(b"x") + body, _ = config.transform_image_edit_request( + model="amazon.nova-canvas-v1:0", + prompt="ignored", + image=img, + image_edit_optional_request_params={"taskType": "BACKGROUND_REMOVAL"}, + litellm_params={}, # type: ignore[arg-type] + headers={}, + ) + assert body["taskType"] == "BACKGROUND_REMOVAL" + assert "image" in body["backgroundRemovalParams"] + + +def test_transform_request_background_removal_omits_image_generation_config(): + """AWS Nova Canvas does not allow imageGenerationConfig on BACKGROUND_REMOVAL.""" + config = BedrockAmazonNovaCanvasImageEditConfig() + img = io.BytesIO(b"x") + body, _ = config.transform_image_edit_request( + model="amazon.nova-canvas-v1:0", + prompt="ignored", + image=img, + image_edit_optional_request_params={ + "taskType": "BACKGROUND_REMOVAL", + "size": "512x512", + "seed": 42, + "quality": "standard", + "cfgScale": 7.5, + "n": 2, + }, + litellm_params={}, # type: ignore[arg-type] + headers={}, + ) + assert body["taskType"] == "BACKGROUND_REMOVAL" + assert "imageGenerationConfig" not in body + + +def test_transform_request_image_variation_includes_image_generation_config(): + """Non-BACKGROUND_REMOVAL tasks may include imageGenerationConfig when params are set.""" + config = BedrockAmazonNovaCanvasImageEditConfig() + img = io.BytesIO(b"x") + body, _ = config.transform_image_edit_request( + model="amazon.nova-canvas-v1:0", + prompt="warm", + image=img, + image_edit_optional_request_params={"size": "1024x1024", "seed": 1}, + litellm_params={}, # type: ignore[arg-type] + headers={}, + ) + assert body["taskType"] == "IMAGE_VARIATION" + assert "imageGenerationConfig" in body + assert body["imageGenerationConfig"]["width"] == 1024 + assert body["imageGenerationConfig"]["height"] == 1024 + assert body["imageGenerationConfig"]["seed"] == 1 + + +def test_map_openai_params_unknown_quality_not_silently_dropped(): + """Non-Nova quality strings (e.g. OpenAI 'auto') must remain for downstream handling.""" + config = BedrockAmazonNovaCanvasImageEditConfig() + mapped = config.map_openai_params( + cast(ImageEditOptionalRequestParams, {"quality": "auto"}), + model="amazon.nova-canvas-v1:0", + drop_params=False, + ) + assert mapped.get("quality") == "auto" + + +def test_transform_request_unknown_quality_reaches_image_generation_config(): + """Unknown quality after map_openai_params is forwarded so callers are not silently ignored.""" + config = BedrockAmazonNovaCanvasImageEditConfig() + img = io.BytesIO(b"x") + op = config.map_openai_params( + cast(ImageEditOptionalRequestParams, {"quality": "auto"}), + model="amazon.nova-canvas-v1:0", + drop_params=False, + ) + body, _ = config.transform_image_edit_request( + model="amazon.nova-canvas-v1:0", + prompt="x", + image=img, + image_edit_optional_request_params=op, + litellm_params={}, # type: ignore[arg-type] + headers={}, + ) + assert body["imageGenerationConfig"]["quality"] == "auto" + + +def test_is_nova_canvas_image_edit_model_uses_model_cost_flag(monkeypatch): + """Routing uses supports_nova_canvas_image_edit in model_cost, not a hardcoded name substring.""" + fake_id = "amazon.custom-bedrock-image-edit-v99:0" + monkeypatch.setitem( + litellm.model_cost, + fake_id, + { + "litellm_provider": "bedrock", + "mode": "image_generation", + "supports_nova_canvas_image_edit": True, + }, + ) + assert ( + BedrockAmazonNovaCanvasImageEditConfig._is_nova_canvas_image_edit_model(fake_id) + is True + ) + + monkeypatch.setitem( + litellm.model_cost, + "amazon.not-nova-canvas-v1:0", + { + "litellm_provider": "bedrock", + "mode": "image_generation", + }, + ) + assert ( + BedrockAmazonNovaCanvasImageEditConfig._is_nova_canvas_image_edit_model( + "amazon.not-nova-canvas-v1:0" + ) + is False + ) + + # Name-shaped ids do not route without supports_nova_canvas_image_edit (no substring heuristic). + monkeypatch.setitem( + litellm.model_cost, + "amazon.nova-canvas-v2:0", + { + "litellm_provider": "bedrock", + "mode": "image_generation", + }, + ) + assert ( + BedrockAmazonNovaCanvasImageEditConfig._is_nova_canvas_image_edit_model( + "amazon.nova-canvas-v2:0" + ) + is False + ) + + +def test_transform_response_to_openai_format(): + """Response maps images[] to ImageResponse.data b64_json.""" + config = BedrockAmazonNovaCanvasImageEditConfig() + resp = httpx.Response( + 200, + json={"images": ["YmFzZTY0X2E=", "YmFzZTY0X2I="]}, + ) + model_response = config.transform_image_edit_response( + model="amazon.nova-canvas-v1:0", + raw_response=resp, + logging_obj=None, # type: ignore[arg-type] + ) + assert model_response.data is not None + assert len(model_response.data) == 2 + assert model_response.data[0].b64_json == "YmFzZTY0X2E=" + + +def test_transform_response_non_200_raises(): + """HTTP 4xx/5xx with JSON body surfaces a structured error.""" + config = BedrockAmazonNovaCanvasImageEditConfig() + resp = httpx.Response( + 400, + json={"message": "ValidationException: invalid input"}, + ) + with pytest.raises(Exception, match="Nova Canvas image edit error"): + config.transform_image_edit_response( + model="amazon.nova-canvas-v1:0", + raw_response=resp, + logging_obj=None, # type: ignore[arg-type] + ) + + +def test_transform_response_errors_field_raises(): + """Align with Bedrock Stability: top-level ``errors`` in body.""" + config = BedrockAmazonNovaCanvasImageEditConfig() + resp = httpx.Response( + 200, + json={"errors": ["upstream failure"]}, + ) + with pytest.raises(Exception, match="Nova Canvas image edit error"): + config.transform_image_edit_response( + model="amazon.nova-canvas-v1:0", + raw_response=resp, + logging_obj=None, # type: ignore[arg-type] + ) + + +def test_transform_response_allows_errors_field_with_images(): + """Do not treat ``errors`` as fatal when ``images`` is present.""" + config = BedrockAmazonNovaCanvasImageEditConfig() + resp = httpx.Response( + 200, + json={"errors": ["non-fatal warning"], "images": ["YmFzZTY0X2E="]}, + ) + model_response = config.transform_image_edit_response( + model="amazon.nova-canvas-v1:0", + raw_response=resp, + logging_obj=None, # type: ignore[arg-type] + ) + assert model_response.data is not None + assert len(model_response.data) == 1 + assert model_response.data[0].b64_json == "YmFzZTY0X2E=" + + +def test_transform_response_message_or_error_field_raises(): + """API-level error payload when there are no images (status 200).""" + config = BedrockAmazonNovaCanvasImageEditConfig() + resp = httpx.Response( + 200, + json={"message": "ValidationException: task rejected"}, + ) + with pytest.raises(Exception, match="Nova Canvas image edit error"): + config.transform_image_edit_response( + model="amazon.nova-canvas-v1:0", + raw_response=resp, + logging_obj=None, # type: ignore[arg-type] + ) + + +def test_transform_response_allows_informational_message_with_images(): + """Do not treat ``message`` as fatal when ``images`` is present (SDK wrappers).""" + config = BedrockAmazonNovaCanvasImageEditConfig() + resp = httpx.Response( + 200, + json={ + "message": "ok", + "images": ["YmFzZTY0X2E="], + }, + ) + model_response = config.transform_image_edit_response( + model="amazon.nova-canvas-v1:0", + raw_response=resp, + logging_obj=None, # type: ignore[arg-type] + ) + assert model_response.data is not None + assert len(model_response.data) == 1 + assert model_response.data[0].b64_json == "YmFzZTY0X2E=" + + +def test_transform_response_content_filtered_via_error_field(): + """AWS Nova Canvas signals failures (e.g. content filter) via top-level ``error``, not ``finish_reasons``.""" + config = BedrockAmazonNovaCanvasImageEditConfig() + resp = httpx.Response( + 200, + json={"images": [], "error": "CONTENT_FILTERED"}, + ) + with pytest.raises(Exception, match="Nova Canvas image edit error"): + config.transform_image_edit_response( + model="amazon.nova-canvas-v1:0", + raw_response=resp, + logging_obj=None, # type: ignore[arg-type] + ) + + +def test_transform_response_empty_images_without_error_raises(): + """200 with empty ``images`` and no error fields must not return silent empty ImageResponse.""" + config = BedrockAmazonNovaCanvasImageEditConfig() + resp = httpx.Response(200, json={"images": []}) + with pytest.raises(Exception, match="returned no images"): + config.transform_image_edit_response( + model="amazon.nova-canvas-v1:0", + raw_response=resp, + logging_obj=None, # type: ignore[arg-type] + ) diff --git a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py index f69f478278f..ea208007cd6 100644 --- a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py +++ b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py @@ -34,7 +34,9 @@ async def test_bedrock_sse_wrapper_encodes_dict_chunks(): _dummy_stream(), litellm_logging_obj=LiteLLMLoggingObj( model="bedrock/invoke/anthropic.claude-3-sonnet-20240229-v1:0", - messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}], + messages=[ + {"role": "user", "content": "Hello, can you tell me a short joke?"} + ], stream=True, call_type="chat", start_time=datetime.now(), @@ -58,6 +60,74 @@ async def test_bedrock_sse_wrapper_encodes_dict_chunks(): assert collected[1] == b"raw-bytes" +@pytest.mark.asyncio +async def test_bedrock_sse_wrapper_keeps_usage_in_message_start_and_message_delta(): + """Regression test: usage should be available on both message_start and message_delta SSE events.""" + + cfg = AmazonAnthropicClaudeMessagesConfig() + + async def _dummy_stream(): # type: ignore[return-type] + yield { + "type": "message_start", + "message": { + "id": "msg_123", + "type": "message", + "role": "assistant", + "content": [], + "usage": { + "input_tokens": 3, + "output_tokens": 1, + }, + }, + } + yield { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": { + "input_tokens": 3, + "output_tokens": 8, + }, + } + yield { + "type": "message_stop", + "usage": { + "input_tokens": 3, + "cache_creation_input_tokens": 1562, + "cache_read_input_tokens": 32392, + }, + } + + collected: list[bytes] = [] + async for chunk in cfg.bedrock_sse_wrapper( + _dummy_stream(), + litellm_logging_obj=LiteLLMLoggingObj( + model="bedrock/invoke/anthropic.claude-3-sonnet-20240229-v1:0", + messages=[{"role": "user", "content": "Hello"}], + stream=True, + call_type="chat", + start_time=datetime.now(), + litellm_call_id="test_bedrock_sse_wrapper_keeps_usage_in_both_events", + function_id="test_bedrock_sse_wrapper_keeps_usage_in_both_events", + ), + request_body={}, + ): + collected.append(chunk) + + start_chunk = next(c for c in collected if b"event: message_start\n" in c) + delta_chunk = next(c for c in collected if b"event: message_delta\n" in c) + + start_json = json.loads(start_chunk.decode("utf-8").split("data: ", 1)[1].strip()) + delta_json = json.loads(delta_chunk.decode("utf-8").split("data: ", 1)[1].strip()) + + assert "usage" in start_json["message"] + assert start_json["message"]["usage"]["input_tokens"] == 3 + + assert "usage" in delta_json + assert delta_json["usage"]["cache_creation_input_tokens"] == 1562 + assert delta_json["usage"]["cache_read_input_tokens"] == 32392 + assert delta_json["usage"]["input_tokens"] == 3 + 1562 + 32392 + + def test_chunk_parser_usage_transformation(): """Ensure Bedrock invocation metrics are transformed to Anthropic usage keys.""" @@ -96,12 +166,9 @@ def test_remove_ttl_from_cache_control(): { "type": "text", "text": "Hello", - "cache_control": { - "type": "ephemeral", - "ttl": "1h" - } + "cache_control": {"type": "ephemeral", "ttl": "1h"}, } - ] + ], } ] } @@ -122,20 +189,14 @@ def test_remove_ttl_from_cache_control(): { "type": "text", "text": "Hello", - "cache_control": { - "type": "ephemeral", - "ttl": "1h" - } + "cache_control": {"type": "ephemeral", "ttl": "1h"}, }, { "type": "text", "text": "World", - "cache_control": { - "type": "ephemeral", - "ttl": "2h" - } - } - ] + "cache_control": {"type": "ephemeral", "ttl": "2h"}, + }, + ], } ] } @@ -156,11 +217,9 @@ def test_remove_ttl_from_cache_control(): { "type": "text", "text": "Hello", - "cache_control": { - "type": "ephemeral" - } + "cache_control": {"type": "ephemeral"}, } - ] + ], } ] } @@ -232,6 +291,7 @@ def test_remove_custom_field_from_tools(): remove_custom_field_from_tools(request4) assert request4["tools"] is None + def test_remove_scope_from_cache_control(): """Ensure scope field is removed from cache_control for Bedrock (not supported).""" @@ -303,9 +363,9 @@ def test_bedrock_messages_strips_output_config(): headers={}, ) - assert "output_config" not in result, ( - "output_config should be stripped — Bedrock Invoke rejects it" - ) + assert ( + "output_config" not in result + ), "output_config should be stripped — Bedrock Invoke rejects it" # Other params should be preserved assert result.get("max_tokens") == 4096 diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py index 97ca55073df..7bd87ed05d7 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py @@ -572,7 +572,7 @@ async def test_model_armor_streaming_block_yields_sse_error(): assert len(result_chunks) == 1 error_data = json.loads(result_chunks[0].removeprefix("data: ")) assert "error" in error_data - assert error_data["error"]["code"] == 400 + assert int(error_data["error"]["code"]) == 400 @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py index d269a9531fd..d35dbb87a1a 100644 --- a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py @@ -479,3 +479,67 @@ async def test_track_cost_callback_skips_for_falsy_model_and_no_slo(model_value) ) mock_proxy_logging.failed_tracking_alert.assert_not_called() + + +@pytest.mark.asyncio +async def test_async_post_call_failure_hook_uses_actual_start_time(): + """ + Verify that failed requests record the actual request start time + instead of datetime.now(), so the spend log shows the real duration. + + Previously both start_time and end_time were set to datetime.now() + at failure-logging time, resulting in duration=0 for all failures. + """ + from datetime import timedelta + + logger = _ProxyDBLogger() + + user_api_key_dict = UserAPIKeyAuth( + api_key="test_api_key", + user_id="test_user_id", + team_id="test_team_id", + org_id="test_org_id", + end_user_id="test_end_user_id", + ) + + # Simulate a request that started 60 seconds ago + simulated_start = datetime.now() - timedelta(seconds=60) + + mock_logging_obj = MagicMock() + mock_logging_obj.start_time = simulated_start + mock_logging_obj.model_call_details = {} + mock_logging_obj.litellm_trace_id = None + + request_data = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "Hello"}], + "metadata": {}, + "proxy_server_request": {}, + "litellm_logging_obj": mock_logging_obj, + } + + original_exception = Exception("Timeout error") + + with patch( + "litellm.proxy.db.db_spend_update_writer.DBSpendUpdateWriter.update_database", + new_callable=AsyncMock, + ) as mock_update_database: + await logger.async_post_call_failure_hook( + request_data=request_data, + original_exception=original_exception, + user_api_key_dict=user_api_key_dict, + ) + + mock_update_database.assert_called_once() + call_args = mock_update_database.call_args[1] + + # start_time should be the simulated start, not datetime.now() + assert call_args["start_time"] == simulated_start + + # end_time should be close to now (within a few seconds) + time_diff = (datetime.now() - call_args["end_time"]).total_seconds() + assert time_diff < 5, f"end_time should be close to now, was {time_diff}s ago" + + # Duration should be approximately 60 seconds, not 0 + duration = (call_args["end_time"] - call_args["start_time"]).total_seconds() + assert duration >= 55, f"Duration should be ~60s, got {duration}s" diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 0ff276953c6..72092a97f5b 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -33,6 +33,7 @@ from litellm.proxy.management_endpoints.key_management_endpoints import ( _check_org_key_limits, _check_team_key_limits, _common_key_generation_helper, + _enforce_upperbound_key_params, _get_and_validate_existing_key, _list_key_helper, _persist_deleted_verification_tokens, @@ -8435,3 +8436,146 @@ class TestKeyAliasSkipValidationOnUnchanged: # None alias should always pass _validate_key_alias_format(None) + + +# --- Tests: _enforce_upperbound_key_params --- + + +def test_enforce_upperbound_rejects_over_limit_on_generate(): + """Test that key generation is rejected when values exceed upperbound.""" + import litellm + from litellm.types.proxy.management_endpoints.ui_sso import ( + LiteLLM_UpperboundKeyGenerateParams, + ) + + original = litellm.upperbound_key_generate_params + try: + litellm.upperbound_key_generate_params = LiteLLM_UpperboundKeyGenerateParams( + tpm_limit=1000, rpm_limit=100, max_budget=10.0 + ) + data = GenerateKeyRequest(tpm_limit=5000) + with pytest.raises(HTTPException) as exc_info: + _enforce_upperbound_key_params(data, fill_defaults=True) + assert exc_info.value.status_code == 400 + assert "tpm_limit" in str(exc_info.value.detail) + finally: + litellm.upperbound_key_generate_params = original + + +def test_enforce_upperbound_fills_defaults_on_generate(): + """Test that None values are filled with upperbound defaults during generation.""" + import litellm + from litellm.types.proxy.management_endpoints.ui_sso import ( + LiteLLM_UpperboundKeyGenerateParams, + ) + + original = litellm.upperbound_key_generate_params + try: + litellm.upperbound_key_generate_params = LiteLLM_UpperboundKeyGenerateParams( + tpm_limit=1000, rpm_limit=100 + ) + data = GenerateKeyRequest() # tpm_limit=None, rpm_limit=None + _enforce_upperbound_key_params(data, fill_defaults=True) + assert data.tpm_limit == 1000 + assert data.rpm_limit == 100 + finally: + litellm.upperbound_key_generate_params = original + + +def test_enforce_upperbound_skips_none_on_update(): + """Test that None values are NOT filled during update (fill_defaults=False).""" + import litellm + from litellm.types.proxy.management_endpoints.ui_sso import ( + LiteLLM_UpperboundKeyGenerateParams, + ) + + original = litellm.upperbound_key_generate_params + try: + litellm.upperbound_key_generate_params = LiteLLM_UpperboundKeyGenerateParams( + tpm_limit=1000, rpm_limit=100 + ) + data = UpdateKeyRequest(key="sk-test") # tpm_limit=None, rpm_limit=None + _enforce_upperbound_key_params(data, fill_defaults=False) + assert data.tpm_limit is None # should NOT be filled + assert data.rpm_limit is None # should NOT be filled + finally: + litellm.upperbound_key_generate_params = original + + +def test_enforce_upperbound_rejects_over_limit_on_update(): + """Test that key update is rejected when values exceed upperbound.""" + import litellm + from litellm.types.proxy.management_endpoints.ui_sso import ( + LiteLLM_UpperboundKeyGenerateParams, + ) + + original = litellm.upperbound_key_generate_params + try: + litellm.upperbound_key_generate_params = LiteLLM_UpperboundKeyGenerateParams( + tpm_limit=1000, rpm_limit=100, max_budget=10.0 + ) + data = UpdateKeyRequest(key="sk-test", tpm_limit=5000) + with pytest.raises(HTTPException) as exc_info: + _enforce_upperbound_key_params(data, fill_defaults=False) + assert exc_info.value.status_code == 400 + assert "tpm_limit" in str(exc_info.value.detail) + finally: + litellm.upperbound_key_generate_params = original + + +def test_enforce_upperbound_allows_within_limit_on_update(): + """Test that key update passes when values are within upperbound.""" + import litellm + from litellm.types.proxy.management_endpoints.ui_sso import ( + LiteLLM_UpperboundKeyGenerateParams, + ) + + original = litellm.upperbound_key_generate_params + try: + litellm.upperbound_key_generate_params = LiteLLM_UpperboundKeyGenerateParams( + tpm_limit=1000, rpm_limit=100, max_budget=10.0 + ) + data = UpdateKeyRequest(key="sk-test", tpm_limit=500, rpm_limit=50, max_budget=5.0) + _enforce_upperbound_key_params(data, fill_defaults=False) + # Should not raise + assert data.tpm_limit == 500 + assert data.rpm_limit == 50 + assert data.max_budget == 5.0 + finally: + litellm.upperbound_key_generate_params = original + + +def test_enforce_upperbound_duration_over_limit(): + """Test that duration exceeding upperbound is rejected.""" + import litellm + from litellm.types.proxy.management_endpoints.ui_sso import ( + LiteLLM_UpperboundKeyGenerateParams, + ) + + original = litellm.upperbound_key_generate_params + try: + litellm.upperbound_key_generate_params = LiteLLM_UpperboundKeyGenerateParams( + duration="7d" + ) + data = UpdateKeyRequest(key="sk-test", duration="30d") + with pytest.raises(HTTPException) as exc_info: + _enforce_upperbound_key_params(data, fill_defaults=False) + assert exc_info.value.status_code == 400 + assert "duration" in str(exc_info.value.detail) + finally: + litellm.upperbound_key_generate_params = original + + +def test_enforce_upperbound_no_config_is_noop(): + """Test that no enforcement happens when upperbound params are not configured.""" + import litellm + + original = litellm.upperbound_key_generate_params + try: + litellm.upperbound_key_generate_params = None + data = UpdateKeyRequest(key="sk-test", tpm_limit=999999) + _enforce_upperbound_key_params(data, fill_defaults=False) + # Should not raise — no enforcement configured + assert data.tpm_limit == 999999 + finally: + litellm.upperbound_key_generate_params = original diff --git a/tests/test_litellm/proxy/prompts/test_prompt_environment.py b/tests/test_litellm/proxy/prompts/test_prompt_environment.py new file mode 100644 index 00000000000..ecd89afefbe --- /dev/null +++ b/tests/test_litellm/proxy/prompts/test_prompt_environment.py @@ -0,0 +1,253 @@ +import json +import pytest +from unittest.mock import MagicMock +from litellm.types.prompts.init_prompts import ( + PromptInfo, + PromptSpec, + PromptLiteLLMParams, +) + + +def test_prompt_info_default_environment(): + """PromptInfo should default environment to 'development'.""" + info = PromptInfo(prompt_type="db") + assert info.environment == "development" + + +def test_prompt_info_custom_environment(): + """PromptInfo should accept a custom environment.""" + info = PromptInfo(prompt_type="db", environment="production") + assert info.environment == "production" + + +def test_prompt_spec_includes_environment_and_created_by(): + """PromptSpec should carry environment and created_by fields.""" + spec = PromptSpec( + prompt_id="test", + litellm_params=PromptLiteLLMParams( + prompt_id="test", prompt_integration="dotprompt" + ), + prompt_info=PromptInfo(prompt_type="db", environment="staging"), + environment="staging", + created_by="user-123", + ) + assert spec.environment == "staging" + assert spec.created_by == "user-123" + + +def test_prompt_spec_default_environment(): + """PromptSpec environment should default to 'development'.""" + spec = PromptSpec( + prompt_id="test", + litellm_params=PromptLiteLLMParams( + prompt_id="test", prompt_integration="dotprompt" + ), + prompt_info=PromptInfo(prompt_type="db"), + ) + assert spec.environment == "development" + assert spec.created_by is None + + +def test_create_versioned_prompt_spec_includes_environment(): + """create_versioned_prompt_spec should populate environment and created_by from DB row.""" + from litellm.proxy.prompts.prompt_endpoints import create_versioned_prompt_spec + + mock_db_prompt = MagicMock() + mock_db_prompt.model_dump.return_value = { + "id": "uuid-123", + "prompt_id": "test_prompt", + "version": 2, + "environment": "staging", + "created_by": "user-456", + "litellm_params": json.dumps( + { + "prompt_id": "test_prompt", + "prompt_integration": "dotprompt", + } + ), + "prompt_info": json.dumps({"prompt_type": "db", "environment": "staging"}), + "created_at": None, + "updated_at": None, + } + spec = create_versioned_prompt_spec(mock_db_prompt) + assert spec.environment == "staging" + assert spec.created_by == "user-456" + assert spec.prompt_id == "test_prompt.v2" + + +@pytest.mark.asyncio +async def test_create_prompt_stores_environment_and_created_by(): + """create_prompt should pass environment and created_by to the DB.""" + from unittest.mock import AsyncMock, patch + from litellm.proxy._types import UserAPIKeyAuth, LitellmUserRoles + from litellm.proxy.prompts.prompt_endpoints import create_prompt, Prompt + + mock_user_auth = UserAPIKeyAuth( + api_key="sk-1234", + user_role=LitellmUserRoles.PROXY_ADMIN, + user_id="user-789", + ) + + mock_prisma_client = MagicMock() + mock_db_entry = MagicMock() + mock_db_entry.model_dump.return_value = { + "id": "uuid-1", + "prompt_id": "my_prompt", + "version": 1, + "environment": "staging", + "created_by": "user-789", + "litellm_params": json.dumps( + { + "prompt_id": "my_prompt", + "prompt_integration": "dotprompt", + } + ), + "prompt_info": json.dumps({"prompt_type": "db", "environment": "staging"}), + "created_at": None, + "updated_at": None, + } + mock_prisma_client.db.litellm_prompttable.create = AsyncMock( + return_value=mock_db_entry + ) + mock_prisma_client.db.litellm_prompttable.find_many = AsyncMock(return_value=[]) + + request = Prompt( + prompt_id="my_prompt", + litellm_params=PromptLiteLLMParams( + prompt_id="my_prompt", prompt_integration="dotprompt" + ), + prompt_info=PromptInfo(prompt_type="db", environment="staging"), + ) + + with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client): + with patch( + "litellm.proxy.prompts.prompt_registry.IN_MEMORY_PROMPT_REGISTRY" + ) as mock_registry: + mock_registry.initialize_prompt.return_value = PromptSpec( + prompt_id="my_prompt.v1", + litellm_params=request.litellm_params, + prompt_info=request.prompt_info, + environment="staging", + created_by="user-789", + ) + await create_prompt(request=request, user_api_key_dict=mock_user_auth) + + create_call = mock_prisma_client.db.litellm_prompttable.create.call_args + data = create_call.kwargs["data"] + assert data["environment"] == "staging" + assert data["created_by"] == "user-789" + + +@pytest.mark.asyncio +async def test_update_prompt_stores_environment_and_created_by(): + """update_prompt should pass environment and created_by to new version.""" + from unittest.mock import AsyncMock, patch + from litellm.proxy._types import UserAPIKeyAuth, LitellmUserRoles + from litellm.proxy.prompts.prompt_endpoints import update_prompt, Prompt + + mock_user_auth = UserAPIKeyAuth( + api_key="sk-1234", + user_role=LitellmUserRoles.PROXY_ADMIN, + user_id="user-update", + ) + + mock_prisma_client = MagicMock() + mock_existing = MagicMock() + mock_existing.version = 1 + mock_prisma_client.db.litellm_prompttable.find_many = AsyncMock( + return_value=[mock_existing] + ) + + mock_db_entry = MagicMock() + mock_db_entry.model_dump.return_value = { + "id": "uuid-2", + "prompt_id": "my_prompt", + "version": 2, + "environment": "production", + "created_by": "user-update", + "litellm_params": json.dumps( + { + "prompt_id": "my_prompt", + "prompt_integration": "dotprompt", + } + ), + "prompt_info": json.dumps({"prompt_type": "db", "environment": "production"}), + "created_at": None, + "updated_at": None, + } + mock_prisma_client.db.litellm_prompttable.create = AsyncMock( + return_value=mock_db_entry + ) + + request = Prompt( + prompt_id="my_prompt", + litellm_params=PromptLiteLLMParams( + prompt_id="my_prompt", prompt_integration="dotprompt" + ), + prompt_info=PromptInfo(prompt_type="db", environment="production"), + ) + + with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client): + with patch( + "litellm.proxy.prompts.prompt_registry.IN_MEMORY_PROMPT_REGISTRY" + ) as mock_registry: + mock_registry.get_prompt_by_id.return_value = PromptSpec( + prompt_id="my_prompt.v1", + litellm_params=request.litellm_params, + prompt_info=PromptInfo(prompt_type="db"), + ) + mock_registry.initialize_prompt.return_value = PromptSpec( + prompt_id="my_prompt.v2", + litellm_params=request.litellm_params, + prompt_info=request.prompt_info, + environment="production", + created_by="user-update", + ) + await update_prompt( + prompt_id="my_prompt", request=request, user_api_key_dict=mock_user_auth + ) + + create_call = mock_prisma_client.db.litellm_prompttable.create.call_args + data = create_call.kwargs["data"] + assert data["environment"] == "production" + assert data["created_by"] == "user-update" + + +@pytest.mark.asyncio +async def test_delete_prompt_scoped_to_environment(): + """delete_prompt with environment param should scope deletion.""" + from unittest.mock import AsyncMock, patch + from litellm.proxy._types import UserAPIKeyAuth, LitellmUserRoles + from litellm.proxy.prompts.prompt_endpoints import delete_prompt + + mock_user_auth = UserAPIKeyAuth( + api_key="sk-1234", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + + mock_prisma_client = MagicMock() + mock_prisma_client.db.litellm_prompttable.delete_many = AsyncMock(return_value=None) + + with patch( + "litellm.proxy.prompts.prompt_registry.IN_MEMORY_PROMPT_REGISTRY" + ) as mock_registry: + prompt_spec = PromptSpec( + prompt_id="test_prompt.v1", + litellm_params=PromptLiteLLMParams( + prompt_id="test_prompt", prompt_integration="dotprompt" + ), + prompt_info=PromptInfo(prompt_type="db"), + environment="staging", + ) + mock_registry.get_prompt_by_id.return_value = prompt_spec + + with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client): + await delete_prompt( + prompt_id="test_prompt", + user_api_key_dict=mock_user_auth, + environment="staging", + ) + + mock_prisma_client.db.litellm_prompttable.delete_many.assert_called_once_with( + where={"prompt_id": "test_prompt", "environment": "staging"} + ) diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index bdb38ab65f3..50dc3c6c6ec 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -752,6 +752,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "supports_file_search": {"type": "boolean"}, "supports_function_calling": {"type": "boolean"}, "supports_image_input": {"type": "boolean"}, + "supports_nova_canvas_image_edit": {"type": "boolean"}, "supports_parallel_function_calling": {"type": "boolean"}, "supports_pdf_input": {"type": "boolean"}, "supports_prompt_caching": {"type": "boolean"}, diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 2e8518d00a6..a00441e703b 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -202,6 +202,7 @@ export interface Model { interface PromptInfo { prompt_type: string; + environment?: string; } export interface PromptSpec { @@ -211,6 +212,8 @@ export interface PromptSpec { created_at?: string; updated_at?: string; version?: number; // Explicit version number for version history + environment?: string; + created_by?: string; } export interface PromptTemplateBase { @@ -222,6 +225,7 @@ export interface PromptTemplateBase { interface PromptInfoResponse { prompt_spec: PromptSpec; raw_prompt_template: PromptTemplateBase | null; + environments?: string[]; } export interface ListPromptsResponse { @@ -6035,9 +6039,15 @@ export const estimateAttachmentImpactCall = async ( } }; -export const getPromptsList = async (accessToken: string): Promise => { +export const getPromptsList = async ( + accessToken: string, + environment?: string, +): Promise => { try { - const url = proxyBaseUrl ? `${proxyBaseUrl}/prompts/list` : `/prompts/list`; + let url = proxyBaseUrl ? `${proxyBaseUrl}/prompts/list` : `/prompts/list`; + if (environment) { + url += `?environment=${encodeURIComponent(environment)}`; + } const response = await fetch(url, { method: "GET", headers: { @@ -6061,9 +6071,12 @@ export const getPromptsList = async (accessToken: string): Promise => { +export const getPromptInfo = async (accessToken: string, promptId: string, environment?: string): Promise => { try { - const url = proxyBaseUrl ? `${proxyBaseUrl}/prompts/${promptId}/info` : `/prompts/${promptId}/info`; + let url = proxyBaseUrl ? `${proxyBaseUrl}/prompts/${promptId}/info` : `/prompts/${promptId}/info`; + if (environment) { + url += `?environment=${encodeURIComponent(environment)}`; + } const response = await fetch(url, { method: "GET", headers: { @@ -6087,9 +6100,12 @@ export const getPromptInfo = async (accessToken: string, promptId: string): Prom } }; -export const getPromptVersions = async (accessToken: string, promptId: string): Promise => { +export const getPromptVersions = async (accessToken: string, promptId: string, environment?: string): Promise => { try { - const url = proxyBaseUrl ? `${proxyBaseUrl}/prompts/${promptId}/versions` : `/prompts/${promptId}/versions`; + let url = proxyBaseUrl ? `${proxyBaseUrl}/prompts/${promptId}/versions` : `/prompts/${promptId}/versions`; + if (environment) { + url += `?environment=${encodeURIComponent(environment)}`; + } const response = await fetch(url, { method: "GET", headers: { diff --git a/ui/litellm-dashboard/src/components/prompts.tsx b/ui/litellm-dashboard/src/components/prompts.tsx index 24df7e78ab0..8e2e9c8b112 100644 --- a/ui/litellm-dashboard/src/components/prompts.tsx +++ b/ui/litellm-dashboard/src/components/prompts.tsx @@ -1,7 +1,7 @@ import React, { useState, useEffect } from "react"; import { Button } from "@tremor/react"; -import { Modal } from "antd"; +import { Modal, Select } from "antd"; import { getPromptsList, PromptSpec, ListPromptsResponse, deletePromptCall } from "./networking"; import PromptTable from "./prompts/prompt_table"; import PromptInfoView from "./prompts/prompt_info"; @@ -18,6 +18,7 @@ interface PromptsProps { const PromptsPanel: React.FC = ({ accessToken, userRole }) => { const [promptsList, setPromptsList] = useState([]); const [isLoading, setIsLoading] = useState(false); + const [selectedEnvironment, setSelectedEnvironment] = useState(undefined); const [selectedPromptId, setSelectedPromptId] = useState(null); const [isAddModalVisible, setIsAddModalVisible] = useState(false); const [showEditorView, setShowEditorView] = useState(false); @@ -34,7 +35,7 @@ const PromptsPanel: React.FC = ({ accessToken, userRole }) => { setIsLoading(true); try { - const response: ListPromptsResponse = await getPromptsList(accessToken); + const response: ListPromptsResponse = await getPromptsList(accessToken, selectedEnvironment); console.log(`prompts: ${JSON.stringify(response)}`); setPromptsList(response.prompts); } catch (error) { @@ -46,7 +47,7 @@ const PromptsPanel: React.FC = ({ accessToken, userRole }) => { useEffect(() => { fetchPrompts(); - }, [accessToken]); + }, [accessToken, selectedEnvironment]); const handlePromptClick = (promptId: string) => { setSelectedPromptId(promptId); @@ -135,13 +136,25 @@ const PromptsPanel: React.FC = ({ accessToken, userRole }) => { <>

- +
+ Draft Unsaved changes
diff --git a/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/index.tsx b/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/index.tsx index b17808df12e..ea49a4b87f4 100644 --- a/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/index.tsx +++ b/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/index.tsx @@ -1,7 +1,7 @@ import React, { useState } from "react"; import ToolModal from "../tool_modal"; import NotificationsManager from "../../molecules/notifications_manager"; -import { createPromptCall, updatePromptCall } from "../../networking"; +import { createPromptCall, updatePromptCall, getPromptInfo } from "../../networking"; import { PromptType, PromptEditorViewProps, Tool } from "./types"; import { convertToDotPrompt, parseExistingPrompt } from "./utils"; import PromptEditorHeader from "./PromptEditorHeader"; @@ -39,6 +39,7 @@ const PromptEditorView: React.FC = ({ onClose, onSuccess, content: "Enter task specifics. Use {{template_variables}} for dynamic inputs", }, ], + environment: "development", }; }; @@ -208,6 +209,7 @@ const PromptEditorView: React.FC = ({ onClose, onSuccess, }, prompt_info: { prompt_type: "db", + environment: prompt.environment, }, }; @@ -273,6 +275,23 @@ const PromptEditorView: React.FC = ({ onClose, onSuccess, promptModel={prompt.model} promptVariables={extractTemplateVariables()} accessToken={accessToken} + environment={prompt.environment} + onEnvironmentChange={async (env) => { + setPrompt({ ...prompt, environment: env }); + if (editMode && accessToken && initialPromptData?.prompt_spec?.prompt_id) { + try { + const response = await getPromptInfo(accessToken, initialPromptData.prompt_spec.prompt_id, env); + if (response?.prompt_spec) { + const loadedPrompt = parseExistingPrompt(response); + setPrompt({ ...loadedPrompt, environment: env }); + const versionNum = response.prompt_spec.version || 1; + setActiveVersionId(`${response.prompt_spec.prompt_id}.v${versionNum}`); + } + } catch { + // No version in this environment yet — keep current content, just change env + } + } + }} />
diff --git a/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/types.ts b/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/types.ts index 4953c9b9437..37e73d0169b 100644 --- a/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/types.ts +++ b/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/types.ts @@ -20,6 +20,7 @@ export interface PromptType { tools: Tool[]; developerMessage: string; messages: Message[]; + environment: string; } export interface PromptEditorViewProps { diff --git a/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/utils.ts b/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/utils.ts index 523470e34d0..a8dd7f60eb1 100644 --- a/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/utils.ts +++ b/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/utils.ts @@ -246,6 +246,7 @@ export const parseExistingPrompt = (apiResponse: any): PromptType => { parsedBody.messages.length > 0 ? parsedBody.messages : [{ role: "user", content: "Enter task specifics. Use {{template_variables}} for dynamic inputs" }], + environment: apiResponse?.prompt_spec?.environment || apiResponse?.prompt_spec?.prompt_info?.environment || "development", }; }; diff --git a/ui/litellm-dashboard/src/components/prompts/prompt_info.tsx b/ui/litellm-dashboard/src/components/prompts/prompt_info.tsx index dec0baa9505..ccc3560b849 100644 --- a/ui/litellm-dashboard/src/components/prompts/prompt_info.tsx +++ b/ui/litellm-dashboard/src/components/prompts/prompt_info.tsx @@ -1,4 +1,4 @@ -import React, { useState, useEffect } from "react"; +import React, { useState, useEffect, useRef } from "react"; import { Card, Title, @@ -11,19 +11,25 @@ import { TabList, TabPanel, TabPanels, + Table, + TableBody, + TableCell, + TableHead, + TableHeaderCell, + TableRow, } from "@tremor/react"; import { Button, Modal } from "antd"; import { ArrowLeftIcon, TrashIcon, PencilIcon } from "@heroicons/react/outline"; -import { getPromptInfo, PromptSpec, PromptTemplateBase, deletePromptCall } from "@/components/networking"; +import { getPromptInfo, getPromptVersions, PromptSpec, PromptTemplateBase, deletePromptCall } from "@/components/networking"; import { copyToClipboard as utilCopyToClipboard } from "@/utils/dataUtils"; import { CheckIcon, CopyIcon } from "lucide-react"; import NotificationsManager from "../molecules/notifications_manager"; import PromptCodeSnippets from "./prompt_editor_view/PromptCodeSnippets"; -import { - extractModel, - extractTemplateVariables, - getBasePromptId, - getCurrentVersion +import { + extractModel, + extractTemplateVariables, + getBasePromptId, + getCurrentVersion } from "./prompt_utils"; export interface PromptInfoProps { @@ -44,14 +50,31 @@ const PromptInfoView: React.FC = ({ promptId, onClose, accessTo const [showDeleteConfirm, setShowDeleteConfirm] = useState(false); const [isDeleting, setIsDeleting] = useState(false); - const fetchPromptInfo = async () => { + // Environment and version state + const [environments, setEnvironments] = useState([]); + const [selectedEnv, setSelectedEnv] = useState(null); + const [versionHistory, setVersionHistory] = useState([]); + const [selectedVersion, setSelectedVersion] = useState(null); + const [loadingVersions, setLoadingVersions] = useState(false); + + // Initial fetch — no environment filter, gets default + all environments list + const fetchPromptInfo = async (environment?: string) => { try { setLoading(true); if (!accessToken) return; - const response = await getPromptInfo(accessToken, promptId); + const response = await getPromptInfo(accessToken, promptId, environment); setPromptData(response.prompt_spec); setPromptTemplate(response.raw_prompt_template); - setRawApiResponse(response); // Store the raw response for the Raw JSON tab + setRawApiResponse(response); + + // Set environments from response + if (response.environments && response.environments.length > 0) { + setEnvironments(response.environments); + if (!selectedEnv) { + setSelectedEnv(response.prompt_spec.environment || response.environments[0]); + } + } + setSelectedVersion(response.prompt_spec.version || null); } catch (error) { NotificationsManager.fromBackend("Failed to load prompt information"); console.error("Error fetching prompt info:", error); @@ -60,11 +83,46 @@ const PromptInfoView: React.FC = ({ promptId, onClose, accessTo } }; + // Fetch version history for selected environment + const fetchVersionHistory = async (env: string) => { + if (!accessToken) return; + setLoadingVersions(true); + try { + const response = await getPromptVersions(accessToken, promptId, env); + setVersionHistory(response.prompts || []); + } catch { + setVersionHistory([]); + } finally { + setLoadingVersions(false); + } + }; + + const isInitialMount = useRef(true); + useEffect(() => { + setSelectedEnv(null); + setEnvironments([]); + setVersionHistory([]); fetchPromptInfo(); }, [promptId, accessToken]); - if (loading) { + // When environment changes (user clicks tab), re-fetch — skip initial mount + useEffect(() => { + if (isInitialMount.current) { + isInitialMount.current = false; + // Still fetch version history on initial mount once selectedEnv is set + if (selectedEnv && accessToken) { + fetchVersionHistory(selectedEnv); + } + return; + } + if (selectedEnv && accessToken) { + fetchPromptInfo(selectedEnv); + fetchVersionHistory(selectedEnv); + } + }, [selectedEnv]); + + if (loading && !promptData) { return
Loading...
; } @@ -72,7 +130,6 @@ const PromptInfoView: React.FC = ({ promptId, onClose, accessTo return
Prompt not found
; } - // Format date helper function const formatDate = (dateString?: string) => { if (!dateString) return "-"; const date = new Date(dateString); @@ -95,13 +152,12 @@ const PromptInfoView: React.FC = ({ promptId, onClose, accessTo const handleDeleteConfirm = async () => { if (!accessToken || !promptData) return; - setIsDeleting(true); try { await deletePromptCall(accessToken, basePromptId); NotificationsManager.success(`Prompt "${basePromptId}" deleted successfully`); - onDelete?.(); // Call the callback to refresh the parent component - onClose(); // Close the info view + onDelete?.(); + onClose(); } catch (error) { console.error("Error deleting prompt:", error); NotificationsManager.fromBackend("Failed to delete prompt"); @@ -115,10 +171,27 @@ const PromptInfoView: React.FC = ({ promptId, onClose, accessTo setShowDeleteConfirm(false); }; - // Use utility functions to extract prompt data + const handleVersionClick = async (version: PromptSpec) => { + if (!accessToken || !selectedEnv) return; + // Fetch specific version's info + const versionNum = version.version || 1; + setSelectedVersion(versionNum); + try { + const versionedId = `${promptId}.v${versionNum}`; + const response = await getPromptInfo(accessToken, versionedId, selectedEnv); + setPromptData(response.prompt_spec); + setPromptTemplate(response.raw_prompt_template); + setRawApiResponse(response); + } catch { + NotificationsManager.fromBackend(`Failed to load version v${versionNum}`); + } + }; + const promptModel = promptData ? extractModel(promptData) || "gpt-4o" : "gpt-4o"; const basePromptId = getBasePromptId(promptData); const currentVersion = getCurrentVersion(promptData); + const latestVersion = versionHistory.length > 0 ? Math.max(...versionHistory.map(v => v.version || 1)) : null; + const isViewingOldVersion = latestVersion !== null && selectedVersion !== null && selectedVersion < latestVersion; return (
@@ -174,32 +247,75 @@ const PromptInfoView: React.FC = ({ promptId, onClose, accessTo
+ {/* Environment Tabs */} + {environments.length > 0 && ( +
+ {[...environments].sort((a, b) => { + const order: Record = { development: 0, staging: 1, production: 2 }; + return (order[a] ?? 99) - (order[b] ?? 99); + }).map((env) => ( + + ))} +
+ )} + + {/* Old version banner */} + {isViewingOldVersion && ( +
+ + Viewing v{selectedVersion} — not the latest version (v{latestVersion}) + + { + const latest = versionHistory.find(v => v.version === latestVersion); + if (latest) handleVersionClick(latest); + }} + > + Go to latest + +
+ )} + Overview {promptTemplate ? Prompt Template : <>} - {isAdmin ? Details : <>} Raw JSON {/* Overview Panel */} - - - Prompt ID -
- {basePromptId} -
- - + Version
{currentVersion} - - v{currentVersion} - + v{currentVersion}
@@ -207,31 +323,98 @@ const PromptInfoView: React.FC = ({ promptId, onClose, accessTo Prompt Type
{promptData.prompt_info?.prompt_type || "-"} - - {promptData.prompt_info?.prompt_type || "Unknown"} - +
+ + + + Created By +
+ {promptData.created_by || "-"}
Created At
- {formatDate(promptData.created_at)} - Last Updated: {formatDate(promptData.updated_at)} + {formatDate(promptData.created_at)} + Updated: {formatDate(promptData.updated_at)}
- {promptData.litellm_params && Object.keys(promptData.litellm_params).length > 0 && ( - - LiteLLM Parameters -
- 
-                    {JSON.stringify(promptData.litellm_params, null, 2)}
-
-
- - )} + {/* Version History Table */} + + Version History — {selectedEnv} + {loadingVersions ? ( + Loading versions... + ) : versionHistory.length > 0 ? ( + + + + Version + Created By + Date + Actions + + + + {versionHistory.map((v) => { + const vNum = v.version || 1; + const isSelected = vNum === selectedVersion; + const isLatest = vNum === latestVersion; + return ( + handleVersionClick(v)} + > + + + v{vNum} + + {isLatest && ( + latest + )} + + + {v.created_by || "-"} + + + {formatDate(v.created_at)} + + + { + e.stopPropagation(); + // Build a response-like object for the editor + const editData = { + prompt_spec: { + ...v, + prompt_id: basePromptId, + environment: selectedEnv, + }, + raw_prompt_template: isSelected ? promptTemplate : null, + }; + onEdit?.(editData); + }} + > + Edit + + + + ); + })} + +
+ ) : ( + No versions found in {selectedEnv} + )} + {/* Prompt Template Panel */} @@ -283,54 +466,6 @@ const PromptInfoView: React.FC = ({ promptId, onClose, accessTo )} - {/* Details Panel (only for admins) */} - {isAdmin && ( - - - Prompt Details -
-
- Prompt ID -
{basePromptId}
-
- -
- Prompt Type -
{promptData.prompt_info?.prompt_type || "-"}
-
- -
- Created At -
{formatDate(promptData.created_at)}
-
- -
- Last Updated -
{formatDate(promptData.updated_at)}
-
- -
- LiteLLM Parameters -
- 
-                        {JSON.stringify(promptData.litellm_params, null, 2)}
-
-
-
- -
- Prompt Info -
- 
-                        {JSON.stringify(promptData.prompt_info, null, 2)}
-
-
-
-
- - - )} - {/* Raw JSON Panel */} diff --git a/ui/litellm-dashboard/src/components/prompts/prompt_table.tsx b/ui/litellm-dashboard/src/components/prompts/prompt_table.tsx index ee2f2baa193..18bb35d4888 100644 --- a/ui/litellm-dashboard/src/components/prompts/prompt_table.tsx +++ b/ui/litellm-dashboard/src/components/prompts/prompt_table.tsx @@ -184,6 +184,36 @@ const PromptTable: React.FC = ({ ); }, }, + { + header: "Environment", + accessorKey: "environment", + cell: ({ row }) => { + const prompt = row.original; + const env = prompt.environment || "development"; + const colorMap: Record = { + production: "text-red-600 bg-red-50", + staging: "text-yellow-600 bg-yellow-50", + development: "text-green-600 bg-green-50", + }; + return ( + + {env} + + ); + }, + }, + { + header: "Created By", + accessorKey: "created_by", + cell: ({ row }) => { + const prompt = row.original; + return ( + + {prompt.created_by || "-"} + + ); + }, + }, { header: "Type", accessorKey: "prompt_info.prompt_type", From a86f19f6dabcdac84d87055903666eede8940b87 Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Fri, 3 Apr 2026 15:08:14 -0700 Subject: [PATCH 21/25] =?UTF-8?q?bump:=20version=201.83.1=20=E2=86=92=201.?= =?UTF-8?q?83.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 90b88165e86..25226462278 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.83.1" +version = "1.83.2" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -181,7 +181,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.83.1" +version = "1.83.2" version_files = [ "pyproject.toml:^version" ] From 127149c263b1676927adda56bf969a50e0ff132f Mon Sep 17 00:00:00 2001 From: ishaan-berri <155045088+ishaan-berri@users.noreply.github.com> Date: Fri, 3 Apr 2026 17:46:06 -0700 Subject: [PATCH 22/25] bump litellm-proxy-extras to 0.4.64 (#25121) * bump litellm-proxy-extras version to 0.4.64 * bump litellm-proxy-extras==0.4.64 in requirements.txt * bump litellm-proxy-extras==0.4.64 in pyproject.toml --- litellm-proxy-extras/pyproject.toml | 4 ++-- pyproject.toml | 2 +- requirements.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml index 9946ac0af8c..6d35ed0c14d 100644 --- a/litellm-proxy-extras/pyproject.toml +++ b/litellm-proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm-proxy-extras" -version = "0.4.63" +version = "0.4.64" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." authors = ["BerriAI"] readme = "README.md" @@ -22,7 +22,7 @@ requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "0.4.63" +version = "0.4.64" version_files = [ "pyproject.toml:version", "../requirements.txt:litellm-proxy-extras==", diff --git a/pyproject.toml b/pyproject.toml index 25226462278..a9435bec4c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,7 +63,7 @@ boto3 = { version = "1.42.80", optional = true } redisvl = {version = "0.4.1", optional = true, markers = "python_version >= '3.9' and python_version < '3.14'"} mcp = {version = "1.26.0", optional = true, python = ">=3.10"} a2a-sdk = {version = "0.3.25", optional = true, python = ">=3.10"} -litellm-proxy-extras = {version = "0.4.63", optional = true} +litellm-proxy-extras = {version = "0.4.64", optional = true} rich = {version = "13.9.4", optional = true} litellm-enterprise = {version = "0.1.35", optional = true} diskcache = {version = "5.6.3", optional = true} diff --git a/requirements.txt b/requirements.txt index e42d4d54484..403e170e8d8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -56,7 +56,7 @@ grpcio==1.80.0 sentry_sdk==2.21.0 # for sentry error handling detect-secrets==1.5.0 # Enterprise - secret detection / masking in LLM requests tzdata==2025.1 # IANA time zone database -litellm-proxy-extras==0.4.63 # for proxy extras - e.g. prisma migrations +litellm-proxy-extras==0.4.64 # for proxy extras - e.g. prisma migrations llm-sandbox==0.3.31 # for skill execution in sandbox ### LITELLM PACKAGE DEPENDENCIES python-dotenv==1.0.1 # for env From f74cd074196bf8ed6269277fddb3e2708c44dc38 Mon Sep 17 00:00:00 2001 From: michelligabriele Date: Sat, 4 Apr 2026 05:20:44 +0200 Subject: [PATCH 23/25] feat(proxy): add project-level guardrails support (#25087) --- litellm/proxy/_types.py | 4 + litellm/proxy/litellm_pre_call_utils.py | 42 ++++++-- .../proxy/test_litellm_pre_call_utils.py | 95 +++++++++++++++++++ 3 files changed, 134 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index e4702364a3e..4f3e51a2a32 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2753,6 +2753,8 @@ class NewProjectRequest(LiteLLM_BudgetTable): budget_id: Optional[str] = None metadata: Optional[dict] = None tags: Optional[List[str]] = None + guardrails: Optional[List[str]] = None + policies: Optional[List[str]] = None models: List[str] = [] model_rpm_limit: Optional[dict] = None model_tpm_limit: Optional[dict] = None @@ -2785,6 +2787,8 @@ class UpdateProjectRequest(LiteLLM_BudgetTable): team_id: Optional[str] = None metadata: Optional[dict] = None tags: Optional[List[str]] = None + guardrails: Optional[List[str]] = None + policies: Optional[List[str]] = None models: Optional[List[str]] = None model_rpm_limit: Optional[dict] = None model_tpm_limit: Optional[dict] = None diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index ba9577f35d7..fcb8b6db80a 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -1472,17 +1472,19 @@ def _add_guardrails_from_key_or_team_metadata( team_metadata: Optional[dict], data: dict, metadata_variable_name: str, + project_metadata: Optional[dict] = None, ) -> None: """ - Helper add guardrails from key or team metadata to request data + Helper add guardrails from key, team, or project metadata to request data - Key guardrails are set first, then team guardrails are appended (without duplicates). + Key guardrails are set first, then team and project guardrails are appended (without duplicates). Args: key_metadata: The key metadata dictionary to check for guardrails team_metadata: The team metadata dictionary to check for guardrails data: The request data to update metadata_variable_name: The name of the metadata field in data + project_metadata: The project metadata dictionary to check for guardrails """ from litellm.proxy.utils import _premium_user_check @@ -1508,6 +1510,15 @@ def _add_guardrails_from_key_or_team_metadata( _premium_user_check() combined_guardrails.update(team_metadata["guardrails"]) + # Add project-level guardrails (set automatically handles duplicates) + if project_metadata and "guardrails" in project_metadata: + if ( + isinstance(project_metadata["guardrails"], list) + and len(project_metadata["guardrails"]) > 0 + ): + _premium_user_check() + combined_guardrails.update(project_metadata["guardrails"]) + # Set combined guardrails in metadata as list if combined_guardrails: data[metadata_variable_name]["guardrails"] = list(combined_guardrails) @@ -1518,12 +1529,13 @@ def _add_guardrails_from_policies_in_metadata( team_metadata: Optional[dict], data: dict, metadata_variable_name: str, + project_metadata: Optional[dict] = None, ) -> None: """ - Helper to resolve guardrails from policies attached to key/team metadata. + Helper to resolve guardrails from policies attached to key/team/project metadata. This function: - 1. Gets policy names from key and team metadata + 1. Gets policy names from key, team, and project metadata 2. Resolves guardrails from those policies (including inheritance) 3. Adds resolved guardrails to request metadata @@ -1532,6 +1544,7 @@ def _add_guardrails_from_policies_in_metadata( team_metadata: The team metadata dictionary to check for policies data: The request data to update metadata_variable_name: The name of the metadata field in data + project_metadata: The project metadata dictionary to check for policies """ from litellm._logging import verbose_proxy_logger from litellm.proxy.policy_engine.policy_registry import get_policy_registry @@ -1560,6 +1573,15 @@ def _add_guardrails_from_policies_in_metadata( _premium_user_check() policy_names.update(team_metadata["policies"]) + # Add project-level policies + if project_metadata and "policies" in project_metadata: + if ( + isinstance(project_metadata["policies"], list) + and len(project_metadata["policies"]) > 0 + ): + _premium_user_check() + policy_names.update(project_metadata["policies"]) + if not policy_names: return @@ -1641,6 +1663,7 @@ async def move_guardrails_to_metadata( # Early-out: skip all guardrails processing when nothing is configured key_metadata = user_api_key_dict.metadata team_metadata = user_api_key_dict.team_metadata + project_metadata = user_api_key_dict.project_metadata or {} has_key_config = key_metadata and ( "guardrails" in key_metadata or "policies" in key_metadata @@ -1648,12 +1671,15 @@ async def move_guardrails_to_metadata( has_team_config = team_metadata and ( "guardrails" in team_metadata or "policies" in team_metadata ) + has_project_config = project_metadata and ( + "guardrails" in project_metadata or "policies" in project_metadata + ) has_request_config = ( "guardrails" in data or "guardrail_config" in data or "policies" in data ) # Only check policy engine if no local config (avoid import + registry lookup) - if not (has_key_config or has_team_config or has_request_config): + if not (has_key_config or has_team_config or has_project_config or has_request_config): from litellm.proxy.policy_engine.policy_registry import get_policy_registry if not get_policy_registry().is_initialized(): @@ -1661,20 +1687,22 @@ async def move_guardrails_to_metadata( data.pop("policies", None) return - # Check key-level guardrails + # Check key/team/project-level guardrails _add_guardrails_from_key_or_team_metadata( key_metadata=user_api_key_dict.metadata, team_metadata=user_api_key_dict.team_metadata, + project_metadata=project_metadata, data=data, metadata_variable_name=_metadata_variable_name, ) ######################################################################################### - # Add guardrails from policies attached to key/team metadata + # Add guardrails from policies attached to key/team/project metadata ######################################################################################### _add_guardrails_from_policies_in_metadata( key_metadata=user_api_key_dict.metadata, team_metadata=user_api_key_dict.team_metadata, + project_metadata=project_metadata, data=data, metadata_variable_name=_metadata_variable_name, ) diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index bc13cea939e..04af5cd0086 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -1363,6 +1363,101 @@ async def test_request_guardrails_do_not_override_key_guardrails(): assert len(requested_guardrails) == 1 +@pytest.mark.asyncio +async def test_project_guardrails_merge_with_key_and_team(): + """ + Test that project guardrails are merged with key and team guardrails (union semantics). + All three levels should contribute to the final guardrails list without duplicates. + """ + request_mock = MagicMock(spec=Request) + request_mock.url.path = "/chat/completions" + request_mock.url = MagicMock() + request_mock.url.__str__.return_value = "http://localhost/chat/completions" + request_mock.method = "POST" + request_mock.query_params = {} + request_mock.headers = {"Content-Type": "application/json"} + request_mock.client = MagicMock() + request_mock.client.host = "127.0.0.1" + + data = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "test"}], + } + + user_api_key_dict = UserAPIKeyAuth( + api_key="test-key", + metadata={"guardrails": ["key-guardrail-1"]}, + team_metadata={"guardrails": ["team-guardrail-1", "key-guardrail-1"]}, + project_metadata={"guardrails": ["project-guardrail-1", "team-guardrail-1"]}, + ) + + with patch("litellm.proxy.utils._premium_user_check"): + updated_data = await add_litellm_data_to_request( + data=data, + request=request_mock, + user_api_key_dict=user_api_key_dict, + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + metadata = updated_data.get("metadata", {}) + guardrails = metadata.get("guardrails", []) + + # All three sources contribute + assert "key-guardrail-1" in guardrails + assert "team-guardrail-1" in guardrails + assert "project-guardrail-1" in guardrails + # No duplicates + assert guardrails.count("key-guardrail-1") == 1 + assert guardrails.count("team-guardrail-1") == 1 + + +@pytest.mark.asyncio +async def test_project_guardrails_only(): + """ + Test that project guardrails work when key and team have no guardrails configured. + """ + request_mock = MagicMock(spec=Request) + request_mock.url.path = "/chat/completions" + request_mock.url = MagicMock() + request_mock.url.__str__.return_value = "http://localhost/chat/completions" + request_mock.method = "POST" + request_mock.query_params = {} + request_mock.headers = {"Content-Type": "application/json"} + request_mock.client = MagicMock() + request_mock.client.host = "127.0.0.1" + + data = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "test"}], + } + + user_api_key_dict = UserAPIKeyAuth( + api_key="test-key", + metadata={}, + team_metadata={}, + project_metadata={"guardrails": ["project-guardrail-1", "project-guardrail-2"]}, + ) + + with patch("litellm.proxy.utils._premium_user_check"): + updated_data = await add_litellm_data_to_request( + data=data, + request=request_mock, + user_api_key_dict=user_api_key_dict, + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + metadata = updated_data.get("metadata", {}) + guardrails = metadata.get("guardrails", []) + + assert "project-guardrail-1" in guardrails + assert "project-guardrail-2" in guardrails + assert len(guardrails) == 2 + + def test_update_model_if_key_alias_exists(): """ Test that _update_model_if_key_alias_exists properly updates the model when a key alias exists. From a292add9bd53a87e919a79aaeae8bbcb542cd033 Mon Sep 17 00:00:00 2001 From: michelligabriele Date: Sat, 4 Apr 2026 05:25:32 +0200 Subject: [PATCH 24/25] fix(a2a): preserve JSON-RPC envelope for AgentCore A2A-native agents (#25092) --- .../litellm_completion_bridge/handler.py | 14 +- litellm/a2a_protocol/providers/base.py | 6 +- .../providers/bedrock_agentcore/__init__.py | 22 ++ .../providers/bedrock_agentcore/config.py | 61 ++++ .../providers/bedrock_agentcore/handler.py | 134 +++++++ .../bedrock_agentcore/transformation.py | 134 +++++++ .../a2a_protocol/providers/config_manager.py | 12 +- .../a2a_protocol/providers/__init__.py | 0 .../providers/bedrock_agentcore/__init__.py | 0 .../test_bedrock_agentcore_a2a.py | 327 ++++++++++++++++++ 10 files changed, 695 insertions(+), 15 deletions(-) create mode 100644 litellm/a2a_protocol/providers/bedrock_agentcore/__init__.py create mode 100644 litellm/a2a_protocol/providers/bedrock_agentcore/config.py create mode 100644 litellm/a2a_protocol/providers/bedrock_agentcore/handler.py create mode 100644 litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py create mode 100644 tests/test_litellm/a2a_protocol/providers/__init__.py create mode 100644 tests/test_litellm/a2a_protocol/providers/bedrock_agentcore/__init__.py create mode 100644 tests/test_litellm/a2a_protocol/providers/bedrock_agentcore/test_bedrock_agentcore_a2a.py diff --git a/litellm/a2a_protocol/litellm_completion_bridge/handler.py b/litellm/a2a_protocol/litellm_completion_bridge/handler.py index c3d2e415237..53aac1d3e6a 100644 --- a/litellm/a2a_protocol/litellm_completion_bridge/handler.py +++ b/litellm/a2a_protocol/litellm_completion_bridge/handler.py @@ -48,20 +48,19 @@ class A2ACompletionBridgeHandler: # Get provider config for custom_llm_provider custom_llm_provider = litellm_params.get("custom_llm_provider") a2a_provider_config = A2AProviderConfigManager.get_provider_config( - custom_llm_provider=custom_llm_provider + custom_llm_provider=custom_llm_provider, + model=litellm_params.get("model"), ) # If provider config exists, use it if a2a_provider_config is not None: - if api_base is None: - raise ValueError(f"api_base is required for {custom_llm_provider}") - verbose_logger.info(f"A2A: Using provider config for {custom_llm_provider}") response_data = await a2a_provider_config.handle_non_streaming( request_id=request_id, params=params, api_base=api_base, + litellm_params=litellm_params, ) return response_data @@ -147,14 +146,12 @@ class A2ACompletionBridgeHandler: # Get provider config for custom_llm_provider custom_llm_provider = litellm_params.get("custom_llm_provider") a2a_provider_config = A2AProviderConfigManager.get_provider_config( - custom_llm_provider=custom_llm_provider + custom_llm_provider=custom_llm_provider, + model=litellm_params.get("model"), ) # If provider config exists, use it if a2a_provider_config is not None: - if api_base is None: - raise ValueError(f"api_base is required for {custom_llm_provider}") - verbose_logger.info( f"A2A: Using provider config for {custom_llm_provider} (streaming)" ) @@ -163,6 +160,7 @@ class A2ACompletionBridgeHandler: request_id=request_id, params=params, api_base=api_base, + litellm_params=litellm_params, ): yield chunk diff --git a/litellm/a2a_protocol/providers/base.py b/litellm/a2a_protocol/providers/base.py index a2354b3495e..3ac1cb47fc8 100644 --- a/litellm/a2a_protocol/providers/base.py +++ b/litellm/a2a_protocol/providers/base.py @@ -3,7 +3,7 @@ Base configuration for A2A protocol providers. """ from abc import ABC, abstractmethod -from typing import Any, AsyncIterator, Dict +from typing import Any, AsyncIterator, Dict, Optional class BaseA2AProviderConfig(ABC): @@ -19,7 +19,7 @@ class BaseA2AProviderConfig(ABC): self, request_id: str, params: Dict[str, Any], - api_base: str, + api_base: Optional[str] = None, **kwargs, ) -> Dict[str, Any]: """ @@ -41,7 +41,7 @@ class BaseA2AProviderConfig(ABC): self, request_id: str, params: Dict[str, Any], - api_base: str, + api_base: Optional[str] = None, **kwargs, ) -> AsyncIterator[Dict[str, Any]]: """ diff --git a/litellm/a2a_protocol/providers/bedrock_agentcore/__init__.py b/litellm/a2a_protocol/providers/bedrock_agentcore/__init__.py new file mode 100644 index 00000000000..a61d8f98b39 --- /dev/null +++ b/litellm/a2a_protocol/providers/bedrock_agentcore/__init__.py @@ -0,0 +1,22 @@ +""" +Bedrock AgentCore A2A provider. + +Preserves JSON-RPC envelopes for AgentCore agents that speak A2A natively, +bypassing the completion bridge that would otherwise strip the envelope. +""" + +from litellm.a2a_protocol.providers.bedrock_agentcore.config import ( + BedrockAgentCoreA2AConfig, +) +from litellm.a2a_protocol.providers.bedrock_agentcore.handler import ( + BedrockAgentCoreA2AHandler, +) +from litellm.a2a_protocol.providers.bedrock_agentcore.transformation import ( + BedrockAgentCoreA2ATransformation, +) + +__all__ = [ + "BedrockAgentCoreA2AConfig", + "BedrockAgentCoreA2AHandler", + "BedrockAgentCoreA2ATransformation", +] diff --git a/litellm/a2a_protocol/providers/bedrock_agentcore/config.py b/litellm/a2a_protocol/providers/bedrock_agentcore/config.py new file mode 100644 index 00000000000..679e19c23cd --- /dev/null +++ b/litellm/a2a_protocol/providers/bedrock_agentcore/config.py @@ -0,0 +1,61 @@ +""" +Bedrock AgentCore A2A provider configuration. +""" + +from typing import Any, AsyncIterator, Dict, Optional + +from litellm.a2a_protocol.providers.base import BaseA2AProviderConfig +from litellm.a2a_protocol.providers.bedrock_agentcore.handler import ( + BedrockAgentCoreA2AHandler, +) + + +class BedrockAgentCoreA2AConfig(BaseA2AProviderConfig): + """ + Provider configuration for Bedrock AgentCore A2A-native agents. + + AgentCore agents that speak A2A natively expect the full JSON-RPC envelope. + This config bypasses the completion bridge and forwards requests directly, + deriving the endpoint URL from the model ARN and signing with SigV4/JWT. + """ + + async def handle_non_streaming( + self, + request_id: str, + params: Dict[str, Any], + api_base: Optional[str] = None, + **kwargs, + ) -> Dict[str, Any]: + """Handle non-streaming request to AgentCore A2A agent.""" + litellm_params = kwargs.get("litellm_params") + if not litellm_params: + raise ValueError( + "litellm_params is required for BedrockAgentCoreA2AConfig " + "(must contain model with AgentCore ARN)" + ) + return await BedrockAgentCoreA2AHandler.handle_non_streaming( + request_id=request_id, + params=params, + litellm_params=litellm_params, + ) + + async def handle_streaming( + self, + request_id: str, + params: Dict[str, Any], + api_base: Optional[str] = None, + **kwargs, + ) -> AsyncIterator[Dict[str, Any]]: + """Handle streaming request to AgentCore A2A agent.""" + litellm_params = kwargs.get("litellm_params") + if not litellm_params: + raise ValueError( + "litellm_params is required for BedrockAgentCoreA2AConfig " + "(must contain model with AgentCore ARN)" + ) + async for chunk in BedrockAgentCoreA2AHandler.handle_streaming( + request_id=request_id, + params=params, + litellm_params=litellm_params, + ): + yield chunk diff --git a/litellm/a2a_protocol/providers/bedrock_agentcore/handler.py b/litellm/a2a_protocol/providers/bedrock_agentcore/handler.py new file mode 100644 index 00000000000..d7445dfc252 --- /dev/null +++ b/litellm/a2a_protocol/providers/bedrock_agentcore/handler.py @@ -0,0 +1,134 @@ +""" +Handler for Bedrock AgentCore A2A-native agents. + +Sends JSON-RPC envelopes directly to AgentCore endpoints, bypassing the +completion bridge that would otherwise strip the envelope. +""" + +import json +from typing import Any, AsyncIterator, Dict, cast + +from litellm._logging import verbose_logger +from litellm.a2a_protocol.providers.bedrock_agentcore.transformation import ( + BedrockAgentCoreA2ATransformation, +) +from litellm.llms.custom_httpx.http_handler import get_async_httpx_client +from litellm.types.llms.custom_http import httpxSpecialProvider + + +class BedrockAgentCoreA2AHandler: + """ + Handler for Bedrock AgentCore A2A requests. + + Constructs JSON-RPC envelopes, signs them via AmazonAgentCoreConfig, + and POSTs directly to the AgentCore endpoint. + """ + + @staticmethod + async def handle_non_streaming( + request_id: str, + params: Dict[str, Any], + litellm_params: Dict[str, Any], + ) -> Dict[str, Any]: + """ + Handle non-streaming A2A request to AgentCore. + + Args: + request_id: A2A JSON-RPC request ID + params: A2A MessageSendParams containing the message + litellm_params: Agent's litellm_params (model, api_key, etc.) + + Returns: + A2A JSON-RPC response dict from the AgentCore agent + """ + url, headers, body = ( + BedrockAgentCoreA2ATransformation.get_url_and_signed_request( + request_id=request_id, + params=params, + litellm_params=litellm_params, + method="message/send", + ) + ) + + verbose_logger.info( + f"BedrockAgentCore A2A: Sending non-streaming request to {url}" + ) + + client = get_async_httpx_client( + llm_provider=cast(Any, httpxSpecialProvider.A2AProvider), + ) + response = await client.post( + url, + headers=headers, + data=body, + ) + response.raise_for_status() + response_data = response.json() + + if "error" in response_data: + verbose_logger.warning( + f"BedrockAgentCore A2A: Agent returned error: {response_data['error']}" + ) + + return response_data + + @staticmethod + async def handle_streaming( + request_id: str, + params: Dict[str, Any], + litellm_params: Dict[str, Any], + ) -> AsyncIterator[Dict[str, Any]]: + """ + Handle streaming A2A request to AgentCore. + + Args: + request_id: A2A JSON-RPC request ID + params: A2A MessageSendParams containing the message + litellm_params: Agent's litellm_params (model, api_key, etc.) + + Yields: + A2A streaming response events from the AgentCore agent + """ + url, headers, body = ( + BedrockAgentCoreA2ATransformation.get_url_and_signed_request( + request_id=request_id, + params=params, + litellm_params=litellm_params, + method="message/send", + stream=True, + ) + ) + + verbose_logger.info( + f"BedrockAgentCore A2A: Sending streaming request to {url}" + ) + + client = get_async_httpx_client( + llm_provider=cast(Any, httpxSpecialProvider.A2AProvider), + ) + response = await client.post( + url, + headers=headers, + data=body, + stream=True, + ) + response.raise_for_status() + + # Check content type — AgentCore may return JSON instead of SSE + content_type = response.headers.get("content-type", "").lower() + + if "application/json" in content_type: + # Single JSON response fallback (not SSE) + verbose_logger.debug( + "BedrockAgentCore A2A streaming: received JSON instead of SSE, " + "yielding as single event" + ) + response_body = await response.aread() + response_data = json.loads(response_body) + yield response_data + else: + # SSE stream — parse data: lines + async for event in BedrockAgentCoreA2ATransformation.parse_sse_events( + response + ): + yield event diff --git a/litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py b/litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py new file mode 100644 index 00000000000..44dc10fe2b7 --- /dev/null +++ b/litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py @@ -0,0 +1,134 @@ +""" +Transformation layer for Bedrock AgentCore A2A provider. + +Constructs JSON-RPC envelopes, derives AgentCore URLs from model ARNs, +and signs requests via AmazonAgentCoreConfig (SigV4 or JWT). +""" + +import json +from typing import Any, AsyncIterator, Dict, Tuple + +from litellm._logging import verbose_logger +from litellm.llms.bedrock.chat.agentcore.transformation import AmazonAgentCoreConfig + + +class BedrockAgentCoreA2ATransformation: + """ + Request/response transformation for Bedrock AgentCore A2A agents. + + Reuses AmazonAgentCoreConfig for URL construction, ARN parsing, + and request signing. No logic is duplicated. + """ + + @staticmethod + def get_url_and_signed_request( + request_id: str, + params: Dict[str, Any], + litellm_params: Dict[str, Any], + method: str = "message/send", + stream: bool = False, + ) -> Tuple[str, dict, bytes]: + """ + Build the AgentCore URL, construct a JSON-RPC envelope, and sign the request. + + Args: + request_id: A2A JSON-RPC request ID + params: A2A MessageSendParams + litellm_params: Agent's litellm_params (model, api_key, etc.) + method: JSON-RPC method name (default: "message/send") + stream: Whether this is a streaming request + + Returns: + Tuple of (url, signed_headers, signed_body_bytes) + """ + # Extract model and strip the "bedrock/" prefix + # "bedrock/agentcore/arn:aws:..." → "agentcore/arn:aws:..." + model = litellm_params.get("model", "") + if model.startswith("bedrock/"): + agentcore_model = model[len("bedrock/") :] + else: + agentcore_model = model + + # Build optional_params from litellm_params (everything except model and custom_llm_provider) + optional_params = { + k: v + for k, v in litellm_params.items() + if k not in ("model", "custom_llm_provider") + } + + agentcore_config = AmazonAgentCoreConfig() + + # Derive URL from ARN + url = agentcore_config.get_complete_url( + api_base=optional_params.get("api_base"), + api_key=optional_params.get("api_key"), + model=agentcore_model, + optional_params=optional_params, + litellm_params=litellm_params, + stream=stream, + ) + + # Construct JSON-RPC 2.0 envelope + json_rpc_body = { + "jsonrpc": "2.0", + "method": method, + "id": request_id, + "params": params, + } + + # Set required AgentCore session headers (normally set by transform_request, + # which we skip because it also builds {"prompt": "..."}) + headers: dict = {} + session_id = agentcore_config._get_runtime_session_id(optional_params) + headers["X-Amzn-Bedrock-AgentCore-Runtime-Session-Id"] = session_id + runtime_user_id = agentcore_config._get_runtime_user_id(optional_params) + if runtime_user_id: + headers["X-Amzn-Bedrock-AgentCore-Runtime-User-Id"] = runtime_user_id + + # Sign the request (SigV4 or JWT depending on api_key presence) + signed_headers, signed_body = agentcore_config.sign_request( + headers=headers, + optional_params=optional_params, + request_data=json_rpc_body, + api_base=url, + api_key=optional_params.get("api_key"), + model=agentcore_model, + stream=stream, + ) + + # sign_request returns Optional[bytes] — ensure we have bytes + if signed_body is None: + signed_body = json.dumps(json_rpc_body).encode() + + return url, signed_headers, signed_body + + @staticmethod + async def parse_sse_events(response: Any) -> AsyncIterator[Dict[str, Any]]: + """ + Parse SSE events from an httpx streaming response. + + Reads line-by-line, parses `data:` lines as JSON, and yields each parsed dict. + + Args: + response: httpx streaming response + + Yields: + Parsed JSON dicts from SSE data lines + """ + async for line in response.aiter_lines(): + line = line.strip() + if not line: + continue + + if line.startswith("data:"): + data_str = line[len("data:") :].strip() + if not data_str: + continue + try: + event = json.loads(data_str) + yield event + except json.JSONDecodeError: + verbose_logger.debug( + f"BedrockAgentCore A2A: Skipping non-JSON SSE line: {data_str[:100]}" + ) + continue diff --git a/litellm/a2a_protocol/providers/config_manager.py b/litellm/a2a_protocol/providers/config_manager.py index a8b9566c171..d684efd4756 100644 --- a/litellm/a2a_protocol/providers/config_manager.py +++ b/litellm/a2a_protocol/providers/config_manager.py @@ -19,12 +19,14 @@ class A2AProviderConfigManager: @staticmethod def get_provider_config( custom_llm_provider: Optional[str], + model: Optional[str] = None, ) -> Optional[BaseA2AProviderConfig]: """ Get the provider configuration for a given custom_llm_provider. Args: custom_llm_provider: The provider identifier (e.g., "pydantic_ai_agents") + model: The model string (used to distinguish sub-providers, e.g. agentcore vs other bedrock) Returns: Provider configuration instance or None if not found @@ -39,9 +41,11 @@ class A2AProviderConfigManager: return PydanticAIProviderConfig() - # Add more providers here as needed - # elif custom_llm_provider == "another_provider": - # from litellm.a2a_protocol.providers.another_provider.config import AnotherProviderConfig - # return AnotherProviderConfig() + if custom_llm_provider == "bedrock" and model and "agentcore" in model: + from litellm.a2a_protocol.providers.bedrock_agentcore.config import ( + BedrockAgentCoreA2AConfig, + ) + + return BedrockAgentCoreA2AConfig() return None diff --git a/tests/test_litellm/a2a_protocol/providers/__init__.py b/tests/test_litellm/a2a_protocol/providers/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/a2a_protocol/providers/bedrock_agentcore/__init__.py b/tests/test_litellm/a2a_protocol/providers/bedrock_agentcore/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/a2a_protocol/providers/bedrock_agentcore/test_bedrock_agentcore_a2a.py b/tests/test_litellm/a2a_protocol/providers/bedrock_agentcore/test_bedrock_agentcore_a2a.py new file mode 100644 index 00000000000..f21faecaa2c --- /dev/null +++ b/tests/test_litellm/a2a_protocol/providers/bedrock_agentcore/test_bedrock_agentcore_a2a.py @@ -0,0 +1,327 @@ +""" +Tests for Bedrock AgentCore A2A provider. + +Verifies that: +- JSON-RPC envelopes are preserved (not stripped by the completion bridge) +- URLs are derived from the model ARN +- Auth uses JWT Bearer or SigV4 +- Config manager routes "bedrock" correctly +- Handler passes litellm_params and allows api_base=None +""" + +import json + +import pytest +from unittest.mock import AsyncMock, MagicMock, patch + + +SAMPLE_ARN = "arn:aws:bedrock-agentcore:us-west-2:123456789:runtime/my_agent" +SAMPLE_MODEL = f"bedrock/agentcore/{SAMPLE_ARN}" +SAMPLE_PARAMS = { + "message": { + "role": "user", + "parts": [{"kind": "text", "text": "what is 1+1?"}], + "messageId": "msg-001", + } +} +SAMPLE_LITELLM_PARAMS = { + "model": SAMPLE_MODEL, + "custom_llm_provider": "bedrock", + "api_key": "test-jwt-token", +} + + +class TestTransformation: + """Test URL construction and JSON-RPC envelope building.""" + + def test_json_rpc_envelope_structure(self): + """Verify JSON-RPC body has jsonrpc, method, id, and params.""" + from litellm.a2a_protocol.providers.bedrock_agentcore.transformation import ( + BedrockAgentCoreA2ATransformation, + ) + + url, headers, body = ( + BedrockAgentCoreA2ATransformation.get_url_and_signed_request( + request_id="req-001", + params=SAMPLE_PARAMS, + litellm_params=SAMPLE_LITELLM_PARAMS, + method="message/send", + ) + ) + body_dict = json.loads(body) + assert body_dict["jsonrpc"] == "2.0" + assert body_dict["method"] == "message/send" + assert body_dict["id"] == "req-001" + assert body_dict["params"] == SAMPLE_PARAMS + + def test_url_derived_from_arn(self): + """Verify URL is constructed from the ARN, not from api_base.""" + from litellm.a2a_protocol.providers.bedrock_agentcore.transformation import ( + BedrockAgentCoreA2ATransformation, + ) + + url, _, _ = BedrockAgentCoreA2ATransformation.get_url_and_signed_request( + request_id="req-001", + params=SAMPLE_PARAMS, + litellm_params=SAMPLE_LITELLM_PARAMS, + ) + assert "bedrock-agentcore.us-west-2.amazonaws.com" in url + assert "/runtimes/" in url + assert "/invocations" in url + + def test_jwt_auth_uses_bearer_header(self): + """When api_key is set, Authorization header uses Bearer token.""" + from litellm.a2a_protocol.providers.bedrock_agentcore.transformation import ( + BedrockAgentCoreA2ATransformation, + ) + + _, headers, _ = BedrockAgentCoreA2ATransformation.get_url_and_signed_request( + request_id="req-001", + params=SAMPLE_PARAMS, + litellm_params=SAMPLE_LITELLM_PARAMS, + ) + assert headers["Authorization"] == "Bearer test-jwt-token" + + def test_session_id_header_set(self): + """Verify X-Amzn-Bedrock-AgentCore-Runtime-Session-Id is set.""" + from litellm.a2a_protocol.providers.bedrock_agentcore.transformation import ( + BedrockAgentCoreA2ATransformation, + ) + + _, headers, _ = BedrockAgentCoreA2ATransformation.get_url_and_signed_request( + request_id="req-001", + params=SAMPLE_PARAMS, + litellm_params=SAMPLE_LITELLM_PARAMS, + ) + session_id = headers.get("X-Amzn-Bedrock-AgentCore-Runtime-Session-Id", "") + assert len(session_id) >= 33 + + def test_custom_session_id_header(self): + """Verify custom runtimeSessionId is used when provided.""" + from litellm.a2a_protocol.providers.bedrock_agentcore.transformation import ( + BedrockAgentCoreA2ATransformation, + ) + + params_with_session = {**SAMPLE_LITELLM_PARAMS, "runtimeSessionId": "a" * 40} + _, headers, _ = BedrockAgentCoreA2ATransformation.get_url_and_signed_request( + request_id="req-001", + params=SAMPLE_PARAMS, + litellm_params=params_with_session, + ) + assert headers["X-Amzn-Bedrock-AgentCore-Runtime-Session-Id"] == "a" * 40 + + def test_sigv4_auth_when_no_api_key(self): + """When no api_key, falls through to SigV4 signing.""" + from litellm.a2a_protocol.providers.bedrock_agentcore.transformation import ( + BedrockAgentCoreA2ATransformation, + ) + + litellm_params_no_key = { + "model": SAMPLE_MODEL, + "custom_llm_provider": "bedrock", + "aws_access_key_id": "AKIAIOSFODNN7EXAMPLE", + "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "aws_region_name": "us-west-2", + } + + # Mock _sign_request to avoid hitting real botocore credential resolution + fake_sigv4_headers = { + "Authorization": "AWS4-HMAC-SHA256 Credential=AKIA.../bedrock-agentcore/aws4_request", + "Content-Type": "application/json", + "Accept": "application/json, text/event-stream", + } + fake_body = b'{"jsonrpc":"2.0"}' + + with patch( + "litellm.llms.bedrock.chat.agentcore.transformation.AmazonAgentCoreConfig._sign_request", + return_value=(fake_sigv4_headers, fake_body), + ): + _, headers, _ = BedrockAgentCoreA2ATransformation.get_url_and_signed_request( + request_id="req-001", + params=SAMPLE_PARAMS, + litellm_params=litellm_params_no_key, + ) + # SigV4 produces an Authorization header starting with "AWS4-HMAC-SHA256" + assert "Authorization" in headers + assert headers["Authorization"].startswith("AWS4-HMAC-SHA256") + + +class TestNonStreaming: + """Test end-to-end non-streaming flow.""" + + @pytest.mark.asyncio + async def test_json_rpc_body_sent_to_agentcore(self): + """Verify the full JSON-RPC envelope is POSTed, not {"prompt": "..."}.""" + from litellm.a2a_protocol.providers.bedrock_agentcore.config import ( + BedrockAgentCoreA2AConfig, + ) + + mock_response = MagicMock() + mock_response.json.return_value = { + "jsonrpc": "2.0", + "id": "req-001", + "result": { + "message": { + "role": "agent", + "parts": [{"kind": "text", "text": "2"}], + "messageId": "resp-001", + } + }, + } + mock_response.raise_for_status = MagicMock() + + with patch( + "litellm.a2a_protocol.providers.bedrock_agentcore.handler.get_async_httpx_client" + ) as mock_get_client: + mock_client = AsyncMock() + mock_client.post = AsyncMock(return_value=mock_response) + mock_get_client.return_value = mock_client + + config = BedrockAgentCoreA2AConfig() + result = await config.handle_non_streaming( + request_id="req-001", + params=SAMPLE_PARAMS, + litellm_params=SAMPLE_LITELLM_PARAMS, + ) + + # Verify the POST was called + mock_client.post.assert_called_once() + call_kwargs = mock_client.post.call_args + + # Verify sent body is JSON-RPC, not {"prompt": "..."} + sent_body = json.loads(call_kwargs.kwargs["data"]) + assert "jsonrpc" in sent_body + assert "method" in sent_body + assert sent_body["method"] == "message/send" + assert sent_body["params"]["message"]["parts"][0]["text"] == "what is 1+1?" + + # Verify response is passed through + assert result["result"]["message"]["parts"][0]["text"] == "2" + + @pytest.mark.asyncio + async def test_a2a_error_response_passthrough(self): + """JSON-RPC error responses from the agent are returned as-is.""" + from litellm.a2a_protocol.providers.bedrock_agentcore.config import ( + BedrockAgentCoreA2AConfig, + ) + + error_response = { + "jsonrpc": "2.0", + "id": "req-001", + "error": {"code": -32600, "message": "Bad request"}, + } + mock_response = MagicMock() + mock_response.json.return_value = error_response + mock_response.raise_for_status = MagicMock() + + with patch( + "litellm.a2a_protocol.providers.bedrock_agentcore.handler.get_async_httpx_client" + ) as mock_get_client: + mock_client = AsyncMock() + mock_client.post = AsyncMock(return_value=mock_response) + mock_get_client.return_value = mock_client + + config = BedrockAgentCoreA2AConfig() + result = await config.handle_non_streaming( + request_id="req-001", + params=SAMPLE_PARAMS, + litellm_params=SAMPLE_LITELLM_PARAMS, + ) + + assert result["error"]["code"] == -32600 + assert result["error"]["message"] == "Bad request" + + +class TestConfigManager: + """Test that config manager routes 'bedrock' correctly.""" + + def test_bedrock_returns_config(self): + from litellm.a2a_protocol.providers.bedrock_agentcore.config import ( + BedrockAgentCoreA2AConfig, + ) + from litellm.a2a_protocol.providers.config_manager import ( + A2AProviderConfigManager, + ) + + config = A2AProviderConfigManager.get_provider_config( + "bedrock", model=SAMPLE_MODEL + ) + assert config is not None + assert isinstance(config, BedrockAgentCoreA2AConfig) + + def test_bedrock_non_agentcore_returns_none(self): + """Non-agentcore bedrock models should fall through to completion bridge.""" + from litellm.a2a_protocol.providers.config_manager import ( + A2AProviderConfigManager, + ) + + config = A2AProviderConfigManager.get_provider_config( + "bedrock", model="bedrock/anthropic.claude-3-sonnet" + ) + assert config is None + + def test_unknown_provider_returns_none(self): + from litellm.a2a_protocol.providers.config_manager import ( + A2AProviderConfigManager, + ) + + assert A2AProviderConfigManager.get_provider_config("unknown") is None + + +class TestHandlerIntegration: + """Test handler.py changes — litellm_params passed through, api_base not required.""" + + @pytest.mark.asyncio + async def test_provider_config_receives_litellm_params(self): + """Verify handler passes litellm_params to provider config via kwargs.""" + from litellm.a2a_protocol.litellm_completion_bridge.handler import ( + A2ACompletionBridgeHandler, + ) + + mock_config = AsyncMock() + mock_config.handle_non_streaming = AsyncMock( + return_value={"jsonrpc": "2.0", "id": "req-001", "result": {}} + ) + + with patch( + "litellm.a2a_protocol.litellm_completion_bridge.handler.A2AProviderConfigManager.get_provider_config", + return_value=mock_config, + ): + await A2ACompletionBridgeHandler.handle_non_streaming( + request_id="req-001", + params=SAMPLE_PARAMS, + litellm_params=SAMPLE_LITELLM_PARAMS, + api_base=None, + ) + + mock_config.handle_non_streaming.assert_called_once_with( + request_id="req-001", + params=SAMPLE_PARAMS, + api_base=None, + litellm_params=SAMPLE_LITELLM_PARAMS, + ) + + @pytest.mark.asyncio + async def test_api_base_none_allowed_with_provider_config(self): + """api_base=None no longer raises when a provider config is registered.""" + from litellm.a2a_protocol.litellm_completion_bridge.handler import ( + A2ACompletionBridgeHandler, + ) + + mock_config = AsyncMock() + mock_config.handle_non_streaming = AsyncMock( + return_value={"jsonrpc": "2.0", "id": "req-001", "result": {}} + ) + + with patch( + "litellm.a2a_protocol.litellm_completion_bridge.handler.A2AProviderConfigManager.get_provider_config", + return_value=mock_config, + ): + # Should NOT raise ValueError + result = await A2ACompletionBridgeHandler.handle_non_streaming( + request_id="req-001", + params=SAMPLE_PARAMS, + litellm_params=SAMPLE_LITELLM_PARAMS, + api_base=None, + ) + assert result is not None From 4c1ef4234d88cb8b9b01b0f1dd02d8eab3a81d64 Mon Sep 17 00:00:00 2001 From: michelligabriele Date: Sat, 4 Apr 2026 05:41:24 +0200 Subject: [PATCH 25/25] feat(ui): add guardrails support to project create/edit forms (#25100) --- .../hooks/projects/useCreateProject.ts | 1 + .../hooks/projects/useUpdateProject.ts | 1 + .../ProjectModals/EditProjectModal.tsx | 6 ++- .../ProjectModals/ProjectBaseForm.test.tsx | 13 +++++++ .../ProjectModals/ProjectBaseForm.tsx | 37 +++++++++++++++++++ .../ProjectModals/projectFormUtils.test.ts | 16 ++++++++ .../ProjectModals/projectFormUtils.ts | 3 ++ 7 files changed, 76 insertions(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/projects/useCreateProject.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/projects/useCreateProject.ts index 3943f23794e..e206c770b19 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/projects/useCreateProject.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/projects/useCreateProject.ts @@ -17,6 +17,7 @@ export interface ProjectCreateParams { models?: string[]; max_budget?: number; blocked?: boolean; + guardrails?: string[]; metadata?: Record; model_rpm_limit?: Record; model_tpm_limit?: Record; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/projects/useUpdateProject.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/projects/useUpdateProject.ts index e6cd3071f5f..2042c8fc7cd 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/projects/useUpdateProject.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/projects/useUpdateProject.ts @@ -17,6 +17,7 @@ export interface ProjectUpdateParams { models?: string[]; max_budget?: number; blocked?: boolean; + guardrails?: string[]; metadata?: Record; model_rpm_limit?: Record; model_tpm_limit?: Record; diff --git a/ui/litellm-dashboard/src/components/Projects/ProjectModals/EditProjectModal.tsx b/ui/litellm-dashboard/src/components/Projects/ProjectModals/EditProjectModal.tsx index dc3b43ef73c..6c65e518cfd 100644 --- a/ui/litellm-dashboard/src/components/Projects/ProjectModals/EditProjectModal.tsx +++ b/ui/litellm-dashboard/src/components/Projects/ProjectModals/EditProjectModal.tsx @@ -33,6 +33,9 @@ export function EditProjectModal({ const metadataObj = (project.metadata ?? {}) as Record; const rpmLimits = (metadataObj.model_rpm_limit ?? {}) as Record; const tpmLimits = (metadataObj.model_tpm_limit ?? {}) as Record; + const guardrails = (Array.isArray(metadataObj.guardrails) + ? metadataObj.guardrails + : []) as string[]; const modelLimits: ProjectFormValues["modelLimits"] = []; const allLimitModels = new Set([ @@ -48,7 +51,7 @@ export function EditProjectModal({ } // Filter out internal keys from user-facing metadata - const internalKeys = new Set(["model_rpm_limit", "model_tpm_limit"]); + const internalKeys = new Set(["model_rpm_limit", "model_tpm_limit", "guardrails"]); const metadata: ProjectFormValues["metadata"] = []; for (const [key, value] of Object.entries(metadataObj)) { if (!internalKeys.has(key)) { @@ -63,6 +66,7 @@ export function EditProjectModal({ models: project.models ?? [], max_budget: project.litellm_budget_table?.max_budget ?? undefined, isBlocked: project.blocked, + guardrails: guardrails.length > 0 ? guardrails : undefined, modelLimits: modelLimits.length > 0 ? modelLimits : undefined, metadata: metadata.length > 0 ? metadata : undefined, }); diff --git a/ui/litellm-dashboard/src/components/Projects/ProjectModals/ProjectBaseForm.test.tsx b/ui/litellm-dashboard/src/components/Projects/ProjectModals/ProjectBaseForm.test.tsx index 04e3ed64f47..d8532146566 100644 --- a/ui/litellm-dashboard/src/components/Projects/ProjectModals/ProjectBaseForm.test.tsx +++ b/ui/litellm-dashboard/src/components/Projects/ProjectModals/ProjectBaseForm.test.tsx @@ -14,6 +14,10 @@ vi.mock("@/components/organisms/create_key_button", () => ({ fetchTeamModels: vi.fn().mockResolvedValue([]), })); +vi.mock("@/components/networking", () => ({ + getGuardrailsList: vi.fn().mockResolvedValue({ guardrails: [] }), +})); + vi.mock("@/components/key_team_helpers/fetch_available_models_team_key", () => ({ getModelDisplayName: (model: string) => model, })); @@ -86,4 +90,13 @@ describe("ProjectBaseForm", () => { renderWithProviders(); expect(screen.getByText("Advanced Settings")).toBeInTheDocument(); }); + + it("should show a Guardrails field in the Advanced Settings section", async () => { + const user = userEvent.setup(); + renderWithProviders(); + await user.click(screen.getByText("Advanced Settings")); + await waitFor(() => { + expect(screen.getByText("Guardrails")).toBeInTheDocument(); + }); + }); }); diff --git a/ui/litellm-dashboard/src/components/Projects/ProjectModals/ProjectBaseForm.tsx b/ui/litellm-dashboard/src/components/Projects/ProjectModals/ProjectBaseForm.tsx index bf1eca882c3..81d8fabe084 100644 --- a/ui/litellm-dashboard/src/components/Projects/ProjectModals/ProjectBaseForm.tsx +++ b/ui/litellm-dashboard/src/components/Projects/ProjectModals/ProjectBaseForm.tsx @@ -22,6 +22,7 @@ import { useTeams } from "@/app/(dashboard)/hooks/teams/useTeams"; import { Team } from "../../key_team_helpers/key_list"; import { fetchTeamModels } from "../../organisms/create_key_button"; import { getModelDisplayName } from "../../key_team_helpers/fetch_available_models_team_key"; +import { getGuardrailsList } from "@/components/networking"; export interface ProjectFormValues { project_alias: string; @@ -30,6 +31,7 @@ export interface ProjectFormValues { models: string[]; max_budget?: number; isBlocked: boolean; + guardrails?: string[]; modelLimits?: { model: string; tpm?: number; rpm?: number }[]; metadata?: { key: string; value: string }[]; } @@ -46,6 +48,23 @@ export function ProjectBaseForm({ const [selectedTeam, setSelectedTeam] = useState(null); const [modelsToPick, setModelsToPick] = useState([]); + const [guardrailsList, setGuardrailsList] = useState([]); + + useEffect(() => { + const fetchGuardrails = async () => { + if (!accessToken) return; + try { + const response = await getGuardrailsList(accessToken); + const names = response.guardrails.map( + (g: { guardrail_name: string }) => g.guardrail_name + ); + setGuardrailsList(names); + } catch (error) { + console.error("Failed to fetch guardrails:", error); + } + }; + fetchGuardrails(); + }, [accessToken]); // Sync selectedTeam from form value (needed for edit mode pre-fill) const teamIdValue = Form.useWatch("team_id", form); @@ -259,6 +278,24 @@ export function ProjectBaseForm({ + +