Merge branch 'main' into flex-prio

.circleci/config.yml

@@ -1559,6 +1559,25 @@ jobs:
             pip install "pytest-asyncio==0.21.1"
             pip install aiohttp
             pip install apscheduler
+      - run:
+          name: Install dockerize
+          command: |
+            sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
+            sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
+            sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
+      - run:
+          name: Start PostgreSQL Database
+          command: |
+            docker run -d \
+              --name postgres-db \
+              -e POSTGRES_USER=postgres \
+              -e POSTGRES_PASSWORD=postgres \
+              -e POSTGRES_DB=litellm_test \
+              -p 5432:5432 \
+              postgres:14
+      - run:
+          name: Wait for PostgreSQL to be ready
+          command: dockerize -wait tcp://localhost:5432 -timeout 1m
       - attach_workspace:
           at: ~/project
       - run:
@@ -1567,29 +1586,41 @@ jobs:
             zstd -d litellm-docker-database.tar.zst --stdout | docker load
             docker images | grep litellm-docker-database
       - run:
-          name: Run Docker container
+          name: Seed database with real schema
+          command: |
+            docker run -d \
+              -p 4001:4000 \
+              -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \
+              -e LITELLM_MASTER_KEY="sk-1234" \
+              --name schema-seed \
+              --add-host=host.docker.internal:host-gateway \
+              -v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \
+              litellm-docker-database:ci \
+              --config /app/config.yaml \
+              --port 4000 \
+              --use_prisma_db_push
+      - run:
+          name: Wait for schema seed to complete
+          command: dockerize -wait http://localhost:4001 -timeout 5m
+      - run:
+          name: Stop schema seed container
+          command: docker stop schema-seed && docker rm schema-seed
+      - run:
+          name: Run Docker container with bad schema and disabled updates
           command: |
             docker run -d \
               -p 4000:4000 \
-              -e DATABASE_URL=$PROXY_DATABASE_URL \
+              -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \
               -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \
               -e DISABLE_SCHEMA_UPDATE="True" \
+              --name my-app \
+              --add-host=host.docker.internal:host-gateway \
               -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/schema.prisma \
               -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/litellm/proxy/schema.prisma \
               -v $(pwd)/litellm/proxy/example_config_yaml/disable_schema_update.yaml:/app/config.yaml \
-              --name my-app \
               litellm-docker-database:ci \
               --config /app/config.yaml \
               --port 4000
-      - run:
-          name: Install curl and dockerize
-          command: |
-            sudo apt-get update
-            sudo apt-get install -y curl
-            sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
-            sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
-            sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
-
       - run:
           name: Wait for container to be ready
           command: dockerize -wait http://localhost:4000 -timeout 1m
@@ -1997,9 +2028,6 @@ jobs:
               -e OPENAI_API_KEY=$OPENAI_API_KEY \
               -e LITELLM_LICENSE=$LITELLM_LICENSE \
               -e OTEL_EXPORTER="in_memory" \
-              -e APORIA_API_BASE_2=$APORIA_API_BASE_2 \
-              -e APORIA_API_KEY_2=$APORIA_API_KEY_2 \
-              -e APORIA_API_BASE_1=$APORIA_API_BASE_1 \
               -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
               -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
               -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \
@@ -2007,7 +2035,6 @@ jobs:
               -e DD_API_KEY=$DD_API_KEY \
               -e DD_SITE=$DD_SITE \
               -e AWS_REGION_NAME=$AWS_REGION_NAME \
-              -e APORIA_API_KEY_1=$APORIA_API_KEY_1 \
               -e COHERE_API_KEY=$COHERE_API_KEY \
               -e GCS_FLUSH_INTERVAL="1" \
               --add-host host.docker.internal:host-gateway \
@@ -2483,6 +2510,20 @@ jobs:
           name: Build Docker image
           command: |
             docker build -t my-app:latest -f docker/build_from_pip/Dockerfile.build_from_pip .
+      - run:
+          name: Start PostgreSQL Database
+          command: |
+            docker run -d \
+              --name postgres-db \
+              -e POSTGRES_USER=postgres \
+              -e POSTGRES_PASSWORD=postgres \
+              -e POSTGRES_DB=circle_test \
+              -p 5432:5432 \
+              postgres:14
+      - run:
+          name: Wait for PostgreSQL to be ready
+          command: |
+            timeout 60s bash -c 'until docker exec postgres-db pg_isready -U postgres -d circle_test; do sleep 2; done'
       - run:
           name: Run Docker container
           # intentionally give bad redis credentials here
@@ -2490,7 +2531,7 @@ jobs:
           command: |
             docker run -d \
               -p 4000:4000 \
-              -e DATABASE_URL=$PROXY_DATABASE_URL \
+              -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
               -e REDIS_HOST=$REDIS_HOST \
               -e REDIS_PASSWORD=$REDIS_PASSWORD \
               -e REDIS_PORT=$REDIS_PORT \
@@ -2498,18 +2539,15 @@ jobs:
               -e OPENAI_API_KEY=$OPENAI_API_KEY \
               -e LITELLM_LICENSE=$LITELLM_LICENSE \
               -e OTEL_EXPORTER="in_memory" \
-              -e APORIA_API_BASE_2=$APORIA_API_BASE_2 \
-              -e APORIA_API_KEY_2=$APORIA_API_KEY_2 \
-              -e APORIA_API_BASE_1=$APORIA_API_BASE_1 \
               -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
               -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
               -e AWS_REGION_NAME=$AWS_REGION_NAME \
-              -e APORIA_API_KEY_1=$APORIA_API_KEY_1 \
               -e COHERE_API_KEY=$COHERE_API_KEY \
               -e USE_DDTRACE=True \
               -e DD_API_KEY=$DD_API_KEY \
               -e DD_SITE=$DD_SITE \
               -e GCS_FLUSH_INTERVAL="1" \
+              --add-host host.docker.internal:host-gateway \
               --name my-app \
               -v $(pwd)/docker/build_from_pip/litellm_config.yaml:/app/config.yaml \
               my-app:latest \
@@ -2540,8 +2578,11 @@ jobs:
       - run:
           name: Stop and remove first container
           command: |
-            docker stop my-app
-            docker rm my-app
+            docker stop my-app || true
+            docker rm my-app || true
+            docker stop postgres-db || true
+            docker rm postgres-db || true
+          when: always
   proxy_pass_through_endpoint_tests:
     machine:
       image: ubuntu-2204:2023.10.1
@@ -3277,37 +3318,39 @@ jobs:
       - setup_google_dns
       - attach_workspace:
           at: ~/project
+      - run:
+          name: Install dockerize
+          command: |
+            sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
+            sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
+            sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
+      - run:
+          name: Start PostgreSQL Database
+          command: |
+            docker run -d \
+              --name postgres-db \
+              -e POSTGRES_USER=postgres \
+              -e POSTGRES_PASSWORD=postgres \
+              -e POSTGRES_DB=litellm_schema_sync \
+              -p 5432:5432 \
+              postgres:14
+      - run:
+          name: Wait for PostgreSQL to be ready
+          command: dockerize -wait tcp://localhost:5432 -timeout 1m
       - run:
           name: Load Docker Database Image
           command: |
             zstd -d litellm-docker-database.tar.zst --stdout | docker load
             docker images | grep litellm-docker-database
       - run:
-          name: Install Neon CLI
+          name: Run schema sync via prisma db push
           command: |
-            npm i -g neonctl
-      - run:
-          name: Install curl and dockerize
-          command: |
-            sudo apt-get update
-            sudo apt-get install -y curl
-            sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
-            sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
-            sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
-      - run:
-          name: Sync schema on base e2e database
-          command: |
-            BASE_DATABASE_URL=$(neon connection-string \
-              --project-id $NEON_PROJECT_ID \
-              --api-key $NEON_API_KEY \
-              --branch br-fancy-paper-ad1olsb3 \
-              --database-name yuneng-trial-db \
-              --role neondb_owner)
             docker run -d \
               -p 4000:4000 \
-              -e DATABASE_URL=$BASE_DATABASE_URL \
+              -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_schema_sync" \
               -e LITELLM_MASTER_KEY="sk-1234" \
               --name schema-sync \
+              --add-host=host.docker.internal:host-gateway \
               -v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \
               litellm-docker-database:ci \
               --config /app/config.yaml \
@@ -3485,7 +3528,6 @@ workflows:
                 - main
                 - /litellm_.*/
       - prisma_schema_sync:
-          context: e2e_ui_tests
           requires:
             - build_docker_database_image
           filters:
@@ -3493,32 +3535,32 @@ workflows:
               only:
                 - main
                 - /litellm_.*/
-      - e2e_ui_testing:
-          name: e2e_ui_testing_chromium
-          browser: chromium
-          context: e2e_ui_tests
-          requires:
-            - ui_build
-            - build_docker_database_image
-            - prisma_schema_sync
-          filters:
-            branches:
-              only:
-                - main
-                - /litellm_.*/
-      - e2e_ui_testing:
-          name: e2e_ui_testing_firefox
-          browser: firefox
-          context: e2e_ui_tests
-          requires:
-            - ui_build
-            - build_docker_database_image
-            - prisma_schema_sync
-          filters:
-            branches:
-              only:
-                - main
-                - /litellm_.*/
+      # - e2e_ui_testing: # migrate to dynamic db - currently requires neon cli 
+      #     name: e2e_ui_testing_chromium
+      #     browser: chromium
+      #     context: e2e_ui_tests
+      #     requires:
+      #       - ui_build
+      #       - build_docker_database_image
+      #       - prisma_schema_sync
+      #     filters:
+      #       branches:
+      #         only:
+      #           - main
+      #           - /litellm_.*/
+      # - e2e_ui_testing:
+      #     name: e2e_ui_testing_firefox
+      #     browser: firefox
+      #     context: e2e_ui_tests
+      #     requires:
+      #       - ui_build
+      #       - build_docker_database_image
+      #       - prisma_schema_sync
+      #     filters:
+      #       branches:
+      #         only:
+      #           - main
+      #           - /litellm_.*/
       - build_and_test:
           requires:
             - build_docker_database_image

.devcontainer/post-create.sh

@@ -12,6 +12,6 @@ echo "[post-create] Generating Prisma client"
 poetry run prisma generate
 
 echo "[post-create] Installing npm dependencies"
-cd ui/litellm-dashboard && npm install --no-audit --no-fund
+cd ui/litellm-dashboard && npm ci
 
 echo "[post-create] Done"

.github/workflows/create-release.yml

@@ -0,0 +1,93 @@
+name: Create Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Release tag (e.g. v1.83.0-stable)"
+        required: true
+        type: string
+      commit_hash:
+        description: "Full 40-char commit SHA to target"
+        required: true
+        type: string
+
+permissions: {}
+
+jobs:
+  release:
+    name: Create Release
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - name: Validate inputs
+        env:
+          TAG: ${{ inputs.tag }}
+          COMMIT_HASH: ${{ inputs.commit_hash }}
+        run: |
+          if ! echo "${COMMIT_HASH}" | grep -qE '^[0-9a-f]{40}$'; then
+            echo "::error::commit_hash must be a full 40-character commit SHA"
+            exit 1
+          fi
+          if ! echo "${TAG}" | grep -qE '^v[0-9]+\.[0-9]+\.[0-9]+'; then
+            echo "::error::tag must start with vX.Y.Z"
+            exit 1
+          fi
+
+      - name: Create release
+        env:
+          TAG: ${{ inputs.tag }}
+          COMMIT_HASH: ${{ inputs.commit_hash }}
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        with:
+          script: |
+            const tag = process.env.TAG;
+            const commitHash = process.env.COMMIT_HASH;
+
+            const cosignSection = [
+              `## Verify Docker Image Signature`,
+              ``,
+              `All LiteLLM Docker images are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). To verify the integrity of an image before deploying:`,
+              ``,
+              '```bash',
+              `cosign verify \\`,
+              `  --key https://raw.githubusercontent.com/BerriAI/litellm/${tag}/cosign.pub \\`,
+              `  ghcr.io/berriai/litellm:${tag}`,
+              '```',
+              ``,
+              `Expected output:`,
+              ``,
+              '```',
+              `The following checks were performed on each of these signatures:`,
+              `  - The cosign claims were validated`,
+              `  - The signatures were verified against the specified public key`,
+              '```',
+              ``,
+              `---`,
+              ``,
+            ].join('\n');
+
+            try {
+              const response = await github.rest.repos.createRelease({
+                draft: true,
+                generate_release_notes: true,
+                target_commitish: commitHash,
+                name: tag,
+                owner: context.repo.owner,
+                prerelease: false,
+                repo: context.repo.repo,
+                tag_name: tag,
+              });
+
+              const updatedBody = cosignSection + (response.data.body ?? '');
+              await github.rest.repos.updateRelease({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                release_id: response.data.id,
+                body: updatedBody,
+                draft: false,
+              });
+            } catch (error) {
+              core.setFailed(error.message);
+            }

.github/workflows/scorecard.yml

@@ -0,0 +1,47 @@
+name: Scorecard supply-chain security
+
+on:
+  branch_protection_rule:
+  schedule:
+    - cron: '27 12 * * 4'
+  push:
+    branches: ["main"]
+
+permissions: read-all
+
+jobs:
+  analysis:
+    name: Scorecard analysis
+    runs-on: ubuntu-latest
+    if: github.event.repository.default_branch == github.ref_name
+    permissions:
+      security-events: write
+      id-token: write
+      # Uncomment for private repos if needed:
+      # contents: read
+      # actions: read
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          persist-credentials: false
+
+      - name: Run analysis
+        uses: ossf/scorecard-action@f49aabe0b5af0936a0987cfb85d86b75731b0186 # v2.4.1
+        with:
+          results_file: results.sarif
+          results_format: sarif
+          publish_results: true
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+        with:
+          name: SARIF file
+          path: results.sarif
+          retention-days: 5
+
+      - name: Upload to code scanning
+        uses: github/codeql-action/upload-sarif@c10b806170c8ee63ea24152429041b5624f0baf5 # v4.35.1
+        with:
+          sarif_file: results.sarif

.github/workflows/test-litellm-ui-build.yml

@@ -28,7 +28,7 @@ jobs:
           cache-dependency-path: ui/litellm-dashboard/package-lock.json
 
       - name: Install dependencies
-        run: npm install
+        run: npm ci
 
       - name: Build
         run: npm run build

.npmrc

@@ -0,0 +1,5 @@
+# Supply-chain hardening
+# Packages needing lifecycle scripts: npm rebuild <pkg>
+ignore-scripts=true
+# Protects local npm install only — npm ci (used in CI) ignores this
+min-release-age=3d

ci_cd/security_scans.sh

@@ -160,7 +160,7 @@ run_grype_scans() {
         "CVE-2026-0775" # npm cli incorrect permission assignment - no fix available yet, npm is only used at build/prisma-generate time
         "GHSA-3ppc-4f35-3m26" # minimatch ReDoS via repeated wildcards - from nodejs_wheel bundled npm, not used in application runtime code
         "GHSA-83g3-92jg-28cx" # tar arbitrary file read/write via hardlink - from nodejs_wheel bundled npm, not used in application runtime code
-        "CVE-2026-25639" # axios - full fix requires 1.x major version bump; pinned to >=0.30.2 to clear other axios CVEs, upgrade to 1.x in follow-up
+        "CVE-2026-25639" # axios DoS via __proto__ in mergeConfig - transitive dev dep via @neondatabase/api-client, not imported in application code
         "CVE-2026-2297" # Python 3.13 SourcelessFileLoader audit hook bypass - no fix available in base image
         "GHSA-qffp-2rhf-9h96" # tar hardlink path traversal - from nodejs_wheel bundled npm, not used in application runtime code
         "CVE-2026-2673" # OpenSSL 3.6.1 TLS 1.3 key exchange group negotiation issue - no fix available yet

cosign.pub

@@ -0,0 +1,4 @@
+-----BEGIN PUBLIC KEY-----
+MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEKi4ivqGpE231OGH50PKbqy1Y1Kkb
+POJC8+i2Wko82gBOUCe3M0Vw86H/4rhUhfoYEti4gdJ9wZbYmK0I2EE96g==
+-----END PUBLIC KEY-----

docker/Dockerfile.custom_ui

@@ -51,7 +51,7 @@ ENV UI_BASE_PATH="/prod/ui"
 
 # Build the UI with the specified UI_BASE_PATH
 WORKDIR /app/ui/litellm-dashboard
-RUN npm install
+RUN npm ci
 RUN UI_BASE_PATH=$UI_BASE_PATH npm run build
 
 # Create the destination directory

docker/Dockerfile.non_root

@@ -47,7 +47,7 @@ RUN mkdir -p /var/lib/litellm/ui && \
     if [ -f "/app/enterprise/enterprise_ui/enterprise_colors.json" ]; then \
       cp /app/enterprise/enterprise_ui/enterprise_colors.json ./ui_colors.json; \
     fi && \
-    npm install --legacy-peer-deps && \
+    npm ci && \
     npm run build && \
     cp -r /app/ui/litellm-dashboard/out/* /var/lib/litellm/ui/ && \
     mkdir -p /var/lib/litellm/assets && \

docker/build_admin_ui.sh

@@ -40,11 +40,22 @@ else
         exit 1
     fi
 fi
-curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh | bash
+NVM_VERSION="v0.40.4"
+NVM_CHECKSUM="4b7412c49960c7d31e8df72da90c1fb5b8cccb419ac99537b737028d497aba4f"
+NVM_SCRIPT=$(mktemp)
+trap 'rm -f "$NVM_SCRIPT"' EXIT
+curl -fsSL "https://raw.githubusercontent.com/nvm-sh/nvm/${NVM_VERSION}/install.sh" -o "$NVM_SCRIPT"
+if command -v sha256sum &>/dev/null; then
+  echo "${NVM_CHECKSUM}  ${NVM_SCRIPT}" | sha256sum -c -
+elif command -v shasum &>/dev/null; then
+  echo "${NVM_CHECKSUM}  ${NVM_SCRIPT}" | shasum -a 256 -c -
+else
+  echo "No sha256 tool found; cannot verify nvm checksum"; exit 1
+fi || { echo "nvm checksum verification failed"; exit 1; }
+bash "$NVM_SCRIPT"
 source ~/.nvm/nvm.sh
 nvm install v18.17.0
 nvm use v18.17.0
-npm install -g npm
 
 # copy _enterprise.json from this directory to /ui/litellm-dashboard, and rename it to ui_colors.json
 cp enterprise/enterprise_ui/enterprise_colors.json ui/litellm-dashboard/ui_colors.json

docs/my-website/blog/ci_cd_v2_improvements/index.md

@@ -0,0 +1,55 @@
+---
+slug: ci-cd-v2-improvements
+title: "Announcing CI/CD v2 for LiteLLM"
+date: 2026-03-30T21:30:00
+authors:
+  - krrish
+description: "CI/CD v2 introduces isolated environments, stronger security gates, and safer release separation for LiteLLM."
+tags: [engineering, ci-cd, security]
+hide_table_of_contents: false
+---
+
+import Image from '@theme/IdealImage';
+
+The CI/CD v2 is now live for LiteLLM.
+
+<Image
+  img={require('../../img/ci_cd_architecture.png')}
+  style={{width: '700px', height: 'auto', display: 'block'}}
+/>
+
+<br/>
+Building on the roadmap from our [security incident](https://docs.litellm.ai/blog/security-townhall-updates#roadmap), CI/CD v2 introduces isolated environments, stronger security gates, and safer release separation for LiteLLM.
+
+## What changed
+
+- Security scans and unit tests run in isolated environments.
+- Validation and release are separated into different repositories, making it harder for an attacker to reach release credentials.
+- Trusted Publishing for PyPI releases - this means no long-lived credentials are used to publish releases.
+- Immutable Docker release tags - this means no tampering of Docker release tags after they are published [Learn more](https://docs.docker.com/docker-hub/repos/manage/hub-images/immutable-tags/). Note: work for GHCR docker releases is planned as well.
+
+## What's next
+
+Moving forward, we plan on:
+- Adopting OpenSSF (this is a set of security criteria that projects should meet to demonstrate a strong security posture - [Learn more](https://baseline.openssf.org/versions/2026-02-19.html))
+  - We've added Scorecard and Allstar to our Github
+
+- Adding SLSA Build Provenance to our CI/CD pipeline - this means we allow users to independently verify that a release came from us and prevent silent modifications of releases after they are published.
+
+
+We hope that this will mean you can be confident that the releases you are using are safe and from us.
+
+
+## The principle
+
+The new CI/CD pipeline reflects the principles, outlined below, and is designed to be more secure and reliable:
+
+- **Limit** what each package can access
+- **Reduce** the number of sensitive environment variables
+- **Avoid** compromised packages
+- **Prevent** release tampering
+
+
+## How to help: 
+
+Help us plan April's stability sprint - https://github.com/BerriAI/litellm/issues/24825

docs/my-website/blog/security_update_march_2026/index.md

@@ -17,6 +17,10 @@ import VersionVerificationTable from '@site/src/components/VersionVerificationTa
 > **Status:** Active investigation
 > **Last updated:** March 27, 2026
 
+> **Update (March 30):** A new **clean** version of LiteLLM is now available (v1.83.0). This was released by our new [CI/CD v2](https://docs.litellm.ai/blog/ci-cd-v2-improvements) pipeline which added isolated environments, stronger security gates, and safer release separation for LiteLLM.
+
+> **Update (March 27):** Review Townhall updates, including explanation of the incident, what we've done, and what comes next. [Learn more](https://docs.litellm.ai/blog/security-townhall-updates)
+
 > **Update (March 27):** Added [Verified safe versions](#verified-safe-versions) section with SHA-256 checksums for all audited PyPI and Docker releases.
 
 > **Update (March 26):** Added `checkmarx[.]zone` to [Indicators of compromise](#indicators-of-compromise-iocs)
@@ -25,10 +29,10 @@ import VersionVerificationTable from '@site/src/components/VersionVerificationTa
 
 
 ## TLDR; 
-- The compromised PyPI packages were **litellm==1.82.7** and **litellm==1.82.8**. Those packages have now been removed from PyPI.
-- We believe that the compromise originated from the Trivy dependency used in our CI/CD security scanning workflow.
+- The compromised PyPI packages were **litellm==1.82.7** and **litellm==1.82.8**. Those packages were live on March 24, 2026 from 10:39 UTC for about 40 minutes before being quarantined by PyPI.
+- We believe that the compromise originated from the [Trivy dependency](https://www.aquasec.com/blog/trivy-supply-chain-attack-what-you-need-to-know/) used in our CI/CD security scanning workflow.
 - Customers running the official LiteLLM Proxy Docker image were not impacted. That deployment path pins dependencies in requirements.txt and does not rely on the compromised PyPI packages.
-- We are pausing new LiteLLM releases until we complete a broader supply-chain review and confirm the release path is safe.
+- ~~We have paused all new LiteLLM releases until we complete a broader supply-chain review and confirm the release path is safe.~~ **Updated:** We have now released a new **safe** version of LiteLLM (v1.83.0) by our new [CI/CD v2](https://docs.litellm.ai/blog/ci-cd-v2-improvements) pipeline which added isolated environments, stronger security gates, and safer release separation for LiteLLM. We have also verified the codebase is safe and no malicious code was pushed to `main`.
 
 
 ## Overview

docs/my-website/blog/vanta_compliance_recertification/index.md

@@ -0,0 +1,18 @@
+---
+slug: vanta-compliance-recertification
+title: "LiteLLM + Vanta: SOC 2 Type 2 and ISO 27001 Recertification"
+date: 2026-03-30T10:00:00
+authors:
+  - krrish
+description: "LiteLLM is partnering with Vanta on SOC 2 Type 2 and ISO 27001 recertification and engaging independent auditors for verification."
+tags: [security, compliance]
+hide_table_of_contents: true
+---
+
+![LiteLLM x Vanta SOC-2 Recertification](/img/blog/vanta_soc2_recertification.png)
+
+We are partnering with [Vanta](https://www.vanta.com/) to recertify LiteLLM's compliance for SOC 2 Type 2 and ISO 27001.
+
+As part of this process, we are also identifying independent auditors to validate and verify our compliance posture.
+
+This is part of our commitment to being the most secure and transparent AI Gateway possible.

docs/my-website/docs/proxy/high_availability_control_plane.md

@@ -42,9 +42,9 @@ The **High Availability Control Plane** takes a different approach:
 
 <ControlPlaneArchitecture />
 
-The **control plane** is a LiteLLM instance that serves the admin UI and knows about all the workers. It does not proxy LLM requests, it is purely for administration.
+The **control plane** is a LiteLLM instance that serves the admin UI and knows about all the workers. It is **not a router** — it does not proxy or route any LLM requests. It exists purely so admins can switch between workers and manage them from a single UI.
 
-Each **worker** is a fully independent LiteLLM proxy that handles LLM requests for its region or team. Workers have their own users, keys, teams, and budgets.
+Each **worker** is a fully independent LiteLLM proxy that handles LLM requests for its region or team. Workers have their own database, Redis, users, keys, teams, and budgets. No infrastructure is shared between workers.
 
 ## Setup
 

docs/my-website/docs/proxy/jwt_key_mapping.md

@@ -0,0 +1,318 @@
+# JWT → Virtual Key Mapping
+
+:::info Enterprise
+
+JWT → Virtual Key Mapping is an Enterprise feature.
+
+[Get a free trial](https://enterprise.litellm.ai/demo)
+
+:::
+
+Map JWT tokens to LiteLLM virtual keys — so every JWT client gets the same granular controls as a virtual key: model restrictions, spend limits, rate limits, guardrails, and full spend tracking.
+
+**Why this matters:** Standard JWT auth maps a JWT to a *team*. That's a shared boundary — all clients under a team share the same limits. With JWT → Virtual Key Mapping, each individual JWT client (identified by a claim like `client_id`, `azp`, or `sub`) maps to its own virtual key. You get per-client accountability without issuing API keys to your users.
+
+**Common use case:** Your company uses SSO/OIDC. Developers use Claude Code with their identity tokens. You want to enforce per-developer model access and spend limits without giving each person a LiteLLM API key.
+
+---
+
+## How It Works
+
+```mermaid
+sequenceDiagram
+    participant Client as Client (Claude Code / API)
+    participant Proxy as LiteLLM Proxy
+    participant OIDC as OIDC Provider
+    participant DB as Mapping Table
+
+    Client->>Proxy: POST /v1/chat/completions<br/>Authorization: Bearer <JWT>
+
+    Proxy->>OIDC: Verify JWT signature
+    OIDC-->>Proxy: Valid ✓
+
+    Proxy->>Proxy: Extract claim<br/>(e.g. client_id = "alice@corp.com")
+
+    Proxy->>DB: Look up (claim_name, claim_value)
+    alt Mapping found
+        DB-->>Proxy: virtual_key_id = sk-abc123
+        Proxy->>Proxy: Apply virtual key permissions<br/>(models, budget, rate limits)
+        Proxy-->>Client: 200 OK
+    else No mapping — fallback_team_mapping
+        Proxy->>Proxy: Fall through to team JWT auth
+        Proxy-->>Client: 200 OK
+    else No mapping — reject
+        Proxy-->>Client: 403 Forbidden
+    else No mapping — auto_register
+        Proxy->>DB: Create new virtual key + mapping
+        Proxy-->>Client: 200 OK
+    end
+```
+
+---
+
+## Setup
+
+### Prerequisites
+
+Complete [OIDC JWT Auth setup](./token_auth.md) first — you need `JWT_PUBLIC_KEY_URL` configured and `enable_jwt_auth: True` in your proxy config.
+
+### Step 1. Configure the JWT claim to map on
+
+Add `jwt_client_id_field` to your `litellm_jwtauth` config. This is the JWT claim LiteLLM uses as the lookup key:
+
+```yaml
+general_settings:
+  master_key: sk-1234
+  enable_jwt_auth: True
+  litellm_jwtauth:
+    team_id_jwt_field: "team_id"          # existing team mapping (optional)
+    user_id_jwt_field: "sub"
+    jwt_client_id_field: "client_id"      # 👈 claim used for key mapping
+    unregistered_jwt_client_behavior: "fallback_team_mapping"  # see below
+```
+
+**`unregistered_jwt_client_behavior`** controls what happens when a JWT has no registered mapping:
+
+| Value | Behavior |
+|-------|----------|
+| `fallback_team_mapping` | Fall through to team-based JWT auth (default — backward compatible) |
+| `reject` | Return 403 if no mapping found |
+| `auto_register` | Auto-create a virtual key + mapping on first encounter |
+
+### Step 2. Register a JWT client → virtual key mapping
+
+**Option A: Single call (creates key + mapping atomically)**
+
+```bash
+curl -X POST 'http://0.0.0.0:4000/jwt_client/new' \
+  -H 'Authorization: Bearer <PROXY_MASTER_KEY>' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "jwt_claim_name": "client_id",
+    "jwt_claim_value": "dev-alice",
+    "models": ["claude-sonnet-4-5", "claude-haiku-4-5"],
+    "max_budget": 50.0,
+    "budget_duration": "30d",
+    "rpm_limit": 100,
+    "tpm_limit": 50000,
+    "team_id": "engineering"
+  }'
+```
+
+Response includes the virtual key token (only shown on creation):
+
+```json
+{
+  "key": "sk-abc123...",
+  "key_id": "key_123",
+  "mapping_id": "mapping_456",
+  "jwt_claim_name": "client_id",
+  "jwt_claim_value": "dev-alice"
+}
+```
+
+**Option B: Map an existing virtual key**
+
+```bash
+curl -X POST 'http://0.0.0.0:4000/jwt/key/mapping/new' \
+  -H 'Authorization: Bearer <PROXY_MASTER_KEY>' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "jwt_claim_name": "client_id",
+    "jwt_claim_value": "dev-alice",
+    "virtual_key_id": "key_123"
+  }'
+```
+
+### Step 3. Test it
+
+```bash
+# Get a JWT from your OIDC provider (must have client_id: dev-alice)
+JWT_TOKEN="eyJhbG..."
+
+curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
+  -H "Authorization: Bearer $JWT_TOKEN" \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "model": "claude-sonnet-4-5",
+    "messages": [{"role": "user", "content": "Hello"}]
+  }'
+```
+
+The request is now tracked against `dev-alice`'s virtual key — spend, rate limits, and model access enforced per-client.
+
+---
+
+## Walkthrough: Admin grants granular access, team uses Claude Code
+
+This is the full flow for an engineering team using Claude Code with company SSO.
+
+### Admin setup
+
+**1. Create a team for engineering**
+
+```bash
+curl -X POST 'http://0.0.0.0:4000/team/new' \
+  -H 'Authorization: Bearer <MASTER_KEY>' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "team_alias": "engineering",
+    "models": ["claude-sonnet-4-5", "claude-haiku-4-5"]
+  }'
+```
+
+**2. Register each developer with their own key and spend limit**
+
+```bash
+# Alice — senior eng, higher budget
+curl -X POST 'http://0.0.0.0:4000/jwt_client/new' \
+  -H 'Authorization: Bearer <MASTER_KEY>' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "jwt_claim_name": "client_id",
+    "jwt_claim_value": "alice@corp.com",
+    "team_id": "engineering",
+    "models": ["claude-sonnet-4-5", "claude-haiku-4-5"],
+    "max_budget": 200.0,
+    "budget_duration": "30d",
+    "rpm_limit": 200
+  }'
+
+# Bob — contractor, tighter limits
+curl -X POST 'http://0.0.0.0:4000/jwt_client/new' \
+  -H 'Authorization: Bearer <MASTER_KEY>' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "jwt_claim_name": "client_id",
+    "jwt_claim_value": "bob@contractor.com",
+    "team_id": "engineering",
+    "models": ["claude-haiku-4-5"],
+    "max_budget": 20.0,
+    "budget_duration": "30d",
+    "rpm_limit": 30
+  }'
+```
+
+**3. Configure Claude Code to use the proxy**
+
+Set the proxy as the API base in your team's Claude Code config:
+
+```bash
+# Point Claude Code at the LiteLLM proxy instead of Anthropic directly.
+# ANTHROPIC_API_KEY here is the bearer token sent to the proxy — set it to
+# the user's SSO/OIDC JWT token (obtained from your IdP at login).
+export ANTHROPIC_API_KEY="<user-sso-jwt-token>"
+export ANTHROPIC_BASE_URL="http://your-litellm-proxy:4000"
+```
+
+Or in `~/.claude/settings.json`:
+
+```json
+{
+  "env": {
+    "ANTHROPIC_BASE_URL": "http://your-litellm-proxy:4000"
+  }
+}
+```
+
+**4. Developers authenticate with SSO as usual**
+
+When Alice runs Claude Code, her JWT (issued by your IdP with `client_id: alice@corp.com`) goes to the proxy. LiteLLM looks up the mapping, finds her virtual key, and enforces her specific limits — her $200/month budget, 200 RPM cap, and access to Sonnet and Haiku only.
+
+Bob's token maps to his own key — $20/month, Haiku only, 30 RPM.
+
+No API keys distributed. No shared limits. Full per-developer spend visibility in the LiteLLM dashboard.
+
+---
+
+## Managing mappings
+
+**View a mapping + its key settings**
+
+```bash
+curl 'http://0.0.0.0:4000/jwt/key/mapping/info?jwt_claim_name=client_id&jwt_claim_value=alice@corp.com' \
+  -H 'Authorization: Bearer <MASTER_KEY>'
+```
+
+Response includes the linked key's `models`, `max_budget`, `spend`, `rpm_limit`, `expires`, etc.
+
+**Update a mapping**
+
+```bash
+curl -X POST 'http://0.0.0.0:4000/jwt_client/update' \
+  -H 'Authorization: Bearer <MASTER_KEY>' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "jwt_claim_name": "client_id",
+    "jwt_claim_value": "alice@corp.com",
+    "max_budget": 300.0
+  }'
+```
+
+**Delete a mapping**
+
+```bash
+curl -X DELETE 'http://0.0.0.0:4000/jwt/key/mapping/delete' \
+  -H 'Authorization: Bearer <MASTER_KEY>' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "jwt_claim_name": "client_id",
+    "jwt_claim_value": "alice@corp.com"
+  }'
+```
+
+---
+
+## Security
+
+JWT-bound keys are locked down:
+
+- Non-admin users cannot call `/key/update`, `/key/delete`, or `/key/regenerate` on a JWT-bound key. These return 403.
+- JWT-bound keys are automatically restricted to `llm_api_routes` — they can make LLM calls but cannot manage other keys or admin resources.
+- Only proxy admins can create, update, or delete mappings.
+
+---
+
+## Multi-IdP support
+
+If you have users across multiple identity providers that share the same claim values (e.g. two services both have `sub: user-123` from different issuers), set `issuer` when creating the mapping:
+
+```bash
+curl -X POST 'http://0.0.0.0:4000/jwt_client/new' \
+  -H 'Authorization: Bearer <MASTER_KEY>' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "jwt_claim_name": "sub",
+    "jwt_claim_value": "user-123",
+    "issuer": "https://idp-a.corp.com",
+    "models": ["claude-sonnet-4-5"],
+    "max_budget": 50.0
+  }'
+```
+
+Mappings are unique per `(claim_name, claim_value, issuer)` — so `user-123` from IdP A and `user-123` from IdP B resolve to different virtual keys.
+
+---
+
+## What JWT clients can and can't do vs virtual keys
+
+| Capability | Virtual Key | JWT → Key Mapping |
+|---|---|---|
+| Per-client model access | ✅ | ✅ |
+| Per-client spend budget | ✅ | ✅ |
+| Per-client RPM/TPM limits | ✅ | ✅ |
+| Team membership | ✅ | ✅ |
+| Spend tracking in dashboard | ✅ | ✅ |
+| Guardrails | ✅ | ✅ |
+| Key rotation | ✅ | ✅ (admin only) |
+| Key expiry | ✅ | ✅ |
+| No API key distribution needed | ❌ | ✅ |
+| Works with existing SSO/OIDC | ❌ | ✅ |
+
+---
+
+## Related
+
+- [OIDC JWT Auth](./token_auth.md) — base JWT auth setup required before using this feature
+- [Virtual Keys](./virtual_keys.md) — full virtual key documentation
+- [Access Control](./access_control.md) — model and team access control

docs/my-website/docs/proxy/token_auth.md

@@ -16,6 +16,12 @@ Use JWT's to auth admins / users / projects into the proxy.
 :::
 
 
+:::tip JWT → Virtual Key Mapping
+
+Want per-user model restrictions, spend limits, and rate limits without distributing API keys? See **[JWT → Virtual Key Mapping](./jwt_key_mapping.md)** — enterprise-grade granular access control for JWT-authenticated users (e.g. Claude Code + SSO).
+
+:::
+
 ## Usage
 
 ### Step 1. Setup Proxy

docs/my-website/docusaurus.config.js

@@ -285,6 +285,7 @@ const config = {
             to: "docs/enterprise"
           },
           { to: '/blog', label: 'Blog', position: 'left' },
+          { to: '/release_notes', label: 'Release Notes', position: 'left' },
           {
             href: 'https://github.com/BerriAI/litellm',
             position: 'right',

docs/my-website/package.json

@@ -48,27 +48,26 @@
     "node": ">=16.14",
     "npm": ">=8.3.0"
   },
-  "resolutions": {
-    "webpack-dev-server": ">=5.2.1",
-    "form-data": ">=4.0.4",
-    "mermaid": ">=11.10.0",
-    "gray-matter": "4.0.3",
-    "node-forge": ">=1.3.2"
-  },
   "overrides": {
-    "webpack-dev-server": ">=5.2.1",
-    "form-data": ">=4.0.4",
-    "mermaid": ">=11.10.0",
     "gray-matter": "4.0.3",
-    "glob": ">=11.1.0",
-    "tar": ">=7.5.10",
-    "minimatch": ">=10.2.4",
-    "diff": ">=8.0.3",
-    "@isaacs/brace-expansion": ">=5.0.1",
-    "serialize-javascript": ">=7.0.3",
-    "node-forge": ">=1.3.2",
-    "mdast-util-to-hast": ">=13.2.1",
-    "lodash-es": ">=4.17.23",
+    "webpack-dev-server": "5.2.3",
+    "form-data": "4.0.5",
+    "mermaid": "11.12.1",
+    "minimatch": "10.2.4",
+    "serialize-javascript": "7.0.3",
+    "mdast-util-to-hast": "13.2.1",
+    "lodash-es": "4.17.23",
+    "@babel/traverse": "7.28.5",
+    "ws": "8.19.0",
+    "http-proxy-middleware": "3.0.5",
+    "tar-fs": "3.1.1",
+    "webpack-dev-middleware": "5.3.4",
+    "braces": "3.0.3",
+    "webpack": "5.105.3",
+    "serve-static": "2.2.1",
+    "path-to-regexp": "1.9.0",
+    "dompurify": "3.3.2",
+    "svgo": "4.0.1",
     "schema-utils@3": {
       "ajv": "6.14.0"
     },
@@ -83,18 +82,6 @@
     },
     "url-loader": {
       "ajv": "6.14.0"
-    },
-    "@babel/traverse": ">=7.23.2",
-    "ws": ">=7.5.10",
-    "http-proxy-middleware": ">=2.0.9",
-    "tar-fs": ">=2.1.4",
-    "webpack-dev-middleware": ">=5.3.4",
-    "braces": ">=3.0.3",
-    "axios": ">=0.30.2",
-    "webpack": ">=5.94.0",
-    "serve-static": ">=1.16.0",
-    "path-to-regexp": ">=0.1.12",
-    "dompurify": ">=3.3.2",
-    "svgo": ">=3.3.3"
+    }
   }
 }

docs/my-website/release_notes/v1.83.0/index.md

@@ -0,0 +1,62 @@
+---
+title: "v1.83.0 - Official Release (Post Supply Chain Incident)"
+slug: "v1-83-0"
+date: 2026-03-31T00:00:00
+authors:
+  - name: Krrish Dholakia
+    title: CEO, LiteLLM
+    url: https://www.linkedin.com/in/krish-d/
+    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
+  - name: Ishaan Jaff
+    title: CTO, LiteLLM
+    url: https://www.linkedin.com/in/reffajnaahsi/
+    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+hide_table_of_contents: false
+---
+
+## Deploy this version
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+<Tabs>
+<TabItem value="docker" label="Docker">
+
+``` showLineNumbers title="docker run litellm"
+docker run \
+-e STORE_MODEL_IN_DB=True \
+-p 4000:4000 \
+ghcr.io/berriai/litellm:main-1.83.0-nightly
+```
+
+</TabItem>
+<TabItem value="pip" label="Pip">
+
+``` showLineNumbers title="pip install litellm"
+pip install litellm==1.83.0
+```
+
+</TabItem>
+</Tabs>
+
+## Context: First Release After Supply Chain Incident
+
+v1.83.0 is the first LiteLLM release built and published through our new [CI/CD v2 pipeline](https://docs.litellm.ai/blog/ci-cd-v2-improvements), following the [supply chain incident on March 24](https://docs.litellm.ai/blog/security-update-march-2026).
+
+We paused all releases for one week while we:
+1. Completed a forensic review with [Mandiant](https://www.mandiant.com/) and [Veria Labs](https://verialabs.com/)
+2. Rebuilt the release pipeline from scratch with isolated environments and ephemeral credentials
+3. Verified the codebase contains no indicators of compromise
+
+If you have questions about this release or the incident, see our [Security Townhall post](https://docs.litellm.ai/blog/security-townhall-updates) or reach out at `security@berri.ai`.
+
+---
+
+## Links
+
+- **PyPI**: [litellm 1.83.0](https://pypi.org/project/litellm/1.83.0/)
+- **Security update**: [Supply chain incident report](https://docs.litellm.ai/blog/security-update-march-2026)
+- **Security townhall**: [What happened, what we've done, what comes next](https://docs.litellm.ai/blog/security-townhall-updates)
+- **CI/CD v2**: [Announcing CI/CD v2 for LiteLLM](https://docs.litellm.ai/blog/ci-cd-v2-improvements)
+- **April stability sprint**: [Help us plan](https://github.com/BerriAI/litellm/issues/24825)
+

docs/my-website/sidebars.js

@@ -452,6 +452,7 @@ const sidebars = {
           items: [
             "proxy/virtual_keys",
             "proxy/token_auth",
+            "proxy/jwt_key_mapping",
             "proxy/service_accounts",
             "proxy/access_control",
             "proxy/cli_sso",

docs/my-website/src/components/ControlPlaneArchitecture/ControlPlaneArchitecture.tsx

@@ -3,23 +3,56 @@ import styles from './styles.module.css';
 
 /* ────────────────────── Shared small pieces ────────────────────── */
 
-function InfraChip({ color, label }: { color: string; label: string }) {
-  const dotClass =
+function InfraBox({ icon, label, color }: { icon: string; label: string; color: 'green' | 'blue' | 'orange' }) {
+  const colorClass =
     color === 'green'
-      ? styles.infraDotGreen
+      ? styles.infraBoxGreen
       : color === 'blue'
-        ? styles.infraDotBlue
-        : styles.infraDotOrange;
+        ? styles.infraBoxBlue
+        : styles.infraBoxOrange;
 
   return (
-    <span className={styles.infraChip}>
-      <span className={`${styles.infraDot} ${dotClass}`} />
-      {label}
-    </span>
+    <div className={`${styles.infraBox} ${colorClass}`}>
+      <span className={styles.infraBoxIcon}>{icon}</span>
+      <span className={styles.infraBoxLabel}>{label}</span>
+    </div>
   );
 }
 
-/* ────────────────────── Architecture tab ────────────────────── */
+/* ────────────────────── Worker column with infra ────────────────────── */
+
+function WorkerColumn({
+  name,
+  region,
+  subtitle,
+  nodeClass,
+  badgeClass,
+}: {
+  name: string;
+  region: string;
+  subtitle: string;
+  nodeClass: string;
+  badgeClass: string;
+}) {
+  return (
+    <div className={styles.workerColumn}>
+      <div className={`${styles.node} ${styles.nodeWorker} ${nodeClass}`}>
+        <div className={styles.nodeHeader}>
+          <span className={styles.nodeTitle}>{name}</span>
+          <span className={`${styles.badge} ${badgeClass}`}>{region}</span>
+        </div>
+        <div className={styles.nodeSubtitle}>{subtitle}</div>
+        <div className={styles.nodeCaption}>Handles LLM requests</div>
+      </div>
+      <div className={styles.infraStack}>
+        <InfraBox icon="🗄" label="Own Database" color="green" />
+        <InfraBox icon="⚡" label="Own Redis" color="orange" />
+      </div>
+    </div>
+  );
+}
+
+/* ────────────────────── Architecture diagram ────────────────────── */
 
 function ArchitectureView() {
   return (
@@ -36,49 +69,41 @@ function ArchitectureView() {
       <div className={`${styles.node} ${styles.nodeControlPlane}`}>
         <div className={styles.nodeHeader}>
           <span className={styles.nodeTitle}>Control Plane</span>
-          <span className={`${styles.badge} ${styles.badgeBlue}`}>UI</span>
+          <span className={`${styles.badge} ${styles.badgeBlue}`}>ADMIN UI ONLY</span>
         </div>
         <div className={styles.nodeSubtitle}>cp.example.com</div>
-        <div className={styles.infraRow}>
-          <InfraChip color="green" label="Own DB" />
-          <InfraChip color="orange" label="Own Redis" />
-          <InfraChip color="blue" label="Own Key" />
+        <div className={styles.nodeCaption}>
+          Not a router — does not proxy LLM requests.
+          <br />
+          Lets admins switch between workers to manage them.
         </div>
       </div>
 
-      {/* Branch connector */}
-      <div className={styles.connectorBranch}>
-        <div className={`${styles.branchLeg} ${styles.branchLegLeft}`} />
-        <div className={`${styles.branchLeg} ${styles.branchLegRight}`} />
+      {/* Branch connector with label */}
+      <div className={styles.connectorBranchLabeled}>
+        <span className={styles.connectorLabel}>UI management only</span>
+        <div className={styles.connectorBranch}>
+          <div className={`${styles.branchLeg} ${styles.branchLegLeft}`} />
+          <div className={`${styles.branchLeg} ${styles.branchLegRight}`} />
+        </div>
       </div>
 
       {/* Workers */}
       <div className={styles.workersRow}>
-        <div className={`${styles.node} ${styles.nodeWorker} ${styles.nodeWorkerA}`}>
-          <div className={styles.nodeHeader}>
-            <span className={styles.nodeTitle}>Worker A</span>
-            <span className={`${styles.badge} ${styles.badgeGreen}`}>US East</span>
-          </div>
-          <div className={styles.nodeSubtitle}>worker-a.example.com</div>
-          <div className={styles.infraRow}>
-            <InfraChip color="green" label="Own DB" />
-            <InfraChip color="orange" label="Own Redis" />
-            <InfraChip color="blue" label="Own Key" />
-          </div>
-        </div>
-
-        <div className={`${styles.node} ${styles.nodeWorker} ${styles.nodeWorkerB}`}>
-          <div className={styles.nodeHeader}>
-            <span className={styles.nodeTitle}>Worker B</span>
-            <span className={`${styles.badge} ${styles.badgePurple}`}>EU West</span>
-          </div>
-          <div className={styles.nodeSubtitle}>worker-b.example.com</div>
-          <div className={styles.infraRow}>
-            <InfraChip color="green" label="Own DB" />
-            <InfraChip color="orange" label="Own Redis" />
-            <InfraChip color="blue" label="Own Key" />
-          </div>
-        </div>
+        <WorkerColumn
+          name="Worker A"
+          region="US East"
+          subtitle="worker-a.example.com"
+          nodeClass={styles.nodeWorkerA}
+          badgeClass={styles.badgeGreen}
+        />
+        <WorkerColumn
+          name="Worker B"
+          region="EU West"
+          subtitle="worker-b.example.com"
+          nodeClass={styles.nodeWorkerB}
+          badgeClass={styles.badgePurple}
+        />
       </div>
     </div>
   );

docs/my-website/src/components/ControlPlaneArchitecture/styles.module.css

@@ -284,45 +284,67 @@
   color: var(--cp-purple);
 }
 
-/* ── Infrastructure chips ── */
-.infraRow {
-  display: flex;
-  gap: 0.4rem;
-  justify-content: center;
-  flex-wrap: wrap;
-  margin-top: 0.5rem;
+/* ── Node caption ── */
+.nodeCaption {
+  font-size: 0.72rem;
+  color: var(--cp-text-muted);
+  margin-top: 0.4rem;
+  line-height: 1.4;
+  font-style: italic;
 }
 
-.infraChip {
+/* ── Infrastructure boxes (per-worker) ── */
+.infraStack {
+  display: flex;
+  flex-direction: column;
+  gap: 0.35rem;
+  margin-top: 0.5rem;
+  width: 100%;
+}
+
+.infraBox {
   display: flex;
   align-items: center;
-  gap: 0.3rem;
-  font-size: 0.7rem;
-  font-weight: 500;
-  color: var(--cp-text-secondary);
-  background: var(--cp-infra-bg);
-  border: 1px solid var(--cp-infra-border);
-  border-radius: 6px;
-  padding: 0.2rem 0.5rem;
+  gap: 0.5rem;
+  padding: 0.45rem 0.75rem;
+  border-radius: 8px;
+  border: 1.5px solid var(--cp-border);
+  background: var(--cp-card-bg);
 }
 
-.infraDot {
-  width: 6px;
-  height: 6px;
-  border-radius: 50%;
+.infraBoxGreen {
+  border-color: var(--cp-green);
+  background: var(--cp-green-light);
+}
+
+.infraBoxBlue {
+  border-color: var(--cp-accent);
+  background: var(--cp-accent-light);
+}
+
+.infraBoxOrange {
+  border-color: var(--cp-orange);
+  background: var(--cp-orange-light);
+}
+
+.infraBoxIcon {
+  font-size: 0.85rem;
   flex-shrink: 0;
 }
 
-.infraDotGreen {
-  background: var(--cp-green);
+.infraBoxLabel {
+  font-size: 0.75rem;
+  font-weight: 600;
+  color: var(--cp-text);
 }
 
-.infraDotBlue {
-  background: var(--cp-accent);
-}
-
-.infraDotOrange {
-  background: var(--cp-orange);
+/* ── Worker column (card + infra stack) ── */
+.workerColumn {
+  display: flex;
+  flex-direction: column;
+  align-items: stretch;
+  min-width: 220px;
+  max-width: 260px;
 }
 
 /* ── Workers row ── */
@@ -333,6 +355,24 @@
   flex-wrap: wrap;
 }
 
+/* ── Connector with label ── */
+.connectorBranchLabeled {
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+  width: 100%;
+  max-width: 700px;
+}
+
+.connectorLabel {
+  font-size: 0.7rem;
+  color: var(--cp-text-muted);
+  font-weight: 500;
+  text-transform: uppercase;
+  letter-spacing: 0.05em;
+  margin-bottom: 0.25rem;
+}
+
 /* ── Animated flow ── */
 .flowLabel {
   font-size: 0.7rem;
@@ -511,6 +551,16 @@
     max-width: 260px;
   }
 
+  .workerColumn {
+    min-width: auto;
+    width: 100%;
+    max-width: 280px;
+  }
+
+  .connectorBranchLabeled {
+    display: none;
+  }
+
   .connectorBranch {
     display: none;
   }

enterprise/litellm_enterprise/enterprise_callbacks/pagerduty/pagerduty.py

@@ -116,6 +116,7 @@ class PagerDutyAlerting(SlackAlerting):
                 user_api_key_org_id=_meta.get("user_api_key_org_id"),
                 user_api_key_team_id=_meta.get("user_api_key_team_id"),
                 user_api_key_project_id=_meta.get("user_api_key_project_id"),
+                user_api_key_project_alias=_meta.get("user_api_key_project_alias"),
                 user_api_key_user_id=_meta.get("user_api_key_user_id"),
                 user_api_key_team_alias=_meta.get("user_api_key_team_alias"),
                 user_api_key_end_user_id=_meta.get("user_api_key_end_user_id"),
@@ -197,6 +198,7 @@ class PagerDutyAlerting(SlackAlerting):
                 user_api_key_org_id=user_api_key_dict.org_id,
                 user_api_key_team_id=user_api_key_dict.team_id,
                 user_api_key_project_id=user_api_key_dict.project_id,
+                user_api_key_project_alias=user_api_key_dict.project_alias,
                 user_api_key_user_id=user_api_key_dict.user_id,
                 user_api_key_team_alias=user_api_key_dict.team_alias,
                 user_api_key_end_user_id=user_api_key_dict.end_user_id,

litellm-js/proxy/.npmrc

@@ -0,0 +1,5 @@
+# Supply-chain hardening
+# Packages needing lifecycle scripts: npm rebuild <pkg>
+ignore-scripts=true
+# Protects local npm install only — npm ci (used in CI) ignores this
+min-release-age=3d

litellm-js/spend-logs/.npmrc

@@ -0,0 +1,5 @@
+# Supply-chain hardening
+# Packages needing lifecycle scripts: npm rebuild <pkg>
+ignore-scripts=true
+# Protects local npm install only — npm ci (used in CI) ignores this
+min-release-age=3d

litellm-js/spend-logs/Dockerfile

@@ -8,7 +8,7 @@ WORKDIR /app
 COPY ./litellm-js/spend-logs/package*.json ./
 
 # Install dependencies
-RUN npm install
+RUN npm ci
 
 # Install Prisma globally
 RUN npm install -g prisma

litellm-js/spend-logs/package.json

@@ -9,22 +9,5 @@
   "devDependencies": {
     "@types/node": "^20.11.17",
     "tsx": "^4.7.1"
-  },
-  "overrides": {
-    "glob": ">=11.1.0",
-    "tar": ">=7.5.10",
-    "minimatch": ">=10.2.4",
-    "diff": ">=8.0.3",
-    "@isaacs/brace-expansion": ">=5.0.1",
-    "@babel/traverse": ">=7.23.2",
-    "ws": ">=7.5.10",
-    "http-proxy-middleware": ">=2.0.9",
-    "tar-fs": ">=2.1.4",
-    "webpack-dev-middleware": ">=5.3.4",
-    "braces": ">=3.0.3",
-    "axios": ">=0.30.2",
-    "webpack": ">=5.94.0",
-    "serve-static": ">=1.16.0",
-    "path-to-regexp": ">=0.1.12"
   }
-}
+}

litellm-proxy-extras/litellm_proxy_extras/migrations/20260319000000_restore_mcp_approval_fields/migration.sql

@@ -0,0 +1,13 @@
+-- Restore fields dropped by 20260311180521_schema_sync on LiteLLM_MCPServerTable
+-- AlterTable
+ALTER TABLE "LiteLLM_MCPServerTable"
+  ADD COLUMN IF NOT EXISTS "source_url"       TEXT,
+  ADD COLUMN IF NOT EXISTS "approval_status"  TEXT DEFAULT 'active',
+  ADD COLUMN IF NOT EXISTS "submitted_by"     TEXT,
+  ADD COLUMN IF NOT EXISTS "submitted_at"     TIMESTAMP(3),
+  ADD COLUMN IF NOT EXISTS "reviewed_at"      TIMESTAMP(3),
+  ADD COLUMN IF NOT EXISTS "review_notes"     TEXT;
+
+-- CreateIndex
+CREATE INDEX IF NOT EXISTS "LiteLLM_MCPServerTable_approval_status_idx"
+  ON "LiteLLM_MCPServerTable"("approval_status");

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -321,12 +321,11 @@ model LiteLLM_MCPServerTable {
   byok_description      String[] @default([])
   byok_api_key_help_url String?
   source_url            String?
-  // BYOM submission lifecycle
-  approval_status  String?   @default("active")
-  submitted_by     String?
-  submitted_at     DateTime?
-  reviewed_at      DateTime?
-  review_notes     String?
+  approval_status       String?   @default("active")
+  submitted_by          String?
+  submitted_at          DateTime?
+  reviewed_at           DateTime?
+  review_notes          String?
 
   @@index([approval_status])
 }

litellm-proxy-extras/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "litellm-proxy-extras"
-version = "0.4.61"
+version = "0.4.62"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 authors = ["BerriAI"]
 readme = "README.md"
@@ -22,7 +22,7 @@ requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.commitizen]
-version = "0.4.61"
+version = "0.4.62"
 version_files = [
     "pyproject.toml:version",
     "../requirements.txt:litellm-proxy-extras==",

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -56,6 +56,16 @@ if TYPE_CHECKING:
     )
 
 
+def _get_reasoning_items(
+    msg: "AllMessageValues",
+) -> List[ChatCompletionReasoningItem]:
+    """Extract reasoning_items from a message dict with proper typing."""
+    items = msg.get("reasoning_items")  # type: ignore[union-attr]
+    if items:
+        return items  # type: ignore[return-value]
+    return []
+
+
 def _build_reasoning_item(
     item_id: str,
     encrypted_content: Optional[str],
@@ -86,7 +96,9 @@ def _build_reasoning_item(
     }
 
 
-def _reasoning_item_to_response_input(r_item: Dict[str, Any]) -> Dict[str, Any]:
+def _reasoning_item_to_response_input(
+    r_item: Union[ChatCompletionReasoningItem, Dict[str, Any]]
+) -> Dict[str, Any]:
     """Convert a stored ChatCompletionReasoningItem back to a Responses API input item."""
     r_input: Dict[str, Any] = {
         "type": "reasoning",
@@ -246,7 +258,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                     }
                 )
             elif role == "assistant" and tool_calls and isinstance(tool_calls, list):
-                for r_item in msg.get("reasoning_items") or []:
+                for r_item in _get_reasoning_items(msg):
                     input_items.append(_reasoning_item_to_response_input(r_item))
                 for tool_call in tool_calls:
                     function = tool_call.get("function")
@@ -264,7 +276,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                         raise ValueError(f"tool call not supported: {tool_call}")
             elif content is not None:
                 if role == "assistant":
-                    for r_item in msg.get("reasoning_items") or []:
+                    for r_item in _get_reasoning_items(msg):
                         input_items.append(_reasoning_item_to_response_input(r_item))
                 input_items.append(
                     {

litellm/integrations/prometheus.py

@@ -2789,9 +2789,7 @@ class PrometheusLogger(CustomLogger):
             )
             return
 
-        async def fetch_orgs(
-            page_size: int, page: int
-        ) -> Tuple[list, Optional[int]]:
+        async def fetch_orgs(page_size: int, page: int) -> Tuple[list, Optional[int]]:
             skip = (page - 1) * page_size
             orgs = await prisma_client.db.litellm_organizationtable.find_many(
                 skip=skip,

litellm/litellm_core_utils/get_llm_provider_logic.py

@@ -169,9 +169,6 @@ def get_llm_provider(  # noqa: PLR0915
                 return remainder, custom_llm_provider, dynamic_api_key, api_base
             return model, custom_llm_provider, dynamic_api_key, api_base
 
-        if api_key and api_key.startswith("os.environ/"):
-            dynamic_api_key = get_secret_str(api_key)
-
         # Check JSON-configured providers FIRST (before enum-based provider_list)
         provider_prefix = model.split("/", 1)[0]
         if len(model.split("/")) > 1 and JSONProviderRegistry.exists(provider_prefix):

litellm/litellm_core_utils/litellm_logging.py

@@ -3000,9 +3000,10 @@ class Logging(LiteLLMLoggingBaseClass):
                             litellm_call_id=self.model_call_details["litellm_call_id"],
                             print_verbose=print_verbose,
                         )
-                    if (
-                        callable(callback) and customLogger is not None
-                    ):  # custom logger functions
+                    if callable(callback):  # custom logger functions
+                        global customLogger
+                        if customLogger is None:
+                            customLogger = CustomLogger()
                         customLogger.log_event(
                             kwargs=self.model_call_details,
                             response_obj=result,
@@ -3143,9 +3144,10 @@ class Logging(LiteLLMLoggingBaseClass):
                         start_time=start_time,
                         end_time=end_time,
                     )  # type: ignore
-                if (
-                    callable(callback) and customLogger is not None
-                ):  # custom logger functions
+                if callable(callback):  # custom logger functions
+                    global customLogger
+                    if customLogger is None:
+                        customLogger = CustomLogger()
                     await customLogger.async_log_event(
                         kwargs=self.model_call_details,
                         response_obj=result,

litellm/llms/a2a/chat/guardrail_translation/handler.py

@@ -234,37 +234,13 @@ class A2AGuardrailHandler(BaseTranslation):
         then the combined guardrailed text is written into the first chunk that had text
         and all other text parts in other chunks are cleared (in-place).
         """
-        from litellm.llms.a2a.common_utils import extract_text_from_a2a_response
-
-        # Parse each item; keep alignment with responses_so_far (None where unparseable)
-        parsed: List[Optional[Dict[str, Any]]] = [None] * len(responses_so_far)
-        for i, item in enumerate(responses_so_far):
-            if isinstance(item, dict):
-                obj = item
-            elif isinstance(item, str):
-                try:
-                    obj = json.loads(item.strip())
-                except (json.JSONDecodeError, TypeError):
-                    continue
-            else:
-                continue
-            if isinstance(obj.get("result"), dict):
-                parsed[i] = obj
-
-        valid_parsed = [(i, obj) for i, obj in enumerate(parsed) if obj is not None]
+        parsed, valid_parsed = self._parse_streaming_responses(responses_so_far)
         if not valid_parsed:
             return responses_so_far
 
-        # Collect text from each chunk in order (by original index in responses_so_far)
-        text_parts: List[str] = []
-        chunk_indices_with_text: List[int] = []  # indices into valid_parsed
-        for idx, (orig_i, obj) in enumerate(valid_parsed):
-            t = extract_text_from_a2a_response(obj)
-            if t:
-                text_parts.append(t)
-                chunk_indices_with_text.append(orig_i)
-
-        combined_text = "".join(text_parts)
+        combined_text, chunk_indices_with_text = self._collect_text_from_parsed_chunks(
+            valid_parsed
+        )
         if not combined_text:
             return responses_so_far
 
@@ -337,6 +313,43 @@ class A2AGuardrailHandler(BaseTranslation):
 
         return responses_so_far
 
+    def _parse_streaming_responses(
+        self,
+        responses_so_far: List[Any],
+    ) -> Tuple[List[Optional[Dict[str, Any]]], List[Tuple[int, Dict[str, Any]]]]:
+        """Parse JSON-RPC items, returning aligned parsed list and valid entries."""
+        parsed: List[Optional[Dict[str, Any]]] = [None] * len(responses_so_far)
+        for i, item in enumerate(responses_so_far):
+            if isinstance(item, dict):
+                obj = item
+            elif isinstance(item, str):
+                try:
+                    obj = json.loads(item.strip())
+                except (json.JSONDecodeError, TypeError):
+                    continue
+            else:
+                continue
+            if isinstance(obj.get("result"), dict):
+                parsed[i] = obj
+        valid_parsed = [(i, obj) for i, obj in enumerate(parsed) if obj is not None]
+        return parsed, valid_parsed
+
+    def _collect_text_from_parsed_chunks(
+        self,
+        valid_parsed: List[Tuple[int, Dict[str, Any]]],
+    ) -> Tuple[str, List[int]]:
+        """Collect text from parsed chunks, returning combined text and indices."""
+        from litellm.llms.a2a.common_utils import extract_text_from_a2a_response
+
+        text_parts: List[str] = []
+        chunk_indices_with_text: List[int] = []
+        for _idx, (orig_i, obj) in enumerate(valid_parsed):
+            t = extract_text_from_a2a_response(obj)
+            if t:
+                text_parts.append(t)
+                chunk_indices_with_text.append(orig_i)
+        return "".join(text_parts), chunk_indices_with_text
+
     def _extract_texts_from_result(
         self,
         result: Dict[str, Any],

litellm/llms/anthropic/chat/guardrail_translation/handler.py

@@ -277,82 +277,35 @@ class AnthropicMessagesHandler(BaseTranslation):
         images_to_check: List[str] = []
         tool_calls_to_check: List[ChatCompletionToolCallChunk] = []
         task_mappings: List[Tuple[int, Optional[int]]] = []
-        # Track (content_index, None) for each text
-
-        # Handle both dict and object responses
-        response_content: List[Any] = []
-        if isinstance(response, dict):
-            response_content = response.get("content", []) or []
-        elif hasattr(response, "content"):
-            content = getattr(response, "content", None)
-            response_content = content or []
-        else:
-            response_content = []
 
+        response_content = self._get_response_content(response)
         if not response_content:
             return response
 
         # Step 1: Extract all text content and tool calls from response
-        for content_idx, content_block in enumerate(response_content):
-            # Handle both dict and Pydantic object content blocks
-            block_dict: Dict[str, Any] = {}
-            if isinstance(content_block, dict):
-                block_type = content_block.get("type")
-                block_dict = cast(Dict[str, Any], content_block)
-            elif hasattr(content_block, "type"):
-                block_type = getattr(content_block, "type", None)
-                # Convert Pydantic object to dict for processing
-                if hasattr(content_block, "model_dump"):
-                    block_dict = content_block.model_dump()
-                else:
-                    block_dict = {
-                        "type": block_type,
-                        "text": getattr(content_block, "text", None),
-                    }
-            else:
-                continue
-
-            if block_type in ["text", "tool_use"]:
-                self._extract_output_text_and_images(
-                    content_block=block_dict,
-                    content_idx=content_idx,
-                    texts_to_check=texts_to_check,
-                    images_to_check=images_to_check,
-                    task_mappings=task_mappings,
-                    tool_calls_to_check=tool_calls_to_check,
-                )
+        self._extract_from_content_blocks(
+            response_content,
+            texts_to_check,
+            images_to_check,
+            task_mappings,
+            tool_calls_to_check,
+        )
 
         # Step 2: Apply guardrail to all texts in batch
         if texts_to_check or tool_calls_to_check:
-            # Use the real request_data if provided (proxy path), otherwise
-            # create a standalone dict (SDK / direct-call path).
-            if request_data is None:
-                request_data = {"response": response}
-            else:
-                if "response" not in request_data:
-                    request_data["response"] = response
+            request_data = self._prepare_request_data(
+                request_data,
+                response,
+                user_api_key_dict,
+                key="response",
+            )
 
-            # Add user API key metadata with prefixed keys
-            if "litellm_metadata" not in request_data:
-                user_metadata = self.transform_user_api_key_dict_to_metadata(
-                    user_api_key_dict
-                )
-                if user_metadata:
-                    request_data["litellm_metadata"] = user_metadata
-
-            inputs = GenericGuardrailAPIInputs(texts=texts_to_check)
-            if images_to_check:
-                inputs["images"] = images_to_check
-            if tool_calls_to_check:
-                inputs["tool_calls"] = tool_calls_to_check
-            # Include model information from the response if available
-            response_model = None
-            if isinstance(response, dict):
-                response_model = response.get("model")
-            elif hasattr(response, "model"):
-                response_model = getattr(response, "model", None)
-            if response_model:
-                inputs["model"] = response_model
+            inputs = self._build_guardrail_inputs(
+                texts_to_check,
+                images_to_check,
+                tool_calls_to_check,
+                response,
+            )
 
             guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
                 inputs=inputs,
@@ -440,6 +393,95 @@ class AnthropicMessagesHandler(BaseTranslation):
         )
         return responses_so_far
 
+    def _prepare_request_data(
+        self,
+        request_data: Optional[dict],
+        response: Any,
+        user_api_key_dict: Optional[Any],
+        key: str,
+    ) -> dict:
+        """Ensure request_data has the response/responses_so_far key and metadata."""
+        if request_data is None:
+            request_data = {key: response}
+        else:
+            if key not in request_data:
+                request_data[key] = response
+
+        if "litellm_metadata" not in request_data:
+            user_metadata = self.transform_user_api_key_dict_to_metadata(
+                user_api_key_dict
+            )
+            if user_metadata:
+                request_data["litellm_metadata"] = user_metadata
+        return request_data
+
+    @staticmethod
+    def _get_response_content(response: Any) -> List[Any]:
+        """Extract content list from a dict or object response."""
+        if isinstance(response, dict):
+            return response.get("content", []) or []
+        elif hasattr(response, "content"):
+            return getattr(response, "content", None) or []
+        return []
+
+    def _extract_from_content_blocks(
+        self,
+        response_content: List[Any],
+        texts_to_check: List[str],
+        images_to_check: List[str],
+        task_mappings: List[Tuple[int, Optional[int]]],
+        tool_calls_to_check: List["ChatCompletionToolCallChunk"],
+    ) -> None:
+        """Extract text, images, and tool calls from content blocks."""
+        for content_idx, content_block in enumerate(response_content):
+            block_dict: Dict[str, Any] = {}
+            if isinstance(content_block, dict):
+                block_type = content_block.get("type")
+                block_dict = cast(Dict[str, Any], content_block)
+            elif hasattr(content_block, "type"):
+                block_type = getattr(content_block, "type", None)
+                if hasattr(content_block, "model_dump"):
+                    block_dict = content_block.model_dump()
+                else:
+                    block_dict = {
+                        "type": block_type,
+                        "text": getattr(content_block, "text", None),
+                    }
+            else:
+                continue
+
+            if block_type in ["text", "tool_use"]:
+                self._extract_output_text_and_images(
+                    content_block=block_dict,
+                    content_idx=content_idx,
+                    texts_to_check=texts_to_check,
+                    images_to_check=images_to_check,
+                    task_mappings=task_mappings,
+                    tool_calls_to_check=tool_calls_to_check,
+                )
+
+    @staticmethod
+    def _build_guardrail_inputs(
+        texts_to_check: List[str],
+        images_to_check: List[str],
+        tool_calls_to_check: List["ChatCompletionToolCallChunk"],
+        response: Any,
+    ) -> "GenericGuardrailAPIInputs":
+        """Build GenericGuardrailAPIInputs with optional images, tool calls, model."""
+        inputs = GenericGuardrailAPIInputs(texts=texts_to_check)
+        if images_to_check:
+            inputs["images"] = images_to_check
+        if tool_calls_to_check:
+            inputs["tool_calls"] = tool_calls_to_check
+        response_model = None
+        if isinstance(response, dict):
+            response_model = response.get("model")
+        elif hasattr(response, "model"):
+            response_model = getattr(response, "model", None)
+        if response_model:
+            inputs["model"] = response_model
+        return inputs
+
     def get_streaming_string_so_far(self, responses_so_far: List[Any]) -> str:
         """
         Parse streaming responses and extract accumulated text content.

litellm/llms/bedrock/chat/converse_transformation.py

@@ -160,7 +160,8 @@ class AmazonConverseConfig(BaseConfig):
 
             if isinstance(content, list):
                 has_guarded_text = any(
-                    isinstance(item, dict) and item.get("type") == "guarded_text" for item in content
+                    isinstance(item, dict) and item.get("type") == "guarded_text"
+                    for item in content
                 )
                 if has_guarded_text:
                     continue  # Skip this message if it already has guarded_text
@@ -321,9 +322,13 @@ class AmazonConverseConfig(BaseConfig):
 
         # Check if the model is a Nova 2 model (matches nova-2-lite, nova-2-pro, etc.)
         # Also check for nova-2/ spec prefix for imported models
-        return model_without_region.startswith("amazon.nova-2-") or model_without_region.startswith("nova-2/")
+        return model_without_region.startswith(
+            "amazon.nova-2-"
+        ) or model_without_region.startswith("nova-2/")
 
-    def _map_web_search_options(self, web_search_options: dict, model: str) -> Optional[BedrockToolBlock]:
+    def _map_web_search_options(
+        self, web_search_options: dict, model: str
+    ) -> Optional[BedrockToolBlock]:
         """
         Map web_search_options to Nova grounding systemTool.
 
@@ -352,7 +357,9 @@ class AmazonConverseConfig(BaseConfig):
         # (unlike Anthropic), so we just enable grounding with no options
         return BedrockToolBlock(systemTool={"name": "nova_grounding"})
 
-    def _transform_reasoning_effort_to_reasoning_config(self, reasoning_effort: str) -> dict:
+    def _transform_reasoning_effort_to_reasoning_config(
+        self, reasoning_effort: str
+    ) -> dict:
         """
         Transform reasoning_effort parameter to Nova 2 reasoningConfig structure.
 
@@ -397,7 +404,9 @@ class AmazonConverseConfig(BaseConfig):
             }
         }
 
-    def _handle_reasoning_effort_parameter(self, model: str, reasoning_effort: str, optional_params: dict) -> None:
+    def _handle_reasoning_effort_parameter(
+        self, model: str, reasoning_effort: str, optional_params: dict
+    ) -> None:
         """
         Handle the reasoning_effort parameter based on the model type.
 
@@ -434,7 +443,9 @@ class AmazonConverseConfig(BaseConfig):
             optional_params["reasoning_effort"] = reasoning_effort
         elif self._is_nova_2_model(model):
             # Nova 2 models: transform to reasoningConfig
-            reasoning_config = self._transform_reasoning_effort_to_reasoning_config(reasoning_effort)
+            reasoning_config = self._transform_reasoning_effort_to_reasoning_config(
+                reasoning_effort
+            )
             optional_params.update(reasoning_config)
         else:
             # Anthropic and other models: convert to thinking parameter
@@ -478,7 +489,9 @@ class AmazonConverseConfig(BaseConfig):
             "parallel_tool_calls",
         ]
 
-        if "arn" in model:  # we can't infer the model from the arn, so just add all params
+        if (
+            "arn" in model
+        ):  # we can't infer the model from the arn, so just add all params
             supported_params.append("tools")
             supported_params.append("tool_choice")
             supported_params.append("thinking")
@@ -500,7 +513,9 @@ class AmazonConverseConfig(BaseConfig):
             or base_model.startswith("meta.llama3-3")
             or base_model.startswith("meta.llama4")
             or base_model.startswith("amazon.nova")
-            or supports_function_calling(model=model, custom_llm_provider=self.custom_llm_provider)
+            or supports_function_calling(
+                model=model, custom_llm_provider=self.custom_llm_provider
+            )
         ):
             supported_params.append("tools")
 
@@ -510,7 +525,9 @@ class AmazonConverseConfig(BaseConfig):
 
         if litellm.utils.supports_tool_choice(
             model=model, custom_llm_provider=self.custom_llm_provider
-        ) or litellm.utils.supports_tool_choice(model=base_model, custom_llm_provider=self.custom_llm_provider):
+        ) or litellm.utils.supports_tool_choice(
+            model=base_model, custom_llm_provider=self.custom_llm_provider
+        ):
             # only anthropic and mistral support tool choice config. otherwise (E.g. cohere) will fail the call - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ToolChoice.html
             supported_params.append("tool_choice")
 
@@ -529,7 +546,9 @@ class AmazonConverseConfig(BaseConfig):
                 model=model,
                 custom_llm_provider=self.custom_llm_provider,
             )
-            or supports_reasoning(model=base_model, custom_llm_provider=self.custom_llm_provider)
+            or supports_reasoning(
+                model=base_model, custom_llm_provider=self.custom_llm_provider
+            )
         ):
             supported_params.append("thinking")
             supported_params.append("reasoning_effort")
@@ -554,7 +573,9 @@ class AmazonConverseConfig(BaseConfig):
             return ToolChoiceValuesBlock(auto={})
         elif isinstance(tool_choice, dict):
             # only supported for anthropic + mistral models - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ToolChoice.html
-            specific_tool = SpecificToolChoiceBlock(name=tool_choice.get("function", {}).get("name", ""))
+            specific_tool = SpecificToolChoiceBlock(
+                name=tool_choice.get("function", {}).get("name", "")
+            )
             return ToolChoiceValuesBlock(tool=specific_tool)
         else:
             raise litellm.utils.UnsupportedParamsError(
@@ -574,9 +595,15 @@ class AmazonConverseConfig(BaseConfig):
         return ["mp4", "mov", "mkv", "webm", "flv", "mpeg", "mpg", "wmv", "3gp"]
 
     def get_all_supported_content_types(self) -> List[str]:
-        return self.get_supported_image_types() + self.get_supported_document_types() + self.get_supported_video_types()
+        return (
+            self.get_supported_image_types()
+            + self.get_supported_document_types()
+            + self.get_supported_video_types()
+        )
 
-    def is_computer_use_tool_used(self, tools: Optional[List[OpenAIChatCompletionToolParam]], model: str) -> bool:
+    def is_computer_use_tool_used(
+        self, tools: Optional[List[OpenAIChatCompletionToolParam]], model: str
+    ) -> bool:
         """Check if computer use tools are being used in the request."""
         if tools is None:
             return False
@@ -589,7 +616,9 @@ class AmazonConverseConfig(BaseConfig):
                         return True
         return False
 
-    def _transform_computer_use_tools(self, computer_use_tools: List[OpenAIChatCompletionToolParam]) -> List[dict]:
+    def _transform_computer_use_tools(
+        self, computer_use_tools: List[OpenAIChatCompletionToolParam]
+    ) -> List[dict]:
         """Transform computer use tools to Bedrock format."""
         transformed_tools: List[dict] = []
 
@@ -631,7 +660,9 @@ class AmazonConverseConfig(BaseConfig):
 
     def _separate_computer_use_tools(
         self, tools: List[OpenAIChatCompletionToolParam], model: str
-    ) -> Tuple[List[OpenAIChatCompletionToolParam], List[OpenAIChatCompletionToolParam]]:
+    ) -> Tuple[
+        List[OpenAIChatCompletionToolParam], List[OpenAIChatCompletionToolParam]
+    ]:
         """
         Separate computer use tools from regular function tools.
 
@@ -740,18 +771,25 @@ class AmazonConverseConfig(BaseConfig):
         # Recurse into nested schemas
         if "properties" in result and isinstance(result["properties"], dict):
             result["properties"] = {
-                k: AmazonConverseConfig._add_additional_properties_to_schema(v) for k, v in result["properties"].items()
+                k: AmazonConverseConfig._add_additional_properties_to_schema(v)
+                for k, v in result["properties"].items()
             }
         if "items" in result and isinstance(result["items"], dict):
-            result["items"] = AmazonConverseConfig._add_additional_properties_to_schema(result["items"])
+            result["items"] = AmazonConverseConfig._add_additional_properties_to_schema(
+                result["items"]
+            )
         for defs_key in ("$defs", "definitions"):
             if defs_key in result and isinstance(result[defs_key], dict):
                 result[defs_key] = {
-                    k: AmazonConverseConfig._add_additional_properties_to_schema(v) for k, v in result[defs_key].items()
+                    k: AmazonConverseConfig._add_additional_properties_to_schema(v)
+                    for k, v in result[defs_key].items()
                 }
         for key in ("anyOf", "allOf", "oneOf"):
             if key in result and isinstance(result[key], list):
-                result[key] = [AmazonConverseConfig._add_additional_properties_to_schema(item) for item in result[key]]
+                result[key] = [
+                    AmazonConverseConfig._add_additional_properties_to_schema(item)
+                    for item in result[key]
+                ]
 
         return result
 
@@ -781,7 +819,9 @@ class AmazonConverseConfig(BaseConfig):
         }
         """
         if json_schema is not None:
-            json_schema = AmazonConverseConfig._add_additional_properties_to_schema(json_schema)
+            json_schema = AmazonConverseConfig._add_additional_properties_to_schema(
+                json_schema
+            )
         schema_str = json.dumps(json_schema) if json_schema is not None else "{}"
         json_schema_def: JsonSchemaDefinition = {"schema": schema_str}
         if name is not None:
@@ -803,9 +843,14 @@ class AmazonConverseConfig(BaseConfig):
         non_default_params: dict,
         optional_params: dict,
     ):
-        optional_params = self._add_tools_to_optional_params(optional_params=optional_params, tools=tools)
+        optional_params = self._add_tools_to_optional_params(
+            optional_params=optional_params, tools=tools
+        )
 
-        if "meta.llama3-3-70b-instruct-v1:0" in model and non_default_params.get("stream", False) is True:
+        if (
+            "meta.llama3-3-70b-instruct-v1:0" in model
+            and non_default_params.get("stream", False) is True
+        ):
             optional_params["fake_stream"] = True
 
     def map_openai_params(
@@ -944,7 +989,10 @@ class AmazonConverseConfig(BaseConfig):
         if "type" in value and value["type"] == "text":
             return optional_params
 
-        if self._supports_native_structured_outputs(model, self.custom_llm_provider) and json_schema is not None:
+        if (
+            self._supports_native_structured_outputs(model, self.custom_llm_provider)
+            and json_schema is not None
+        ):
             # Use Bedrock's native structured outputs API (outputConfig.textFormat)
             # No synthetic tool injection, no fake_stream needed.
             # Requires an explicit schema — json_object with no schema falls through
@@ -962,10 +1010,14 @@ class AmazonConverseConfig(BaseConfig):
                 json_schema=json_schema,
                 description=description,
             )
-            optional_params = self._add_tools_to_optional_params(optional_params=optional_params, tools=[_tool])
+            optional_params = self._add_tools_to_optional_params(
+                optional_params=optional_params, tools=[_tool]
+            )
 
             if (
-                litellm.utils.supports_tool_choice(model=model, custom_llm_provider=self.custom_llm_provider)
+                litellm.utils.supports_tool_choice(
+                    model=model, custom_llm_provider=self.custom_llm_provider
+                )
                 and not is_thinking_enabled
             ):
                 optional_params["tool_choice"] = ToolChoiceValuesBlock(
@@ -977,7 +1029,9 @@ class AmazonConverseConfig(BaseConfig):
         optional_params["json_mode"] = True
         return optional_params
 
-    def update_optional_params_with_thinking_tokens(self, non_default_params: dict, optional_params: dict):
+    def update_optional_params_with_thinking_tokens(
+        self, non_default_params: dict, optional_params: dict
+    ):
         """
         Handles scenario where max tokens is not specified. For anthropic models (anthropic api/bedrock/vertex ai), this requires having the max tokens being set and being greater than the thinking token budget.
 
@@ -995,9 +1049,13 @@ class AmazonConverseConfig(BaseConfig):
         is_thinking_enabled = self.is_thinking_enabled(optional_params)
         is_max_tokens_in_request = self.is_max_tokens_in_request(non_default_params)
         if is_thinking_enabled and not is_max_tokens_in_request:
-            thinking_token_budget = cast(dict, optional_params["thinking"]).get("budget_tokens", None)
+            thinking_token_budget = cast(dict, optional_params["thinking"]).get(
+                "budget_tokens", None
+            )
             if thinking_token_budget is not None:
-                optional_params["maxTokens"] = thinking_token_budget + DEFAULT_MAX_TOKENS
+                optional_params["maxTokens"] = (
+                    thinking_token_budget + DEFAULT_MAX_TOKENS
+                )
 
     @overload
     def _get_cache_point_block(
@@ -1063,15 +1121,23 @@ class AmazonConverseConfig(BaseConfig):
             if message["role"] == "system":
                 system_prompt_indices.append(idx)
                 if isinstance(message["content"], str) and message["content"]:
-                    system_content_blocks.append(SystemContentBlock(text=message["content"]))
-                    cache_block = self._get_cache_point_block(message, block_type="system", model=model)
+                    system_content_blocks.append(
+                        SystemContentBlock(text=message["content"])
+                    )
+                    cache_block = self._get_cache_point_block(
+                        message, block_type="system", model=model
+                    )
                     if cache_block:
                         system_content_blocks.append(cache_block)
                 elif isinstance(message["content"], list):
                     for m in message["content"]:
                         if m.get("type") == "text" and m.get("text"):
-                            system_content_blocks.append(SystemContentBlock(text=m["text"]))
-                            cache_block = self._get_cache_point_block(m, block_type="system", model=model)
+                            system_content_blocks.append(
+                                SystemContentBlock(text=m["text"])
+                            )
+                            cache_block = self._get_cache_point_block(
+                                m, block_type="system", model=model
+                            )
                             if cache_block:
                                 system_content_blocks.append(cache_block)
         if len(system_prompt_indices) > 0:
@@ -1109,10 +1175,16 @@ class AmazonConverseConfig(BaseConfig):
         # Exceptions should not be stored in optional_params (this is a defensive fix)
         cleaned_params = filter_exceptions_from_params(optional_params)
         inference_params = safe_deep_copy(cleaned_params)
-        supported_converse_params = list(AmazonConverseConfig.__annotations__.keys()) + ["top_k"]
+        supported_converse_params = list(
+            AmazonConverseConfig.__annotations__.keys()
+        ) + ["top_k"]
         supported_tool_call_params = ["tools", "tool_choice"]
         supported_config_params = list(self.get_config_blocks().keys())
-        total_supported_params = supported_converse_params + supported_tool_call_params + supported_config_params
+        total_supported_params = (
+            supported_converse_params
+            + supported_tool_call_params
+            + supported_config_params
+        )
         inference_params.pop("json_mode", None)  # used for handling json_schema
         # Anthropic-only key. Bedrock expects `outputConfig` (camelCase) and
         # will reject `output_config` if it leaks through pass-through routes.
@@ -1123,15 +1195,25 @@ class AmazonConverseConfig(BaseConfig):
         if request_metadata is not None:
             self._validate_request_metadata(request_metadata)
 
-        output_config: Optional[OutputConfigBlock] = inference_params.pop("outputConfig", None)
-        inference_params.pop("output_config", None)  # Bedrock Converse doesn't support it
+        output_config: Optional[OutputConfigBlock] = inference_params.pop(
+            "outputConfig", None
+        )
+        inference_params.pop(
+            "output_config", None
+        )  # Bedrock Converse doesn't support it
 
         # keep supported params in 'inference_params', and set all model-specific params in 'additional_request_params'
-        additional_request_params = {k: v for k, v in inference_params.items() if k not in total_supported_params}
-        inference_params = {k: v for k, v in inference_params.items() if k in total_supported_params}
+        additional_request_params = {
+            k: v for k, v in inference_params.items() if k not in total_supported_params
+        }
+        inference_params = {
+            k: v for k, v in inference_params.items() if k in total_supported_params
+        }
 
         # Handle parallel_tool_calls configuration
-        parallel_tool_use_config = additional_request_params.pop("_parallel_tool_use_config", None)
+        parallel_tool_use_config = additional_request_params.pop(
+            "_parallel_tool_use_config", None
+        )
         if parallel_tool_use_config is not None and is_claude_4_5_on_bedrock(model):
             for key, value in parallel_tool_use_config.items():
                 if (
@@ -1146,7 +1228,9 @@ class AmazonConverseConfig(BaseConfig):
         additional_request_params.pop("parallel_tool_calls", None)
 
         # Only set the topK value in for models that support it
-        additional_request_params.update(self._handle_top_k_value(model, inference_params))
+        additional_request_params.update(
+            self._handle_top_k_value(model, inference_params)
+        )
 
         # Filter out internal/MCP-related parameters that shouldn't be sent to the API
         # These are LiteLLM internal parameters, not API parameters
@@ -1155,7 +1239,9 @@ class AmazonConverseConfig(BaseConfig):
         # Filter out non-serializable objects (exceptions, callables, logging objects, etc.)
         # from additional_request_params to prevent JSON serialization errors
         # This filters: Exception objects, callable objects (functions), Logging objects, etc.
-        additional_request_params = filter_exceptions_from_params(additional_request_params)
+        additional_request_params = filter_exceptions_from_params(
+            additional_request_params
+        )
 
         return (
             inference_params,
@@ -1202,7 +1288,9 @@ class AmazonConverseConfig(BaseConfig):
         # Only separate tools if computer use tools are actually present
         if filtered_tools and self.is_computer_use_tool_used(filtered_tools, model):
             # Separate computer use tools from regular function tools
-            computer_use_tools, regular_tools = self._separate_computer_use_tools(filtered_tools, model)
+            computer_use_tools, regular_tools = self._separate_computer_use_tools(
+                filtered_tools, model
+            )
 
             # Process regular function tools using existing logic
             bedrock_tools = _bedrock_tools_pt(regular_tools)
@@ -1263,7 +1351,9 @@ class AmazonConverseConfig(BaseConfig):
 
                 anthropic_beta_list.append(computer_use_header)
                 # Transform computer use tools to proper Bedrock format
-                transformed_computer_tools = self._transform_computer_use_tools(computer_use_tools)
+                transformed_computer_tools = self._transform_computer_use_tools(
+                    computer_use_tools
+                )
                 additional_request_params["tools"] = transformed_computer_tools
         else:
             # No computer use tools, process all tools as regular tools
@@ -1292,9 +1382,15 @@ class AmazonConverseConfig(BaseConfig):
         """
         Bedrock doesn't support tool calling without `tools=` param specified.
         """
-        if "tools" not in optional_params and messages is not None and has_tool_call_blocks(messages):
+        if (
+            "tools" not in optional_params
+            and messages is not None
+            and has_tool_call_blocks(messages)
+        ):
             if litellm.modify_params:
-                optional_params["tools"] = add_dummy_tool(custom_llm_provider="bedrock_converse")
+                optional_params["tools"] = add_dummy_tool(
+                    custom_llm_provider="bedrock_converse"
+                )
             else:
                 raise litellm.UnsupportedParamsError(
                     message="Bedrock doesn't support tool calling without `tools=` param specified. Pass `tools=` param OR set `litellm.modify_params = True` // `litellm_settings::modify_params: True` to add dummy tool to the request.",
@@ -1348,7 +1444,9 @@ class AmazonConverseConfig(BaseConfig):
 
         bedrock_tool_config: Optional[ToolConfigBlock] = None
         if len(bedrock_tools) > 0:
-            tool_choice_values: ToolChoiceValuesBlock = inference_params.pop("tool_choice", None)
+            tool_choice_values: ToolChoiceValuesBlock = inference_params.pop(
+                "tool_choice", None
+            )
             bedrock_tool_config = ToolConfigBlock(
                 tools=bedrock_tools,
             )
@@ -1358,7 +1456,9 @@ class AmazonConverseConfig(BaseConfig):
         data: CommonRequestObject = {
             "additionalModelRequestFields": additional_request_params,
             "system": system_content_blocks,
-            "inferenceConfig": self._transform_inference_params(inference_params=inference_params),
+            "inferenceConfig": self._transform_inference_params(
+                inference_params=inference_params
+            ),
         }
 
         # Handle all config blocks
@@ -1388,10 +1488,14 @@ class AmazonConverseConfig(BaseConfig):
         litellm_params: dict,
         headers: Optional[dict] = None,
     ) -> RequestObject:
-        messages, system_content_blocks = self._transform_system_message(messages, model=model)
+        messages, system_content_blocks = self._transform_system_message(
+            messages, model=model
+        )
 
         # Convert last user message to guarded_text if guardrailConfig is present
-        messages = self._convert_consecutive_user_messages_to_guarded_text(messages, optional_params)
+        messages = self._convert_consecutive_user_messages_to_guarded_text(
+            messages, optional_params
+        )
         ## TRANSFORMATION ##
 
         _data: CommonRequestObject = self._transform_request_helper(
@@ -1402,11 +1506,13 @@ class AmazonConverseConfig(BaseConfig):
             headers=headers,
         )
 
-        bedrock_messages = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async(
-            messages=messages,
-            model=model,
-            llm_provider="bedrock_converse",
-            user_continue_message=litellm_params.pop("user_continue_message", None),
+        bedrock_messages = (
+            await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async(
+                messages=messages,
+                model=model,
+                llm_provider="bedrock_converse",
+                user_continue_message=litellm_params.pop("user_continue_message", None),
+            )
         )
 
         data: RequestObject = {"messages": bedrock_messages, **_data}
@@ -1440,10 +1546,14 @@ class AmazonConverseConfig(BaseConfig):
         litellm_params: dict,
         headers: Optional[dict] = None,
     ) -> RequestObject:
-        messages, system_content_blocks = self._transform_system_message(messages, model=model)
+        messages, system_content_blocks = self._transform_system_message(
+            messages, model=model
+        )
 
         # Convert last user message to guarded_text if guardrailConfig is present
-        messages = self._convert_consecutive_user_messages_to_guarded_text(messages, optional_params)
+        messages = self._convert_consecutive_user_messages_to_guarded_text(
+            messages, optional_params
+        )
 
         _data: CommonRequestObject = self._transform_request_helper(
             model=model,
@@ -1492,7 +1602,9 @@ class AmazonConverseConfig(BaseConfig):
             encoding=encoding,
         )
 
-    def _transform_reasoning_content(self, reasoning_content_blocks: List[BedrockConverseReasoningContentBlock]) -> str:
+    def _transform_reasoning_content(
+        self, reasoning_content_blocks: List[BedrockConverseReasoningContentBlock]
+    ) -> str:
         """
         Extract the reasoning text from the reasoning content blocks
 
@@ -1508,7 +1620,9 @@ class AmazonConverseConfig(BaseConfig):
         self, thinking_blocks: List[BedrockConverseReasoningContentBlock]
     ) -> List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]]:
         """Return a consistent format for thinking blocks between Anthropic and Bedrock."""
-        thinking_blocks_list: List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]] = []
+        thinking_blocks_list: List[
+            Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]
+        ] = []
         for block in thinking_blocks:
             if "reasoningText" in block:
                 _thinking_block = ChatCompletionThinkingBlock(type="thinking")
@@ -1544,11 +1658,21 @@ class AmazonConverseConfig(BaseConfig):
             cache_creation_input_tokens = usage["cacheWriteInputTokens"]
             input_tokens += cache_creation_input_tokens
 
-        prompt_tokens_details = PromptTokensDetailsWrapper(cached_tokens=cache_read_input_tokens)
-        reasoning_tokens = token_counter(text=reasoning_content, count_response_tokens=True) if reasoning_content else 0
+        prompt_tokens_details = PromptTokensDetailsWrapper(
+            cached_tokens=cache_read_input_tokens
+        )
+        reasoning_tokens = (
+            token_counter(text=reasoning_content, count_response_tokens=True)
+            if reasoning_content
+            else 0
+        )
         completion_tokens_details = CompletionTokensDetailsWrapper(
             reasoning_tokens=reasoning_tokens,
-            text_tokens=(output_tokens - reasoning_tokens if reasoning_tokens > 0 else output_tokens),
+            text_tokens=(
+                output_tokens - reasoning_tokens
+                if reasoning_tokens > 0
+                else output_tokens
+            ),
         )
         openai_usage = Usage(
             prompt_tokens=input_tokens,
@@ -1563,7 +1687,9 @@ class AmazonConverseConfig(BaseConfig):
 
     def get_tool_call_names(
         self,
-        tools: Optional[Union[List[ToolBlock], List[OpenAIChatCompletionToolParam]]] = None,
+        tools: Optional[
+            Union[List[ToolBlock], List[OpenAIChatCompletionToolParam]]
+        ] = None,
     ) -> List[str]:
         if tools is None:
             return []
@@ -1602,8 +1728,13 @@ class AmazonConverseConfig(BaseConfig):
             try:
                 tool_call_names = self.get_tool_call_names(tools)
                 json_content = json.loads(message.content)
-                if json_content.get("type") == "function" and json_content.get("name") in tool_call_names:
-                    tool_calls = [ChatCompletionMessageToolCall(function=Function(**json_content))]
+                if (
+                    json_content.get("type") == "function"
+                    and json_content.get("name") in tool_call_names
+                ):
+                    tool_calls = [
+                        ChatCompletionMessageToolCall(function=Function(**json_content))
+                    ]
 
                     message.tool_calls = tool_calls
                     message.content = None
@@ -1632,7 +1763,9 @@ class AmazonConverseConfig(BaseConfig):
         """
         content_str = ""
         tools: List[ChatCompletionToolCallChunk] = []
-        reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = None
+        reasoningContentBlocks: Optional[
+            List[BedrockConverseReasoningContentBlock]
+        ] = None
         citationsContentBlocks: Optional[List[CitationsContentBlock]] = None
         for idx, content in enumerate(content_blocks):
             """
@@ -1649,7 +1782,9 @@ class AmazonConverseConfig(BaseConfig):
             if "toolUse" in content:
                 ## check tool name was formatted by litellm
                 _response_tool_name = content["toolUse"]["name"]
-                response_tool_name = get_bedrock_tool_name(response_tool_name=_response_tool_name)
+                response_tool_name = get_bedrock_tool_name(
+                    response_tool_name=_response_tool_name
+                )
                 _function_chunk = ChatCompletionToolCallFunctionChunk(
                     name=response_tool_name,
                     arguments=json.dumps(content["toolUse"]["input"]),
@@ -1700,7 +1835,11 @@ class AmazonConverseConfig(BaseConfig):
         """
         try:
             response_data = json.loads(json_str)
-            if isinstance(response_data, dict) and "properties" in response_data and len(response_data) == 1:
+            if (
+                isinstance(response_data, dict)
+                and "properties" in response_data
+                and len(response_data) == 1
+            ):
                 response_data = response_data["properties"]
                 return json.dumps(response_data)
         except json.JSONDecodeError:
@@ -1724,7 +1863,11 @@ class AmazonConverseConfig(BaseConfig):
         if not json_mode or not tools:
             return tools if tools else None
 
-        json_tool_indices = [i for i, t in enumerate(tools) if t["function"].get("name") == RESPONSE_FORMAT_TOOL_NAME]
+        json_tool_indices = [
+            i
+            for i, t in enumerate(tools)
+            if t["function"].get("name") == RESPONSE_FORMAT_TOOL_NAME
+        ]
 
         if not json_tool_indices:
             # No json_tool_call found, return tools unchanged
@@ -1732,10 +1875,14 @@ class AmazonConverseConfig(BaseConfig):
 
         if len(json_tool_indices) == len(tools):
             # All tools are json_tool_call — convert first one to content
-            verbose_logger.debug("Processing JSON tool call response for response_format")
+            verbose_logger.debug(
+                "Processing JSON tool call response for response_format"
+            )
             json_mode_content_str: Optional[str] = tools[0]["function"].get("arguments")
             if json_mode_content_str is not None:
-                json_mode_content_str = AmazonConverseConfig._unwrap_bedrock_properties(json_mode_content_str)
+                json_mode_content_str = AmazonConverseConfig._unwrap_bedrock_properties(
+                    json_mode_content_str
+                )
                 chat_completion_message["content"] = json_mode_content_str
             return None
 
@@ -1745,9 +1892,13 @@ class AmazonConverseConfig(BaseConfig):
         first_idx = json_tool_indices[0]
         json_mode_args = tools[first_idx]["function"].get("arguments")
         if json_mode_args is not None:
-            json_mode_args = AmazonConverseConfig._unwrap_bedrock_properties(json_mode_args)
+            json_mode_args = AmazonConverseConfig._unwrap_bedrock_properties(
+                json_mode_args
+            )
             existing = chat_completion_message.get("content") or ""
-            chat_completion_message["content"] = existing + json_mode_args if existing else json_mode_args
+            chat_completion_message["content"] = (
+                existing + json_mode_args if existing else json_mode_args
+            )
 
         real_tools = [t for i, t in enumerate(tools) if i not in json_tool_indices]
         return real_tools if real_tools else None
@@ -1825,7 +1976,9 @@ class AmazonConverseConfig(BaseConfig):
         chat_completion_message: ChatCompletionResponseMessage = {"role": "assistant"}
         content_str = ""
         tools: List[ChatCompletionToolCallChunk] = []
-        reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = None
+        reasoningContentBlocks: Optional[
+            List[BedrockConverseReasoningContentBlock]
+        ] = None
         citationsContentBlocks: Optional[List[CitationsContentBlock]] = None
 
         if message is not None:
@@ -1844,11 +1997,17 @@ class AmazonConverseConfig(BaseConfig):
             provider_specific_fields["citationsContent"] = citationsContentBlocks
 
         if provider_specific_fields:
-            chat_completion_message["provider_specific_fields"] = provider_specific_fields
+            chat_completion_message[
+                "provider_specific_fields"
+            ] = provider_specific_fields
 
         if reasoningContentBlocks is not None:
-            chat_completion_message["reasoning_content"] = self._transform_reasoning_content(reasoningContentBlocks)
-            chat_completion_message["thinking_blocks"] = self._transform_thinking_blocks(reasoningContentBlocks)
+            chat_completion_message[
+                "reasoning_content"
+            ] = self._transform_reasoning_content(reasoningContentBlocks)
+            chat_completion_message[
+                "thinking_blocks"
+            ] = self._transform_thinking_blocks(reasoningContentBlocks)
         chat_completion_message["content"] = content_str
         filtered_tools = self._filter_json_mode_tools(
             json_mode=json_mode,

litellm/llms/openai/chat/guardrail_translation/handler.py

@@ -86,9 +86,9 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
             if tool_calls_to_check:
                 inputs["tool_calls"] = tool_calls_to_check  # type: ignore
             if messages:
-                inputs["structured_messages"] = (
-                    messages  # pass the openai /chat/completions messages to the guardrail, as-is
-                )
+                inputs[
+                    "structured_messages"
+                ] = messages  # pass the openai /chat/completions messages to the guardrail, as-is
             # Pass tools (function definitions) to the guardrail
             tools = data.get("tools")
             if tools:

litellm/proxy/auth/auth_checks.py

@@ -3078,10 +3078,7 @@ async def _team_max_budget_check(
         BudgetExceededError if the team is over it's max budget.
         Triggers a budget alert if the team is over it's max budget.
     """
-    if (
-        team_object is not None
-        and team_object.max_budget is not None
-    ):
+    if team_object is not None and team_object.max_budget is not None:
         from litellm.proxy.proxy_server import get_current_spend
 
         # Read spend from cross-pod counter (Redis-first) or cached object (fallback)

litellm/proxy/auth/handle_jwt.py

@@ -7,6 +7,7 @@ JWT token must have 'litellm_proxy_admin' in scope.
 """
 
 import fnmatch
+import hashlib
 import os
 import re
 from typing import Any, List, Literal, Optional, Set, Tuple, cast
@@ -620,9 +621,7 @@ class JWTHandler:
             )
 
         # Check cache first
-        cache_key = (
-            f"oidc_userinfo_{token[:20]}"  # Use first 20 chars of token as cache key
-        )
+        cache_key = f"oidc_userinfo_{hashlib.sha256(token.encode()).hexdigest()}"
         cached_userinfo = await self.user_api_key_cache.async_get_cache(cache_key)
 
         if cached_userinfo is not None:
@@ -1397,8 +1396,12 @@ class JWTAuthManager:
         request_headers: Optional[dict] = None,
     ) -> JWTAuthBuilderResult:
         """Main authentication and authorization builder"""
-        # Check if OIDC UserInfo endpoint is enabled
-        if jwt_handler.litellm_jwtauth.oidc_userinfo_enabled:
+        # Check if OIDC UserInfo endpoint is enabled, but fall back to standard
+        # JWT auth if the token itself is a well-formed JWT (3-part structure).
+        if (
+            jwt_handler.litellm_jwtauth.oidc_userinfo_enabled
+            and not jwt_handler.is_jwt(token=api_key)
+        ):
             verbose_proxy_logger.debug(
                 "OIDC UserInfo is enabled. Fetching user info from UserInfo endpoint."
             )

litellm/proxy/auth/login_utils.py

@@ -20,7 +20,6 @@ from litellm.proxy._types import (
     ProxyException,
     UpdateUserRequest,
     UserAPIKeyAuth,
-    hash_token,
 )
 from litellm.proxy.management_endpoints.internal_user_endpoints import user_update
 from litellm.proxy.management_endpoints.key_management_endpoints import (
@@ -29,11 +28,29 @@ from litellm.proxy.management_endpoints.key_management_endpoints import (
 from litellm.proxy.management_endpoints.ui_sso import (
     get_disabled_non_admin_personal_key_creation,
 )
-from litellm.proxy.utils import PrismaClient, get_server_root_path
+from litellm.proxy.utils import (
+    PrismaClient,
+    get_server_root_path,
+    hash_password,
+    verify_password,
+)
 from litellm.secret_managers.main import get_secret_bool
 from litellm.types.proxy.ui_sso import ReturnedUITokenObject
 
 
+async def _rehash_password_if_needed(user_id: str, password: str, stored: str) -> None:
+    """Rehash legacy password (SHA256) to scrypt on successful login."""
+    if stored.startswith("scrypt:"):
+        return
+    from litellm.proxy.proxy_server import prisma_client
+
+    if prisma_client is not None:
+        await prisma_client.db.litellm_usertable.update(
+            where={"user_id": user_id},
+            data={"password": hash_password(password)},
+        )
+
+
 def get_ui_credentials(master_key: Optional[str]) -> tuple[str, str]:
     """
     Get UI username and password from environment variables or master key.
@@ -254,13 +271,8 @@ async def authenticate_user(  # noqa: PLR0915
                 code=401,
             )
 
-        # check if password == _user_row.password
-        hash_password = hash_token(token=password)
-        if secrets.compare_digest(
-            password.encode("utf-8"), _password.encode("utf-8")
-        ) or secrets.compare_digest(
-            hash_password.encode("utf-8"), _password.encode("utf-8")
-        ):
+        if verify_password(password, _password):
+            await _rehash_password_if_needed(_user_row.user_id, password, _password)
             if os.getenv("DATABASE_URL") is not None:
                 response = await generate_key_helper_fn(
                     request_type="key",

litellm/proxy/auth/user_api_key_auth.py

@@ -686,7 +686,10 @@ async def _user_api_key_auth_builder(  # noqa: PLR0915
                 if jwt_handler.litellm_jwtauth.virtual_key_claim_field is not None:
                     # Decode JWT to get claims without running full auth_builder
                     jwt_claims: Optional[dict]
-                    if jwt_handler.litellm_jwtauth.oidc_userinfo_enabled:
+                    if (
+                        jwt_handler.litellm_jwtauth.oidc_userinfo_enabled
+                        and not jwt_handler.is_jwt(token=api_key)
+                    ):
                         jwt_claims = await jwt_handler.get_oidc_userinfo(token=api_key)
                     else:
                         jwt_claims = await jwt_handler.auth_jwt(token=api_key)

litellm/proxy/common_request_processing.py

@@ -293,19 +293,20 @@ def _override_openai_response_model(
        we preserve the actual model that was used (the fallback model).
     2. If the request was to an Azure Model Router, we preserve the actual model
        that was used (e.g., gpt-5-nano-2025-08-07) instead of the router model.
+    3. If this was a fastest_response batch completion, use the winning model's
+       model group name instead of the comma-separated list the client sent.
     """
     if not requested_model:
         return
 
-    # Check if a fallback occurred - if so, preserve the actual model used
     hidden_params = getattr(response_obj, "_hidden_params", {}) or {}
     if isinstance(hidden_params, dict):
+        # Check if a fallback occurred - if so, preserve the actual model used
         fallback_headers = hidden_params.get("additional_headers", {}) or {}
         attempted_fallbacks = fallback_headers.get(
             "x-litellm-attempted-fallbacks", None
         )
         if attempted_fallbacks is not None and attempted_fallbacks > 0:
-            # A fallback occurred - preserve the actual model that was used
             verbose_proxy_logger.debug(
                 "%s: fallback detected (attempted_fallbacks=%d), preserving actual model used instead of overriding to requested model.",
                 log_context,
@@ -313,6 +314,25 @@ def _override_openai_response_model(
             )
             return
 
+        # For fastest_response batch completions, use the winning model's group
+        # name rather than the comma-separated list the client sent.
+        if hidden_params.get("fastest_response_batch_completion"):
+            winning_model = fallback_headers.get("x-litellm-model-group")
+            if winning_model:
+                verbose_proxy_logger.debug(
+                    "%s: fastest_response detected, using winning model group=%r instead of requested=%r.",
+                    log_context,
+                    winning_model,
+                    requested_model,
+                )
+                requested_model = winning_model
+            else:
+                verbose_proxy_logger.debug(
+                    "%s: fastest_response detected but no model group header found, preserving actual model from response.",
+                    log_context,
+                )
+                return
+
     # Check if this is an Azure Model Router request - if so, preserve the actual model used
     if _is_azure_model_router_request(requested_model):
         verbose_proxy_logger.debug(

litellm/proxy/common_utils/reset_budget_job.py

@@ -66,10 +66,8 @@ class ResetBudgetJob:
         try:
             from litellm.proxy.proxy_server import spend_counter_cache
 
-            memberships = (
-                await self.prisma_client.db.litellm_teammembership.find_many(
-                    where={"budget_id": {"in": budget_ids}}
-                )
+            memberships = await self.prisma_client.db.litellm_teammembership.find_many(
+                where={"budget_id": {"in": budget_ids}}
             )
             for m in memberships:
                 counter_key = f"spend:team_member:{m.user_id}:{m.team_id}"
@@ -574,7 +572,11 @@ class ResetBudgetJob:
             counter_key = None
             if item_type == "key" and hasattr(item, "token") and item.token is not None:
                 counter_key = f"spend:key:{item.token}"
-            elif item_type == "team" and hasattr(item, "team_id") and item.team_id is not None:
+            elif (
+                item_type == "team"
+                and hasattr(item, "team_id")
+                and item.team_id is not None
+            ):
                 counter_key = f"spend:team:{item.team_id}"
 
             if counter_key is not None:

litellm/proxy/example_config_yaml/otel_test_config.yaml

@@ -48,21 +48,9 @@ litellm_settings:
   disable_end_user_cost_tracking_prometheus_only: True
 
 guardrails:
-  - guardrail_name: "aporia-pre-guard"
-    litellm_params:
-      guardrail: aporia  # supported values: "aporia", "bedrock", "lakera"
-      mode: "post_call"
-      api_key: os.environ/APORIA_API_KEY_1
-      api_base: os.environ/APORIA_API_BASE_1
-  - guardrail_name: "aporia-post-guard"
-    litellm_params:
-      guardrail: aporia  # supported values: "aporia", "bedrock", "lakera"
-      mode: "post_call"
-      api_key: os.environ/APORIA_API_KEY_2
-      api_base: os.environ/APORIA_API_BASE_2
   - guardrail_name: "bedrock-pre-guard"
     litellm_params:
-      guardrail: bedrock  # supported values: "aporia", "bedrock", "lakera"
+      guardrail: bedrock  # supported values: "bedrock", "lakera"
       mode: "during_call"
       guardrailIdentifier: ff6ujrregl1q
       guardrailVersion: "DRAFT"

litellm/proxy/management_endpoints/budget_management_endpoints.py

@@ -15,7 +15,6 @@ All /budget management endpoints
 from datetime import timedelta
 
 from fastapi import APIRouter, Depends, HTTPException
-from prisma.errors import UniqueViolationError
 
 from litellm.litellm_core_utils.duration_parser import duration_in_seconds
 from litellm.proxy._types import *
@@ -48,6 +47,8 @@ async def new_budget(
     - model_max_budget: Optional[dict] - Specify max budget for a given model. Example: {"openai/gpt-4o-mini": {"max_budget": 100.0, "budget_duration": "1d", "tpm_limit": 100000, "rpm_limit": 100000}}
     - budget_reset_at: Optional[datetime] - Datetime when the initial budget is reset. Default is now.
     """
+    from prisma.errors import UniqueViolationError
+
     from litellm.proxy.proxy_server import litellm_proxy_admin_name, prisma_client
 
     if prisma_client is None:
@@ -99,7 +100,9 @@ async def new_budget(
                 "updated_by": user_api_key_dict.user_id or litellm_proxy_admin_name,
             }  # type: ignore
         )
-    except UniqueViolationError:
+    except Exception as e:
+        if not isinstance(e, UniqueViolationError):
+            raise
         raise HTTPException(
             status_code=400,
             detail={

litellm/proxy/management_endpoints/internal_user_endpoints.py

@@ -41,7 +41,7 @@ from litellm.proxy.management_endpoints.key_management_endpoints import (
     prepare_metadata_fields,
 )
 from litellm.proxy.management_helpers.utils import management_endpoint_wrapper
-from litellm.proxy.utils import handle_exception_on_proxy
+from litellm.proxy.utils import handle_exception_on_proxy, hash_password
 from litellm.types.proxy.management_endpoints.common_daily_activity import (
     SpendAnalyticsPaginatedResponse,
 )
@@ -58,6 +58,22 @@ if TYPE_CHECKING:
 router = APIRouter()
 
 
+def _hash_password_in_dict(data: dict) -> None:
+    """Hash password field in-place if present."""
+    if "password" in data and data["password"] is not None:
+        data["password"] = hash_password(data["password"])
+
+
+def _strip_password_from_response(response) -> None:
+    """Strip password from API response (handles dicts, nested dicts, and Prisma models)."""
+    if isinstance(response, dict):
+        response.pop("password", None)
+        if isinstance(response.get("data"), dict):
+            response["data"].pop("password", None)
+        elif hasattr(response.get("data"), "__dict__"):
+            response["data"].__dict__.pop("password", None)
+
+
 def _update_internal_new_user_params(data_json: dict, data: NewUserRequest) -> dict:
     if "user_id" in data_json and data_json["user_id"] is None:
         data_json["user_id"] = str(uuid.uuid4())
@@ -438,6 +454,7 @@ async def new_user(
 
         data_json = data.json()  # type: ignore
         data_json = _update_internal_new_user_params(data_json, data)
+        _hash_password_in_dict(data_json)
         teams = data.teams
         if teams is None:
             teams = check_if_default_team_set()
@@ -591,6 +608,98 @@ def get_user_id_from_request(request: Request) -> Optional[str]:
     return user_id
 
 
+def _normalize_user_info_user_id(
+    request: Request, user_id: Optional[str]
+) -> Optional[str]:
+    """Normalize URL-decoded user_id while preserving '+' characters."""
+    if user_id is not None and " " in user_id:
+        return get_user_id_from_request(request=request)
+    return user_id
+
+
+async def _get_user_info_teams(
+    prisma_client: Any,
+    user_id: Optional[str],
+    user_info: Optional[Any],
+    user_api_key_dict: UserAPIKeyAuth,
+) -> tuple[list[Any], Optional[list[Any]]]:
+    """Fetch and merge teams from membership + user.teams field."""
+    from litellm.proxy.management_endpoints.team_endpoints import list_team
+
+    team_list: list[Any] = []
+    team_id_list: list[str] = []
+
+    teams_1 = await list_team(
+        http_request=Request(
+            scope={"type": "http", "path": "/user/info"},
+        ),
+        user_id=user_id,
+        user_api_key_dict=user_api_key_dict,
+    )
+
+    if teams_1 is not None and isinstance(teams_1, list):
+        team_list = teams_1
+        team_id_list = [team.team_id for team in teams_1]
+
+    teams_2: Optional[list[Any]] = None
+    target_team_ids = getattr(user_info, "teams", None)
+
+    if target_team_ids and isinstance(target_team_ids, list):
+        teams_2 = await prisma_client.get_data(
+            team_id_list=target_team_ids,
+            table_name="team",
+            query_type="find_all",
+        )
+    elif user_api_key_dict.user_id is not None and user_id is None:
+        caller_user_info = await prisma_client.get_data(
+            user_id=user_api_key_dict.user_id
+        )
+        caller_team_ids = getattr(caller_user_info, "teams", None)
+        if caller_team_ids:
+            teams_2 = await prisma_client.get_data(
+                team_id_list=caller_team_ids,
+                table_name="team",
+                query_type="find_all",
+            )
+
+    if teams_2 is not None and isinstance(teams_2, list):
+        for team in teams_2:
+            if team.team_id not in team_id_list:
+                team_list.append(team)
+                team_id_list.append(team.team_id)
+
+    return team_list, teams_1
+
+
+def _build_user_info_response(
+    user_id: Optional[str],
+    user_info: Optional[Any],
+    keys: Optional[List[LiteLLM_VerificationToken]],
+    team_list: list[Any],
+    teams_1: Optional[list[Any]],
+) -> UserInfoResponse:
+    """Create UserInfoResponse while filtering sensitive fields."""
+    if user_info is None and keys is not None:
+        spend = sum(getattr(k, "spend", 0) for k in keys)
+        user_info = {"spend": spend}
+
+    returned_keys = _process_keys_for_user_info(keys=keys, all_teams=teams_1)
+    team_list.sort(key=lambda x: (getattr(x, "team_alias", "") or ""))
+
+    _user_info = (
+        user_info.model_dump() if isinstance(user_info, BaseModel) else user_info
+    )
+    if isinstance(_user_info, dict):
+        _user_info.pop("password", None)
+
+    return UserInfoResponse(
+        user_id=user_id,
+        user_info=_user_info,
+        keys=returned_keys,
+        teams=team_list,
+    )
+
+
 @router.get(
     "/user/info",
     tags=["Internal User management"],
@@ -598,7 +707,7 @@ def get_user_id_from_request(request: Request) -> Optional[str]:
     response_model=UserInfoResponse,
 )
 @management_endpoint_wrapper
-async def user_info(
+async def user_info(  # noqa: PLR0915
     request: Request,
     user_id: Optional[str] = fastapi.Query(
         default=None, description="User ID in the request parameters"
@@ -621,11 +730,7 @@ async def user_info(
     from litellm.proxy.proxy_server import prisma_client
 
     try:
-        # Handle URL encoding properly by getting user_id from the original request
-        if (
-            user_id is not None and " " in user_id
-        ):  # if user_id is not None and contains a space, get the user_id from the request - this is to handle the case where the user_id is encoded in the url
-            user_id = get_user_id_from_request(request=request)
+        user_id = _normalize_user_info_user_id(request=request, user_id=user_id)
 
         if prisma_client is None:
             raise Exception(
@@ -652,57 +757,13 @@ async def user_info(
                 detail=f"User {user_id} not found",
             )
 
-        ## GET ALL TEAMS ##
-        team_list = []
-        team_id_list = []
-        from litellm.proxy.management_endpoints.team_endpoints import list_team
-
-        teams_1 = await list_team(
-            http_request=Request(
-                scope={"type": "http", "path": "/user/info"},
-            ),
+        team_list, teams_1 = await _get_user_info_teams(
+            prisma_client=prisma_client,
             user_id=user_id,
+            user_info=user_info,
             user_api_key_dict=user_api_key_dict,
         )
 
-        if teams_1 is not None and isinstance(teams_1, list):
-            team_list = teams_1
-            for team in teams_1:
-                team_id_list.append(team.team_id)
-
-        teams_2: Optional[Any] = None
-        if user_info is not None:
-            # *NEW* get all teams in user 'teams' field
-            teams_2 = await prisma_client.get_data(
-                team_id_list=user_info.teams, table_name="team", query_type="find_all"
-            )
-
-            if teams_2 is not None and isinstance(teams_2, list):
-                for team in teams_2:
-                    if team.team_id not in team_id_list:
-                        team_list.append(team)
-                        team_id_list.append(team.team_id)
-
-        elif (
-            user_api_key_dict.user_id is not None and user_id is None
-        ):  # the key querying the endpoint is the one asking for it's teams
-            caller_user_info = await prisma_client.get_data(
-                user_id=user_api_key_dict.user_id
-            )
-            # *NEW* get all teams in user 'teams' field
-            if caller_user_info is not None:
-                teams_2 = await prisma_client.get_data(
-                    team_id_list=caller_user_info.teams,
-                    table_name="team",
-                    query_type="find_all",
-                )
-
-            if teams_2 is not None and isinstance(teams_2, list):
-                for team in teams_2:
-                    if team.team_id not in team_id_list:
-                        team_list.append(team)
-                        team_id_list.append(team.team_id)
-
         ## GET ALL KEYS ##
         keys = await prisma_client.get_data(
             user_id=user_id,
@@ -710,21 +771,12 @@ async def user_info(
             query_type="find_all",
         )
 
-        if user_info is None and keys is not None:
-            ## make sure we still return a total spend ##
-            spend = 0
-            for k in keys:
-                spend += getattr(k, "spend", 0)
-            user_info = {"spend": spend}
-
-        ## REMOVE HASHED TOKEN INFO before returning ##
-        returned_keys = _process_keys_for_user_info(keys=keys, all_teams=teams_1)
-        team_list.sort(key=lambda x: (getattr(x, "team_alias", "") or ""))
-        _user_info = (
-            user_info.model_dump() if isinstance(user_info, BaseModel) else user_info
-        )
-        response_data = UserInfoResponse(
-            user_id=user_id, user_info=_user_info, keys=returned_keys, teams=team_list
+        response_data = _build_user_info_response(
+            user_id=user_id,
+            user_info=user_info,
+            keys=keys,
+            team_list=team_list,
+            teams_1=teams_1,
         )
 
         return response_data
@@ -950,6 +1002,8 @@ async def _get_user_info_for_proxy_admin(user_api_key_dict: UserAPIKeyAuth):
                 if isinstance(admin_user_info, BaseModel)
                 else admin_user_info
             )
+            if isinstance(admin_user_info, dict):
+                admin_user_info.pop("password", None)
 
     return UserInfoResponse(
         user_id=admin_user_id,
@@ -1089,6 +1143,8 @@ async def _update_single_user_helper(
         data_json=data_json, data=user_request
     )
 
+    _hash_password_in_dict(non_default_values)
+
     # Get existing user data for audit logging and metadata preparation
     existing_user_row: Optional[BaseModel] = None
     if user_request.user_id:
@@ -1205,6 +1261,7 @@ async def _update_single_user_helper(
             status_code=400,
             detail={"error": "Failed to update user"},
         )
+    _strip_password_from_response(response)
     return response
 
 
@@ -2189,9 +2246,7 @@ async def _resolve_team_org_filter(
     proxy_logging_obj: Any,
 ) -> List[str]:
     """Look up the team and return its org as a filter list, or raise 403."""
-    from litellm.proxy.management_endpoints.common_utils import (
-        _is_user_team_admin,
-    )
+    from litellm.proxy.management_endpoints.common_utils import _is_user_team_admin
 
     try:
         team_obj = await get_team_object(

litellm/proxy/management_endpoints/key_management_endpoints.py

@@ -4181,13 +4181,13 @@ async def list_keys(
     user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
     page: int = Query(1, description="Page number", ge=1),
     size: int = Query(10, description="Page size", ge=1, le=100),
-    user_id: Optional[str] = Query(None, description="Filter keys by user ID"),
+    user_id: Optional[str] = Query(None, description="Filter keys by user ID. Supports partial matching (substring, case-insensitive)."),
     team_id: Optional[str] = Query(None, description="Filter keys by team ID"),
     organization_id: Optional[str] = Query(
         None, description="Filter keys by organization ID"
     ),
     key_hash: Optional[str] = Query(None, description="Filter keys by key hash"),
-    key_alias: Optional[str] = Query(None, description="Filter keys by key alias"),
+    key_alias: Optional[str] = Query(None, description="Filter keys by key alias. Supports partial matching (substring, case-insensitive)."),
     return_full_object: bool = Query(False, description="Return full key object"),
     include_team_keys: bool = Query(
         False, description="Include all keys for teams that user is an admin of."
@@ -4280,10 +4280,12 @@ async def list_keys(
         else:
             admin_team_ids = None
 
-        if not user_id and user_api_key_dict.user_role not in [
+        use_substring_matching = user_api_key_dict.user_role in [
             LitellmUserRoles.PROXY_ADMIN.value,
             LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY.value,
-        ]:
+        ]
+
+        if not user_id and not use_substring_matching:
             user_id = user_api_key_dict.user_id
 
         response = await _list_key_helper(
@@ -4305,6 +4307,7 @@ async def list_keys(
             status=status,
             project_id=project_id,
             access_group_id=access_group_id,
+            use_substring_matching=use_substring_matching,
         )
 
         verbose_proxy_logger.debug("Successfully prepared response")
@@ -4522,6 +4525,7 @@ def _build_key_filter_conditions(
     include_created_by_keys: bool = False,
     project_id: Optional[str] = None,
     access_group_id: Optional[str] = None,
+    use_substring_matching: bool = False,
 ) -> Dict[str, Union[str, Dict[str, Any], List[Dict[str, Any]]]]:
     """Build filter conditions for key listing.
 
@@ -4543,9 +4547,21 @@ def _build_key_filter_conditions(
     # Base conditions for user's own keys
     user_condition: Dict[str, Any] = {}
     if user_id and isinstance(user_id, str):
-        user_condition["user_id"] = user_id
+        if use_substring_matching:
+            user_condition["user_id"] = {
+                "contains": user_id,
+                "mode": "insensitive",
+            }
+        else:
+            user_condition["user_id"] = user_id
     if key_alias and isinstance(key_alias, str):
-        user_condition["key_alias"] = key_alias
+        if use_substring_matching:
+            user_condition["key_alias"] = {
+                "contains": key_alias,
+                "mode": "insensitive",
+            }
+        else:
+            user_condition["key_alias"] = key_alias
     if exclude_team_id and isinstance(exclude_team_id, str):
         user_condition["team_id"] = {"not": exclude_team_id}
     if organization_id and isinstance(organization_id, str):
@@ -4648,6 +4664,7 @@ async def _list_key_helper(
     status: Optional[str] = None,
     project_id: Optional[str] = None,
     access_group_id: Optional[str] = None,
+    use_substring_matching: bool = False,
 ) -> KeyListResponseObject:
     """
     Helper function to list keys
@@ -4683,6 +4700,7 @@ async def _list_key_helper(
         include_created_by_keys=include_created_by_keys,
         project_id=project_id,
         access_group_id=access_group_id,
+        use_substring_matching=use_substring_matching,
     )
 
     # Calculate skip for pagination

litellm/proxy/management_endpoints/ui_sso.py

@@ -827,7 +827,9 @@ async def get_generic_sso_response(
     ],  # sso specific jwt handler - used for restricted sso group access control
     generic_client_id: str,
     redirect_url: str,
-) -> Tuple[Union[OpenID, dict], Optional[dict], Optional[dict]]:  # (result, received_response, access_token_payload)
+) -> Tuple[
+    Union[OpenID, dict], Optional[dict], Optional[dict]
+]:  # (result, received_response, access_token_payload)
     # make generic sso provider
     from fastapi_sso.sso.base import DiscoveryDocument
     from fastapi_sso.sso.generic import create_provider
@@ -1369,7 +1371,11 @@ async def auth_callback(request: Request, state: Optional[str] = None):  # noqa:
         )
 
     elif generic_client_id is not None:
-        result, received_response, access_token_payload = await get_generic_sso_response(
+        (
+            result,
+            received_response,
+            access_token_payload,
+        ) = await get_generic_sso_response(
             request=request,
             jwt_handler=jwt_handler,
             generic_client_id=generic_client_id,

litellm/proxy/management_helpers/audit_logs.py

@@ -51,7 +51,11 @@ def _build_audit_log_payload(
     if request_data.updated_at is not None:
         updated_at = request_data.updated_at.isoformat()
 
-    table_name_str: str = request_data.table_name.value if isinstance(request_data.table_name, LitellmTableNames) else str(request_data.table_name)
+    table_name_str: str = (
+        request_data.table_name.value
+        if isinstance(request_data.table_name, LitellmTableNames)
+        else str(request_data.table_name)
+    )
 
     return StandardAuditLogPayload(
         id=request_data.id,
@@ -89,7 +93,9 @@ async def _dispatch_audit_log_to_callbacks(
 
     for callback in litellm.audit_log_callbacks:
         try:
-            resolved: Optional[CustomLogger] = callback if isinstance(callback, CustomLogger) else None
+            resolved: Optional[CustomLogger] = (
+                callback if isinstance(callback, CustomLogger) else None
+            )
             if isinstance(callback, str):
                 resolved = _resolve_audit_log_callback(callback)
                 if resolved is None:
@@ -138,9 +144,7 @@ async def create_object_audit_log(
         return
 
     _changed_by = (
-        litellm_changed_by
-        or user_api_key_dict.user_id
-        or litellm_proxy_admin_name
+        litellm_changed_by or user_api_key_dict.user_id or litellm_proxy_admin_name
     )
 
     await create_audit_log_for_update(

litellm/proxy/proxy_server.py

@@ -503,7 +503,9 @@ from litellm.proxy.utils import (
     get_error_message_str,
     get_server_root_path,
     handle_exception_on_proxy,
+    hash_password,
     hash_token,
+    migrate_passwords_to_scrypt_async,
     model_dump_with_preserved_fields,
     update_spend,
 )
@@ -870,6 +872,17 @@ async def proxy_startup_event(app: FastAPI):  # noqa: PLR0915
             user_api_key_cache=user_api_key_cache,
         )
 
+    if prisma_client is not None:
+
+        async def _run_pw_migration():
+            try:
+                result = await migrate_passwords_to_scrypt_async(prisma_client)
+                verbose_proxy_logger.info(f"Password migration: {result}")
+            except Exception as e:
+                verbose_proxy_logger.warning(f"Password migration skipped: {e}")
+
+        asyncio.create_task(_run_pw_migration())
+
     ProxyStartupEvent._initialize_startup_logging(
         llm_router=llm_router,
         proxy_logging_obj=proxy_logging_obj,
@@ -1714,9 +1727,7 @@ async def get_current_spend(counter_key: str, fallback_spend: float) -> float:
     # 1. Try Redis first (cross-pod authoritative)
     if spend_counter_cache.redis_cache is not None:
         try:
-            val = await spend_counter_cache.redis_cache.async_get_cache(
-                key=counter_key
-            )
+            val = await spend_counter_cache.redis_cache.async_get_cache(key=counter_key)
             if val is not None:
                 return float(val)
         except Exception as e:
@@ -1820,9 +1831,7 @@ async def _init_and_increment_spend_counter(
                 key=counter_key, value=base_spend
             )
 
-    await spend_counter_cache.async_increment_cache(
-        key=counter_key, value=increment
-    )
+    await spend_counter_cache.async_increment_cache(key=counter_key, value=increment)
 
 
 async def update_cache(  # noqa: PLR0915
@@ -5721,6 +5730,11 @@ def _restamp_streaming_chunk_model(
     if _is_azure_model_router_request(requested_model_from_client):
         return chunk, model_mismatch_logged
 
+    # For fastest_response batch completions, preserve the winning model's name
+    # instead of stamping the comma-separated list the client sent.
+    if request_data.get("fastest_response", False):
+        return chunk, model_mismatch_logged
+
     downstream_model = (
         chunk.get("model") if isinstance(chunk, dict) else getattr(chunk, "model", None)
     )
@@ -9215,7 +9229,7 @@ def _add_team_models_to_all_models(
 
     for team_object in team_db_objects_typed:
         if (
-            len(team_object.models) == 0  # empty list = all model access
+            not team_object.models  # None or empty list = all model access
             or SpecialModelNames.all_proxy_models.value in team_object.models
         ):
             model_list = llm_router.get_model_list()
@@ -9249,6 +9263,75 @@ def _add_team_models_to_all_models(
     return team_models
 
 
+async def _add_access_group_models_to_team_models(
+    team_db_objects_typed: List[LiteLLM_TeamTable],
+    llm_router: Router,
+    prisma_client: PrismaClient,
+    team_models: Dict[str, Set[str]],
+) -> Dict[str, Set[str]]:
+    """
+    Resolve models reachable via team access groups and merge them into team_models.
+
+    Batch-fetches all distinct access groups in a single DB query, then resolves
+    each eligible team's access group models via the pre-fetched map.
+
+    This ensures models associated with a team only through access groups
+    (not directly in team.models) are included in the UI model listing.
+    """
+    # First pass: identify eligible teams and collect all distinct access group IDs
+    eligible_teams: List[LiteLLM_TeamTable] = []
+    all_access_group_ids: Set[str] = set()
+
+    for team_object in team_db_objects_typed:
+        if not team_object.access_group_ids:
+            continue
+
+        # Skip teams with empty models list — they already have access to everything
+        # (handled by _add_team_models_to_all_models)
+        if (
+            not team_object.models
+            or SpecialModelNames.all_proxy_models.value in team_object.models
+        ):
+            continue
+
+        eligible_teams.append(team_object)
+        all_access_group_ids.update(team_object.access_group_ids)
+
+    if not eligible_teams:
+        return team_models
+
+    # Single batch fetch for all access groups
+    access_group_rows = (
+        await prisma_client.db.litellm_accessgrouptable.find_many(
+            where={"access_group_id": {"in": list(all_access_group_ids)}}
+        )
+    )
+    ag_model_map: Dict[str, List[str]] = {
+        row.access_group_id: row.access_model_names or []
+        for row in access_group_rows
+    }
+
+    # Second pass: resolve deployments for each eligible team
+    for team_object in eligible_teams:
+        model_names: Set[str] = set()
+        for ag_id in team_object.access_group_ids:
+            model_names.update(ag_model_map.get(ag_id, []))
+
+        for model_name in model_names:
+            deployments = llm_router.get_model_list(
+                model_name=model_name, team_id=team_object.team_id
+            )
+            if deployments is not None:
+                for deployment in deployments:
+                    model_id = deployment.get("model_info", {}).get("id", None)
+                    if model_id is not None:
+                        team_models.setdefault(model_id, set()).add(
+                            team_object.team_id
+                        )
+
+    return team_models
+
+
 async def get_all_team_models(
     user_teams: Union[List[str], Literal["*"]],
     prisma_client: PrismaClient,
@@ -9285,6 +9368,14 @@ async def get_all_team_models(
         llm_router=llm_router,
     )
 
+    # Also resolve models reachable via team access groups
+    team_models = await _add_access_group_models_to_team_models(
+        team_db_objects_typed=team_db_objects_typed,
+        llm_router=llm_router,
+        prisma_client=prisma_client,
+        team_models=team_models,
+    )
+
     # convert set to list
     returned_team_models: Dict[str, List[str]] = {}
     for model_id, team_ids in team_models.items():
@@ -9792,7 +9883,7 @@ async def _filter_models_by_team_id(
     team_accessible_model_ids: Set[str] = set()
 
     if (
-        len(team_object.models) == 0  # empty list = all model access
+        not team_object.models  # empty list = all model access
         or SpecialModelNames.all_proxy_models.value in team_object.models
     ):
         # Team has access to all models
@@ -11678,9 +11769,9 @@ async def claim_onboarding_link(data: InvitationClaim):
             },
         )
     ### UPDATE USER OBJECT ###
-    hash_password = hash_token(token=data.password)
+    hashed_pw = hash_password(data.password)
     user_obj = await prisma_client.db.litellm_usertable.update(
-        where={"user_id": invite_obj.user_id}, data={"password": hash_password}
+        where={"user_id": invite_obj.user_id}, data={"password": hashed_pw}
     )
 
     if user_obj is None:
@@ -11700,6 +11791,8 @@ async def claim_onboarding_link(data: InvitationClaim):
         },
     )
 
+    if user_obj and hasattr(user_obj, "__dict__"):
+        user_obj.__dict__.pop("password", None)
     return user_obj
 
 
@@ -12138,7 +12231,10 @@ async def invitation_delete(
     dependencies=[Depends(user_api_key_auth)],
     include_in_schema=False,
 )
-async def update_config(config_info: ConfigYAML):  # noqa: PLR0915
+async def update_config(  # noqa: PLR0915
+    config_info: ConfigYAML,
+    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+):
     """
     For Admin UI - allows admin to update config via UI
 
@@ -12146,6 +12242,10 @@ async def update_config(config_info: ConfigYAML):  # noqa: PLR0915
     """
     global llm_router, llm_model_list, general_settings, proxy_config, proxy_logging_obj, master_key, prisma_client
     try:
+        if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN:
+            raise HTTPException(
+                status_code=403, detail="Only proxy admins can update config"
+            )
         import base64
 
         """

litellm/proxy/spend_tracking/spend_management_endpoints.py

@@ -66,6 +66,15 @@ async def spend_key_fn():
         )
 
 
+def _strip_password_from_users(users) -> None:
+    """Strip password field from a list of user objects."""
+    for user in users if isinstance(users, list) else [users]:
+        if user and hasattr(user, "__dict__"):
+            user.__dict__.pop("password", None)
+        elif isinstance(user, dict):
+            user.pop("password", None)
+
+
 @router.get(
     "/spend/users",
     tags=["Budget & Spend Tracking"],
@@ -105,13 +114,15 @@ async def spend_user_fn(
             user_info = await prisma_client.get_data(
                 table_name="user", query_type="find_unique", user_id=user_id
             )
-            return [user_info]
+            result = [user_info]
         else:
             user_info = await prisma_client.get_data(
                 table_name="user", query_type="find_all"
             )
+            result = user_info
 
-        return user_info
+        _strip_password_from_users(result)
+        return result
 
     except Exception as e:
         raise HTTPException(

litellm/proxy/spend_tracking/spend_tracking_utils.py

@@ -90,6 +90,7 @@ def _get_spend_logs_metadata(
             user_api_key_alias=None,
             user_api_key_team_id=None,
             user_api_key_project_id=None,
+            user_api_key_project_alias=None,
             user_api_key_org_id=None,
             user_api_key_user_id=None,
             user_api_key_team_alias=None,

litellm/proxy/utils.py

@@ -4554,6 +4554,72 @@ def hash_token(token: str):
     return hashed_token
 
 
+def hash_password(password: str) -> str:
+    """Hash a password using scrypt with a random salt."""
+    import base64
+    import hashlib
+    import os
+
+    salt = os.urandom(16)
+    dk = hashlib.scrypt(password.encode(), salt=salt, n=16384, r=8, p=1, dklen=32)
+    return "scrypt:" + base64.b64encode(salt + dk).decode()
+
+
+def verify_password(password: str, stored: str) -> bool:
+    """Verify a password against a stored hash. Supports scrypt and SHA256."""
+    import base64
+    import hashlib
+    import secrets
+
+    if stored.startswith("scrypt:"):
+        try:
+            raw = base64.b64decode(stored[7:])
+            salt, dk = raw[:16], raw[16:]
+            dk2 = hashlib.scrypt(
+                password.encode(), salt=salt, n=16384, r=8, p=1, dklen=32
+            )
+            return secrets.compare_digest(dk, dk2)
+        except Exception:
+            return False
+    # SHA256 fallback (not vulnerable to pass-the-hash: checks sha256(input) == stored)
+    if len(stored) == 64 and all(c in "0123456789abcdef" for c in stored):
+        return secrets.compare_digest(
+            hashlib.sha256(password.encode()).hexdigest().encode(), stored.encode()
+        )
+    return False
+
+
+async def migrate_passwords_to_scrypt_async(prisma_client) -> str:
+    """
+    Migrate plaintext passwords in the DB to scrypt. SHA256 passwords
+    are left alone (they migrate on next login via the SHA256 fallback).
+    Skips quickly if no plaintext passwords exist.
+    """
+    all_with_pw = await prisma_client.db.litellm_usertable.find_many(
+        where={"password": {"not": None}},
+    )
+
+    def _is_sha256_hex(s: str) -> bool:
+        return len(s) == 64 and all(c in "0123456789abcdef" for c in s)
+
+    plaintext_users = [
+        u
+        for u in all_with_pw
+        if u.password
+        and not u.password.startswith("scrypt:")
+        and not _is_sha256_hex(u.password)
+    ]
+    if not plaintext_users:
+        return "No plaintext passwords found"
+
+    for user in plaintext_users:
+        await prisma_client.db.litellm_usertable.update(
+            where={"user_id": user.user_id},
+            data={"password": hash_password(user.password)},
+        )
+    return f"Migrated {len(plaintext_users)} plaintext passwords to scrypt"
+
+
 def _hash_token_if_needed(token: str) -> str:
     """
     Hash the token if it's a string and starts with "sk-"

litellm/responses/litellm_completion_transformation/streaming_iterator.py

@@ -845,6 +845,15 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
         )
         event.__dict__["sequence_number"] = self._sequence_number
         self._pending_response_events.append(event)
+
+        # Emit content_part.added immediately after output_item.added for message
+        # items. The OpenAI Responses spec requires this event before any
+        # output_text.delta events so downstream parsers can initialize the
+        # text part structure.
+        if not self.sent_content_part_added_event:
+            self.sent_content_part_added_event = True
+            content_part_event = self.create_content_part_added_event()
+            self._pending_response_events.append(content_part_event)
         return
 
     async def __anext__(

litellm/responses/litellm_completion_transformation/transformation.py

@@ -1202,22 +1202,32 @@ class LiteLLMCompletionResponsesConfig:
 
         return [chat_completion_response_message]
 
+    @staticmethod
+    def _resolve_file_id(item: Dict[str, Any]) -> Optional[str]:
+        """
+        Return the effective file_id for a Responses API input_file item.
+        Explicit file_id takes precedence; file_url is used as fallback so
+        downstream providers (Anthropic, Gemini) can handle the URL natively.
+        """
+        return item.get("file_id") or item.get("file_url") or None
+
     @staticmethod
     def _transform_input_file_item_to_file_item(item: Dict[str, Any]) -> Dict[str, Any]:
         """
         Transform a Responses API input_file item to a Chat Completion file item
 
         Args:
-            item: Dictionary containing input_file type with file_id and/or file_data
+            item: Dictionary containing input_file type with file_id, file_data, and/or file_url
 
         Returns:
             Dictionary with transformed file structure for Chat Completion
         """
         file_dict: Dict[str, Any] = {}
-        keys = ["file_id", "file_data"]
-        for key in keys:
-            if item.get(key):
-                file_dict[key] = item.get(key)
+        file_id = LiteLLMCompletionResponsesConfig._resolve_file_id(item)
+        if file_id:
+            file_dict["file_id"] = file_id
+        if item.get("file_data"):
+            file_dict["file_data"] = item["file_data"]
 
         new_item: Dict[str, Any] = {"type": "file", "file": file_dict}
         return new_item
@@ -2113,9 +2123,9 @@ class LiteLLMCompletionResponsesConfig:
                 hasattr(completion_details, "reasoning_tokens")
                 and completion_details.reasoning_tokens is not None
             ):
-                output_details_dict[
-                    "reasoning_tokens"
-                ] = completion_details.reasoning_tokens
+                output_details_dict["reasoning_tokens"] = (
+                    completion_details.reasoning_tokens
+                )
             else:
                 output_details_dict["reasoning_tokens"] = 0
 

litellm/setup_wizard.py

@@ -71,7 +71,7 @@ PROVIDERS: List[Dict] = [
         "id": "azure",
         "name": "Azure OpenAI",
         "description": "GPT-4o via Azure",
-        "env_key": "AZURE_API_KEY",
+        "env_key": "AZURE_AI_API_KEY",
         "key_hint": "your-azure-key",
         "test_model": None,  # needs deployment name — skip validation
         "models": [],
@@ -428,7 +428,7 @@ class SetupWizard:
                     f"  {blue('❯')} Azure endpoint URL {grey(p.get('api_base_hint', ''))}: "
                 )
                 if api_base:
-                    env_vars[f"_LITELLM_AZURE_API_BASE_{p['id'].upper()}"] = api_base
+                    env_vars[f"_LITELLM_AZURE_AI_API_BASE_{p['id'].upper()}"] = api_base
                 deployment = _styled_input(
                     f"  {blue('❯')} Azure deployment name {grey('(e.g. my-gpt4o)')}: "
                 )
@@ -557,7 +557,7 @@ class SetupWizard:
                         f'      api_base: "{_yaml_escape(str(p["api_base"]))}"'
                     )
                 elif p.get("needs_api_base"):
-                    azure_base_key = f"_LITELLM_AZURE_API_BASE_{p['id'].upper()}"
+                    azure_base_key = f"_LITELLM_AZURE_AI_API_BASE_{p['id'].upper()}"
                     if azure_base_key in env_copy:
                         lines.append(
                             f'      api_base: "{_yaml_escape(env_copy.pop(azure_base_key))}"'

litellm/types/llms/openai.py

@@ -747,6 +747,7 @@ class ChatCompletionAssistantMessage(OpenAIChatCompletionAssistantMessage, total
     thinking_blocks: Optional[
         List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]]
     ]
+    reasoning_items: Optional[List[ChatCompletionReasoningItem]]
 
 
 class ChatCompletionToolMessage(TypedDict):

litellm/utils.py

@@ -1495,10 +1495,6 @@ def client(original_function):  # noqa: PLR0915
             )
             logging_obj._llm_caching_handler = _llm_caching_handler
 
-            # CHECK FOR 'os.environ/' in kwargs
-            for k, v in kwargs.items():
-                if v is not None and isinstance(v, str) and v.startswith("os.environ/"):
-                    kwargs[k] = litellm.get_secret(v)
             # [OPTIONAL] CHECK BUDGET
             if litellm.max_budget:
                 if litellm._current_cost > litellm.max_budget:

package.json

@@ -11,20 +11,10 @@
     "jest": "^29.7.0"
   },
   "overrides": {
-    "glob": ">=11.1.0",
-    "tar": ">=7.5.11",
-    "minimatch": ">=10.2.4",
-    "diff": ">=8.0.3",
-    "@isaacs/brace-expansion": ">=5.0.1",
-    "@babel/traverse": ">=7.23.2",
-    "ws": ">=7.5.10",
-    "http-proxy-middleware": ">=2.0.9",
-    "tar-fs": ">=2.1.4",
-    "webpack-dev-middleware": ">=5.3.4",
-    "braces": ">=3.0.3",
-    "axios": ">=0.30.2",
-    "webpack": ">=5.94.0",
-    "serve-static": ">=1.16.0",
-    "path-to-regexp": ">=0.1.12"
+    "glob": "13.0.0",
+    "minimatch": "10.1.1",
+    "@isaacs/brace-expansion": "5.0.0",
+    "@babel/traverse": "7.28.5",
+    "braces": "3.0.3"
   }
 }

poetry.lock

@@ -3219,15 +3219,15 @@ files = [
 
 [[package]]
 name = "litellm-proxy-extras"
-version = "0.4.61"
+version = "0.4.62"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 optional = true
 python-versions = "!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*,>=3.8"
 groups = ["main"]
 markers = "extra == \"proxy\""
 files = [
-    {file = "litellm_proxy_extras-0.4.61-py3-none-any.whl", hash = "sha256:9bd1e57ef51972cacff52172ef5d70b0ff689f57f3d240877667301ab8f8590e"},
-    {file = "litellm_proxy_extras-0.4.61.tar.gz", hash = "sha256:dce8e39b1547abf90d912ddd0f2a876beadf789d700ef04c165362d78ad56aee"},
+    {file = "litellm_proxy_extras-0.4.62-py3-none-any.whl", hash = "sha256:cf91c1a83d94000b8997ee29d9e8d505d1ad80c26f111803bef5365174c97de0"},
+    {file = "litellm_proxy_extras-0.4.62.tar.gz", hash = "sha256:0d87db1cda9851717e5294f2fa2c7ae1f34d7476d99e24e6462702e11a2cdd88"},
 ]
 
 [[package]]
@@ -8009,4 +8009,4 @@ utils = ["numpydoc"]
 [metadata]
 lock-version = "2.1"
 python-versions = ">=3.9,<4.0"
-content-hash = "8dad0e86d75e574f12c57c9f32614b7b4ea2181e931874046d43a398aef1e998"
+content-hash = "b238a24abf451e3a7ce954d2f565fb95172cedfb2f1b56d6ed985b67e23fed36"

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "litellm"
-version = "1.82.6"
+version = "1.83.0"
 description = "Library to easily interface with LLM API providers"
 authors = ["BerriAI"]
 license = "MIT"
@@ -61,7 +61,7 @@ boto3 = { version = "^1.40.76", optional = true }
 redisvl = {version = "^0.4.1", optional = true, markers = "python_version >= '3.9' and python_version < '3.14'"}
 mcp = {version = ">=1.25.0,<2.0.0", optional = true, python = ">=3.10"}
 a2a-sdk = {version = "^0.3.22", optional = true, python = ">=3.10"}
-litellm-proxy-extras = {version = "^0.4.61", optional = true}
+litellm-proxy-extras = {version = "^0.4.62", optional = true}
 rich = {version = "^13.7.1", optional = true}
 litellm-enterprise = {version = "0.1.35", optional = true}
 diskcache = {version = "^5.6.1", optional = true}
@@ -184,7 +184,7 @@ requires = ["poetry-core", "wheel"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.commitizen]
-version = "1.82.6"
+version = "1.83.0"
 version_files = [
     "pyproject.toml:^version"
 ]

requirements.txt

@@ -57,7 +57,7 @@ grpcio>=1.75.0; python_version >= "3.14"
 sentry_sdk==2.21.0 # for sentry error handling
 detect-secrets==1.5.0 # Enterprise - secret detection / masking in LLM requests
 tzdata==2025.1 # IANA time zone database
-litellm-proxy-extras==0.4.61 # for proxy extras - e.g. prisma migrations
+litellm-proxy-extras==0.4.62 # for proxy extras - e.g. prisma migrations
 llm-sandbox==0.3.31 # for skill execution in sandbox
 ### LITELLM PACKAGE DEPENDENCIES
 python-dotenv==1.0.1 # for env

security.md

@@ -1,5 +1,49 @@
 # Data Privacy and Security
 
+
+## Security Vulnerability Reporting Guidelines
+
+We value the security community's role in protecting our systems and users. To report a security vulnerability:
+
+- File a private vulnerability report on GitHub: [Report a vulnerability](https://github.com/BerriAI/litellm/security/advisories/new)
+- Include steps to reproduce the issue
+- Provide any relevant additional information
+
+### Vulnerability Categories
+
+We classify vulnerabilities into the following categories:
+
+**P0: Supply Chain Attacks**
+
+Attacks that compromise our CI/CD pipeline, allowing a malicious actor to point our PyPI package or Docker images (GHCR or Docker Hub) to vulnerable or tampered artifacts.
+
+**P1: Unauthenticated Proxy Access**
+
+Application-level attacks where an unauthenticated user is able to gain access to protected data on a LiteLLM proxy instance that should be protected (e.g api keys).
+
+**P2: Authenticated Malicious Actions**
+
+Application-level attacks where an authenticated user is able to perform actions beyond their intended permissions, such as privilege escalation or unauthorized data access.
+
+### Bug Bounty Program
+
+We offer bounties for responsibly disclosed vulnerabilities based on severity:
+
+**Note that currently only P0/P1 reports are eligible for a bounty, though submissions for P2 bugs are still encouraged**
+
+| Severity | Bounty Range | Example |
+|----------|-------------|---------|
+| **Critical** | $1,500 - $3,000 | P0 supply chain compromise |
+| **High** | $500 - $1,500 | P1 unauthenticated proxy access |
+| **Medium** | N/A | P2 authenticated privilege escalation |
+| **Low** | N/A | Minor information disclosure, low-impact misconfigurations |
+
+To qualify for a bounty, reports must include clear reproduction steps and must not involve systems or accounts you do not own. We review all submissions promptly and will follow up within 5 business days.
+
+### Known Non-Issues
+
+- Attacks that require a misconfiguration on setup (e.g not setting a `master_key` on the proxy configuration), are **explicitly not in scope** and are not considered vulnerable.
+
 ## Security Measures
 
 ### LiteLLM Github
@@ -12,11 +56,6 @@
 - For installation and configuration, see: [Self-hosting guided](https://docs.litellm.ai/docs/proxy/deploy)
 - **Telemetry** We run no telemetry when you self host LiteLLM
 
-
-:::info
-✨ SSO is free for up to 5 users. After that, an enterprise license is required. [Get Started with Enterprise here](https://www.litellm.ai/enterprise)
-:::
-
 ### LiteLLM Cloud
 
 - We encrypt all data stored using your `LITELLM_MASTER_KEY` and in transit using TLS.
@@ -37,13 +76,3 @@ LiteLLM supports the following data regions:
 - Europe, Frankfurt, Germany (AWS/GCP `eu-central-1`)
 
 All data, user accounts, and infrastructure are completely separated between these two regions
-
-### Security Vulnerability Reporting Guidelines
-
-We value the security community's role in protecting our systems and users. To report a security vulnerability:
-
-- Email support@berri.ai with details
-- Include steps to reproduce the issue
-- Provide any relevant additional information
-
-We'll review all reports promptly. Note that we don't currently offer a bug bounty program.

tests/audio_tests/test_audio_speech.py

@@ -342,7 +342,7 @@ async def test_azure_ava_tts_async():
 
     try:
         response = await litellm.aspeech(
-            model="azure/speech/azure-tts",
+            model="azure/tts",
             voice="alloy",
             input="Hello, this is a test of Azure text to speech",
             api_base=api_base,
@@ -377,10 +377,6 @@ async def test_azure_ava_tts_async():
 
         print(f"Azure TTS audio saved to: {speech_file_path}")
 
-        # assert response cost is greater than 0
-        print("Response cost: ", response._hidden_params["response_cost"])
-        assert response._hidden_params["response_cost"] > 0
-
     except Exception as e:
         pytest.fail(f"Test failed with exception: {str(e)}")
 

tests/batches_tests/test_fine_tuning_api.py

@@ -125,7 +125,6 @@ async def test_create_fine_tune_jobs_async():
 
 @pytest.mark.asyncio()
 async def test_create_vertex_fine_tune_jobs_mocked():
-    load_vertex_ai_credentials()
     # Define reusable variables for the test
     project_id = "633608382793"
     location = "us-central1"
@@ -164,7 +163,10 @@ async def test_create_vertex_fine_tune_jobs_mocked():
         with patch(
             "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
             return_value=mock_response,
-        ) as mock_post:
+        ) as mock_post, patch(
+            "litellm.llms.vertex_ai.vertex_llm_base.VertexBase._ensure_access_token",
+            return_value=("fake-token", project_id),
+        ):
             create_fine_tuning_response = await litellm.acreate_fine_tuning_job(
                 model=base_model,
                 custom_llm_provider="vertex_ai",
@@ -218,7 +220,6 @@ async def test_create_vertex_fine_tune_jobs_mocked():
 
 @pytest.mark.asyncio()
 async def test_create_vertex_fine_tune_jobs_mocked_with_hyperparameters():
-    load_vertex_ai_credentials()
     # Define reusable variables for the test
     project_id = "633608382793"
     location = "us-central1"
@@ -257,7 +258,10 @@ async def test_create_vertex_fine_tune_jobs_mocked_with_hyperparameters():
         with patch(
             "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
             return_value=mock_response,
-        ) as mock_post:
+        ) as mock_post, patch(
+            "litellm.llms.vertex_ai.vertex_llm_base.VertexBase._ensure_access_token",
+            return_value=("fake-token", project_id),
+        ):
             create_fine_tuning_response = await litellm.acreate_fine_tuning_job(
                 model=base_model,
                 custom_llm_provider="vertex_ai",
@@ -406,29 +410,6 @@ def test_convert_basic_openai_request_to_vertex_request():
     )
 
 
-@pytest.mark.asyncio()
-@pytest.mark.skip(reason="skipping - we run mock tests for vertex ai")
-async def test_create_vertex_fine_tune_jobs():
-    verbose_logger.setLevel(logging.DEBUG)
-    # load_vertex_ai_credentials()
-
-    vertex_credentials = os.getenv("GCS_PATH_SERVICE_ACCOUNT")
-    print("creating fine tuning job")
-    create_fine_tuning_response = await litellm.acreate_fine_tuning_job(
-        model="gemini-1.0-pro-002",
-        custom_llm_provider="vertex_ai",
-        training_file="gs://cloud-samples-data/ai-platform/generative_ai/sft_train_data.jsonl",
-        vertex_project="pathrise-convert-1606954137718",
-        vertex_location="us-central1",
-        vertex_credentials=vertex_credentials,
-    )
-    print("vertex ai create fine tuning response=", create_fine_tuning_response)
-
-    assert create_fine_tuning_response.id is not None
-    assert create_fine_tuning_response.model == "gemini-1.0-pro-002"
-    assert create_fine_tuning_response.object == "fine_tuning.job"
-
-
 @pytest.mark.asyncio
 async def test_mock_openai_create_fine_tune_job():
     """Test that create_fine_tuning_job sends correct parameters to OpenAI"""
@@ -536,7 +517,6 @@ async def test_mock_openai_retrieve_fine_tune_job():
         except Exception as e:
             print("error=", e)
 
-
         # Verify the request
         mock_retrieve.assert_called_once_with(fine_tuning_job_id="ft-123")
 
@@ -544,7 +524,9 @@ async def test_mock_openai_retrieve_fine_tune_job():
 @pytest.mark.asyncio
 async def test_mock_azure_create_fine_tune_job_with_azure_specific_params():
     """Test that Azure-specific parameters are passed through extra_body"""
-    from openai.types.fine_tuning.fine_tuning_job import Hyperparameters as OAIHyperparameters
+    from openai.types.fine_tuning.fine_tuning_job import (
+        Hyperparameters as OAIHyperparameters,
+    )
     from litellm.types.utils import LiteLLMFineTuningJob
 
     mock_response = LiteLLMFineTuningJob(
@@ -564,7 +546,9 @@ async def test_mock_azure_create_fine_tune_job_with_azure_specific_params():
     async def mock_async_create(*args, **kwargs):
         return mock_response
 
-    with patch("litellm.llms.azure.fine_tuning.handler.AzureOpenAIFineTuningAPI.create_fine_tuning_job") as mock_create:
+    with patch(
+        "litellm.llms.azure.fine_tuning.handler.AzureOpenAIFineTuningAPI.create_fine_tuning_job"
+    ) as mock_create:
         mock_create.return_value = mock_async_create()
 
         response = await litellm.acreate_fine_tuning_job(
@@ -575,10 +559,7 @@ async def test_mock_azure_create_fine_tune_job_with_azure_specific_params():
             api_key="test-key",
             api_version="2025-04-01-preview",
             trainingType=1,
-            hyperparameters={
-                "n_epochs": 3,
-                "prompt_loss_weight": 0.1
-            },
+            hyperparameters={"n_epochs": 3, "prompt_loss_weight": 0.1},
         )
 
         # Verify the request
@@ -590,7 +571,7 @@ async def test_mock_azure_create_fine_tune_job_with_azure_specific_params():
         assert create_data["model"] == "gpt-4.1-mini-2025-04-14"
         assert create_data["training_file"] == "file-123"
         assert create_data["hyperparameters"] == {"n_epochs": 3}
-        
+
         # Azure-specific parameters should be in extra_body
         assert "extra_body" in create_data
         assert create_data["extra_body"]["trainingType"] == 1

tests/batches_tests/test_openai_batches_and_files.py

@@ -54,7 +54,7 @@ def load_vertex_ai_credentials():
     print("loading vertex ai credentials")
     os.environ["GCS_FLUSH_INTERVAL"] = "1"
     filepath = os.path.dirname(os.path.abspath(__file__))
-    vertex_key_path = filepath + "/pathrise-convert-1606954137718.json"
+    vertex_key_path = filepath + "/vertex_key.json"
 
     # Read the existing content of the file or create an empty dictionary
     try:

tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py

@@ -1608,15 +1608,24 @@ async def test_initialize_remaining_budget_metrics_exception_handling(
         mock_teamtable = MagicMock()
         mock_teamtable.count = MagicMock(side_effect=Exception("Team count error"))
 
+        # Mock litellm_organizationtable to raise an exception for org budget metrics
+        mock_orgtable = MagicMock()
+        mock_orgtable.find_many = MagicMock(
+            side_effect=Exception("Org database error")
+        )
+        mock_orgtable.count = MagicMock(side_effect=Exception("Org count error"))
+
         mock_db = MagicMock()
         mock_db.litellm_usertable = mock_usertable
         mock_db.litellm_teamtable = mock_teamtable
+        mock_db.litellm_organizationtable = mock_orgtable
         mock_prisma.db = mock_db
 
         # Mock the Prometheus metrics
         prometheus_logger.litellm_remaining_team_budget_metric = MagicMock()
         prometheus_logger.litellm_remaining_api_key_budget_metric = MagicMock()
         prometheus_logger.litellm_remaining_user_budget_metric = MagicMock()
+        prometheus_logger.litellm_remaining_org_budget_metric = MagicMock()
         prometheus_logger.litellm_total_users_metric = MagicMock()
         prometheus_logger.litellm_teams_count_metric = MagicMock()
 
@@ -1625,8 +1634,8 @@ async def test_initialize_remaining_budget_metrics_exception_handling(
             # Call the function
             await prometheus_logger._initialize_remaining_budget_metrics()
 
-            # Verify all four errors were logged (teams, keys, users, and user/team count)
-            assert mock_logger.call_count == 4
+            # Verify all five errors were logged (teams, keys, users, orgs, and user/team count)
+            assert mock_logger.call_count == 5
             assert (
                 "Error initializing teams budget metrics"
                 in mock_logger.call_args_list[0][0][0]
@@ -1640,14 +1649,19 @@ async def test_initialize_remaining_budget_metrics_exception_handling(
                 in mock_logger.call_args_list[2][0][0]
             )
             assert (
-                "Error initializing user/team count metrics"
+                "Error initializing orgs budget metrics"
                 in mock_logger.call_args_list[3][0][0]
             )
+            assert (
+                "Error initializing user/team count metrics"
+                in mock_logger.call_args_list[4][0][0]
+            )
 
         # Verify the metrics were never called
         prometheus_logger.litellm_remaining_team_budget_metric.assert_not_called()
         prometheus_logger.litellm_remaining_api_key_budget_metric.assert_not_called()
         prometheus_logger.litellm_remaining_user_budget_metric.assert_not_called()
+        prometheus_logger.litellm_remaining_org_budget_metric.assert_not_called()
         prometheus_logger.litellm_total_users_metric.assert_not_called()
         prometheus_logger.litellm_teams_count_metric.assert_not_called()
 

tests/enterprise/litellm_enterprise/proxy/auth/test_user_api_key_auth.py

@@ -72,7 +72,7 @@ async def test_enterprise_custom_auth_returns_string():
                 auth_obj = await _user_api_key_auth_builder(
                     request=request,
                     api_key="my-custom-key",
-                    AZURE_AI_API_KEY_header="",
+                    azure_api_key_header="",
                     anthropic_api_key_header=None,
                     google_ai_studio_api_key_header=None,
                     azure_apim_header=None,

tests/image_gen_tests/test_image_edits.py

@@ -158,7 +158,7 @@ class TestAzureAIFlux2ImageEdit(BaseLLMImageEditTest):
         return {
             "model": "azure_ai/flux.2-pro",
             "image": SINGLE_TEST_IMAGE,
-            "api_base": "https://litellm-ci-cd-prod.services.ai.azure.com",
+            "api_base": os.getenv("AZURE_AI_API_BASE"),
             "api_key": os.getenv("AZURE_AI_API_KEY"),
             "api_version": "preview",
         }

tests/image_gen_tests/test_image_generation.py

@@ -113,7 +113,7 @@ class TestVertexImageGeneration(BaseImageGenTest):
         litellm.in_memory_llm_clients_cache = InMemoryCache()
         return {
             "model": "vertex_ai/imagen-3.0-fast-generate-001",
-            "vertex_ai_project": "pathrise-convert-1606954137718",
+            "vertex_ai_project": "litellm-ci-cd",
             "vertex_ai_location": "us-central1",
             "n": 1,
         }
@@ -129,7 +129,7 @@ class TestVertexAIGeminiImageGeneration(BaseImageGenTest):
         litellm.in_memory_llm_clients_cache = InMemoryCache()
         return {
             "model": "vertex_ai/gemini-2.5-flash-image",
-            "vertex_ai_project": "pathrise-convert-1606954137718",
+            "vertex_ai_project": "litellm-ci-cd",
             "vertex_ai_location": "us-central1",
             "n": 1,
             "size": "1024x1024",

tests/litellm_utils_tests/test_azure_ai_anthropic_token_counter.py

@@ -26,22 +26,20 @@ class TestAzureAIAnthropicTokenCounter(BaseTokenCounterTest):
         return AzureAIAnthropicTokenCounter()
 
     def get_test_model(self) -> str:
-        return "claude-3-5-sonnet"
+        return "claude-sonnet-4-6"
 
     def get_test_messages(self) -> List[Dict[str, Any]]:
-        return [
-            {"role": "user", "content": "Hello, how are you today?"}
-        ]
+        return [{"role": "user", "content": "Hello, how are you today?"}]
 
     def get_deployment_config(self) -> Dict[str, Any]:
-        api_key = os.getenv("AZURE_AI_API_KEY")
-        api_base = os.getenv("AZURE_AI_API_BASE")
-        
+        api_key = os.getenv("AZURE_ANTHROPIC_API_KEY")
+        api_base = os.getenv("AZURE_AI_SWEDEN_API_BASE")
+
         if not api_key:
             pytest.skip("AZURE_AI_API_KEY not set")
         if not api_base:
             pytest.skip("AZURE_AI_API_BASE not set")
-            
+
         return {
             "litellm_params": {
                 "api_key": api_key,

tests/litellm_utils_tests/test_hashicorp.py

@@ -27,6 +27,19 @@ from litellm.secret_managers.hashicorp_secret_manager import HashicorpSecretMana
 @pytest.fixture
 def hashicorp_secret_manager():
     """Provide a fresh HashicorpSecretManager per test to avoid shared state."""
+    has_token = bool(os.getenv("HCP_VAULT_TOKEN"))
+    has_approle = bool(
+        os.getenv("HCP_VAULT_APPROLE_ROLE_ID")
+        and os.getenv("HCP_VAULT_APPROLE_SECRET_ID")
+    )
+    has_tls_cert = bool(
+        os.getenv("HCP_VAULT_CLIENT_CERT") and os.getenv("HCP_VAULT_CLIENT_KEY")
+    )
+    if not (has_token or has_approle or has_tls_cert):
+        pytest.skip(
+            "Skipping Hashicorp tests: set HCP_VAULT_TOKEN, AppRole vars, or TLS cert vars."
+        )
+
     manager = HashicorpSecretManager()
     manager.vault_addr = "https://test-cluster-public-vault-0f98180c.e98296b2.z1.hashicorp.cloud:8200"
     manager.vault_namespace = "admin"
@@ -253,7 +266,7 @@ async def test_hashicorp_secret_manager_delete_secret_with_team_overrides(
         assert called_url == expected_url
 
 
-def test_hashicorp_secret_manager_tls_cert_auth(monkeypatch, hashicorp_secret_manager):
+def test_hashicorp_secret_manager_tls_cert_auth(monkeypatch):
     monkeypatch.setenv("HCP_VAULT_TOKEN", "test-client-token-12345")
     print("HCP_VAULT_TOKEN=", os.getenv("HCP_VAULT_TOKEN"))
     # Mock both httpx.post and httpx.Client
@@ -301,7 +314,7 @@ def test_hashicorp_secret_manager_tls_cert_auth(monkeypatch, hashicorp_secret_ma
         assert test_manager.cache.get_cache("hcp_vault_token") == "test-client-token-12345"
 
 
-def test_hashicorp_secret_manager_approle_auth(monkeypatch, hashicorp_secret_manager):
+def test_hashicorp_secret_manager_approle_auth(monkeypatch):
     """
     Test AppRole authentication makes the expected POST request to the correct URL.
     """

tests/litellm_utils_tests/test_litellm_overhead.py

@@ -101,7 +101,7 @@ async def test_litellm_overhead_non_streaming(model):
         kwargs["vertex_project"] = "fake-project"
         kwargs["vertex_location"] = "us-central1"
     if model == "openai/self_hosted":
-        kwargs["api_base"] = "https://exampleopenaiendpoint-production.up.railway.app/"
+        kwargs["api_base"] = os.environ.get("FAKE_OPENAI_API_BASE")
 
     async def _run():
         return await litellm.acompletion(**kwargs)

tests/litellm_utils_tests/test_secret_manager.py

@@ -1,3 +1,4 @@
+import base64
 import os
 import sys
 import time
@@ -24,7 +25,7 @@ from litellm.secret_managers.main import (
     get_secret,
     _should_read_secret_from_secret_manager,
 )
-from unittest.mock import AsyncMock
+from unittest.mock import AsyncMock, patch, MagicMock
 
 
 def load_vertex_ai_credentials():
@@ -221,53 +222,79 @@ def test_oidc_env_path():
         del os.environ[env_var_name]
 
 
-@pytest.mark.flaky(retries=6, delay=1)
 def test_google_secret_manager():
     """
     Test that we can get a secret from Google Secret Manager
     """
-    os.environ["GOOGLE_SECRET_MANAGER_PROJECT_ID"] = "pathrise-convert-1606954137718"
+    os.environ["GOOGLE_SECRET_MANAGER_PROJECT_ID"] = "litellm-ci-cd"
 
     from litellm.secret_managers.google_secret_manager import GoogleSecretManager
 
-    load_vertex_ai_credentials()
-    secret_manager = GoogleSecretManager()
+    mock_response = MagicMock()
+    mock_response.status_code = 200
+    mock_response.json.return_value = {
+        "payload": {
+            "data": base64.b64encode(b"anything").decode("utf-8"),
+        }
+    }
 
-    secret_val = secret_manager.get_secret_from_google_secret_manager(
-        secret_name="OPENAI_API_KEY"
-    )
-    print("secret_val: {}".format(secret_val))
+    with patch(
+        "litellm.proxy.proxy_server.premium_user", True
+    ), patch.object(
+        GoogleSecretManager,
+        "sync_construct_request_headers",
+        return_value={"Authorization": "Bearer mock_token"},
+    ):
+        secret_manager = GoogleSecretManager()
+        secret_manager.sync_httpx_client = MagicMock()
+        secret_manager.sync_httpx_client.get.return_value = mock_response
 
-    assert (
-        secret_val == "anything"
-    ), "did not get expected secret value. expect 'anything', got '{}'".format(
-        secret_val
-    )
+        secret_val = secret_manager.get_secret_from_google_secret_manager(
+            secret_name="OPENAI_API_KEY"
+        )
+        print("secret_val: {}".format(secret_val))
+
+        assert (
+            secret_val == "anything"
+        ), "did not get expected secret value. expect 'anything', got '{}'".format(
+            secret_val
+        )
+
+        secret_manager.sync_httpx_client.get.assert_called_once()
+        call_url = secret_manager.sync_httpx_client.get.call_args[1]["url"]
+        assert "projects/litellm-ci-cd/secrets/OPENAI_API_KEY" in call_url
 
 
 def test_google_secret_manager_read_in_memory():
     """
-    Test that Google Secret manager returs in memory value when it exists
+    Test that Google Secret manager returns in memory value when it exists
     """
     from litellm.secret_managers.google_secret_manager import GoogleSecretManager
 
-    load_vertex_ai_credentials()
-    os.environ["GOOGLE_SECRET_MANAGER_PROJECT_ID"] = "pathrise-convert-1606954137718"
-    secret_manager = GoogleSecretManager()
-    secret_manager.cache.cache_dict["UNIQUE_KEY"] = None
-    secret_manager.cache.cache_dict["UNIQUE_KEY_2"] = "lite-llm"
+    os.environ["GOOGLE_SECRET_MANAGER_PROJECT_ID"] = "litellm-ci-cd"
 
-    secret_val = secret_manager.get_secret_from_google_secret_manager(
-        secret_name="UNIQUE_KEY"
-    )
-    print("secret_val: {}".format(secret_val))
-    assert secret_val == None
+    with patch(
+        "litellm.proxy.proxy_server.premium_user", True
+    ), patch.object(
+        GoogleSecretManager,
+        "sync_construct_request_headers",
+        return_value={"Authorization": "Bearer mock_token"},
+    ):
+        secret_manager = GoogleSecretManager()
+        secret_manager.cache.cache_dict["UNIQUE_KEY"] = None
+        secret_manager.cache.cache_dict["UNIQUE_KEY_2"] = "lite-llm"
 
-    secret_val = secret_manager.get_secret_from_google_secret_manager(
-        secret_name="UNIQUE_KEY_2"
-    )
-    print("secret_val: {}".format(secret_val))
-    assert secret_val == "lite-llm"
+        secret_val = secret_manager.get_secret_from_google_secret_manager(
+            secret_name="UNIQUE_KEY"
+        )
+        print("secret_val: {}".format(secret_val))
+        assert secret_val is None
+
+        secret_val = secret_manager.get_secret_from_google_secret_manager(
+            secret_name="UNIQUE_KEY_2"
+        )
+        print("secret_val: {}".format(secret_val))
+        assert secret_val == "lite-llm"
 
 
 def test_should_read_secret_from_secret_manager():
@@ -337,6 +364,7 @@ def test_get_secret_with_access_mode():
     litellm._key_management_settings = KeyManagementSettings()
     del os.environ[test_secret_name]
 
+
 def test_key_management_settings_defaults():
     """
     Test that KeyManagementSettings initializes with correct default values.

tests/llm_responses_api_testing/test_manus_files_all_methods.py

@@ -1,71 +0,0 @@
-"""
-E2E test for all Manus Files API methods.
-"""
-
-import os
-import pytest
-import litellm
-
-
-@pytest.mark.asyncio
-async def test_manus_files_api_e2e_all_methods():
-    """
-    E2E test for Manus Files API: create, retrieve, list, delete.
-    """
-    litellm._turn_on_debug()
-
-    api_key = os.getenv("MANUS_API_KEY")
-    if api_key is None:
-        pytest.skip("MANUS_API_KEY not set")
-
-    # Create a simple test file content
-    test_content = b"This is a test file for Manus Files API - all methods test."
-    test_filename = "test_file_all_methods.txt"
-
-    # Step 1: Create file
-    print("Step 1: Creating file...")
-    created_file = await litellm.acreate_file(
-        file=(test_filename, test_content),
-        purpose="assistants",
-        custom_llm_provider="manus",
-        api_key=api_key,
-    )
-    print(f"Created file: {created_file}")
-    assert created_file.filename == test_filename
-    assert created_file.status == "uploaded"
-    # Note: Manus doesn't return bytes in initial response
-    file_id = created_file.id
-
-    # Step 2: Retrieve file
-    print(f"\nStep 2: Retrieving file {file_id}...")
-    retrieved_file = await litellm.afile_retrieve(
-        file_id=file_id,
-        custom_llm_provider="manus",
-        api_key=api_key,
-    )
-    print(f"Retrieved file: {retrieved_file}")
-    assert retrieved_file.id == file_id
-    assert retrieved_file.filename == test_filename
-
-    # Step 3: List files
-    print("\nStep 3: Listing files...")
-    files_list = await litellm.afile_list(
-        custom_llm_provider="manus",
-        api_key=api_key,
-    )
-    print(f"Files list: {files_list}")
-    assert isinstance(files_list, list)
-    assert any(f.id == file_id for f in files_list)
-
-    # Step 4: Delete file
-    print(f"\nStep 4: Deleting file {file_id}...")
-    deleted_file = await litellm.afile_delete(
-        file_id=file_id,
-        custom_llm_provider="manus",
-        api_key=api_key,
-    )
-    print(f"Deleted file: {deleted_file}")
-    assert deleted_file.id == file_id
-    assert deleted_file.deleted is True
-
-    print("\n✅ All Manus Files API methods working!")

tests/llm_responses_api_testing/test_manus_responses_api.py

@@ -1,127 +0,0 @@
-import os
-import sys
-import pytest
-import asyncio
-from typing import Optional
-from unittest.mock import patch, AsyncMock
-
-sys.path.insert(0, os.path.abspath("../.."))
-import litellm
-from litellm.integrations.custom_logger import CustomLogger
-import json
-from litellm.types.utils import StandardLoggingPayload
-from litellm.types.llms.openai import (
-    ResponseCompletedEvent,
-    ResponsesAPIResponse,
-    ResponseAPIUsage,
-    IncompleteDetails,
-)
-from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
-from base_responses_api import BaseResponsesAPITest
-
-
-@pytest.mark.asyncio
-async def test_manus_responses_api_with_agent_profile():
-    """
-    Test that Manus API correctly extracts agent profile from model name
-    and includes task_mode and agent_profile in the request.
-    """
-    litellm._turn_on_debug()
-    
-    response = await litellm.aresponses(
-        model="manus/manus-1.6-lite",
-        input="What's the color of the sky?",
-        api_key=os.getenv("MANUS_API_KEY"),
-        max_output_tokens=50,
-    )
-    
-    print("Manus response=", json.dumps(response, indent=4, default=str))
-
-    ## Get the status of the response
-    got_response = await litellm.aget_responses(
-        response_id=response.id,
-        custom_llm_provider="manus",
-        api_key=os.getenv("MANUS_API_KEY"),
-    )
-    print("GET API MANUS RESPONSE=", json.dumps(got_response, indent=4, default=str))
-    if got_response.status == "completed":
-        assert got_response.output is not None
-        assert len(got_response.output) > 0
-
-
-
-@pytest.mark.asyncio
-async def test_manus_responses_api_with_file_upload():
-    """
-    Test that uploads a file via Files API and then passes it to Responses API.
-    """
-    litellm._turn_on_debug()
-    
-    api_key = os.getenv("MANUS_API_KEY")
-    if api_key is None:
-        pytest.skip("MANUS_API_KEY not set")
-    
-    # Step 1: Upload a file
-    test_content = b"Warren Buffett's 2023 Letter to Shareholders\n\nKey Points:\n1. Long-term value creation\n2. Capital allocation strategy\n3. Market volatility perspective"
-    test_filename = "buffett_letter_summary.txt"
-    
-    print("Step 1: Uploading file...")
-    uploaded_file = await litellm.acreate_file(
-        file=(test_filename, test_content),
-        purpose="assistants",
-        custom_llm_provider="manus",
-        api_key=api_key,
-    )
-    print(f"Uploaded file: {uploaded_file}")
-    assert uploaded_file.id is not None
-    file_id = uploaded_file.id
-    
-    # Step 2: Create a response with the uploaded file
-    print(f"\nStep 2: Creating response with file {file_id}...")
-    response = await litellm.aresponses(
-        model="manus/manus-1.6-lite",
-        input=[
-            {
-                "role": "user",
-                "content": [
-                    {
-                        "type": "input_text",
-                        "text": "Summarize the key points from this letter.",
-                    },
-                    {
-                        "type": "input_file",
-                        "file_id": file_id,
-                    },
-                ],
-            },
-        ],
-        api_key=api_key,
-        max_output_tokens=100,
-    )
-    
-    print(f"Response created: {response}")
-    print(f"Response type: {type(response)}")
-    print(f"Response has id: {hasattr(response, 'id')}")
-    
-    # Handle both dict and ResponsesAPIResponse object
-    if isinstance(response, dict):
-        response_id = response.get("id")
-    else:
-        response_id = getattr(response, "id", None)
-    
-    assert response_id is not None, f"Response ID is None. Response: {response}"
-
-    
-    # Step 3: Clean up - delete the file
-    print(f"\nStep 4: Cleaning up - deleting file {file_id}...")
-    deleted_file = await litellm.afile_delete(
-        file_id=file_id,
-        custom_llm_provider="manus",
-        api_key=api_key,
-    )
-    print(f"Deleted file: {deleted_file}")
-    assert deleted_file.deleted is True
-    
-    print("\n✅ File upload and responses API integration test passed!")
-
-

tests/llm_translation/test_anthropic_completion.py

@@ -1600,6 +1600,7 @@ def test_anthropic_via_responses_api():
         ResponsesAPIStreamEvents.RESPONSE_CREATED,
         ResponsesAPIStreamEvents.RESPONSE_IN_PROGRESS,
         ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED,
+        ResponsesAPIStreamEvents.CONTENT_PART_ADDED,
         ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA,  # Can occur multiple times
         ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE,
         ResponsesAPIStreamEvents.CONTENT_PART_DONE,

tests/llm_translation/test_azure_ai.py

@@ -34,6 +34,8 @@ from litellm import completion
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.types.utils import StandardLoggingPayload
 
+AZURE_AI_API_BASE = os.getenv("AZURE_AI_API_BASE")
+
 
 @pytest.mark.parametrize(
     "model_group_header, expected_model",
@@ -295,14 +297,15 @@ def test_completion_azure():
         litellm.set_verbose = False
         ## Test azure call
         response = completion(
-            model="azure/gpt-4.1-mini",
+            model="azure_ai/gpt-4.1-mini",
+            api_base=os.getenv("AZURE_AI_API_BASE"),
             messages=[
                 {
                     "role": "user",
                     "content": "Hello, how are you?",
                 }
             ],
-            api_key="os.environ/AZURE_AI_API_KEY",
+            api_key=os.getenv("AZURE_AI_API_KEY"),
         )
         print(f"response: {response}")
         print(f"response hidden params: {response._hidden_params}")
@@ -318,8 +321,8 @@ def test_completion_azure():
 @pytest.mark.parametrize(
     "api_base",
     [
-        "https://litellm-ci-cd-prod.cognitiveservices.azure.com/",
-        "https://litellm-ci-cd-prod.cognitiveservices.azure.com/openai/deployments/gpt-4.1-mini/chat/completions?api-version=2023-03-15-preview",
+        AZURE_AI_API_BASE,
+        f"{AZURE_AI_API_BASE}/openai/deployments/gpt-4.1-mini/chat/completions?api-version=2023-03-15-preview",
     ],
 )
 def test_completion_azure_ai_gpt_4o_with_flexible_api_base(api_base):
@@ -397,7 +400,7 @@ async def test_azure_ai_model_router_streaming_model_in_chunk():
     response = await litellm.acompletion(
         model="azure_ai/azure-model-router",
         messages=[{"role": "user", "content": "hi"}],
-        api_base="https://ishaa-mh6uutut-swedencentral.cognitiveservices.azure.com/openai/v1/",
+        api_base=os.getenv("AZURE_MODEL_ROUTER_API_BASE"),
         api_key=os.getenv("AZURE_MODEL_ROUTER_API_KEY"),
         stream=True,
     )
@@ -475,7 +478,7 @@ async def test_azure_ai_model_router_streaming_cost_with_stream_options():
         response = await litellm.acompletion(
             model="azure_ai/azure-model-router",
             messages=[{"role": "user", "content": "hi"}],
-            api_base="https://ishaa-mh6uutut-swedencentral.cognitiveservices.azure.com/openai/v1/",
+            api_base=os.getenv("AZURE_MODEL_ROUTER_API_BASE"),
             api_key=os.getenv("AZURE_MODEL_ROUTER_API_KEY"),
             stream=True,
             stream_options={"include_usage": True},

tests/llm_translation/test_bedrock_completion.py

@@ -2472,7 +2472,6 @@ def test_bedrock_error_handling_streaming():
         "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf",
         # "https://raw.githubusercontent.com/datasets/gdp/master/data/gdp.csv",
         "https://www.cmu.edu/blackboard/files/evaluate/tests-example.xls",
-        "http://www.krishdholakia.com/",
         # "https://raw.githubusercontent.com/datasets/sample-data/master/README.txt", # invalid url
         "https://raw.githubusercontent.com/mdn/content/main/README.md",
     ],

tests/llm_translation/test_cloudflare.py

@@ -1,42 +1,145 @@
-import os
-import sys
-import traceback
-
-from dotenv import load_dotenv
-
-load_dotenv()
-import io
-import os
-
-sys.path.insert(
-    0, os.path.abspath("../..")
-)  # Adds the parent directory to the system path
+import asyncio
 import json
+from typing import Any, Dict
+from unittest.mock import AsyncMock, MagicMock, patch
 
+import httpx
 import pytest
 
-import litellm
-from litellm import RateLimitError, Timeout, completion, completion_cost, embedding
+from litellm import acompletion, completion
+from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
+
+FAKE_API_BASE = "https://fake-cloudflare.example.com/client/v4/accounts/fake-acct/ai/run/"
+FAKE_API_KEY = "fake-cf-api-key"
 
 
-# Cloud flare AI test
-@pytest.mark.asyncio
-@pytest.mark.parametrize("stream", [True, False])
-async def test_completion_cloudflare(stream):
-    try:
-        litellm.set_verbose = False
-        response = await litellm.acompletion(
-            model="cloudflare/@cf/meta/llama-2-7b-chat-int8",
-            messages=[{"content": "what llm are you", "role": "user"}],
-            max_tokens=15,
-            stream=stream,
-        )
-        print(response)
-        if stream is True:
-            async for chunk in response:
-                print(chunk)
-        else:
-            print(response)
+def _make_mock_response(json_data: Dict[str, Any]) -> MagicMock:
+    mock = MagicMock(spec=httpx.Response)
+    mock.status_code = 200
+    mock.headers = {"content-type": "application/json"}
+    mock.json.return_value = json_data
+    mock.text = json.dumps(json_data)
+    return mock
 
-    except Exception as e:
-        pytest.fail(f"Error occurred: {e}")
+
+def _chat_response() -> Dict[str, Any]:
+    return {
+        "result": {
+            "response": "I am a large language model created to assist you.",
+        },
+        "success": True,
+        "errors": [],
+        "messages": [],
+    }
+
+
+def _streaming_chunks() -> list[str]:
+    return [
+        json.dumps({"response": "I am"}),
+        json.dumps({"response": " a language"}),
+        json.dumps({"response": " model."}),
+    ]
+
+
+@pytest.mark.parametrize("sync_mode", [True, False])
+def test_completion_cloudflare(sync_mode):
+    messages = [{"role": "user", "content": "what llm are you"}]
+    mock_resp = _make_mock_response(_chat_response())
+
+    if sync_mode:
+        with patch.object(HTTPHandler, "post", return_value=mock_resp) as mock_post:
+            response = completion(
+                model="cloudflare/@cf/meta/llama-2-7b-chat-int8",
+                messages=messages,
+                max_tokens=15,
+                api_base=FAKE_API_BASE,
+                api_key=FAKE_API_KEY,
+            )
+            mock_post.assert_called_once()
+    else:
+        with patch.object(
+            AsyncHTTPHandler, "post", new_callable=AsyncMock, return_value=mock_resp
+        ) as mock_post:
+            response = asyncio.run(
+                acompletion(
+                    model="cloudflare/@cf/meta/llama-2-7b-chat-int8",
+                    messages=messages,
+                    max_tokens=15,
+                    api_base=FAKE_API_BASE,
+                    api_key=FAKE_API_KEY,
+                )
+            )
+            mock_post.assert_called_once()
+
+    assert response is not None
+    assert response.choices[0].message.content is not None
+    assert "language model" in response.choices[0].message.content.lower()
+
+
+@pytest.mark.parametrize("sync_mode", [True, False])
+def test_completion_cloudflare_stream(sync_mode):
+    messages = [{"role": "user", "content": "what llm are you"}]
+    raw_chunks = _streaming_chunks()
+
+    if sync_mode:
+
+        def _iter_lines():
+            for chunk in raw_chunks:
+                yield f"data: {chunk}"
+            yield "data: [DONE]"
+
+        mock_resp = MagicMock()
+        mock_resp.iter_lines.return_value = _iter_lines()
+        mock_resp.status_code = 200
+        mock_resp.headers = {"content-type": "text/event-stream"}
+
+        with patch.object(HTTPHandler, "post", return_value=mock_resp) as mock_post:
+            response = completion(
+                model="cloudflare/@cf/meta/llama-2-7b-chat-int8",
+                messages=messages,
+                max_tokens=15,
+                stream=True,
+                api_base=FAKE_API_BASE,
+                api_key=FAKE_API_KEY,
+            )
+            chunks_received = list(response)
+            mock_post.assert_called_once()
+    else:
+
+        async def _aiter_lines():
+            for chunk in raw_chunks:
+                yield f"data: {chunk}"
+            yield "data: [DONE]"
+
+        mock_resp = MagicMock()
+        mock_resp.aiter_lines.return_value = _aiter_lines()
+        mock_resp.status_code = 200
+        mock_resp.headers = {"content-type": "text/event-stream"}
+
+        async def _run():
+            with patch.object(
+                AsyncHTTPHandler, "post", new_callable=AsyncMock, return_value=mock_resp
+            ) as mock_post:
+                resp = await acompletion(
+                    model="cloudflare/@cf/meta/llama-2-7b-chat-int8",
+                    messages=messages,
+                    max_tokens=15,
+                    stream=True,
+                    api_base=FAKE_API_BASE,
+                    api_key=FAKE_API_KEY,
+                )
+                received = []
+                async for chunk in resp:
+                    received.append(chunk)
+                mock_post.assert_called_once()
+                return received
+
+        chunks_received = asyncio.run(_run())
+
+    assert len(chunks_received) > 0
+    content = "".join(
+        c.choices[0].delta.content
+        for c in chunks_received
+        if c.choices[0].delta.content
+    )
+    assert "language" in content.lower()

tests/llm_translation/test_fireworks_ai_translation.py

@@ -77,18 +77,6 @@ def test_map_response_format():
     }
 
 
-@pytest.mark.skip(reason="fireworks is having an active outage")
-class TestFireworksAIChatCompletion(BaseLLMChatTest):
-    def get_base_completion_call_args(self) -> dict:
-        return {
-            "model": "fireworks_ai/accounts/fireworks/models/llama-v3p1-8b-instruct"
-        }
-
-    def test_tool_call_no_arguments(self, tool_call_no_arguments):
-        """Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833"""
-        pass
-
-
 class TestFireworksAIAudioTranscription(BaseLLMAudioTranscriptionTest):
     def get_base_audio_transcription_call_args(self) -> dict:
         return {
@@ -252,7 +240,5 @@ def test_global_disable_flag_with_transform_messages_helper(monkeypatch):
         json_data = json.loads(mock_post.call_args.kwargs["data"])
         assert (
             "#transform=inline"
-            not in json_data["messages"][0]["content"][1]["image_url"][
-                "url"
-            ]
+            not in json_data["messages"][0]["content"][1]["image_url"]["url"]
         )

tests/llm_translation/test_gemini.py

@@ -532,7 +532,7 @@ def test_gemini_with_grounding():
     ## Check streaming
 
     response = completion(
-        model="gemini/gemini-2.0-flash",
+        model="gemini/gemini-2.5-flash",
         messages=[{"role": "user", "content": "What is the capital of France?"}],
         tools=tools,
         stream=True,
@@ -566,7 +566,7 @@ def test_gemini_with_empty_function_call_arguments():
         }
     ]
     response = completion(
-        model="gemini/gemini-2.0-flash",
+        model="gemini/gemini-2.5-flash",
         messages=[{"role": "user", "content": "What is the capital of France?"}],
         tools=tools,
     )
@@ -775,7 +775,7 @@ def test_gemini_tool_use():
             {"role": "system", "content": "You are a helpful assistant."},
             {"role": "user", "content": "What's the weather like in Lima, Peru today?"},
         ],
-        "model": "gemini/gemini-2.0-flash",
+        "model": "gemini/gemini-2.5-flash",
         "tools": [
             {
                 "type": "function",

tests/llm_translation/test_snowflake.py

@@ -1,12 +1,10 @@
-import os
-import sys
+import asyncio
 import json
+import os
 import httpx
 from typing import Any, Dict, List
-from unittest.mock import Mock, MagicMock, patch
-from dotenv import load_dotenv
+from unittest.mock import AsyncMock, MagicMock, patch
 
-load_dotenv()
 import pytest
 
 from litellm import completion, acompletion, responses
@@ -14,202 +12,68 @@ from litellm.exceptions import APIConnectionError
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
 
 
-def mock_snowflake_chat_response() -> Dict[str, Any]:
+@pytest.mark.skip(reason="Requires Snowflake credentials - run manually when needed")
+def test_snowflake_tool_calling_responses_api():
     """
-    Mock response for Snowflake chat completion.
+    Test Snowflake tool calling with Responses API.
+    Requires SNOWFLAKE_JWT and SNOWFLAKE_ACCOUNT_ID environment variables.
     """
-    return {
-        "id": "chatcmpl-snowflake-123",
-        "object": "chat.completion",
-        "created": 1700000000,
-        "model": "mistral-7b",
-        "choices": [
-            {
-                "index": 0,
-                "message": {
-                    "role": "assistant",
-                    "content": "The sky above is painted blue,\nWith clouds of white and morning dew.\nA canvas vast, serene and bright,\nThat fills my heart with pure delight.",
+    import litellm
+
+    # Skip if credentials not available
+    if not os.getenv("SNOWFLAKE_JWT") or not os.getenv("SNOWFLAKE_ACCOUNT_ID"):
+        pytest.skip("Snowflake credentials not available")
+
+    litellm.drop_params = False  # We now support tools!
+
+    tools = [
+        {
+            "type": "function",
+            "name": "get_weather",
+            "description": "Get the current weather in a given location",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "location": {
+                        "type": "string",
+                        "description": "The city and state, e.g. San Francisco, CA",
+                    }
                 },
-                "finish_reason": "stop",
-            }
-        ],
-        "usage": {
-            "prompt_tokens": 10,
-            "completion_tokens": 30,
-            "total_tokens": 40,
-        },
-    }
-
-
-def mock_snowflake_streaming_response_chunks() -> List[str]:
-    """
-    Mock streaming response chunks for Snowflake.
-    """
-    return [
-        json.dumps(
-            {
-                "id": "chatcmpl-snowflake-stream-123",
-                "object": "chat.completion.chunk",
-                "created": 1700000000,
-                "model": "mistral-7b",
-                "choices": [
-                    {
-                        "index": 0,
-                        "delta": {"role": "assistant", "content": "The"},
-                        "finish_reason": None,
-                    }
-                ],
-            }
-        ),
-        json.dumps(
-            {
-                "id": "chatcmpl-snowflake-stream-123",
-                "object": "chat.completion.chunk",
-                "created": 1700000000,
-                "model": "mistral-7b",
-                "choices": [
-                    {
-                        "index": 0,
-                        "delta": {"content": " sky"},
-                        "finish_reason": None,
-                    }
-                ],
-            }
-        ),
-        json.dumps(
-            {
-                "id": "chatcmpl-snowflake-stream-123",
-                "object": "chat.completion.chunk",
-                "created": 1700000000,
-                "model": "mistral-7b",
-                "choices": [
-                    {
-                        "index": 0,
-                        "delta": {"content": " is blue"},
-                        "finish_reason": "stop",
-                    }
-                ],
-            }
-        ),
+                "required": ["location"],
+            },
+        }
     ]
 
+    try:
+        # Test with tool_choice to force tool use
+        response = responses(
+            model="snowflake/claude-3-5-sonnet",
+            input="What's the weather in Paris?",
+            tools=tools,
+            tool_choice={"type": "function", "function": {"name": "get_weather"}},
+            max_output_tokens=200,
+        )
 
-@pytest.mark.parametrize("sync_mode", [True, False])
-def test_chat_completion_snowflake(sync_mode):
-    """
-    Test Snowflake chat completion with mocked HTTP responses.
-    """
-    messages = [
-        {
-            "role": "user",
-            "content": "Write me a poem about the blue sky",
-        },
-    ]
+        assert response is not None
+        assert hasattr(response, "output")
+        assert len(response.output) > 0
 
-    mock_response = Mock(spec=httpx.Response)
-    mock_response.status_code = 200
-    mock_response.json.return_value = mock_snowflake_chat_response()
+        # Verify tool call was made
+        tool_call_found = False
+        for item in response.output:
+            if hasattr(item, "type") and item.type == "function_call":
+                tool_call_found = True
+                assert item.name == "get_weather"
+                assert hasattr(item, "arguments")
+                print(f"✅ Tool call detected: {item.name}({item.arguments})")
+                break
 
-    if sync_mode:
-        sync_handler = HTTPHandler()
-        with patch.object(HTTPHandler, "post", return_value=mock_response):
-            response = completion(
-                model="snowflake/mistral-7b",
-                messages=messages,
-                api_base="https://exampleopenaiendpoint-production.up.railway.app/v1/chat/completions",
-                client=sync_handler,
-            )
-            assert response is not None
-            assert response.choices[0].message.content is not None
-            assert "sky" in response.choices[0].message.content.lower()
-    else:
-        async_handler = AsyncHTTPHandler()
-        with patch.object(AsyncHTTPHandler, "post", return_value=mock_response):
-            import asyncio
+        assert tool_call_found, "Expected tool call but none was found"
 
-            response = asyncio.run(
-                acompletion(
-                    model="snowflake/mistral-7b",
-                    messages=messages,
-                    api_base="https://exampleopenaiendpoint-production.up.railway.app/v1/chat/completions",
-                    client=async_handler,
-                )
-            )
-            assert response is not None
-            assert response.choices[0].message.content is not None
-            assert "sky" in response.choices[0].message.content.lower()
-
-
-@pytest.mark.parametrize("sync_mode", [True, False])
-def test_chat_completion_snowflake_stream(sync_mode):
-    """
-    Test Snowflake streaming chat completion with mocked HTTP responses.
-    """
-    messages = [
-        {
-            "role": "user",
-            "content": "Write me a poem about the blue sky",
-        },
-    ]
-
-    if sync_mode:
-        sync_handler = HTTPHandler()
-        mock_chunks = mock_snowflake_streaming_response_chunks()
-
-        def mock_iter_lines():
-            for chunk in mock_chunks:
-                for line in [f"data: {chunk}", "data: [DONE]"]:
-                    yield line
-
-        mock_response = MagicMock()
-        mock_response.iter_lines.side_effect = mock_iter_lines
-        mock_response.status_code = 200
-
-        with patch.object(HTTPHandler, "post", return_value=mock_response):
-            response = completion(
-                model="snowflake/mistral-7b",
-                messages=messages,
-                max_tokens=100,
-                stream=True,
-                api_base="https://exampleopenaiendpoint-production.up.railway.app/v1/chat/completions",
-                client=sync_handler,
-            )
-
-            chunks_received = []
-            for chunk in response:
-                chunks_received.append(chunk)
-
-            assert len(chunks_received) > 0
-    else:
-        async_handler = AsyncHTTPHandler()
-        mock_chunks = mock_snowflake_streaming_response_chunks()
-
-        async def mock_iter_lines():
-            for chunk in mock_chunks:
-                for line in [f"data: {chunk}", "data: [DONE]"]:
-                    yield line
-
-        mock_response = MagicMock()
-        mock_response.iter_lines.side_effect = mock_iter_lines
-        mock_response.status_code = 200
-
-        with patch.object(AsyncHTTPHandler, "post", return_value=mock_response):
-            import asyncio
-
-            async def test_async_stream():
-                response = await acompletion(
-                    model="snowflake/mistral-7b",
-                    messages=messages,
-                    max_tokens=100,
-                    stream=True,
-                    api_base="https://exampleopenaiendpoint-production.up.railway.app/v1/chat/completions",
-                    client=async_handler,
-                )
-
-                chunks_received = []
-                async for chunk in response:
-                    chunks_received.append(chunk)
-
-                assert len(chunks_received) > 0
-
-            asyncio.run(test_async_stream())
+    except APIConnectionError as e:
+        if "JWT token is invalid" in str(e):
+            pytest.skip("Invalid Snowflake JWT token")
+        elif "Application failed to respond" in str(e) or "502" in str(e):
+            pytest.skip(f"Snowflake API unavailable: {e}")
+        else:
+            raise

tests/llm_translation/test_watsonx.py

@@ -13,6 +13,16 @@ import pytest
 from typing import Optional
 
 
+@pytest.fixture(autouse=True)
+def watsonx_env_vars(monkeypatch):
+    """Set required WatsonX env vars so the provider passes validation.
+    Also clear WATSONX_ZENAPIKEY/WATSONX_TOKEN so they don't bypass the IAM token mock."""
+    monkeypatch.setenv("WATSONX_URL", "https://us-south.ml.cloud.ibm.com")
+    monkeypatch.setenv("WATSONX_PROJECT_ID", "test-project-id")
+    monkeypatch.delenv("WATSONX_ZENAPIKEY", raising=False)
+    monkeypatch.delenv("WATSONX_TOKEN", raising=False)
+
+
 @pytest.fixture
 def watsonx_chat_completion_call():
     def _call(

tests/local_testing/adroit-crow-413218-bc47f303efc9.json

@@ -1,13 +0,0 @@
-{
-  "type": "service_account",
-  "project_id": "pathrise-convert-1606954137718",
-  "private_key_id": "",
-  "private_key": "",
-  "client_email": "test-adroit-crow@pathrise-convert-1606954137718.iam.gserviceaccount.com",
-  "client_id": "104886546564708740969",
-  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
-  "token_uri": "https://oauth2.googleapis.com/token",
-  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
-  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/test-adroit-crow%40pathrise-convert-1606954137718.iam.gserviceaccount.com",
-  "universe_domain": "googleapis.com"
-}

tests/local_testing/test_amazing_vertex_completion.py

@@ -167,43 +167,6 @@ async def test_get_response():
         pytest.fail(f"An error occurred - {str(e)}")
 
 
-@pytest.mark.skip(
-    reason="Local test. Vertex AI Quota is low. Leads to rate limit errors on ci/cd."
-)
-@pytest.mark.flaky(retries=3, delay=1)
-def test_vertex_ai_anthropic_streaming():
-    try:
-        load_vertex_ai_credentials()
-
-        # litellm.set_verbose = True
-
-        model = "claude-3-5-sonnet@20240620"
-
-        vertex_ai_project = "pathrise-convert-1606954137718"
-        vertex_ai_location = "asia-southeast1"
-        json_obj = get_vertex_ai_creds_json()
-        vertex_credentials = json.dumps(json_obj)
-
-        response = completion(
-            model="vertex_ai/" + model,
-            messages=[{"role": "user", "content": "hi"}],
-            temperature=0.7,
-            vertex_ai_project=vertex_ai_project,
-            vertex_ai_location=vertex_ai_location,
-            stream=True,
-        )
-        # print("\nModel Response", response)
-        for idx, chunk in enumerate(response):
-            print(f"chunk: {chunk}")
-            streaming_format_tests(idx=idx, chunk=chunk)
-
-    # raise Exception("it worked!")
-    except litellm.RateLimitError as e:
-        pass
-    except Exception as e:
-        pytest.fail(f"Error occurred: {e}")
-
-
 # test_vertex_ai_anthropic_streaming()
 
 
@@ -394,10 +357,7 @@ async def test_async_vertexai_response_basic():
         user_message = "Hello, how are you?"
         messages = [{"content": user_message, "role": "user"}]
         response = await acompletion(
-            model="gemini-2.5-flash",
-            messages=messages, 
-            temperature=0.7, 
-            timeout=5
+            model="gemini-2.5-flash", messages=messages, temperature=0.7, timeout=5
         )
         print(f"response: {response}")
     except litellm.NotFoundError as e:
@@ -414,8 +374,6 @@ async def test_async_vertexai_response_basic():
         pytest.fail(f"An exception occurred: {e}")
 
 
-
-
 @pytest.mark.flaky(retries=3, delay=1)
 @pytest.mark.asyncio
 async def test_async_vertexai_streaming_response():
@@ -739,7 +697,9 @@ def test_gemini_pro_grounding(value_in_dict):
 
 
 # @pytest.mark.skip(reason="exhausted vertex quota. need to refactor to mock the call")
-@pytest.mark.parametrize("model", ["vertex_ai_beta/gemini-2.5-flash-lite"])  # "vertex_ai",
+@pytest.mark.parametrize(
+    "model", ["vertex_ai_beta/gemini-2.5-flash-lite"]
+)  # "vertex_ai",
 @pytest.mark.parametrize("sync_mode", [True])  # "vertex_ai",
 @pytest.mark.asyncio
 @pytest.mark.flaky(retries=6, delay=2)
@@ -808,7 +768,17 @@ async def test_gemini_pro_function_calling_httpx(model, sync_mode):
     except Exception as e:
         error_msg = str(e)
         # Skip test for known transient API issues
-        if any(x in error_msg for x in ["429 Quota exceeded", "503", "Service unavailable", "timeout", "Timeout", "UNAVAILABLE"]):
+        if any(
+            x in error_msg
+            for x in [
+                "429 Quota exceeded",
+                "503",
+                "Service unavailable",
+                "timeout",
+                "Timeout",
+                "UNAVAILABLE",
+            ]
+        ):
             pytest.skip(f"Transient API error: {error_msg}")
         else:
             pytest.fail(f"An unexpected exception occurred - {error_msg}")
@@ -1396,12 +1366,14 @@ async def test_gemini_pro_json_schema_args_sent_httpx(
                 # Gemini 2.x+ uses response_json_schema, Gemini 1.x uses response_schema
                 gen_config = mock_call.call_args.kwargs["json"]["generationConfig"]
                 assert (
-                    "response_schema" in gen_config or "response_json_schema" in gen_config
+                    "response_schema" in gen_config
+                    or "response_json_schema" in gen_config
                 ), f"Expected response_schema or response_json_schema in {gen_config}"
             else:
                 gen_config = mock_call.call_args.kwargs["json"]["generationConfig"]
                 assert (
-                    "response_schema" not in gen_config and "response_json_schema" not in gen_config
+                    "response_schema" not in gen_config
+                    and "response_json_schema" not in gen_config
                 )
                 assert (
                     "Use this JSON schema:"
@@ -1577,7 +1549,8 @@ async def test_gemini_pro_json_schema_args_sent_httpx_openai_schema(
                 # Gemini 2.x+ uses response_json_schema, Gemini 1.x uses response_schema
                 gen_config = mock_call.call_args.kwargs["json"]["generationConfig"]
                 assert (
-                    "response_schema" in gen_config or "response_json_schema" in gen_config
+                    "response_schema" in gen_config
+                    or "response_json_schema" in gen_config
                 ), f"Expected response_schema or response_json_schema in {gen_config}"
                 assert (
                     "response_mime_type"
@@ -1592,7 +1565,8 @@ async def test_gemini_pro_json_schema_args_sent_httpx_openai_schema(
             else:
                 gen_config = mock_call.call_args.kwargs["json"]["generationConfig"]
                 assert (
-                    "response_schema" not in gen_config and "response_json_schema" not in gen_config
+                    "response_schema" not in gen_config
+                    and "response_json_schema" not in gen_config
                 )
                 assert (
                     "Use this JSON schema:"
@@ -2313,8 +2287,6 @@ def test_prompt_factory_nested():
         ), "'text' value not a string."
 
 
-
-
 @pytest.mark.asyncio
 async def test_completion_fine_tuned_model():
     load_vertex_ai_credentials()
@@ -2380,7 +2352,7 @@ async def test_completion_fine_tuned_model():
         # this is the fine-tuned model endpoint
         assert (
             url[0]
-            == "https://us-central1-aiplatform.googleapis.com/v1/projects/pathrise-convert-1606954137718/locations/us-central1/endpoints/4965075652664360960:generateContent"
+            == "https://us-central1-aiplatform.googleapis.com/v1/projects/litellm-ci-cd/locations/us-central1/endpoints/4965075652664360960:generateContent"
         )
 
         print("call args = ", kwargs)
@@ -2579,20 +2551,20 @@ async def test_gemini_context_caching_anthropic_format(sync_mode):
 async def test_gemini_context_caching_disabled_flag(sync_mode):
     """
     Test that disable_anthropic_gemini_context_caching_transform flag properly disables context caching.
-    
+
     When the flag is set to True, messages with cache_control should not trigger caching API calls.
     """
     from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
 
     litellm.set_verbose = True
-    
+
     # Store original value to restore later
     original_flag_value = litellm.disable_anthropic_gemini_context_caching_transform
-    
+
     try:
         # Enable the disable flag
         litellm.disable_anthropic_gemini_context_caching_transform = True
-        
+
         gemini_context_caching_messages = [
             # System Message with cache_control
             {
@@ -2633,13 +2605,15 @@ async def test_gemini_context_caching_disabled_flag(sync_mode):
                 ],
             },
         ]
-        
+
         if sync_mode:
             client = HTTPHandler(concurrent_limit=1)
         else:
             client = AsyncHTTPHandler(concurrent_limit=1)
-            
-        with patch.object(client, "post", side_effect=mock_gemini_request) as mock_client:
+
+        with patch.object(
+            client, "post", side_effect=mock_gemini_request
+        ) as mock_client:
             try:
                 if sync_mode:
                     response = litellm.completion(
@@ -2662,24 +2636,32 @@ async def test_gemini_context_caching_disabled_flag(sync_mode):
                 print(e)
 
             # When caching is disabled, should only make 1 call (no separate cache creation call)
-            assert mock_client.call_count == 1, f"Expected 1 call when caching is disabled, got {mock_client.call_count}"
+            assert (
+                mock_client.call_count == 1
+            ), f"Expected 1 call when caching is disabled, got {mock_client.call_count}"
 
             first_call_args = mock_client.call_args_list[0].kwargs
             first_call_positional_args = mock_client.call_args_list[0].args
 
             print(f"first_call_args with caching disabled: {first_call_args}")
-            print(f"first_call_positional_args with caching disabled: {first_call_positional_args}")
+            print(
+                f"first_call_positional_args with caching disabled: {first_call_positional_args}"
+            )
 
             # Assert that cachedContents is NOT in the URL when caching is disabled
-            url = first_call_args.get("url", first_call_positional_args[0] if first_call_positional_args else "")
-            assert "cachedContents" not in url, "cachedContents should not be in URL when caching is disabled"
-            
+            url = first_call_args.get(
+                "url",
+                first_call_positional_args[0] if first_call_positional_args else "",
+            )
+            assert (
+                "cachedContents" not in url
+            ), "cachedContents should not be in URL when caching is disabled"
+
     finally:
         # Restore original flag value
         litellm.disable_anthropic_gemini_context_caching_transform = original_flag_value
 
 
-
 @pytest.mark.asyncio
 async def test_partner_models_httpx_ai21():
     litellm.set_verbose = True
@@ -2776,7 +2758,7 @@ async def test_partner_models_httpx_ai21():
 
         assert (
             url[0]
-            == "https://us-central1-aiplatform.googleapis.com/v1beta1/projects/pathrise-convert-1606954137718/locations/us-central1/publishers/ai21/models/jamba-1.5-mini@001:rawPredict"
+            == "https://us-central1-aiplatform.googleapis.com/v1beta1/projects/litellm-ci-cd/locations/us-central1/publishers/ai21/models/jamba-1.5-mini@001:rawPredict"
         )
 
         # json loads kwargs
@@ -2920,7 +2902,9 @@ def test_gemini_function_call_parameter_in_messages():
                 "contents": [
                     {
                         "role": "user",
-                        "parts": [{"text": "search for weather in boston (use `search`)"}],
+                        "parts": [
+                            {"text": "search for weather in boston (use `search`)"}
+                        ],
                     },
                     {
                         "role": "model",
@@ -2947,7 +2931,9 @@ def test_gemini_function_call_parameter_in_messages():
                         ],
                     },
                 ],
-                "system_instruction": {"parts": [{"text": "Use search for most queries."}]},
+                "system_instruction": {
+                    "parts": [{"text": "Use search for most queries."}]
+                },
                 "tools": [
                     {
                         "function_declarations": [
@@ -3578,8 +3564,9 @@ def test_gemini_tool_calling_working_demo():
                 },
             }
         ],
+        "vertex_location": "global",
     }
-    response = completion(model="vertex_ai/gemini-2.0-flash", **args)
+    response = completion(model="vertex_ai/gemini-3-flash-preview", **args)
     print(response)
 
 
@@ -3650,8 +3637,9 @@ def test_gemini_tool_calling_not_working():
                 },
             }
         ],
+        "vertex_location": "global",
     }
-    response = completion(model="vertex_ai/gemini-2.0-flash", **args)
+    response = completion(model="vertex_ai/gemini-3-flash-preview", **args)
     print(response)
 
 
@@ -3746,8 +3734,8 @@ def test_gemini_nullable_object_tool_schema_httpx():
     load_vertex_ai_credentials()
     litellm._turn_on_debug()
 
-
-    tools = [{
+    tools = [
+        {
             "type": "function",
             "strict": True,
             "function": {
@@ -3760,7 +3748,7 @@ def test_gemini_nullable_object_tool_schema_httpx():
                     "properties": {
                         "ticket_id": {
                             "type": "string",
-                            "description": "Unique identifier for the support ticket"
+                            "description": "Unique identifier for the support ticket",
                         },
                         "customer_context": {
                             "type": ["object", "null"],
@@ -3770,18 +3758,19 @@ def test_gemini_nullable_object_tool_schema_httpx():
                             "properties": {
                                 "user_id": {
                                     "type": "string",
-                                    "description": "Internal user identifier"
+                                    "description": "Internal user identifier",
                                 },
                                 "plan": {
                                     "type": "string",
-                                    "description": "Subscription plan name (e.g. pro, enterprise)"
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }]
+                                    "description": "Subscription plan name (e.g. pro, enterprise)",
+                                },
+                            },
+                        },
+                    },
+                },
+            },
+        }
+    ]
 
     response = litellm.completion(
         model="vertex_ai/gemini-2.5-flash",
@@ -3986,17 +3975,23 @@ def test_vertex_ai_gemini_audio_ogg():
         for part in content["parts"]
         if "file_data" in part
     ]
-    assert len(file_data_parts) == 1, f"Expected 1 file_data part, got: {file_data_parts}"
+    assert (
+        len(file_data_parts) == 1
+    ), f"Expected 1 file_data part, got: {file_data_parts}"
     file_data = file_data_parts[0]["file_data"]
-    assert file_data["mime_type"] == "audio/ogg", f"Expected audio/ogg, got: {file_data['mime_type']}"
-    assert "En-us-public.ogg" in file_data["file_uri"], f"Unexpected file_uri: {file_data['file_uri']}"
+    assert (
+        file_data["mime_type"] == "audio/ogg"
+    ), f"Expected audio/ogg, got: {file_data['mime_type']}"
+    assert (
+        "En-us-public.ogg" in file_data["file_uri"]
+    ), f"Unexpected file_uri: {file_data['file_uri']}"
     print(response)
 
 
 @pytest.mark.asyncio
 async def test_vertex_ai_deepseek():
     """Test that deepseek models use the correct v1 API endpoint instead of v1beta1."""
-    # load_vertex_ai_credentials()
+    load_vertex_ai_credentials()
     litellm._turn_on_debug()
     from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
 
@@ -4041,7 +4036,7 @@ def test_gemini_grounding_on_streaming():
     load_vertex_ai_credentials()
     # litellm._turn_on_debug()
     args = {
-        "model": "vertex_ai/gemini-2.0-flash",
+        "model": "vertex_ai/gemini-3-flash-preview",
         "messages": [
             {
                 "role": "user",
@@ -4053,6 +4048,7 @@ def test_gemini_grounding_on_streaming():
                 ],
             }
         ],
+        "vertex_location": "global",
         "stream": True,
         "tools": [{"googleSearch": {}}],
         "fallbacks": [],
@@ -4075,11 +4071,20 @@ def test_gemini_google_maps_tool_simple():
     litellm._turn_on_debug()
 
     tools = [{"googleMaps": {"enableWidget": True}}]
-    tools_with_location = [{"googleMaps": {"enableWidget": True, "latitude": 37.7749, "longitude": -122.4194, "languageCode": "en_US"}}]
+    tools_with_location = [
+        {
+            "googleMaps": {
+                "enableWidget": True,
+                "latitude": 37.7749,
+                "longitude": -122.4194,
+                "languageCode": "en_US",
+            }
+        }
+    ]
     try:
         for tools in [tools, tools_with_location]:
             response = completion(
-                model="vertex_ai/gemini-2.0-flash",
+                model="vertex_ai/gemini-3-flash-preview",
                 messages=[
                     {
                         "role": "user",
@@ -4087,6 +4092,7 @@ def test_gemini_google_maps_tool_simple():
                     }
                 ],
                 tools=tools,
+                vertex_location="global",
             )
         print(f"Response: {response.model_dump_json(indent=4)}")
         assert response.choices[0].message.content is not None
@@ -4094,4 +4100,3 @@ def test_gemini_google_maps_tool_simple():
         pass
     except Exception as e:
         pytest.fail(f"Error occurred: {e}")
-

tests/local_testing/test_assistants.py

@@ -288,6 +288,8 @@ async def test_aarun_thread_litellm(sync_mode, provider, is_streaming):
                         thread_id=_new_thread.id, custom_llm_provider=provider
                     )
                     assert isinstance(messages.data[0], Message)
+                elif run.status == "failed" and run.last_error and "No connection matching model" in run.last_error.message:
+                    pytest.skip(f"Azure deployment not found: {run.last_error.message}")
                 else:
                     pytest.fail(
                         "An unexpected error occurred when running the thread, {}".format(
@@ -320,6 +322,8 @@ async def test_aarun_thread_litellm(sync_mode, provider, is_streaming):
                         thread_id=_new_thread.id, custom_llm_provider=provider
                     )
                     assert isinstance(messages.data[0], Message)
+                elif run.status == "failed" and run.last_error and "No connection matching model" in run.last_error.message:
+                    pytest.skip(f"Azure deployment not found: {run.last_error.message}")
                 else:
                     pytest.fail(
                         "An unexpected error occurred when running the thread, {}".format(

tests/local_testing/test_caching_ssl.py

@@ -69,10 +69,10 @@ def test_caching_router():
         ]
         litellm.cache = Cache(
             type="redis",
-            host="os.environ/REDIS_HOST_2",
-            port="os.environ/REDIS_PORT_2",
-            password="os.environ/REDIS_PASSWORD_2",
-            ssl="os.environ/REDIS_SSL_2",
+            host="os.environ/REDIS_HOST",
+            port="os.environ/REDIS_PORT",
+            password="os.environ/REDIS_PASSWORD",
+            ssl="os.environ/REDIS_SSL",
         )
         router = Router(
             model_list=model_list,

tests/local_testing/test_completion.py

@@ -868,78 +868,6 @@ def test_completion_mistral_api_modified_input():
             pytest.fail(f"Error occurred: {e}")
 
 
-# def test_completion_oobabooga():
-#     try:
-#         response = completion(
-#             model="oobabooga/vicuna-1.3b", messages=messages, api_base="http://127.0.0.1:5000"
-#         )
-#         # Add any assertions here to check the response
-#         print(response)
-#     except Exception as e:
-#         pytest.fail(f"Error occurred: {e}")
-
-# test_completion_oobabooga()
-# aleph alpha
-# def test_completion_aleph_alpha():
-#     try:
-#         response = completion(
-#             model="luminous-base", messages=messages, logger_fn=logger_fn
-#         )
-#         # Add any assertions here to check the response
-#         print(response)
-#     except Exception as e:
-#         pytest.fail(f"Error occurred: {e}")
-# test_completion_aleph_alpha()
-
-
-# def test_completion_aleph_alpha_control_models():
-#     try:
-#         response = completion(
-#             model="luminous-base-control", messages=messages, logger_fn=logger_fn
-#         )
-#         # Add any assertions here to check the response
-#         print(response)
-#     except Exception as e:
-#         pytest.fail(f"Error occurred: {e}")
-# test_completion_aleph_alpha_control_models()
-
-import openai
-
-
-def test_completion_gpt4_turbo():
-    litellm.set_verbose = True
-    try:
-        response = completion(
-            model="gpt-4-1106-preview",
-            messages=messages,
-            max_completion_tokens=10,
-        )
-        print(response)
-    except openai.RateLimitError:
-        print("got a rate liimt error")
-        pass
-    except Exception as e:
-        pytest.fail(f"Error occurred: {e}")
-
-
-# test_completion_gpt4_turbo()
-
-
-def test_completion_gpt4_turbo_0125():
-    try:
-        response = completion(
-            model="gpt-4-0125-preview",
-            messages=messages,
-            max_tokens=10,
-        )
-        print(response)
-    except openai.RateLimitError:
-        print("got a rate liimt error")
-        pass
-    except Exception as e:
-        pytest.fail(f"Error occurred: {e}")
-
-
 @pytest.mark.skip(reason="this test is flaky")
 def test_completion_gpt4_vision():
     try:
@@ -2438,7 +2366,7 @@ def test_azure_openai_ad_token():
 # test_azure_openai_ad_token()
 
 
-# test_completion_azure()
+
 def test_completion_azure2():
     # test if we can pass api_base, api_version and api_key in compleition()
     try:
@@ -2560,7 +2488,6 @@ def test_completion_azure_with_litellm_key():
         pytest.fail(f"Error occurred: {e}")
 
 
-# test_completion_azure()
 
 
 import asyncio
@@ -3292,6 +3219,13 @@ def test_petals():
 ## test deep infra
 @pytest.mark.parametrize("drop_params", [True, False])
 def test_completion_deep_infra(drop_params):
+    """Test that DeepInfra requests are shaped correctly without making real API calls."""
+    from unittest.mock import MagicMock, patch
+    from openai import OpenAI
+    from openai.types.chat import ChatCompletion, ChatCompletionMessage
+    from openai.types.chat.chat_completion import Choice
+    import httpx
+
     litellm.set_verbose = False
     model_name = "deepinfra/meta-llama/Llama-2-70b-chat-hf"
     tools = [
@@ -3320,7 +3254,51 @@ def test_completion_deep_infra(drop_params):
             "content": "What's the weather like in Boston today in Fahrenheit?",
         }
     ]
-    try:
+
+    mock_response = ChatCompletion(
+        id="chatcmpl-mock",
+        choices=[
+            Choice(
+                finish_reason="stop",
+                index=0,
+                message=ChatCompletionMessage(
+                    content="It's sunny.", role="assistant"
+                ),
+            )
+        ],
+        created=1234567890,
+        model="meta-llama/Llama-2-70b-chat-hf",
+        object="chat.completion",
+        usage={"completion_tokens": 5, "prompt_tokens": 20, "total_tokens": 25},
+    )
+
+    mock_raw = MagicMock()
+    mock_raw.parse.return_value = mock_response
+    mock_raw.headers = httpx.Headers({"content-type": "application/json"})
+    mock_raw.status_code = 200
+
+    with patch(
+        "litellm.llms.openai.openai.OpenAIChatCompletion.make_sync_openai_chat_completion_request",
+        return_value=(mock_raw, mock_response),
+    ) as mock_create:
+        if drop_params is False:
+            # DeepInfra doesn't support tool_choice, should raise UnsupportedParamsError
+            with pytest.raises(litellm.exceptions.UnsupportedParamsError):
+                completion(
+                    model=model_name,
+                    messages=messages,
+                    temperature=0,
+                    max_tokens=10,
+                    tools=tools,
+                    tool_choice={
+                        "type": "function",
+                        "function": {"name": "get_current_weather"},
+                    },
+                    drop_params=drop_params,
+                    api_key="fake-api-key",
+                )
+            return
+
         response = completion(
             model=model_name,
             messages=messages,
@@ -3332,33 +3310,75 @@ def test_completion_deep_infra(drop_params):
                 "function": {"name": "get_current_weather"},
             },
             drop_params=drop_params,
+            api_key="fake-api-key",
         )
-        # Add any assertions here to check the response
-        print(response)
-    except Exception as e:
-        if drop_params is True:
-            pytest.fail(f"Error occurred: {e}")
+
+        # Verify the call was made
+        mock_create.assert_called_once()
+        call_kwargs = mock_create.call_args.kwargs
+
+        # Verify request shape
+        data = call_kwargs["data"]
+        assert data["model"] == "meta-llama/Llama-2-70b-chat-hf"
+        assert data["messages"] == messages
+        assert data["temperature"] == 0
+        assert data["max_tokens"] == 10
+        # tool_choice should be dropped for unsupported params
+        assert "tool_choice" not in data
 
 
 # test_completion_deep_infra()
 
 
 def test_completion_deep_infra_mistral():
-    print("deep infra test with temp=0")
+    """Test that DeepInfra Mistral requests are shaped correctly without making real API calls."""
+    from unittest.mock import MagicMock, patch
+    from openai.types.chat import ChatCompletion, ChatCompletionMessage
+    from openai.types.chat.chat_completion import Choice
+    import httpx
+
     model_name = "deepinfra/mistralai/Mistral-7B-Instruct-v0.1"
-    try:
+
+    mock_response = ChatCompletion(
+        id="chatcmpl-mock",
+        choices=[
+            Choice(
+                finish_reason="stop",
+                index=0,
+                message=ChatCompletionMessage(
+                    content="Hello!", role="assistant"
+                ),
+            )
+        ],
+        created=1234567890,
+        model="mistralai/Mistral-7B-Instruct-v0.1",
+        object="chat.completion",
+        usage={"completion_tokens": 5, "prompt_tokens": 20, "total_tokens": 25},
+    )
+
+    mock_raw = MagicMock()
+    mock_raw.parse.return_value = mock_response
+    mock_raw.headers = httpx.Headers({"content-type": "application/json"})
+    mock_raw.status_code = 200
+
+    with patch(
+        "litellm.llms.openai.openai.OpenAIChatCompletion.make_sync_openai_chat_completion_request",
+        return_value=(mock_raw, mock_response),
+    ) as mock_create:
         response = completion(
             model=model_name,
             messages=messages,
-            temperature=0.01,  # mistrail fails with temperature=0
+            temperature=0.01,
             max_tokens=10,
+            api_key="fake-api-key",
         )
-        # Add any assertions here to check the response
-        print(response)
-    except litellm.exceptions.Timeout as e:
-        pass
-    except Exception as e:
-        pytest.fail(f"Error occurred: {e}")
+
+        mock_create.assert_called_once()
+        call_kwargs = mock_create.call_args.kwargs
+        data = call_kwargs["data"]
+        assert data["model"] == "mistralai/Mistral-7B-Instruct-v0.1"
+        assert data["temperature"] == 0.01
+        assert data["max_tokens"] == 10
 
 
 # test_completion_deep_infra_mistral()

tests/local_testing/test_embedding.py

@@ -549,7 +549,7 @@ def test_bedrock_embedding_cohere():
                 "good morning from litellm, attempting to embed data",
                 "lets test a second string for good measure",
             ],
-            aws_region_name="os.environ/AWS_REGION_NAME",
+            aws_region_name="us-west-2",
         )
         assert isinstance(
             response["data"][0]["embedding"], list

tests/local_testing/test_exceptions.py

@@ -24,7 +24,7 @@ from litellm import (  # AuthenticationError,; RateLimitError,; ServiceUnavailab
     embedding,
 )
 
-litellm.vertex_project = "pathrise-convert-1606954137718"
+litellm.vertex_project = "litellm-ci-cd"
 litellm.vertex_location = "us-central1"
 litellm.num_retries = 0
 

tests/local_testing/test_gcs_bucket.py

@@ -21,177 +21,68 @@ from litellm.integrations.gcs_bucket.gcs_bucket import (
     StandardLoggingPayload,
 )
 from litellm.types.utils import StandardCallbackDynamicParams
-from unittest.mock import patch
+from litellm.types.integrations.gcs_bucket import GCSLoggingConfig
+from unittest.mock import patch, AsyncMock, MagicMock
 
 verbose_logger.setLevel(logging.DEBUG)
 
 
-def load_vertex_ai_credentials():
-    # Define the path to the vertex_key.json file
-    print("loading vertex ai credentials")
-    os.environ["GCS_FLUSH_INTERVAL"] = "1"
-    os.environ["GCS_USE_BATCHED_LOGGING"] = "false"
-    filepath = os.path.dirname(os.path.abspath(__file__))
-    vertex_key_path = filepath + "/vertex_key.json"
-
-    # Read the existing content of the file or create an empty dictionary
-    try:
-        with open(vertex_key_path, "r") as file:
-            # Read the file content
-            print("Read vertexai file path")
-            content = file.read()
-
-            # If the file is empty or not valid JSON, create an empty dictionary
-            if not content or not content.strip():
-                service_account_key_data = {}
-            else:
-                # Attempt to load the existing JSON content
-                file.seek(0)
-                service_account_key_data = json.load(file)
-    except FileNotFoundError:
-        # If the file doesn't exist, create an empty dictionary
-        service_account_key_data = {}
-
-    # Update the service_account_key_data with environment variables
-    private_key_id = os.environ.get("VERTEX_AI_PRIVATE_KEY_ID", "")
-    private_key = os.environ.get("VERTEX_AI_PRIVATE_KEY", "")
-    private_key = private_key.replace("\\n", "\n")
-    service_account_key_data["private_key_id"] = private_key_id
-    service_account_key_data["private_key"] = private_key
-
-    # Create a temporary file
-    with tempfile.NamedTemporaryFile(mode="w+", delete=False) as temp_file:
-        # Write the updated content to the temporary files
-        json.dump(service_account_key_data, temp_file, indent=2)
-
-    # Export the temporary file as GOOGLE_APPLICATION_CREDENTIALS
-    os.environ["GCS_PATH_SERVICE_ACCOUNT"] = os.path.abspath(temp_file.name)
-    print("created gcs path service account=", os.environ["GCS_PATH_SERVICE_ACCOUNT"])
+def _make_mock_gcs_logging_config():
+    return GCSLoggingConfig(
+        bucket_name="test-bucket",
+        vertex_instance=MagicMock(),
+        path_service_account=None,
+    )
 
 
 @pytest.mark.asyncio
 async def test_aaabasic_gcs_logger():
-    load_vertex_ai_credentials()
-    gcs_logger = GCSBucketLogger()
-    print("GCSBucketLogger", gcs_logger)
+    os.environ["GCS_FLUSH_INTERVAL"] = "1"
+    os.environ["GCS_USE_BATCHED_LOGGING"] = "false"
+    os.environ["GCS_BUCKET_NAME"] = "test-bucket"
 
-    litellm.callbacks = [gcs_logger]
-    response = await litellm.acompletion(
-        model="gpt-3.5-turbo",
-        temperature=0.7,
-        messages=[{"role": "user", "content": "This is a test"}],
-        max_tokens=10,
-        user="ishaan-2",
-        mock_response="Hi!",
-        metadata={
-            "tags": ["model-anthropic-claude-v2.1", "app-ishaan-prod"],
-            "user_api_key": "a1b2c3d4e5f6789012345678901234567890abcdef1234567890abcdef123456",
-            "user_api_key_alias": None,
-            "user_api_end_user_max_budget": None,
-            "litellm_api_version": "0.0.0",
-            "global_max_parallel_requests": None,
-            "user_api_key_user_id": "116544810872468347480",
-            "user_api_key_org_id": None,
-            "user_api_key_team_id": None,
-            "user_api_key_team_alias": None,
-            "user_api_key_metadata": {},
-            "requester_ip_address": "127.0.0.1",
-            "requester_metadata": {"foo": "bar"},
-            "spend_logs_metadata": {"hello": "world"},
-            "headers": {
-                "content-type": "application/json",
-                "user-agent": "PostmanRuntime/7.32.3",
-                "accept": "*/*",
-                "postman-token": "92300061-eeaa-423b-a420-0b44896ecdc4",
-                "host": "localhost:4000",
-                "accept-encoding": "gzip, deflate, br",
-                "connection": "keep-alive",
-                "content-length": "163",
-            },
-            "endpoint": "http://localhost:4000/chat/completions",
-            "model_group": "gpt-3.5-turbo",
-            "model_info": {
-                "id": "4bad40a1eb6bebd1682800f16f44b9f06c52a6703444c99c7f9f32e9de3693b4",
-                "db_model": False,
-            },
-            "api_base": "https://openai-gpt-4-test-v-1.openai.azure.com/",
-            "caching_groups": None,
-            "raw_request": "\n\nPOST Request Sent from LiteLLM:\ncurl -X POST \\\nhttps://openai-gpt-4-test-v-1.openai.azure.com//openai/ \\\n-H 'Authorization: *****' \\\n-d '{'model': 'chatgpt-v-3', 'messages': [{'role': 'system', 'content': 'you are a helpful assistant.\\n'}, {'role': 'user', 'content': 'bom dia'}], 'stream': False, 'max_tokens': 10, 'user': '116544810872468347480', 'extra_body': {}}'\n",
-        },
-    )
+    captured_payloads = []
 
-    print("response", response)
+    async def mock_log_json_data_on_gcs(
+        self, headers, bucket_name, object_name, logging_payload
+    ):
+        captured_payloads.append(
+            {
+                "bucket_name": bucket_name,
+                "object_name": object_name,
+                "logging_payload": logging_payload,
+            }
+        )
+        return {"kind": "storage#object", "name": object_name}
 
-    await asyncio.sleep(5)
+    with patch(
+        "litellm.proxy.proxy_server.premium_user", True
+    ), patch.object(
+        GCSBucketLogger,
+        "construct_request_headers",
+        new_callable=AsyncMock,
+        return_value={"Authorization": "Bearer mock_token"},
+    ), patch.object(
+        GCSBucketLogger,
+        "get_gcs_logging_config",
+        new_callable=AsyncMock,
+        return_value=_make_mock_gcs_logging_config(),
+    ), patch.object(
+        GCSBucketLogger,
+        "_log_json_data_on_gcs",
+        mock_log_json_data_on_gcs,
+    ):
+        gcs_logger = GCSBucketLogger()
 
-    # Get the current date
-    # Get the current date
-    current_date = datetime.now().strftime("%Y-%m-%d")
-
-    # Modify the object_name to include the date-based folder
-    object_name = f"{current_date}%2F{response.id}"
-
-    print("object_name", object_name)
-
-    # Check if object landed on GCS
-    object_from_gcs = await gcs_logger.download_gcs_object(object_name=object_name)
-    print("object from gcs=", object_from_gcs)
-    # convert object_from_gcs from bytes to DICT
-    parsed_data = json.loads(object_from_gcs)
-    print("object_from_gcs as dict", parsed_data)
-
-    print("type of object_from_gcs", type(parsed_data))
-
-    gcs_payload = StandardLoggingPayload(**parsed_data)
-
-    print("gcs_payload", gcs_payload)
-
-    assert gcs_payload["model"] == "gpt-3.5-turbo"
-    assert gcs_payload["messages"] == [{"role": "user", "content": "This is a test"}]
-
-    assert gcs_payload["response"]["choices"][0]["message"]["content"] == "Hi!"
-
-    assert gcs_payload["response_cost"] > 0.0
-
-    assert gcs_payload["status"] == "success"
-
-    assert (
-        gcs_payload["metadata"]["user_api_key_hash"]
-        == "a1b2c3d4e5f6789012345678901234567890abcdef1234567890abcdef123456"
-    )
-    assert gcs_payload["metadata"]["user_api_key_user_id"] == "116544810872468347480"
-
-    assert gcs_payload["metadata"]["requester_metadata"] == {"foo": "bar"}
-
-    # Delete Object from GCS
-    print("deleting object from GCS")
-    await gcs_logger.delete_gcs_object(object_name=object_name)
-
-
-@pytest.mark.asyncio
-async def test_basic_gcs_logger_failure():
-    load_vertex_ai_credentials()
-    gcs_logger = GCSBucketLogger()
-    print("GCSBucketLogger", gcs_logger)
-
-    gcs_log_id = f"failure-test-{uuid.uuid4().hex}"
-
-    litellm.callbacks = [gcs_logger]
-
-    try:
+        litellm.callbacks = [gcs_logger]
         response = await litellm.acompletion(
             model="gpt-3.5-turbo",
             temperature=0.7,
             messages=[{"role": "user", "content": "This is a test"}],
             max_tokens=10,
             user="ishaan-2",
-            mock_response=litellm.BadRequestError(
-                model="gpt-3.5-turbo",
-                message="Error: 400: Bad Request: Invalid API key, please check your API key and try again.",
-                llm_provider="openai",
-            ),
+            mock_response="Hi!",
             metadata={
-                "gcs_log_id": gcs_log_id,
                 "tags": ["model-anthropic-claude-v2.1", "app-ishaan-prod"],
                 "user_api_key": "a1b2c3d4e5f6789012345678901234567890abcdef1234567890abcdef123456",
                 "user_api_key_alias": None,
@@ -204,6 +95,7 @@ async def test_basic_gcs_logger_failure():
                 "user_api_key_team_alias": None,
                 "user_api_key_metadata": {},
                 "requester_ip_address": "127.0.0.1",
+                "requester_metadata": {"foo": "bar"},
                 "spend_logs_metadata": {"hello": "world"},
                 "headers": {
                     "content-type": "application/json",
@@ -226,557 +118,153 @@ async def test_basic_gcs_logger_failure():
                 "raw_request": "\n\nPOST Request Sent from LiteLLM:\ncurl -X POST \\\nhttps://openai-gpt-4-test-v-1.openai.azure.com//openai/ \\\n-H 'Authorization: *****' \\\n-d '{'model': 'chatgpt-v-3', 'messages': [{'role': 'system', 'content': 'you are a helpful assistant.\\n'}, {'role': 'user', 'content': 'bom dia'}], 'stream': False, 'max_tokens': 10, 'user': '116544810872468347480', 'extra_body': {}}'\n",
             },
         )
-    except Exception:
-        pass
 
-    await asyncio.sleep(5)
+        print("response", response)
 
-    # Get the current date
-    # Get the current date
-    current_date = datetime.now().strftime("%Y-%m-%d")
+        await asyncio.sleep(3)
 
-    # Modify the object_name to include the date-based folder
-    object_name = gcs_log_id
-
-    print("object_name", object_name)
-
-    # Check if object landed on GCS
-    object_from_gcs = await gcs_logger.download_gcs_object(object_name=object_name)
-    print("object from gcs=", object_from_gcs)
-    # convert object_from_gcs from bytes to DICT
-    parsed_data = json.loads(object_from_gcs)
-    print("object_from_gcs as dict", parsed_data)
-
-    print("type of object_from_gcs", type(parsed_data))
-
-    gcs_payload = StandardLoggingPayload(**parsed_data)
-
-    print("gcs_payload", gcs_payload)
-
-    assert gcs_payload["model"] == "gpt-3.5-turbo"
-    assert gcs_payload["messages"] == [{"role": "user", "content": "This is a test"}]
-
-    assert gcs_payload["response_cost"] == 0
-    assert gcs_payload["status"] == "failure"
-
-    assert (
-        gcs_payload["metadata"]["user_api_key_hash"]
-        == "a1b2c3d4e5f6789012345678901234567890abcdef1234567890abcdef123456"
-    )
-    assert gcs_payload["metadata"]["user_api_key_user_id"] == "116544810872468347480"
-
-    # Delete Object from GCS
-    print("deleting object from GCS")
-    await gcs_logger.delete_gcs_object(object_name=object_name)
-
-
-@pytest.mark.skip(reason="This test is flaky")
-@pytest.mark.asyncio
-async def test_basic_gcs_logging_per_request_with_callback_set():
-    """
-    Test GCS Bucket logging per request
-
-    Request 1 - pass gcs_bucket_name in kwargs
-    Request 2 - don't pass gcs_bucket_name in kwargs - ensure 'litellm-testing-bucket'
-    """
-    import logging
-    from litellm._logging import verbose_logger
-
-    verbose_logger.setLevel(logging.DEBUG)
-    load_vertex_ai_credentials()
-    gcs_logger = GCSBucketLogger()
-    print("GCSBucketLogger", gcs_logger)
-    litellm.callbacks = [gcs_logger]
-
-    GCS_BUCKET_NAME = "example-bucket-1-litellm"
-    standard_callback_dynamic_params: StandardCallbackDynamicParams = (
-        StandardCallbackDynamicParams(gcs_bucket_name=GCS_BUCKET_NAME)
-    )
-
-    try:
-        response = await litellm.acompletion(
-            model="gpt-4o-mini",
-            temperature=0.7,
-            messages=[{"role": "user", "content": "This is a test"}],
-            max_tokens=10,
-            user="ishaan-2",
-            gcs_bucket_name=GCS_BUCKET_NAME,
+        assert len(captured_payloads) == 1, (
+            f"Expected 1 GCS upload, got {len(captured_payloads)}"
         )
-    except:
-        pass
 
-    await asyncio.sleep(5)
+        gcs_payload = captured_payloads[0]["logging_payload"]
 
-    # Get the current date
-    # Get the current date
-    current_date = datetime.now().strftime("%Y-%m-%d")
+        assert gcs_payload["model"] == "gpt-3.5-turbo"
+        assert gcs_payload["messages"] == [
+            {"role": "user", "content": "This is a test"}
+        ]
 
-    # Modify the object_name to include the date-based folder
-    object_name = f"{current_date}%2F{response.id}"
+        assert gcs_payload["response"]["choices"][0]["message"]["content"] == "Hi!"
 
-    print("object_name", object_name)
+        assert gcs_payload["response_cost"] > 0.0
 
-    # Check if object landed on GCS
-    object_from_gcs = await gcs_logger.download_gcs_object(
-        object_name=object_name,
-        standard_callback_dynamic_params=standard_callback_dynamic_params,
-    )
-    print("object from gcs=", object_from_gcs)
-    # convert object_from_gcs from bytes to DICT
-    parsed_data = json.loads(object_from_gcs)
-    print("object_from_gcs as dict", parsed_data)
-
-    print("type of object_from_gcs", type(parsed_data))
-
-    gcs_payload = StandardLoggingPayload(**parsed_data)
-
-    assert gcs_payload["model"] == "gpt-4o-mini"
-    assert gcs_payload["messages"] == [{"role": "user", "content": "This is a test"}]
-
-    assert gcs_payload["response_cost"] > 0.0
-
-    assert gcs_payload["status"] == "success"
-
-    # clean up the object from GCS
-    await gcs_logger.delete_gcs_object(
-        object_name=object_name,
-        standard_callback_dynamic_params=standard_callback_dynamic_params,
-    )
-
-    # Request 2 - don't pass gcs_bucket_name in kwargs - ensure 'litellm-testing-bucket'
-    try:
-        response = await litellm.acompletion(
-            model="gpt-4o-mini",
-            temperature=0.7,
-            messages=[{"role": "user", "content": "This is a test"}],
-            max_tokens=10,
-            user="ishaan-2",
-            mock_response="Hi!",
-        )
-    except:
-        pass
-
-    await asyncio.sleep(5)
-
-    # Get the current date
-    # Get the current date
-    current_date = datetime.now().strftime("%Y-%m-%d")
-    standard_callback_dynamic_params = StandardCallbackDynamicParams(
-        gcs_bucket_name="litellm-testing-bucket"
-    )
-
-    # Modify the object_name to include the date-based folder
-    object_name = f"{current_date}%2F{response.id}"
-
-    print("object_name", object_name)
-
-    # Check if object landed on GCS
-    object_from_gcs = await gcs_logger.download_gcs_object(
-        object_name=object_name,
-        standard_callback_dynamic_params=standard_callback_dynamic_params,
-    )
-    print("object from gcs=", object_from_gcs)
-    # convert object_from_gcs from bytes to DICT
-    parsed_data = json.loads(object_from_gcs)
-    print("object_from_gcs as dict", parsed_data)
-
-    print("type of object_from_gcs", type(parsed_data))
-
-    gcs_payload = StandardLoggingPayload(**parsed_data)
-
-    assert gcs_payload["model"] == "gpt-4o-mini"
-    assert gcs_payload["messages"] == [{"role": "user", "content": "This is a test"}]
-
-    assert gcs_payload["response_cost"] > 0.0
-
-    assert gcs_payload["status"] == "success"
-
-    # clean up the object from GCS
-    await gcs_logger.delete_gcs_object(
-        object_name=object_name,
-        standard_callback_dynamic_params=standard_callback_dynamic_params,
-    )
-
-
-@pytest.mark.skip(reason="This test is flaky")
-@pytest.mark.asyncio
-async def test_basic_gcs_logging_per_request_with_no_litellm_callback_set():
-    """
-    Test GCS Bucket logging per request
-
-    key difference: no litellm.callbacks set
-
-    Request 1 - pass gcs_bucket_name in kwargs
-    Request 2 - don't pass gcs_bucket_name in kwargs - ensure 'litellm-testing-bucket'
-    """
-    import logging
-    from litellm._logging import verbose_logger
-
-    verbose_logger.setLevel(logging.DEBUG)
-    load_vertex_ai_credentials()
-    gcs_logger = GCSBucketLogger()
-
-    GCS_BUCKET_NAME = "example-bucket-1-litellm"
-    standard_callback_dynamic_params: StandardCallbackDynamicParams = (
-        StandardCallbackDynamicParams(gcs_bucket_name=GCS_BUCKET_NAME)
-    )
-
-    try:
-        response = await litellm.acompletion(
-            model="gpt-4o-mini",
-            temperature=0.7,
-            messages=[{"role": "user", "content": "This is a test"}],
-            max_tokens=10,
-            user="ishaan-2",
-            gcs_bucket_name=GCS_BUCKET_NAME,
-            success_callback=["gcs_bucket"],
-            failure_callback=["gcs_bucket"],
-        )
-    except:
-        pass
-
-    await asyncio.sleep(5)
-
-    # Get the current date
-    # Get the current date
-    current_date = datetime.now().strftime("%Y-%m-%d")
-
-    # Modify the object_name to include the date-based folder
-    object_name = f"{current_date}%2F{response.id}"
-
-    print("object_name", object_name)
-
-    # Check if object landed on GCS
-    object_from_gcs = await gcs_logger.download_gcs_object(
-        object_name=object_name,
-        standard_callback_dynamic_params=standard_callback_dynamic_params,
-    )
-    print("object from gcs=", object_from_gcs)
-    # convert object_from_gcs from bytes to DICT
-    parsed_data = json.loads(object_from_gcs)
-    print("object_from_gcs as dict", parsed_data)
-
-    print("type of object_from_gcs", type(parsed_data))
-
-    gcs_payload = StandardLoggingPayload(**parsed_data)
-
-    assert gcs_payload["model"] == "gpt-4o-mini"
-    assert gcs_payload["messages"] == [{"role": "user", "content": "This is a test"}]
-
-    assert gcs_payload["response_cost"] > 0.0
-
-    assert gcs_payload["status"] == "success"
-
-    # clean up the object from GCS
-    await gcs_logger.delete_gcs_object(
-        object_name=object_name,
-        standard_callback_dynamic_params=standard_callback_dynamic_params,
-    )
-
-    # make a failure request - assert that failure callback is hit
-    gcs_log_id = f"failure-test-{uuid.uuid4().hex}"
-    try:
-        response = await litellm.acompletion(
-            model="gpt-4o-mini",
-            temperature=0.7,
-            messages=[{"role": "user", "content": "This is a test"}],
-            max_tokens=10,
-            user="ishaan-2",
-            mock_response=litellm.BadRequestError(
-                model="gpt-3.5-turbo",
-                message="Error: 400: Bad Request: Invalid API key, please check your API key and try again.",
-                llm_provider="openai",
-            ),
-            success_callback=["gcs_bucket"],
-            failure_callback=["gcs_bucket"],
-            gcs_bucket_name=GCS_BUCKET_NAME,
-            metadata={
-                "gcs_log_id": gcs_log_id,
-            },
-        )
-    except:
-        pass
-
-    await asyncio.sleep(5)
-
-    # check if the failure object is logged in GCS
-    object_from_gcs = await gcs_logger.download_gcs_object(
-        object_name=gcs_log_id,
-        standard_callback_dynamic_params=standard_callback_dynamic_params,
-    )
-    print("object from gcs=", object_from_gcs)
-    # convert object_from_gcs from bytes to DICT
-    parsed_data = json.loads(object_from_gcs)
-    print("object_from_gcs as dict", parsed_data)
-
-    gcs_payload = StandardLoggingPayload(**parsed_data)
-
-    assert gcs_payload["model"] == "gpt-4o-mini"
-    assert gcs_payload["messages"] == [{"role": "user", "content": "This is a test"}]
-
-    assert gcs_payload["response_cost"] == 0
-    assert gcs_payload["status"] == "failure"
-
-    # clean up the object from GCS
-    await gcs_logger.delete_gcs_object(
-        object_name=gcs_log_id,
-        standard_callback_dynamic_params=standard_callback_dynamic_params,
-    )
-
-
-@pytest.mark.skip(reason="This test is flaky")
-@pytest.mark.asyncio
-async def test_aaaget_gcs_logging_config_without_service_account():
-    """
-    Test the get_gcs_logging_config works for IAM auth on GCS
-    1. Key based logging without a service account
-    2. Default Callback without a service account
-    """
-    load_vertex_ai_credentials()
-    _old_gcs_bucket_name = os.environ.get("GCS_BUCKET_NAME")
-    os.environ.pop("GCS_BUCKET_NAME", None)
-
-    _old_gcs_service_acct = os.environ.get("GCS_PATH_SERVICE_ACCOUNT")
-    os.environ.pop("GCS_PATH_SERVICE_ACCOUNT", None)
-
-    # Mock the load_auth function to avoid credential loading issues
-    # Test 1: With standard_callback_dynamic_params (with service account)
-    gcs_logger = GCSBucketLogger()
-
-    dynamic_params = StandardCallbackDynamicParams(
-        gcs_bucket_name="dynamic-bucket",
-    )
-    config = await gcs_logger.get_gcs_logging_config(
-        {"standard_callback_dynamic_params": dynamic_params}
-    )
-
-    assert config["bucket_name"] == "dynamic-bucket"
-    assert config["path_service_account"] is None
-    assert config["vertex_instance"] is not None
-
-    # Test 2: With standard_callback_dynamic_params (without service account - this is IAM auth)
-    dynamic_params = StandardCallbackDynamicParams(
-        gcs_bucket_name="dynamic-bucket", gcs_path_service_account=None
-    )
-
-    config = await gcs_logger.get_gcs_logging_config(
-        {"standard_callback_dynamic_params": dynamic_params}
-    )
-
-    assert config["bucket_name"] == "dynamic-bucket"
-    assert config["path_service_account"] is None
-    assert config["vertex_instance"] is not None
-
-    # Test 5: With missing bucket name
-    with pytest.raises(ValueError, match="GCS_BUCKET_NAME is not set"):
-        gcs_logger = GCSBucketLogger(bucket_name=None)
-        await gcs_logger.get_gcs_logging_config({})
-
-    if _old_gcs_bucket_name is not None:
-        os.environ["GCS_BUCKET_NAME"] = _old_gcs_bucket_name
-
-    if _old_gcs_service_acct is not None:
-        os.environ["GCS_PATH_SERVICE_ACCOUNT"] = _old_gcs_service_acct
-
-
-@pytest.mark.skip(reason="This test is flaky")
-@pytest.mark.asyncio
-async def test_basic_gcs_logger_with_folder_in_bucket_name():
-    load_vertex_ai_credentials()
-    gcs_logger = GCSBucketLogger()
-
-    bucket_name = "litellm-testing-bucket/test-folder-logs"
-
-    old_bucket_name = os.environ.get("GCS_BUCKET_NAME")
-    os.environ["GCS_BUCKET_NAME"] = bucket_name
-    print("GCSBucketLogger", gcs_logger)
-
-    litellm.callbacks = [gcs_logger]
-    response = await litellm.acompletion(
-        model="gpt-3.5-turbo",
-        temperature=0.7,
-        messages=[{"role": "user", "content": "This is a test"}],
-        max_tokens=10,
-        user="ishaan-2",
-        mock_response="Hi!",
-        metadata={
-            "tags": ["model-anthropic-claude-v2.1", "app-ishaan-prod"],
-            "user_api_key": "a1b2c3d4e5f6789012345678901234567890abcdef1234567890abcdef123456",
-            "user_api_key_alias": None,
-            "user_api_end_user_max_budget": None,
-            "litellm_api_version": "0.0.0",
-            "global_max_parallel_requests": None,
-            "user_api_key_user_id": "116544810872468347480",
-            "user_api_key_org_id": None,
-            "user_api_key_team_id": None,
-            "user_api_key_team_alias": None,
-            "user_api_key_metadata": {},
-            "requester_ip_address": "127.0.0.1",
-            "requester_metadata": {"foo": "bar"},
-            "spend_logs_metadata": {"hello": "world"},
-            "headers": {
-                "content-type": "application/json",
-                "user-agent": "PostmanRuntime/7.32.3",
-                "accept": "*/*",
-                "postman-token": "92300061-eeaa-423b-a420-0b44896ecdc4",
-                "host": "localhost:4000",
-                "accept-encoding": "gzip, deflate, br",
-                "connection": "keep-alive",
-                "content-length": "163",
-            },
-            "endpoint": "http://localhost:4000/chat/completions",
-            "model_group": "gpt-3.5-turbo",
-            "model_info": {
-                "id": "4bad40a1eb6bebd1682800f16f44b9f06c52a6703444c99c7f9f32e9de3693b4",
-                "db_model": False,
-            },
-            "api_base": "https://openai-gpt-4-test-v-1.openai.azure.com/",
-            "caching_groups": None,
-            "raw_request": "\n\nPOST Request Sent from LiteLLM:\ncurl -X POST \\\nhttps://openai-gpt-4-test-v-1.openai.azure.com//openai/ \\\n-H 'Authorization: *****' \\\n-d '{'model': 'chatgpt-v-3', 'messages': [{'role': 'system', 'content': 'you are a helpful assistant.\\n'}, {'role': 'user', 'content': 'bom dia'}], 'stream': False, 'max_tokens': 10, 'user': '116544810872468347480', 'extra_body': {}}'\n",
-        },
-    )
-
-    print("response", response)
-
-    await asyncio.sleep(5)
-
-    # Get the current date
-    # Get the current date
-    current_date = datetime.now().strftime("%Y-%m-%d")
-
-    # Modify the object_name to include the date-based folder
-    object_name = f"{current_date}%2F{response.id}"
-
-    print("object_name", object_name)
-
-    # Check if object landed on GCS
-    object_from_gcs = await gcs_logger.download_gcs_object(object_name=object_name)
-    print("object from gcs=", object_from_gcs)
-    # convert object_from_gcs from bytes to DICT
-    parsed_data = json.loads(object_from_gcs)
-    print("object_from_gcs as dict", parsed_data)
-
-    print("type of object_from_gcs", type(parsed_data))
-
-    gcs_payload = StandardLoggingPayload(**parsed_data)
-
-    print("gcs_payload", gcs_payload)
-
-    assert gcs_payload["model"] == "gpt-3.5-turbo"
-    assert gcs_payload["messages"] == [{"role": "user", "content": "This is a test"}]
-
-    assert gcs_payload["response"]["choices"][0]["message"]["content"] == "Hi!"
-
-    assert gcs_payload["response_cost"] > 0.0
-
-    assert gcs_payload["status"] == "success"
-
-    assert (
-        gcs_payload["metadata"]["user_api_key_hash"]
-        == "a1b2c3d4e5f6789012345678901234567890abcdef1234567890abcdef123456"
-    )
-    assert gcs_payload["metadata"]["user_api_key_user_id"] == "116544810872468347480"
-
-    assert gcs_payload["metadata"]["requester_metadata"] == {"foo": "bar"}
-
-    # Delete Object from GCS
-    print("deleting object from GCS")
-    await gcs_logger.delete_gcs_object(object_name=object_name)
-
-    # clean up
-    if old_bucket_name is not None:
-        os.environ["GCS_BUCKET_NAME"] = old_bucket_name
-
-
-@pytest.mark.skip(reason="This test is flaky on ci/cd")
-def test_create_file_e2e():
-    """
-    Asserts 'create_file' is called with the correct arguments
-    """
-    load_vertex_ai_credentials()
-    test_file_content = b"test audio content"
-    test_file = ("test.wav", test_file_content, "audio/wav")
-
-    from litellm import create_file
-
-    response = create_file(
-        file=test_file,
-        purpose="user_data",
-        custom_llm_provider="vertex_ai",
-    )
-    print("response", response)
-    assert response is not None
-
-
-@pytest.mark.skip(reason="This test is flaky on ci/cd")
-def test_create_file_e2e_jsonl():
-    """
-    Asserts 'create_file' is called with the correct arguments
-    """
-    load_vertex_ai_credentials()
-    from litellm.llms.custom_httpx.http_handler import HTTPHandler
-
-    client = HTTPHandler()
-
-    example_jsonl = [
-        {
-            "custom_id": "request-1",
-            "method": "POST",
-            "url": "/v1/chat/completions",
-            "body": {
-                "model": "gemini-1.5-flash-001",
-                "messages": [
-                    {"role": "system", "content": "You are a helpful assistant."},
-                    {"role": "user", "content": "Hello world!"},
-                ],
-                "max_tokens": 10,
-            },
-        },
-        {
-            "custom_id": "request-2",
-            "method": "POST",
-            "url": "/v1/chat/completions",
-            "body": {
-                "model": "gemini-1.5-flash-001",
-                "messages": [
-                    {"role": "system", "content": "You are an unhelpful assistant."},
-                    {"role": "user", "content": "Hello world!"},
-                ],
-                "max_tokens": 10,
-            },
-        },
-    ]
-
-    # Create and write to the file
-    file_path = "example.jsonl"
-    with open(file_path, "w") as f:
-        for item in example_jsonl:
-            f.write(json.dumps(item) + "\n")
-
-    # Verify file content
-    with open(file_path, "r") as f:
-        content = f.read()
-        print("File content:", content)
-        assert len(content) > 0, "File is empty"
-
-    from litellm import create_file
-
-    with patch.object(client, "post") as mock_create_file:
-        try:
-            response = create_file(
-                file=open(file_path, "rb"),
-                purpose="user_data",
-                custom_llm_provider="vertex_ai",
-                client=client,
-            )
-        except Exception as e:
-            print("error", e)
-
-        mock_create_file.assert_called_once()
-
-        print(f"kwargs: {mock_create_file.call_args.kwargs}")
+        assert gcs_payload["status"] == "success"
 
         assert (
-            mock_create_file.call_args.kwargs["data"] is not None
-            and len(mock_create_file.call_args.kwargs["data"]) > 0
+            gcs_payload["metadata"]["user_api_key_hash"]
+            == "a1b2c3d4e5f6789012345678901234567890abcdef1234567890abcdef123456"
+        )
+        assert (
+            gcs_payload["metadata"]["user_api_key_user_id"] == "116544810872468347480"
+        )
+
+        assert gcs_payload["metadata"]["requester_metadata"] == {"foo": "bar"}
+
+
+@pytest.mark.asyncio
+async def test_basic_gcs_logger_failure():
+    os.environ["GCS_FLUSH_INTERVAL"] = "1"
+    os.environ["GCS_USE_BATCHED_LOGGING"] = "false"
+    os.environ["GCS_BUCKET_NAME"] = "test-bucket"
+
+    captured_payloads = []
+
+    async def mock_log_json_data_on_gcs(
+        self, headers, bucket_name, object_name, logging_payload
+    ):
+        captured_payloads.append(
+            {
+                "bucket_name": bucket_name,
+                "object_name": object_name,
+                "logging_payload": logging_payload,
+            }
+        )
+        return {"kind": "storage#object", "name": object_name}
+
+    gcs_log_id = f"failure-test-{uuid.uuid4().hex}"
+
+    with patch(
+        "litellm.proxy.proxy_server.premium_user", True
+    ), patch.object(
+        GCSBucketLogger,
+        "construct_request_headers",
+        new_callable=AsyncMock,
+        return_value={"Authorization": "Bearer mock_token"},
+    ), patch.object(
+        GCSBucketLogger,
+        "get_gcs_logging_config",
+        new_callable=AsyncMock,
+        return_value=_make_mock_gcs_logging_config(),
+    ), patch.object(
+        GCSBucketLogger,
+        "_log_json_data_on_gcs",
+        mock_log_json_data_on_gcs,
+    ):
+        gcs_logger = GCSBucketLogger()
+
+        litellm.callbacks = [gcs_logger]
+
+        try:
+            response = await litellm.acompletion(
+                model="gpt-3.5-turbo",
+                temperature=0.7,
+                messages=[{"role": "user", "content": "This is a test"}],
+                max_tokens=10,
+                user="ishaan-2",
+                mock_response=litellm.BadRequestError(
+                    model="gpt-3.5-turbo",
+                    message="Error: 400: Bad Request: Invalid API key, please check your API key and try again.",
+                    llm_provider="openai",
+                ),
+                metadata={
+                    "gcs_log_id": gcs_log_id,
+                    "tags": ["model-anthropic-claude-v2.1", "app-ishaan-prod"],
+                    "user_api_key": "a1b2c3d4e5f6789012345678901234567890abcdef1234567890abcdef123456",
+                    "user_api_key_alias": None,
+                    "user_api_end_user_max_budget": None,
+                    "litellm_api_version": "0.0.0",
+                    "global_max_parallel_requests": None,
+                    "user_api_key_user_id": "116544810872468347480",
+                    "user_api_key_org_id": None,
+                    "user_api_key_team_id": None,
+                    "user_api_key_team_alias": None,
+                    "user_api_key_metadata": {},
+                    "requester_ip_address": "127.0.0.1",
+                    "spend_logs_metadata": {"hello": "world"},
+                    "headers": {
+                        "content-type": "application/json",
+                        "user-agent": "PostmanRuntime/7.32.3",
+                        "accept": "*/*",
+                        "postman-token": "92300061-eeaa-423b-a420-0b44896ecdc4",
+                        "host": "localhost:4000",
+                        "accept-encoding": "gzip, deflate, br",
+                        "connection": "keep-alive",
+                        "content-length": "163",
+                    },
+                    "endpoint": "http://localhost:4000/chat/completions",
+                    "model_group": "gpt-3.5-turbo",
+                    "model_info": {
+                        "id": "4bad40a1eb6bebd1682800f16f44b9f06c52a6703444c99c7f9f32e9de3693b4",
+                        "db_model": False,
+                    },
+                    "api_base": "https://openai-gpt-4-test-v-1.openai.azure.com/",
+                    "caching_groups": None,
+                    "raw_request": "\n\nPOST Request Sent from LiteLLM:\ncurl -X POST \\\nhttps://openai-gpt-4-test-v-1.openai.azure.com//openai/ \\\n-H 'Authorization: *****' \\\n-d '{'model': 'chatgpt-v-3', 'messages': [{'role': 'system', 'content': 'you are a helpful assistant.\\n'}, {'role': 'user', 'content': 'bom dia'}], 'stream': False, 'max_tokens': 10, 'user': '116544810872468347480', 'extra_body': {}}'\n",
+                },
+            )
+        except Exception:
+            pass
+
+        await asyncio.sleep(3)
+
+        assert len(captured_payloads) == 1, (
+            f"Expected 1 GCS upload, got {len(captured_payloads)}"
+        )
+
+        gcs_payload = captured_payloads[0]["logging_payload"]
+
+        assert gcs_payload["model"] == "gpt-3.5-turbo"
+        assert gcs_payload["messages"] == [
+            {"role": "user", "content": "This is a test"}
+        ]
+
+        assert gcs_payload["response_cost"] == 0
+        assert gcs_payload["status"] == "failure"
+
+        assert (
+            gcs_payload["metadata"]["user_api_key_hash"]
+            == "a1b2c3d4e5f6789012345678901234567890abcdef1234567890abcdef123456"
+        )
+        assert (
+            gcs_payload["metadata"]["user_api_key_user_id"] == "116544810872468347480"
         )

tests/local_testing/test_router_client_init.py

@@ -71,9 +71,7 @@ def test_router_init_with_neither_api_key_nor_azure_service_principal_with_secre
 
 @patch("azure.identity.get_bearer_token_provider")
 @patch("azure.identity.ClientSecretCredential")
-@patch("litellm.secret_managers.get_azure_ad_token_provider.os")
 def test_router_init_azure_service_principal_with_secret_with_environment_variables(
-    mocked_os_lib: MagicMock,
     mocked_credential: MagicMock,
     mocked_get_bearer_token_provider: MagicMock,
     monkeypatch,
@@ -86,21 +84,18 @@ def test_router_init_azure_service_principal_with_secret_with_environment_variab
     and environment variables.
     """
     monkeypatch.delenv("AZURE_AI_API_KEY", raising=False)
+    monkeypatch.delenv("AZURE_OPENAI_API_KEY", raising=False)
+    monkeypatch.delenv("AZURE_API_KEY", raising=False)
     litellm.enable_azure_ad_token_refresh = True
     # mock the token provider function
     mocked_func_generating_token = MagicMock(return_value="test_token")
     mocked_get_bearer_token_provider.return_value = mocked_func_generating_token
 
-    # mock the environment variables with mocked credentials
-    environment_variables_expected_to_use = {
-        "AZURE_CLIENT_ID": "test_client_id",
-        "AZURE_CLIENT_SECRET": "test_client_secret",
-        "AZURE_TENANT_ID": "test_tenant_id",
-    }
-    mocked_environ = PropertyMock(return_value=environment_variables_expected_to_use)
-    # Because of the way mock attributes are stored you can’t directly attach a PropertyMock to a mock object.
-    # https://docs.python.org/3.11/library/unittest.mock.html#unittest.mock.PropertyMock
-    type(mocked_os_lib).environ = mocked_environ
+    # set environment variables with mocked credentials using monkeypatch
+    # so both common_utils._resolve_env_var and get_azure_ad_token_provider see them
+    monkeypatch.setenv("AZURE_CLIENT_ID", "test_client_id")
+    monkeypatch.setenv("AZURE_CLIENT_SECRET", "test_client_secret")
+    monkeypatch.setenv("AZURE_TENANT_ID", "test_tenant_id")
 
     # define the model list
     model_list = [

tests/local_testing/test_simple_shuffle.py

@@ -1,53 +0,0 @@
-# What is this?
-## unit tests for 'simple-shuffle'
-
-import sys, os, asyncio, time, random
-from datetime import datetime
-import traceback
-from dotenv import load_dotenv
-
-load_dotenv()
-import os
-
-sys.path.insert(
-    0, os.path.abspath("../..")
-)  # Adds the parent directory to the system path
-import pytest
-from litellm import Router
-
-"""
-Test random shuffle 
-- async 
-- sync 
-"""
-
-
-async def test_simple_shuffle():
-    model_list = [
-        {
-            "model_name": "azure-model",
-            "litellm_params": {
-                "model": "azure/gpt-turbo",
-                "api_key": "os.environ/AZURE_FRANCE_API_KEY",
-                "api_base": "https://openai-france-1234.openai.azure.com",
-                "rpm": 1440,
-            },
-            "model_info": {"id": 1},
-        },
-        {
-            "model_name": "azure-model",
-            "litellm_params": {
-                "model": "azure/gpt-35-turbo",
-                "api_key": "os.environ/AZURE_EUROPE_API_KEY",
-                "api_base": "https://my-endpoint-europe-berri-992.openai.azure.com",
-                "rpm": 6,
-            },
-            "model_info": {"id": 2},
-        },
-    ]
-    router = Router(
-        model_list=model_list,
-        routing_strategy="usage-based-routing-v2",
-        set_verbose=False,
-        num_retries=3,
-    )  # type: ignore