diff --git a/.circleci/config.yml b/.circleci/config.yml index 16019fe047b..d4943b59c9a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1559,6 +1559,25 @@ jobs: pip install "pytest-asyncio==0.21.1" pip install aiohttp pip install apscheduler + - run: + name: Install dockerize + command: | + sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + sudo rm dockerize-linux-amd64-v0.6.1.tar.gz + - run: + name: Start PostgreSQL Database + command: | + docker run -d \ + --name postgres-db \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=litellm_test \ + -p 5432:5432 \ + postgres:14 + - run: + name: Wait for PostgreSQL to be ready + command: dockerize -wait tcp://localhost:5432 -timeout 1m - attach_workspace: at: ~/project - run: @@ -1567,29 +1586,41 @@ jobs: zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - run: - name: Run Docker container + name: Seed database with real schema + command: | + docker run -d \ + -p 4001:4000 \ + -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \ + -e LITELLM_MASTER_KEY="sk-1234" \ + --name schema-seed \ + --add-host=host.docker.internal:host-gateway \ + -v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \ + litellm-docker-database:ci \ + --config /app/config.yaml \ + --port 4000 \ + --use_prisma_db_push + - run: + name: Wait for schema seed to complete + command: dockerize -wait http://localhost:4001 -timeout 5m + - run: + name: Stop schema seed container + command: docker stop schema-seed && docker rm schema-seed + - run: + name: Run Docker container with bad schema and disabled updates command: | docker run -d \ -p 4000:4000 \ - -e DATABASE_URL=$PROXY_DATABASE_URL \ + -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \ -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ -e DISABLE_SCHEMA_UPDATE="True" \ + --name my-app \ + --add-host=host.docker.internal:host-gateway \ -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/schema.prisma \ -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/litellm/proxy/schema.prisma \ -v $(pwd)/litellm/proxy/example_config_yaml/disable_schema_update.yaml:/app/config.yaml \ - --name my-app \ litellm-docker-database:ci \ --config /app/config.yaml \ --port 4000 - - run: - name: Install curl and dockerize - command: | - sudo apt-get update - sudo apt-get install -y curl - sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - sudo rm dockerize-linux-amd64-v0.6.1.tar.gz - - run: name: Wait for container to be ready command: dockerize -wait http://localhost:4000 -timeout 1m @@ -1997,9 +2028,6 @@ jobs: -e OPENAI_API_KEY=$OPENAI_API_KEY \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ -e OTEL_EXPORTER="in_memory" \ - -e APORIA_API_BASE_2=$APORIA_API_BASE_2 \ - -e APORIA_API_KEY_2=$APORIA_API_KEY_2 \ - -e APORIA_API_BASE_1=$APORIA_API_BASE_1 \ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ @@ -2007,7 +2035,6 @@ jobs: -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ -e AWS_REGION_NAME=$AWS_REGION_NAME \ - -e APORIA_API_KEY_1=$APORIA_API_KEY_1 \ -e COHERE_API_KEY=$COHERE_API_KEY \ -e GCS_FLUSH_INTERVAL="1" \ --add-host host.docker.internal:host-gateway \ @@ -2483,6 +2510,20 @@ jobs: name: Build Docker image command: | docker build -t my-app:latest -f docker/build_from_pip/Dockerfile.build_from_pip . + - run: + name: Start PostgreSQL Database + command: | + docker run -d \ + --name postgres-db \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=circle_test \ + -p 5432:5432 \ + postgres:14 + - run: + name: Wait for PostgreSQL to be ready + command: | + timeout 60s bash -c 'until docker exec postgres-db pg_isready -U postgres -d circle_test; do sleep 2; done' - run: name: Run Docker container # intentionally give bad redis credentials here @@ -2490,7 +2531,7 @@ jobs: command: | docker run -d \ -p 4000:4000 \ - -e DATABASE_URL=$PROXY_DATABASE_URL \ + -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e REDIS_HOST=$REDIS_HOST \ -e REDIS_PASSWORD=$REDIS_PASSWORD \ -e REDIS_PORT=$REDIS_PORT \ @@ -2498,18 +2539,15 @@ jobs: -e OPENAI_API_KEY=$OPENAI_API_KEY \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ -e OTEL_EXPORTER="in_memory" \ - -e APORIA_API_BASE_2=$APORIA_API_BASE_2 \ - -e APORIA_API_KEY_2=$APORIA_API_KEY_2 \ - -e APORIA_API_BASE_1=$APORIA_API_BASE_1 \ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -e AWS_REGION_NAME=$AWS_REGION_NAME \ - -e APORIA_API_KEY_1=$APORIA_API_KEY_1 \ -e COHERE_API_KEY=$COHERE_API_KEY \ -e USE_DDTRACE=True \ -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ -e GCS_FLUSH_INTERVAL="1" \ + --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/docker/build_from_pip/litellm_config.yaml:/app/config.yaml \ my-app:latest \ @@ -2540,8 +2578,11 @@ jobs: - run: name: Stop and remove first container command: | - docker stop my-app - docker rm my-app + docker stop my-app || true + docker rm my-app || true + docker stop postgres-db || true + docker rm postgres-db || true + when: always proxy_pass_through_endpoint_tests: machine: image: ubuntu-2204:2023.10.1 @@ -3277,37 +3318,39 @@ jobs: - setup_google_dns - attach_workspace: at: ~/project + - run: + name: Install dockerize + command: | + sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + sudo rm dockerize-linux-amd64-v0.6.1.tar.gz + - run: + name: Start PostgreSQL Database + command: | + docker run -d \ + --name postgres-db \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=litellm_schema_sync \ + -p 5432:5432 \ + postgres:14 + - run: + name: Wait for PostgreSQL to be ready + command: dockerize -wait tcp://localhost:5432 -timeout 1m - run: name: Load Docker Database Image command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database - run: - name: Install Neon CLI + name: Run schema sync via prisma db push command: | - npm i -g neonctl - - run: - name: Install curl and dockerize - command: | - sudo apt-get update - sudo apt-get install -y curl - sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - sudo rm dockerize-linux-amd64-v0.6.1.tar.gz - - run: - name: Sync schema on base e2e database - command: | - BASE_DATABASE_URL=$(neon connection-string \ - --project-id $NEON_PROJECT_ID \ - --api-key $NEON_API_KEY \ - --branch br-fancy-paper-ad1olsb3 \ - --database-name yuneng-trial-db \ - --role neondb_owner) docker run -d \ -p 4000:4000 \ - -e DATABASE_URL=$BASE_DATABASE_URL \ + -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_schema_sync" \ -e LITELLM_MASTER_KEY="sk-1234" \ --name schema-sync \ + --add-host=host.docker.internal:host-gateway \ -v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \ litellm-docker-database:ci \ --config /app/config.yaml \ @@ -3485,7 +3528,6 @@ workflows: - main - /litellm_.*/ - prisma_schema_sync: - context: e2e_ui_tests requires: - build_docker_database_image filters: @@ -3493,32 +3535,32 @@ workflows: only: - main - /litellm_.*/ - - e2e_ui_testing: - name: e2e_ui_testing_chromium - browser: chromium - context: e2e_ui_tests - requires: - - ui_build - - build_docker_database_image - - prisma_schema_sync - filters: - branches: - only: - - main - - /litellm_.*/ - - e2e_ui_testing: - name: e2e_ui_testing_firefox - browser: firefox - context: e2e_ui_tests - requires: - - ui_build - - build_docker_database_image - - prisma_schema_sync - filters: - branches: - only: - - main - - /litellm_.*/ + # - e2e_ui_testing: # migrate to dynamic db - currently requires neon cli + # name: e2e_ui_testing_chromium + # browser: chromium + # context: e2e_ui_tests + # requires: + # - ui_build + # - build_docker_database_image + # - prisma_schema_sync + # filters: + # branches: + # only: + # - main + # - /litellm_.*/ + # - e2e_ui_testing: + # name: e2e_ui_testing_firefox + # browser: firefox + # context: e2e_ui_tests + # requires: + # - ui_build + # - build_docker_database_image + # - prisma_schema_sync + # filters: + # branches: + # only: + # - main + # - /litellm_.*/ - build_and_test: requires: - build_docker_database_image diff --git a/.devcontainer/post-create.sh b/.devcontainer/post-create.sh index bd72e91a20f..484baa9041d 100644 --- a/.devcontainer/post-create.sh +++ b/.devcontainer/post-create.sh @@ -12,6 +12,6 @@ echo "[post-create] Generating Prisma client" poetry run prisma generate echo "[post-create] Installing npm dependencies" -cd ui/litellm-dashboard && npm install --no-audit --no-fund +cd ui/litellm-dashboard && npm ci echo "[post-create] Done" \ No newline at end of file diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml new file mode 100644 index 00000000000..2ae01823a96 --- /dev/null +++ b/.github/workflows/create-release.yml @@ -0,0 +1,93 @@ +name: Create Release + +on: + workflow_dispatch: + inputs: + tag: + description: "Release tag (e.g. v1.83.0-stable)" + required: true + type: string + commit_hash: + description: "Full 40-char commit SHA to target" + required: true + type: string + +permissions: {} + +jobs: + release: + name: Create Release + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Validate inputs + env: + TAG: ${{ inputs.tag }} + COMMIT_HASH: ${{ inputs.commit_hash }} + run: | + if ! echo "${COMMIT_HASH}" | grep -qE '^[0-9a-f]{40}$'; then + echo "::error::commit_hash must be a full 40-character commit SHA" + exit 1 + fi + if ! echo "${TAG}" | grep -qE '^v[0-9]+\.[0-9]+\.[0-9]+'; then + echo "::error::tag must start with vX.Y.Z" + exit 1 + fi + + - name: Create release + env: + TAG: ${{ inputs.tag }} + COMMIT_HASH: ${{ inputs.commit_hash }} + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + const tag = process.env.TAG; + const commitHash = process.env.COMMIT_HASH; + + const cosignSection = [ + `## Verify Docker Image Signature`, + ``, + `All LiteLLM Docker images are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). To verify the integrity of an image before deploying:`, + ``, + '```bash', + `cosign verify \\`, + ` --key https://raw.githubusercontent.com/BerriAI/litellm/${tag}/cosign.pub \\`, + ` ghcr.io/berriai/litellm:${tag}`, + '```', + ``, + `Expected output:`, + ``, + '```', + `The following checks were performed on each of these signatures:`, + ` - The cosign claims were validated`, + ` - The signatures were verified against the specified public key`, + '```', + ``, + `---`, + ``, + ].join('\n'); + + try { + const response = await github.rest.repos.createRelease({ + draft: true, + generate_release_notes: true, + target_commitish: commitHash, + name: tag, + owner: context.repo.owner, + prerelease: false, + repo: context.repo.repo, + tag_name: tag, + }); + + const updatedBody = cosignSection + (response.data.body ?? ''); + await github.rest.repos.updateRelease({ + owner: context.repo.owner, + repo: context.repo.repo, + release_id: response.data.id, + body: updatedBody, + draft: false, + }); + } catch (error) { + core.setFailed(error.message); + } diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml new file mode 100644 index 00000000000..7cd12bb219c --- /dev/null +++ b/.github/workflows/scorecard.yml @@ -0,0 +1,47 @@ +name: Scorecard supply-chain security + +on: + branch_protection_rule: + schedule: + - cron: '27 12 * * 4' + push: + branches: ["main"] + +permissions: read-all + +jobs: + analysis: + name: Scorecard analysis + runs-on: ubuntu-latest + if: github.event.repository.default_branch == github.ref_name + permissions: + security-events: write + id-token: write + # Uncomment for private repos if needed: + # contents: read + # actions: read + + steps: + - name: Checkout code + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + + - name: Run analysis + uses: ossf/scorecard-action@f49aabe0b5af0936a0987cfb85d86b75731b0186 # v2.4.1 + with: + results_file: results.sarif + results_format: sarif + publish_results: true + + - name: Upload artifact + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + with: + name: SARIF file + path: results.sarif + retention-days: 5 + + - name: Upload to code scanning + uses: github/codeql-action/upload-sarif@c10b806170c8ee63ea24152429041b5624f0baf5 # v4.35.1 + with: + sarif_file: results.sarif diff --git a/.github/workflows/test-litellm-ui-build.yml b/.github/workflows/test-litellm-ui-build.yml index 6b0b3a413a6..bef568298e0 100644 --- a/.github/workflows/test-litellm-ui-build.yml +++ b/.github/workflows/test-litellm-ui-build.yml @@ -28,7 +28,7 @@ jobs: cache-dependency-path: ui/litellm-dashboard/package-lock.json - name: Install dependencies - run: npm install + run: npm ci - name: Build run: npm run build diff --git a/.npmrc b/.npmrc new file mode 100644 index 00000000000..168e81a1c4e --- /dev/null +++ b/.npmrc @@ -0,0 +1,5 @@ +# Supply-chain hardening +# Packages needing lifecycle scripts: npm rebuild +ignore-scripts=true +# Protects local npm install only — npm ci (used in CI) ignores this +min-release-age=3d diff --git a/ci_cd/security_scans.sh b/ci_cd/security_scans.sh index 061e454465a..ee33393c8b1 100755 --- a/ci_cd/security_scans.sh +++ b/ci_cd/security_scans.sh @@ -160,7 +160,7 @@ run_grype_scans() { "CVE-2026-0775" # npm cli incorrect permission assignment - no fix available yet, npm is only used at build/prisma-generate time "GHSA-3ppc-4f35-3m26" # minimatch ReDoS via repeated wildcards - from nodejs_wheel bundled npm, not used in application runtime code "GHSA-83g3-92jg-28cx" # tar arbitrary file read/write via hardlink - from nodejs_wheel bundled npm, not used in application runtime code - "CVE-2026-25639" # axios - full fix requires 1.x major version bump; pinned to >=0.30.2 to clear other axios CVEs, upgrade to 1.x in follow-up + "CVE-2026-25639" # axios DoS via __proto__ in mergeConfig - transitive dev dep via @neondatabase/api-client, not imported in application code "CVE-2026-2297" # Python 3.13 SourcelessFileLoader audit hook bypass - no fix available in base image "GHSA-qffp-2rhf-9h96" # tar hardlink path traversal - from nodejs_wheel bundled npm, not used in application runtime code "CVE-2026-2673" # OpenSSL 3.6.1 TLS 1.3 key exchange group negotiation issue - no fix available yet diff --git a/cosign.pub b/cosign.pub new file mode 100644 index 00000000000..2c2a555ab09 --- /dev/null +++ b/cosign.pub @@ -0,0 +1,4 @@ +-----BEGIN PUBLIC KEY----- +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEKi4ivqGpE231OGH50PKbqy1Y1Kkb +POJC8+i2Wko82gBOUCe3M0Vw86H/4rhUhfoYEti4gdJ9wZbYmK0I2EE96g== +-----END PUBLIC KEY----- diff --git a/docker/Dockerfile.custom_ui b/docker/Dockerfile.custom_ui index c1bd9a383fa..9ab416944f5 100644 --- a/docker/Dockerfile.custom_ui +++ b/docker/Dockerfile.custom_ui @@ -51,7 +51,7 @@ ENV UI_BASE_PATH="/prod/ui" # Build the UI with the specified UI_BASE_PATH WORKDIR /app/ui/litellm-dashboard -RUN npm install +RUN npm ci RUN UI_BASE_PATH=$UI_BASE_PATH npm run build # Create the destination directory diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index db3981fb7e7..c224894779c 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -47,7 +47,7 @@ RUN mkdir -p /var/lib/litellm/ui && \ if [ -f "/app/enterprise/enterprise_ui/enterprise_colors.json" ]; then \ cp /app/enterprise/enterprise_ui/enterprise_colors.json ./ui_colors.json; \ fi && \ - npm install --legacy-peer-deps && \ + npm ci && \ npm run build && \ cp -r /app/ui/litellm-dashboard/out/* /var/lib/litellm/ui/ && \ mkdir -p /var/lib/litellm/assets && \ diff --git a/docker/build_admin_ui.sh b/docker/build_admin_ui.sh index 5373ad0e3d9..efb2bac3535 100755 --- a/docker/build_admin_ui.sh +++ b/docker/build_admin_ui.sh @@ -40,11 +40,22 @@ else exit 1 fi fi -curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh | bash +NVM_VERSION="v0.40.4" +NVM_CHECKSUM="4b7412c49960c7d31e8df72da90c1fb5b8cccb419ac99537b737028d497aba4f" +NVM_SCRIPT=$(mktemp) +trap 'rm -f "$NVM_SCRIPT"' EXIT +curl -fsSL "https://raw.githubusercontent.com/nvm-sh/nvm/${NVM_VERSION}/install.sh" -o "$NVM_SCRIPT" +if command -v sha256sum &>/dev/null; then + echo "${NVM_CHECKSUM} ${NVM_SCRIPT}" | sha256sum -c - +elif command -v shasum &>/dev/null; then + echo "${NVM_CHECKSUM} ${NVM_SCRIPT}" | shasum -a 256 -c - +else + echo "No sha256 tool found; cannot verify nvm checksum"; exit 1 +fi || { echo "nvm checksum verification failed"; exit 1; } +bash "$NVM_SCRIPT" source ~/.nvm/nvm.sh nvm install v18.17.0 nvm use v18.17.0 -npm install -g npm # copy _enterprise.json from this directory to /ui/litellm-dashboard, and rename it to ui_colors.json cp enterprise/enterprise_ui/enterprise_colors.json ui/litellm-dashboard/ui_colors.json diff --git a/docs/my-website/blog/ci_cd_v2_improvements/index.md b/docs/my-website/blog/ci_cd_v2_improvements/index.md new file mode 100644 index 00000000000..84f8f7bda6b --- /dev/null +++ b/docs/my-website/blog/ci_cd_v2_improvements/index.md @@ -0,0 +1,55 @@ +--- +slug: ci-cd-v2-improvements +title: "Announcing CI/CD v2 for LiteLLM" +date: 2026-03-30T21:30:00 +authors: + - krrish +description: "CI/CD v2 introduces isolated environments, stronger security gates, and safer release separation for LiteLLM." +tags: [engineering, ci-cd, security] +hide_table_of_contents: false +--- + +import Image from '@theme/IdealImage'; + +The CI/CD v2 is now live for LiteLLM. + + + +
+Building on the roadmap from our [security incident](https://docs.litellm.ai/blog/security-townhall-updates#roadmap), CI/CD v2 introduces isolated environments, stronger security gates, and safer release separation for LiteLLM. + +## What changed + +- Security scans and unit tests run in isolated environments. +- Validation and release are separated into different repositories, making it harder for an attacker to reach release credentials. +- Trusted Publishing for PyPI releases - this means no long-lived credentials are used to publish releases. +- Immutable Docker release tags - this means no tampering of Docker release tags after they are published [Learn more](https://docs.docker.com/docker-hub/repos/manage/hub-images/immutable-tags/). Note: work for GHCR docker releases is planned as well. + +## What's next + +Moving forward, we plan on: +- Adopting OpenSSF (this is a set of security criteria that projects should meet to demonstrate a strong security posture - [Learn more](https://baseline.openssf.org/versions/2026-02-19.html)) + - We've added Scorecard and Allstar to our Github + +- Adding SLSA Build Provenance to our CI/CD pipeline - this means we allow users to independently verify that a release came from us and prevent silent modifications of releases after they are published. + + +We hope that this will mean you can be confident that the releases you are using are safe and from us. + + +## The principle + +The new CI/CD pipeline reflects the principles, outlined below, and is designed to be more secure and reliable: + +- **Limit** what each package can access +- **Reduce** the number of sensitive environment variables +- **Avoid** compromised packages +- **Prevent** release tampering + + +## How to help: + +Help us plan April's stability sprint - https://github.com/BerriAI/litellm/issues/24825 \ No newline at end of file diff --git a/docs/my-website/blog/security_update_march_2026/index.md b/docs/my-website/blog/security_update_march_2026/index.md index 93ae7daac3a..1c298fe372f 100644 --- a/docs/my-website/blog/security_update_march_2026/index.md +++ b/docs/my-website/blog/security_update_march_2026/index.md @@ -17,6 +17,10 @@ import VersionVerificationTable from '@site/src/components/VersionVerificationTa > **Status:** Active investigation > **Last updated:** March 27, 2026 +> **Update (March 30):** A new **clean** version of LiteLLM is now available (v1.83.0). This was released by our new [CI/CD v2](https://docs.litellm.ai/blog/ci-cd-v2-improvements) pipeline which added isolated environments, stronger security gates, and safer release separation for LiteLLM. + +> **Update (March 27):** Review Townhall updates, including explanation of the incident, what we've done, and what comes next. [Learn more](https://docs.litellm.ai/blog/security-townhall-updates) + > **Update (March 27):** Added [Verified safe versions](#verified-safe-versions) section with SHA-256 checksums for all audited PyPI and Docker releases. > **Update (March 26):** Added `checkmarx[.]zone` to [Indicators of compromise](#indicators-of-compromise-iocs) @@ -25,10 +29,10 @@ import VersionVerificationTable from '@site/src/components/VersionVerificationTa ## TLDR; -- The compromised PyPI packages were **litellm==1.82.7** and **litellm==1.82.8**. Those packages have now been removed from PyPI. -- We believe that the compromise originated from the Trivy dependency used in our CI/CD security scanning workflow. +- The compromised PyPI packages were **litellm==1.82.7** and **litellm==1.82.8**. Those packages were live on March 24, 2026 from 10:39 UTC for about 40 minutes before being quarantined by PyPI. +- We believe that the compromise originated from the [Trivy dependency](https://www.aquasec.com/blog/trivy-supply-chain-attack-what-you-need-to-know/) used in our CI/CD security scanning workflow. - Customers running the official LiteLLM Proxy Docker image were not impacted. That deployment path pins dependencies in requirements.txt and does not rely on the compromised PyPI packages. -- We are pausing new LiteLLM releases until we complete a broader supply-chain review and confirm the release path is safe. +- ~~We have paused all new LiteLLM releases until we complete a broader supply-chain review and confirm the release path is safe.~~ **Updated:** We have now released a new **safe** version of LiteLLM (v1.83.0) by our new [CI/CD v2](https://docs.litellm.ai/blog/ci-cd-v2-improvements) pipeline which added isolated environments, stronger security gates, and safer release separation for LiteLLM. We have also verified the codebase is safe and no malicious code was pushed to `main`. ## Overview diff --git a/docs/my-website/blog/vanta_compliance_recertification/index.md b/docs/my-website/blog/vanta_compliance_recertification/index.md new file mode 100644 index 00000000000..d05c113967f --- /dev/null +++ b/docs/my-website/blog/vanta_compliance_recertification/index.md @@ -0,0 +1,18 @@ +--- +slug: vanta-compliance-recertification +title: "LiteLLM + Vanta: SOC 2 Type 2 and ISO 27001 Recertification" +date: 2026-03-30T10:00:00 +authors: + - krrish +description: "LiteLLM is partnering with Vanta on SOC 2 Type 2 and ISO 27001 recertification and engaging independent auditors for verification." +tags: [security, compliance] +hide_table_of_contents: true +--- + +![LiteLLM x Vanta SOC-2 Recertification](/img/blog/vanta_soc2_recertification.png) + +We are partnering with [Vanta](https://www.vanta.com/) to recertify LiteLLM's compliance for SOC 2 Type 2 and ISO 27001. + +As part of this process, we are also identifying independent auditors to validate and verify our compliance posture. + +This is part of our commitment to being the most secure and transparent AI Gateway possible. diff --git a/docs/my-website/docs/proxy/high_availability_control_plane.md b/docs/my-website/docs/proxy/high_availability_control_plane.md index 324fba3a180..4cc6d2952fb 100644 --- a/docs/my-website/docs/proxy/high_availability_control_plane.md +++ b/docs/my-website/docs/proxy/high_availability_control_plane.md @@ -42,9 +42,9 @@ The **High Availability Control Plane** takes a different approach: -The **control plane** is a LiteLLM instance that serves the admin UI and knows about all the workers. It does not proxy LLM requests, it is purely for administration. +The **control plane** is a LiteLLM instance that serves the admin UI and knows about all the workers. It is **not a router** — it does not proxy or route any LLM requests. It exists purely so admins can switch between workers and manage them from a single UI. -Each **worker** is a fully independent LiteLLM proxy that handles LLM requests for its region or team. Workers have their own users, keys, teams, and budgets. +Each **worker** is a fully independent LiteLLM proxy that handles LLM requests for its region or team. Workers have their own database, Redis, users, keys, teams, and budgets. No infrastructure is shared between workers. ## Setup diff --git a/docs/my-website/docs/proxy/jwt_key_mapping.md b/docs/my-website/docs/proxy/jwt_key_mapping.md new file mode 100644 index 00000000000..452bf821016 --- /dev/null +++ b/docs/my-website/docs/proxy/jwt_key_mapping.md @@ -0,0 +1,318 @@ +# JWT → Virtual Key Mapping + +:::info Enterprise + +JWT → Virtual Key Mapping is an Enterprise feature. + +[Get a free trial](https://enterprise.litellm.ai/demo) + +::: + +Map JWT tokens to LiteLLM virtual keys — so every JWT client gets the same granular controls as a virtual key: model restrictions, spend limits, rate limits, guardrails, and full spend tracking. + +**Why this matters:** Standard JWT auth maps a JWT to a *team*. That's a shared boundary — all clients under a team share the same limits. With JWT → Virtual Key Mapping, each individual JWT client (identified by a claim like `client_id`, `azp`, or `sub`) maps to its own virtual key. You get per-client accountability without issuing API keys to your users. + +**Common use case:** Your company uses SSO/OIDC. Developers use Claude Code with their identity tokens. You want to enforce per-developer model access and spend limits without giving each person a LiteLLM API key. + +--- + +## How It Works + +```mermaid +sequenceDiagram + participant Client as Client (Claude Code / API) + participant Proxy as LiteLLM Proxy + participant OIDC as OIDC Provider + participant DB as Mapping Table + + Client->>Proxy: POST /v1/chat/completions
Authorization: Bearer + + Proxy->>OIDC: Verify JWT signature + OIDC-->>Proxy: Valid ✓ + + Proxy->>Proxy: Extract claim
(e.g. client_id = "alice@corp.com") + + Proxy->>DB: Look up (claim_name, claim_value) + alt Mapping found + DB-->>Proxy: virtual_key_id = sk-abc123 + Proxy->>Proxy: Apply virtual key permissions
(models, budget, rate limits) + Proxy-->>Client: 200 OK + else No mapping — fallback_team_mapping + Proxy->>Proxy: Fall through to team JWT auth + Proxy-->>Client: 200 OK + else No mapping — reject + Proxy-->>Client: 403 Forbidden + else No mapping — auto_register + Proxy->>DB: Create new virtual key + mapping + Proxy-->>Client: 200 OK + end +``` + +--- + +## Setup + +### Prerequisites + +Complete [OIDC JWT Auth setup](./token_auth.md) first — you need `JWT_PUBLIC_KEY_URL` configured and `enable_jwt_auth: True` in your proxy config. + +### Step 1. Configure the JWT claim to map on + +Add `jwt_client_id_field` to your `litellm_jwtauth` config. This is the JWT claim LiteLLM uses as the lookup key: + +```yaml +general_settings: + master_key: sk-1234 + enable_jwt_auth: True + litellm_jwtauth: + team_id_jwt_field: "team_id" # existing team mapping (optional) + user_id_jwt_field: "sub" + jwt_client_id_field: "client_id" # 👈 claim used for key mapping + unregistered_jwt_client_behavior: "fallback_team_mapping" # see below +``` + +**`unregistered_jwt_client_behavior`** controls what happens when a JWT has no registered mapping: + +| Value | Behavior | +|-------|----------| +| `fallback_team_mapping` | Fall through to team-based JWT auth (default — backward compatible) | +| `reject` | Return 403 if no mapping found | +| `auto_register` | Auto-create a virtual key + mapping on first encounter | + +### Step 2. Register a JWT client → virtual key mapping + +**Option A: Single call (creates key + mapping atomically)** + +```bash +curl -X POST 'http://0.0.0.0:4000/jwt_client/new' \ + -H 'Authorization: Bearer ' \ + -H 'Content-Type: application/json' \ + -d '{ + "jwt_claim_name": "client_id", + "jwt_claim_value": "dev-alice", + "models": ["claude-sonnet-4-5", "claude-haiku-4-5"], + "max_budget": 50.0, + "budget_duration": "30d", + "rpm_limit": 100, + "tpm_limit": 50000, + "team_id": "engineering" + }' +``` + +Response includes the virtual key token (only shown on creation): + +```json +{ + "key": "sk-abc123...", + "key_id": "key_123", + "mapping_id": "mapping_456", + "jwt_claim_name": "client_id", + "jwt_claim_value": "dev-alice" +} +``` + +**Option B: Map an existing virtual key** + +```bash +curl -X POST 'http://0.0.0.0:4000/jwt/key/mapping/new' \ + -H 'Authorization: Bearer ' \ + -H 'Content-Type: application/json' \ + -d '{ + "jwt_claim_name": "client_id", + "jwt_claim_value": "dev-alice", + "virtual_key_id": "key_123" + }' +``` + +### Step 3. Test it + +```bash +# Get a JWT from your OIDC provider (must have client_id: dev-alice) +JWT_TOKEN="eyJhbG..." + +curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \ + -H "Authorization: Bearer $JWT_TOKEN" \ + -H 'Content-Type: application/json' \ + -d '{ + "model": "claude-sonnet-4-5", + "messages": [{"role": "user", "content": "Hello"}] + }' +``` + +The request is now tracked against `dev-alice`'s virtual key — spend, rate limits, and model access enforced per-client. + +--- + +## Walkthrough: Admin grants granular access, team uses Claude Code + +This is the full flow for an engineering team using Claude Code with company SSO. + +### Admin setup + +**1. Create a team for engineering** + +```bash +curl -X POST 'http://0.0.0.0:4000/team/new' \ + -H 'Authorization: Bearer ' \ + -H 'Content-Type: application/json' \ + -d '{ + "team_alias": "engineering", + "models": ["claude-sonnet-4-5", "claude-haiku-4-5"] + }' +``` + +**2. Register each developer with their own key and spend limit** + +```bash +# Alice — senior eng, higher budget +curl -X POST 'http://0.0.0.0:4000/jwt_client/new' \ + -H 'Authorization: Bearer ' \ + -H 'Content-Type: application/json' \ + -d '{ + "jwt_claim_name": "client_id", + "jwt_claim_value": "alice@corp.com", + "team_id": "engineering", + "models": ["claude-sonnet-4-5", "claude-haiku-4-5"], + "max_budget": 200.0, + "budget_duration": "30d", + "rpm_limit": 200 + }' + +# Bob — contractor, tighter limits +curl -X POST 'http://0.0.0.0:4000/jwt_client/new' \ + -H 'Authorization: Bearer ' \ + -H 'Content-Type: application/json' \ + -d '{ + "jwt_claim_name": "client_id", + "jwt_claim_value": "bob@contractor.com", + "team_id": "engineering", + "models": ["claude-haiku-4-5"], + "max_budget": 20.0, + "budget_duration": "30d", + "rpm_limit": 30 + }' +``` + +**3. Configure Claude Code to use the proxy** + +Set the proxy as the API base in your team's Claude Code config: + +```bash +# Point Claude Code at the LiteLLM proxy instead of Anthropic directly. +# ANTHROPIC_API_KEY here is the bearer token sent to the proxy — set it to +# the user's SSO/OIDC JWT token (obtained from your IdP at login). +export ANTHROPIC_API_KEY="" +export ANTHROPIC_BASE_URL="http://your-litellm-proxy:4000" +``` + +Or in `~/.claude/settings.json`: + +```json +{ + "env": { + "ANTHROPIC_BASE_URL": "http://your-litellm-proxy:4000" + } +} +``` + +**4. Developers authenticate with SSO as usual** + +When Alice runs Claude Code, her JWT (issued by your IdP with `client_id: alice@corp.com`) goes to the proxy. LiteLLM looks up the mapping, finds her virtual key, and enforces her specific limits — her $200/month budget, 200 RPM cap, and access to Sonnet and Haiku only. + +Bob's token maps to his own key — $20/month, Haiku only, 30 RPM. + +No API keys distributed. No shared limits. Full per-developer spend visibility in the LiteLLM dashboard. + +--- + +## Managing mappings + +**View a mapping + its key settings** + +```bash +curl 'http://0.0.0.0:4000/jwt/key/mapping/info?jwt_claim_name=client_id&jwt_claim_value=alice@corp.com' \ + -H 'Authorization: Bearer ' +``` + +Response includes the linked key's `models`, `max_budget`, `spend`, `rpm_limit`, `expires`, etc. + +**Update a mapping** + +```bash +curl -X POST 'http://0.0.0.0:4000/jwt_client/update' \ + -H 'Authorization: Bearer ' \ + -H 'Content-Type: application/json' \ + -d '{ + "jwt_claim_name": "client_id", + "jwt_claim_value": "alice@corp.com", + "max_budget": 300.0 + }' +``` + +**Delete a mapping** + +```bash +curl -X DELETE 'http://0.0.0.0:4000/jwt/key/mapping/delete' \ + -H 'Authorization: Bearer ' \ + -H 'Content-Type: application/json' \ + -d '{ + "jwt_claim_name": "client_id", + "jwt_claim_value": "alice@corp.com" + }' +``` + +--- + +## Security + +JWT-bound keys are locked down: + +- Non-admin users cannot call `/key/update`, `/key/delete`, or `/key/regenerate` on a JWT-bound key. These return 403. +- JWT-bound keys are automatically restricted to `llm_api_routes` — they can make LLM calls but cannot manage other keys or admin resources. +- Only proxy admins can create, update, or delete mappings. + +--- + +## Multi-IdP support + +If you have users across multiple identity providers that share the same claim values (e.g. two services both have `sub: user-123` from different issuers), set `issuer` when creating the mapping: + +```bash +curl -X POST 'http://0.0.0.0:4000/jwt_client/new' \ + -H 'Authorization: Bearer ' \ + -H 'Content-Type: application/json' \ + -d '{ + "jwt_claim_name": "sub", + "jwt_claim_value": "user-123", + "issuer": "https://idp-a.corp.com", + "models": ["claude-sonnet-4-5"], + "max_budget": 50.0 + }' +``` + +Mappings are unique per `(claim_name, claim_value, issuer)` — so `user-123` from IdP A and `user-123` from IdP B resolve to different virtual keys. + +--- + +## What JWT clients can and can't do vs virtual keys + +| Capability | Virtual Key | JWT → Key Mapping | +|---|---|---| +| Per-client model access | ✅ | ✅ | +| Per-client spend budget | ✅ | ✅ | +| Per-client RPM/TPM limits | ✅ | ✅ | +| Team membership | ✅ | ✅ | +| Spend tracking in dashboard | ✅ | ✅ | +| Guardrails | ✅ | ✅ | +| Key rotation | ✅ | ✅ (admin only) | +| Key expiry | ✅ | ✅ | +| No API key distribution needed | ❌ | ✅ | +| Works with existing SSO/OIDC | ❌ | ✅ | + +--- + +## Related + +- [OIDC JWT Auth](./token_auth.md) — base JWT auth setup required before using this feature +- [Virtual Keys](./virtual_keys.md) — full virtual key documentation +- [Access Control](./access_control.md) — model and team access control diff --git a/docs/my-website/docs/proxy/token_auth.md b/docs/my-website/docs/proxy/token_auth.md index bc6fde7c840..c287ab03644 100644 --- a/docs/my-website/docs/proxy/token_auth.md +++ b/docs/my-website/docs/proxy/token_auth.md @@ -16,6 +16,12 @@ Use JWT's to auth admins / users / projects into the proxy. ::: +:::tip JWT → Virtual Key Mapping + +Want per-user model restrictions, spend limits, and rate limits without distributing API keys? See **[JWT → Virtual Key Mapping](./jwt_key_mapping.md)** — enterprise-grade granular access control for JWT-authenticated users (e.g. Claude Code + SSO). + +::: + ## Usage ### Step 1. Setup Proxy diff --git a/docs/my-website/docusaurus.config.js b/docs/my-website/docusaurus.config.js index 9e1fae6a34f..81c08609e5d 100644 --- a/docs/my-website/docusaurus.config.js +++ b/docs/my-website/docusaurus.config.js @@ -285,6 +285,7 @@ const config = { to: "docs/enterprise" }, { to: '/blog', label: 'Blog', position: 'left' }, + { to: '/release_notes', label: 'Release Notes', position: 'left' }, { href: 'https://github.com/BerriAI/litellm', position: 'right', diff --git a/docs/my-website/img/ci_cd_architecture.png b/docs/my-website/img/ci_cd_architecture.png new file mode 100644 index 00000000000..111567c11b0 Binary files /dev/null and b/docs/my-website/img/ci_cd_architecture.png differ diff --git a/docs/my-website/package.json b/docs/my-website/package.json index 20462de2dd7..2802d24b57e 100644 --- a/docs/my-website/package.json +++ b/docs/my-website/package.json @@ -48,27 +48,26 @@ "node": ">=16.14", "npm": ">=8.3.0" }, - "resolutions": { - "webpack-dev-server": ">=5.2.1", - "form-data": ">=4.0.4", - "mermaid": ">=11.10.0", - "gray-matter": "4.0.3", - "node-forge": ">=1.3.2" - }, "overrides": { - "webpack-dev-server": ">=5.2.1", - "form-data": ">=4.0.4", - "mermaid": ">=11.10.0", "gray-matter": "4.0.3", - "glob": ">=11.1.0", - "tar": ">=7.5.10", - "minimatch": ">=10.2.4", - "diff": ">=8.0.3", - "@isaacs/brace-expansion": ">=5.0.1", - "serialize-javascript": ">=7.0.3", - "node-forge": ">=1.3.2", - "mdast-util-to-hast": ">=13.2.1", - "lodash-es": ">=4.17.23", + "webpack-dev-server": "5.2.3", + "form-data": "4.0.5", + "mermaid": "11.12.1", + "minimatch": "10.2.4", + "serialize-javascript": "7.0.3", + "mdast-util-to-hast": "13.2.1", + "lodash-es": "4.17.23", + "@babel/traverse": "7.28.5", + "ws": "8.19.0", + "http-proxy-middleware": "3.0.5", + "tar-fs": "3.1.1", + "webpack-dev-middleware": "5.3.4", + "braces": "3.0.3", + "webpack": "5.105.3", + "serve-static": "2.2.1", + "path-to-regexp": "1.9.0", + "dompurify": "3.3.2", + "svgo": "4.0.1", "schema-utils@3": { "ajv": "6.14.0" }, @@ -83,18 +82,6 @@ }, "url-loader": { "ajv": "6.14.0" - }, - "@babel/traverse": ">=7.23.2", - "ws": ">=7.5.10", - "http-proxy-middleware": ">=2.0.9", - "tar-fs": ">=2.1.4", - "webpack-dev-middleware": ">=5.3.4", - "braces": ">=3.0.3", - "axios": ">=0.30.2", - "webpack": ">=5.94.0", - "serve-static": ">=1.16.0", - "path-to-regexp": ">=0.1.12", - "dompurify": ">=3.3.2", - "svgo": ">=3.3.3" + } } } diff --git a/docs/my-website/release_notes/v1.83.0/index.md b/docs/my-website/release_notes/v1.83.0/index.md new file mode 100644 index 00000000000..35e8a494ee8 --- /dev/null +++ b/docs/my-website/release_notes/v1.83.0/index.md @@ -0,0 +1,62 @@ +--- +title: "v1.83.0 - Official Release (Post Supply Chain Incident)" +slug: "v1-83-0" +date: 2026-03-31T00:00:00 +authors: + - name: Krrish Dholakia + title: CEO, LiteLLM + url: https://www.linkedin.com/in/krish-d/ + image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg + - name: Ishaan Jaff + title: CTO, LiteLLM + url: https://www.linkedin.com/in/reffajnaahsi/ + image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg +hide_table_of_contents: false +--- + +## Deploy this version + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + + + + +``` showLineNumbers title="docker run litellm" +docker run \ +-e STORE_MODEL_IN_DB=True \ +-p 4000:4000 \ +ghcr.io/berriai/litellm:main-1.83.0-nightly +``` + + + + +``` showLineNumbers title="pip install litellm" +pip install litellm==1.83.0 +``` + + + + +## Context: First Release After Supply Chain Incident + +v1.83.0 is the first LiteLLM release built and published through our new [CI/CD v2 pipeline](https://docs.litellm.ai/blog/ci-cd-v2-improvements), following the [supply chain incident on March 24](https://docs.litellm.ai/blog/security-update-march-2026). + +We paused all releases for one week while we: +1. Completed a forensic review with [Mandiant](https://www.mandiant.com/) and [Veria Labs](https://verialabs.com/) +2. Rebuilt the release pipeline from scratch with isolated environments and ephemeral credentials +3. Verified the codebase contains no indicators of compromise + +If you have questions about this release or the incident, see our [Security Townhall post](https://docs.litellm.ai/blog/security-townhall-updates) or reach out at `security@berri.ai`. + +--- + +## Links + +- **PyPI**: [litellm 1.83.0](https://pypi.org/project/litellm/1.83.0/) +- **Security update**: [Supply chain incident report](https://docs.litellm.ai/blog/security-update-march-2026) +- **Security townhall**: [What happened, what we've done, what comes next](https://docs.litellm.ai/blog/security-townhall-updates) +- **CI/CD v2**: [Announcing CI/CD v2 for LiteLLM](https://docs.litellm.ai/blog/ci-cd-v2-improvements) +- **April stability sprint**: [Help us plan](https://github.com/BerriAI/litellm/issues/24825) + diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index cc2d7800a21..b514ea2234c 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -452,6 +452,7 @@ const sidebars = { items: [ "proxy/virtual_keys", "proxy/token_auth", + "proxy/jwt_key_mapping", "proxy/service_accounts", "proxy/access_control", "proxy/cli_sso", diff --git a/docs/my-website/src/components/ControlPlaneArchitecture/ControlPlaneArchitecture.tsx b/docs/my-website/src/components/ControlPlaneArchitecture/ControlPlaneArchitecture.tsx index 5cc41979b02..d296e0ce29f 100644 --- a/docs/my-website/src/components/ControlPlaneArchitecture/ControlPlaneArchitecture.tsx +++ b/docs/my-website/src/components/ControlPlaneArchitecture/ControlPlaneArchitecture.tsx @@ -3,23 +3,56 @@ import styles from './styles.module.css'; /* ────────────────────── Shared small pieces ────────────────────── */ -function InfraChip({ color, label }: { color: string; label: string }) { - const dotClass = +function InfraBox({ icon, label, color }: { icon: string; label: string; color: 'green' | 'blue' | 'orange' }) { + const colorClass = color === 'green' - ? styles.infraDotGreen + ? styles.infraBoxGreen : color === 'blue' - ? styles.infraDotBlue - : styles.infraDotOrange; + ? styles.infraBoxBlue + : styles.infraBoxOrange; return ( - - - {label} - +
+ {icon} + {label} +
); } -/* ────────────────────── Architecture tab ────────────────────── */ +/* ────────────────────── Worker column with infra ────────────────────── */ + +function WorkerColumn({ + name, + region, + subtitle, + nodeClass, + badgeClass, +}: { + name: string; + region: string; + subtitle: string; + nodeClass: string; + badgeClass: string; +}) { + return ( +
+
+
+ {name} + {region} +
+
{subtitle}
+
Handles LLM requests
+
+
+ + +
+
+ ); +} + +/* ────────────────────── Architecture diagram ────────────────────── */ function ArchitectureView() { return ( @@ -36,49 +69,41 @@ function ArchitectureView() {
Control Plane - UI + ADMIN UI ONLY
cp.example.com
-
- - - +
+ Not a router — does not proxy LLM requests. +
+ Lets admins switch between workers to manage them.
- {/* Branch connector */} -
-
-
+ {/* Branch connector with label */} +
+ UI management only +
+
+
+
{/* Workers */}
-
-
- Worker A - US East -
-
worker-a.example.com
-
- - - -
-
- -
-
- Worker B - EU West -
-
worker-b.example.com
-
- - - -
-
+ +
); diff --git a/docs/my-website/src/components/ControlPlaneArchitecture/styles.module.css b/docs/my-website/src/components/ControlPlaneArchitecture/styles.module.css index 8400a8920e0..3084c5ad44a 100644 --- a/docs/my-website/src/components/ControlPlaneArchitecture/styles.module.css +++ b/docs/my-website/src/components/ControlPlaneArchitecture/styles.module.css @@ -284,45 +284,67 @@ color: var(--cp-purple); } -/* ── Infrastructure chips ── */ -.infraRow { - display: flex; - gap: 0.4rem; - justify-content: center; - flex-wrap: wrap; - margin-top: 0.5rem; +/* ── Node caption ── */ +.nodeCaption { + font-size: 0.72rem; + color: var(--cp-text-muted); + margin-top: 0.4rem; + line-height: 1.4; + font-style: italic; } -.infraChip { +/* ── Infrastructure boxes (per-worker) ── */ +.infraStack { + display: flex; + flex-direction: column; + gap: 0.35rem; + margin-top: 0.5rem; + width: 100%; +} + +.infraBox { display: flex; align-items: center; - gap: 0.3rem; - font-size: 0.7rem; - font-weight: 500; - color: var(--cp-text-secondary); - background: var(--cp-infra-bg); - border: 1px solid var(--cp-infra-border); - border-radius: 6px; - padding: 0.2rem 0.5rem; + gap: 0.5rem; + padding: 0.45rem 0.75rem; + border-radius: 8px; + border: 1.5px solid var(--cp-border); + background: var(--cp-card-bg); } -.infraDot { - width: 6px; - height: 6px; - border-radius: 50%; +.infraBoxGreen { + border-color: var(--cp-green); + background: var(--cp-green-light); +} + +.infraBoxBlue { + border-color: var(--cp-accent); + background: var(--cp-accent-light); +} + +.infraBoxOrange { + border-color: var(--cp-orange); + background: var(--cp-orange-light); +} + +.infraBoxIcon { + font-size: 0.85rem; flex-shrink: 0; } -.infraDotGreen { - background: var(--cp-green); +.infraBoxLabel { + font-size: 0.75rem; + font-weight: 600; + color: var(--cp-text); } -.infraDotBlue { - background: var(--cp-accent); -} - -.infraDotOrange { - background: var(--cp-orange); +/* ── Worker column (card + infra stack) ── */ +.workerColumn { + display: flex; + flex-direction: column; + align-items: stretch; + min-width: 220px; + max-width: 260px; } /* ── Workers row ── */ @@ -333,6 +355,24 @@ flex-wrap: wrap; } +/* ── Connector with label ── */ +.connectorBranchLabeled { + display: flex; + flex-direction: column; + align-items: center; + width: 100%; + max-width: 700px; +} + +.connectorLabel { + font-size: 0.7rem; + color: var(--cp-text-muted); + font-weight: 500; + text-transform: uppercase; + letter-spacing: 0.05em; + margin-bottom: 0.25rem; +} + /* ── Animated flow ── */ .flowLabel { font-size: 0.7rem; @@ -511,6 +551,16 @@ max-width: 260px; } + .workerColumn { + min-width: auto; + width: 100%; + max-width: 280px; + } + + .connectorBranchLabeled { + display: none; + } + .connectorBranch { display: none; } diff --git a/docs/my-website/static/img/blog/vanta_soc2_recertification.png b/docs/my-website/static/img/blog/vanta_soc2_recertification.png new file mode 100644 index 00000000000..c1c5644fbe5 Binary files /dev/null and b/docs/my-website/static/img/blog/vanta_soc2_recertification.png differ diff --git a/enterprise/litellm_enterprise/enterprise_callbacks/pagerduty/pagerduty.py b/enterprise/litellm_enterprise/enterprise_callbacks/pagerduty/pagerduty.py index b6c9104b232..de02a0c4dab 100644 --- a/enterprise/litellm_enterprise/enterprise_callbacks/pagerduty/pagerduty.py +++ b/enterprise/litellm_enterprise/enterprise_callbacks/pagerduty/pagerduty.py @@ -116,6 +116,7 @@ class PagerDutyAlerting(SlackAlerting): user_api_key_org_id=_meta.get("user_api_key_org_id"), user_api_key_team_id=_meta.get("user_api_key_team_id"), user_api_key_project_id=_meta.get("user_api_key_project_id"), + user_api_key_project_alias=_meta.get("user_api_key_project_alias"), user_api_key_user_id=_meta.get("user_api_key_user_id"), user_api_key_team_alias=_meta.get("user_api_key_team_alias"), user_api_key_end_user_id=_meta.get("user_api_key_end_user_id"), @@ -197,6 +198,7 @@ class PagerDutyAlerting(SlackAlerting): user_api_key_org_id=user_api_key_dict.org_id, user_api_key_team_id=user_api_key_dict.team_id, user_api_key_project_id=user_api_key_dict.project_id, + user_api_key_project_alias=user_api_key_dict.project_alias, user_api_key_user_id=user_api_key_dict.user_id, user_api_key_team_alias=user_api_key_dict.team_alias, user_api_key_end_user_id=user_api_key_dict.end_user_id, diff --git a/litellm-js/proxy/.npmrc b/litellm-js/proxy/.npmrc new file mode 100644 index 00000000000..168e81a1c4e --- /dev/null +++ b/litellm-js/proxy/.npmrc @@ -0,0 +1,5 @@ +# Supply-chain hardening +# Packages needing lifecycle scripts: npm rebuild +ignore-scripts=true +# Protects local npm install only — npm ci (used in CI) ignores this +min-release-age=3d diff --git a/litellm-js/spend-logs/.npmrc b/litellm-js/spend-logs/.npmrc new file mode 100644 index 00000000000..168e81a1c4e --- /dev/null +++ b/litellm-js/spend-logs/.npmrc @@ -0,0 +1,5 @@ +# Supply-chain hardening +# Packages needing lifecycle scripts: npm rebuild +ignore-scripts=true +# Protects local npm install only — npm ci (used in CI) ignores this +min-release-age=3d diff --git a/litellm-js/spend-logs/Dockerfile b/litellm-js/spend-logs/Dockerfile index a325b5cbc91..5040dc74bf6 100644 --- a/litellm-js/spend-logs/Dockerfile +++ b/litellm-js/spend-logs/Dockerfile @@ -8,7 +8,7 @@ WORKDIR /app COPY ./litellm-js/spend-logs/package*.json ./ # Install dependencies -RUN npm install +RUN npm ci # Install Prisma globally RUN npm install -g prisma diff --git a/litellm-js/spend-logs/package.json b/litellm-js/spend-logs/package.json index a40b0fc2a83..8bccbdba27b 100644 --- a/litellm-js/spend-logs/package.json +++ b/litellm-js/spend-logs/package.json @@ -9,22 +9,5 @@ "devDependencies": { "@types/node": "^20.11.17", "tsx": "^4.7.1" - }, - "overrides": { - "glob": ">=11.1.0", - "tar": ">=7.5.10", - "minimatch": ">=10.2.4", - "diff": ">=8.0.3", - "@isaacs/brace-expansion": ">=5.0.1", - "@babel/traverse": ">=7.23.2", - "ws": ">=7.5.10", - "http-proxy-middleware": ">=2.0.9", - "tar-fs": ">=2.1.4", - "webpack-dev-middleware": ">=5.3.4", - "braces": ">=3.0.3", - "axios": ">=0.30.2", - "webpack": ">=5.94.0", - "serve-static": ">=1.16.0", - "path-to-regexp": ">=0.1.12" } -} \ No newline at end of file +} diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260319000000_restore_mcp_approval_fields/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260319000000_restore_mcp_approval_fields/migration.sql new file mode 100644 index 00000000000..fa8724046c2 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260319000000_restore_mcp_approval_fields/migration.sql @@ -0,0 +1,13 @@ +-- Restore fields dropped by 20260311180521_schema_sync on LiteLLM_MCPServerTable +-- AlterTable +ALTER TABLE "LiteLLM_MCPServerTable" + ADD COLUMN IF NOT EXISTS "source_url" TEXT, + ADD COLUMN IF NOT EXISTS "approval_status" TEXT DEFAULT 'active', + ADD COLUMN IF NOT EXISTS "submitted_by" TEXT, + ADD COLUMN IF NOT EXISTS "submitted_at" TIMESTAMP(3), + ADD COLUMN IF NOT EXISTS "reviewed_at" TIMESTAMP(3), + ADD COLUMN IF NOT EXISTS "review_notes" TEXT; + +-- CreateIndex +CREATE INDEX IF NOT EXISTS "LiteLLM_MCPServerTable_approval_status_idx" + ON "LiteLLM_MCPServerTable"("approval_status"); diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 46be6b31e1f..d8d6015ce87 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -321,12 +321,11 @@ model LiteLLM_MCPServerTable { byok_description String[] @default([]) byok_api_key_help_url String? source_url String? - // BYOM submission lifecycle - approval_status String? @default("active") - submitted_by String? - submitted_at DateTime? - reviewed_at DateTime? - review_notes String? + approval_status String? @default("active") + submitted_by String? + submitted_at DateTime? + reviewed_at DateTime? + review_notes String? @@index([approval_status]) } diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml index f03cdf61c2a..55c964557fd 100644 --- a/litellm-proxy-extras/pyproject.toml +++ b/litellm-proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm-proxy-extras" -version = "0.4.61" +version = "0.4.62" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." authors = ["BerriAI"] readme = "README.md" @@ -22,7 +22,7 @@ requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "0.4.61" +version = "0.4.62" version_files = [ "pyproject.toml:version", "../requirements.txt:litellm-proxy-extras==", diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 6e6070f7f2c..ff1bc0d3839 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -56,6 +56,16 @@ if TYPE_CHECKING: ) +def _get_reasoning_items( + msg: "AllMessageValues", +) -> List[ChatCompletionReasoningItem]: + """Extract reasoning_items from a message dict with proper typing.""" + items = msg.get("reasoning_items") # type: ignore[union-attr] + if items: + return items # type: ignore[return-value] + return [] + + def _build_reasoning_item( item_id: str, encrypted_content: Optional[str], @@ -86,7 +96,9 @@ def _build_reasoning_item( } -def _reasoning_item_to_response_input(r_item: Dict[str, Any]) -> Dict[str, Any]: +def _reasoning_item_to_response_input( + r_item: Union[ChatCompletionReasoningItem, Dict[str, Any]] +) -> Dict[str, Any]: """Convert a stored ChatCompletionReasoningItem back to a Responses API input item.""" r_input: Dict[str, Any] = { "type": "reasoning", @@ -246,7 +258,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): } ) elif role == "assistant" and tool_calls and isinstance(tool_calls, list): - for r_item in msg.get("reasoning_items") or []: + for r_item in _get_reasoning_items(msg): input_items.append(_reasoning_item_to_response_input(r_item)) for tool_call in tool_calls: function = tool_call.get("function") @@ -264,7 +276,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): raise ValueError(f"tool call not supported: {tool_call}") elif content is not None: if role == "assistant": - for r_item in msg.get("reasoning_items") or []: + for r_item in _get_reasoning_items(msg): input_items.append(_reasoning_item_to_response_input(r_item)) input_items.append( { diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 294b9d464f5..fb5fc253ae4 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -2789,9 +2789,7 @@ class PrometheusLogger(CustomLogger): ) return - async def fetch_orgs( - page_size: int, page: int - ) -> Tuple[list, Optional[int]]: + async def fetch_orgs(page_size: int, page: int) -> Tuple[list, Optional[int]]: skip = (page - 1) * page_size orgs = await prisma_client.db.litellm_organizationtable.find_many( skip=skip, diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index c26759eaf0a..0a2f07bcb25 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -169,9 +169,6 @@ def get_llm_provider( # noqa: PLR0915 return remainder, custom_llm_provider, dynamic_api_key, api_base return model, custom_llm_provider, dynamic_api_key, api_base - if api_key and api_key.startswith("os.environ/"): - dynamic_api_key = get_secret_str(api_key) - # Check JSON-configured providers FIRST (before enum-based provider_list) provider_prefix = model.split("/", 1)[0] if len(model.split("/")) > 1 and JSONProviderRegistry.exists(provider_prefix): diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 53478dcbb65..7395b65626f 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -3000,9 +3000,10 @@ class Logging(LiteLLMLoggingBaseClass): litellm_call_id=self.model_call_details["litellm_call_id"], print_verbose=print_verbose, ) - if ( - callable(callback) and customLogger is not None - ): # custom logger functions + if callable(callback): # custom logger functions + global customLogger + if customLogger is None: + customLogger = CustomLogger() customLogger.log_event( kwargs=self.model_call_details, response_obj=result, @@ -3143,9 +3144,10 @@ class Logging(LiteLLMLoggingBaseClass): start_time=start_time, end_time=end_time, ) # type: ignore - if ( - callable(callback) and customLogger is not None - ): # custom logger functions + if callable(callback): # custom logger functions + global customLogger + if customLogger is None: + customLogger = CustomLogger() await customLogger.async_log_event( kwargs=self.model_call_details, response_obj=result, diff --git a/litellm/llms/a2a/chat/guardrail_translation/handler.py b/litellm/llms/a2a/chat/guardrail_translation/handler.py index 91e7e9d5550..3d6037b1f8f 100644 --- a/litellm/llms/a2a/chat/guardrail_translation/handler.py +++ b/litellm/llms/a2a/chat/guardrail_translation/handler.py @@ -234,37 +234,13 @@ class A2AGuardrailHandler(BaseTranslation): then the combined guardrailed text is written into the first chunk that had text and all other text parts in other chunks are cleared (in-place). """ - from litellm.llms.a2a.common_utils import extract_text_from_a2a_response - - # Parse each item; keep alignment with responses_so_far (None where unparseable) - parsed: List[Optional[Dict[str, Any]]] = [None] * len(responses_so_far) - for i, item in enumerate(responses_so_far): - if isinstance(item, dict): - obj = item - elif isinstance(item, str): - try: - obj = json.loads(item.strip()) - except (json.JSONDecodeError, TypeError): - continue - else: - continue - if isinstance(obj.get("result"), dict): - parsed[i] = obj - - valid_parsed = [(i, obj) for i, obj in enumerate(parsed) if obj is not None] + parsed, valid_parsed = self._parse_streaming_responses(responses_so_far) if not valid_parsed: return responses_so_far - # Collect text from each chunk in order (by original index in responses_so_far) - text_parts: List[str] = [] - chunk_indices_with_text: List[int] = [] # indices into valid_parsed - for idx, (orig_i, obj) in enumerate(valid_parsed): - t = extract_text_from_a2a_response(obj) - if t: - text_parts.append(t) - chunk_indices_with_text.append(orig_i) - - combined_text = "".join(text_parts) + combined_text, chunk_indices_with_text = self._collect_text_from_parsed_chunks( + valid_parsed + ) if not combined_text: return responses_so_far @@ -337,6 +313,43 @@ class A2AGuardrailHandler(BaseTranslation): return responses_so_far + def _parse_streaming_responses( + self, + responses_so_far: List[Any], + ) -> Tuple[List[Optional[Dict[str, Any]]], List[Tuple[int, Dict[str, Any]]]]: + """Parse JSON-RPC items, returning aligned parsed list and valid entries.""" + parsed: List[Optional[Dict[str, Any]]] = [None] * len(responses_so_far) + for i, item in enumerate(responses_so_far): + if isinstance(item, dict): + obj = item + elif isinstance(item, str): + try: + obj = json.loads(item.strip()) + except (json.JSONDecodeError, TypeError): + continue + else: + continue + if isinstance(obj.get("result"), dict): + parsed[i] = obj + valid_parsed = [(i, obj) for i, obj in enumerate(parsed) if obj is not None] + return parsed, valid_parsed + + def _collect_text_from_parsed_chunks( + self, + valid_parsed: List[Tuple[int, Dict[str, Any]]], + ) -> Tuple[str, List[int]]: + """Collect text from parsed chunks, returning combined text and indices.""" + from litellm.llms.a2a.common_utils import extract_text_from_a2a_response + + text_parts: List[str] = [] + chunk_indices_with_text: List[int] = [] + for _idx, (orig_i, obj) in enumerate(valid_parsed): + t = extract_text_from_a2a_response(obj) + if t: + text_parts.append(t) + chunk_indices_with_text.append(orig_i) + return "".join(text_parts), chunk_indices_with_text + def _extract_texts_from_result( self, result: Dict[str, Any], diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index b6139a2f84d..d31a0a091e9 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -277,82 +277,35 @@ class AnthropicMessagesHandler(BaseTranslation): images_to_check: List[str] = [] tool_calls_to_check: List[ChatCompletionToolCallChunk] = [] task_mappings: List[Tuple[int, Optional[int]]] = [] - # Track (content_index, None) for each text - - # Handle both dict and object responses - response_content: List[Any] = [] - if isinstance(response, dict): - response_content = response.get("content", []) or [] - elif hasattr(response, "content"): - content = getattr(response, "content", None) - response_content = content or [] - else: - response_content = [] + response_content = self._get_response_content(response) if not response_content: return response # Step 1: Extract all text content and tool calls from response - for content_idx, content_block in enumerate(response_content): - # Handle both dict and Pydantic object content blocks - block_dict: Dict[str, Any] = {} - if isinstance(content_block, dict): - block_type = content_block.get("type") - block_dict = cast(Dict[str, Any], content_block) - elif hasattr(content_block, "type"): - block_type = getattr(content_block, "type", None) - # Convert Pydantic object to dict for processing - if hasattr(content_block, "model_dump"): - block_dict = content_block.model_dump() - else: - block_dict = { - "type": block_type, - "text": getattr(content_block, "text", None), - } - else: - continue - - if block_type in ["text", "tool_use"]: - self._extract_output_text_and_images( - content_block=block_dict, - content_idx=content_idx, - texts_to_check=texts_to_check, - images_to_check=images_to_check, - task_mappings=task_mappings, - tool_calls_to_check=tool_calls_to_check, - ) + self._extract_from_content_blocks( + response_content, + texts_to_check, + images_to_check, + task_mappings, + tool_calls_to_check, + ) # Step 2: Apply guardrail to all texts in batch if texts_to_check or tool_calls_to_check: - # Use the real request_data if provided (proxy path), otherwise - # create a standalone dict (SDK / direct-call path). - if request_data is None: - request_data = {"response": response} - else: - if "response" not in request_data: - request_data["response"] = response + request_data = self._prepare_request_data( + request_data, + response, + user_api_key_dict, + key="response", + ) - # Add user API key metadata with prefixed keys - if "litellm_metadata" not in request_data: - user_metadata = self.transform_user_api_key_dict_to_metadata( - user_api_key_dict - ) - if user_metadata: - request_data["litellm_metadata"] = user_metadata - - inputs = GenericGuardrailAPIInputs(texts=texts_to_check) - if images_to_check: - inputs["images"] = images_to_check - if tool_calls_to_check: - inputs["tool_calls"] = tool_calls_to_check - # Include model information from the response if available - response_model = None - if isinstance(response, dict): - response_model = response.get("model") - elif hasattr(response, "model"): - response_model = getattr(response, "model", None) - if response_model: - inputs["model"] = response_model + inputs = self._build_guardrail_inputs( + texts_to_check, + images_to_check, + tool_calls_to_check, + response, + ) guardrailed_inputs = await guardrail_to_apply.apply_guardrail( inputs=inputs, @@ -440,6 +393,95 @@ class AnthropicMessagesHandler(BaseTranslation): ) return responses_so_far + def _prepare_request_data( + self, + request_data: Optional[dict], + response: Any, + user_api_key_dict: Optional[Any], + key: str, + ) -> dict: + """Ensure request_data has the response/responses_so_far key and metadata.""" + if request_data is None: + request_data = {key: response} + else: + if key not in request_data: + request_data[key] = response + + if "litellm_metadata" not in request_data: + user_metadata = self.transform_user_api_key_dict_to_metadata( + user_api_key_dict + ) + if user_metadata: + request_data["litellm_metadata"] = user_metadata + return request_data + + @staticmethod + def _get_response_content(response: Any) -> List[Any]: + """Extract content list from a dict or object response.""" + if isinstance(response, dict): + return response.get("content", []) or [] + elif hasattr(response, "content"): + return getattr(response, "content", None) or [] + return [] + + def _extract_from_content_blocks( + self, + response_content: List[Any], + texts_to_check: List[str], + images_to_check: List[str], + task_mappings: List[Tuple[int, Optional[int]]], + tool_calls_to_check: List["ChatCompletionToolCallChunk"], + ) -> None: + """Extract text, images, and tool calls from content blocks.""" + for content_idx, content_block in enumerate(response_content): + block_dict: Dict[str, Any] = {} + if isinstance(content_block, dict): + block_type = content_block.get("type") + block_dict = cast(Dict[str, Any], content_block) + elif hasattr(content_block, "type"): + block_type = getattr(content_block, "type", None) + if hasattr(content_block, "model_dump"): + block_dict = content_block.model_dump() + else: + block_dict = { + "type": block_type, + "text": getattr(content_block, "text", None), + } + else: + continue + + if block_type in ["text", "tool_use"]: + self._extract_output_text_and_images( + content_block=block_dict, + content_idx=content_idx, + texts_to_check=texts_to_check, + images_to_check=images_to_check, + task_mappings=task_mappings, + tool_calls_to_check=tool_calls_to_check, + ) + + @staticmethod + def _build_guardrail_inputs( + texts_to_check: List[str], + images_to_check: List[str], + tool_calls_to_check: List["ChatCompletionToolCallChunk"], + response: Any, + ) -> "GenericGuardrailAPIInputs": + """Build GenericGuardrailAPIInputs with optional images, tool calls, model.""" + inputs = GenericGuardrailAPIInputs(texts=texts_to_check) + if images_to_check: + inputs["images"] = images_to_check + if tool_calls_to_check: + inputs["tool_calls"] = tool_calls_to_check + response_model = None + if isinstance(response, dict): + response_model = response.get("model") + elif hasattr(response, "model"): + response_model = getattr(response, "model", None) + if response_model: + inputs["model"] = response_model + return inputs + def get_streaming_string_so_far(self, responses_so_far: List[Any]) -> str: """ Parse streaming responses and extract accumulated text content. diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 90b501a9cbb..5cfb00d69b6 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -160,7 +160,8 @@ class AmazonConverseConfig(BaseConfig): if isinstance(content, list): has_guarded_text = any( - isinstance(item, dict) and item.get("type") == "guarded_text" for item in content + isinstance(item, dict) and item.get("type") == "guarded_text" + for item in content ) if has_guarded_text: continue # Skip this message if it already has guarded_text @@ -321,9 +322,13 @@ class AmazonConverseConfig(BaseConfig): # Check if the model is a Nova 2 model (matches nova-2-lite, nova-2-pro, etc.) # Also check for nova-2/ spec prefix for imported models - return model_without_region.startswith("amazon.nova-2-") or model_without_region.startswith("nova-2/") + return model_without_region.startswith( + "amazon.nova-2-" + ) or model_without_region.startswith("nova-2/") - def _map_web_search_options(self, web_search_options: dict, model: str) -> Optional[BedrockToolBlock]: + def _map_web_search_options( + self, web_search_options: dict, model: str + ) -> Optional[BedrockToolBlock]: """ Map web_search_options to Nova grounding systemTool. @@ -352,7 +357,9 @@ class AmazonConverseConfig(BaseConfig): # (unlike Anthropic), so we just enable grounding with no options return BedrockToolBlock(systemTool={"name": "nova_grounding"}) - def _transform_reasoning_effort_to_reasoning_config(self, reasoning_effort: str) -> dict: + def _transform_reasoning_effort_to_reasoning_config( + self, reasoning_effort: str + ) -> dict: """ Transform reasoning_effort parameter to Nova 2 reasoningConfig structure. @@ -397,7 +404,9 @@ class AmazonConverseConfig(BaseConfig): } } - def _handle_reasoning_effort_parameter(self, model: str, reasoning_effort: str, optional_params: dict) -> None: + def _handle_reasoning_effort_parameter( + self, model: str, reasoning_effort: str, optional_params: dict + ) -> None: """ Handle the reasoning_effort parameter based on the model type. @@ -434,7 +443,9 @@ class AmazonConverseConfig(BaseConfig): optional_params["reasoning_effort"] = reasoning_effort elif self._is_nova_2_model(model): # Nova 2 models: transform to reasoningConfig - reasoning_config = self._transform_reasoning_effort_to_reasoning_config(reasoning_effort) + reasoning_config = self._transform_reasoning_effort_to_reasoning_config( + reasoning_effort + ) optional_params.update(reasoning_config) else: # Anthropic and other models: convert to thinking parameter @@ -478,7 +489,9 @@ class AmazonConverseConfig(BaseConfig): "parallel_tool_calls", ] - if "arn" in model: # we can't infer the model from the arn, so just add all params + if ( + "arn" in model + ): # we can't infer the model from the arn, so just add all params supported_params.append("tools") supported_params.append("tool_choice") supported_params.append("thinking") @@ -500,7 +513,9 @@ class AmazonConverseConfig(BaseConfig): or base_model.startswith("meta.llama3-3") or base_model.startswith("meta.llama4") or base_model.startswith("amazon.nova") - or supports_function_calling(model=model, custom_llm_provider=self.custom_llm_provider) + or supports_function_calling( + model=model, custom_llm_provider=self.custom_llm_provider + ) ): supported_params.append("tools") @@ -510,7 +525,9 @@ class AmazonConverseConfig(BaseConfig): if litellm.utils.supports_tool_choice( model=model, custom_llm_provider=self.custom_llm_provider - ) or litellm.utils.supports_tool_choice(model=base_model, custom_llm_provider=self.custom_llm_provider): + ) or litellm.utils.supports_tool_choice( + model=base_model, custom_llm_provider=self.custom_llm_provider + ): # only anthropic and mistral support tool choice config. otherwise (E.g. cohere) will fail the call - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ToolChoice.html supported_params.append("tool_choice") @@ -529,7 +546,9 @@ class AmazonConverseConfig(BaseConfig): model=model, custom_llm_provider=self.custom_llm_provider, ) - or supports_reasoning(model=base_model, custom_llm_provider=self.custom_llm_provider) + or supports_reasoning( + model=base_model, custom_llm_provider=self.custom_llm_provider + ) ): supported_params.append("thinking") supported_params.append("reasoning_effort") @@ -554,7 +573,9 @@ class AmazonConverseConfig(BaseConfig): return ToolChoiceValuesBlock(auto={}) elif isinstance(tool_choice, dict): # only supported for anthropic + mistral models - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ToolChoice.html - specific_tool = SpecificToolChoiceBlock(name=tool_choice.get("function", {}).get("name", "")) + specific_tool = SpecificToolChoiceBlock( + name=tool_choice.get("function", {}).get("name", "") + ) return ToolChoiceValuesBlock(tool=specific_tool) else: raise litellm.utils.UnsupportedParamsError( @@ -574,9 +595,15 @@ class AmazonConverseConfig(BaseConfig): return ["mp4", "mov", "mkv", "webm", "flv", "mpeg", "mpg", "wmv", "3gp"] def get_all_supported_content_types(self) -> List[str]: - return self.get_supported_image_types() + self.get_supported_document_types() + self.get_supported_video_types() + return ( + self.get_supported_image_types() + + self.get_supported_document_types() + + self.get_supported_video_types() + ) - def is_computer_use_tool_used(self, tools: Optional[List[OpenAIChatCompletionToolParam]], model: str) -> bool: + def is_computer_use_tool_used( + self, tools: Optional[List[OpenAIChatCompletionToolParam]], model: str + ) -> bool: """Check if computer use tools are being used in the request.""" if tools is None: return False @@ -589,7 +616,9 @@ class AmazonConverseConfig(BaseConfig): return True return False - def _transform_computer_use_tools(self, computer_use_tools: List[OpenAIChatCompletionToolParam]) -> List[dict]: + def _transform_computer_use_tools( + self, computer_use_tools: List[OpenAIChatCompletionToolParam] + ) -> List[dict]: """Transform computer use tools to Bedrock format.""" transformed_tools: List[dict] = [] @@ -631,7 +660,9 @@ class AmazonConverseConfig(BaseConfig): def _separate_computer_use_tools( self, tools: List[OpenAIChatCompletionToolParam], model: str - ) -> Tuple[List[OpenAIChatCompletionToolParam], List[OpenAIChatCompletionToolParam]]: + ) -> Tuple[ + List[OpenAIChatCompletionToolParam], List[OpenAIChatCompletionToolParam] + ]: """ Separate computer use tools from regular function tools. @@ -740,18 +771,25 @@ class AmazonConverseConfig(BaseConfig): # Recurse into nested schemas if "properties" in result and isinstance(result["properties"], dict): result["properties"] = { - k: AmazonConverseConfig._add_additional_properties_to_schema(v) for k, v in result["properties"].items() + k: AmazonConverseConfig._add_additional_properties_to_schema(v) + for k, v in result["properties"].items() } if "items" in result and isinstance(result["items"], dict): - result["items"] = AmazonConverseConfig._add_additional_properties_to_schema(result["items"]) + result["items"] = AmazonConverseConfig._add_additional_properties_to_schema( + result["items"] + ) for defs_key in ("$defs", "definitions"): if defs_key in result and isinstance(result[defs_key], dict): result[defs_key] = { - k: AmazonConverseConfig._add_additional_properties_to_schema(v) for k, v in result[defs_key].items() + k: AmazonConverseConfig._add_additional_properties_to_schema(v) + for k, v in result[defs_key].items() } for key in ("anyOf", "allOf", "oneOf"): if key in result and isinstance(result[key], list): - result[key] = [AmazonConverseConfig._add_additional_properties_to_schema(item) for item in result[key]] + result[key] = [ + AmazonConverseConfig._add_additional_properties_to_schema(item) + for item in result[key] + ] return result @@ -781,7 +819,9 @@ class AmazonConverseConfig(BaseConfig): } """ if json_schema is not None: - json_schema = AmazonConverseConfig._add_additional_properties_to_schema(json_schema) + json_schema = AmazonConverseConfig._add_additional_properties_to_schema( + json_schema + ) schema_str = json.dumps(json_schema) if json_schema is not None else "{}" json_schema_def: JsonSchemaDefinition = {"schema": schema_str} if name is not None: @@ -803,9 +843,14 @@ class AmazonConverseConfig(BaseConfig): non_default_params: dict, optional_params: dict, ): - optional_params = self._add_tools_to_optional_params(optional_params=optional_params, tools=tools) + optional_params = self._add_tools_to_optional_params( + optional_params=optional_params, tools=tools + ) - if "meta.llama3-3-70b-instruct-v1:0" in model and non_default_params.get("stream", False) is True: + if ( + "meta.llama3-3-70b-instruct-v1:0" in model + and non_default_params.get("stream", False) is True + ): optional_params["fake_stream"] = True def map_openai_params( @@ -944,7 +989,10 @@ class AmazonConverseConfig(BaseConfig): if "type" in value and value["type"] == "text": return optional_params - if self._supports_native_structured_outputs(model, self.custom_llm_provider) and json_schema is not None: + if ( + self._supports_native_structured_outputs(model, self.custom_llm_provider) + and json_schema is not None + ): # Use Bedrock's native structured outputs API (outputConfig.textFormat) # No synthetic tool injection, no fake_stream needed. # Requires an explicit schema — json_object with no schema falls through @@ -962,10 +1010,14 @@ class AmazonConverseConfig(BaseConfig): json_schema=json_schema, description=description, ) - optional_params = self._add_tools_to_optional_params(optional_params=optional_params, tools=[_tool]) + optional_params = self._add_tools_to_optional_params( + optional_params=optional_params, tools=[_tool] + ) if ( - litellm.utils.supports_tool_choice(model=model, custom_llm_provider=self.custom_llm_provider) + litellm.utils.supports_tool_choice( + model=model, custom_llm_provider=self.custom_llm_provider + ) and not is_thinking_enabled ): optional_params["tool_choice"] = ToolChoiceValuesBlock( @@ -977,7 +1029,9 @@ class AmazonConverseConfig(BaseConfig): optional_params["json_mode"] = True return optional_params - def update_optional_params_with_thinking_tokens(self, non_default_params: dict, optional_params: dict): + def update_optional_params_with_thinking_tokens( + self, non_default_params: dict, optional_params: dict + ): """ Handles scenario where max tokens is not specified. For anthropic models (anthropic api/bedrock/vertex ai), this requires having the max tokens being set and being greater than the thinking token budget. @@ -995,9 +1049,13 @@ class AmazonConverseConfig(BaseConfig): is_thinking_enabled = self.is_thinking_enabled(optional_params) is_max_tokens_in_request = self.is_max_tokens_in_request(non_default_params) if is_thinking_enabled and not is_max_tokens_in_request: - thinking_token_budget = cast(dict, optional_params["thinking"]).get("budget_tokens", None) + thinking_token_budget = cast(dict, optional_params["thinking"]).get( + "budget_tokens", None + ) if thinking_token_budget is not None: - optional_params["maxTokens"] = thinking_token_budget + DEFAULT_MAX_TOKENS + optional_params["maxTokens"] = ( + thinking_token_budget + DEFAULT_MAX_TOKENS + ) @overload def _get_cache_point_block( @@ -1063,15 +1121,23 @@ class AmazonConverseConfig(BaseConfig): if message["role"] == "system": system_prompt_indices.append(idx) if isinstance(message["content"], str) and message["content"]: - system_content_blocks.append(SystemContentBlock(text=message["content"])) - cache_block = self._get_cache_point_block(message, block_type="system", model=model) + system_content_blocks.append( + SystemContentBlock(text=message["content"]) + ) + cache_block = self._get_cache_point_block( + message, block_type="system", model=model + ) if cache_block: system_content_blocks.append(cache_block) elif isinstance(message["content"], list): for m in message["content"]: if m.get("type") == "text" and m.get("text"): - system_content_blocks.append(SystemContentBlock(text=m["text"])) - cache_block = self._get_cache_point_block(m, block_type="system", model=model) + system_content_blocks.append( + SystemContentBlock(text=m["text"]) + ) + cache_block = self._get_cache_point_block( + m, block_type="system", model=model + ) if cache_block: system_content_blocks.append(cache_block) if len(system_prompt_indices) > 0: @@ -1109,10 +1175,16 @@ class AmazonConverseConfig(BaseConfig): # Exceptions should not be stored in optional_params (this is a defensive fix) cleaned_params = filter_exceptions_from_params(optional_params) inference_params = safe_deep_copy(cleaned_params) - supported_converse_params = list(AmazonConverseConfig.__annotations__.keys()) + ["top_k"] + supported_converse_params = list( + AmazonConverseConfig.__annotations__.keys() + ) + ["top_k"] supported_tool_call_params = ["tools", "tool_choice"] supported_config_params = list(self.get_config_blocks().keys()) - total_supported_params = supported_converse_params + supported_tool_call_params + supported_config_params + total_supported_params = ( + supported_converse_params + + supported_tool_call_params + + supported_config_params + ) inference_params.pop("json_mode", None) # used for handling json_schema # Anthropic-only key. Bedrock expects `outputConfig` (camelCase) and # will reject `output_config` if it leaks through pass-through routes. @@ -1123,15 +1195,25 @@ class AmazonConverseConfig(BaseConfig): if request_metadata is not None: self._validate_request_metadata(request_metadata) - output_config: Optional[OutputConfigBlock] = inference_params.pop("outputConfig", None) - inference_params.pop("output_config", None) # Bedrock Converse doesn't support it + output_config: Optional[OutputConfigBlock] = inference_params.pop( + "outputConfig", None + ) + inference_params.pop( + "output_config", None + ) # Bedrock Converse doesn't support it # keep supported params in 'inference_params', and set all model-specific params in 'additional_request_params' - additional_request_params = {k: v for k, v in inference_params.items() if k not in total_supported_params} - inference_params = {k: v for k, v in inference_params.items() if k in total_supported_params} + additional_request_params = { + k: v for k, v in inference_params.items() if k not in total_supported_params + } + inference_params = { + k: v for k, v in inference_params.items() if k in total_supported_params + } # Handle parallel_tool_calls configuration - parallel_tool_use_config = additional_request_params.pop("_parallel_tool_use_config", None) + parallel_tool_use_config = additional_request_params.pop( + "_parallel_tool_use_config", None + ) if parallel_tool_use_config is not None and is_claude_4_5_on_bedrock(model): for key, value in parallel_tool_use_config.items(): if ( @@ -1146,7 +1228,9 @@ class AmazonConverseConfig(BaseConfig): additional_request_params.pop("parallel_tool_calls", None) # Only set the topK value in for models that support it - additional_request_params.update(self._handle_top_k_value(model, inference_params)) + additional_request_params.update( + self._handle_top_k_value(model, inference_params) + ) # Filter out internal/MCP-related parameters that shouldn't be sent to the API # These are LiteLLM internal parameters, not API parameters @@ -1155,7 +1239,9 @@ class AmazonConverseConfig(BaseConfig): # Filter out non-serializable objects (exceptions, callables, logging objects, etc.) # from additional_request_params to prevent JSON serialization errors # This filters: Exception objects, callable objects (functions), Logging objects, etc. - additional_request_params = filter_exceptions_from_params(additional_request_params) + additional_request_params = filter_exceptions_from_params( + additional_request_params + ) return ( inference_params, @@ -1202,7 +1288,9 @@ class AmazonConverseConfig(BaseConfig): # Only separate tools if computer use tools are actually present if filtered_tools and self.is_computer_use_tool_used(filtered_tools, model): # Separate computer use tools from regular function tools - computer_use_tools, regular_tools = self._separate_computer_use_tools(filtered_tools, model) + computer_use_tools, regular_tools = self._separate_computer_use_tools( + filtered_tools, model + ) # Process regular function tools using existing logic bedrock_tools = _bedrock_tools_pt(regular_tools) @@ -1263,7 +1351,9 @@ class AmazonConverseConfig(BaseConfig): anthropic_beta_list.append(computer_use_header) # Transform computer use tools to proper Bedrock format - transformed_computer_tools = self._transform_computer_use_tools(computer_use_tools) + transformed_computer_tools = self._transform_computer_use_tools( + computer_use_tools + ) additional_request_params["tools"] = transformed_computer_tools else: # No computer use tools, process all tools as regular tools @@ -1292,9 +1382,15 @@ class AmazonConverseConfig(BaseConfig): """ Bedrock doesn't support tool calling without `tools=` param specified. """ - if "tools" not in optional_params and messages is not None and has_tool_call_blocks(messages): + if ( + "tools" not in optional_params + and messages is not None + and has_tool_call_blocks(messages) + ): if litellm.modify_params: - optional_params["tools"] = add_dummy_tool(custom_llm_provider="bedrock_converse") + optional_params["tools"] = add_dummy_tool( + custom_llm_provider="bedrock_converse" + ) else: raise litellm.UnsupportedParamsError( message="Bedrock doesn't support tool calling without `tools=` param specified. Pass `tools=` param OR set `litellm.modify_params = True` // `litellm_settings::modify_params: True` to add dummy tool to the request.", @@ -1348,7 +1444,9 @@ class AmazonConverseConfig(BaseConfig): bedrock_tool_config: Optional[ToolConfigBlock] = None if len(bedrock_tools) > 0: - tool_choice_values: ToolChoiceValuesBlock = inference_params.pop("tool_choice", None) + tool_choice_values: ToolChoiceValuesBlock = inference_params.pop( + "tool_choice", None + ) bedrock_tool_config = ToolConfigBlock( tools=bedrock_tools, ) @@ -1358,7 +1456,9 @@ class AmazonConverseConfig(BaseConfig): data: CommonRequestObject = { "additionalModelRequestFields": additional_request_params, "system": system_content_blocks, - "inferenceConfig": self._transform_inference_params(inference_params=inference_params), + "inferenceConfig": self._transform_inference_params( + inference_params=inference_params + ), } # Handle all config blocks @@ -1388,10 +1488,14 @@ class AmazonConverseConfig(BaseConfig): litellm_params: dict, headers: Optional[dict] = None, ) -> RequestObject: - messages, system_content_blocks = self._transform_system_message(messages, model=model) + messages, system_content_blocks = self._transform_system_message( + messages, model=model + ) # Convert last user message to guarded_text if guardrailConfig is present - messages = self._convert_consecutive_user_messages_to_guarded_text(messages, optional_params) + messages = self._convert_consecutive_user_messages_to_guarded_text( + messages, optional_params + ) ## TRANSFORMATION ## _data: CommonRequestObject = self._transform_request_helper( @@ -1402,11 +1506,13 @@ class AmazonConverseConfig(BaseConfig): headers=headers, ) - bedrock_messages = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, - model=model, - llm_provider="bedrock_converse", - user_continue_message=litellm_params.pop("user_continue_message", None), + bedrock_messages = ( + await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model=model, + llm_provider="bedrock_converse", + user_continue_message=litellm_params.pop("user_continue_message", None), + ) ) data: RequestObject = {"messages": bedrock_messages, **_data} @@ -1440,10 +1546,14 @@ class AmazonConverseConfig(BaseConfig): litellm_params: dict, headers: Optional[dict] = None, ) -> RequestObject: - messages, system_content_blocks = self._transform_system_message(messages, model=model) + messages, system_content_blocks = self._transform_system_message( + messages, model=model + ) # Convert last user message to guarded_text if guardrailConfig is present - messages = self._convert_consecutive_user_messages_to_guarded_text(messages, optional_params) + messages = self._convert_consecutive_user_messages_to_guarded_text( + messages, optional_params + ) _data: CommonRequestObject = self._transform_request_helper( model=model, @@ -1492,7 +1602,9 @@ class AmazonConverseConfig(BaseConfig): encoding=encoding, ) - def _transform_reasoning_content(self, reasoning_content_blocks: List[BedrockConverseReasoningContentBlock]) -> str: + def _transform_reasoning_content( + self, reasoning_content_blocks: List[BedrockConverseReasoningContentBlock] + ) -> str: """ Extract the reasoning text from the reasoning content blocks @@ -1508,7 +1620,9 @@ class AmazonConverseConfig(BaseConfig): self, thinking_blocks: List[BedrockConverseReasoningContentBlock] ) -> List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]]: """Return a consistent format for thinking blocks between Anthropic and Bedrock.""" - thinking_blocks_list: List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]] = [] + thinking_blocks_list: List[ + Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock] + ] = [] for block in thinking_blocks: if "reasoningText" in block: _thinking_block = ChatCompletionThinkingBlock(type="thinking") @@ -1544,11 +1658,21 @@ class AmazonConverseConfig(BaseConfig): cache_creation_input_tokens = usage["cacheWriteInputTokens"] input_tokens += cache_creation_input_tokens - prompt_tokens_details = PromptTokensDetailsWrapper(cached_tokens=cache_read_input_tokens) - reasoning_tokens = token_counter(text=reasoning_content, count_response_tokens=True) if reasoning_content else 0 + prompt_tokens_details = PromptTokensDetailsWrapper( + cached_tokens=cache_read_input_tokens + ) + reasoning_tokens = ( + token_counter(text=reasoning_content, count_response_tokens=True) + if reasoning_content + else 0 + ) completion_tokens_details = CompletionTokensDetailsWrapper( reasoning_tokens=reasoning_tokens, - text_tokens=(output_tokens - reasoning_tokens if reasoning_tokens > 0 else output_tokens), + text_tokens=( + output_tokens - reasoning_tokens + if reasoning_tokens > 0 + else output_tokens + ), ) openai_usage = Usage( prompt_tokens=input_tokens, @@ -1563,7 +1687,9 @@ class AmazonConverseConfig(BaseConfig): def get_tool_call_names( self, - tools: Optional[Union[List[ToolBlock], List[OpenAIChatCompletionToolParam]]] = None, + tools: Optional[ + Union[List[ToolBlock], List[OpenAIChatCompletionToolParam]] + ] = None, ) -> List[str]: if tools is None: return [] @@ -1602,8 +1728,13 @@ class AmazonConverseConfig(BaseConfig): try: tool_call_names = self.get_tool_call_names(tools) json_content = json.loads(message.content) - if json_content.get("type") == "function" and json_content.get("name") in tool_call_names: - tool_calls = [ChatCompletionMessageToolCall(function=Function(**json_content))] + if ( + json_content.get("type") == "function" + and json_content.get("name") in tool_call_names + ): + tool_calls = [ + ChatCompletionMessageToolCall(function=Function(**json_content)) + ] message.tool_calls = tool_calls message.content = None @@ -1632,7 +1763,9 @@ class AmazonConverseConfig(BaseConfig): """ content_str = "" tools: List[ChatCompletionToolCallChunk] = [] - reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = None + reasoningContentBlocks: Optional[ + List[BedrockConverseReasoningContentBlock] + ] = None citationsContentBlocks: Optional[List[CitationsContentBlock]] = None for idx, content in enumerate(content_blocks): """ @@ -1649,7 +1782,9 @@ class AmazonConverseConfig(BaseConfig): if "toolUse" in content: ## check tool name was formatted by litellm _response_tool_name = content["toolUse"]["name"] - response_tool_name = get_bedrock_tool_name(response_tool_name=_response_tool_name) + response_tool_name = get_bedrock_tool_name( + response_tool_name=_response_tool_name + ) _function_chunk = ChatCompletionToolCallFunctionChunk( name=response_tool_name, arguments=json.dumps(content["toolUse"]["input"]), @@ -1700,7 +1835,11 @@ class AmazonConverseConfig(BaseConfig): """ try: response_data = json.loads(json_str) - if isinstance(response_data, dict) and "properties" in response_data and len(response_data) == 1: + if ( + isinstance(response_data, dict) + and "properties" in response_data + and len(response_data) == 1 + ): response_data = response_data["properties"] return json.dumps(response_data) except json.JSONDecodeError: @@ -1724,7 +1863,11 @@ class AmazonConverseConfig(BaseConfig): if not json_mode or not tools: return tools if tools else None - json_tool_indices = [i for i, t in enumerate(tools) if t["function"].get("name") == RESPONSE_FORMAT_TOOL_NAME] + json_tool_indices = [ + i + for i, t in enumerate(tools) + if t["function"].get("name") == RESPONSE_FORMAT_TOOL_NAME + ] if not json_tool_indices: # No json_tool_call found, return tools unchanged @@ -1732,10 +1875,14 @@ class AmazonConverseConfig(BaseConfig): if len(json_tool_indices) == len(tools): # All tools are json_tool_call — convert first one to content - verbose_logger.debug("Processing JSON tool call response for response_format") + verbose_logger.debug( + "Processing JSON tool call response for response_format" + ) json_mode_content_str: Optional[str] = tools[0]["function"].get("arguments") if json_mode_content_str is not None: - json_mode_content_str = AmazonConverseConfig._unwrap_bedrock_properties(json_mode_content_str) + json_mode_content_str = AmazonConverseConfig._unwrap_bedrock_properties( + json_mode_content_str + ) chat_completion_message["content"] = json_mode_content_str return None @@ -1745,9 +1892,13 @@ class AmazonConverseConfig(BaseConfig): first_idx = json_tool_indices[0] json_mode_args = tools[first_idx]["function"].get("arguments") if json_mode_args is not None: - json_mode_args = AmazonConverseConfig._unwrap_bedrock_properties(json_mode_args) + json_mode_args = AmazonConverseConfig._unwrap_bedrock_properties( + json_mode_args + ) existing = chat_completion_message.get("content") or "" - chat_completion_message["content"] = existing + json_mode_args if existing else json_mode_args + chat_completion_message["content"] = ( + existing + json_mode_args if existing else json_mode_args + ) real_tools = [t for i, t in enumerate(tools) if i not in json_tool_indices] return real_tools if real_tools else None @@ -1825,7 +1976,9 @@ class AmazonConverseConfig(BaseConfig): chat_completion_message: ChatCompletionResponseMessage = {"role": "assistant"} content_str = "" tools: List[ChatCompletionToolCallChunk] = [] - reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = None + reasoningContentBlocks: Optional[ + List[BedrockConverseReasoningContentBlock] + ] = None citationsContentBlocks: Optional[List[CitationsContentBlock]] = None if message is not None: @@ -1844,11 +1997,17 @@ class AmazonConverseConfig(BaseConfig): provider_specific_fields["citationsContent"] = citationsContentBlocks if provider_specific_fields: - chat_completion_message["provider_specific_fields"] = provider_specific_fields + chat_completion_message[ + "provider_specific_fields" + ] = provider_specific_fields if reasoningContentBlocks is not None: - chat_completion_message["reasoning_content"] = self._transform_reasoning_content(reasoningContentBlocks) - chat_completion_message["thinking_blocks"] = self._transform_thinking_blocks(reasoningContentBlocks) + chat_completion_message[ + "reasoning_content" + ] = self._transform_reasoning_content(reasoningContentBlocks) + chat_completion_message[ + "thinking_blocks" + ] = self._transform_thinking_blocks(reasoningContentBlocks) chat_completion_message["content"] = content_str filtered_tools = self._filter_json_mode_tools( json_mode=json_mode, diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 0ce9d57a65f..f854cdb13d0 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -86,9 +86,9 @@ class OpenAIChatCompletionsHandler(BaseTranslation): if tool_calls_to_check: inputs["tool_calls"] = tool_calls_to_check # type: ignore if messages: - inputs["structured_messages"] = ( - messages # pass the openai /chat/completions messages to the guardrail, as-is - ) + inputs[ + "structured_messages" + ] = messages # pass the openai /chat/completions messages to the guardrail, as-is # Pass tools (function definitions) to the guardrail tools = data.get("tools") if tools: diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 0efc3dab638..c00a351bdd8 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -3078,10 +3078,7 @@ async def _team_max_budget_check( BudgetExceededError if the team is over it's max budget. Triggers a budget alert if the team is over it's max budget. """ - if ( - team_object is not None - and team_object.max_budget is not None - ): + if team_object is not None and team_object.max_budget is not None: from litellm.proxy.proxy_server import get_current_spend # Read spend from cross-pod counter (Redis-first) or cached object (fallback) diff --git a/litellm/proxy/auth/handle_jwt.py b/litellm/proxy/auth/handle_jwt.py index 880ce3fb322..6ad372f3b5a 100644 --- a/litellm/proxy/auth/handle_jwt.py +++ b/litellm/proxy/auth/handle_jwt.py @@ -7,6 +7,7 @@ JWT token must have 'litellm_proxy_admin' in scope. """ import fnmatch +import hashlib import os import re from typing import Any, List, Literal, Optional, Set, Tuple, cast @@ -620,9 +621,7 @@ class JWTHandler: ) # Check cache first - cache_key = ( - f"oidc_userinfo_{token[:20]}" # Use first 20 chars of token as cache key - ) + cache_key = f"oidc_userinfo_{hashlib.sha256(token.encode()).hexdigest()}" cached_userinfo = await self.user_api_key_cache.async_get_cache(cache_key) if cached_userinfo is not None: @@ -1397,8 +1396,12 @@ class JWTAuthManager: request_headers: Optional[dict] = None, ) -> JWTAuthBuilderResult: """Main authentication and authorization builder""" - # Check if OIDC UserInfo endpoint is enabled - if jwt_handler.litellm_jwtauth.oidc_userinfo_enabled: + # Check if OIDC UserInfo endpoint is enabled, but fall back to standard + # JWT auth if the token itself is a well-formed JWT (3-part structure). + if ( + jwt_handler.litellm_jwtauth.oidc_userinfo_enabled + and not jwt_handler.is_jwt(token=api_key) + ): verbose_proxy_logger.debug( "OIDC UserInfo is enabled. Fetching user info from UserInfo endpoint." ) diff --git a/litellm/proxy/auth/login_utils.py b/litellm/proxy/auth/login_utils.py index 702f9751506..34085d5685a 100644 --- a/litellm/proxy/auth/login_utils.py +++ b/litellm/proxy/auth/login_utils.py @@ -20,7 +20,6 @@ from litellm.proxy._types import ( ProxyException, UpdateUserRequest, UserAPIKeyAuth, - hash_token, ) from litellm.proxy.management_endpoints.internal_user_endpoints import user_update from litellm.proxy.management_endpoints.key_management_endpoints import ( @@ -29,11 +28,29 @@ from litellm.proxy.management_endpoints.key_management_endpoints import ( from litellm.proxy.management_endpoints.ui_sso import ( get_disabled_non_admin_personal_key_creation, ) -from litellm.proxy.utils import PrismaClient, get_server_root_path +from litellm.proxy.utils import ( + PrismaClient, + get_server_root_path, + hash_password, + verify_password, +) from litellm.secret_managers.main import get_secret_bool from litellm.types.proxy.ui_sso import ReturnedUITokenObject +async def _rehash_password_if_needed(user_id: str, password: str, stored: str) -> None: + """Rehash legacy password (SHA256) to scrypt on successful login.""" + if stored.startswith("scrypt:"): + return + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is not None: + await prisma_client.db.litellm_usertable.update( + where={"user_id": user_id}, + data={"password": hash_password(password)}, + ) + + def get_ui_credentials(master_key: Optional[str]) -> tuple[str, str]: """ Get UI username and password from environment variables or master key. @@ -254,13 +271,8 @@ async def authenticate_user( # noqa: PLR0915 code=401, ) - # check if password == _user_row.password - hash_password = hash_token(token=password) - if secrets.compare_digest( - password.encode("utf-8"), _password.encode("utf-8") - ) or secrets.compare_digest( - hash_password.encode("utf-8"), _password.encode("utf-8") - ): + if verify_password(password, _password): + await _rehash_password_if_needed(_user_row.user_id, password, _password) if os.getenv("DATABASE_URL") is not None: response = await generate_key_helper_fn( request_type="key", diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 9dd2ab18c8a..53ae08aefb1 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -686,7 +686,10 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 if jwt_handler.litellm_jwtauth.virtual_key_claim_field is not None: # Decode JWT to get claims without running full auth_builder jwt_claims: Optional[dict] - if jwt_handler.litellm_jwtauth.oidc_userinfo_enabled: + if ( + jwt_handler.litellm_jwtauth.oidc_userinfo_enabled + and not jwt_handler.is_jwt(token=api_key) + ): jwt_claims = await jwt_handler.get_oidc_userinfo(token=api_key) else: jwt_claims = await jwt_handler.auth_jwt(token=api_key) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index d1aebe4dceb..9e8a1585c48 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -293,19 +293,20 @@ def _override_openai_response_model( we preserve the actual model that was used (the fallback model). 2. If the request was to an Azure Model Router, we preserve the actual model that was used (e.g., gpt-5-nano-2025-08-07) instead of the router model. + 3. If this was a fastest_response batch completion, use the winning model's + model group name instead of the comma-separated list the client sent. """ if not requested_model: return - # Check if a fallback occurred - if so, preserve the actual model used hidden_params = getattr(response_obj, "_hidden_params", {}) or {} if isinstance(hidden_params, dict): + # Check if a fallback occurred - if so, preserve the actual model used fallback_headers = hidden_params.get("additional_headers", {}) or {} attempted_fallbacks = fallback_headers.get( "x-litellm-attempted-fallbacks", None ) if attempted_fallbacks is not None and attempted_fallbacks > 0: - # A fallback occurred - preserve the actual model that was used verbose_proxy_logger.debug( "%s: fallback detected (attempted_fallbacks=%d), preserving actual model used instead of overriding to requested model.", log_context, @@ -313,6 +314,25 @@ def _override_openai_response_model( ) return + # For fastest_response batch completions, use the winning model's group + # name rather than the comma-separated list the client sent. + if hidden_params.get("fastest_response_batch_completion"): + winning_model = fallback_headers.get("x-litellm-model-group") + if winning_model: + verbose_proxy_logger.debug( + "%s: fastest_response detected, using winning model group=%r instead of requested=%r.", + log_context, + winning_model, + requested_model, + ) + requested_model = winning_model + else: + verbose_proxy_logger.debug( + "%s: fastest_response detected but no model group header found, preserving actual model from response.", + log_context, + ) + return + # Check if this is an Azure Model Router request - if so, preserve the actual model used if _is_azure_model_router_request(requested_model): verbose_proxy_logger.debug( diff --git a/litellm/proxy/common_utils/reset_budget_job.py b/litellm/proxy/common_utils/reset_budget_job.py index 5db919659ed..bcfaed24398 100644 --- a/litellm/proxy/common_utils/reset_budget_job.py +++ b/litellm/proxy/common_utils/reset_budget_job.py @@ -66,10 +66,8 @@ class ResetBudgetJob: try: from litellm.proxy.proxy_server import spend_counter_cache - memberships = ( - await self.prisma_client.db.litellm_teammembership.find_many( - where={"budget_id": {"in": budget_ids}} - ) + memberships = await self.prisma_client.db.litellm_teammembership.find_many( + where={"budget_id": {"in": budget_ids}} ) for m in memberships: counter_key = f"spend:team_member:{m.user_id}:{m.team_id}" @@ -574,7 +572,11 @@ class ResetBudgetJob: counter_key = None if item_type == "key" and hasattr(item, "token") and item.token is not None: counter_key = f"spend:key:{item.token}" - elif item_type == "team" and hasattr(item, "team_id") and item.team_id is not None: + elif ( + item_type == "team" + and hasattr(item, "team_id") + and item.team_id is not None + ): counter_key = f"spend:team:{item.team_id}" if counter_key is not None: diff --git a/litellm/proxy/example_config_yaml/otel_test_config.yaml b/litellm/proxy/example_config_yaml/otel_test_config.yaml index 4af95d21b62..fc506a792eb 100644 --- a/litellm/proxy/example_config_yaml/otel_test_config.yaml +++ b/litellm/proxy/example_config_yaml/otel_test_config.yaml @@ -48,21 +48,9 @@ litellm_settings: disable_end_user_cost_tracking_prometheus_only: True guardrails: - - guardrail_name: "aporia-pre-guard" - litellm_params: - guardrail: aporia # supported values: "aporia", "bedrock", "lakera" - mode: "post_call" - api_key: os.environ/APORIA_API_KEY_1 - api_base: os.environ/APORIA_API_BASE_1 - - guardrail_name: "aporia-post-guard" - litellm_params: - guardrail: aporia # supported values: "aporia", "bedrock", "lakera" - mode: "post_call" - api_key: os.environ/APORIA_API_KEY_2 - api_base: os.environ/APORIA_API_BASE_2 - guardrail_name: "bedrock-pre-guard" litellm_params: - guardrail: bedrock # supported values: "aporia", "bedrock", "lakera" + guardrail: bedrock # supported values: "bedrock", "lakera" mode: "during_call" guardrailIdentifier: ff6ujrregl1q guardrailVersion: "DRAFT" diff --git a/litellm/proxy/management_endpoints/budget_management_endpoints.py b/litellm/proxy/management_endpoints/budget_management_endpoints.py index 37f13269b1c..41a98fa4ad9 100644 --- a/litellm/proxy/management_endpoints/budget_management_endpoints.py +++ b/litellm/proxy/management_endpoints/budget_management_endpoints.py @@ -15,7 +15,6 @@ All /budget management endpoints from datetime import timedelta from fastapi import APIRouter, Depends, HTTPException -from prisma.errors import UniqueViolationError from litellm.litellm_core_utils.duration_parser import duration_in_seconds from litellm.proxy._types import * @@ -48,6 +47,8 @@ async def new_budget( - model_max_budget: Optional[dict] - Specify max budget for a given model. Example: {"openai/gpt-4o-mini": {"max_budget": 100.0, "budget_duration": "1d", "tpm_limit": 100000, "rpm_limit": 100000}} - budget_reset_at: Optional[datetime] - Datetime when the initial budget is reset. Default is now. """ + from prisma.errors import UniqueViolationError + from litellm.proxy.proxy_server import litellm_proxy_admin_name, prisma_client if prisma_client is None: @@ -99,7 +100,9 @@ async def new_budget( "updated_by": user_api_key_dict.user_id or litellm_proxy_admin_name, } # type: ignore ) - except UniqueViolationError: + except Exception as e: + if not isinstance(e, UniqueViolationError): + raise raise HTTPException( status_code=400, detail={ diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index 7b459cd8502..646779e6f81 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -41,7 +41,7 @@ from litellm.proxy.management_endpoints.key_management_endpoints import ( prepare_metadata_fields, ) from litellm.proxy.management_helpers.utils import management_endpoint_wrapper -from litellm.proxy.utils import handle_exception_on_proxy +from litellm.proxy.utils import handle_exception_on_proxy, hash_password from litellm.types.proxy.management_endpoints.common_daily_activity import ( SpendAnalyticsPaginatedResponse, ) @@ -58,6 +58,22 @@ if TYPE_CHECKING: router = APIRouter() +def _hash_password_in_dict(data: dict) -> None: + """Hash password field in-place if present.""" + if "password" in data and data["password"] is not None: + data["password"] = hash_password(data["password"]) + + +def _strip_password_from_response(response) -> None: + """Strip password from API response (handles dicts, nested dicts, and Prisma models).""" + if isinstance(response, dict): + response.pop("password", None) + if isinstance(response.get("data"), dict): + response["data"].pop("password", None) + elif hasattr(response.get("data"), "__dict__"): + response["data"].__dict__.pop("password", None) + + def _update_internal_new_user_params(data_json: dict, data: NewUserRequest) -> dict: if "user_id" in data_json and data_json["user_id"] is None: data_json["user_id"] = str(uuid.uuid4()) @@ -438,6 +454,7 @@ async def new_user( data_json = data.json() # type: ignore data_json = _update_internal_new_user_params(data_json, data) + _hash_password_in_dict(data_json) teams = data.teams if teams is None: teams = check_if_default_team_set() @@ -591,6 +608,98 @@ def get_user_id_from_request(request: Request) -> Optional[str]: return user_id +def _normalize_user_info_user_id( + request: Request, user_id: Optional[str] +) -> Optional[str]: + """Normalize URL-decoded user_id while preserving '+' characters.""" + if user_id is not None and " " in user_id: + return get_user_id_from_request(request=request) + return user_id + + +async def _get_user_info_teams( + prisma_client: Any, + user_id: Optional[str], + user_info: Optional[Any], + user_api_key_dict: UserAPIKeyAuth, +) -> tuple[list[Any], Optional[list[Any]]]: + """Fetch and merge teams from membership + user.teams field.""" + from litellm.proxy.management_endpoints.team_endpoints import list_team + + team_list: list[Any] = [] + team_id_list: list[str] = [] + + teams_1 = await list_team( + http_request=Request( + scope={"type": "http", "path": "/user/info"}, + ), + user_id=user_id, + user_api_key_dict=user_api_key_dict, + ) + + if teams_1 is not None and isinstance(teams_1, list): + team_list = teams_1 + team_id_list = [team.team_id for team in teams_1] + + teams_2: Optional[list[Any]] = None + target_team_ids = getattr(user_info, "teams", None) + + if target_team_ids and isinstance(target_team_ids, list): + teams_2 = await prisma_client.get_data( + team_id_list=target_team_ids, + table_name="team", + query_type="find_all", + ) + elif user_api_key_dict.user_id is not None and user_id is None: + caller_user_info = await prisma_client.get_data( + user_id=user_api_key_dict.user_id + ) + caller_team_ids = getattr(caller_user_info, "teams", None) + if caller_team_ids: + teams_2 = await prisma_client.get_data( + team_id_list=caller_team_ids, + table_name="team", + query_type="find_all", + ) + + if teams_2 is not None and isinstance(teams_2, list): + for team in teams_2: + if team.team_id not in team_id_list: + team_list.append(team) + team_id_list.append(team.team_id) + + return team_list, teams_1 + + +def _build_user_info_response( + user_id: Optional[str], + user_info: Optional[Any], + keys: Optional[List[LiteLLM_VerificationToken]], + team_list: list[Any], + teams_1: Optional[list[Any]], +) -> UserInfoResponse: + """Create UserInfoResponse while filtering sensitive fields.""" + if user_info is None and keys is not None: + spend = sum(getattr(k, "spend", 0) for k in keys) + user_info = {"spend": spend} + + returned_keys = _process_keys_for_user_info(keys=keys, all_teams=teams_1) + team_list.sort(key=lambda x: (getattr(x, "team_alias", "") or "")) + + _user_info = ( + user_info.model_dump() if isinstance(user_info, BaseModel) else user_info + ) + if isinstance(_user_info, dict): + _user_info.pop("password", None) + + return UserInfoResponse( + user_id=user_id, + user_info=_user_info, + keys=returned_keys, + teams=team_list, + ) + + @router.get( "/user/info", tags=["Internal User management"], @@ -598,7 +707,7 @@ def get_user_id_from_request(request: Request) -> Optional[str]: response_model=UserInfoResponse, ) @management_endpoint_wrapper -async def user_info( +async def user_info( # noqa: PLR0915 request: Request, user_id: Optional[str] = fastapi.Query( default=None, description="User ID in the request parameters" @@ -621,11 +730,7 @@ async def user_info( from litellm.proxy.proxy_server import prisma_client try: - # Handle URL encoding properly by getting user_id from the original request - if ( - user_id is not None and " " in user_id - ): # if user_id is not None and contains a space, get the user_id from the request - this is to handle the case where the user_id is encoded in the url - user_id = get_user_id_from_request(request=request) + user_id = _normalize_user_info_user_id(request=request, user_id=user_id) if prisma_client is None: raise Exception( @@ -652,57 +757,13 @@ async def user_info( detail=f"User {user_id} not found", ) - ## GET ALL TEAMS ## - team_list = [] - team_id_list = [] - from litellm.proxy.management_endpoints.team_endpoints import list_team - - teams_1 = await list_team( - http_request=Request( - scope={"type": "http", "path": "/user/info"}, - ), + team_list, teams_1 = await _get_user_info_teams( + prisma_client=prisma_client, user_id=user_id, + user_info=user_info, user_api_key_dict=user_api_key_dict, ) - if teams_1 is not None and isinstance(teams_1, list): - team_list = teams_1 - for team in teams_1: - team_id_list.append(team.team_id) - - teams_2: Optional[Any] = None - if user_info is not None: - # *NEW* get all teams in user 'teams' field - teams_2 = await prisma_client.get_data( - team_id_list=user_info.teams, table_name="team", query_type="find_all" - ) - - if teams_2 is not None and isinstance(teams_2, list): - for team in teams_2: - if team.team_id not in team_id_list: - team_list.append(team) - team_id_list.append(team.team_id) - - elif ( - user_api_key_dict.user_id is not None and user_id is None - ): # the key querying the endpoint is the one asking for it's teams - caller_user_info = await prisma_client.get_data( - user_id=user_api_key_dict.user_id - ) - # *NEW* get all teams in user 'teams' field - if caller_user_info is not None: - teams_2 = await prisma_client.get_data( - team_id_list=caller_user_info.teams, - table_name="team", - query_type="find_all", - ) - - if teams_2 is not None and isinstance(teams_2, list): - for team in teams_2: - if team.team_id not in team_id_list: - team_list.append(team) - team_id_list.append(team.team_id) - ## GET ALL KEYS ## keys = await prisma_client.get_data( user_id=user_id, @@ -710,21 +771,12 @@ async def user_info( query_type="find_all", ) - if user_info is None and keys is not None: - ## make sure we still return a total spend ## - spend = 0 - for k in keys: - spend += getattr(k, "spend", 0) - user_info = {"spend": spend} - - ## REMOVE HASHED TOKEN INFO before returning ## - returned_keys = _process_keys_for_user_info(keys=keys, all_teams=teams_1) - team_list.sort(key=lambda x: (getattr(x, "team_alias", "") or "")) - _user_info = ( - user_info.model_dump() if isinstance(user_info, BaseModel) else user_info - ) - response_data = UserInfoResponse( - user_id=user_id, user_info=_user_info, keys=returned_keys, teams=team_list + response_data = _build_user_info_response( + user_id=user_id, + user_info=user_info, + keys=keys, + team_list=team_list, + teams_1=teams_1, ) return response_data @@ -950,6 +1002,8 @@ async def _get_user_info_for_proxy_admin(user_api_key_dict: UserAPIKeyAuth): if isinstance(admin_user_info, BaseModel) else admin_user_info ) + if isinstance(admin_user_info, dict): + admin_user_info.pop("password", None) return UserInfoResponse( user_id=admin_user_id, @@ -1089,6 +1143,8 @@ async def _update_single_user_helper( data_json=data_json, data=user_request ) + _hash_password_in_dict(non_default_values) + # Get existing user data for audit logging and metadata preparation existing_user_row: Optional[BaseModel] = None if user_request.user_id: @@ -1205,6 +1261,7 @@ async def _update_single_user_helper( status_code=400, detail={"error": "Failed to update user"}, ) + _strip_password_from_response(response) return response @@ -2189,9 +2246,7 @@ async def _resolve_team_org_filter( proxy_logging_obj: Any, ) -> List[str]: """Look up the team and return its org as a filter list, or raise 403.""" - from litellm.proxy.management_endpoints.common_utils import ( - _is_user_team_admin, - ) + from litellm.proxy.management_endpoints.common_utils import _is_user_team_admin try: team_obj = await get_team_object( diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 831922ec3f9..c140de98191 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -4181,13 +4181,13 @@ async def list_keys( user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), page: int = Query(1, description="Page number", ge=1), size: int = Query(10, description="Page size", ge=1, le=100), - user_id: Optional[str] = Query(None, description="Filter keys by user ID"), + user_id: Optional[str] = Query(None, description="Filter keys by user ID. Supports partial matching (substring, case-insensitive)."), team_id: Optional[str] = Query(None, description="Filter keys by team ID"), organization_id: Optional[str] = Query( None, description="Filter keys by organization ID" ), key_hash: Optional[str] = Query(None, description="Filter keys by key hash"), - key_alias: Optional[str] = Query(None, description="Filter keys by key alias"), + key_alias: Optional[str] = Query(None, description="Filter keys by key alias. Supports partial matching (substring, case-insensitive)."), return_full_object: bool = Query(False, description="Return full key object"), include_team_keys: bool = Query( False, description="Include all keys for teams that user is an admin of." @@ -4280,10 +4280,12 @@ async def list_keys( else: admin_team_ids = None - if not user_id and user_api_key_dict.user_role not in [ + use_substring_matching = user_api_key_dict.user_role in [ LitellmUserRoles.PROXY_ADMIN.value, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY.value, - ]: + ] + + if not user_id and not use_substring_matching: user_id = user_api_key_dict.user_id response = await _list_key_helper( @@ -4305,6 +4307,7 @@ async def list_keys( status=status, project_id=project_id, access_group_id=access_group_id, + use_substring_matching=use_substring_matching, ) verbose_proxy_logger.debug("Successfully prepared response") @@ -4522,6 +4525,7 @@ def _build_key_filter_conditions( include_created_by_keys: bool = False, project_id: Optional[str] = None, access_group_id: Optional[str] = None, + use_substring_matching: bool = False, ) -> Dict[str, Union[str, Dict[str, Any], List[Dict[str, Any]]]]: """Build filter conditions for key listing. @@ -4543,9 +4547,21 @@ def _build_key_filter_conditions( # Base conditions for user's own keys user_condition: Dict[str, Any] = {} if user_id and isinstance(user_id, str): - user_condition["user_id"] = user_id + if use_substring_matching: + user_condition["user_id"] = { + "contains": user_id, + "mode": "insensitive", + } + else: + user_condition["user_id"] = user_id if key_alias and isinstance(key_alias, str): - user_condition["key_alias"] = key_alias + if use_substring_matching: + user_condition["key_alias"] = { + "contains": key_alias, + "mode": "insensitive", + } + else: + user_condition["key_alias"] = key_alias if exclude_team_id and isinstance(exclude_team_id, str): user_condition["team_id"] = {"not": exclude_team_id} if organization_id and isinstance(organization_id, str): @@ -4648,6 +4664,7 @@ async def _list_key_helper( status: Optional[str] = None, project_id: Optional[str] = None, access_group_id: Optional[str] = None, + use_substring_matching: bool = False, ) -> KeyListResponseObject: """ Helper function to list keys @@ -4683,6 +4700,7 @@ async def _list_key_helper( include_created_by_keys=include_created_by_keys, project_id=project_id, access_group_id=access_group_id, + use_substring_matching=use_substring_matching, ) # Calculate skip for pagination diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index a5a62813b9f..0bfe8eb75bc 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -827,7 +827,9 @@ async def get_generic_sso_response( ], # sso specific jwt handler - used for restricted sso group access control generic_client_id: str, redirect_url: str, -) -> Tuple[Union[OpenID, dict], Optional[dict], Optional[dict]]: # (result, received_response, access_token_payload) +) -> Tuple[ + Union[OpenID, dict], Optional[dict], Optional[dict] +]: # (result, received_response, access_token_payload) # make generic sso provider from fastapi_sso.sso.base import DiscoveryDocument from fastapi_sso.sso.generic import create_provider @@ -1369,7 +1371,11 @@ async def auth_callback(request: Request, state: Optional[str] = None): # noqa: ) elif generic_client_id is not None: - result, received_response, access_token_payload = await get_generic_sso_response( + ( + result, + received_response, + access_token_payload, + ) = await get_generic_sso_response( request=request, jwt_handler=jwt_handler, generic_client_id=generic_client_id, diff --git a/litellm/proxy/management_helpers/audit_logs.py b/litellm/proxy/management_helpers/audit_logs.py index b9020222f1f..7599e11bdef 100644 --- a/litellm/proxy/management_helpers/audit_logs.py +++ b/litellm/proxy/management_helpers/audit_logs.py @@ -51,7 +51,11 @@ def _build_audit_log_payload( if request_data.updated_at is not None: updated_at = request_data.updated_at.isoformat() - table_name_str: str = request_data.table_name.value if isinstance(request_data.table_name, LitellmTableNames) else str(request_data.table_name) + table_name_str: str = ( + request_data.table_name.value + if isinstance(request_data.table_name, LitellmTableNames) + else str(request_data.table_name) + ) return StandardAuditLogPayload( id=request_data.id, @@ -89,7 +93,9 @@ async def _dispatch_audit_log_to_callbacks( for callback in litellm.audit_log_callbacks: try: - resolved: Optional[CustomLogger] = callback if isinstance(callback, CustomLogger) else None + resolved: Optional[CustomLogger] = ( + callback if isinstance(callback, CustomLogger) else None + ) if isinstance(callback, str): resolved = _resolve_audit_log_callback(callback) if resolved is None: @@ -138,9 +144,7 @@ async def create_object_audit_log( return _changed_by = ( - litellm_changed_by - or user_api_key_dict.user_id - or litellm_proxy_admin_name + litellm_changed_by or user_api_key_dict.user_id or litellm_proxy_admin_name ) await create_audit_log_for_update( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 853ceb63c9d..6c977a25696 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -503,7 +503,9 @@ from litellm.proxy.utils import ( get_error_message_str, get_server_root_path, handle_exception_on_proxy, + hash_password, hash_token, + migrate_passwords_to_scrypt_async, model_dump_with_preserved_fields, update_spend, ) @@ -870,6 +872,17 @@ async def proxy_startup_event(app: FastAPI): # noqa: PLR0915 user_api_key_cache=user_api_key_cache, ) + if prisma_client is not None: + + async def _run_pw_migration(): + try: + result = await migrate_passwords_to_scrypt_async(prisma_client) + verbose_proxy_logger.info(f"Password migration: {result}") + except Exception as e: + verbose_proxy_logger.warning(f"Password migration skipped: {e}") + + asyncio.create_task(_run_pw_migration()) + ProxyStartupEvent._initialize_startup_logging( llm_router=llm_router, proxy_logging_obj=proxy_logging_obj, @@ -1714,9 +1727,7 @@ async def get_current_spend(counter_key: str, fallback_spend: float) -> float: # 1. Try Redis first (cross-pod authoritative) if spend_counter_cache.redis_cache is not None: try: - val = await spend_counter_cache.redis_cache.async_get_cache( - key=counter_key - ) + val = await spend_counter_cache.redis_cache.async_get_cache(key=counter_key) if val is not None: return float(val) except Exception as e: @@ -1820,9 +1831,7 @@ async def _init_and_increment_spend_counter( key=counter_key, value=base_spend ) - await spend_counter_cache.async_increment_cache( - key=counter_key, value=increment - ) + await spend_counter_cache.async_increment_cache(key=counter_key, value=increment) async def update_cache( # noqa: PLR0915 @@ -5721,6 +5730,11 @@ def _restamp_streaming_chunk_model( if _is_azure_model_router_request(requested_model_from_client): return chunk, model_mismatch_logged + # For fastest_response batch completions, preserve the winning model's name + # instead of stamping the comma-separated list the client sent. + if request_data.get("fastest_response", False): + return chunk, model_mismatch_logged + downstream_model = ( chunk.get("model") if isinstance(chunk, dict) else getattr(chunk, "model", None) ) @@ -9215,7 +9229,7 @@ def _add_team_models_to_all_models( for team_object in team_db_objects_typed: if ( - len(team_object.models) == 0 # empty list = all model access + not team_object.models # None or empty list = all model access or SpecialModelNames.all_proxy_models.value in team_object.models ): model_list = llm_router.get_model_list() @@ -9249,6 +9263,75 @@ def _add_team_models_to_all_models( return team_models +async def _add_access_group_models_to_team_models( + team_db_objects_typed: List[LiteLLM_TeamTable], + llm_router: Router, + prisma_client: PrismaClient, + team_models: Dict[str, Set[str]], +) -> Dict[str, Set[str]]: + """ + Resolve models reachable via team access groups and merge them into team_models. + + Batch-fetches all distinct access groups in a single DB query, then resolves + each eligible team's access group models via the pre-fetched map. + + This ensures models associated with a team only through access groups + (not directly in team.models) are included in the UI model listing. + """ + # First pass: identify eligible teams and collect all distinct access group IDs + eligible_teams: List[LiteLLM_TeamTable] = [] + all_access_group_ids: Set[str] = set() + + for team_object in team_db_objects_typed: + if not team_object.access_group_ids: + continue + + # Skip teams with empty models list — they already have access to everything + # (handled by _add_team_models_to_all_models) + if ( + not team_object.models + or SpecialModelNames.all_proxy_models.value in team_object.models + ): + continue + + eligible_teams.append(team_object) + all_access_group_ids.update(team_object.access_group_ids) + + if not eligible_teams: + return team_models + + # Single batch fetch for all access groups + access_group_rows = ( + await prisma_client.db.litellm_accessgrouptable.find_many( + where={"access_group_id": {"in": list(all_access_group_ids)}} + ) + ) + ag_model_map: Dict[str, List[str]] = { + row.access_group_id: row.access_model_names or [] + for row in access_group_rows + } + + # Second pass: resolve deployments for each eligible team + for team_object in eligible_teams: + model_names: Set[str] = set() + for ag_id in team_object.access_group_ids: + model_names.update(ag_model_map.get(ag_id, [])) + + for model_name in model_names: + deployments = llm_router.get_model_list( + model_name=model_name, team_id=team_object.team_id + ) + if deployments is not None: + for deployment in deployments: + model_id = deployment.get("model_info", {}).get("id", None) + if model_id is not None: + team_models.setdefault(model_id, set()).add( + team_object.team_id + ) + + return team_models + + async def get_all_team_models( user_teams: Union[List[str], Literal["*"]], prisma_client: PrismaClient, @@ -9285,6 +9368,14 @@ async def get_all_team_models( llm_router=llm_router, ) + # Also resolve models reachable via team access groups + team_models = await _add_access_group_models_to_team_models( + team_db_objects_typed=team_db_objects_typed, + llm_router=llm_router, + prisma_client=prisma_client, + team_models=team_models, + ) + # convert set to list returned_team_models: Dict[str, List[str]] = {} for model_id, team_ids in team_models.items(): @@ -9792,7 +9883,7 @@ async def _filter_models_by_team_id( team_accessible_model_ids: Set[str] = set() if ( - len(team_object.models) == 0 # empty list = all model access + not team_object.models # empty list = all model access or SpecialModelNames.all_proxy_models.value in team_object.models ): # Team has access to all models @@ -11678,9 +11769,9 @@ async def claim_onboarding_link(data: InvitationClaim): }, ) ### UPDATE USER OBJECT ### - hash_password = hash_token(token=data.password) + hashed_pw = hash_password(data.password) user_obj = await prisma_client.db.litellm_usertable.update( - where={"user_id": invite_obj.user_id}, data={"password": hash_password} + where={"user_id": invite_obj.user_id}, data={"password": hashed_pw} ) if user_obj is None: @@ -11700,6 +11791,8 @@ async def claim_onboarding_link(data: InvitationClaim): }, ) + if user_obj and hasattr(user_obj, "__dict__"): + user_obj.__dict__.pop("password", None) return user_obj @@ -12138,7 +12231,10 @@ async def invitation_delete( dependencies=[Depends(user_api_key_auth)], include_in_schema=False, ) -async def update_config(config_info: ConfigYAML): # noqa: PLR0915 +async def update_config( # noqa: PLR0915 + config_info: ConfigYAML, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): """ For Admin UI - allows admin to update config via UI @@ -12146,6 +12242,10 @@ async def update_config(config_info: ConfigYAML): # noqa: PLR0915 """ global llm_router, llm_model_list, general_settings, proxy_config, proxy_logging_obj, master_key, prisma_client try: + if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: + raise HTTPException( + status_code=403, detail="Only proxy admins can update config" + ) import base64 """ diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index b3b4b55af19..97fa6c39fb5 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -66,6 +66,15 @@ async def spend_key_fn(): ) +def _strip_password_from_users(users) -> None: + """Strip password field from a list of user objects.""" + for user in users if isinstance(users, list) else [users]: + if user and hasattr(user, "__dict__"): + user.__dict__.pop("password", None) + elif isinstance(user, dict): + user.pop("password", None) + + @router.get( "/spend/users", tags=["Budget & Spend Tracking"], @@ -105,13 +114,15 @@ async def spend_user_fn( user_info = await prisma_client.get_data( table_name="user", query_type="find_unique", user_id=user_id ) - return [user_info] + result = [user_info] else: user_info = await prisma_client.get_data( table_name="user", query_type="find_all" ) + result = user_info - return user_info + _strip_password_from_users(result) + return result except Exception as e: raise HTTPException( diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 3eacc19a6df..8ea93453ed3 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -90,6 +90,7 @@ def _get_spend_logs_metadata( user_api_key_alias=None, user_api_key_team_id=None, user_api_key_project_id=None, + user_api_key_project_alias=None, user_api_key_org_id=None, user_api_key_user_id=None, user_api_key_team_alias=None, diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 75c803aad65..2e24468cefd 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -4554,6 +4554,72 @@ def hash_token(token: str): return hashed_token +def hash_password(password: str) -> str: + """Hash a password using scrypt with a random salt.""" + import base64 + import hashlib + import os + + salt = os.urandom(16) + dk = hashlib.scrypt(password.encode(), salt=salt, n=16384, r=8, p=1, dklen=32) + return "scrypt:" + base64.b64encode(salt + dk).decode() + + +def verify_password(password: str, stored: str) -> bool: + """Verify a password against a stored hash. Supports scrypt and SHA256.""" + import base64 + import hashlib + import secrets + + if stored.startswith("scrypt:"): + try: + raw = base64.b64decode(stored[7:]) + salt, dk = raw[:16], raw[16:] + dk2 = hashlib.scrypt( + password.encode(), salt=salt, n=16384, r=8, p=1, dklen=32 + ) + return secrets.compare_digest(dk, dk2) + except Exception: + return False + # SHA256 fallback (not vulnerable to pass-the-hash: checks sha256(input) == stored) + if len(stored) == 64 and all(c in "0123456789abcdef" for c in stored): + return secrets.compare_digest( + hashlib.sha256(password.encode()).hexdigest().encode(), stored.encode() + ) + return False + + +async def migrate_passwords_to_scrypt_async(prisma_client) -> str: + """ + Migrate plaintext passwords in the DB to scrypt. SHA256 passwords + are left alone (they migrate on next login via the SHA256 fallback). + Skips quickly if no plaintext passwords exist. + """ + all_with_pw = await prisma_client.db.litellm_usertable.find_many( + where={"password": {"not": None}}, + ) + + def _is_sha256_hex(s: str) -> bool: + return len(s) == 64 and all(c in "0123456789abcdef" for c in s) + + plaintext_users = [ + u + for u in all_with_pw + if u.password + and not u.password.startswith("scrypt:") + and not _is_sha256_hex(u.password) + ] + if not plaintext_users: + return "No plaintext passwords found" + + for user in plaintext_users: + await prisma_client.db.litellm_usertable.update( + where={"user_id": user.user_id}, + data={"password": hash_password(user.password)}, + ) + return f"Migrated {len(plaintext_users)} plaintext passwords to scrypt" + + def _hash_token_if_needed(token: str) -> str: """ Hash the token if it's a string and starts with "sk-" diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 0672b03bcd7..8e75ffdff61 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -845,6 +845,15 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): ) event.__dict__["sequence_number"] = self._sequence_number self._pending_response_events.append(event) + + # Emit content_part.added immediately after output_item.added for message + # items. The OpenAI Responses spec requires this event before any + # output_text.delta events so downstream parsers can initialize the + # text part structure. + if not self.sent_content_part_added_event: + self.sent_content_part_added_event = True + content_part_event = self.create_content_part_added_event() + self._pending_response_events.append(content_part_event) return async def __anext__( diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index b6479a36998..9075373f1cf 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -1202,22 +1202,32 @@ class LiteLLMCompletionResponsesConfig: return [chat_completion_response_message] + @staticmethod + def _resolve_file_id(item: Dict[str, Any]) -> Optional[str]: + """ + Return the effective file_id for a Responses API input_file item. + Explicit file_id takes precedence; file_url is used as fallback so + downstream providers (Anthropic, Gemini) can handle the URL natively. + """ + return item.get("file_id") or item.get("file_url") or None + @staticmethod def _transform_input_file_item_to_file_item(item: Dict[str, Any]) -> Dict[str, Any]: """ Transform a Responses API input_file item to a Chat Completion file item Args: - item: Dictionary containing input_file type with file_id and/or file_data + item: Dictionary containing input_file type with file_id, file_data, and/or file_url Returns: Dictionary with transformed file structure for Chat Completion """ file_dict: Dict[str, Any] = {} - keys = ["file_id", "file_data"] - for key in keys: - if item.get(key): - file_dict[key] = item.get(key) + file_id = LiteLLMCompletionResponsesConfig._resolve_file_id(item) + if file_id: + file_dict["file_id"] = file_id + if item.get("file_data"): + file_dict["file_data"] = item["file_data"] new_item: Dict[str, Any] = {"type": "file", "file": file_dict} return new_item @@ -2113,9 +2123,9 @@ class LiteLLMCompletionResponsesConfig: hasattr(completion_details, "reasoning_tokens") and completion_details.reasoning_tokens is not None ): - output_details_dict[ - "reasoning_tokens" - ] = completion_details.reasoning_tokens + output_details_dict["reasoning_tokens"] = ( + completion_details.reasoning_tokens + ) else: output_details_dict["reasoning_tokens"] = 0 diff --git a/litellm/setup_wizard.py b/litellm/setup_wizard.py index ee5918e1273..666915669fa 100644 --- a/litellm/setup_wizard.py +++ b/litellm/setup_wizard.py @@ -71,7 +71,7 @@ PROVIDERS: List[Dict] = [ "id": "azure", "name": "Azure OpenAI", "description": "GPT-4o via Azure", - "env_key": "AZURE_API_KEY", + "env_key": "AZURE_AI_API_KEY", "key_hint": "your-azure-key", "test_model": None, # needs deployment name — skip validation "models": [], @@ -428,7 +428,7 @@ class SetupWizard: f" {blue('❯')} Azure endpoint URL {grey(p.get('api_base_hint', ''))}: " ) if api_base: - env_vars[f"_LITELLM_AZURE_API_BASE_{p['id'].upper()}"] = api_base + env_vars[f"_LITELLM_AZURE_AI_API_BASE_{p['id'].upper()}"] = api_base deployment = _styled_input( f" {blue('❯')} Azure deployment name {grey('(e.g. my-gpt4o)')}: " ) @@ -557,7 +557,7 @@ class SetupWizard: f' api_base: "{_yaml_escape(str(p["api_base"]))}"' ) elif p.get("needs_api_base"): - azure_base_key = f"_LITELLM_AZURE_API_BASE_{p['id'].upper()}" + azure_base_key = f"_LITELLM_AZURE_AI_API_BASE_{p['id'].upper()}" if azure_base_key in env_copy: lines.append( f' api_base: "{_yaml_escape(env_copy.pop(azure_base_key))}"' diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 9e669655761..80b6190db8f 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -747,6 +747,7 @@ class ChatCompletionAssistantMessage(OpenAIChatCompletionAssistantMessage, total thinking_blocks: Optional[ List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]] ] + reasoning_items: Optional[List[ChatCompletionReasoningItem]] class ChatCompletionToolMessage(TypedDict): diff --git a/litellm/utils.py b/litellm/utils.py index 83d1242f3fd..e2dac1c9f62 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1495,10 +1495,6 @@ def client(original_function): # noqa: PLR0915 ) logging_obj._llm_caching_handler = _llm_caching_handler - # CHECK FOR 'os.environ/' in kwargs - for k, v in kwargs.items(): - if v is not None and isinstance(v, str) and v.startswith("os.environ/"): - kwargs[k] = litellm.get_secret(v) # [OPTIONAL] CHECK BUDGET if litellm.max_budget: if litellm._current_cost > litellm.max_budget: diff --git a/package.json b/package.json index 70fcb01afc7..9a7ebf4ae05 100644 --- a/package.json +++ b/package.json @@ -11,20 +11,10 @@ "jest": "^29.7.0" }, "overrides": { - "glob": ">=11.1.0", - "tar": ">=7.5.11", - "minimatch": ">=10.2.4", - "diff": ">=8.0.3", - "@isaacs/brace-expansion": ">=5.0.1", - "@babel/traverse": ">=7.23.2", - "ws": ">=7.5.10", - "http-proxy-middleware": ">=2.0.9", - "tar-fs": ">=2.1.4", - "webpack-dev-middleware": ">=5.3.4", - "braces": ">=3.0.3", - "axios": ">=0.30.2", - "webpack": ">=5.94.0", - "serve-static": ">=1.16.0", - "path-to-regexp": ">=0.1.12" + "glob": "13.0.0", + "minimatch": "10.1.1", + "@isaacs/brace-expansion": "5.0.0", + "@babel/traverse": "7.28.5", + "braces": "3.0.3" } } diff --git a/poetry.lock b/poetry.lock index 958e0ac65b8..e369d1be75f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3219,15 +3219,15 @@ files = [ [[package]] name = "litellm-proxy-extras" -version = "0.4.61" +version = "0.4.62" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." optional = true python-versions = "!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*,>=3.8" groups = ["main"] markers = "extra == \"proxy\"" files = [ - {file = "litellm_proxy_extras-0.4.61-py3-none-any.whl", hash = "sha256:9bd1e57ef51972cacff52172ef5d70b0ff689f57f3d240877667301ab8f8590e"}, - {file = "litellm_proxy_extras-0.4.61.tar.gz", hash = "sha256:dce8e39b1547abf90d912ddd0f2a876beadf789d700ef04c165362d78ad56aee"}, + {file = "litellm_proxy_extras-0.4.62-py3-none-any.whl", hash = "sha256:cf91c1a83d94000b8997ee29d9e8d505d1ad80c26f111803bef5365174c97de0"}, + {file = "litellm_proxy_extras-0.4.62.tar.gz", hash = "sha256:0d87db1cda9851717e5294f2fa2c7ae1f34d7476d99e24e6462702e11a2cdd88"}, ] [[package]] @@ -8009,4 +8009,4 @@ utils = ["numpydoc"] [metadata] lock-version = "2.1" python-versions = ">=3.9,<4.0" -content-hash = "8dad0e86d75e574f12c57c9f32614b7b4ea2181e931874046d43a398aef1e998" +content-hash = "b238a24abf451e3a7ce954d2f565fb95172cedfb2f1b56d6ed985b67e23fed36" diff --git a/pyproject.toml b/pyproject.toml index ced8c4eb712..8c95a4421bb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.82.6" +version = "1.83.0" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -61,7 +61,7 @@ boto3 = { version = "^1.40.76", optional = true } redisvl = {version = "^0.4.1", optional = true, markers = "python_version >= '3.9' and python_version < '3.14'"} mcp = {version = ">=1.25.0,<2.0.0", optional = true, python = ">=3.10"} a2a-sdk = {version = "^0.3.22", optional = true, python = ">=3.10"} -litellm-proxy-extras = {version = "^0.4.61", optional = true} +litellm-proxy-extras = {version = "^0.4.62", optional = true} rich = {version = "^13.7.1", optional = true} litellm-enterprise = {version = "0.1.35", optional = true} diskcache = {version = "^5.6.1", optional = true} @@ -184,7 +184,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.82.6" +version = "1.83.0" version_files = [ "pyproject.toml:^version" ] diff --git a/requirements.txt b/requirements.txt index d9e188c6e64..1cccdf8ccc8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -57,7 +57,7 @@ grpcio>=1.75.0; python_version >= "3.14" sentry_sdk==2.21.0 # for sentry error handling detect-secrets==1.5.0 # Enterprise - secret detection / masking in LLM requests tzdata==2025.1 # IANA time zone database -litellm-proxy-extras==0.4.61 # for proxy extras - e.g. prisma migrations +litellm-proxy-extras==0.4.62 # for proxy extras - e.g. prisma migrations llm-sandbox==0.3.31 # for skill execution in sandbox ### LITELLM PACKAGE DEPENDENCIES python-dotenv==1.0.1 # for env diff --git a/security.md b/security.md index 2da073661c5..c6cd64ddaac 100644 --- a/security.md +++ b/security.md @@ -1,5 +1,49 @@ # Data Privacy and Security + +## Security Vulnerability Reporting Guidelines + +We value the security community's role in protecting our systems and users. To report a security vulnerability: + +- File a private vulnerability report on GitHub: [Report a vulnerability](https://github.com/BerriAI/litellm/security/advisories/new) +- Include steps to reproduce the issue +- Provide any relevant additional information + +### Vulnerability Categories + +We classify vulnerabilities into the following categories: + +**P0: Supply Chain Attacks** + +Attacks that compromise our CI/CD pipeline, allowing a malicious actor to point our PyPI package or Docker images (GHCR or Docker Hub) to vulnerable or tampered artifacts. + +**P1: Unauthenticated Proxy Access** + +Application-level attacks where an unauthenticated user is able to gain access to protected data on a LiteLLM proxy instance that should be protected (e.g api keys). + +**P2: Authenticated Malicious Actions** + +Application-level attacks where an authenticated user is able to perform actions beyond their intended permissions, such as privilege escalation or unauthorized data access. + +### Bug Bounty Program + +We offer bounties for responsibly disclosed vulnerabilities based on severity: + +**Note that currently only P0/P1 reports are eligible for a bounty, though submissions for P2 bugs are still encouraged** + +| Severity | Bounty Range | Example | +|----------|-------------|---------| +| **Critical** | $1,500 - $3,000 | P0 supply chain compromise | +| **High** | $500 - $1,500 | P1 unauthenticated proxy access | +| **Medium** | N/A | P2 authenticated privilege escalation | +| **Low** | N/A | Minor information disclosure, low-impact misconfigurations | + +To qualify for a bounty, reports must include clear reproduction steps and must not involve systems or accounts you do not own. We review all submissions promptly and will follow up within 5 business days. + +### Known Non-Issues + +- Attacks that require a misconfiguration on setup (e.g not setting a `master_key` on the proxy configuration), are **explicitly not in scope** and are not considered vulnerable. + ## Security Measures ### LiteLLM Github @@ -12,11 +56,6 @@ - For installation and configuration, see: [Self-hosting guided](https://docs.litellm.ai/docs/proxy/deploy) - **Telemetry** We run no telemetry when you self host LiteLLM - -:::info -✨ SSO is free for up to 5 users. After that, an enterprise license is required. [Get Started with Enterprise here](https://www.litellm.ai/enterprise) -::: - ### LiteLLM Cloud - We encrypt all data stored using your `LITELLM_MASTER_KEY` and in transit using TLS. @@ -37,13 +76,3 @@ LiteLLM supports the following data regions: - Europe, Frankfurt, Germany (AWS/GCP `eu-central-1`) All data, user accounts, and infrastructure are completely separated between these two regions - -### Security Vulnerability Reporting Guidelines - -We value the security community's role in protecting our systems and users. To report a security vulnerability: - -- Email support@berri.ai with details -- Include steps to reproduce the issue -- Provide any relevant additional information - -We'll review all reports promptly. Note that we don't currently offer a bug bounty program. diff --git a/tests/audio_tests/azure_speech.mp3 b/tests/audio_tests/azure_speech.mp3 index 27835b83a61..ec41d428bcf 100644 Binary files a/tests/audio_tests/azure_speech.mp3 and b/tests/audio_tests/azure_speech.mp3 differ diff --git a/tests/audio_tests/test_audio_speech.py b/tests/audio_tests/test_audio_speech.py index abf0df22a6a..46d45158910 100644 --- a/tests/audio_tests/test_audio_speech.py +++ b/tests/audio_tests/test_audio_speech.py @@ -342,7 +342,7 @@ async def test_azure_ava_tts_async(): try: response = await litellm.aspeech( - model="azure/speech/azure-tts", + model="azure/tts", voice="alloy", input="Hello, this is a test of Azure text to speech", api_base=api_base, @@ -377,10 +377,6 @@ async def test_azure_ava_tts_async(): print(f"Azure TTS audio saved to: {speech_file_path}") - # assert response cost is greater than 0 - print("Response cost: ", response._hidden_params["response_cost"]) - assert response._hidden_params["response_cost"] > 0 - except Exception as e: pytest.fail(f"Test failed with exception: {str(e)}") diff --git a/tests/batches_tests/test_fine_tuning_api.py b/tests/batches_tests/test_fine_tuning_api.py index 034f4e1b495..cb570ff3c39 100644 --- a/tests/batches_tests/test_fine_tuning_api.py +++ b/tests/batches_tests/test_fine_tuning_api.py @@ -125,7 +125,6 @@ async def test_create_fine_tune_jobs_async(): @pytest.mark.asyncio() async def test_create_vertex_fine_tune_jobs_mocked(): - load_vertex_ai_credentials() # Define reusable variables for the test project_id = "633608382793" location = "us-central1" @@ -164,7 +163,10 @@ async def test_create_vertex_fine_tune_jobs_mocked(): with patch( "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", return_value=mock_response, - ) as mock_post: + ) as mock_post, patch( + "litellm.llms.vertex_ai.vertex_llm_base.VertexBase._ensure_access_token", + return_value=("fake-token", project_id), + ): create_fine_tuning_response = await litellm.acreate_fine_tuning_job( model=base_model, custom_llm_provider="vertex_ai", @@ -218,7 +220,6 @@ async def test_create_vertex_fine_tune_jobs_mocked(): @pytest.mark.asyncio() async def test_create_vertex_fine_tune_jobs_mocked_with_hyperparameters(): - load_vertex_ai_credentials() # Define reusable variables for the test project_id = "633608382793" location = "us-central1" @@ -257,7 +258,10 @@ async def test_create_vertex_fine_tune_jobs_mocked_with_hyperparameters(): with patch( "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", return_value=mock_response, - ) as mock_post: + ) as mock_post, patch( + "litellm.llms.vertex_ai.vertex_llm_base.VertexBase._ensure_access_token", + return_value=("fake-token", project_id), + ): create_fine_tuning_response = await litellm.acreate_fine_tuning_job( model=base_model, custom_llm_provider="vertex_ai", @@ -406,29 +410,6 @@ def test_convert_basic_openai_request_to_vertex_request(): ) -@pytest.mark.asyncio() -@pytest.mark.skip(reason="skipping - we run mock tests for vertex ai") -async def test_create_vertex_fine_tune_jobs(): - verbose_logger.setLevel(logging.DEBUG) - # load_vertex_ai_credentials() - - vertex_credentials = os.getenv("GCS_PATH_SERVICE_ACCOUNT") - print("creating fine tuning job") - create_fine_tuning_response = await litellm.acreate_fine_tuning_job( - model="gemini-1.0-pro-002", - custom_llm_provider="vertex_ai", - training_file="gs://cloud-samples-data/ai-platform/generative_ai/sft_train_data.jsonl", - vertex_project="pathrise-convert-1606954137718", - vertex_location="us-central1", - vertex_credentials=vertex_credentials, - ) - print("vertex ai create fine tuning response=", create_fine_tuning_response) - - assert create_fine_tuning_response.id is not None - assert create_fine_tuning_response.model == "gemini-1.0-pro-002" - assert create_fine_tuning_response.object == "fine_tuning.job" - - @pytest.mark.asyncio async def test_mock_openai_create_fine_tune_job(): """Test that create_fine_tuning_job sends correct parameters to OpenAI""" @@ -536,7 +517,6 @@ async def test_mock_openai_retrieve_fine_tune_job(): except Exception as e: print("error=", e) - # Verify the request mock_retrieve.assert_called_once_with(fine_tuning_job_id="ft-123") @@ -544,7 +524,9 @@ async def test_mock_openai_retrieve_fine_tune_job(): @pytest.mark.asyncio async def test_mock_azure_create_fine_tune_job_with_azure_specific_params(): """Test that Azure-specific parameters are passed through extra_body""" - from openai.types.fine_tuning.fine_tuning_job import Hyperparameters as OAIHyperparameters + from openai.types.fine_tuning.fine_tuning_job import ( + Hyperparameters as OAIHyperparameters, + ) from litellm.types.utils import LiteLLMFineTuningJob mock_response = LiteLLMFineTuningJob( @@ -564,7 +546,9 @@ async def test_mock_azure_create_fine_tune_job_with_azure_specific_params(): async def mock_async_create(*args, **kwargs): return mock_response - with patch("litellm.llms.azure.fine_tuning.handler.AzureOpenAIFineTuningAPI.create_fine_tuning_job") as mock_create: + with patch( + "litellm.llms.azure.fine_tuning.handler.AzureOpenAIFineTuningAPI.create_fine_tuning_job" + ) as mock_create: mock_create.return_value = mock_async_create() response = await litellm.acreate_fine_tuning_job( @@ -575,10 +559,7 @@ async def test_mock_azure_create_fine_tune_job_with_azure_specific_params(): api_key="test-key", api_version="2025-04-01-preview", trainingType=1, - hyperparameters={ - "n_epochs": 3, - "prompt_loss_weight": 0.1 - }, + hyperparameters={"n_epochs": 3, "prompt_loss_weight": 0.1}, ) # Verify the request @@ -590,7 +571,7 @@ async def test_mock_azure_create_fine_tune_job_with_azure_specific_params(): assert create_data["model"] == "gpt-4.1-mini-2025-04-14" assert create_data["training_file"] == "file-123" assert create_data["hyperparameters"] == {"n_epochs": 3} - + # Azure-specific parameters should be in extra_body assert "extra_body" in create_data assert create_data["extra_body"]["trainingType"] == 1 diff --git a/tests/batches_tests/test_openai_batches_and_files.py b/tests/batches_tests/test_openai_batches_and_files.py index e4e1180eb13..0aed224c256 100644 --- a/tests/batches_tests/test_openai_batches_and_files.py +++ b/tests/batches_tests/test_openai_batches_and_files.py @@ -54,7 +54,7 @@ def load_vertex_ai_credentials(): print("loading vertex ai credentials") os.environ["GCS_FLUSH_INTERVAL"] = "1" filepath = os.path.dirname(os.path.abspath(__file__)) - vertex_key_path = filepath + "/pathrise-convert-1606954137718.json" + vertex_key_path = filepath + "/vertex_key.json" # Read the existing content of the file or create an empty dictionary try: diff --git a/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py b/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py index 61b1b1f8185..967ef2a5fec 100644 --- a/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py +++ b/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py @@ -1608,15 +1608,24 @@ async def test_initialize_remaining_budget_metrics_exception_handling( mock_teamtable = MagicMock() mock_teamtable.count = MagicMock(side_effect=Exception("Team count error")) + # Mock litellm_organizationtable to raise an exception for org budget metrics + mock_orgtable = MagicMock() + mock_orgtable.find_many = MagicMock( + side_effect=Exception("Org database error") + ) + mock_orgtable.count = MagicMock(side_effect=Exception("Org count error")) + mock_db = MagicMock() mock_db.litellm_usertable = mock_usertable mock_db.litellm_teamtable = mock_teamtable + mock_db.litellm_organizationtable = mock_orgtable mock_prisma.db = mock_db # Mock the Prometheus metrics prometheus_logger.litellm_remaining_team_budget_metric = MagicMock() prometheus_logger.litellm_remaining_api_key_budget_metric = MagicMock() prometheus_logger.litellm_remaining_user_budget_metric = MagicMock() + prometheus_logger.litellm_remaining_org_budget_metric = MagicMock() prometheus_logger.litellm_total_users_metric = MagicMock() prometheus_logger.litellm_teams_count_metric = MagicMock() @@ -1625,8 +1634,8 @@ async def test_initialize_remaining_budget_metrics_exception_handling( # Call the function await prometheus_logger._initialize_remaining_budget_metrics() - # Verify all four errors were logged (teams, keys, users, and user/team count) - assert mock_logger.call_count == 4 + # Verify all five errors were logged (teams, keys, users, orgs, and user/team count) + assert mock_logger.call_count == 5 assert ( "Error initializing teams budget metrics" in mock_logger.call_args_list[0][0][0] @@ -1640,14 +1649,19 @@ async def test_initialize_remaining_budget_metrics_exception_handling( in mock_logger.call_args_list[2][0][0] ) assert ( - "Error initializing user/team count metrics" + "Error initializing orgs budget metrics" in mock_logger.call_args_list[3][0][0] ) + assert ( + "Error initializing user/team count metrics" + in mock_logger.call_args_list[4][0][0] + ) # Verify the metrics were never called prometheus_logger.litellm_remaining_team_budget_metric.assert_not_called() prometheus_logger.litellm_remaining_api_key_budget_metric.assert_not_called() prometheus_logger.litellm_remaining_user_budget_metric.assert_not_called() + prometheus_logger.litellm_remaining_org_budget_metric.assert_not_called() prometheus_logger.litellm_total_users_metric.assert_not_called() prometheus_logger.litellm_teams_count_metric.assert_not_called() diff --git a/tests/enterprise/litellm_enterprise/proxy/auth/test_user_api_key_auth.py b/tests/enterprise/litellm_enterprise/proxy/auth/test_user_api_key_auth.py index 0a2930aebb0..a45df5df008 100644 --- a/tests/enterprise/litellm_enterprise/proxy/auth/test_user_api_key_auth.py +++ b/tests/enterprise/litellm_enterprise/proxy/auth/test_user_api_key_auth.py @@ -72,7 +72,7 @@ async def test_enterprise_custom_auth_returns_string(): auth_obj = await _user_api_key_auth_builder( request=request, api_key="my-custom-key", - AZURE_AI_API_KEY_header="", + azure_api_key_header="", anthropic_api_key_header=None, google_ai_studio_api_key_header=None, azure_apim_header=None, diff --git a/tests/image_gen_tests/test_image_edit.png b/tests/image_gen_tests/test_image_edit.png index 3b9d865ce6f..6ccc3026c6e 100644 Binary files a/tests/image_gen_tests/test_image_edit.png and b/tests/image_gen_tests/test_image_edit.png differ diff --git a/tests/image_gen_tests/test_image_edits.py b/tests/image_gen_tests/test_image_edits.py index 8fe2e25a994..3504065a109 100644 --- a/tests/image_gen_tests/test_image_edits.py +++ b/tests/image_gen_tests/test_image_edits.py @@ -158,7 +158,7 @@ class TestAzureAIFlux2ImageEdit(BaseLLMImageEditTest): return { "model": "azure_ai/flux.2-pro", "image": SINGLE_TEST_IMAGE, - "api_base": "https://litellm-ci-cd-prod.services.ai.azure.com", + "api_base": os.getenv("AZURE_AI_API_BASE"), "api_key": os.getenv("AZURE_AI_API_KEY"), "api_version": "preview", } diff --git a/tests/image_gen_tests/test_image_generation.py b/tests/image_gen_tests/test_image_generation.py index aca5c41de38..b22a18b49b8 100644 --- a/tests/image_gen_tests/test_image_generation.py +++ b/tests/image_gen_tests/test_image_generation.py @@ -113,7 +113,7 @@ class TestVertexImageGeneration(BaseImageGenTest): litellm.in_memory_llm_clients_cache = InMemoryCache() return { "model": "vertex_ai/imagen-3.0-fast-generate-001", - "vertex_ai_project": "pathrise-convert-1606954137718", + "vertex_ai_project": "litellm-ci-cd", "vertex_ai_location": "us-central1", "n": 1, } @@ -129,7 +129,7 @@ class TestVertexAIGeminiImageGeneration(BaseImageGenTest): litellm.in_memory_llm_clients_cache = InMemoryCache() return { "model": "vertex_ai/gemini-2.5-flash-image", - "vertex_ai_project": "pathrise-convert-1606954137718", + "vertex_ai_project": "litellm-ci-cd", "vertex_ai_location": "us-central1", "n": 1, "size": "1024x1024", diff --git a/tests/litellm_utils_tests/test_azure_ai_anthropic_token_counter.py b/tests/litellm_utils_tests/test_azure_ai_anthropic_token_counter.py index 031502cbece..2686c28cb1c 100644 --- a/tests/litellm_utils_tests/test_azure_ai_anthropic_token_counter.py +++ b/tests/litellm_utils_tests/test_azure_ai_anthropic_token_counter.py @@ -26,22 +26,20 @@ class TestAzureAIAnthropicTokenCounter(BaseTokenCounterTest): return AzureAIAnthropicTokenCounter() def get_test_model(self) -> str: - return "claude-3-5-sonnet" + return "claude-sonnet-4-6" def get_test_messages(self) -> List[Dict[str, Any]]: - return [ - {"role": "user", "content": "Hello, how are you today?"} - ] + return [{"role": "user", "content": "Hello, how are you today?"}] def get_deployment_config(self) -> Dict[str, Any]: - api_key = os.getenv("AZURE_AI_API_KEY") - api_base = os.getenv("AZURE_AI_API_BASE") - + api_key = os.getenv("AZURE_ANTHROPIC_API_KEY") + api_base = os.getenv("AZURE_AI_SWEDEN_API_BASE") + if not api_key: pytest.skip("AZURE_AI_API_KEY not set") if not api_base: pytest.skip("AZURE_AI_API_BASE not set") - + return { "litellm_params": { "api_key": api_key, diff --git a/tests/litellm_utils_tests/test_hashicorp.py b/tests/litellm_utils_tests/test_hashicorp.py index e4d69da6ac6..ef755306eff 100644 --- a/tests/litellm_utils_tests/test_hashicorp.py +++ b/tests/litellm_utils_tests/test_hashicorp.py @@ -27,6 +27,19 @@ from litellm.secret_managers.hashicorp_secret_manager import HashicorpSecretMana @pytest.fixture def hashicorp_secret_manager(): """Provide a fresh HashicorpSecretManager per test to avoid shared state.""" + has_token = bool(os.getenv("HCP_VAULT_TOKEN")) + has_approle = bool( + os.getenv("HCP_VAULT_APPROLE_ROLE_ID") + and os.getenv("HCP_VAULT_APPROLE_SECRET_ID") + ) + has_tls_cert = bool( + os.getenv("HCP_VAULT_CLIENT_CERT") and os.getenv("HCP_VAULT_CLIENT_KEY") + ) + if not (has_token or has_approle or has_tls_cert): + pytest.skip( + "Skipping Hashicorp tests: set HCP_VAULT_TOKEN, AppRole vars, or TLS cert vars." + ) + manager = HashicorpSecretManager() manager.vault_addr = "https://test-cluster-public-vault-0f98180c.e98296b2.z1.hashicorp.cloud:8200" manager.vault_namespace = "admin" @@ -253,7 +266,7 @@ async def test_hashicorp_secret_manager_delete_secret_with_team_overrides( assert called_url == expected_url -def test_hashicorp_secret_manager_tls_cert_auth(monkeypatch, hashicorp_secret_manager): +def test_hashicorp_secret_manager_tls_cert_auth(monkeypatch): monkeypatch.setenv("HCP_VAULT_TOKEN", "test-client-token-12345") print("HCP_VAULT_TOKEN=", os.getenv("HCP_VAULT_TOKEN")) # Mock both httpx.post and httpx.Client @@ -301,7 +314,7 @@ def test_hashicorp_secret_manager_tls_cert_auth(monkeypatch, hashicorp_secret_ma assert test_manager.cache.get_cache("hcp_vault_token") == "test-client-token-12345" -def test_hashicorp_secret_manager_approle_auth(monkeypatch, hashicorp_secret_manager): +def test_hashicorp_secret_manager_approle_auth(monkeypatch): """ Test AppRole authentication makes the expected POST request to the correct URL. """ diff --git a/tests/litellm_utils_tests/test_litellm_overhead.py b/tests/litellm_utils_tests/test_litellm_overhead.py index 006fbea8d4b..76a1894327f 100644 --- a/tests/litellm_utils_tests/test_litellm_overhead.py +++ b/tests/litellm_utils_tests/test_litellm_overhead.py @@ -101,7 +101,7 @@ async def test_litellm_overhead_non_streaming(model): kwargs["vertex_project"] = "fake-project" kwargs["vertex_location"] = "us-central1" if model == "openai/self_hosted": - kwargs["api_base"] = "https://exampleopenaiendpoint-production.up.railway.app/" + kwargs["api_base"] = os.environ.get("FAKE_OPENAI_API_BASE") async def _run(): return await litellm.acompletion(**kwargs) diff --git a/tests/litellm_utils_tests/test_secret_manager.py b/tests/litellm_utils_tests/test_secret_manager.py index 7569c673ece..a1190193ea5 100644 --- a/tests/litellm_utils_tests/test_secret_manager.py +++ b/tests/litellm_utils_tests/test_secret_manager.py @@ -1,3 +1,4 @@ +import base64 import os import sys import time @@ -24,7 +25,7 @@ from litellm.secret_managers.main import ( get_secret, _should_read_secret_from_secret_manager, ) -from unittest.mock import AsyncMock +from unittest.mock import AsyncMock, patch, MagicMock def load_vertex_ai_credentials(): @@ -221,53 +222,79 @@ def test_oidc_env_path(): del os.environ[env_var_name] -@pytest.mark.flaky(retries=6, delay=1) def test_google_secret_manager(): """ Test that we can get a secret from Google Secret Manager """ - os.environ["GOOGLE_SECRET_MANAGER_PROJECT_ID"] = "pathrise-convert-1606954137718" + os.environ["GOOGLE_SECRET_MANAGER_PROJECT_ID"] = "litellm-ci-cd" from litellm.secret_managers.google_secret_manager import GoogleSecretManager - load_vertex_ai_credentials() - secret_manager = GoogleSecretManager() + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "payload": { + "data": base64.b64encode(b"anything").decode("utf-8"), + } + } - secret_val = secret_manager.get_secret_from_google_secret_manager( - secret_name="OPENAI_API_KEY" - ) - print("secret_val: {}".format(secret_val)) + with patch( + "litellm.proxy.proxy_server.premium_user", True + ), patch.object( + GoogleSecretManager, + "sync_construct_request_headers", + return_value={"Authorization": "Bearer mock_token"}, + ): + secret_manager = GoogleSecretManager() + secret_manager.sync_httpx_client = MagicMock() + secret_manager.sync_httpx_client.get.return_value = mock_response - assert ( - secret_val == "anything" - ), "did not get expected secret value. expect 'anything', got '{}'".format( - secret_val - ) + secret_val = secret_manager.get_secret_from_google_secret_manager( + secret_name="OPENAI_API_KEY" + ) + print("secret_val: {}".format(secret_val)) + + assert ( + secret_val == "anything" + ), "did not get expected secret value. expect 'anything', got '{}'".format( + secret_val + ) + + secret_manager.sync_httpx_client.get.assert_called_once() + call_url = secret_manager.sync_httpx_client.get.call_args[1]["url"] + assert "projects/litellm-ci-cd/secrets/OPENAI_API_KEY" in call_url def test_google_secret_manager_read_in_memory(): """ - Test that Google Secret manager returs in memory value when it exists + Test that Google Secret manager returns in memory value when it exists """ from litellm.secret_managers.google_secret_manager import GoogleSecretManager - load_vertex_ai_credentials() - os.environ["GOOGLE_SECRET_MANAGER_PROJECT_ID"] = "pathrise-convert-1606954137718" - secret_manager = GoogleSecretManager() - secret_manager.cache.cache_dict["UNIQUE_KEY"] = None - secret_manager.cache.cache_dict["UNIQUE_KEY_2"] = "lite-llm" + os.environ["GOOGLE_SECRET_MANAGER_PROJECT_ID"] = "litellm-ci-cd" - secret_val = secret_manager.get_secret_from_google_secret_manager( - secret_name="UNIQUE_KEY" - ) - print("secret_val: {}".format(secret_val)) - assert secret_val == None + with patch( + "litellm.proxy.proxy_server.premium_user", True + ), patch.object( + GoogleSecretManager, + "sync_construct_request_headers", + return_value={"Authorization": "Bearer mock_token"}, + ): + secret_manager = GoogleSecretManager() + secret_manager.cache.cache_dict["UNIQUE_KEY"] = None + secret_manager.cache.cache_dict["UNIQUE_KEY_2"] = "lite-llm" - secret_val = secret_manager.get_secret_from_google_secret_manager( - secret_name="UNIQUE_KEY_2" - ) - print("secret_val: {}".format(secret_val)) - assert secret_val == "lite-llm" + secret_val = secret_manager.get_secret_from_google_secret_manager( + secret_name="UNIQUE_KEY" + ) + print("secret_val: {}".format(secret_val)) + assert secret_val is None + + secret_val = secret_manager.get_secret_from_google_secret_manager( + secret_name="UNIQUE_KEY_2" + ) + print("secret_val: {}".format(secret_val)) + assert secret_val == "lite-llm" def test_should_read_secret_from_secret_manager(): @@ -337,6 +364,7 @@ def test_get_secret_with_access_mode(): litellm._key_management_settings = KeyManagementSettings() del os.environ[test_secret_name] + def test_key_management_settings_defaults(): """ Test that KeyManagementSettings initializes with correct default values. diff --git a/tests/llm_responses_api_testing/test_manus_files_all_methods.py b/tests/llm_responses_api_testing/test_manus_files_all_methods.py deleted file mode 100644 index 39311441f59..00000000000 --- a/tests/llm_responses_api_testing/test_manus_files_all_methods.py +++ /dev/null @@ -1,71 +0,0 @@ -""" -E2E test for all Manus Files API methods. -""" - -import os -import pytest -import litellm - - -@pytest.mark.asyncio -async def test_manus_files_api_e2e_all_methods(): - """ - E2E test for Manus Files API: create, retrieve, list, delete. - """ - litellm._turn_on_debug() - - api_key = os.getenv("MANUS_API_KEY") - if api_key is None: - pytest.skip("MANUS_API_KEY not set") - - # Create a simple test file content - test_content = b"This is a test file for Manus Files API - all methods test." - test_filename = "test_file_all_methods.txt" - - # Step 1: Create file - print("Step 1: Creating file...") - created_file = await litellm.acreate_file( - file=(test_filename, test_content), - purpose="assistants", - custom_llm_provider="manus", - api_key=api_key, - ) - print(f"Created file: {created_file}") - assert created_file.filename == test_filename - assert created_file.status == "uploaded" - # Note: Manus doesn't return bytes in initial response - file_id = created_file.id - - # Step 2: Retrieve file - print(f"\nStep 2: Retrieving file {file_id}...") - retrieved_file = await litellm.afile_retrieve( - file_id=file_id, - custom_llm_provider="manus", - api_key=api_key, - ) - print(f"Retrieved file: {retrieved_file}") - assert retrieved_file.id == file_id - assert retrieved_file.filename == test_filename - - # Step 3: List files - print("\nStep 3: Listing files...") - files_list = await litellm.afile_list( - custom_llm_provider="manus", - api_key=api_key, - ) - print(f"Files list: {files_list}") - assert isinstance(files_list, list) - assert any(f.id == file_id for f in files_list) - - # Step 4: Delete file - print(f"\nStep 4: Deleting file {file_id}...") - deleted_file = await litellm.afile_delete( - file_id=file_id, - custom_llm_provider="manus", - api_key=api_key, - ) - print(f"Deleted file: {deleted_file}") - assert deleted_file.id == file_id - assert deleted_file.deleted is True - - print("\n✅ All Manus Files API methods working!") diff --git a/tests/llm_responses_api_testing/test_manus_responses_api.py b/tests/llm_responses_api_testing/test_manus_responses_api.py deleted file mode 100644 index 6a2aed3812d..00000000000 --- a/tests/llm_responses_api_testing/test_manus_responses_api.py +++ /dev/null @@ -1,127 +0,0 @@ -import os -import sys -import pytest -import asyncio -from typing import Optional -from unittest.mock import patch, AsyncMock - -sys.path.insert(0, os.path.abspath("../..")) -import litellm -from litellm.integrations.custom_logger import CustomLogger -import json -from litellm.types.utils import StandardLoggingPayload -from litellm.types.llms.openai import ( - ResponseCompletedEvent, - ResponsesAPIResponse, - ResponseAPIUsage, - IncompleteDetails, -) -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler -from base_responses_api import BaseResponsesAPITest - - -@pytest.mark.asyncio -async def test_manus_responses_api_with_agent_profile(): - """ - Test that Manus API correctly extracts agent profile from model name - and includes task_mode and agent_profile in the request. - """ - litellm._turn_on_debug() - - response = await litellm.aresponses( - model="manus/manus-1.6-lite", - input="What's the color of the sky?", - api_key=os.getenv("MANUS_API_KEY"), - max_output_tokens=50, - ) - - print("Manus response=", json.dumps(response, indent=4, default=str)) - - ## Get the status of the response - got_response = await litellm.aget_responses( - response_id=response.id, - custom_llm_provider="manus", - api_key=os.getenv("MANUS_API_KEY"), - ) - print("GET API MANUS RESPONSE=", json.dumps(got_response, indent=4, default=str)) - if got_response.status == "completed": - assert got_response.output is not None - assert len(got_response.output) > 0 - - - -@pytest.mark.asyncio -async def test_manus_responses_api_with_file_upload(): - """ - Test that uploads a file via Files API and then passes it to Responses API. - """ - litellm._turn_on_debug() - - api_key = os.getenv("MANUS_API_KEY") - if api_key is None: - pytest.skip("MANUS_API_KEY not set") - - # Step 1: Upload a file - test_content = b"Warren Buffett's 2023 Letter to Shareholders\n\nKey Points:\n1. Long-term value creation\n2. Capital allocation strategy\n3. Market volatility perspective" - test_filename = "buffett_letter_summary.txt" - - print("Step 1: Uploading file...") - uploaded_file = await litellm.acreate_file( - file=(test_filename, test_content), - purpose="assistants", - custom_llm_provider="manus", - api_key=api_key, - ) - print(f"Uploaded file: {uploaded_file}") - assert uploaded_file.id is not None - file_id = uploaded_file.id - - # Step 2: Create a response with the uploaded file - print(f"\nStep 2: Creating response with file {file_id}...") - response = await litellm.aresponses( - model="manus/manus-1.6-lite", - input=[ - { - "role": "user", - "content": [ - { - "type": "input_text", - "text": "Summarize the key points from this letter.", - }, - { - "type": "input_file", - "file_id": file_id, - }, - ], - }, - ], - api_key=api_key, - max_output_tokens=100, - ) - - print(f"Response created: {response}") - print(f"Response type: {type(response)}") - print(f"Response has id: {hasattr(response, 'id')}") - - # Handle both dict and ResponsesAPIResponse object - if isinstance(response, dict): - response_id = response.get("id") - else: - response_id = getattr(response, "id", None) - - assert response_id is not None, f"Response ID is None. Response: {response}" - - - # Step 3: Clean up - delete the file - print(f"\nStep 4: Cleaning up - deleting file {file_id}...") - deleted_file = await litellm.afile_delete( - file_id=file_id, - custom_llm_provider="manus", - api_key=api_key, - ) - print(f"Deleted file: {deleted_file}") - assert deleted_file.deleted is True - - print("\n✅ File upload and responses API integration test passed!") - - diff --git a/tests/llm_translation/test_anthropic_completion.py b/tests/llm_translation/test_anthropic_completion.py index b6a98e4b2c7..fdf8c24ac9e 100644 --- a/tests/llm_translation/test_anthropic_completion.py +++ b/tests/llm_translation/test_anthropic_completion.py @@ -1600,6 +1600,7 @@ def test_anthropic_via_responses_api(): ResponsesAPIStreamEvents.RESPONSE_CREATED, ResponsesAPIStreamEvents.RESPONSE_IN_PROGRESS, ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED, + ResponsesAPIStreamEvents.CONTENT_PART_ADDED, ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, # Can occur multiple times ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE, ResponsesAPIStreamEvents.CONTENT_PART_DONE, diff --git a/tests/llm_translation/test_azure_ai.py b/tests/llm_translation/test_azure_ai.py index b6f53e4e955..d2d893a611b 100644 --- a/tests/llm_translation/test_azure_ai.py +++ b/tests/llm_translation/test_azure_ai.py @@ -34,6 +34,8 @@ from litellm import completion from litellm.integrations.custom_logger import CustomLogger from litellm.types.utils import StandardLoggingPayload +AZURE_AI_API_BASE = os.getenv("AZURE_AI_API_BASE") + @pytest.mark.parametrize( "model_group_header, expected_model", @@ -295,14 +297,15 @@ def test_completion_azure(): litellm.set_verbose = False ## Test azure call response = completion( - model="azure/gpt-4.1-mini", + model="azure_ai/gpt-4.1-mini", + api_base=os.getenv("AZURE_AI_API_BASE"), messages=[ { "role": "user", "content": "Hello, how are you?", } ], - api_key="os.environ/AZURE_AI_API_KEY", + api_key=os.getenv("AZURE_AI_API_KEY"), ) print(f"response: {response}") print(f"response hidden params: {response._hidden_params}") @@ -318,8 +321,8 @@ def test_completion_azure(): @pytest.mark.parametrize( "api_base", [ - "https://litellm-ci-cd-prod.cognitiveservices.azure.com/", - "https://litellm-ci-cd-prod.cognitiveservices.azure.com/openai/deployments/gpt-4.1-mini/chat/completions?api-version=2023-03-15-preview", + AZURE_AI_API_BASE, + f"{AZURE_AI_API_BASE}/openai/deployments/gpt-4.1-mini/chat/completions?api-version=2023-03-15-preview", ], ) def test_completion_azure_ai_gpt_4o_with_flexible_api_base(api_base): @@ -397,7 +400,7 @@ async def test_azure_ai_model_router_streaming_model_in_chunk(): response = await litellm.acompletion( model="azure_ai/azure-model-router", messages=[{"role": "user", "content": "hi"}], - api_base="https://ishaa-mh6uutut-swedencentral.cognitiveservices.azure.com/openai/v1/", + api_base=os.getenv("AZURE_MODEL_ROUTER_API_BASE"), api_key=os.getenv("AZURE_MODEL_ROUTER_API_KEY"), stream=True, ) @@ -475,7 +478,7 @@ async def test_azure_ai_model_router_streaming_cost_with_stream_options(): response = await litellm.acompletion( model="azure_ai/azure-model-router", messages=[{"role": "user", "content": "hi"}], - api_base="https://ishaa-mh6uutut-swedencentral.cognitiveservices.azure.com/openai/v1/", + api_base=os.getenv("AZURE_MODEL_ROUTER_API_BASE"), api_key=os.getenv("AZURE_MODEL_ROUTER_API_KEY"), stream=True, stream_options={"include_usage": True}, diff --git a/tests/llm_translation/test_bedrock_completion.py b/tests/llm_translation/test_bedrock_completion.py index f57c77db54d..c7c179fd49e 100644 --- a/tests/llm_translation/test_bedrock_completion.py +++ b/tests/llm_translation/test_bedrock_completion.py @@ -2472,7 +2472,6 @@ def test_bedrock_error_handling_streaming(): "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf", # "https://raw.githubusercontent.com/datasets/gdp/master/data/gdp.csv", "https://www.cmu.edu/blackboard/files/evaluate/tests-example.xls", - "http://www.krishdholakia.com/", # "https://raw.githubusercontent.com/datasets/sample-data/master/README.txt", # invalid url "https://raw.githubusercontent.com/mdn/content/main/README.md", ], diff --git a/tests/llm_translation/test_cloudflare.py b/tests/llm_translation/test_cloudflare.py index 109e5a86321..5d8e3e5990e 100644 --- a/tests/llm_translation/test_cloudflare.py +++ b/tests/llm_translation/test_cloudflare.py @@ -1,42 +1,145 @@ -import os -import sys -import traceback - -from dotenv import load_dotenv - -load_dotenv() -import io -import os - -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path +import asyncio import json +from typing import Any, Dict +from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest -import litellm -from litellm import RateLimitError, Timeout, completion, completion_cost, embedding +from litellm import acompletion, completion +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler + +FAKE_API_BASE = "https://fake-cloudflare.example.com/client/v4/accounts/fake-acct/ai/run/" +FAKE_API_KEY = "fake-cf-api-key" -# Cloud flare AI test -@pytest.mark.asyncio -@pytest.mark.parametrize("stream", [True, False]) -async def test_completion_cloudflare(stream): - try: - litellm.set_verbose = False - response = await litellm.acompletion( - model="cloudflare/@cf/meta/llama-2-7b-chat-int8", - messages=[{"content": "what llm are you", "role": "user"}], - max_tokens=15, - stream=stream, - ) - print(response) - if stream is True: - async for chunk in response: - print(chunk) - else: - print(response) +def _make_mock_response(json_data: Dict[str, Any]) -> MagicMock: + mock = MagicMock(spec=httpx.Response) + mock.status_code = 200 + mock.headers = {"content-type": "application/json"} + mock.json.return_value = json_data + mock.text = json.dumps(json_data) + return mock - except Exception as e: - pytest.fail(f"Error occurred: {e}") + +def _chat_response() -> Dict[str, Any]: + return { + "result": { + "response": "I am a large language model created to assist you.", + }, + "success": True, + "errors": [], + "messages": [], + } + + +def _streaming_chunks() -> list[str]: + return [ + json.dumps({"response": "I am"}), + json.dumps({"response": " a language"}), + json.dumps({"response": " model."}), + ] + + +@pytest.mark.parametrize("sync_mode", [True, False]) +def test_completion_cloudflare(sync_mode): + messages = [{"role": "user", "content": "what llm are you"}] + mock_resp = _make_mock_response(_chat_response()) + + if sync_mode: + with patch.object(HTTPHandler, "post", return_value=mock_resp) as mock_post: + response = completion( + model="cloudflare/@cf/meta/llama-2-7b-chat-int8", + messages=messages, + max_tokens=15, + api_base=FAKE_API_BASE, + api_key=FAKE_API_KEY, + ) + mock_post.assert_called_once() + else: + with patch.object( + AsyncHTTPHandler, "post", new_callable=AsyncMock, return_value=mock_resp + ) as mock_post: + response = asyncio.run( + acompletion( + model="cloudflare/@cf/meta/llama-2-7b-chat-int8", + messages=messages, + max_tokens=15, + api_base=FAKE_API_BASE, + api_key=FAKE_API_KEY, + ) + ) + mock_post.assert_called_once() + + assert response is not None + assert response.choices[0].message.content is not None + assert "language model" in response.choices[0].message.content.lower() + + +@pytest.mark.parametrize("sync_mode", [True, False]) +def test_completion_cloudflare_stream(sync_mode): + messages = [{"role": "user", "content": "what llm are you"}] + raw_chunks = _streaming_chunks() + + if sync_mode: + + def _iter_lines(): + for chunk in raw_chunks: + yield f"data: {chunk}" + yield "data: [DONE]" + + mock_resp = MagicMock() + mock_resp.iter_lines.return_value = _iter_lines() + mock_resp.status_code = 200 + mock_resp.headers = {"content-type": "text/event-stream"} + + with patch.object(HTTPHandler, "post", return_value=mock_resp) as mock_post: + response = completion( + model="cloudflare/@cf/meta/llama-2-7b-chat-int8", + messages=messages, + max_tokens=15, + stream=True, + api_base=FAKE_API_BASE, + api_key=FAKE_API_KEY, + ) + chunks_received = list(response) + mock_post.assert_called_once() + else: + + async def _aiter_lines(): + for chunk in raw_chunks: + yield f"data: {chunk}" + yield "data: [DONE]" + + mock_resp = MagicMock() + mock_resp.aiter_lines.return_value = _aiter_lines() + mock_resp.status_code = 200 + mock_resp.headers = {"content-type": "text/event-stream"} + + async def _run(): + with patch.object( + AsyncHTTPHandler, "post", new_callable=AsyncMock, return_value=mock_resp + ) as mock_post: + resp = await acompletion( + model="cloudflare/@cf/meta/llama-2-7b-chat-int8", + messages=messages, + max_tokens=15, + stream=True, + api_base=FAKE_API_BASE, + api_key=FAKE_API_KEY, + ) + received = [] + async for chunk in resp: + received.append(chunk) + mock_post.assert_called_once() + return received + + chunks_received = asyncio.run(_run()) + + assert len(chunks_received) > 0 + content = "".join( + c.choices[0].delta.content + for c in chunks_received + if c.choices[0].delta.content + ) + assert "language" in content.lower() diff --git a/tests/llm_translation/test_fireworks_ai_translation.py b/tests/llm_translation/test_fireworks_ai_translation.py index 24c0d546e25..1cc6aabdca8 100644 --- a/tests/llm_translation/test_fireworks_ai_translation.py +++ b/tests/llm_translation/test_fireworks_ai_translation.py @@ -77,18 +77,6 @@ def test_map_response_format(): } -@pytest.mark.skip(reason="fireworks is having an active outage") -class TestFireworksAIChatCompletion(BaseLLMChatTest): - def get_base_completion_call_args(self) -> dict: - return { - "model": "fireworks_ai/accounts/fireworks/models/llama-v3p1-8b-instruct" - } - - def test_tool_call_no_arguments(self, tool_call_no_arguments): - """Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833""" - pass - - class TestFireworksAIAudioTranscription(BaseLLMAudioTranscriptionTest): def get_base_audio_transcription_call_args(self) -> dict: return { @@ -252,7 +240,5 @@ def test_global_disable_flag_with_transform_messages_helper(monkeypatch): json_data = json.loads(mock_post.call_args.kwargs["data"]) assert ( "#transform=inline" - not in json_data["messages"][0]["content"][1]["image_url"][ - "url" - ] + not in json_data["messages"][0]["content"][1]["image_url"]["url"] ) diff --git a/tests/llm_translation/test_gemini.py b/tests/llm_translation/test_gemini.py index b10a7d699c2..1ad71d25a05 100644 --- a/tests/llm_translation/test_gemini.py +++ b/tests/llm_translation/test_gemini.py @@ -532,7 +532,7 @@ def test_gemini_with_grounding(): ## Check streaming response = completion( - model="gemini/gemini-2.0-flash", + model="gemini/gemini-2.5-flash", messages=[{"role": "user", "content": "What is the capital of France?"}], tools=tools, stream=True, @@ -566,7 +566,7 @@ def test_gemini_with_empty_function_call_arguments(): } ] response = completion( - model="gemini/gemini-2.0-flash", + model="gemini/gemini-2.5-flash", messages=[{"role": "user", "content": "What is the capital of France?"}], tools=tools, ) @@ -775,7 +775,7 @@ def test_gemini_tool_use(): {"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "What's the weather like in Lima, Peru today?"}, ], - "model": "gemini/gemini-2.0-flash", + "model": "gemini/gemini-2.5-flash", "tools": [ { "type": "function", diff --git a/tests/llm_translation/test_snowflake.py b/tests/llm_translation/test_snowflake.py index 63878c8fc69..6861c2c7eca 100644 --- a/tests/llm_translation/test_snowflake.py +++ b/tests/llm_translation/test_snowflake.py @@ -1,12 +1,10 @@ -import os -import sys +import asyncio import json +import os import httpx from typing import Any, Dict, List -from unittest.mock import Mock, MagicMock, patch -from dotenv import load_dotenv +from unittest.mock import AsyncMock, MagicMock, patch -load_dotenv() import pytest from litellm import completion, acompletion, responses @@ -14,202 +12,68 @@ from litellm.exceptions import APIConnectionError from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler -def mock_snowflake_chat_response() -> Dict[str, Any]: +@pytest.mark.skip(reason="Requires Snowflake credentials - run manually when needed") +def test_snowflake_tool_calling_responses_api(): """ - Mock response for Snowflake chat completion. + Test Snowflake tool calling with Responses API. + Requires SNOWFLAKE_JWT and SNOWFLAKE_ACCOUNT_ID environment variables. """ - return { - "id": "chatcmpl-snowflake-123", - "object": "chat.completion", - "created": 1700000000, - "model": "mistral-7b", - "choices": [ - { - "index": 0, - "message": { - "role": "assistant", - "content": "The sky above is painted blue,\nWith clouds of white and morning dew.\nA canvas vast, serene and bright,\nThat fills my heart with pure delight.", + import litellm + + # Skip if credentials not available + if not os.getenv("SNOWFLAKE_JWT") or not os.getenv("SNOWFLAKE_ACCOUNT_ID"): + pytest.skip("Snowflake credentials not available") + + litellm.drop_params = False # We now support tools! + + tools = [ + { + "type": "function", + "name": "get_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + } }, - "finish_reason": "stop", - } - ], - "usage": { - "prompt_tokens": 10, - "completion_tokens": 30, - "total_tokens": 40, - }, - } - - -def mock_snowflake_streaming_response_chunks() -> List[str]: - """ - Mock streaming response chunks for Snowflake. - """ - return [ - json.dumps( - { - "id": "chatcmpl-snowflake-stream-123", - "object": "chat.completion.chunk", - "created": 1700000000, - "model": "mistral-7b", - "choices": [ - { - "index": 0, - "delta": {"role": "assistant", "content": "The"}, - "finish_reason": None, - } - ], - } - ), - json.dumps( - { - "id": "chatcmpl-snowflake-stream-123", - "object": "chat.completion.chunk", - "created": 1700000000, - "model": "mistral-7b", - "choices": [ - { - "index": 0, - "delta": {"content": " sky"}, - "finish_reason": None, - } - ], - } - ), - json.dumps( - { - "id": "chatcmpl-snowflake-stream-123", - "object": "chat.completion.chunk", - "created": 1700000000, - "model": "mistral-7b", - "choices": [ - { - "index": 0, - "delta": {"content": " is blue"}, - "finish_reason": "stop", - } - ], - } - ), + "required": ["location"], + }, + } ] + try: + # Test with tool_choice to force tool use + response = responses( + model="snowflake/claude-3-5-sonnet", + input="What's the weather in Paris?", + tools=tools, + tool_choice={"type": "function", "function": {"name": "get_weather"}}, + max_output_tokens=200, + ) -@pytest.mark.parametrize("sync_mode", [True, False]) -def test_chat_completion_snowflake(sync_mode): - """ - Test Snowflake chat completion with mocked HTTP responses. - """ - messages = [ - { - "role": "user", - "content": "Write me a poem about the blue sky", - }, - ] + assert response is not None + assert hasattr(response, "output") + assert len(response.output) > 0 - mock_response = Mock(spec=httpx.Response) - mock_response.status_code = 200 - mock_response.json.return_value = mock_snowflake_chat_response() + # Verify tool call was made + tool_call_found = False + for item in response.output: + if hasattr(item, "type") and item.type == "function_call": + tool_call_found = True + assert item.name == "get_weather" + assert hasattr(item, "arguments") + print(f"✅ Tool call detected: {item.name}({item.arguments})") + break - if sync_mode: - sync_handler = HTTPHandler() - with patch.object(HTTPHandler, "post", return_value=mock_response): - response = completion( - model="snowflake/mistral-7b", - messages=messages, - api_base="https://exampleopenaiendpoint-production.up.railway.app/v1/chat/completions", - client=sync_handler, - ) - assert response is not None - assert response.choices[0].message.content is not None - assert "sky" in response.choices[0].message.content.lower() - else: - async_handler = AsyncHTTPHandler() - with patch.object(AsyncHTTPHandler, "post", return_value=mock_response): - import asyncio + assert tool_call_found, "Expected tool call but none was found" - response = asyncio.run( - acompletion( - model="snowflake/mistral-7b", - messages=messages, - api_base="https://exampleopenaiendpoint-production.up.railway.app/v1/chat/completions", - client=async_handler, - ) - ) - assert response is not None - assert response.choices[0].message.content is not None - assert "sky" in response.choices[0].message.content.lower() - - -@pytest.mark.parametrize("sync_mode", [True, False]) -def test_chat_completion_snowflake_stream(sync_mode): - """ - Test Snowflake streaming chat completion with mocked HTTP responses. - """ - messages = [ - { - "role": "user", - "content": "Write me a poem about the blue sky", - }, - ] - - if sync_mode: - sync_handler = HTTPHandler() - mock_chunks = mock_snowflake_streaming_response_chunks() - - def mock_iter_lines(): - for chunk in mock_chunks: - for line in [f"data: {chunk}", "data: [DONE]"]: - yield line - - mock_response = MagicMock() - mock_response.iter_lines.side_effect = mock_iter_lines - mock_response.status_code = 200 - - with patch.object(HTTPHandler, "post", return_value=mock_response): - response = completion( - model="snowflake/mistral-7b", - messages=messages, - max_tokens=100, - stream=True, - api_base="https://exampleopenaiendpoint-production.up.railway.app/v1/chat/completions", - client=sync_handler, - ) - - chunks_received = [] - for chunk in response: - chunks_received.append(chunk) - - assert len(chunks_received) > 0 - else: - async_handler = AsyncHTTPHandler() - mock_chunks = mock_snowflake_streaming_response_chunks() - - async def mock_iter_lines(): - for chunk in mock_chunks: - for line in [f"data: {chunk}", "data: [DONE]"]: - yield line - - mock_response = MagicMock() - mock_response.iter_lines.side_effect = mock_iter_lines - mock_response.status_code = 200 - - with patch.object(AsyncHTTPHandler, "post", return_value=mock_response): - import asyncio - - async def test_async_stream(): - response = await acompletion( - model="snowflake/mistral-7b", - messages=messages, - max_tokens=100, - stream=True, - api_base="https://exampleopenaiendpoint-production.up.railway.app/v1/chat/completions", - client=async_handler, - ) - - chunks_received = [] - async for chunk in response: - chunks_received.append(chunk) - - assert len(chunks_received) > 0 - - asyncio.run(test_async_stream()) + except APIConnectionError as e: + if "JWT token is invalid" in str(e): + pytest.skip("Invalid Snowflake JWT token") + elif "Application failed to respond" in str(e) or "502" in str(e): + pytest.skip(f"Snowflake API unavailable: {e}") + else: + raise diff --git a/tests/llm_translation/test_watsonx.py b/tests/llm_translation/test_watsonx.py index d6a82b44969..ce02d3aac6f 100644 --- a/tests/llm_translation/test_watsonx.py +++ b/tests/llm_translation/test_watsonx.py @@ -13,6 +13,16 @@ import pytest from typing import Optional +@pytest.fixture(autouse=True) +def watsonx_env_vars(monkeypatch): + """Set required WatsonX env vars so the provider passes validation. + Also clear WATSONX_ZENAPIKEY/WATSONX_TOKEN so they don't bypass the IAM token mock.""" + monkeypatch.setenv("WATSONX_URL", "https://us-south.ml.cloud.ibm.com") + monkeypatch.setenv("WATSONX_PROJECT_ID", "test-project-id") + monkeypatch.delenv("WATSONX_ZENAPIKEY", raising=False) + monkeypatch.delenv("WATSONX_TOKEN", raising=False) + + @pytest.fixture def watsonx_chat_completion_call(): def _call( diff --git a/tests/local_testing/adroit-crow-413218-bc47f303efc9.json b/tests/local_testing/adroit-crow-413218-bc47f303efc9.json deleted file mode 100644 index 7e02c821360..00000000000 --- a/tests/local_testing/adroit-crow-413218-bc47f303efc9.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "type": "service_account", - "project_id": "pathrise-convert-1606954137718", - "private_key_id": "", - "private_key": "", - "client_email": "test-adroit-crow@pathrise-convert-1606954137718.iam.gserviceaccount.com", - "client_id": "104886546564708740969", - "auth_uri": "https://accounts.google.com/o/oauth2/auth", - "token_uri": "https://oauth2.googleapis.com/token", - "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", - "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/test-adroit-crow%40pathrise-convert-1606954137718.iam.gserviceaccount.com", - "universe_domain": "googleapis.com" -} diff --git a/tests/local_testing/test_amazing_vertex_completion.py b/tests/local_testing/test_amazing_vertex_completion.py index 6f7c371bdb5..a1684e23769 100644 --- a/tests/local_testing/test_amazing_vertex_completion.py +++ b/tests/local_testing/test_amazing_vertex_completion.py @@ -167,43 +167,6 @@ async def test_get_response(): pytest.fail(f"An error occurred - {str(e)}") -@pytest.mark.skip( - reason="Local test. Vertex AI Quota is low. Leads to rate limit errors on ci/cd." -) -@pytest.mark.flaky(retries=3, delay=1) -def test_vertex_ai_anthropic_streaming(): - try: - load_vertex_ai_credentials() - - # litellm.set_verbose = True - - model = "claude-3-5-sonnet@20240620" - - vertex_ai_project = "pathrise-convert-1606954137718" - vertex_ai_location = "asia-southeast1" - json_obj = get_vertex_ai_creds_json() - vertex_credentials = json.dumps(json_obj) - - response = completion( - model="vertex_ai/" + model, - messages=[{"role": "user", "content": "hi"}], - temperature=0.7, - vertex_ai_project=vertex_ai_project, - vertex_ai_location=vertex_ai_location, - stream=True, - ) - # print("\nModel Response", response) - for idx, chunk in enumerate(response): - print(f"chunk: {chunk}") - streaming_format_tests(idx=idx, chunk=chunk) - - # raise Exception("it worked!") - except litellm.RateLimitError as e: - pass - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - # test_vertex_ai_anthropic_streaming() @@ -394,10 +357,7 @@ async def test_async_vertexai_response_basic(): user_message = "Hello, how are you?" messages = [{"content": user_message, "role": "user"}] response = await acompletion( - model="gemini-2.5-flash", - messages=messages, - temperature=0.7, - timeout=5 + model="gemini-2.5-flash", messages=messages, temperature=0.7, timeout=5 ) print(f"response: {response}") except litellm.NotFoundError as e: @@ -414,8 +374,6 @@ async def test_async_vertexai_response_basic(): pytest.fail(f"An exception occurred: {e}") - - @pytest.mark.flaky(retries=3, delay=1) @pytest.mark.asyncio async def test_async_vertexai_streaming_response(): @@ -739,7 +697,9 @@ def test_gemini_pro_grounding(value_in_dict): # @pytest.mark.skip(reason="exhausted vertex quota. need to refactor to mock the call") -@pytest.mark.parametrize("model", ["vertex_ai_beta/gemini-2.5-flash-lite"]) # "vertex_ai", +@pytest.mark.parametrize( + "model", ["vertex_ai_beta/gemini-2.5-flash-lite"] +) # "vertex_ai", @pytest.mark.parametrize("sync_mode", [True]) # "vertex_ai", @pytest.mark.asyncio @pytest.mark.flaky(retries=6, delay=2) @@ -808,7 +768,17 @@ async def test_gemini_pro_function_calling_httpx(model, sync_mode): except Exception as e: error_msg = str(e) # Skip test for known transient API issues - if any(x in error_msg for x in ["429 Quota exceeded", "503", "Service unavailable", "timeout", "Timeout", "UNAVAILABLE"]): + if any( + x in error_msg + for x in [ + "429 Quota exceeded", + "503", + "Service unavailable", + "timeout", + "Timeout", + "UNAVAILABLE", + ] + ): pytest.skip(f"Transient API error: {error_msg}") else: pytest.fail(f"An unexpected exception occurred - {error_msg}") @@ -1396,12 +1366,14 @@ async def test_gemini_pro_json_schema_args_sent_httpx( # Gemini 2.x+ uses response_json_schema, Gemini 1.x uses response_schema gen_config = mock_call.call_args.kwargs["json"]["generationConfig"] assert ( - "response_schema" in gen_config or "response_json_schema" in gen_config + "response_schema" in gen_config + or "response_json_schema" in gen_config ), f"Expected response_schema or response_json_schema in {gen_config}" else: gen_config = mock_call.call_args.kwargs["json"]["generationConfig"] assert ( - "response_schema" not in gen_config and "response_json_schema" not in gen_config + "response_schema" not in gen_config + and "response_json_schema" not in gen_config ) assert ( "Use this JSON schema:" @@ -1577,7 +1549,8 @@ async def test_gemini_pro_json_schema_args_sent_httpx_openai_schema( # Gemini 2.x+ uses response_json_schema, Gemini 1.x uses response_schema gen_config = mock_call.call_args.kwargs["json"]["generationConfig"] assert ( - "response_schema" in gen_config or "response_json_schema" in gen_config + "response_schema" in gen_config + or "response_json_schema" in gen_config ), f"Expected response_schema or response_json_schema in {gen_config}" assert ( "response_mime_type" @@ -1592,7 +1565,8 @@ async def test_gemini_pro_json_schema_args_sent_httpx_openai_schema( else: gen_config = mock_call.call_args.kwargs["json"]["generationConfig"] assert ( - "response_schema" not in gen_config and "response_json_schema" not in gen_config + "response_schema" not in gen_config + and "response_json_schema" not in gen_config ) assert ( "Use this JSON schema:" @@ -2313,8 +2287,6 @@ def test_prompt_factory_nested(): ), "'text' value not a string." - - @pytest.mark.asyncio async def test_completion_fine_tuned_model(): load_vertex_ai_credentials() @@ -2380,7 +2352,7 @@ async def test_completion_fine_tuned_model(): # this is the fine-tuned model endpoint assert ( url[0] - == "https://us-central1-aiplatform.googleapis.com/v1/projects/pathrise-convert-1606954137718/locations/us-central1/endpoints/4965075652664360960:generateContent" + == "https://us-central1-aiplatform.googleapis.com/v1/projects/litellm-ci-cd/locations/us-central1/endpoints/4965075652664360960:generateContent" ) print("call args = ", kwargs) @@ -2579,20 +2551,20 @@ async def test_gemini_context_caching_anthropic_format(sync_mode): async def test_gemini_context_caching_disabled_flag(sync_mode): """ Test that disable_anthropic_gemini_context_caching_transform flag properly disables context caching. - + When the flag is set to True, messages with cache_control should not trigger caching API calls. """ from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler litellm.set_verbose = True - + # Store original value to restore later original_flag_value = litellm.disable_anthropic_gemini_context_caching_transform - + try: # Enable the disable flag litellm.disable_anthropic_gemini_context_caching_transform = True - + gemini_context_caching_messages = [ # System Message with cache_control { @@ -2633,13 +2605,15 @@ async def test_gemini_context_caching_disabled_flag(sync_mode): ], }, ] - + if sync_mode: client = HTTPHandler(concurrent_limit=1) else: client = AsyncHTTPHandler(concurrent_limit=1) - - with patch.object(client, "post", side_effect=mock_gemini_request) as mock_client: + + with patch.object( + client, "post", side_effect=mock_gemini_request + ) as mock_client: try: if sync_mode: response = litellm.completion( @@ -2662,24 +2636,32 @@ async def test_gemini_context_caching_disabled_flag(sync_mode): print(e) # When caching is disabled, should only make 1 call (no separate cache creation call) - assert mock_client.call_count == 1, f"Expected 1 call when caching is disabled, got {mock_client.call_count}" + assert ( + mock_client.call_count == 1 + ), f"Expected 1 call when caching is disabled, got {mock_client.call_count}" first_call_args = mock_client.call_args_list[0].kwargs first_call_positional_args = mock_client.call_args_list[0].args print(f"first_call_args with caching disabled: {first_call_args}") - print(f"first_call_positional_args with caching disabled: {first_call_positional_args}") + print( + f"first_call_positional_args with caching disabled: {first_call_positional_args}" + ) # Assert that cachedContents is NOT in the URL when caching is disabled - url = first_call_args.get("url", first_call_positional_args[0] if first_call_positional_args else "") - assert "cachedContents" not in url, "cachedContents should not be in URL when caching is disabled" - + url = first_call_args.get( + "url", + first_call_positional_args[0] if first_call_positional_args else "", + ) + assert ( + "cachedContents" not in url + ), "cachedContents should not be in URL when caching is disabled" + finally: # Restore original flag value litellm.disable_anthropic_gemini_context_caching_transform = original_flag_value - @pytest.mark.asyncio async def test_partner_models_httpx_ai21(): litellm.set_verbose = True @@ -2776,7 +2758,7 @@ async def test_partner_models_httpx_ai21(): assert ( url[0] - == "https://us-central1-aiplatform.googleapis.com/v1beta1/projects/pathrise-convert-1606954137718/locations/us-central1/publishers/ai21/models/jamba-1.5-mini@001:rawPredict" + == "https://us-central1-aiplatform.googleapis.com/v1beta1/projects/litellm-ci-cd/locations/us-central1/publishers/ai21/models/jamba-1.5-mini@001:rawPredict" ) # json loads kwargs @@ -2920,7 +2902,9 @@ def test_gemini_function_call_parameter_in_messages(): "contents": [ { "role": "user", - "parts": [{"text": "search for weather in boston (use `search`)"}], + "parts": [ + {"text": "search for weather in boston (use `search`)"} + ], }, { "role": "model", @@ -2947,7 +2931,9 @@ def test_gemini_function_call_parameter_in_messages(): ], }, ], - "system_instruction": {"parts": [{"text": "Use search for most queries."}]}, + "system_instruction": { + "parts": [{"text": "Use search for most queries."}] + }, "tools": [ { "function_declarations": [ @@ -3578,8 +3564,9 @@ def test_gemini_tool_calling_working_demo(): }, } ], + "vertex_location": "global", } - response = completion(model="vertex_ai/gemini-2.0-flash", **args) + response = completion(model="vertex_ai/gemini-3-flash-preview", **args) print(response) @@ -3650,8 +3637,9 @@ def test_gemini_tool_calling_not_working(): }, } ], + "vertex_location": "global", } - response = completion(model="vertex_ai/gemini-2.0-flash", **args) + response = completion(model="vertex_ai/gemini-3-flash-preview", **args) print(response) @@ -3746,8 +3734,8 @@ def test_gemini_nullable_object_tool_schema_httpx(): load_vertex_ai_credentials() litellm._turn_on_debug() - - tools = [{ + tools = [ + { "type": "function", "strict": True, "function": { @@ -3760,7 +3748,7 @@ def test_gemini_nullable_object_tool_schema_httpx(): "properties": { "ticket_id": { "type": "string", - "description": "Unique identifier for the support ticket" + "description": "Unique identifier for the support ticket", }, "customer_context": { "type": ["object", "null"], @@ -3770,18 +3758,19 @@ def test_gemini_nullable_object_tool_schema_httpx(): "properties": { "user_id": { "type": "string", - "description": "Internal user identifier" + "description": "Internal user identifier", }, "plan": { "type": "string", - "description": "Subscription plan name (e.g. pro, enterprise)" - } - } - } - } - } - } - }] + "description": "Subscription plan name (e.g. pro, enterprise)", + }, + }, + }, + }, + }, + }, + } + ] response = litellm.completion( model="vertex_ai/gemini-2.5-flash", @@ -3986,17 +3975,23 @@ def test_vertex_ai_gemini_audio_ogg(): for part in content["parts"] if "file_data" in part ] - assert len(file_data_parts) == 1, f"Expected 1 file_data part, got: {file_data_parts}" + assert ( + len(file_data_parts) == 1 + ), f"Expected 1 file_data part, got: {file_data_parts}" file_data = file_data_parts[0]["file_data"] - assert file_data["mime_type"] == "audio/ogg", f"Expected audio/ogg, got: {file_data['mime_type']}" - assert "En-us-public.ogg" in file_data["file_uri"], f"Unexpected file_uri: {file_data['file_uri']}" + assert ( + file_data["mime_type"] == "audio/ogg" + ), f"Expected audio/ogg, got: {file_data['mime_type']}" + assert ( + "En-us-public.ogg" in file_data["file_uri"] + ), f"Unexpected file_uri: {file_data['file_uri']}" print(response) @pytest.mark.asyncio async def test_vertex_ai_deepseek(): """Test that deepseek models use the correct v1 API endpoint instead of v1beta1.""" - # load_vertex_ai_credentials() + load_vertex_ai_credentials() litellm._turn_on_debug() from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler @@ -4041,7 +4036,7 @@ def test_gemini_grounding_on_streaming(): load_vertex_ai_credentials() # litellm._turn_on_debug() args = { - "model": "vertex_ai/gemini-2.0-flash", + "model": "vertex_ai/gemini-3-flash-preview", "messages": [ { "role": "user", @@ -4053,6 +4048,7 @@ def test_gemini_grounding_on_streaming(): ], } ], + "vertex_location": "global", "stream": True, "tools": [{"googleSearch": {}}], "fallbacks": [], @@ -4075,11 +4071,20 @@ def test_gemini_google_maps_tool_simple(): litellm._turn_on_debug() tools = [{"googleMaps": {"enableWidget": True}}] - tools_with_location = [{"googleMaps": {"enableWidget": True, "latitude": 37.7749, "longitude": -122.4194, "languageCode": "en_US"}}] + tools_with_location = [ + { + "googleMaps": { + "enableWidget": True, + "latitude": 37.7749, + "longitude": -122.4194, + "languageCode": "en_US", + } + } + ] try: for tools in [tools, tools_with_location]: response = completion( - model="vertex_ai/gemini-2.0-flash", + model="vertex_ai/gemini-3-flash-preview", messages=[ { "role": "user", @@ -4087,6 +4092,7 @@ def test_gemini_google_maps_tool_simple(): } ], tools=tools, + vertex_location="global", ) print(f"Response: {response.model_dump_json(indent=4)}") assert response.choices[0].message.content is not None @@ -4094,4 +4100,3 @@ def test_gemini_google_maps_tool_simple(): pass except Exception as e: pytest.fail(f"Error occurred: {e}") - diff --git a/tests/local_testing/test_assistants.py b/tests/local_testing/test_assistants.py index 93ebba3c345..f5e8540a4c7 100644 --- a/tests/local_testing/test_assistants.py +++ b/tests/local_testing/test_assistants.py @@ -288,6 +288,8 @@ async def test_aarun_thread_litellm(sync_mode, provider, is_streaming): thread_id=_new_thread.id, custom_llm_provider=provider ) assert isinstance(messages.data[0], Message) + elif run.status == "failed" and run.last_error and "No connection matching model" in run.last_error.message: + pytest.skip(f"Azure deployment not found: {run.last_error.message}") else: pytest.fail( "An unexpected error occurred when running the thread, {}".format( @@ -320,6 +322,8 @@ async def test_aarun_thread_litellm(sync_mode, provider, is_streaming): thread_id=_new_thread.id, custom_llm_provider=provider ) assert isinstance(messages.data[0], Message) + elif run.status == "failed" and run.last_error and "No connection matching model" in run.last_error.message: + pytest.skip(f"Azure deployment not found: {run.last_error.message}") else: pytest.fail( "An unexpected error occurred when running the thread, {}".format( diff --git a/tests/local_testing/test_caching_ssl.py b/tests/local_testing/test_caching_ssl.py index a8a38e747a7..21782963250 100644 --- a/tests/local_testing/test_caching_ssl.py +++ b/tests/local_testing/test_caching_ssl.py @@ -69,10 +69,10 @@ def test_caching_router(): ] litellm.cache = Cache( type="redis", - host="os.environ/REDIS_HOST_2", - port="os.environ/REDIS_PORT_2", - password="os.environ/REDIS_PASSWORD_2", - ssl="os.environ/REDIS_SSL_2", + host="os.environ/REDIS_HOST", + port="os.environ/REDIS_PORT", + password="os.environ/REDIS_PASSWORD", + ssl="os.environ/REDIS_SSL", ) router = Router( model_list=model_list, diff --git a/tests/local_testing/test_completion.py b/tests/local_testing/test_completion.py index d5da0719e0e..ef34c9f85b0 100644 --- a/tests/local_testing/test_completion.py +++ b/tests/local_testing/test_completion.py @@ -868,78 +868,6 @@ def test_completion_mistral_api_modified_input(): pytest.fail(f"Error occurred: {e}") -# def test_completion_oobabooga(): -# try: -# response = completion( -# model="oobabooga/vicuna-1.3b", messages=messages, api_base="http://127.0.0.1:5000" -# ) -# # Add any assertions here to check the response -# print(response) -# except Exception as e: -# pytest.fail(f"Error occurred: {e}") - -# test_completion_oobabooga() -# aleph alpha -# def test_completion_aleph_alpha(): -# try: -# response = completion( -# model="luminous-base", messages=messages, logger_fn=logger_fn -# ) -# # Add any assertions here to check the response -# print(response) -# except Exception as e: -# pytest.fail(f"Error occurred: {e}") -# test_completion_aleph_alpha() - - -# def test_completion_aleph_alpha_control_models(): -# try: -# response = completion( -# model="luminous-base-control", messages=messages, logger_fn=logger_fn -# ) -# # Add any assertions here to check the response -# print(response) -# except Exception as e: -# pytest.fail(f"Error occurred: {e}") -# test_completion_aleph_alpha_control_models() - -import openai - - -def test_completion_gpt4_turbo(): - litellm.set_verbose = True - try: - response = completion( - model="gpt-4-1106-preview", - messages=messages, - max_completion_tokens=10, - ) - print(response) - except openai.RateLimitError: - print("got a rate liimt error") - pass - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -# test_completion_gpt4_turbo() - - -def test_completion_gpt4_turbo_0125(): - try: - response = completion( - model="gpt-4-0125-preview", - messages=messages, - max_tokens=10, - ) - print(response) - except openai.RateLimitError: - print("got a rate liimt error") - pass - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - @pytest.mark.skip(reason="this test is flaky") def test_completion_gpt4_vision(): try: @@ -2438,7 +2366,7 @@ def test_azure_openai_ad_token(): # test_azure_openai_ad_token() -# test_completion_azure() + def test_completion_azure2(): # test if we can pass api_base, api_version and api_key in compleition() try: @@ -2560,7 +2488,6 @@ def test_completion_azure_with_litellm_key(): pytest.fail(f"Error occurred: {e}") -# test_completion_azure() import asyncio @@ -3292,6 +3219,13 @@ def test_petals(): ## test deep infra @pytest.mark.parametrize("drop_params", [True, False]) def test_completion_deep_infra(drop_params): + """Test that DeepInfra requests are shaped correctly without making real API calls.""" + from unittest.mock import MagicMock, patch + from openai import OpenAI + from openai.types.chat import ChatCompletion, ChatCompletionMessage + from openai.types.chat.chat_completion import Choice + import httpx + litellm.set_verbose = False model_name = "deepinfra/meta-llama/Llama-2-70b-chat-hf" tools = [ @@ -3320,7 +3254,51 @@ def test_completion_deep_infra(drop_params): "content": "What's the weather like in Boston today in Fahrenheit?", } ] - try: + + mock_response = ChatCompletion( + id="chatcmpl-mock", + choices=[ + Choice( + finish_reason="stop", + index=0, + message=ChatCompletionMessage( + content="It's sunny.", role="assistant" + ), + ) + ], + created=1234567890, + model="meta-llama/Llama-2-70b-chat-hf", + object="chat.completion", + usage={"completion_tokens": 5, "prompt_tokens": 20, "total_tokens": 25}, + ) + + mock_raw = MagicMock() + mock_raw.parse.return_value = mock_response + mock_raw.headers = httpx.Headers({"content-type": "application/json"}) + mock_raw.status_code = 200 + + with patch( + "litellm.llms.openai.openai.OpenAIChatCompletion.make_sync_openai_chat_completion_request", + return_value=(mock_raw, mock_response), + ) as mock_create: + if drop_params is False: + # DeepInfra doesn't support tool_choice, should raise UnsupportedParamsError + with pytest.raises(litellm.exceptions.UnsupportedParamsError): + completion( + model=model_name, + messages=messages, + temperature=0, + max_tokens=10, + tools=tools, + tool_choice={ + "type": "function", + "function": {"name": "get_current_weather"}, + }, + drop_params=drop_params, + api_key="fake-api-key", + ) + return + response = completion( model=model_name, messages=messages, @@ -3332,33 +3310,75 @@ def test_completion_deep_infra(drop_params): "function": {"name": "get_current_weather"}, }, drop_params=drop_params, + api_key="fake-api-key", ) - # Add any assertions here to check the response - print(response) - except Exception as e: - if drop_params is True: - pytest.fail(f"Error occurred: {e}") + + # Verify the call was made + mock_create.assert_called_once() + call_kwargs = mock_create.call_args.kwargs + + # Verify request shape + data = call_kwargs["data"] + assert data["model"] == "meta-llama/Llama-2-70b-chat-hf" + assert data["messages"] == messages + assert data["temperature"] == 0 + assert data["max_tokens"] == 10 + # tool_choice should be dropped for unsupported params + assert "tool_choice" not in data # test_completion_deep_infra() def test_completion_deep_infra_mistral(): - print("deep infra test with temp=0") + """Test that DeepInfra Mistral requests are shaped correctly without making real API calls.""" + from unittest.mock import MagicMock, patch + from openai.types.chat import ChatCompletion, ChatCompletionMessage + from openai.types.chat.chat_completion import Choice + import httpx + model_name = "deepinfra/mistralai/Mistral-7B-Instruct-v0.1" - try: + + mock_response = ChatCompletion( + id="chatcmpl-mock", + choices=[ + Choice( + finish_reason="stop", + index=0, + message=ChatCompletionMessage( + content="Hello!", role="assistant" + ), + ) + ], + created=1234567890, + model="mistralai/Mistral-7B-Instruct-v0.1", + object="chat.completion", + usage={"completion_tokens": 5, "prompt_tokens": 20, "total_tokens": 25}, + ) + + mock_raw = MagicMock() + mock_raw.parse.return_value = mock_response + mock_raw.headers = httpx.Headers({"content-type": "application/json"}) + mock_raw.status_code = 200 + + with patch( + "litellm.llms.openai.openai.OpenAIChatCompletion.make_sync_openai_chat_completion_request", + return_value=(mock_raw, mock_response), + ) as mock_create: response = completion( model=model_name, messages=messages, - temperature=0.01, # mistrail fails with temperature=0 + temperature=0.01, max_tokens=10, + api_key="fake-api-key", ) - # Add any assertions here to check the response - print(response) - except litellm.exceptions.Timeout as e: - pass - except Exception as e: - pytest.fail(f"Error occurred: {e}") + + mock_create.assert_called_once() + call_kwargs = mock_create.call_args.kwargs + data = call_kwargs["data"] + assert data["model"] == "mistralai/Mistral-7B-Instruct-v0.1" + assert data["temperature"] == 0.01 + assert data["max_tokens"] == 10 # test_completion_deep_infra_mistral() diff --git a/tests/local_testing/test_embedding.py b/tests/local_testing/test_embedding.py index e6776e3329b..c43cad78b1e 100644 --- a/tests/local_testing/test_embedding.py +++ b/tests/local_testing/test_embedding.py @@ -549,7 +549,7 @@ def test_bedrock_embedding_cohere(): "good morning from litellm, attempting to embed data", "lets test a second string for good measure", ], - aws_region_name="os.environ/AWS_REGION_NAME", + aws_region_name="us-west-2", ) assert isinstance( response["data"][0]["embedding"], list diff --git a/tests/local_testing/test_exceptions.py b/tests/local_testing/test_exceptions.py index 123442d1cf6..e02d9e21171 100644 --- a/tests/local_testing/test_exceptions.py +++ b/tests/local_testing/test_exceptions.py @@ -24,7 +24,7 @@ from litellm import ( # AuthenticationError,; RateLimitError,; ServiceUnavailab embedding, ) -litellm.vertex_project = "pathrise-convert-1606954137718" +litellm.vertex_project = "litellm-ci-cd" litellm.vertex_location = "us-central1" litellm.num_retries = 0 diff --git a/tests/local_testing/test_gcs_bucket.py b/tests/local_testing/test_gcs_bucket.py index 018a79b53f0..9d72ff873ad 100644 --- a/tests/local_testing/test_gcs_bucket.py +++ b/tests/local_testing/test_gcs_bucket.py @@ -21,177 +21,68 @@ from litellm.integrations.gcs_bucket.gcs_bucket import ( StandardLoggingPayload, ) from litellm.types.utils import StandardCallbackDynamicParams -from unittest.mock import patch +from litellm.types.integrations.gcs_bucket import GCSLoggingConfig +from unittest.mock import patch, AsyncMock, MagicMock verbose_logger.setLevel(logging.DEBUG) -def load_vertex_ai_credentials(): - # Define the path to the vertex_key.json file - print("loading vertex ai credentials") - os.environ["GCS_FLUSH_INTERVAL"] = "1" - os.environ["GCS_USE_BATCHED_LOGGING"] = "false" - filepath = os.path.dirname(os.path.abspath(__file__)) - vertex_key_path = filepath + "/vertex_key.json" - - # Read the existing content of the file or create an empty dictionary - try: - with open(vertex_key_path, "r") as file: - # Read the file content - print("Read vertexai file path") - content = file.read() - - # If the file is empty or not valid JSON, create an empty dictionary - if not content or not content.strip(): - service_account_key_data = {} - else: - # Attempt to load the existing JSON content - file.seek(0) - service_account_key_data = json.load(file) - except FileNotFoundError: - # If the file doesn't exist, create an empty dictionary - service_account_key_data = {} - - # Update the service_account_key_data with environment variables - private_key_id = os.environ.get("VERTEX_AI_PRIVATE_KEY_ID", "") - private_key = os.environ.get("VERTEX_AI_PRIVATE_KEY", "") - private_key = private_key.replace("\\n", "\n") - service_account_key_data["private_key_id"] = private_key_id - service_account_key_data["private_key"] = private_key - - # Create a temporary file - with tempfile.NamedTemporaryFile(mode="w+", delete=False) as temp_file: - # Write the updated content to the temporary files - json.dump(service_account_key_data, temp_file, indent=2) - - # Export the temporary file as GOOGLE_APPLICATION_CREDENTIALS - os.environ["GCS_PATH_SERVICE_ACCOUNT"] = os.path.abspath(temp_file.name) - print("created gcs path service account=", os.environ["GCS_PATH_SERVICE_ACCOUNT"]) +def _make_mock_gcs_logging_config(): + return GCSLoggingConfig( + bucket_name="test-bucket", + vertex_instance=MagicMock(), + path_service_account=None, + ) @pytest.mark.asyncio async def test_aaabasic_gcs_logger(): - load_vertex_ai_credentials() - gcs_logger = GCSBucketLogger() - print("GCSBucketLogger", gcs_logger) + os.environ["GCS_FLUSH_INTERVAL"] = "1" + os.environ["GCS_USE_BATCHED_LOGGING"] = "false" + os.environ["GCS_BUCKET_NAME"] = "test-bucket" - litellm.callbacks = [gcs_logger] - response = await litellm.acompletion( - model="gpt-3.5-turbo", - temperature=0.7, - messages=[{"role": "user", "content": "This is a test"}], - max_tokens=10, - user="ishaan-2", - mock_response="Hi!", - metadata={ - "tags": ["model-anthropic-claude-v2.1", "app-ishaan-prod"], - "user_api_key": "a1b2c3d4e5f6789012345678901234567890abcdef1234567890abcdef123456", - "user_api_key_alias": None, - "user_api_end_user_max_budget": None, - "litellm_api_version": "0.0.0", - "global_max_parallel_requests": None, - "user_api_key_user_id": "116544810872468347480", - "user_api_key_org_id": None, - "user_api_key_team_id": None, - "user_api_key_team_alias": None, - "user_api_key_metadata": {}, - "requester_ip_address": "127.0.0.1", - "requester_metadata": {"foo": "bar"}, - "spend_logs_metadata": {"hello": "world"}, - "headers": { - "content-type": "application/json", - "user-agent": "PostmanRuntime/7.32.3", - "accept": "*/*", - "postman-token": "92300061-eeaa-423b-a420-0b44896ecdc4", - "host": "localhost:4000", - "accept-encoding": "gzip, deflate, br", - "connection": "keep-alive", - "content-length": "163", - }, - "endpoint": "http://localhost:4000/chat/completions", - "model_group": "gpt-3.5-turbo", - "model_info": { - "id": "4bad40a1eb6bebd1682800f16f44b9f06c52a6703444c99c7f9f32e9de3693b4", - "db_model": False, - }, - "api_base": "https://openai-gpt-4-test-v-1.openai.azure.com/", - "caching_groups": None, - "raw_request": "\n\nPOST Request Sent from LiteLLM:\ncurl -X POST \\\nhttps://openai-gpt-4-test-v-1.openai.azure.com//openai/ \\\n-H 'Authorization: *****' \\\n-d '{'model': 'chatgpt-v-3', 'messages': [{'role': 'system', 'content': 'you are a helpful assistant.\\n'}, {'role': 'user', 'content': 'bom dia'}], 'stream': False, 'max_tokens': 10, 'user': '116544810872468347480', 'extra_body': {}}'\n", - }, - ) + captured_payloads = [] - print("response", response) + async def mock_log_json_data_on_gcs( + self, headers, bucket_name, object_name, logging_payload + ): + captured_payloads.append( + { + "bucket_name": bucket_name, + "object_name": object_name, + "logging_payload": logging_payload, + } + ) + return {"kind": "storage#object", "name": object_name} - await asyncio.sleep(5) + with patch( + "litellm.proxy.proxy_server.premium_user", True + ), patch.object( + GCSBucketLogger, + "construct_request_headers", + new_callable=AsyncMock, + return_value={"Authorization": "Bearer mock_token"}, + ), patch.object( + GCSBucketLogger, + "get_gcs_logging_config", + new_callable=AsyncMock, + return_value=_make_mock_gcs_logging_config(), + ), patch.object( + GCSBucketLogger, + "_log_json_data_on_gcs", + mock_log_json_data_on_gcs, + ): + gcs_logger = GCSBucketLogger() - # Get the current date - # Get the current date - current_date = datetime.now().strftime("%Y-%m-%d") - - # Modify the object_name to include the date-based folder - object_name = f"{current_date}%2F{response.id}" - - print("object_name", object_name) - - # Check if object landed on GCS - object_from_gcs = await gcs_logger.download_gcs_object(object_name=object_name) - print("object from gcs=", object_from_gcs) - # convert object_from_gcs from bytes to DICT - parsed_data = json.loads(object_from_gcs) - print("object_from_gcs as dict", parsed_data) - - print("type of object_from_gcs", type(parsed_data)) - - gcs_payload = StandardLoggingPayload(**parsed_data) - - print("gcs_payload", gcs_payload) - - assert gcs_payload["model"] == "gpt-3.5-turbo" - assert gcs_payload["messages"] == [{"role": "user", "content": "This is a test"}] - - assert gcs_payload["response"]["choices"][0]["message"]["content"] == "Hi!" - - assert gcs_payload["response_cost"] > 0.0 - - assert gcs_payload["status"] == "success" - - assert ( - gcs_payload["metadata"]["user_api_key_hash"] - == "a1b2c3d4e5f6789012345678901234567890abcdef1234567890abcdef123456" - ) - assert gcs_payload["metadata"]["user_api_key_user_id"] == "116544810872468347480" - - assert gcs_payload["metadata"]["requester_metadata"] == {"foo": "bar"} - - # Delete Object from GCS - print("deleting object from GCS") - await gcs_logger.delete_gcs_object(object_name=object_name) - - -@pytest.mark.asyncio -async def test_basic_gcs_logger_failure(): - load_vertex_ai_credentials() - gcs_logger = GCSBucketLogger() - print("GCSBucketLogger", gcs_logger) - - gcs_log_id = f"failure-test-{uuid.uuid4().hex}" - - litellm.callbacks = [gcs_logger] - - try: + litellm.callbacks = [gcs_logger] response = await litellm.acompletion( model="gpt-3.5-turbo", temperature=0.7, messages=[{"role": "user", "content": "This is a test"}], max_tokens=10, user="ishaan-2", - mock_response=litellm.BadRequestError( - model="gpt-3.5-turbo", - message="Error: 400: Bad Request: Invalid API key, please check your API key and try again.", - llm_provider="openai", - ), + mock_response="Hi!", metadata={ - "gcs_log_id": gcs_log_id, "tags": ["model-anthropic-claude-v2.1", "app-ishaan-prod"], "user_api_key": "a1b2c3d4e5f6789012345678901234567890abcdef1234567890abcdef123456", "user_api_key_alias": None, @@ -204,6 +95,7 @@ async def test_basic_gcs_logger_failure(): "user_api_key_team_alias": None, "user_api_key_metadata": {}, "requester_ip_address": "127.0.0.1", + "requester_metadata": {"foo": "bar"}, "spend_logs_metadata": {"hello": "world"}, "headers": { "content-type": "application/json", @@ -226,557 +118,153 @@ async def test_basic_gcs_logger_failure(): "raw_request": "\n\nPOST Request Sent from LiteLLM:\ncurl -X POST \\\nhttps://openai-gpt-4-test-v-1.openai.azure.com//openai/ \\\n-H 'Authorization: *****' \\\n-d '{'model': 'chatgpt-v-3', 'messages': [{'role': 'system', 'content': 'you are a helpful assistant.\\n'}, {'role': 'user', 'content': 'bom dia'}], 'stream': False, 'max_tokens': 10, 'user': '116544810872468347480', 'extra_body': {}}'\n", }, ) - except Exception: - pass - await asyncio.sleep(5) + print("response", response) - # Get the current date - # Get the current date - current_date = datetime.now().strftime("%Y-%m-%d") + await asyncio.sleep(3) - # Modify the object_name to include the date-based folder - object_name = gcs_log_id - - print("object_name", object_name) - - # Check if object landed on GCS - object_from_gcs = await gcs_logger.download_gcs_object(object_name=object_name) - print("object from gcs=", object_from_gcs) - # convert object_from_gcs from bytes to DICT - parsed_data = json.loads(object_from_gcs) - print("object_from_gcs as dict", parsed_data) - - print("type of object_from_gcs", type(parsed_data)) - - gcs_payload = StandardLoggingPayload(**parsed_data) - - print("gcs_payload", gcs_payload) - - assert gcs_payload["model"] == "gpt-3.5-turbo" - assert gcs_payload["messages"] == [{"role": "user", "content": "This is a test"}] - - assert gcs_payload["response_cost"] == 0 - assert gcs_payload["status"] == "failure" - - assert ( - gcs_payload["metadata"]["user_api_key_hash"] - == "a1b2c3d4e5f6789012345678901234567890abcdef1234567890abcdef123456" - ) - assert gcs_payload["metadata"]["user_api_key_user_id"] == "116544810872468347480" - - # Delete Object from GCS - print("deleting object from GCS") - await gcs_logger.delete_gcs_object(object_name=object_name) - - -@pytest.mark.skip(reason="This test is flaky") -@pytest.mark.asyncio -async def test_basic_gcs_logging_per_request_with_callback_set(): - """ - Test GCS Bucket logging per request - - Request 1 - pass gcs_bucket_name in kwargs - Request 2 - don't pass gcs_bucket_name in kwargs - ensure 'litellm-testing-bucket' - """ - import logging - from litellm._logging import verbose_logger - - verbose_logger.setLevel(logging.DEBUG) - load_vertex_ai_credentials() - gcs_logger = GCSBucketLogger() - print("GCSBucketLogger", gcs_logger) - litellm.callbacks = [gcs_logger] - - GCS_BUCKET_NAME = "example-bucket-1-litellm" - standard_callback_dynamic_params: StandardCallbackDynamicParams = ( - StandardCallbackDynamicParams(gcs_bucket_name=GCS_BUCKET_NAME) - ) - - try: - response = await litellm.acompletion( - model="gpt-4o-mini", - temperature=0.7, - messages=[{"role": "user", "content": "This is a test"}], - max_tokens=10, - user="ishaan-2", - gcs_bucket_name=GCS_BUCKET_NAME, + assert len(captured_payloads) == 1, ( + f"Expected 1 GCS upload, got {len(captured_payloads)}" ) - except: - pass - await asyncio.sleep(5) + gcs_payload = captured_payloads[0]["logging_payload"] - # Get the current date - # Get the current date - current_date = datetime.now().strftime("%Y-%m-%d") + assert gcs_payload["model"] == "gpt-3.5-turbo" + assert gcs_payload["messages"] == [ + {"role": "user", "content": "This is a test"} + ] - # Modify the object_name to include the date-based folder - object_name = f"{current_date}%2F{response.id}" + assert gcs_payload["response"]["choices"][0]["message"]["content"] == "Hi!" - print("object_name", object_name) + assert gcs_payload["response_cost"] > 0.0 - # Check if object landed on GCS - object_from_gcs = await gcs_logger.download_gcs_object( - object_name=object_name, - standard_callback_dynamic_params=standard_callback_dynamic_params, - ) - print("object from gcs=", object_from_gcs) - # convert object_from_gcs from bytes to DICT - parsed_data = json.loads(object_from_gcs) - print("object_from_gcs as dict", parsed_data) - - print("type of object_from_gcs", type(parsed_data)) - - gcs_payload = StandardLoggingPayload(**parsed_data) - - assert gcs_payload["model"] == "gpt-4o-mini" - assert gcs_payload["messages"] == [{"role": "user", "content": "This is a test"}] - - assert gcs_payload["response_cost"] > 0.0 - - assert gcs_payload["status"] == "success" - - # clean up the object from GCS - await gcs_logger.delete_gcs_object( - object_name=object_name, - standard_callback_dynamic_params=standard_callback_dynamic_params, - ) - - # Request 2 - don't pass gcs_bucket_name in kwargs - ensure 'litellm-testing-bucket' - try: - response = await litellm.acompletion( - model="gpt-4o-mini", - temperature=0.7, - messages=[{"role": "user", "content": "This is a test"}], - max_tokens=10, - user="ishaan-2", - mock_response="Hi!", - ) - except: - pass - - await asyncio.sleep(5) - - # Get the current date - # Get the current date - current_date = datetime.now().strftime("%Y-%m-%d") - standard_callback_dynamic_params = StandardCallbackDynamicParams( - gcs_bucket_name="litellm-testing-bucket" - ) - - # Modify the object_name to include the date-based folder - object_name = f"{current_date}%2F{response.id}" - - print("object_name", object_name) - - # Check if object landed on GCS - object_from_gcs = await gcs_logger.download_gcs_object( - object_name=object_name, - standard_callback_dynamic_params=standard_callback_dynamic_params, - ) - print("object from gcs=", object_from_gcs) - # convert object_from_gcs from bytes to DICT - parsed_data = json.loads(object_from_gcs) - print("object_from_gcs as dict", parsed_data) - - print("type of object_from_gcs", type(parsed_data)) - - gcs_payload = StandardLoggingPayload(**parsed_data) - - assert gcs_payload["model"] == "gpt-4o-mini" - assert gcs_payload["messages"] == [{"role": "user", "content": "This is a test"}] - - assert gcs_payload["response_cost"] > 0.0 - - assert gcs_payload["status"] == "success" - - # clean up the object from GCS - await gcs_logger.delete_gcs_object( - object_name=object_name, - standard_callback_dynamic_params=standard_callback_dynamic_params, - ) - - -@pytest.mark.skip(reason="This test is flaky") -@pytest.mark.asyncio -async def test_basic_gcs_logging_per_request_with_no_litellm_callback_set(): - """ - Test GCS Bucket logging per request - - key difference: no litellm.callbacks set - - Request 1 - pass gcs_bucket_name in kwargs - Request 2 - don't pass gcs_bucket_name in kwargs - ensure 'litellm-testing-bucket' - """ - import logging - from litellm._logging import verbose_logger - - verbose_logger.setLevel(logging.DEBUG) - load_vertex_ai_credentials() - gcs_logger = GCSBucketLogger() - - GCS_BUCKET_NAME = "example-bucket-1-litellm" - standard_callback_dynamic_params: StandardCallbackDynamicParams = ( - StandardCallbackDynamicParams(gcs_bucket_name=GCS_BUCKET_NAME) - ) - - try: - response = await litellm.acompletion( - model="gpt-4o-mini", - temperature=0.7, - messages=[{"role": "user", "content": "This is a test"}], - max_tokens=10, - user="ishaan-2", - gcs_bucket_name=GCS_BUCKET_NAME, - success_callback=["gcs_bucket"], - failure_callback=["gcs_bucket"], - ) - except: - pass - - await asyncio.sleep(5) - - # Get the current date - # Get the current date - current_date = datetime.now().strftime("%Y-%m-%d") - - # Modify the object_name to include the date-based folder - object_name = f"{current_date}%2F{response.id}" - - print("object_name", object_name) - - # Check if object landed on GCS - object_from_gcs = await gcs_logger.download_gcs_object( - object_name=object_name, - standard_callback_dynamic_params=standard_callback_dynamic_params, - ) - print("object from gcs=", object_from_gcs) - # convert object_from_gcs from bytes to DICT - parsed_data = json.loads(object_from_gcs) - print("object_from_gcs as dict", parsed_data) - - print("type of object_from_gcs", type(parsed_data)) - - gcs_payload = StandardLoggingPayload(**parsed_data) - - assert gcs_payload["model"] == "gpt-4o-mini" - assert gcs_payload["messages"] == [{"role": "user", "content": "This is a test"}] - - assert gcs_payload["response_cost"] > 0.0 - - assert gcs_payload["status"] == "success" - - # clean up the object from GCS - await gcs_logger.delete_gcs_object( - object_name=object_name, - standard_callback_dynamic_params=standard_callback_dynamic_params, - ) - - # make a failure request - assert that failure callback is hit - gcs_log_id = f"failure-test-{uuid.uuid4().hex}" - try: - response = await litellm.acompletion( - model="gpt-4o-mini", - temperature=0.7, - messages=[{"role": "user", "content": "This is a test"}], - max_tokens=10, - user="ishaan-2", - mock_response=litellm.BadRequestError( - model="gpt-3.5-turbo", - message="Error: 400: Bad Request: Invalid API key, please check your API key and try again.", - llm_provider="openai", - ), - success_callback=["gcs_bucket"], - failure_callback=["gcs_bucket"], - gcs_bucket_name=GCS_BUCKET_NAME, - metadata={ - "gcs_log_id": gcs_log_id, - }, - ) - except: - pass - - await asyncio.sleep(5) - - # check if the failure object is logged in GCS - object_from_gcs = await gcs_logger.download_gcs_object( - object_name=gcs_log_id, - standard_callback_dynamic_params=standard_callback_dynamic_params, - ) - print("object from gcs=", object_from_gcs) - # convert object_from_gcs from bytes to DICT - parsed_data = json.loads(object_from_gcs) - print("object_from_gcs as dict", parsed_data) - - gcs_payload = StandardLoggingPayload(**parsed_data) - - assert gcs_payload["model"] == "gpt-4o-mini" - assert gcs_payload["messages"] == [{"role": "user", "content": "This is a test"}] - - assert gcs_payload["response_cost"] == 0 - assert gcs_payload["status"] == "failure" - - # clean up the object from GCS - await gcs_logger.delete_gcs_object( - object_name=gcs_log_id, - standard_callback_dynamic_params=standard_callback_dynamic_params, - ) - - -@pytest.mark.skip(reason="This test is flaky") -@pytest.mark.asyncio -async def test_aaaget_gcs_logging_config_without_service_account(): - """ - Test the get_gcs_logging_config works for IAM auth on GCS - 1. Key based logging without a service account - 2. Default Callback without a service account - """ - load_vertex_ai_credentials() - _old_gcs_bucket_name = os.environ.get("GCS_BUCKET_NAME") - os.environ.pop("GCS_BUCKET_NAME", None) - - _old_gcs_service_acct = os.environ.get("GCS_PATH_SERVICE_ACCOUNT") - os.environ.pop("GCS_PATH_SERVICE_ACCOUNT", None) - - # Mock the load_auth function to avoid credential loading issues - # Test 1: With standard_callback_dynamic_params (with service account) - gcs_logger = GCSBucketLogger() - - dynamic_params = StandardCallbackDynamicParams( - gcs_bucket_name="dynamic-bucket", - ) - config = await gcs_logger.get_gcs_logging_config( - {"standard_callback_dynamic_params": dynamic_params} - ) - - assert config["bucket_name"] == "dynamic-bucket" - assert config["path_service_account"] is None - assert config["vertex_instance"] is not None - - # Test 2: With standard_callback_dynamic_params (without service account - this is IAM auth) - dynamic_params = StandardCallbackDynamicParams( - gcs_bucket_name="dynamic-bucket", gcs_path_service_account=None - ) - - config = await gcs_logger.get_gcs_logging_config( - {"standard_callback_dynamic_params": dynamic_params} - ) - - assert config["bucket_name"] == "dynamic-bucket" - assert config["path_service_account"] is None - assert config["vertex_instance"] is not None - - # Test 5: With missing bucket name - with pytest.raises(ValueError, match="GCS_BUCKET_NAME is not set"): - gcs_logger = GCSBucketLogger(bucket_name=None) - await gcs_logger.get_gcs_logging_config({}) - - if _old_gcs_bucket_name is not None: - os.environ["GCS_BUCKET_NAME"] = _old_gcs_bucket_name - - if _old_gcs_service_acct is not None: - os.environ["GCS_PATH_SERVICE_ACCOUNT"] = _old_gcs_service_acct - - -@pytest.mark.skip(reason="This test is flaky") -@pytest.mark.asyncio -async def test_basic_gcs_logger_with_folder_in_bucket_name(): - load_vertex_ai_credentials() - gcs_logger = GCSBucketLogger() - - bucket_name = "litellm-testing-bucket/test-folder-logs" - - old_bucket_name = os.environ.get("GCS_BUCKET_NAME") - os.environ["GCS_BUCKET_NAME"] = bucket_name - print("GCSBucketLogger", gcs_logger) - - litellm.callbacks = [gcs_logger] - response = await litellm.acompletion( - model="gpt-3.5-turbo", - temperature=0.7, - messages=[{"role": "user", "content": "This is a test"}], - max_tokens=10, - user="ishaan-2", - mock_response="Hi!", - metadata={ - "tags": ["model-anthropic-claude-v2.1", "app-ishaan-prod"], - "user_api_key": "a1b2c3d4e5f6789012345678901234567890abcdef1234567890abcdef123456", - "user_api_key_alias": None, - "user_api_end_user_max_budget": None, - "litellm_api_version": "0.0.0", - "global_max_parallel_requests": None, - "user_api_key_user_id": "116544810872468347480", - "user_api_key_org_id": None, - "user_api_key_team_id": None, - "user_api_key_team_alias": None, - "user_api_key_metadata": {}, - "requester_ip_address": "127.0.0.1", - "requester_metadata": {"foo": "bar"}, - "spend_logs_metadata": {"hello": "world"}, - "headers": { - "content-type": "application/json", - "user-agent": "PostmanRuntime/7.32.3", - "accept": "*/*", - "postman-token": "92300061-eeaa-423b-a420-0b44896ecdc4", - "host": "localhost:4000", - "accept-encoding": "gzip, deflate, br", - "connection": "keep-alive", - "content-length": "163", - }, - "endpoint": "http://localhost:4000/chat/completions", - "model_group": "gpt-3.5-turbo", - "model_info": { - "id": "4bad40a1eb6bebd1682800f16f44b9f06c52a6703444c99c7f9f32e9de3693b4", - "db_model": False, - }, - "api_base": "https://openai-gpt-4-test-v-1.openai.azure.com/", - "caching_groups": None, - "raw_request": "\n\nPOST Request Sent from LiteLLM:\ncurl -X POST \\\nhttps://openai-gpt-4-test-v-1.openai.azure.com//openai/ \\\n-H 'Authorization: *****' \\\n-d '{'model': 'chatgpt-v-3', 'messages': [{'role': 'system', 'content': 'you are a helpful assistant.\\n'}, {'role': 'user', 'content': 'bom dia'}], 'stream': False, 'max_tokens': 10, 'user': '116544810872468347480', 'extra_body': {}}'\n", - }, - ) - - print("response", response) - - await asyncio.sleep(5) - - # Get the current date - # Get the current date - current_date = datetime.now().strftime("%Y-%m-%d") - - # Modify the object_name to include the date-based folder - object_name = f"{current_date}%2F{response.id}" - - print("object_name", object_name) - - # Check if object landed on GCS - object_from_gcs = await gcs_logger.download_gcs_object(object_name=object_name) - print("object from gcs=", object_from_gcs) - # convert object_from_gcs from bytes to DICT - parsed_data = json.loads(object_from_gcs) - print("object_from_gcs as dict", parsed_data) - - print("type of object_from_gcs", type(parsed_data)) - - gcs_payload = StandardLoggingPayload(**parsed_data) - - print("gcs_payload", gcs_payload) - - assert gcs_payload["model"] == "gpt-3.5-turbo" - assert gcs_payload["messages"] == [{"role": "user", "content": "This is a test"}] - - assert gcs_payload["response"]["choices"][0]["message"]["content"] == "Hi!" - - assert gcs_payload["response_cost"] > 0.0 - - assert gcs_payload["status"] == "success" - - assert ( - gcs_payload["metadata"]["user_api_key_hash"] - == "a1b2c3d4e5f6789012345678901234567890abcdef1234567890abcdef123456" - ) - assert gcs_payload["metadata"]["user_api_key_user_id"] == "116544810872468347480" - - assert gcs_payload["metadata"]["requester_metadata"] == {"foo": "bar"} - - # Delete Object from GCS - print("deleting object from GCS") - await gcs_logger.delete_gcs_object(object_name=object_name) - - # clean up - if old_bucket_name is not None: - os.environ["GCS_BUCKET_NAME"] = old_bucket_name - - -@pytest.mark.skip(reason="This test is flaky on ci/cd") -def test_create_file_e2e(): - """ - Asserts 'create_file' is called with the correct arguments - """ - load_vertex_ai_credentials() - test_file_content = b"test audio content" - test_file = ("test.wav", test_file_content, "audio/wav") - - from litellm import create_file - - response = create_file( - file=test_file, - purpose="user_data", - custom_llm_provider="vertex_ai", - ) - print("response", response) - assert response is not None - - -@pytest.mark.skip(reason="This test is flaky on ci/cd") -def test_create_file_e2e_jsonl(): - """ - Asserts 'create_file' is called with the correct arguments - """ - load_vertex_ai_credentials() - from litellm.llms.custom_httpx.http_handler import HTTPHandler - - client = HTTPHandler() - - example_jsonl = [ - { - "custom_id": "request-1", - "method": "POST", - "url": "/v1/chat/completions", - "body": { - "model": "gemini-1.5-flash-001", - "messages": [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Hello world!"}, - ], - "max_tokens": 10, - }, - }, - { - "custom_id": "request-2", - "method": "POST", - "url": "/v1/chat/completions", - "body": { - "model": "gemini-1.5-flash-001", - "messages": [ - {"role": "system", "content": "You are an unhelpful assistant."}, - {"role": "user", "content": "Hello world!"}, - ], - "max_tokens": 10, - }, - }, - ] - - # Create and write to the file - file_path = "example.jsonl" - with open(file_path, "w") as f: - for item in example_jsonl: - f.write(json.dumps(item) + "\n") - - # Verify file content - with open(file_path, "r") as f: - content = f.read() - print("File content:", content) - assert len(content) > 0, "File is empty" - - from litellm import create_file - - with patch.object(client, "post") as mock_create_file: - try: - response = create_file( - file=open(file_path, "rb"), - purpose="user_data", - custom_llm_provider="vertex_ai", - client=client, - ) - except Exception as e: - print("error", e) - - mock_create_file.assert_called_once() - - print(f"kwargs: {mock_create_file.call_args.kwargs}") + assert gcs_payload["status"] == "success" assert ( - mock_create_file.call_args.kwargs["data"] is not None - and len(mock_create_file.call_args.kwargs["data"]) > 0 + gcs_payload["metadata"]["user_api_key_hash"] + == "a1b2c3d4e5f6789012345678901234567890abcdef1234567890abcdef123456" + ) + assert ( + gcs_payload["metadata"]["user_api_key_user_id"] == "116544810872468347480" + ) + + assert gcs_payload["metadata"]["requester_metadata"] == {"foo": "bar"} + + +@pytest.mark.asyncio +async def test_basic_gcs_logger_failure(): + os.environ["GCS_FLUSH_INTERVAL"] = "1" + os.environ["GCS_USE_BATCHED_LOGGING"] = "false" + os.environ["GCS_BUCKET_NAME"] = "test-bucket" + + captured_payloads = [] + + async def mock_log_json_data_on_gcs( + self, headers, bucket_name, object_name, logging_payload + ): + captured_payloads.append( + { + "bucket_name": bucket_name, + "object_name": object_name, + "logging_payload": logging_payload, + } + ) + return {"kind": "storage#object", "name": object_name} + + gcs_log_id = f"failure-test-{uuid.uuid4().hex}" + + with patch( + "litellm.proxy.proxy_server.premium_user", True + ), patch.object( + GCSBucketLogger, + "construct_request_headers", + new_callable=AsyncMock, + return_value={"Authorization": "Bearer mock_token"}, + ), patch.object( + GCSBucketLogger, + "get_gcs_logging_config", + new_callable=AsyncMock, + return_value=_make_mock_gcs_logging_config(), + ), patch.object( + GCSBucketLogger, + "_log_json_data_on_gcs", + mock_log_json_data_on_gcs, + ): + gcs_logger = GCSBucketLogger() + + litellm.callbacks = [gcs_logger] + + try: + response = await litellm.acompletion( + model="gpt-3.5-turbo", + temperature=0.7, + messages=[{"role": "user", "content": "This is a test"}], + max_tokens=10, + user="ishaan-2", + mock_response=litellm.BadRequestError( + model="gpt-3.5-turbo", + message="Error: 400: Bad Request: Invalid API key, please check your API key and try again.", + llm_provider="openai", + ), + metadata={ + "gcs_log_id": gcs_log_id, + "tags": ["model-anthropic-claude-v2.1", "app-ishaan-prod"], + "user_api_key": "a1b2c3d4e5f6789012345678901234567890abcdef1234567890abcdef123456", + "user_api_key_alias": None, + "user_api_end_user_max_budget": None, + "litellm_api_version": "0.0.0", + "global_max_parallel_requests": None, + "user_api_key_user_id": "116544810872468347480", + "user_api_key_org_id": None, + "user_api_key_team_id": None, + "user_api_key_team_alias": None, + "user_api_key_metadata": {}, + "requester_ip_address": "127.0.0.1", + "spend_logs_metadata": {"hello": "world"}, + "headers": { + "content-type": "application/json", + "user-agent": "PostmanRuntime/7.32.3", + "accept": "*/*", + "postman-token": "92300061-eeaa-423b-a420-0b44896ecdc4", + "host": "localhost:4000", + "accept-encoding": "gzip, deflate, br", + "connection": "keep-alive", + "content-length": "163", + }, + "endpoint": "http://localhost:4000/chat/completions", + "model_group": "gpt-3.5-turbo", + "model_info": { + "id": "4bad40a1eb6bebd1682800f16f44b9f06c52a6703444c99c7f9f32e9de3693b4", + "db_model": False, + }, + "api_base": "https://openai-gpt-4-test-v-1.openai.azure.com/", + "caching_groups": None, + "raw_request": "\n\nPOST Request Sent from LiteLLM:\ncurl -X POST \\\nhttps://openai-gpt-4-test-v-1.openai.azure.com//openai/ \\\n-H 'Authorization: *****' \\\n-d '{'model': 'chatgpt-v-3', 'messages': [{'role': 'system', 'content': 'you are a helpful assistant.\\n'}, {'role': 'user', 'content': 'bom dia'}], 'stream': False, 'max_tokens': 10, 'user': '116544810872468347480', 'extra_body': {}}'\n", + }, + ) + except Exception: + pass + + await asyncio.sleep(3) + + assert len(captured_payloads) == 1, ( + f"Expected 1 GCS upload, got {len(captured_payloads)}" + ) + + gcs_payload = captured_payloads[0]["logging_payload"] + + assert gcs_payload["model"] == "gpt-3.5-turbo" + assert gcs_payload["messages"] == [ + {"role": "user", "content": "This is a test"} + ] + + assert gcs_payload["response_cost"] == 0 + assert gcs_payload["status"] == "failure" + + assert ( + gcs_payload["metadata"]["user_api_key_hash"] + == "a1b2c3d4e5f6789012345678901234567890abcdef1234567890abcdef123456" + ) + assert ( + gcs_payload["metadata"]["user_api_key_user_id"] == "116544810872468347480" ) diff --git a/tests/local_testing/test_router_client_init.py b/tests/local_testing/test_router_client_init.py index 6cf8b4e59b7..f2b82b651dd 100644 --- a/tests/local_testing/test_router_client_init.py +++ b/tests/local_testing/test_router_client_init.py @@ -71,9 +71,7 @@ def test_router_init_with_neither_api_key_nor_azure_service_principal_with_secre @patch("azure.identity.get_bearer_token_provider") @patch("azure.identity.ClientSecretCredential") -@patch("litellm.secret_managers.get_azure_ad_token_provider.os") def test_router_init_azure_service_principal_with_secret_with_environment_variables( - mocked_os_lib: MagicMock, mocked_credential: MagicMock, mocked_get_bearer_token_provider: MagicMock, monkeypatch, @@ -86,21 +84,18 @@ def test_router_init_azure_service_principal_with_secret_with_environment_variab and environment variables. """ monkeypatch.delenv("AZURE_AI_API_KEY", raising=False) + monkeypatch.delenv("AZURE_OPENAI_API_KEY", raising=False) + monkeypatch.delenv("AZURE_API_KEY", raising=False) litellm.enable_azure_ad_token_refresh = True # mock the token provider function mocked_func_generating_token = MagicMock(return_value="test_token") mocked_get_bearer_token_provider.return_value = mocked_func_generating_token - # mock the environment variables with mocked credentials - environment_variables_expected_to_use = { - "AZURE_CLIENT_ID": "test_client_id", - "AZURE_CLIENT_SECRET": "test_client_secret", - "AZURE_TENANT_ID": "test_tenant_id", - } - mocked_environ = PropertyMock(return_value=environment_variables_expected_to_use) - # Because of the way mock attributes are stored you can’t directly attach a PropertyMock to a mock object. - # https://docs.python.org/3.11/library/unittest.mock.html#unittest.mock.PropertyMock - type(mocked_os_lib).environ = mocked_environ + # set environment variables with mocked credentials using monkeypatch + # so both common_utils._resolve_env_var and get_azure_ad_token_provider see them + monkeypatch.setenv("AZURE_CLIENT_ID", "test_client_id") + monkeypatch.setenv("AZURE_CLIENT_SECRET", "test_client_secret") + monkeypatch.setenv("AZURE_TENANT_ID", "test_tenant_id") # define the model list model_list = [ diff --git a/tests/local_testing/test_simple_shuffle.py b/tests/local_testing/test_simple_shuffle.py deleted file mode 100644 index 8837e91126b..00000000000 --- a/tests/local_testing/test_simple_shuffle.py +++ /dev/null @@ -1,53 +0,0 @@ -# What is this? -## unit tests for 'simple-shuffle' - -import sys, os, asyncio, time, random -from datetime import datetime -import traceback -from dotenv import load_dotenv - -load_dotenv() -import os - -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path -import pytest -from litellm import Router - -""" -Test random shuffle -- async -- sync -""" - - -async def test_simple_shuffle(): - model_list = [ - { - "model_name": "azure-model", - "litellm_params": { - "model": "azure/gpt-turbo", - "api_key": "os.environ/AZURE_FRANCE_API_KEY", - "api_base": "https://openai-france-1234.openai.azure.com", - "rpm": 1440, - }, - "model_info": {"id": 1}, - }, - { - "model_name": "azure-model", - "litellm_params": { - "model": "azure/gpt-35-turbo", - "api_key": "os.environ/AZURE_EUROPE_API_KEY", - "api_base": "https://my-endpoint-europe-berri-992.openai.azure.com", - "rpm": 6, - }, - "model_info": {"id": 2}, - }, - ] - router = Router( - model_list=model_list, - routing_strategy="usage-based-routing-v2", - set_verbose=False, - num_retries=3, - ) # type: ignore diff --git a/tests/local_testing/test_streaming.py b/tests/local_testing/test_streaming.py index 97d6875f161..3aed0699603 100644 --- a/tests/local_testing/test_streaming.py +++ b/tests/local_testing/test_streaming.py @@ -471,85 +471,6 @@ def test_completion_azure_stream(): # test_completion_azure_stream() -@pytest.mark.skip("Skipping predibase streaming test - ran out of credits") -@pytest.mark.parametrize("sync_mode", [True, False]) -@pytest.mark.asyncio -async def test_completion_predibase_streaming(sync_mode): - try: - litellm.set_verbose = True - litellm._turn_on_debug() - if sync_mode: - response = completion( - model="predibase/llama-3-8b-instruct", - timeout=5, - tenant_id="c4768f95", - max_tokens=10, - api_base="https://serving.app.predibase.com", - api_key=os.getenv("PREDIBASE_API_KEY"), - messages=[{"role": "user", "content": "What is the meaning of life?"}], - stream=True, - ) - - complete_response = "" - for idx, init_chunk in enumerate(response): - chunk, finished = streaming_format_tests(idx, init_chunk) - complete_response += chunk - custom_llm_provider = init_chunk._hidden_params["custom_llm_provider"] - print(f"custom_llm_provider: {custom_llm_provider}") - assert custom_llm_provider == "predibase" - if finished: - assert isinstance( - init_chunk.choices[0], litellm.utils.StreamingChoices - ) - break - if complete_response.strip() == "": - raise Exception("Empty response received") - else: - response = await litellm.acompletion( - model="predibase/llama-3-8b-instruct", - tenant_id="c4768f95", - timeout=5, - max_tokens=10, - api_base="https://serving.app.predibase.com", - api_key=os.getenv("PREDIBASE_API_KEY"), - messages=[{"role": "user", "content": "What is the meaning of life?"}], - stream=True, - ) - - # await response - - complete_response = "" - idx = 0 - async for init_chunk in response: - chunk, finished = streaming_format_tests(idx, init_chunk) - complete_response += chunk - custom_llm_provider = init_chunk._hidden_params["custom_llm_provider"] - print(f"custom_llm_provider: {custom_llm_provider}") - assert custom_llm_provider == "predibase" - idx += 1 - if finished: - assert isinstance( - init_chunk.choices[0], litellm.utils.StreamingChoices - ) - break - if complete_response.strip() == "": - raise Exception("Empty response received") - - print(f"complete_response: {complete_response}") - except litellm.Timeout: - pass - except litellm.InternalServerError: - pass - except litellm.ServiceUnavailableError: - pass - except litellm.APIConnectionError: - pass - except Exception as e: - print("ERROR class", e.__class__) - print("ERROR message", e) - print("ERROR traceback", traceback.format_exc()) - - pytest.fail(f"Error occurred: {e}") def test_completion_azure_function_calling_stream(): @@ -936,49 +857,6 @@ def test_completion_mistral_api_mistral_large_function_call_with_streaming(): # test_completion_mistral_api_stream() -def test_completion_deep_infra_stream(): - # deep infra,currently includes role in the 2nd chunk - # waiting for them to make a fix on this - litellm.set_verbose = True - try: - messages = [ - {"role": "system", "content": "You are a helpful assistant."}, - { - "role": "user", - "content": "how does a court case get to the Supreme Court?", - }, - ] - print("testing deep infra streaming") - response = completion( - model="deepinfra/meta-llama/Llama-2-70b-chat-hf", - messages=messages, - stream=True, - max_tokens=80, - ) - - complete_response = "" - # Add any assertions here to check the response - has_finish_reason = False - for idx, chunk in enumerate(response): - chunk, finished = streaming_format_tests(idx, chunk) - if finished: - has_finish_reason = True - break - complete_response += chunk - if has_finish_reason == False: - raise Exception("finish reason not set") - if complete_response.strip() == "": - raise Exception("Empty response received") - print(f"completion_response: {complete_response}") - except Exception as e: - if "Model busy, retry later" in str(e): - pass - pytest.fail(f"Error occurred: {e}") - - -# test_completion_deep_infra_stream() - - @pytest.mark.skip() def test_completion_nlp_cloud_stream(): try: @@ -1067,7 +945,6 @@ def test_vertex_ai_stream(provider): load_vertex_ai_credentials() litellm.set_verbose = True - litellm.vertex_project = "pathrise-convert-1606954137718" import random test_models = ["gemini-2.5-flash-lite"] @@ -1186,6 +1063,7 @@ def test_vertex_ai_stream(provider): # test_completion_vertexai_stream_bad_key() +@pytest.mark.skip(reason="Replicate extremely flaky.") @pytest.mark.parametrize("sync_mode", [False, True]) @pytest.mark.asyncio async def test_completion_replicate_llama3_streaming(sync_mode): diff --git a/tests/logging_callback_tests/gcs_pub_sub_body/spend_logs_payload.json b/tests/logging_callback_tests/gcs_pub_sub_body/spend_logs_payload.json index 7dcbd2467fd..a4c50d3c575 100644 --- a/tests/logging_callback_tests/gcs_pub_sub_body/spend_logs_payload.json +++ b/tests/logging_callback_tests/gcs_pub_sub_body/spend_logs_payload.json @@ -11,7 +11,7 @@ "user": "", "team_id": "", "organization_id": "", - "metadata": "{\"applied_guardrails\": [], \"batch_models\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"guardrail_information\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null}", + "metadata": "{\"applied_guardrails\": [], \"batch_models\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"guardrail_information\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null}", "cache_key": "Cache OFF", "spend": 0.00022500000000000002, "total_tokens": 30, diff --git a/tests/logging_callback_tests/test_amazing_s3_logs.py b/tests/logging_callback_tests/test_amazing_s3_logs.py index c1e6884278c..987e09264c1 100644 --- a/tests/logging_callback_tests/test_amazing_s3_logs.py +++ b/tests/logging_callback_tests/test_amazing_s3_logs.py @@ -78,52 +78,47 @@ async def test_basic_s3_logging(sync_mode, streaming): @pytest.mark.parametrize("streaming", [True]) @pytest.mark.flaky(retries=3, delay=1) async def test_basic_s3_v2_logging(streaming): - from blockbuster import BlockBuster + from unittest.mock import AsyncMock, MagicMock, patch from litellm.integrations.s3_v2 import S3Logger + litellm.s3_callback_params = { + "s3_bucket_name": "load-testing-oct", + "s3_aws_secret_access_key": "test-secret", + "s3_aws_access_key_id": "test-key", + "s3_region_name": "us-west-2", + } + s3_v2_logger = S3Logger(s3_flush_interval=1) litellm.callbacks = [s3_v2_logger] - blockbuster = BlockBuster() - blockbuster.activate() - litellm._turn_on_debug() - litellm.callbacks = ["s3_v2"] - litellm.s3_callback_params = { - "s3_bucket_name": "load-testing-oct", - "s3_aws_secret_access_key": "os.environ/AWS_SECRET_ACCESS_KEY", - "s3_aws_access_key_id": "os.environ/AWS_ACCESS_KEY_ID", - "s3_region_name": "us-west-2", - } + uploaded_keys: list = [] + original_upload = s3_v2_logger.async_upload_data_to_s3 + + async def mock_upload(batch_logging_element): + uploaded_keys.append(batch_logging_element.s3_object_key) + + s3_v2_logger.async_upload_data_to_s3 = mock_upload + litellm.set_verbose = True response_id = None response = await litellm.acompletion( model="gpt-4o-mini", messages=[{"role": "user", "content": "This is a test"}], + mock_response="It's simple to use and easy to get started", stream=streaming, ) if streaming: async for chunk in response: - print(chunk) response_id = chunk.id else: response_id = response.id - await asyncio.sleep(30) - print(f"response: {response}") + await asyncio.sleep(5) - # stop blockbuster - blockbuster.deactivate() - - total_objects, all_s3_keys = list_all_s3_objects("load-testing-oct") - - print(f"all_s3_keys: {all_s3_keys}") - - # assert that atlest one key has response.id in it - assert any(response_id in key for key in all_s3_keys) - s3 = boto3.client("s3") - # delete all objects - for key in all_s3_keys: - s3.delete_object(Bucket="load-testing-oct", Key=key) + assert len(uploaded_keys) > 0, "S3 upload was never called" + assert any(response_id in key for key in uploaded_keys), ( + f"Expected response_id={response_id} in one of the uploaded S3 keys: {uploaded_keys}" + ) @pytest.mark.asyncio @@ -230,9 +225,6 @@ def list_all_s3_objects(bucket_name): return total_objects, all_s3_keys -list_all_s3_objects("load-testing-oct") - - @pytest.mark.skip(reason="AWS Suspended Account") def test_s3_logging(): # all s3 requests need to be in one test function diff --git a/tests/logging_callback_tests/test_azure_blob_storage.py b/tests/logging_callback_tests/test_azure_blob_storage.py deleted file mode 100644 index a90f253cc9e..00000000000 --- a/tests/logging_callback_tests/test_azure_blob_storage.py +++ /dev/null @@ -1,45 +0,0 @@ -import io -import os -import sys - - -sys.path.insert(0, os.path.abspath("../..")) - -import asyncio -import gzip -import json -import logging -import time -from unittest.mock import AsyncMock, patch - -import pytest - -import litellm -from litellm import completion -from litellm._logging import verbose_logger -from litellm.integrations.datadog.datadog import * -from datetime import datetime, timedelta -from litellm.types.utils import ( - StandardLoggingPayload, - StandardLoggingModelInformation, - StandardLoggingMetadata, - StandardLoggingHiddenParams, -) -from litellm.integrations.azure_storage.azure_storage import AzureBlobStorageLogger - -verbose_logger.setLevel(logging.DEBUG) - - -@pytest.mark.asyncio -async def test_azure_blob_storage(): - azure_storage_logger = AzureBlobStorageLogger(flush_interval=1) - litellm.callbacks = [azure_storage_logger] - - response = await litellm.acompletion( - model="gpt-4o", - messages=[{"role": "user", "content": "Hello, world!"}], - ) - print(response) - - await asyncio.sleep(3) - pass diff --git a/tests/logging_callback_tests/test_gcs_pub_sub.py b/tests/logging_callback_tests/test_gcs_pub_sub.py index 540fb59ab01..aa846e34f63 100644 --- a/tests/logging_callback_tests/test_gcs_pub_sub.py +++ b/tests/logging_callback_tests/test_gcs_pub_sub.py @@ -190,7 +190,9 @@ async def test_async_gcs_pub_sub(): mock_post.return_value.text = "Accepted" # Initialize the GcsPubSubLogger and set the mock - gcs_pub_sub_logger = GcsPubSubLogger(flush_interval=1) + gcs_pub_sub_logger = GcsPubSubLogger( + project_id="STUBBED_PROJECT_ID", topic_id="STUBBED_TOPIC_ID", flush_interval=1 + ) gcs_pub_sub_logger.async_httpx_client.post = mock_post mock_construct_request_headers = AsyncMock() @@ -215,7 +217,7 @@ async def test_async_gcs_pub_sub(): print("sent to url", actual_url) assert ( actual_url - == "https://pubsub.googleapis.com/v1/projects/reliableKeys/topics/litellmDB:publish" + == "https://pubsub.googleapis.com/v1/projects/STUBBED_PROJECT_ID/topics/STUBBED_TOPIC_ID:publish" ) actual_request = mock_post.call_args[1]["json"] @@ -245,7 +247,9 @@ async def test_async_gcs_pub_sub_v1(): mock_post.return_value.text = "Accepted" # Initialize the GcsPubSubLogger and set the mock - gcs_pub_sub_logger = GcsPubSubLogger(flush_interval=1) + gcs_pub_sub_logger = GcsPubSubLogger( + project_id="STUBBED_PROJECT_ID", topic_id="STUBBED_TOPIC_ID", flush_interval=1 + ) gcs_pub_sub_logger.async_httpx_client.post = mock_post mock_construct_request_headers = AsyncMock() @@ -270,7 +274,7 @@ async def test_async_gcs_pub_sub_v1(): print("sent to url", actual_url) assert ( actual_url - == "https://pubsub.googleapis.com/v1/projects/reliableKeys/topics/litellmDB:publish" + == "https://pubsub.googleapis.com/v1/projects/STUBBED_PROJECT_ID/topics/STUBBED_TOPIC_ID:publish" ) actual_request = mock_post.call_args[1]["json"] diff --git a/tests/old_proxy_tests/tests/test_vtx_sdk_embedding.py b/tests/old_proxy_tests/tests/test_vtx_sdk_embedding.py index ff7f835a8da..a71718a204a 100644 --- a/tests/old_proxy_tests/tests/test_vtx_sdk_embedding.py +++ b/tests/old_proxy_tests/tests/test_vtx_sdk_embedding.py @@ -37,7 +37,7 @@ class CredentialsWrapper(Credentials): credentials = CredentialsWrapper(token=LITELLM_PROXY_API_KEY) vertexai.init( - project="pathrise-convert-1606954137718", + project="litellm-ci-cd", location="us-central1", api_endpoint=LITELLM_PROXY_BASE, credentials=credentials, diff --git a/tests/pass_through_tests/test_assembly_ai.py b/tests/pass_through_tests/test_assembly_ai.py index 2d01ef2c1be..31bdf24009a 100644 --- a/tests/pass_through_tests/test_assembly_ai.py +++ b/tests/pass_through_tests/test_assembly_ai.py @@ -2,43 +2,76 @@ This test ensures that the proxy can passthrough requests to assemblyai """ +import time + import pytest -import assemblyai as aai +import httpx import aiohttp import asyncio -import time TEST_MASTER_KEY = "sk-1234" TEST_BASE_URL = "http://0.0.0.0:4000/assemblyai" -def test_assemblyai_basic_transcribe(): - print("making basic transcribe request to assemblyai passthrough") +def _transcribe_and_verify(virtual_key: str, base_url: str): + file_url = "https://assembly.ai/wildfires.mp3" + headers = { + "Authorization": f"Bearer {virtual_key}", + "Content-Type": "application/json", + } + create_payload = { + "audio_url": file_url, + "speech_models": ["universal-2"], + } - # Replace with your API key - aai.settings.api_key = f"Bearer {TEST_MASTER_KEY}" - aai.settings.base_url = TEST_BASE_URL + create_response = httpx.post( + url=f"{base_url}/v2/transcript", + headers=headers, + json=create_payload, + timeout=60.0, + ) + if create_response.status_code != 200: + pytest.fail( + "Failed to create transcript request: " + f"status={create_response.status_code}, body={create_response.text}" + ) - # URL of the file to transcribe - FILE_URL = "https://assembly.ai/wildfires.mp3" - - # You can also transcribe a local file by passing in a file path - # FILE_URL = './path/to/file.mp3' - - transcriber = aai.Transcriber() - transcript = transcriber.transcribe(FILE_URL) - print(transcript) - print(transcript.id) - if transcript.id: - transcript.delete_by_id(transcript.id) - else: + transcript = create_response.json() + transcript_id = transcript.get("id") + if not transcript_id: pytest.fail("Failed to get transcript id") - if transcript.status == aai.TranscriptStatus.error: - print(transcript.error) - pytest.fail(f"Failed to transcribe file error: {transcript.error}") - else: - print(transcript.text) + for _ in range(60): + poll_response = httpx.get( + url=f"{base_url}/v2/transcript/{transcript_id}", + headers=headers, + timeout=30.0, + ) + if poll_response.status_code != 200: + pytest.fail( + "Failed to poll transcript status: " + f"status={poll_response.status_code}, body={poll_response.text}" + ) + transcript = poll_response.json() + if transcript.get("status") in ("completed", "error"): + break + time.sleep(1) + + httpx.delete( + url=f"{base_url}/v2/transcript/{transcript_id}", + headers=headers, + timeout=30.0, + ) + + if transcript.get("status") == "error": + pytest.fail(f"Failed to transcribe file error: {transcript.get('error')}") + + print(transcript.get("text")) + + +def test_assemblyai_basic_transcribe(): + print("making basic transcribe request to assemblyai passthrough") + _transcribe_and_verify(TEST_MASTER_KEY, TEST_BASE_URL) async def generate_key(calling_key: str) -> str: @@ -59,37 +92,10 @@ async def generate_key(calling_key: str) -> str: @pytest.mark.asyncio async def test_assemblyai_transcribe_with_non_admin_key(): - # Generate a non-admin key using the helper non_admin_key = await generate_key(TEST_MASTER_KEY) print(f"Generated non-admin key: {non_admin_key}") - # Use the non-admin key to transcribe - # Replace with your API key - aai.settings.api_key = f"Bearer {non_admin_key}" - aai.settings.base_url = TEST_BASE_URL - - # URL of the file to transcribe - FILE_URL = "https://assembly.ai/wildfires.mp3" - - # You can also transcribe a local file by passing in a file path - # FILE_URL = './path/to/file.mp3' - request_start_time = time.time() - - transcriber = aai.Transcriber() - transcript = transcriber.transcribe(FILE_URL) - print(transcript) - print(transcript.id) - if transcript.id: - transcript.delete_by_id(transcript.id) - else: - pytest.fail("Failed to get transcript id") - - if transcript.status == aai.TranscriptStatus.error: - print(transcript.error) - pytest.fail(f"Failed to transcribe file error: {transcript.error}") - else: - print(transcript.text) - + _transcribe_and_verify(non_admin_key, TEST_BASE_URL) request_end_time = time.time() print(f"Request took {request_end_time - request_start_time} seconds") diff --git a/tests/pass_through_tests/test_local_vertex.js b/tests/pass_through_tests/test_local_vertex.js index 231858cdf9a..149635e2d6f 100644 --- a/tests/pass_through_tests/test_local_vertex.js +++ b/tests/pass_through_tests/test_local_vertex.js @@ -3,7 +3,7 @@ const { VertexAI, RequestOptions } = require('@google-cloud/vertexai'); const vertexAI = new VertexAI({ - project: 'pathrise-convert-1606954137718', + project: 'litellm-ci-cd', location: 'us-central1', apiEndpoint: "127.0.0.1:4000/vertex-ai" }); diff --git a/tests/pass_through_tests/test_vertex.test.js b/tests/pass_through_tests/test_vertex.test.js index c10889e3a60..cfb494da670 100644 --- a/tests/pass_through_tests/test_vertex.test.js +++ b/tests/pass_through_tests/test_vertex.test.js @@ -61,7 +61,7 @@ beforeAll(() => { describe('Vertex AI Tests', () => { test('should successfully generate content from Vertex AI', async () => { const vertexAI = new VertexAI({ - project: 'pathrise-convert-1606954137718', + project: 'litellm-ci-cd', location: 'us-central1', apiEndpoint: "localhost:4000/vertex-ai" }); @@ -100,7 +100,7 @@ describe('Vertex AI Tests', () => { test('should successfully generate non-streaming content from Vertex AI', async () => { - const vertexAI = new VertexAI({project: 'pathrise-convert-1606954137718', location: 'us-central1', apiEndpoint: "localhost:4000/vertex-ai"}); + const vertexAI = new VertexAI({project: 'litellm-ci-cd', location: 'us-central1', apiEndpoint: "localhost:4000/vertex-ai"}); const customHeaders = new Headers({"x-litellm-api-key": "sk-1234"}); const requestOptions = {customHeaders: customHeaders}; const generativeModel = vertexAI.getGenerativeModel({model: 'gemini-2.5-flash-lite'}, requestOptions); diff --git a/tests/pass_through_tests/test_vertex_ai.py b/tests/pass_through_tests/test_vertex_ai.py index 2f5ec8eaa3e..ba27a4cc460 100644 --- a/tests/pass_through_tests/test_vertex_ai.py +++ b/tests/pass_through_tests/test_vertex_ai.py @@ -98,7 +98,7 @@ async def test_basic_vertex_ai_pass_through_with_spendlog(): load_vertex_ai_credentials() vertexai.init( - project="pathrise-convert-1606954137718", + project="litellm-ci-cd", location="us-central1", api_endpoint=f"{LITE_LLM_ENDPOINT}/vertex_ai", api_transport="rest", @@ -138,7 +138,7 @@ async def test_basic_vertex_ai_pass_through_streaming_with_spendlog(): load_vertex_ai_credentials() vertexai.init( - project="pathrise-convert-1606954137718", + project="litellm-ci-cd", location="us-central1", api_endpoint=f"{LITE_LLM_ENDPOINT}/vertex_ai", api_transport="rest", @@ -177,7 +177,7 @@ async def test_vertex_ai_pass_through_endpoint_context_caching(): # load_vertex_ai_credentials() vertexai.init( - project="pathrise-convert-1606954137718", + project="litellm-ci-cd", location="us-central1", api_endpoint=f"{LITE_LLM_ENDPOINT}/vertex_ai", api_transport="rest", diff --git a/tests/pass_through_tests/test_vertex_with_spend.test.js b/tests/pass_through_tests/test_vertex_with_spend.test.js index 6a5643918aa..142a1cec8ff 100644 --- a/tests/pass_through_tests/test_vertex_with_spend.test.js +++ b/tests/pass_through_tests/test_vertex_with_spend.test.js @@ -70,7 +70,7 @@ jest.retryTimes(3); describe('Vertex AI Tests', () => { test('should successfully generate non-streaming content with tags', async () => { const vertexAI = new VertexAI({ - project: 'pathrise-convert-1606954137718', + project: 'litellm-ci-cd', location: 'us-central1', apiEndpoint: "127.0.0.1:4000/vertex_ai" }); @@ -129,7 +129,7 @@ describe('Vertex AI Tests', () => { test('should successfully generate streaming content with tags', async () => { const vertexAI = new VertexAI({ - project: 'pathrise-convert-1606954137718', + project: 'litellm-ci-cd', location: 'us-central1', apiEndpoint: "127.0.0.1:4000/vertex_ai" }); diff --git a/tests/pass_through_unit_tests/test_unit_test_passthrough_router.py b/tests/pass_through_unit_tests/test_unit_test_passthrough_router.py index 3f62f557915..8e016b68d05 100644 --- a/tests/pass_through_unit_tests/test_unit_test_passthrough_router.py +++ b/tests/pass_through_unit_tests/test_unit_test_passthrough_router.py @@ -102,7 +102,7 @@ class TestPassthroughEndpointRouter(unittest.TestCase): mock_get_secret.return_value = "env_azure_key" result = self.router.get_credentials("azure", None) self.assertEqual(result, "env_azure_key") - mock_get_secret.assert_called_once_with("AZURE_AI_API_KEY") + mock_get_secret.assert_called_once_with("AZURE_API_KEY") def test_default_env_variable_method(self): """ diff --git a/tests/proxy_admin_ui_tests/.npmrc b/tests/proxy_admin_ui_tests/.npmrc new file mode 100644 index 00000000000..168e81a1c4e --- /dev/null +++ b/tests/proxy_admin_ui_tests/.npmrc @@ -0,0 +1,5 @@ +# Supply-chain hardening +# Packages needing lifecycle scripts: npm rebuild +ignore-scripts=true +# Protects local npm install only — npm ci (used in CI) ignores this +min-release-age=3d diff --git a/tests/proxy_admin_ui_tests/package.json b/tests/proxy_admin_ui_tests/package.json index ac726b64b76..20dfed7a8a5 100644 --- a/tests/proxy_admin_ui_tests/package.json +++ b/tests/proxy_admin_ui_tests/package.json @@ -10,22 +10,5 @@ "devDependencies": { "@playwright/test": "^1.47.2", "@types/node": "^22.5.5" - }, - "overrides": { - "glob": ">=11.1.0", - "tar": ">=7.5.10", - "minimatch": ">=10.2.4", - "diff": ">=8.0.3", - "@isaacs/brace-expansion": ">=5.0.1", - "@babel/traverse": ">=7.23.2", - "ws": ">=7.5.10", - "http-proxy-middleware": ">=2.0.9", - "tar-fs": ">=2.1.4", - "webpack-dev-middleware": ">=5.3.4", - "braces": ">=3.0.3", - "axios": ">=0.30.2", - "webpack": ">=5.94.0", - "serve-static": ">=1.16.0", - "path-to-regexp": ">=0.1.12" } -} \ No newline at end of file +} diff --git a/tests/proxy_admin_ui_tests/ui_unit_tests/.npmrc b/tests/proxy_admin_ui_tests/ui_unit_tests/.npmrc new file mode 100644 index 00000000000..168e81a1c4e --- /dev/null +++ b/tests/proxy_admin_ui_tests/ui_unit_tests/.npmrc @@ -0,0 +1,5 @@ +# Supply-chain hardening +# Packages needing lifecycle scripts: npm rebuild +ignore-scripts=true +# Protects local npm install only — npm ci (used in CI) ignores this +min-release-age=3d diff --git a/tests/proxy_admin_ui_tests/ui_unit_tests/package.json b/tests/proxy_admin_ui_tests/ui_unit_tests/package.json index 9f1c689721c..c6a3e6e260e 100644 --- a/tests/proxy_admin_ui_tests/ui_unit_tests/package.json +++ b/tests/proxy_admin_ui_tests/ui_unit_tests/package.json @@ -24,20 +24,11 @@ "react-dom": "^18.2.0" }, "overrides": { - "glob": ">=11.1.0", - "tar": ">=7.5.10", - "minimatch": ">=10.2.4", - "diff": ">=8.0.3", - "@isaacs/brace-expansion": ">=5.0.1", - "@babel/traverse": ">=7.23.2", - "ws": ">=7.5.10", - "http-proxy-middleware": ">=2.0.9", - "tar-fs": ">=2.1.4", - "webpack-dev-middleware": ">=5.3.4", - "braces": ">=3.0.3", - "axios": ">=0.30.2", - "webpack": ">=5.94.0", - "serve-static": ">=1.16.0", - "path-to-regexp": ">=0.1.12" + "glob": "13.0.0", + "minimatch": "10.1.1", + "@isaacs/brace-expansion": "5.0.0", + "@babel/traverse": "7.28.5", + "ws": "8.18.3", + "braces": "3.0.3" } } \ No newline at end of file diff --git a/tests/proxy_unit_tests/adroit-crow-413218-bc47f303efc9.json b/tests/proxy_unit_tests/adroit-crow-413218-bc47f303efc9.json deleted file mode 100644 index 7e02c821360..00000000000 --- a/tests/proxy_unit_tests/adroit-crow-413218-bc47f303efc9.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "type": "service_account", - "project_id": "pathrise-convert-1606954137718", - "private_key_id": "", - "private_key": "", - "client_email": "test-adroit-crow@pathrise-convert-1606954137718.iam.gserviceaccount.com", - "client_id": "104886546564708740969", - "auth_uri": "https://accounts.google.com/o/oauth2/auth", - "token_uri": "https://oauth2.googleapis.com/token", - "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", - "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/test-adroit-crow%40pathrise-convert-1606954137718.iam.gserviceaccount.com", - "universe_domain": "googleapis.com" -} diff --git a/tests/proxy_unit_tests/test_jwt_key_mapping.py b/tests/proxy_unit_tests/test_jwt_key_mapping.py index b67dd2792f8..66e5b3839bc 100644 --- a/tests/proxy_unit_tests/test_jwt_key_mapping.py +++ b/tests/proxy_unit_tests/test_jwt_key_mapping.py @@ -135,6 +135,81 @@ async def test_jwt_to_virtual_key_mapping_no_mapping(): prisma_client.db.litellm_jwtkeymapping.find_first.assert_not_called() +# ────────────────────────────────────────────── +# Tests: OIDC / JWT routing in user_api_key_auth +# ────────────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_virtual_key_mapping_oidc_enabled_jwt_token_uses_auth_jwt(): + """ + Regression test for the is_jwt routing fix in user_api_key_auth.py. + + When oidc_userinfo_enabled=True and virtual_key_claim_field is set, but + the token is a well-formed JWT (3-part header.payload.sig), the virtual-key + claim lookup must call auth_jwt — not get_oidc_userinfo. + """ + # Three-part token: is_jwt() returns True + api_key = "eyJhbGciOiJSUzI1NiJ9.eyJlbWFpbCI6InVzZXJAZXhhbXBsZS5jb20ifQ.sig" + + jwt_handler = JWTHandler() + jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth( + oidc_userinfo_enabled=True, + virtual_key_claim_field="email", + ) + + # Confirm our fixture token is treated as a JWT + assert jwt_handler.is_jwt(token=api_key) is True + + auth_jwt_mock = AsyncMock(return_value={"email": "user@example.com", "sub": "123"}) + oidc_userinfo_mock = AsyncMock(return_value={"email": "user@example.com"}) + + # Simulate the routing condition from user_api_key_auth.py + if jwt_handler.litellm_jwtauth.oidc_userinfo_enabled and not jwt_handler.is_jwt( + token=api_key + ): + jwt_claims = await oidc_userinfo_mock(token=api_key) + else: + jwt_claims = await auth_jwt_mock(token=api_key) + + auth_jwt_mock.assert_called_once_with(token=api_key) + oidc_userinfo_mock.assert_not_called() + assert jwt_claims["email"] == "user@example.com" + + +@pytest.mark.asyncio +async def test_virtual_key_mapping_oidc_enabled_opaque_token_uses_oidc_userinfo(): + """ + Complement of the test above: when oidc_userinfo_enabled=True and the token + is an opaque access token (not a JWT), the virtual-key claim lookup must + call get_oidc_userinfo — not auth_jwt. + """ + # Opaque token: no dots → is_jwt() returns False + api_key = "some_opaque_access_token_with_no_dots" + + jwt_handler = JWTHandler() + jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth( + oidc_userinfo_enabled=True, + virtual_key_claim_field="email", + ) + + assert jwt_handler.is_jwt(token=api_key) is False + + auth_jwt_mock = AsyncMock(return_value={"email": "user@example.com"}) + oidc_userinfo_mock = AsyncMock(return_value={"email": "user@example.com", "sub": "123"}) + + if jwt_handler.litellm_jwtauth.oidc_userinfo_enabled and not jwt_handler.is_jwt( + token=api_key + ): + jwt_claims = await oidc_userinfo_mock(token=api_key) + else: + jwt_claims = await auth_jwt_mock(token=api_key) + + oidc_userinfo_mock.assert_called_once_with(token=api_key) + auth_jwt_mock.assert_not_called() + assert jwt_claims["sub"] == "123" + + # ────────────────────────────────────────────── # Tests: _to_response redacts hashed token # ────────────────────────────────────────────── diff --git a/tests/proxy_unit_tests/test_proxy_server.py b/tests/proxy_unit_tests/test_proxy_server.py index 59b9297d1b2..047193055d8 100644 --- a/tests/proxy_unit_tests/test_proxy_server.py +++ b/tests/proxy_unit_tests/test_proxy_server.py @@ -2787,7 +2787,9 @@ async def test_update_config_success_callback_normalization(): # Update config with mixed-case callbacks - expect normalization to lowercase config_update = ConfigYAML(litellm_settings={"success_callback": ["SQS", "sQs"]}) - await proxy_server.update_config(config_update) + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + admin_user = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-test") + await proxy_server.update_config(config_update, user_api_key_dict=admin_user) saved = mock_proxy_config.saved_config assert saved is not None, "save_config was not called" diff --git a/tests/store_model_in_db_tests/test_adding_passthrough_model.py b/tests/store_model_in_db_tests/test_adding_passthrough_model.py index 80cd7e1aabf..c8212b849a1 100644 --- a/tests/store_model_in_db_tests/test_adding_passthrough_model.py +++ b/tests/store_model_in_db_tests/test_adding_passthrough_model.py @@ -11,7 +11,6 @@ Cases to cover """ import time -import assemblyai as aai import pytest import httpx import os @@ -175,28 +174,56 @@ def make_assemblyai_basic_transcribe_request( virtual_key: str, assemblyai_base_url: str ): print("making basic transcribe request to assemblyai passthrough") + file_url = "https://assembly.ai/wildfires.mp3" + headers = { + "Authorization": f"Bearer {virtual_key}", + "Content-Type": "application/json", + } + create_payload = { + "audio_url": file_url, + "speech_models": ["universal-2"], + } - # Replace with your API key - aai.settings.api_key = f"Bearer {virtual_key}" - aai.settings.base_url = assemblyai_base_url + create_response = httpx.post( + url=f"{assemblyai_base_url}/v2/transcript", + headers=headers, + json=create_payload, + timeout=60.0, + ) + if create_response.status_code != 200: + pytest.fail( + "Failed to create transcript request: " + f"status={create_response.status_code}, body={create_response.text}" + ) - # URL of the file to transcribe - FILE_URL = "https://assembly.ai/wildfires.mp3" - - # You can also transcribe a local file by passing in a file path - # FILE_URL = './path/to/file.mp3' - - transcriber = aai.Transcriber() - transcript = transcriber.transcribe(FILE_URL) - print(transcript) - print(transcript.id) - if transcript.id: - transcript.delete_by_id(transcript.id) - else: + transcript = create_response.json() + transcript_id = transcript.get("id") + if not transcript_id: pytest.fail("Failed to get transcript id") - if transcript.status == aai.TranscriptStatus.error: - print(transcript.error) - pytest.fail(f"Failed to transcribe file error: {transcript.error}") - else: - print(transcript.text) + for _ in range(60): + poll_response = httpx.get( + url=f"{assemblyai_base_url}/v2/transcript/{transcript_id}", + headers=headers, + timeout=30.0, + ) + if poll_response.status_code != 200: + pytest.fail( + "Failed to poll transcript status: " + f"status={poll_response.status_code}, body={poll_response.text}" + ) + transcript = poll_response.json() + if transcript.get("status") in ("completed", "error"): + break + time.sleep(1) + + httpx.delete( + url=f"{assemblyai_base_url}/v2/transcript/{transcript_id}", + headers=headers, + timeout=30.0, + ) + + if transcript.get("status") == "error": + pytest.fail(f"Failed to transcribe file error: {transcript.get('error')}") + + print(transcript.get("text")) diff --git a/tests/test_litellm/llms/snowflake/chat/test_snowflake_chat_transformation.py b/tests/test_litellm/llms/snowflake/chat/test_snowflake_chat_transformation.py index 5bb4942dde6..5b18618fdf5 100644 --- a/tests/test_litellm/llms/snowflake/chat/test_snowflake_chat_transformation.py +++ b/tests/test_litellm/llms/snowflake/chat/test_snowflake_chat_transformation.py @@ -1,18 +1,22 @@ """ Unit tests for Snowflake chat transformation -Tests tool calling request/response transformations +Tests tool calling request/response transformations and chat completions """ +import asyncio import os import copy import json +from typing import Any, Dict, List -from unittest.mock import patch -from unittest.mock import MagicMock +from unittest.mock import AsyncMock, patch, Mock, MagicMock import httpx +import pytest import litellm +from litellm import completion, acompletion +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.snowflake.chat.transformation import SnowflakeConfig from litellm.types.utils import ModelResponse @@ -438,3 +442,174 @@ class TestSnowFlakeCompletion: os.environ.pop("SNOWFLAKE_ACCOUNT_ID", None) os.environ.pop("SNOWFLAKE_JWT", None) + + +FAKE_API_BASE = "https://fake-snowflake.example.com/api/v2/cortex/inference:chat" + + +def _make_mock_response(json_data: Dict[str, Any]) -> MagicMock: + mock = MagicMock(spec=httpx.Response) + mock.status_code = 200 + mock.headers = {"content-type": "application/json"} + mock.json.return_value = json_data + mock.text = json.dumps(json_data) + return mock + + +def _chat_response() -> Dict[str, Any]: + return { + "id": "chatcmpl-snowflake-123", + "object": "chat.completion", + "created": 1700000000, + "model": "mistral-7b", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "The sky above is painted blue,\nWith clouds of white and morning dew.", + }, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 10, + "completion_tokens": 30, + "total_tokens": 40, + }, + } + + +def _streaming_chunks() -> List[str]: + base = { + "id": "chatcmpl-snowflake-stream-123", + "object": "chat.completion.chunk", + "created": 1700000000, + "model": "mistral-7b", + } + deltas = [ + {"role": "assistant", "content": "The"}, + {"content": " sky"}, + {"content": " is blue"}, + ] + chunks = [] + for i, delta in enumerate(deltas): + finish = "stop" if i == len(deltas) - 1 else None + chunks.append( + json.dumps( + { + **base, + "choices": [ + {"index": 0, "delta": delta, "finish_reason": finish} + ], + } + ) + ) + return chunks + + +class TestSnowflakeChatCompletion: + """End-to-end chat completion tests (mocked HTTP).""" + + messages = [{"role": "user", "content": "Write me a poem about the blue sky"}] + + @pytest.mark.parametrize("sync_mode", [True, False]) + def test_chat_completion_snowflake(self, sync_mode): + mock_resp = _make_mock_response(_chat_response()) + + if sync_mode: + with patch.object(HTTPHandler, "post", return_value=mock_resp) as mock_post: + response = completion( + model="snowflake/mistral-7b", + messages=self.messages, + api_key="fake-jwt", + account_id="FAKE-ACCOUNT", + api_base=FAKE_API_BASE, + ) + mock_post.assert_called_once() + else: + with patch.object( + AsyncHTTPHandler, "post", new_callable=AsyncMock, return_value=mock_resp + ) as mock_post: + response = asyncio.run( + acompletion( + model="snowflake/mistral-7b", + messages=self.messages, + api_key="fake-jwt", + account_id="FAKE-ACCOUNT", + api_base=FAKE_API_BASE, + ) + ) + mock_post.assert_called_once() + + assert response is not None + assert response.choices[0].message.content is not None + assert "sky" in response.choices[0].message.content.lower() + assert response.usage.prompt_tokens == 10 + assert response.usage.completion_tokens == 30 + + @pytest.mark.parametrize("sync_mode", [True, False]) + def test_chat_completion_snowflake_stream(self, sync_mode): + raw_chunks = _streaming_chunks() + + if sync_mode: + + def _iter_lines(): + for chunk in raw_chunks: + yield f"data: {chunk}" + yield "data: [DONE]" + + mock_resp = MagicMock() + mock_resp.iter_lines.return_value = _iter_lines() + mock_resp.status_code = 200 + mock_resp.headers = {"content-type": "text/event-stream"} + + with patch.object(HTTPHandler, "post", return_value=mock_resp) as mock_post: + response = completion( + model="snowflake/mistral-7b", + messages=self.messages, + max_tokens=100, + stream=True, + api_key="fake-jwt", + account_id="FAKE-ACCOUNT", + api_base=FAKE_API_BASE, + ) + chunks_received = list(response) + mock_post.assert_called_once() + else: + + async def _aiter_lines(): + for chunk in raw_chunks: + yield f"data: {chunk}" + yield "data: [DONE]" + + mock_resp = MagicMock() + mock_resp.aiter_lines.return_value = _aiter_lines() + mock_resp.status_code = 200 + mock_resp.headers = {"content-type": "text/event-stream"} + + async def _run(): + with patch.object( + AsyncHTTPHandler, "post", new_callable=AsyncMock, return_value=mock_resp + ) as mock_post: + resp = await acompletion( + model="snowflake/mistral-7b", + messages=self.messages, + max_tokens=100, + stream=True, + api_key="fake-jwt", + account_id="FAKE-ACCOUNT", + api_base=FAKE_API_BASE, + ) + received = [] + async for chunk in resp: + received.append(chunk) + mock_post.assert_called_once() + return received + + chunks_received = asyncio.run(_run()) + + assert len(chunks_received) > 0 + content = "".join( + c.choices[0].delta.content for c in chunks_received if c.choices[0].delta.content + ) diff --git a/tests/test_litellm/proxy/auth/test_handle_jwt.py b/tests/test_litellm/proxy/auth/test_handle_jwt.py index ada67fbba88..5303da6fbcf 100644 --- a/tests/test_litellm/proxy/auth/test_handle_jwt.py +++ b/tests/test_litellm/proxy/auth/test_handle_jwt.py @@ -305,24 +305,28 @@ async def test_sync_user_role_and_teams(): # Create mock objects for required types mock_user_api_key_cache = MagicMock() mock_proxy_logging_obj = MagicMock() - + jwt_handler = JWTHandler() jwt_handler.update_environment( prisma_client=None, user_api_key_cache=mock_user_api_key_cache, litellm_jwtauth=LiteLLM_JWTAuth( jwt_litellm_role_map=[ - JWTLiteLLMRoleMap(jwt_role="ADMIN", litellm_role=LitellmUserRoles.PROXY_ADMIN) + JWTLiteLLMRoleMap( + jwt_role="ADMIN", litellm_role=LitellmUserRoles.PROXY_ADMIN + ) ], roles_jwt_field="roles", team_ids_jwt_field="my_id_teams", - sync_user_role_and_teams=True + sync_user_role_and_teams=True, ), ) token = {"roles": ["ADMIN"], "my_id_teams": ["team1", "team2"]} - user = LiteLLM_UserTable(user_id="u1", user_role=LitellmUserRoles.INTERNAL_USER.value, teams=["team2"]) + user = LiteLLM_UserTable( + user_id="u1", user_role=LitellmUserRoles.INTERNAL_USER.value, teams=["team2"] + ) prisma = AsyncMock() prisma.db.litellm_usertable.update = AsyncMock() @@ -350,7 +354,9 @@ async def test_sync_user_role_and_teams_cache_invalidation_on_role_change(): user_api_key_cache=AsyncMock(), litellm_jwtauth=LiteLLM_JWTAuth( jwt_litellm_role_map=[ - JWTLiteLLMRoleMap(jwt_role="ADMIN", litellm_role=LitellmUserRoles.PROXY_ADMIN) + JWTLiteLLMRoleMap( + jwt_role="ADMIN", litellm_role=LitellmUserRoles.PROXY_ADMIN + ) ], roles_jwt_field="roles", team_ids_jwt_field="my_id_teams", @@ -375,7 +381,9 @@ async def test_sync_user_role_and_teams_cache_invalidation_on_role_change(): mock_cache.async_set_cache.assert_called_once() call_kwargs = mock_cache.async_set_cache.call_args assert call_kwargs.kwargs["key"] == "u1" - assert call_kwargs.kwargs["value"]["user_role"] == LitellmUserRoles.PROXY_ADMIN.value + assert ( + call_kwargs.kwargs["value"]["user_role"] == LitellmUserRoles.PROXY_ADMIN.value + ) @pytest.mark.asyncio @@ -389,7 +397,9 @@ async def test_sync_user_role_and_teams_cache_invalidation_on_team_change(): user_api_key_cache=AsyncMock(), litellm_jwtauth=LiteLLM_JWTAuth( jwt_litellm_role_map=[ - JWTLiteLLMRoleMap(jwt_role="ADMIN", litellm_role=LitellmUserRoles.PROXY_ADMIN) + JWTLiteLLMRoleMap( + jwt_role="ADMIN", litellm_role=LitellmUserRoles.PROXY_ADMIN + ) ], roles_jwt_field="roles", team_ids_jwt_field="my_id_teams", @@ -432,7 +442,9 @@ async def test_sync_user_role_and_teams_no_cache_write_when_nothing_changes(): user_api_key_cache=AsyncMock(), litellm_jwtauth=LiteLLM_JWTAuth( jwt_litellm_role_map=[ - JWTLiteLLMRoleMap(jwt_role="ADMIN", litellm_role=LitellmUserRoles.PROXY_ADMIN) + JWTLiteLLMRoleMap( + jwt_role="ADMIN", litellm_role=LitellmUserRoles.PROXY_ADMIN + ) ], roles_jwt_field="roles", team_ids_jwt_field="my_id_teams", @@ -463,7 +475,7 @@ async def test_map_jwt_role_to_litellm_role(): # Create mock objects for required types mock_user_api_key_cache = MagicMock() - + jwt_handler = JWTHandler() jwt_handler.update_environment( prisma_client=None, @@ -471,13 +483,21 @@ async def test_map_jwt_role_to_litellm_role(): litellm_jwtauth=LiteLLM_JWTAuth( jwt_litellm_role_map=[ # Exact match - JWTLiteLLMRoleMap(jwt_role="ADMIN", litellm_role=LitellmUserRoles.PROXY_ADMIN), + JWTLiteLLMRoleMap( + jwt_role="ADMIN", litellm_role=LitellmUserRoles.PROXY_ADMIN + ), # Wildcard patterns - JWTLiteLLMRoleMap(jwt_role="user_*", litellm_role=LitellmUserRoles.INTERNAL_USER), - JWTLiteLLMRoleMap(jwt_role="team_?", litellm_role=LitellmUserRoles.TEAM), - JWTLiteLLMRoleMap(jwt_role="dev_[123]", litellm_role=LitellmUserRoles.INTERNAL_USER), + JWTLiteLLMRoleMap( + jwt_role="user_*", litellm_role=LitellmUserRoles.INTERNAL_USER + ), + JWTLiteLLMRoleMap( + jwt_role="team_?", litellm_role=LitellmUserRoles.TEAM + ), + JWTLiteLLMRoleMap( + jwt_role="dev_[123]", litellm_role=LitellmUserRoles.INTERNAL_USER + ), ], - roles_jwt_field="roles" + roles_jwt_field="roles", ), ) @@ -547,7 +567,9 @@ async def test_map_jwt_role_to_litellm_role(): # Test patterns that don't match character classes jwt_handler.litellm_jwtauth.jwt_litellm_role_map = [ - JWTLiteLLMRoleMap(jwt_role="dev_[123]", litellm_role=LitellmUserRoles.INTERNAL_USER), + JWTLiteLLMRoleMap( + jwt_role="dev_[123]", litellm_role=LitellmUserRoles.INTERNAL_USER + ), ] token = {"roles": ["dev_4"]} # 4 is not in [123] result = jwt_handler.map_jwt_role_to_litellm_role(token) @@ -570,7 +592,7 @@ async def test_map_jwt_role_to_litellm_role(): async def test_nested_jwt_field_access(): """ Test that all JWT fields support dot notation for nested access - + This test verifies that: 1. All JWT field methods can access nested values using dot notation 2. Backward compatibility is maintained for flat field names @@ -581,33 +603,18 @@ async def test_nested_jwt_field_access(): # Create JWT handler jwt_handler = JWTHandler() - + # Test token with nested claims nested_token = { - "user": { - "sub": "u123", - "email": "user@example.com" - }, - "resource_access": { - "my-client": { - "roles": ["admin", "user"] - } - }, + "user": {"sub": "u123", "email": "user@example.com"}, + "resource_access": {"my-client": {"roles": ["admin", "user"]}}, "groups": ["team1", "team2"], - "organization": { - "id": "org456" - }, - "profile": { - "object_id": "obj789" - }, - "customer": { - "end_user_id": "customer123" - }, - "tenant": { - "team_id": "team456" - } + "organization": {"id": "org456"}, + "profile": {"object_id": "obj789"}, + "customer": {"end_user_id": "customer123"}, + "tenant": {"team_id": "team456"}, } - + # Test flat token for backward compatibility flat_token = { "sub": "u123", @@ -617,13 +624,13 @@ async def test_nested_jwt_field_access(): "org_id": "org456", "object_id": "obj789", "end_user_id": "customer123", - "team_id": "team456" + "team_id": "team456", } # Test 1: user_id_jwt_field with nested access jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth(user_id_jwt_field="user.sub") assert jwt_handler.get_user_id(nested_token, None) == "u123" - + # Test 1b: user_id_jwt_field with flat access (backward compatibility) jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth(user_id_jwt_field="sub") assert jwt_handler.get_user_id(flat_token, None) == "u123" @@ -631,7 +638,7 @@ async def test_nested_jwt_field_access(): # Test 2: user_email_jwt_field with nested access jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth(user_email_jwt_field="user.email") assert jwt_handler.get_user_email(nested_token, None) == "user@example.com" - + # Test 2b: user_email_jwt_field with flat access (backward compatibility) jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth(user_email_jwt_field="email") assert jwt_handler.get_user_email(flat_token, None) == "user@example.com" @@ -639,7 +646,7 @@ async def test_nested_jwt_field_access(): # Test 3: team_ids_jwt_field with nested access jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth(team_ids_jwt_field="groups") assert jwt_handler.get_team_ids_from_jwt(nested_token) == ["team1", "team2"] - + # Test 3b: team_ids_jwt_field with flat access (backward compatibility) jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth(team_ids_jwt_field="groups") assert jwt_handler.get_team_ids_from_jwt(flat_token) == ["team1", "team2"] @@ -647,30 +654,37 @@ async def test_nested_jwt_field_access(): # Test 4: org_id_jwt_field with nested access jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth(org_id_jwt_field="organization.id") assert jwt_handler.get_org_id(nested_token, None) == "org456" - + # Test 4b: org_id_jwt_field with flat access (backward compatibility) jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth(org_id_jwt_field="org_id") assert jwt_handler.get_org_id(flat_token, None) == "org456" # Test 5: object_id_jwt_field with nested access (requires role_mappings) from litellm.proxy._types import LitellmUserRoles, RoleMapping + jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth( object_id_jwt_field="profile.object_id", - role_mappings=[RoleMapping(role="admin", internal_role=LitellmUserRoles.INTERNAL_USER)] + role_mappings=[ + RoleMapping(role="admin", internal_role=LitellmUserRoles.INTERNAL_USER) + ], ) assert jwt_handler.get_object_id(nested_token, None) == "obj789" - + # Test 5b: object_id_jwt_field with flat access (backward compatibility) jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth( object_id_jwt_field="object_id", - role_mappings=[RoleMapping(role="admin", internal_role=LitellmUserRoles.INTERNAL_USER)] + role_mappings=[ + RoleMapping(role="admin", internal_role=LitellmUserRoles.INTERNAL_USER) + ], ) assert jwt_handler.get_object_id(flat_token, None) == "obj789" # Test 6: end_user_id_jwt_field with nested access - jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth(end_user_id_jwt_field="customer.end_user_id") + jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth( + end_user_id_jwt_field="customer.end_user_id" + ) assert jwt_handler.get_end_user_id(nested_token, None) == "customer123" - + # Test 6b: end_user_id_jwt_field with flat access (backward compatibility) jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth(end_user_id_jwt_field="end_user_id") assert jwt_handler.get_end_user_id(flat_token, None) == "customer123" @@ -678,19 +692,21 @@ async def test_nested_jwt_field_access(): # Test 7: team_id_jwt_field with nested access jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth(team_id_jwt_field="tenant.team_id") assert jwt_handler.get_team_id(nested_token, None) == "team456" - + # Test 7b: team_id_jwt_field with flat access (backward compatibility) jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth(team_id_jwt_field="team_id") assert jwt_handler.get_team_id(flat_token, None) == "team456" # Test 8: roles_jwt_field with deeply nested access (already supported, but testing) - jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth(roles_jwt_field="resource_access.my-client.roles") + jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth( + roles_jwt_field="resource_access.my-client.roles" + ) assert jwt_handler.get_jwt_role(nested_token, []) == ["admin", "user"] # Test 9: user_roles_jwt_field with nested access (already supported, but testing) jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth( user_roles_jwt_field="resource_access.my-client.roles", - user_allowed_roles=["admin", "user"] + user_allowed_roles=["admin", "user"], ) assert jwt_handler.get_user_roles(nested_token, []) == ["admin", "user"] @@ -699,7 +715,7 @@ async def test_nested_jwt_field_access(): async def test_nested_jwt_field_missing_paths(): """ Test handling of missing nested paths in JWT tokens - + This test verifies that: 1. Missing nested paths return appropriate defaults 2. Partial paths that exist but don't have the final key return defaults @@ -710,7 +726,7 @@ async def test_nested_jwt_field_missing_paths(): # Create JWT handler jwt_handler = JWTHandler() - + # Test token with missing nested paths incomplete_token = { "user": { @@ -718,9 +734,7 @@ async def test_nested_jwt_field_missing_paths(): # missing "sub" and "email" }, "resource_access": { - "other-client": { - "roles": ["viewer"] - } + "other-client": {"roles": ["viewer"]} # missing "my-client" } # missing "organization", "profile", "customer", "tenant", "groups" @@ -732,7 +746,10 @@ async def test_nested_jwt_field_missing_paths(): # Test 2: Missing user.email should return default jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth(user_email_jwt_field="user.email") - assert jwt_handler.get_user_email(incomplete_token, "default@example.com") == "default@example.com" + assert ( + jwt_handler.get_user_email(incomplete_token, "default@example.com") + == "default@example.com" + ) # Test 3: Missing groups should return empty list jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth(team_ids_jwt_field="groups") @@ -744,40 +761,53 @@ async def test_nested_jwt_field_missing_paths(): # Test 5: Missing profile.object_id should return default (requires role_mappings) from litellm.proxy._types import LitellmUserRoles, RoleMapping + jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth( object_id_jwt_field="profile.object_id", - role_mappings=[RoleMapping(role="admin", internal_role=LitellmUserRoles.INTERNAL_USER)] + role_mappings=[ + RoleMapping(role="admin", internal_role=LitellmUserRoles.INTERNAL_USER) + ], ) assert jwt_handler.get_object_id(incomplete_token, "default_obj") == "default_obj" # Test 6: Missing customer.end_user_id should return default - jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth(end_user_id_jwt_field="customer.end_user_id") - assert jwt_handler.get_end_user_id(incomplete_token, "default_customer") == "default_customer" + jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth( + end_user_id_jwt_field="customer.end_user_id" + ) + assert ( + jwt_handler.get_end_user_id(incomplete_token, "default_customer") + == "default_customer" + ) # Test 7: Missing tenant.team_id should use team_id_default fallback jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth( - team_id_jwt_field="tenant.team_id", - team_id_default="fallback_team" + team_id_jwt_field="tenant.team_id", team_id_default="fallback_team" ) assert jwt_handler.get_team_id(incomplete_token, "default_team") == "fallback_team" # Test 8: Missing resource_access.my-client.roles should return default - jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth(roles_jwt_field="resource_access.my-client.roles") - assert jwt_handler.get_jwt_role(incomplete_token, ["default_role"]) == ["default_role"] + jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth( + roles_jwt_field="resource_access.my-client.roles" + ) + assert jwt_handler.get_jwt_role(incomplete_token, ["default_role"]) == [ + "default_role" + ] # Test 9: Missing nested user roles should return default jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth( user_roles_jwt_field="resource_access.my-client.roles", - user_allowed_roles=["admin", "user"] + user_allowed_roles=["admin", "user"], ) - assert jwt_handler.get_user_roles(incomplete_token, ["default_user_role"]) == ["default_user_role"] + assert jwt_handler.get_user_roles(incomplete_token, ["default_user_role"]) == [ + "default_user_role" + ] -@pytest.mark.asyncio +@pytest.mark.asyncio async def test_metadata_prefix_handling_in_nested_fields(): """ Test that metadata. prefix is properly handled in nested JWT field access - + The get_nested_value function should remove metadata. prefix before traversing """ from litellm.proxy._types import LiteLLM_JWTAuth @@ -785,17 +815,19 @@ async def test_metadata_prefix_handling_in_nested_fields(): # Create JWT handler jwt_handler = JWTHandler() - + # Test token with proper structure for metadata prefix removal token = { "user": { "email": "user@example.com" # This will be accessed when metadata.user.email is used }, - "sub": "u123" + "sub": "u123", } # Test 1: metadata.user.email should access user.email after prefix removal - jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth(user_email_jwt_field="metadata.user.email") + jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth( + user_email_jwt_field="metadata.user.email" + ) # The get_nested_value function removes "metadata." prefix, so "metadata.user.email" becomes "user.email" assert jwt_handler.get_user_email(token, None) == "user@example.com" @@ -871,24 +903,21 @@ async def test_auth_builder_returns_team_membership_object(): # Create mock objects from litellm.proxy._types import LiteLLM_BudgetTable, LiteLLM_TeamMembership - + mock_team_membership = LiteLLM_TeamMembership( user_id=_user_id, team_id=_team_id, budget_id="budget_123", spend=10.5, litellm_budget_table=LiteLLM_BudgetTable( - budget_id="budget_123", - rpm_limit=100, - tpm_limit=5000 - ) + budget_id="budget_123", rpm_limit=100, tpm_limit=5000 + ), ) - + user_object = LiteLLM_UserTable( - user_id=_user_id, - user_role=LitellmUserRoles.INTERNAL_USER + user_id=_user_id, user_role=LitellmUserRoles.INTERNAL_USER ) - + team_object = LiteLLM_TeamTable(team_id=_team_id) # Create mock JWT handler @@ -958,12 +987,24 @@ async def test_auth_builder_returns_team_membership_object(): ) # Verify that team_membership_object is returned - assert result["team_membership"] is not None, "team_membership should be present" - assert result["team_membership"] == mock_team_membership, "team_membership should match the mock object" - assert result["team_membership"].user_id == _user_id, "team_membership user_id should match" - assert result["team_membership"].team_id == _team_id, "team_membership team_id should match" - assert result["team_membership"].budget_id == "budget_123", "team_membership budget_id should match" - assert result["team_membership"].spend == 10.5, "team_membership spend should match" + assert ( + result["team_membership"] is not None + ), "team_membership should be present" + assert ( + result["team_membership"] == mock_team_membership + ), "team_membership should match the mock object" + assert ( + result["team_membership"].user_id == _user_id + ), "team_membership user_id should match" + assert ( + result["team_membership"].team_id == _team_id + ), "team_membership team_id should match" + assert ( + result["team_membership"].budget_id == "budget_123" + ), "team_membership budget_id should match" + assert ( + result["team_membership"].spend == 10.5 + ), "team_membership spend should match" @pytest.mark.asyncio @@ -979,16 +1020,16 @@ async def test_auth_builder_with_oidc_userinfo_enabled(): request_data = {"model": "gpt-4"} general_settings = {"enforce_rbac": False} route = "/chat/completions" - + user_object = LiteLLM_UserTable( user_id="test_user_1", user_role=LitellmUserRoles.INTERNAL_USER ) - + # Create JWT handler with OIDC UserInfo enabled jwt_handler = JWTHandler() user_api_key_cache = DualCache() proxy_logging_obj = ProxyLogging(user_api_key_cache=user_api_key_cache) - + jwt_handler.update_environment( prisma_client=None, user_api_key_cache=user_api_key_cache, @@ -999,14 +1040,14 @@ async def test_auth_builder_with_oidc_userinfo_enabled(): user_email_jwt_field="email", ), ) - + # Mock OIDC UserInfo response userinfo_response = { "sub": "test_user_1", "email": "test@example.com", "scope": "", } - + # Mock all the dependencies with patch.object( jwt_handler, "get_oidc_userinfo", new_callable=AsyncMock @@ -1057,7 +1098,7 @@ async def test_auth_builder_with_oidc_userinfo_enabled(): ) as mock_sync_user: # Set up mock return values mock_get_userinfo.return_value = userinfo_response - + # Call auth_builder result = await JWTAuthManager.auth_builder( api_key=api_key, @@ -1070,11 +1111,11 @@ async def test_auth_builder_with_oidc_userinfo_enabled(): parent_otel_span=None, proxy_logging_obj=proxy_logging_obj, ) - + # Verify that get_oidc_userinfo was called instead of auth_jwt mock_get_userinfo.assert_called_once_with(token=api_key) mock_auth_jwt.assert_not_called() # Should not be called when OIDC is enabled - + # Verify the result assert result["user_id"] == "test_user_1" assert result["user_object"] == user_object @@ -1093,16 +1134,16 @@ async def test_auth_builder_with_oidc_userinfo_disabled(): request_data = {"model": "gpt-4"} general_settings = {"enforce_rbac": False} route = "/chat/completions" - + user_object = LiteLLM_UserTable( user_id="test_user_1", user_role=LitellmUserRoles.INTERNAL_USER ) - + # Create JWT handler with OIDC UserInfo disabled jwt_handler = JWTHandler() user_api_key_cache = DualCache() proxy_logging_obj = ProxyLogging(user_api_key_cache=user_api_key_cache) - + jwt_handler.update_environment( prisma_client=None, user_api_key_cache=user_api_key_cache, @@ -1111,13 +1152,13 @@ async def test_auth_builder_with_oidc_userinfo_disabled(): user_id_jwt_field="sub", ), ) - + # Mock JWT validation response jwt_response = { "sub": "test_user_1", "scope": "", } - + # Mock all the dependencies with patch.object( jwt_handler, "get_oidc_userinfo", new_callable=AsyncMock @@ -1168,7 +1209,7 @@ async def test_auth_builder_with_oidc_userinfo_disabled(): ) as mock_sync_user: # Set up mock return values mock_auth_jwt.return_value = jwt_response - + # Call auth_builder result = await JWTAuthManager.auth_builder( api_key=api_key, @@ -1181,16 +1222,125 @@ async def test_auth_builder_with_oidc_userinfo_disabled(): parent_otel_span=None, proxy_logging_obj=proxy_logging_obj, ) - + # Verify that auth_jwt was called instead of get_oidc_userinfo mock_auth_jwt.assert_called_once_with(token=api_key) mock_get_userinfo.assert_not_called() # Should not be called when OIDC is disabled - + # Verify the result assert result["user_id"] == "test_user_1" assert result["user_object"] == user_object +@pytest.mark.asyncio +async def test_auth_builder_oidc_enabled_falls_back_to_jwt_auth_for_jwt_tokens(): + """ + Regression test for the is_jwt routing fix. + + When oidc_userinfo_enabled=True but the supplied token is a well-formed + JWT (three dot-separated parts), auth_builder must call auth_jwt and skip + get_oidc_userinfo. Sending a standard JWT to the OIDC UserInfo endpoint + is incorrect — the endpoint expects an opaque access token. + """ + from litellm.caching import DualCache + from litellm.proxy.utils import ProxyLogging + + # Three-part token: recognised as a JWT by is_jwt() + api_key = "eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJ0ZXN0X3VzZXIifQ.some_signature" + request_data = {"model": "gpt-4"} + general_settings = {"enforce_rbac": False} + route = "/chat/completions" + + user_object = LiteLLM_UserTable( + user_id="test_user_1", user_role=LitellmUserRoles.INTERNAL_USER + ) + + jwt_handler = JWTHandler() + user_api_key_cache = DualCache() + proxy_logging_obj = ProxyLogging(user_api_key_cache=user_api_key_cache) + + jwt_handler.update_environment( + prisma_client=None, + user_api_key_cache=user_api_key_cache, + litellm_jwtauth=LiteLLM_JWTAuth( + oidc_userinfo_enabled=True, + oidc_userinfo_endpoint="https://example.com/oauth2/userinfo", + user_id_jwt_field="sub", + ), + ) + + jwt_response = {"sub": "test_user_1", "scope": ""} + + with patch.object( + jwt_handler, "get_oidc_userinfo", new_callable=AsyncMock + ) as mock_get_userinfo, patch.object( + jwt_handler, "auth_jwt", new_callable=AsyncMock + ) as mock_auth_jwt, patch.object( + JWTAuthManager, "check_rbac_role", new_callable=AsyncMock + ), patch.object( + jwt_handler, "get_rbac_role", return_value=None + ), patch.object( + jwt_handler, "get_scopes", return_value=[] + ), patch.object( + jwt_handler, "get_object_id", return_value=None + ), patch.object( + JWTAuthManager, + "get_user_info", + new_callable=AsyncMock, + return_value=("test_user_1", None, None), + ), patch.object( + jwt_handler, "get_org_id", return_value=None + ), patch.object( + jwt_handler, "get_end_user_id", return_value=None + ), patch.object( + JWTAuthManager, "check_admin_access", new_callable=AsyncMock, return_value=None + ), patch.object( + JWTAuthManager, + "find_and_validate_specific_team_id", + new_callable=AsyncMock, + return_value=(None, None), + ), patch.object( + JWTAuthManager, "get_all_team_ids", return_value=set() + ), patch.object( + JWTAuthManager, + "find_team_with_model_access", + new_callable=AsyncMock, + return_value=(None, None), + ), patch.object( + JWTAuthManager, + "get_objects", + new_callable=AsyncMock, + return_value=(user_object, None, None, None), + ), patch.object( + JWTAuthManager, "map_user_to_teams", new_callable=AsyncMock + ), patch.object( + JWTAuthManager, "validate_object_id", return_value=True + ), patch.object( + JWTAuthManager, "sync_user_role_and_teams", new_callable=AsyncMock + ): + mock_auth_jwt.return_value = jwt_response + + result = await JWTAuthManager.auth_builder( + api_key=api_key, + jwt_handler=jwt_handler, + request_data=request_data, + general_settings=general_settings, + route=route, + prisma_client=None, + user_api_key_cache=user_api_key_cache, + parent_otel_span=None, + proxy_logging_obj=proxy_logging_obj, + ) + + # Token is a JWT, so standard JWT auth must be used even when + # oidc_userinfo_enabled is True. + mock_auth_jwt.assert_called_once_with(token=api_key) + mock_get_userinfo.assert_not_called() + + assert result["user_id"] == "test_user_1" + assert result["user_object"] == user_object + + def test_get_team_id_from_header(): """Test get_team_id_from_header returns team when valid, None when missing, raises on invalid.""" from fastapi import HTTPException @@ -1236,17 +1386,33 @@ async def test_auth_builder_uses_team_from_header_e2e(): ) team_object = LiteLLM_TeamTable(team_id="team-2") - user_object = LiteLLM_UserTable(user_id="user-1", user_role=LitellmUserRoles.INTERNAL_USER) + user_object = LiteLLM_UserTable( + user_id="user-1", user_role=LitellmUserRoles.INTERNAL_USER + ) - with patch.object(jwt_handler, "auth_jwt", new_callable=AsyncMock) as mock_auth_jwt, \ - patch.object(JWTAuthManager, "check_rbac_role", new_callable=AsyncMock), \ - patch.object(JWTAuthManager, "check_admin_access", new_callable=AsyncMock, return_value=None), \ - patch("litellm.proxy.auth.handle_jwt.get_team_object", new_callable=AsyncMock) as mock_get_team, \ - patch.object(JWTAuthManager, "get_objects", new_callable=AsyncMock, return_value=(user_object, None, None, None)), \ - patch.object(JWTAuthManager, "map_user_to_teams", new_callable=AsyncMock), \ - patch.object(JWTAuthManager, "sync_user_role_and_teams", new_callable=AsyncMock): - - mock_auth_jwt.return_value = {"sub": "user-1", "scope": "", "groups": ["team-1", "team-2"]} + with patch.object( + jwt_handler, "auth_jwt", new_callable=AsyncMock + ) as mock_auth_jwt, patch.object( + JWTAuthManager, "check_rbac_role", new_callable=AsyncMock + ), patch.object( + JWTAuthManager, "check_admin_access", new_callable=AsyncMock, return_value=None + ), patch( + "litellm.proxy.auth.handle_jwt.get_team_object", new_callable=AsyncMock + ) as mock_get_team, patch.object( + JWTAuthManager, + "get_objects", + new_callable=AsyncMock, + return_value=(user_object, None, None, None), + ), patch.object( + JWTAuthManager, "map_user_to_teams", new_callable=AsyncMock + ), patch.object( + JWTAuthManager, "sync_user_role_and_teams", new_callable=AsyncMock + ): + mock_auth_jwt.return_value = { + "sub": "user-1", + "scope": "", + "groups": ["team-1", "team-2"], + } mock_get_team.return_value = team_object result = await JWTAuthManager.auth_builder( @@ -1275,29 +1441,29 @@ async def test_get_team_alias_with_nested_fields(): from litellm.proxy.auth.handle_jwt import JWTHandler jwt_handler = JWTHandler() - + # Test token with nested team name nested_token = { - "organization": { - "team": { - "name": "engineering-team" - } - }, - "team_name": "flat-team" + "organization": {"team": {"name": "engineering-team"}}, + "team_name": "flat-team", } - + # Test nested access - jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth(team_alias_jwt_field="organization.team.name") + jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth( + team_alias_jwt_field="organization.team.name" + ) assert jwt_handler.get_team_alias(nested_token, None) == "engineering-team" - + # Test flat access (backward compatibility) jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth(team_alias_jwt_field="team_name") assert jwt_handler.get_team_alias(nested_token, None) == "flat-team" - + # Test missing field returns default - jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth(team_alias_jwt_field="nonexistent.field") + jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth( + team_alias_jwt_field="nonexistent.field" + ) assert jwt_handler.get_team_alias(nested_token, "default-team") == "default-team" - + # Test with team_alias_jwt_field not configured jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth() # team_alias_jwt_field is None assert jwt_handler.get_team_alias(nested_token, "default") is None @@ -1312,23 +1478,22 @@ async def test_is_required_team_id_with_team_alias_field(): from litellm.proxy.auth.handle_jwt import JWTHandler jwt_handler = JWTHandler() - + # Neither field set - should return False jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth() assert jwt_handler.is_required_team_id() is False - + # Only team_id_jwt_field set - should return True jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth(team_id_jwt_field="team_id") assert jwt_handler.is_required_team_id() is True - + # Only team_alias_jwt_field set - should return True jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth(team_alias_jwt_field="team_name") assert jwt_handler.is_required_team_id() is True - + # Both fields set - should return True jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth( - team_id_jwt_field="team_id", - team_alias_jwt_field="team_name" + team_id_jwt_field="team_id", team_alias_jwt_field="team_name" ) assert jwt_handler.is_required_team_id() is True @@ -1348,30 +1513,24 @@ async def test_find_and_validate_specific_team_id_with_team_alias(): jwt_handler = JWTHandler() user_api_key_cache = DualCache() proxy_logging_obj = ProxyLogging(user_api_key_cache=user_api_key_cache) - + jwt_handler.update_environment( prisma_client=None, user_api_key_cache=user_api_key_cache, - litellm_jwtauth=LiteLLM_JWTAuth( - team_alias_jwt_field="team_alias" - ), + litellm_jwtauth=LiteLLM_JWTAuth(team_alias_jwt_field="team_alias"), ) - + # Token with team name (no team_id) - jwt_token = { - "sub": "user-1", - "team_alias": "my-team" - } - + jwt_token = {"sub": "user-1", "team_alias": "my-team"} + # Mock team object returned by get_team_object_by_alias team_object = LiteLLM_TeamTable(team_id="resolved-team-id", team_alias="my-team") - + with patch( - "litellm.proxy.auth.handle_jwt.get_team_object_by_alias", - new_callable=AsyncMock + "litellm.proxy.auth.handle_jwt.get_team_object_by_alias", new_callable=AsyncMock ) as mock_get_by_alias: mock_get_by_alias.return_value = team_object - + team_id, result_team = await JWTAuthManager.find_and_validate_specific_team_id( jwt_handler=jwt_handler, jwt_valid_token=jwt_token, @@ -1380,7 +1539,7 @@ async def test_find_and_validate_specific_team_id_with_team_alias(): parent_otel_span=None, proxy_logging_obj=proxy_logging_obj, ) - + # Should have resolved team_id from team name assert team_id == "resolved-team-id" assert result_team == team_object @@ -1408,35 +1567,28 @@ async def test_find_and_validate_team_id_takes_precedence_over_name(): jwt_handler = JWTHandler() user_api_key_cache = DualCache() proxy_logging_obj = ProxyLogging(user_api_key_cache=user_api_key_cache) - + jwt_handler.update_environment( prisma_client=None, user_api_key_cache=user_api_key_cache, litellm_jwtauth=LiteLLM_JWTAuth( - team_id_jwt_field="team_id", - team_alias_jwt_field="team_alias" + team_id_jwt_field="team_id", team_alias_jwt_field="team_alias" ), ) - + # Token with both team_id and team name - jwt_token = { - "sub": "user-1", - "team_id": "direct-team-id", - "team_alias": "my-team" - } - + jwt_token = {"sub": "user-1", "team_id": "direct-team-id", "team_alias": "my-team"} + # Mock team object returned by get_team_object (by ID) team_object = LiteLLM_TeamTable(team_id="direct-team-id") - + with patch( - "litellm.proxy.auth.handle_jwt.get_team_object", - new_callable=AsyncMock + "litellm.proxy.auth.handle_jwt.get_team_object", new_callable=AsyncMock ) as mock_get_by_id, patch( - "litellm.proxy.auth.handle_jwt.get_team_object_by_alias", - new_callable=AsyncMock + "litellm.proxy.auth.handle_jwt.get_team_object_by_alias", new_callable=AsyncMock ) as mock_get_by_alias: mock_get_by_id.return_value = team_object - + team_id, result_team = await JWTAuthManager.find_and_validate_specific_team_id( jwt_handler=jwt_handler, jwt_valid_token=jwt_token, @@ -1445,7 +1597,7 @@ async def test_find_and_validate_team_id_takes_precedence_over_name(): parent_otel_span=None, proxy_logging_obj=proxy_logging_obj, ) - + # Should use team_id directly, not resolve by name assert team_id == "direct-team-id" assert result_team == team_object @@ -1466,7 +1618,7 @@ async def test_find_and_validate_raises_when_required_team_not_found(): jwt_handler = JWTHandler() user_api_key_cache = DualCache() proxy_logging_obj = ProxyLogging(user_api_key_cache=user_api_key_cache) - + jwt_handler.update_environment( prisma_client=None, user_api_key_cache=user_api_key_cache, @@ -1474,12 +1626,10 @@ async def test_find_and_validate_raises_when_required_team_not_found(): team_alias_jwt_field="team_alias" # Required, but not in token ), ) - + # Token without team info - jwt_token = { - "sub": "user-1" - } - + jwt_token = {"sub": "user-1"} + with pytest.raises(Exception) as exc_info: await JWTAuthManager.find_and_validate_specific_team_id( jwt_handler=jwt_handler, @@ -1489,7 +1639,7 @@ async def test_find_and_validate_raises_when_required_team_not_found(): parent_otel_span=None, proxy_logging_obj=proxy_logging_obj, ) - + assert "No team found in token" in str(exc_info.value) assert "team_alias field 'team_alias'" in str(exc_info.value) @@ -1503,29 +1653,29 @@ async def test_get_org_alias_with_nested_fields(): from litellm.proxy.auth.handle_jwt import JWTHandler jwt_handler = JWTHandler() - + # Test token with nested org name nested_token = { - "company": { - "organization": { - "name": "acme-corp" - } - }, - "org_name": "flat-org" + "company": {"organization": {"name": "acme-corp"}}, + "org_name": "flat-org", } - + # Test nested access - jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth(org_alias_jwt_field="company.organization.name") + jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth( + org_alias_jwt_field="company.organization.name" + ) assert jwt_handler.get_org_alias(nested_token, None) == "acme-corp" - + # Test flat access jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth(org_alias_jwt_field="org_name") assert jwt_handler.get_org_alias(nested_token, None) == "flat-org" - + # Test missing field returns default - jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth(org_alias_jwt_field="nonexistent.field") + jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth( + org_alias_jwt_field="nonexistent.field" + ) assert jwt_handler.get_org_alias(nested_token, "default-org") == "default-org" - + # Test with org_alias_jwt_field not configured jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth() assert jwt_handler.get_org_alias(nested_token, "default") is None @@ -1544,15 +1694,13 @@ async def test_get_objects_resolves_org_by_name(): jwt_handler = JWTHandler() user_api_key_cache = DualCache() proxy_logging_obj = ProxyLogging(user_api_key_cache=user_api_key_cache) - + jwt_handler.update_environment( prisma_client=None, user_api_key_cache=user_api_key_cache, - litellm_jwtauth=LiteLLM_JWTAuth( - org_alias_jwt_field="org_alias" - ), + litellm_jwtauth=LiteLLM_JWTAuth(org_alias_jwt_field="org_alias"), ) - + # Mock org object returned by get_org_object_by_alias org_object = LiteLLM_OrganizationTable( organization_id="resolved-org-id", @@ -1560,15 +1708,14 @@ async def test_get_objects_resolves_org_by_name(): budget_id="budget-1", created_by="admin", updated_by="admin", - models=[] + models=[], ) - + with patch( - "litellm.proxy.auth.handle_jwt.get_org_object_by_alias", - new_callable=AsyncMock + "litellm.proxy.auth.handle_jwt.get_org_object_by_alias", new_callable=AsyncMock ) as mock_get_by_alias: mock_get_by_alias.return_value = org_object - + ( result_user_obj, result_org_obj, @@ -1589,7 +1736,7 @@ async def test_get_objects_resolves_org_by_name(): route="/chat/completions", org_alias="my-org", ) - + # Should resolve org by alias - org_id can be derived from org_object.organization_id assert result_org_obj == org_object assert result_org_obj.organization_id == "resolved-org-id" @@ -1643,7 +1790,9 @@ async def test_resolve_jwks_url_resolves_oidc_discovery_document(): litellm_jwtauth=LiteLLM_JWTAuth(), ) - discovery_url = "https://login.microsoftonline.com/tenant/.well-known/openid-configuration" + discovery_url = ( + "https://login.microsoftonline.com/tenant/.well-known/openid-configuration" + ) jwks_url = "https://login.microsoftonline.com/tenant/discovery/keys" mock_response = MagicMock() @@ -1674,7 +1823,9 @@ async def test_resolve_jwks_url_caches_resolved_jwks_uri(): litellm_jwtauth=LiteLLM_JWTAuth(), ) - discovery_url = "https://login.microsoftonline.com/tenant/.well-known/openid-configuration" + discovery_url = ( + "https://login.microsoftonline.com/tenant/.well-known/openid-configuration" + ) jwks_url = "https://login.microsoftonline.com/tenant/discovery/keys" mock_response = MagicMock() @@ -1818,9 +1969,9 @@ async def test_find_and_validate_specific_team_id_hints_bracket_notation(): error_msg = str(exc_info.value) # Should mention the bad field name and suggest the fix assert "roles.0" in error_msg, f"Expected field name in: {error_msg}" - assert "roles" in error_msg and "list" in error_msg, ( - f"Expected hint about using 'roles' instead: {error_msg}" - ) + assert ( + "roles" in error_msg and "list" in error_msg + ), f"Expected hint about using 'roles' instead: {error_msg}" @pytest.mark.asyncio @@ -1848,9 +1999,9 @@ async def test_find_and_validate_specific_team_id_hints_bracket_index_notation() error_msg = str(exc_info.value) assert "roles[0]" in error_msg, f"Expected field name in: {error_msg}" - assert "roles" in error_msg and "list" in error_msg, ( - f"Expected hint about using 'roles' instead: {error_msg}" - ) + assert ( + "roles" in error_msg and "list" in error_msg + ), f"Expected hint about using 'roles' instead: {error_msg}" @pytest.mark.asyncio @@ -1878,4 +2029,3 @@ async def test_find_and_validate_specific_team_id_no_hint_for_valid_field(): error_msg = str(exc_info.value) assert "Hint" not in error_msg - diff --git a/tests/test_litellm/proxy/auth/test_password_hashing.py b/tests/test_litellm/proxy/auth/test_password_hashing.py new file mode 100644 index 00000000000..be4ae21264f --- /dev/null +++ b/tests/test_litellm/proxy/auth/test_password_hashing.py @@ -0,0 +1,69 @@ +"""Tests for password hashing and verification utilities.""" + +import hashlib + +import pytest + +from litellm.proxy.utils import hash_password, verify_password + + +class TestHashPassword: + def test_produces_scrypt_prefix(self): + assert hash_password("test").startswith("scrypt:") + + def test_unique_salt_per_call(self): + assert hash_password("same") != hash_password("same") + + def test_output_length(self): + # "scrypt:" (7) + base64(48 bytes) (64) = 71 + assert len(hash_password("test")) == 71 + + +class TestVerifyPassword: + def test_correct_password(self): + h = hash_password("correct") + assert verify_password("correct", h) is True + + def test_wrong_password(self): + h = hash_password("correct") + assert verify_password("wrong", h) is False + + def test_empty_password(self): + h = hash_password("") + assert verify_password("", h) is True + assert verify_password("notempty", h) is False + + def test_unicode_password(self): + h = hash_password("pässwörd") + assert verify_password("pässwörd", h) is True + assert verify_password("password", h) is False + + def test_long_password(self): + pw = "a" * 1000 + h = hash_password(pw) + assert verify_password(pw, h) is True + + +class TestVerifyPasswordFallbacks: + def test_sha256_fallback(self): + stored = hashlib.sha256("oldpass".encode()).hexdigest() + assert verify_password("oldpass", stored) is True + assert verify_password("wrong", stored) is False + + def test_no_plaintext_fallback(self): + # Plaintext fallback removed to prevent pass-the-hash attacks + assert verify_password("plaintext", "plaintext") is False + + def test_scrypt_preferred_over_fallbacks(self): + h = hash_password("test") + # Scrypt hash should not accidentally match as plaintext or SHA256 + assert verify_password("test", h) is True + assert h.startswith("scrypt:") + + def test_sha256_not_confused_with_plaintext(self): + # A 64-char hex string that isn't a valid SHA256 of the password + fake_hex = "a" * 64 + assert verify_password("test", fake_hex) is False + + def test_scrypt_invalid_base64_rejected(self): + assert verify_password("test", "scrypt:not-valid-base64!!!") is False diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py index ca70b4bfa8e..74c7f9bca5f 100644 --- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py +++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py @@ -28,7 +28,7 @@ def test_get_api_key(): assert get_api_key( custom_litellm_key_header=None, api_key=bearer_token, - AZURE_AI_API_KEY_header=None, + azure_api_key_header=None, anthropic_api_key_header=None, google_ai_studio_api_key_header=None, azure_apim_header=None, @@ -59,7 +59,7 @@ def test_get_api_key_with_custom_litellm_key_header( assert get_api_key( custom_litellm_key_header=custom_litellm_key_header, api_key=None, - AZURE_AI_API_KEY_header=None, + azure_api_key_header=None, anthropic_api_key_header=None, google_ai_studio_api_key_header=None, azure_apim_header=None, @@ -371,7 +371,7 @@ async def test_proxy_admin_expired_key_from_cache(): await _user_api_key_auth_builder( request=request, api_key=f"Bearer {api_key}", # Add Bearer prefix - AZURE_AI_API_KEY_header="", + azure_api_key_header="", anthropic_api_key_header=None, google_ai_studio_api_key_header=None, azure_apim_header=None, @@ -842,7 +842,7 @@ async def test_user_api_key_auth_builder_no_blocking_calls(): await _user_api_key_auth_builder( request=request, api_key=f"Bearer {api_key}", - AZURE_AI_API_KEY_header="", + azure_api_key_header="", anthropic_api_key_header=None, google_ai_studio_api_key_header=None, azure_apim_header=None, diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 12ec79d3e0b..0ff276953c6 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -6588,7 +6588,7 @@ async def test_build_key_filter_member_team_service_accounts(): # Should have 2 conditions: user's own keys + member team service accounts assert len(or_conditions) == 2 - # First: user's own keys + # First: user's own keys (exact match — non-admin callers use exact matching) user_cond = or_conditions[0] assert user_cond["user_id"] == user_id @@ -6988,6 +6988,98 @@ async def test_build_key_filter_team_id_scoped(): ) +@pytest.mark.asyncio +async def test_build_key_filter_admin_substring_matching(): + """ + Admin callers get substring (contains + insensitive) matching for user_id + and key_alias when use_substring_matching=True. + """ + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _build_key_filter_conditions, + ) + + user_id = "alice" + key_alias = "prod" + + where = _build_key_filter_conditions( + user_id=user_id, + team_id=None, + organization_id=None, + key_alias=key_alias, + key_hash=None, + exclude_team_id=None, + admin_team_ids=None, + member_team_ids=None, + include_created_by_keys=False, + use_substring_matching=True, + ) + + # Single OR condition is flattened into the top-level where dict + assert where["user_id"] == {"contains": user_id, "mode": "insensitive"} + assert where["key_alias"] == {"contains": key_alias, "mode": "insensitive"} + + +@pytest.mark.asyncio +async def test_build_key_filter_non_admin_exact_matching(): + """ + Non-admin callers get exact matching for user_id and key_alias when + use_substring_matching=False (the default). This prevents a user whose + ID is a substring of another user's ID from seeing that user's keys. + """ + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _build_key_filter_conditions, + ) + + user_id = "alice@example.com" + key_alias = "my-key" + + where = _build_key_filter_conditions( + user_id=user_id, + team_id=None, + organization_id=None, + key_alias=key_alias, + key_hash=None, + exclude_team_id=None, + admin_team_ids=None, + member_team_ids=None, + include_created_by_keys=False, + use_substring_matching=False, + ) + + # Single OR condition is flattened into the top-level where dict + # Exact match — no contains/insensitive wrapping + assert where["user_id"] == user_id + assert where["key_alias"] == key_alias + + +@pytest.mark.asyncio +async def test_build_key_filter_default_is_exact_matching(): + """ + The default for use_substring_matching is False, ensuring backward + compatibility — callers that don't pass the flag get exact matching. + """ + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _build_key_filter_conditions, + ) + + user_id = "user-123" + + where = _build_key_filter_conditions( + user_id=user_id, + team_id=None, + organization_id=None, + key_alias=None, + key_hash=None, + exclude_team_id=None, + admin_team_ids=None, + member_team_ids=None, + include_created_by_keys=False, + ) + + # Single OR condition is flattened into the top-level where dict + assert where["user_id"] == user_id + + @pytest.mark.asyncio async def test_get_member_team_ids(): """ diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py index 2e566ab6222..c3e3d1ecbd9 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py @@ -14,6 +14,7 @@ sys.path.insert( ) # Adds the parent directory to the system path from litellm.proxy._types import ( LiteLLM_ModelTable, + LiteLLM_ProxyModelTable, LiteLLM_TeamTable, LitellmUserRoles, Member, @@ -1176,3 +1177,102 @@ class TestModelInfoEndpoint: assert result["id"] == "team-model-1" assert result["object"] == "model" assert result["owned_by"] == "custom" + + +class TestAddAndDeleteModelLifecycle: + """ + Mock replacement for test_add_and_delete_models in tests/test_models.py. + + The original integration test required a live proxy + OPENAI_API_KEY. + This test verifies the same lifecycle (add → delete → double-delete fails) + by calling the endpoint handlers directly with mocked DB. + """ + + @pytest.mark.asyncio + async def test_add_then_delete_model(self): + """ + - Add model via add_new_model → returns model_id + - Delete model via delete_model → returns success + - Delete same model again → raises (model not found) + """ + from litellm.proxy.management_endpoints.model_management_endpoints import ( + add_new_model, + delete_model as delete_model_endpoint, + ) + from litellm.proxy.management_endpoints.model_management_endpoints import ( + ModelInfoDelete, + ) + + model_id = "lifecycle-test-model-123" + admin_user = UserAPIKeyAuth( + user_id="test-admin", user_role=LitellmUserRoles.PROXY_ADMIN + ) + + # Build a real LiteLLM_ProxyModelTable for the DB mock to return + db_row = LiteLLM_ProxyModelTable( + model_id=model_id, + model_name="lifecycle-model", + litellm_params={"model": "openai/gpt-4.1-nano"}, + model_info={"id": model_id}, + created_by="test-admin", + updated_by="test-admin", + ) + + mock_prisma = MagicMock() + mock_prisma.db = MagicMock() + mock_prisma.db.litellm_proxymodeltable = AsyncMock() + mock_prisma.db.litellm_proxymodeltable.create = AsyncMock(return_value=db_row) + mock_prisma.db.litellm_proxymodeltable.find_unique = AsyncMock( + return_value=db_row + ) + mock_prisma.db.litellm_proxymodeltable.delete = AsyncMock(return_value=db_row) + + mock_proxy_config = MagicMock() + mock_proxy_config.add_deployment = AsyncMock() + + mock_router = MagicMock() + mock_router.delete_deployment = MagicMock() + + _PS = "litellm.proxy.proxy_server" + _ENCRYPT = "litellm.proxy.management_endpoints.model_management_endpoints.encrypt_value_helper" + with patch(f"{_PS}.prisma_client", mock_prisma), \ + patch(f"{_PS}.store_model_in_db", True), \ + patch(f"{_PS}.proxy_config", mock_proxy_config), \ + patch(f"{_PS}.proxy_logging_obj", MagicMock()), \ + patch(f"{_PS}.general_settings", {}), \ + patch(f"{_PS}.premium_user", True), \ + patch(f"{_PS}.llm_router", mock_router), \ + patch(_ENCRYPT, side_effect=lambda value, **kwargs: value): + + # --- ADD --- + add_result = await add_new_model( + model_params=Deployment( + model_name="lifecycle-model", + litellm_params=LiteLLM_Params( + model="openai/gpt-4.1-nano", api_key="fake-key" + ), + model_info={"id": model_id}, + ), + user_api_key_dict=admin_user, + ) + assert add_result.model_id == model_id + + # --- DELETE --- + delete_result = await delete_model_endpoint( + model_info=ModelInfoDelete(id=model_id), + user_api_key_dict=admin_user, + ) + assert "deleted successfully" in delete_result["message"] + + # --- DELETE again should fail (model not found) --- + mock_prisma.db.litellm_proxymodeltable.find_unique = AsyncMock( + return_value=None + ) + from litellm.proxy.proxy_server import ProxyException + + with pytest.raises(ProxyException) as exc_info: + await delete_model_endpoint( + model_info=ModelInfoDelete(id=model_id), + user_api_key_dict=admin_user, + ) + assert str(exc_info.value.code) == "400" diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 223f0b335f2..45ec4b726a8 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -1431,6 +1431,83 @@ class TestOverrideOpenAIResponseModel: assert response_obj.model == actual_model_used assert response_obj.model != requested_model + def test_override_model_uses_winning_model_for_fastest_response(self): + """ + Test that when fastest_response batch completion is used with a + comma-separated model list, the response model is set to the winning + model's group name (not the comma-separated list). + """ + requested_model = "openai/gpt-4o,gemini/gemini-2.5-flash" + winning_model_group = "gemini/gemini-2.5-flash" + downstream_model = "gemini-2.5-flash" + + response_obj = MagicMock() + response_obj.model = downstream_model + response_obj._hidden_params = { + "fastest_response_batch_completion": True, + "additional_headers": { + "x-litellm-model-group": winning_model_group, + }, + } + + _override_openai_response_model( + response_obj=response_obj, + requested_model=requested_model, + log_context="test_context", + ) + + assert response_obj.model == winning_model_group + assert response_obj.model != requested_model + + def test_override_model_preserves_response_when_fastest_response_no_model_group( + self, + ): + """ + Test that when fastest_response is set but no model group header is + available, the actual downstream model is preserved. + """ + requested_model = "openai/gpt-4o,gemini/gemini-2.5-flash" + downstream_model = "gpt-4o-2024-08-06" + + response_obj = MagicMock() + response_obj.model = downstream_model + response_obj._hidden_params = { + "fastest_response_batch_completion": True, + "additional_headers": {}, + } + + _override_openai_response_model( + response_obj=response_obj, + requested_model=requested_model, + log_context="test_context", + ) + + assert response_obj.model == downstream_model + + def test_override_model_normal_when_fastest_response_not_set(self): + """ + Test that when fastest_response_batch_completion is not set, the + normal override behavior applies (model is set to requested_model). + """ + requested_model = "openai/gpt-4o" + downstream_model = "gpt-4o-2024-08-06" + + response_obj = MagicMock() + response_obj.model = downstream_model + response_obj._hidden_params = { + "additional_headers": { + "x-litellm-model-group": "openai/gpt-4o", + }, + } + + _override_openai_response_model( + response_obj=response_obj, + requested_model=requested_model, + log_context="test_context", + ) + + assert response_obj.model == requested_model + class TestIsAzureModelRouterRequest: """Tests for _is_azure_model_router_request helper""" diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 11bae9009a2..daabed0def1 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -988,6 +988,7 @@ async def test_get_all_team_models(): mock_instance = MagicMock() mock_instance.team_id = kwargs["team_id"] mock_instance.models = kwargs["models"] + mock_instance.access_group_ids = kwargs.get("access_group_ids") return mock_instance mock_team_table_class.side_effect = mock_team_table_constructor @@ -1108,6 +1109,283 @@ def test_add_team_models_to_all_models(): assert result == {"gpt-4-model-2": {"team1"}} +@pytest.mark.asyncio +async def test_add_access_group_models_to_team_models(): + """ + Test that models reachable via team access groups are included in team_models. + + Scenario: A team has models=["gpt-4"] and access_group_ids=["premium"]. + The "premium" access group contains ["claude-3", "gemini"]. + After resolution, the team should see gpt-4 (direct) + claude-3/gemini (via access group). + """ + from litellm.proxy._types import LiteLLM_TeamTable + from litellm.proxy.proxy_server import _add_access_group_models_to_team_models + + # Team with specific models AND access groups + team_with_access_groups = MagicMock(spec=LiteLLM_TeamTable) + team_with_access_groups.team_id = "team1" + team_with_access_groups.models = ["gpt-4"] # non-empty = specific models + team_with_access_groups.access_group_ids = ["premium"] + + # Team with no access groups — should be skipped + team_without_access_groups = MagicMock(spec=LiteLLM_TeamTable) + team_without_access_groups.team_id = "team2" + team_without_access_groups.models = ["gpt-4"] + team_without_access_groups.access_group_ids = None + + # Team with empty access_group_ids list — should be skipped + team_empty_access_groups = MagicMock(spec=LiteLLM_TeamTable) + team_empty_access_groups.team_id = "team2b" + team_empty_access_groups.models = ["gpt-4"] + team_empty_access_groups.access_group_ids = [] + + # Team with empty models (all access) — should be skipped + team_all_access = MagicMock(spec=LiteLLM_TeamTable) + team_all_access.team_id = "team3" + team_all_access.models = [] + team_all_access.access_group_ids = ["premium"] + + # Team with all-proxy-models sentinel (all access) — should be skipped + team_all_proxy = MagicMock(spec=LiteLLM_TeamTable) + team_all_proxy.team_id = "team4" + team_all_proxy.models = ["all-proxy-models"] + team_all_proxy.access_group_ids = ["premium"] + + # Mock router + mock_router = MagicMock() + + def mock_get_model_list(model_name, team_id=None): + if model_name == "claude-3": + return [{"model_info": {"id": "claude-3-id"}}] + elif model_name == "gemini": + return [{"model_info": {"id": "gemini-id"}}] + return None + + mock_router.get_model_list.side_effect = mock_get_model_list + + # Pre-existing team_models (e.g., from _add_team_models_to_all_models) + existing_team_models = { + "gpt-4-id": {"team1"}, + } + + # Mock prisma client with batch find_many returning access group rows + mock_ag_row = MagicMock() + mock_ag_row.access_group_id = "premium" + mock_ag_row.access_model_names = ["claude-3", "gemini"] + + mock_prisma_client = MagicMock() + mock_prisma_client.db.litellm_accessgrouptable.find_many = AsyncMock( + return_value=[mock_ag_row] + ) + + result = await _add_access_group_models_to_team_models( + team_db_objects_typed=[ + team_with_access_groups, + team_without_access_groups, + team_empty_access_groups, + team_all_access, + team_all_proxy, + ], + llm_router=mock_router, + prisma_client=mock_prisma_client, + team_models=existing_team_models, + ) + + # Single batch query with only the eligible team's access group IDs + mock_prisma_client.db.litellm_accessgrouptable.find_many.assert_called_once() + call_args = mock_prisma_client.db.litellm_accessgrouptable.find_many.call_args + queried_ids = call_args[1]["where"]["access_group_id"]["in"] + assert set(queried_ids) == {"premium"} + + # Original model still present + assert "gpt-4-id" in result + assert "team1" in result["gpt-4-id"] + + # Access group models added for team1 + assert "claude-3-id" in result + assert "team1" in result["claude-3-id"] + assert "gemini-id" in result + assert "team1" in result["gemini-id"] + + # Skipped teams should NOT have added these models + for skipped_team in ["team2", "team2b", "team3", "team4"]: + assert skipped_team not in result.get("claude-3-id", set()) + assert skipped_team not in result.get("gemini-id", set()) + + +@pytest.mark.asyncio +async def test_add_access_group_models_multiple_teams_shared_group(): + """ + Test that multiple teams sharing the same access group each get the models, + and only one batch DB query is made. + """ + from litellm.proxy._types import LiteLLM_TeamTable + from litellm.proxy.proxy_server import _add_access_group_models_to_team_models + + team_a = MagicMock(spec=LiteLLM_TeamTable) + team_a.team_id = "team-a" + team_a.models = ["gpt-4"] + team_a.access_group_ids = ["shared-group"] + + team_b = MagicMock(spec=LiteLLM_TeamTable) + team_b.team_id = "team-b" + team_b.models = ["gpt-3.5"] + team_b.access_group_ids = ["shared-group", "extra-group"] + + mock_router = MagicMock() + + def mock_get_model_list(model_name, team_id=None): + if model_name == "claude-3": + return [{"model_info": {"id": "claude-3-id"}}] + elif model_name == "gemini": + return [{"model_info": {"id": "gemini-id"}}] + return None + + mock_router.get_model_list.side_effect = mock_get_model_list + + mock_shared_row = MagicMock() + mock_shared_row.access_group_id = "shared-group" + mock_shared_row.access_model_names = ["claude-3"] + + mock_extra_row = MagicMock() + mock_extra_row.access_group_id = "extra-group" + mock_extra_row.access_model_names = ["gemini"] + + mock_prisma_client = MagicMock() + mock_prisma_client.db.litellm_accessgrouptable.find_many = AsyncMock( + return_value=[mock_shared_row, mock_extra_row] + ) + + result = await _add_access_group_models_to_team_models( + team_db_objects_typed=[team_a, team_b], + llm_router=mock_router, + prisma_client=mock_prisma_client, + team_models={}, + ) + + # Single batch query for both groups + mock_prisma_client.db.litellm_accessgrouptable.find_many.assert_called_once() + call_args = mock_prisma_client.db.litellm_accessgrouptable.find_many.call_args + queried_ids = set(call_args[1]["where"]["access_group_id"]["in"]) + assert queried_ids == {"shared-group", "extra-group"} + + # Both teams get claude-3 from the shared group + assert "claude-3-id" in result + assert "team-a" in result["claude-3-id"] + assert "team-b" in result["claude-3-id"] + + # Only team-b gets gemini (from extra-group) + assert "gemini-id" in result + assert "team-b" in result["gemini-id"] + assert "team-a" not in result["gemini-id"] + + +@pytest.mark.asyncio +async def test_add_access_group_models_no_eligible_teams(): + """ + When no teams have access groups, find_many should not be called at all. + """ + from litellm.proxy._types import LiteLLM_TeamTable + from litellm.proxy.proxy_server import _add_access_group_models_to_team_models + + team = MagicMock(spec=LiteLLM_TeamTable) + team.team_id = "team1" + team.models = ["gpt-4"] + team.access_group_ids = None + + mock_prisma_client = MagicMock() + mock_prisma_client.db.litellm_accessgrouptable.find_many = AsyncMock() + + result = await _add_access_group_models_to_team_models( + team_db_objects_typed=[team], + llm_router=MagicMock(), + prisma_client=mock_prisma_client, + team_models={"existing-id": {"team1"}}, + ) + + # No DB call made + mock_prisma_client.db.litellm_accessgrouptable.find_many.assert_not_called() + + # Original data unchanged + assert result == {"existing-id": {"team1"}} + + +@pytest.mark.asyncio +async def test_get_all_team_models_with_access_groups(): + """ + End-to-end test: get_all_team_models includes models from access groups. + + Scenario: User is on team1 which has models=["gpt-4"] and + access_group_ids=["premium"]. The "premium" group has ["claude-3"]. + The result should include both gpt-4 and claude-3 deployments for team1. + """ + from litellm.proxy.proxy_server import get_all_team_models + + mock_team1 = MagicMock() + mock_team1.model_dump.return_value = { + "team_id": "team1", + "models": ["gpt-4"], + "team_alias": "Team 1", + "access_group_ids": ["premium"], + } + + # Mock access group row returned by batch find_many + mock_ag_row = MagicMock() + mock_ag_row.access_group_id = "premium" + mock_ag_row.access_model_names = ["claude-3"] + + mock_prisma_client = MagicMock() + mock_db = MagicMock() + mock_litellm_teamtable = MagicMock() + mock_prisma_client.db = mock_db + mock_db.litellm_teamtable = mock_litellm_teamtable + mock_litellm_teamtable.find_many = AsyncMock(return_value=[mock_team1]) + mock_db.litellm_accessgrouptable = MagicMock() + mock_db.litellm_accessgrouptable.find_many = AsyncMock( + return_value=[mock_ag_row] + ) + + mock_router = MagicMock() + + def mock_get_model_list(model_name, team_id=None): + if model_name == "gpt-4": + return [{"model_info": {"id": "gpt-4-deploy-1"}}] + elif model_name == "claude-3": + return [{"model_info": {"id": "claude-3-deploy-1"}}] + return None + + mock_router.get_model_list.side_effect = mock_get_model_list + + with patch("litellm.proxy.proxy_server.LiteLLM_TeamTable") as mock_tt_class: + + def mock_team_table_constructor(**kwargs): + mock_instance = MagicMock() + mock_instance.team_id = kwargs["team_id"] + mock_instance.models = kwargs["models"] + mock_instance.access_group_ids = kwargs.get("access_group_ids") + return mock_instance + + mock_tt_class.side_effect = mock_team_table_constructor + + result = await get_all_team_models( + user_teams=["team1"], + prisma_client=mock_prisma_client, + llm_router=mock_router, + ) + + # gpt-4 from direct team.models + assert "gpt-4-deploy-1" in result + assert "team1" in result["gpt-4-deploy-1"] + + # claude-3 from access group + assert "claude-3-deploy-1" in result + assert "team1" in result["claude-3-deploy-1"] + + # Return type is Dict[str, List[str]] + assert isinstance(result["gpt-4-deploy-1"], list) + assert isinstance(result["claude-3-deploy-1"], list) + + @pytest.mark.asyncio async def test_delete_deployment_type_mismatch(): """ diff --git a/tests/test_litellm/proxy/test_response_model_sanitization.py b/tests/test_litellm/proxy/test_response_model_sanitization.py index 22785bbcb9e..b7253d98333 100644 --- a/tests/test_litellm/proxy/test_response_model_sanitization.py +++ b/tests/test_litellm/proxy/test_response_model_sanitization.py @@ -273,3 +273,61 @@ async def test_proxy_streaming_azure_model_router_preserves_actual_model(monkeyp # Azure Model Router: preserve actual model used, not the router model assert payload["model"] == actual_model_used assert payload["model"] != router_model + + +@pytest.mark.asyncio +async def test_proxy_streaming_fastest_response_preserves_winning_model(monkeypatch): + """ + Regression test for fastest_response streaming: + + When the client sends a comma-separated model list with fastest_response=True, + the streaming chunks should preserve the winning model's name from the + downstream response, NOT override to the comma-separated list. + """ + comma_separated_models = "openai/gpt-4o,gemini/gemini-2.5-flash" + winning_model = "gemini-2.5-flash" + + from litellm.proxy import proxy_server + from litellm.proxy._types import UserAPIKeyAuth + + async def _iterator_hook( + user_api_key_dict: UserAPIKeyAuth, + response: AsyncGenerator, + request_data: dict, + ): + yield _make_model_response_stream_chunk(model=winning_model) + + monkeypatch.setattr( + proxy_server.proxy_logging_obj, + "async_post_call_streaming_iterator_hook", + _iterator_hook, + ) + monkeypatch.setattr( + proxy_server.proxy_logging_obj, + "async_post_call_streaming_hook", + AsyncMock(side_effect=lambda **kwargs: kwargs["response"]), + ) + + user_api_key_dict = UserAPIKeyAuth(api_key="sk-1234") + + gen = proxy_server.async_data_generator( + response=MagicMock(), + user_api_key_dict=user_api_key_dict, + request_data={ + "model": comma_separated_models, + "_litellm_client_requested_model": comma_separated_models, + "fastest_response": True, + }, + ) + + chunks = [] + async for item in gen: + chunks.append(item) + + assert len(chunks) >= 2 + first = chunks[0] + assert first.startswith("data: ") + + payload = json.loads(first[len("data: ") :].strip()) + assert payload["model"] == winning_model + assert payload["model"] != comma_separated_models diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py index a931a9bc93c..4e44ef9e50c 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -6,20 +6,20 @@ sys.path.insert( ) # Adds the parent directory to the system path from litellm.responses.litellm_completion_transformation.transformation import ( - LiteLLMCompletionResponsesConfig, TOOL_CALLS_CACHE, + LiteLLMCompletionResponsesConfig, ) from litellm.types.llms.openai import ( ChatCompletionResponseMessage, ChatCompletionToolMessage, ) from litellm.types.utils import ( + ChatCompletionMessageToolCall, Choices, CompletionTokensDetailsWrapper, + Function, Message, ModelResponse, - Function, - ChatCompletionMessageToolCall, PromptTokensDetailsWrapper, Usage, ) @@ -130,6 +130,31 @@ class TestLiteLLMCompletionResponsesConfig: assert "extra_field" not in result["file"] assert "another_field" not in result["file"] + def test_transform_input_file_item_to_file_item_with_file_url(self): + """file_url should be mapped to file_id for downstream URL handling""" + result = ( + LiteLLMCompletionResponsesConfig._transform_input_file_item_to_file_item( + {"type": "input_file", "file_url": "https://example.com/doc.pdf"} + ) + ) + assert result == { + "type": "file", + "file": {"file_id": "https://example.com/doc.pdf"}, + } + + def test_transform_input_file_item_file_id_takes_precedence_over_file_url(self): + """explicit file_id should not be overwritten by file_url""" + result = ( + LiteLLMCompletionResponsesConfig._transform_input_file_item_to_file_item( + { + "type": "input_file", + "file_id": "file-abc123", + "file_url": "https://example.com/doc.pdf", + } + ) + ) + assert result == {"type": "file", "file": {"file_id": "file-abc123"}} + def test_transform_input_image_item_to_image_item_with_image_url(self): """Test transformation of input_image item with image_url to Chat Completion image format""" # Setup @@ -144,7 +169,10 @@ class TestLiteLLMCompletionResponsesConfig: ) # Assert - expected = {"type": "image_url", "image_url": {"url": image_url, "detail": "high"}} + expected = { + "type": "image_url", + "image_url": {"url": image_url, "detail": "high"}, + } assert result == expected assert result["type"] == "image_url" assert result["image_url"]["url"] == image_url @@ -164,7 +192,10 @@ class TestLiteLLMCompletionResponsesConfig: ) # Assert - expected = {"type": "image_url", "image_url": {"url": image_url, "detail": "high"}} + expected = { + "type": "image_url", + "image_url": {"url": image_url, "detail": "high"}, + } assert result == expected assert result["type"] == "image_url" assert result["image_url"]["url"] == image_url @@ -184,7 +215,10 @@ class TestLiteLLMCompletionResponsesConfig: ) # Assert - expected = {"type": "image_url", "image_url": {"url": image_url, "detail": "auto"}} + expected = { + "type": "image_url", + "image_url": {"url": image_url, "detail": "auto"}, + } assert result == expected assert result["type"] == "image_url" assert result["image_url"]["url"] == image_url @@ -227,7 +261,10 @@ class TestLiteLLMCompletionResponsesConfig: ) # Assert - expected = {"type": "image_url", "image_url": {"url": "https://example.com/image.png", "detail": "auto"}} + expected = { + "type": "image_url", + "image_url": {"url": "https://example.com/image.png", "detail": "auto"}, + } assert result == expected assert result["type"] == "image_url" assert result["image_url"]["url"] == "https://example.com/image.png" @@ -265,9 +302,7 @@ class TestLiteLLMCompletionResponsesConfig: # Assert assert hasattr(responses_api_response, "output") - assert ( - len(responses_api_response.output) >= 2 - ) + assert len(responses_api_response.output) >= 2 reasoning_items = [ item for item in responses_api_response.output if item.type == "reasoning" @@ -277,8 +312,10 @@ class TestLiteLLMCompletionResponsesConfig: reasoning_item = reasoning_items[0] # Note: ID auto-generation was disabled, so reasoning items may not have IDs # Only assert ID format if an ID is present - if hasattr(reasoning_item, 'id') and reasoning_item.id: - assert reasoning_item.id.startswith("rs_"), f"Expected ID to start with 'rs_', got: {reasoning_item.id}" + if hasattr(reasoning_item, "id") and reasoning_item.id: + assert reasoning_item.id.startswith( + "rs_" + ), f"Expected ID to start with 'rs_', got: {reasoning_item.id}" assert reasoning_item.status == "completed" assert reasoning_item.role == "assistant" assert len(reasoning_item.content) == 1 @@ -386,7 +423,7 @@ class TestLiteLLMCompletionResponsesConfig: """ Test that transforming a chat completion response with 'stop' finish_reason results in 'completed' status in the responses API response. - + This is the main test case for GitHub issue #15714. """ chat_completion_response = ModelResponse( @@ -406,12 +443,10 @@ class TestLiteLLMCompletionResponsesConfig: ], ) - responses_api_response = ( - LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( - request_input="this is a test", - responses_api_request={}, - chat_completion_response=chat_completion_response, - ) + responses_api_response = LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( + request_input="this is a test", + responses_api_request={}, + chat_completion_response=chat_completion_response, ) assert responses_api_response.status == "completed" @@ -427,7 +462,7 @@ class TestLiteLLMCompletionResponsesConfig: def test_transform_chat_completion_response_output_item_status(self): """ Test that output items in the transformed response also have valid status values. - + This verifies the fix for GitHub issue #15714. """ chat_completion_response = ModelResponse( @@ -447,12 +482,10 @@ class TestLiteLLMCompletionResponsesConfig: ], ) - responses_api_response = ( - LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( - request_input="this is a test", - responses_api_request={}, - chat_completion_response=chat_completion_response, - ) + responses_api_response = LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( + request_input="this is a test", + responses_api_request={}, + chat_completion_response=chat_completion_response, ) message_items = [ @@ -543,6 +576,7 @@ class TestLiteLLMCompletionResponsesConfig: assert hasattr(responses_api_response, "_hidden_params") assert responses_api_response._hidden_params == {} + class TestFunctionCallTransformation: """Test cases for function_call input transformation""" @@ -552,30 +586,38 @@ class TestFunctionCallTransformation: "type": "function_call", "name": "get_weather", "arguments": '{"location": "test"}', - "call_id": "test_id" + "call_id": "test_id", } - + function_call_output_item = { "type": "function_call_output", "call_id": "test_id", - "output": "result" + "output": "result", } - - regular_message = { - "type": "message", - "role": "user", - "content": "Hello" - } - + + regular_message = {"type": "message", "role": "user", "content": "Hello"} + # Test function_call detection - assert LiteLLMCompletionResponsesConfig._is_input_item_function_call(function_call_item) - assert not LiteLLMCompletionResponsesConfig._is_input_item_function_call(function_call_output_item) - assert not LiteLLMCompletionResponsesConfig._is_input_item_function_call(regular_message) - + assert LiteLLMCompletionResponsesConfig._is_input_item_function_call( + function_call_item + ) + assert not LiteLLMCompletionResponsesConfig._is_input_item_function_call( + function_call_output_item + ) + assert not LiteLLMCompletionResponsesConfig._is_input_item_function_call( + regular_message + ) + # Test function_call_output detection (should still work) - assert LiteLLMCompletionResponsesConfig._is_input_item_tool_call_output(function_call_output_item) - assert not LiteLLMCompletionResponsesConfig._is_input_item_tool_call_output(function_call_item) - assert not LiteLLMCompletionResponsesConfig._is_input_item_tool_call_output(regular_message) + assert LiteLLMCompletionResponsesConfig._is_input_item_tool_call_output( + function_call_output_item + ) + assert not LiteLLMCompletionResponsesConfig._is_input_item_tool_call_output( + function_call_item + ) + assert not LiteLLMCompletionResponsesConfig._is_input_item_tool_call_output( + regular_message + ) def test_function_call_transformation(self): """Test that function_call items are correctly transformed to assistant messages with tool calls""" @@ -585,28 +627,28 @@ class TestFunctionCallTransformation: "arguments": '{"location": "São Paulo, Brazil"}', "call_id": "call_123", "id": "call_123", - "status": "completed" + "status": "completed", } - + result = LiteLLMCompletionResponsesConfig._transform_responses_api_function_call_to_chat_completion_message( function_call=function_call_item ) - + assert len(result) == 1 message = result[0] - + # Should be an assistant message assert message.get("role") == "assistant" assert message.get("content") is None # Function calls don't have content - + # Should have tool calls tool_calls = message.get("tool_calls", []) assert len(tool_calls) == 1 - + tool_call = tool_calls[0] assert tool_call.get("id") == "call_123" assert tool_call.get("type") == "function" - + function = tool_call.get("function", {}) assert function.get("name") == "get_weather" assert function.get("arguments") == '{"location": "São Paulo, Brazil"}' @@ -617,7 +659,7 @@ class TestFunctionCallTransformation: { "type": "message", "role": "user", - "content": "How is the weather in São Paulo today ?" + "content": "How is the weather in São Paulo today ?", }, { "type": "function_call", @@ -625,49 +667,51 @@ class TestFunctionCallTransformation: "call_id": "call_1fe70e2a-a596-45ef-b72c-9b8567c460e5", "name": "get_weather", "id": "call_1fe70e2a-a596-45ef-b72c-9b8567c460e5", - "status": "completed" + "status": "completed", }, { "type": "function_call_output", "call_id": "call_1fe70e2a-a596-45ef-b72c-9b8567c460e5", - "output": "Rainy" - } + "output": "Rainy", + }, ] - + # This should not raise an error (previously would raise "Invalid content type: ") messages = LiteLLMCompletionResponsesConfig._transform_response_input_param_to_chat_completion_message( input=test_input ) - + assert len(messages) == 3 - + # First message: user message user_msg = messages[0] assert user_msg.get("role") == "user" assert user_msg.get("content") == "How is the weather in São Paulo today ?" - + # Second message: assistant message with tool call assistant_msg = messages[1] assert assistant_msg.get("role") == "assistant" assert assistant_msg.get("tool_calls") is not None assert len(assistant_msg.get("tool_calls", [])) == 1 - + tool_call = assistant_msg.get("tool_calls")[0] assert tool_call.get("function", {}).get("name") == "get_weather" - + # Third message: tool output tool_msg = messages[2] assert tool_msg.get("role") == "tool" assert tool_msg.get("content") == "Rainy" - assert tool_msg.get("tool_call_id") == "call_1fe70e2a-a596-45ef-b72c-9b8567c460e5" + assert ( + tool_msg.get("tool_call_id") == "call_1fe70e2a-a596-45ef-b72c-9b8567c460e5" + ) def test_complete_request_transformation_with_function_calls(self): """Test the complete request transformation that would be used by the responses API""" test_input = [ { "type": "message", - "role": "user", - "content": "How is the weather in São Paulo today ?" + "role": "user", + "content": "How is the weather in São Paulo today ?", }, { "type": "function_call", @@ -675,15 +719,15 @@ class TestFunctionCallTransformation: "call_id": "call_1fe70e2a-a596-45ef-b72c-9b8567c460e5", "name": "get_weather", "id": "call_1fe70e2a-a596-45ef-b72c-9b8567c460e5", - "status": "completed" + "status": "completed", }, { "type": "function_call_output", "call_id": "call_1fe70e2a-a596-45ef-b72c-9b8567c460e5", - "output": "Rainy" - } + "output": "Rainy", + }, ] - + tools = [ { "type": "function", @@ -694,44 +738,41 @@ class TestFunctionCallTransformation: "properties": { "location": { "type": "string", - "description": "City and country e.g. Bogotá, Colombia" + "description": "City and country e.g. Bogotá, Colombia", } }, "required": ["location"], - "additionalProperties": False - } + "additionalProperties": False, + }, } ] - - responses_api_request = { - "store": False, - "tools": tools - } - + + responses_api_request = {"store": False, "tools": tools} + # This should work without errors for non-OpenAI models result = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( model="gemini/gemini-2.0-flash", input=test_input, responses_api_request=responses_api_request, - extra_headers={"X-Test-Header": "test-value"} + extra_headers={"X-Test-Header": "test-value"}, ) - + assert "messages" in result assert "model" in result assert "tools" in result - + messages = result["messages"] assert len(messages) == 3 assert result["model"] == "gemini/gemini-2.0-flash" - + # Verify the structure is correct for chat completion user_msg = messages[0] assert user_msg["role"] == "user" - - assistant_msg = messages[1] + + assistant_msg = messages[1] assert assistant_msg["role"] == "assistant" assert "tool_calls" in assistant_msg - + tool_msg = messages[2] assert tool_msg["role"] == "tool" @@ -743,18 +784,18 @@ class TestFunctionCallTransformation: "type": "function_call", "name": "get_weather", "arguments": '{"location": "test"}', - "id": "fallback_id" # Only has 'id', not 'call_id' + "id": "fallback_id", # Only has 'id', not 'call_id' } - + result = LiteLLMCompletionResponsesConfig._transform_responses_api_function_call_to_chat_completion_message( function_call=function_call_item ) - + assert len(result) == 1 message = result[0] tool_calls = message.get("tool_calls", []) assert len(tool_calls) == 1 - + tool_call = tool_calls[0] assert tool_call.get("id") == "fallback_id" @@ -778,7 +819,11 @@ class TestFunctionCallTransformation: messages_missing_tool_calls = [ {"role": "user", "content": "Search for python bugs"}, {"role": "assistant", "content": None, "tool_calls": []}, - {"role": "tool", "content": "Found 5 results", "tool_call_id": tool_call_id}, + { + "role": "tool", + "content": "Found 5 results", + "tool_call_id": tool_call_id, + }, ] try: @@ -830,7 +875,11 @@ class TestFunctionCallTransformation: messages_missing_tool_calls = [ {"role": "user", "content": "Search using attr object"}, {"role": "assistant", "content": None, "tool_calls": []}, - {"role": "tool", "content": "Found 3 results", "tool_call_id": tool_call_id}, + { + "role": "tool", + "content": "Found 3 results", + "tool_call_id": tool_call_id, + }, ] try: @@ -859,7 +908,9 @@ class TestToolChoiceTransformation: Test that {"type": "tool"} is transformed to "required". This fixes the Anthropic error: "tool_choice.tool.name: Field required" """ - result = LiteLLMCompletionResponsesConfig._transform_tool_choice({"type": "tool"}) + result = LiteLLMCompletionResponsesConfig._transform_tool_choice( + {"type": "tool"} + ) assert result == "required" def test_transform_tool_choice_preserves_function_with_name(self): @@ -877,12 +928,20 @@ class TestContentTypeTransformation: Test that 'tool_result' content type is transformed to 'text'. This fixes: Invalid user message - content type 'tool_result' not valid. """ - result = LiteLLMCompletionResponsesConfig._get_chat_completion_request_content_type("tool_result") + result = ( + LiteLLMCompletionResponsesConfig._get_chat_completion_request_content_type( + "tool_result" + ) + ) assert result == "text" def test_input_text_content_type_transformed_to_text(self): """Test that 'input_text' content type is transformed to 'text'""" - result = LiteLLMCompletionResponsesConfig._get_chat_completion_request_content_type("input_text") + result = ( + LiteLLMCompletionResponsesConfig._get_chat_completion_request_content_type( + "input_text" + ) + ) assert result == "text" def test_none_text_blocks_filtered_out(self): @@ -896,7 +955,9 @@ class TestContentTypeTransformation: {"type": "text", "text": None}, # Should be filtered out {"type": "text", "text": "another valid"}, ] - result = LiteLLMCompletionResponsesConfig._transform_responses_api_content_to_chat_completion_content(content) + result = LiteLLMCompletionResponsesConfig._transform_responses_api_content_to_chat_completion_content( + content + ) assert len(result) == 2 assert result[0]["text"] == "valid text" assert result[1]["text"] == "another valid" @@ -911,14 +972,16 @@ class TestToolTransformation: # Create a Vertex AI tool using the enum value vertex_tool = {VertexToolName.CODE_EXECUTION.value: {}} - + tools = [vertex_tool] - + # Execute - result_tools, web_search_options = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( - tools=tools + result_tools, web_search_options = ( + LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( + tools=tools + ) ) - + # Assert assert len(result_tools) == 1 assert result_tools[0] == vertex_tool @@ -930,18 +993,18 @@ class TestToolTransformation: "type": "mcp", "server_label": "zapier", "server_url": "https://mcp.zapier.com/api/mcp/mcp", - "headers": { - "Authorization": "Bearer token123" - }, + "headers": {"Authorization": "Bearer token123"}, } - + tools = [mcp_tool] - + # Execute - result_tools, web_search_options = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( - tools=tools + result_tools, web_search_options = ( + LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( + tools=tools + ) ) - + # Assert assert len(result_tools) == 1 assert result_tools[0] == mcp_tool @@ -953,16 +1016,18 @@ class TestToolTransformation: computer_use_tool = { "type": "computer_use", "display_width_px": 1024, - "display_height_px": 768 + "display_height_px": 768, } - + tools = [computer_use_tool] - + # Execute - result_tools, web_search_options = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( - tools=tools + result_tools, web_search_options = ( + LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( + tools=tools + ) ) - + # Assert assert len(result_tools) == 1 assert result_tools[0] == computer_use_tool @@ -974,16 +1039,18 @@ class TestToolTransformation: web_search_tool = { "type": "web_search_preview", "search_context_size": "medium", - "user_location": {"country": "US"} + "user_location": {"country": "US"}, } - + tools = [web_search_tool] - + # Execute - result_tools, web_search_options = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( - tools=tools + result_tools, web_search_options = ( + LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( + tools=tools + ) ) - + # Assert assert len(result_tools) == 0 # Web search is not added to tools assert web_search_options is not None @@ -998,24 +1065,24 @@ class TestToolTransformation: "description": "Get weather for a location", "parameters": { "type": "object", - "properties": { - "location": {"type": "string"} - }, - "required": ["location"] + "properties": {"location": {"type": "string"}}, + "required": ["location"], }, "cache_control": {"type": "ephemeral"}, "defer_loading": True, "allowed_callers": ["user"], - "input_examples": [{"location": "San Francisco"}] + "input_examples": [{"location": "San Francisco"}], } - + tools = [function_tool] - + # Execute - result_tools, web_search_options = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( - tools=tools + result_tools, web_search_options = ( + LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( + tools=tools + ) ) - + # Assert assert len(result_tools) == 1 result_tool = result_tools[0] @@ -1035,16 +1102,18 @@ class TestToolTransformation: "name": "search", "description": "Search function", "parameters": {"type": "object"}, - "cache_control": {"type": "ephemeral"} + "cache_control": {"type": "ephemeral"}, } - + tools = [function_tool] - + # Execute - result_tools, _ = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( - tools=tools + result_tools, _ = ( + LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( + tools=tools + ) ) - + # Assert assert len(result_tools) == 1 result_tool = result_tools[0] @@ -1059,19 +1128,19 @@ class TestToolTransformation: "description": "A simple function", "parameters": { "type": "object", - "properties": { - "param": {"type": "string"} - } - } + "properties": {"param": {"type": "string"}}, + }, } - + tools = [function_tool] - + # Execute - result_tools, _ = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( - tools=tools + result_tools, _ = ( + LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( + tools=tools + ) ) - + # Assert assert len(result_tools) == 1 result_tool = result_tools[0] @@ -1087,16 +1156,18 @@ class TestToolTransformation: """Test that code_execution tools are passed through as-is""" code_execution_tool = { "type": "code_execution_20250825", - "name": "python_code_execution" + "name": "python_code_execution", } - + tools = [code_execution_tool] - + # Execute - result_tools, _ = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( - tools=tools + result_tools, _ = ( + LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( + tools=tools + ) ) - + # Assert assert len(result_tools) == 1 assert result_tools[0]["type"] == "code_execution_20250825" @@ -1105,21 +1176,23 @@ class TestToolTransformation: """Test that tool_search tools are passed through as-is""" tool_search_regex = { "name": "tool_search_tool_regex", - "description": "Search tools using regex" + "description": "Search tools using regex", } - + tool_search_bm25 = { "name": "tool_search_tool_bm25", - "description": "Search tools using BM25" + "description": "Search tools using BM25", } - + tools = [tool_search_regex, tool_search_bm25] - + # Execute - result_tools, _ = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( - tools=tools + result_tools, _ = ( + LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( + tools=tools + ) ) - + # Assert assert len(result_tools) == 2 assert result_tools[0]["name"] == "tool_search_tool_regex" @@ -1128,7 +1201,7 @@ class TestToolTransformation: def test_transform_mixed_tools_list(self): """Test transforming a mixed list of different tool types""" from litellm.types.llms.vertex_ai import VertexToolName - + tools = [ # Regular function tool with anthropic fields { @@ -1136,40 +1209,38 @@ class TestToolTransformation: "name": "get_weather", "description": "Get weather", "parameters": {"type": "object"}, - "cache_control": {"type": "ephemeral"} + "cache_control": {"type": "ephemeral"}, }, # MCP tool - { - "type": "mcp", - "server_label": "zapier" - }, + {"type": "mcp", "server_label": "zapier"}, # Web search tool - { - "type": "web_search_preview", - "search_context_size": "high" - }, + {"type": "web_search_preview", "search_context_size": "high"}, # Vertex AI tool - {VertexToolName.CODE_EXECUTION.value: {}} + {VertexToolName.CODE_EXECUTION.value: {}}, ] - + # Execute - result_tools, web_search_options = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( - tools=tools + result_tools, web_search_options = ( + LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( + tools=tools + ) ) - + # Assert - assert len(result_tools) == 3 # function, mcp, vertex (web_search becomes options) + assert ( + len(result_tools) == 3 + ) # function, mcp, vertex (web_search becomes options) assert web_search_options is not None - + # Check function tool func_tools = [t for t in result_tools if t.get("type") == "function"] assert len(func_tools) == 1 assert func_tools[0]["cache_control"]["type"] == "ephemeral" - + # Check MCP tool mcp_tools = [t for t in result_tools if t.get("type") == "mcp"] assert len(mcp_tools) == 1 - + # Check web search was converted to options assert web_search_options.get("search_context_size") == "high" @@ -1179,20 +1250,18 @@ class TestToolTransformation: "type": "function", "name": "test_function", "description": "Test function", - "parameters": { - "properties": { - "arg": {"type": "string"} - } - } + "parameters": {"properties": {"arg": {"type": "string"}}}, } - + tools = [function_tool] - + # Execute - result_tools, _ = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( - tools=tools + result_tools, _ = ( + LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( + tools=tools + ) ) - + # Assert assert len(result_tools) == 1 result_tool = result_tools[0] @@ -1205,16 +1274,18 @@ class TestToolTransformation: "type": "function", "name": "test_function", "description": "Test function", - "parameters": {} + "parameters": {}, } - + tools = [function_tool] - + # Execute - result_tools, _ = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( - tools=tools + result_tools, _ = ( + LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( + tools=tools + ) ) - + # Assert assert len(result_tools) == 1 result_tool = result_tools[0] @@ -1225,16 +1296,18 @@ class TestToolTransformation: function_tool = { "type": "function", "name": "test_function", - "description": "Test function" + "description": "Test function", } - + tools = [function_tool] - + # Execute - result_tools, _ = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( - tools=tools + result_tools, _ = ( + LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( + tools=tools + ) ) - + # Assert assert len(result_tools) == 1 result_tool = result_tools[0] @@ -1246,27 +1319,27 @@ class TestToolTransformation: "type": "function", "name": "test_function", "description": "Test function", - "parameters": { - "type": "object", - "properties": { - "arg": {"type": "string"} - } - } + "parameters": {"type": "object", "properties": {"arg": {"type": "string"}}}, } - + tools = [function_tool] - + # Execute - result_tools, _ = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( - tools=tools + result_tools, _ = ( + LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( + tools=tools + ) ) - + # Assert assert len(result_tools) == 1 result_tool = result_tools[0] assert result_tool["function"]["parameters"]["type"] == "object" assert "properties" in result_tool["function"]["parameters"] - assert result_tool["function"]["parameters"]["properties"]["arg"]["type"] == "string" + assert ( + result_tool["function"]["parameters"]["properties"]["arg"]["type"] + == "string" + ) class TestUsageTransformation: @@ -1434,12 +1507,12 @@ class TestUsageTransformation: assert response_usage.input_tokens == 13 assert response_usage.output_tokens == 100 assert response_usage.total_tokens == 113 - + # Verify input_tokens_details assert response_usage.input_tokens_details is not None assert response_usage.input_tokens_details.cached_tokens == 5 assert response_usage.input_tokens_details.text_tokens == 8 - + # Verify output_tokens_details assert response_usage.output_tokens_details is not None assert response_usage.output_tokens_details.reasoning_tokens == 50 @@ -1543,7 +1616,9 @@ class TestUsageTransformation: Choices( finish_reason="stop", index=0, - message=Message(content="Here is the generated image.", role="assistant"), + message=Message( + content="Here is the generated image.", role="assistant" + ), ) ], ) @@ -1569,7 +1644,7 @@ class TestStreamingIDConsistency: Test that all streaming events use the same item_id throughout the stream. This fixes the issue where text-start, text-delta, and text-end events had different IDs, breaking SDK text accumulation. - + Reproduces: https://github.com/BerriAI/litellm/issues/14962 """ from unittest.mock import Mock @@ -1645,25 +1720,27 @@ class TestStreamingIDConsistency: # Assert: All events should use the same item_id (from the first chunk) assert event1 is not None, "First event should not be None" assert event2 is not None, "Second event should not be None" - + # Extract item_ids from events item_id_1 = getattr(event1, "item_id", None) item_id_2 = getattr(event2, "item_id", None) - + assert item_id_1 is not None, "First event should have an item_id" assert item_id_2 is not None, "Second event should have an item_id" - + # The critical assertion: IDs should match across all events assert item_id_1 == item_id_2, ( f"Item IDs should be consistent across streaming events. " f"Got {item_id_1} and {item_id_2}. " f"This breaks SDK text accumulation (issue #14962)." ) - + # Verify the cached ID is set and matches assert iterator._cached_item_id is not None, "Iterator should cache the item_id" assert iterator._cached_item_id == item_id_1, "Cached ID should match event IDs" - assert iterator._cached_item_id == "chatcmpl-first-id", "Should use the first chunk's ID" + assert ( + iterator._cached_item_id == "chatcmpl-first-id" + ), "Should use the first chunk's ID" def test_streaming_iterator_initial_events_use_cached_id(self): """ @@ -1704,7 +1781,7 @@ class TestStreamingIDConsistency: f"Initial events should use consistent IDs. " f"Got output_item_id={output_item_id}, content_part_id={content_part_id}" ) - + # Verify it matches the cached ID assert iterator._cached_item_id is not None assert iterator._cached_item_id == output_item_id @@ -1753,7 +1830,9 @@ class TestStreamingIDConsistency: # Create done events text_done_event = iterator.create_output_text_done_event(complete_response) - content_done_event = iterator.create_output_content_part_done_event(complete_response) + content_done_event = iterator.create_output_content_part_done_event( + complete_response + ) item_done_event = iterator.create_output_item_done_event(complete_response) # Extract IDs @@ -1765,12 +1844,12 @@ class TestStreamingIDConsistency: assert text_done_id is not None, "Text done event should have an item_id" assert content_done_id is not None, "Content done event should have an item_id" assert item_done_id is not None, "Item done event should have an id" - + assert text_done_id == content_done_id == item_done_id, ( f"All done events should use consistent IDs. " f"Got text_done={text_done_id}, content_done={content_done_id}, item_done={item_done_id}" ) - + # Verify it matches the cached ID assert iterator._cached_item_id is not None assert iterator._cached_item_id == text_done_id @@ -1826,13 +1905,14 @@ class TestStreamingIDConsistency: # The single assistant message must contain BOTH tool_calls assistant_messages = [ - m for m in messages + m + for m in messages if (m.get("role") if isinstance(m, dict) else getattr(m, "role", None)) == "assistant" ] - assert len(assistant_messages) == 1, ( - f"Expected 1 assistant message, got {len(assistant_messages)}" - ) + assert ( + len(assistant_messages) == 1 + ), f"Expected 1 assistant message, got {len(assistant_messages)}" assistant_msg = assistant_messages[0] tool_calls = ( @@ -1840,9 +1920,9 @@ class TestStreamingIDConsistency: if isinstance(assistant_msg, dict) else getattr(assistant_msg, "tool_calls", None) ) - assert tool_calls is not None and len(tool_calls) == 2, ( - f"Expected 2 tool_calls in the merged assistant message, got: {tool_calls}" - ) + assert ( + tool_calls is not None and len(tool_calls) == 2 + ), f"Expected 2 tool_calls in the merged assistant message, got: {tool_calls}" call_ids = [ (tc.get("id") if isinstance(tc, dict) else getattr(tc, "id", None)) @@ -1853,13 +1933,14 @@ class TestStreamingIDConsistency: # Both tool messages must be present tool_messages = [ - m for m in messages + m + for m in messages if (m.get("role") if isinstance(m, dict) else getattr(m, "role", None)) == "tool" ] - assert len(tool_messages) == 2, ( - f"Expected 2 tool messages, got {len(tool_messages)}" - ) + assert ( + len(tool_messages) == 2 + ), f"Expected 2 tool messages, got {len(tool_messages)}" def test_single_tool_call_still_works_after_merge_fix(self): """ @@ -1890,7 +1971,12 @@ class TestStreamingIDConsistency: assert "assistant" in roles assert "tool" in roles - assistant_messages = [m for m in messages if (m.get("role") if isinstance(m, dict) else getattr(m, "role", None)) == "assistant"] + assistant_messages = [ + m + for m in messages + if (m.get("role") if isinstance(m, dict) else getattr(m, "role", None)) + == "assistant" + ] assert len(assistant_messages) == 1 tool_calls = ( @@ -1899,3 +1985,97 @@ class TestStreamingIDConsistency: else getattr(assistant_messages[0], "tool_calls", None) ) assert tool_calls is not None and len(tool_calls) == 1 + + +class TestEnsureOutputItemContentPartAdded: + """Test that _ensure_output_item_for_chunk emits content_part.added after + output_item.added for message items.""" + + def _make_iterator(self): + """Create a minimal LiteLLMCompletionStreamingIterator for testing.""" + from unittest.mock import MagicMock + + from litellm.responses.litellm_completion_transformation.streaming_iterator import ( + LiteLLMCompletionStreamingIterator, + ) + + iterator = LiteLLMCompletionStreamingIterator.__new__( + LiteLLMCompletionStreamingIterator + ) + iterator.sent_output_item_added_event = False + iterator.sent_content_part_added_event = False + iterator._sequence_number = 0 + iterator._cached_item_id = None + iterator._cached_reasoning_item_id = None + iterator._reasoning_active = False + iterator._pending_response_events = [] + return iterator + + def _make_text_chunk(self): + """Create a mock ModelResponseStream with a text delta.""" + from unittest.mock import MagicMock + + chunk = MagicMock() + delta = MagicMock() + delta.reasoning_content = None + delta.tool_calls = None + chunk.choices = [MagicMock(delta=delta)] + return chunk + + def _make_reasoning_chunk(self): + """Create a mock ModelResponseStream with a reasoning delta.""" + from unittest.mock import MagicMock + + chunk = MagicMock() + delta = MagicMock() + delta.reasoning_content = "thinking..." + delta.tool_calls = None + chunk.choices = [MagicMock(delta=delta)] + return chunk + + def test_message_item_emits_content_part_added(self): + """content_part.added must follow output_item.added for message items.""" + from litellm.types.llms.openai import ( + ContentPartAddedEvent, + OutputItemAddedEvent, + ResponsesAPIStreamEvents, + ) + + iterator = self._make_iterator() + chunk = self._make_text_chunk() + + iterator._ensure_output_item_for_chunk(chunk) + + events = iterator._pending_response_events + assert len(events) == 2 + assert isinstance(events[0], OutputItemAddedEvent) + assert events[0].type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED + assert isinstance(events[1], ContentPartAddedEvent) + assert events[1].type == ResponsesAPIStreamEvents.CONTENT_PART_ADDED + assert events[1].part.type == "output_text" + assert iterator.sent_content_part_added_event is True + + def test_reasoning_item_does_not_emit_content_part_added(self): + """Reasoning items should not get a content_part.added event.""" + from litellm.types.llms.openai import OutputItemAddedEvent + + iterator = self._make_iterator() + chunk = self._make_reasoning_chunk() + + iterator._ensure_output_item_for_chunk(chunk) + + events = iterator._pending_response_events + assert len(events) == 1 + assert isinstance(events[0], OutputItemAddedEvent) + assert iterator.sent_content_part_added_event is False + + def test_only_emits_once(self): + """Calling _ensure_output_item_for_chunk twice should not duplicate events.""" + iterator = self._make_iterator() + chunk = self._make_text_chunk() + + iterator._ensure_output_item_for_chunk(chunk) + iterator._ensure_output_item_for_chunk(chunk) + + events = iterator._pending_response_events + assert len(events) == 2 diff --git a/tests/test_litellm/router_utils/test_health_check_routing.py b/tests/test_litellm/router_utils/test_router_health_check_routing.py similarity index 100% rename from tests/test_litellm/router_utils/test_health_check_routing.py rename to tests/test_litellm/router_utils/test_router_health_check_routing.py diff --git a/tests/test_models.py b/tests/test_models.py index a4b7c6a44fd..151fb70b665 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -268,6 +268,9 @@ async def delete_model(session, model_id="123", key="sk-1234"): return await response.json() +@pytest.mark.skip( + reason="Requires live proxy + OPENAI_API_KEY. Deterministic mock version in tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py::TestAddAndDeleteModelLifecycle" +) @pytest.mark.asyncio async def test_add_and_delete_models(): """ diff --git a/ui/litellm-dashboard/.npmrc b/ui/litellm-dashboard/.npmrc new file mode 100644 index 00000000000..168e81a1c4e --- /dev/null +++ b/ui/litellm-dashboard/.npmrc @@ -0,0 +1,5 @@ +# Supply-chain hardening +# Packages needing lifecycle scripts: npm rebuild +ignore-scripts=true +# Protects local npm install only — npm ci (used in CI) ignores this +min-release-age=3d diff --git a/ui/litellm-dashboard/build_ui.sh b/ui/litellm-dashboard/build_ui.sh index cd6ec901904..aa346c12edc 100755 --- a/ui/litellm-dashboard/build_ui.sh +++ b/ui/litellm-dashboard/build_ui.sh @@ -2,8 +2,20 @@ # Check if nvm is not installed if ! command -v nvm &> /dev/null; then - # Install nvm - curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh | bash + # Install nvm with checksum verification + NVM_VERSION="v0.40.4" + NVM_CHECKSUM="4b7412c49960c7d31e8df72da90c1fb5b8cccb419ac99537b737028d497aba4f" + NVM_SCRIPT=$(mktemp) + trap 'rm -f "$NVM_SCRIPT"' EXIT + curl -fsSL "https://raw.githubusercontent.com/nvm-sh/nvm/${NVM_VERSION}/install.sh" -o "$NVM_SCRIPT" + if command -v sha256sum &>/dev/null; then + echo "${NVM_CHECKSUM} ${NVM_SCRIPT}" | sha256sum -c - + elif command -v shasum &>/dev/null; then + echo "${NVM_CHECKSUM} ${NVM_SCRIPT}" | shasum -a 256 -c - + else + echo "No sha256 tool found; cannot verify nvm checksum"; exit 1 + fi || { echo "nvm checksum verification failed"; exit 1; } + bash "$NVM_SCRIPT" # Source nvm script in the current session export NVM_DIR="$HOME/.nvm" diff --git a/ui/litellm-dashboard/build_ui_custom_path.sh b/ui/litellm-dashboard/build_ui_custom_path.sh index f947f87d3b7..a92927f8ea7 100755 --- a/ui/litellm-dashboard/build_ui_custom_path.sh +++ b/ui/litellm-dashboard/build_ui_custom_path.sh @@ -12,8 +12,20 @@ UI_BASE_PATH="$1" # Check if nvm is not installed if ! command -v nvm &> /dev/null; then - # Install nvm - curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh | bash + # Install nvm with checksum verification + NVM_VERSION="v0.40.4" + NVM_CHECKSUM="4b7412c49960c7d31e8df72da90c1fb5b8cccb419ac99537b737028d497aba4f" + NVM_SCRIPT=$(mktemp) + trap 'rm -f "$NVM_SCRIPT"' EXIT + curl -fsSL "https://raw.githubusercontent.com/nvm-sh/nvm/${NVM_VERSION}/install.sh" -o "$NVM_SCRIPT" + if command -v sha256sum &>/dev/null; then + echo "${NVM_CHECKSUM} ${NVM_SCRIPT}" | sha256sum -c - + elif command -v shasum &>/dev/null; then + echo "${NVM_CHECKSUM} ${NVM_SCRIPT}" | shasum -a 256 -c - + else + echo "No sha256 tool found; cannot verify nvm checksum"; exit 1 + fi || { echo "nvm checksum verification failed"; exit 1; } + bash "$NVM_SCRIPT" # Source nvm script in the current session export NVM_DIR="$HOME/.nvm" diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index fce2e09b54c..7b763eaa66a 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -82,28 +82,14 @@ "vitest": "^3.2.4" }, "overrides": { - "diff": ">=8.0.3", - "prismjs": ">=1.30.0", - "webpack-dev-server": ">=5.2.1", - "mermaid": ">=11.10.0", - "js-yaml": ">=4.1.1", - "glob": ">=11.1.0", - "tar": ">=7.5.11", - "minimatch": ">=10.2.4", - "@isaacs/brace-expansion": ">=5.0.1", - "node-forge": ">=1.3.2", - "lodash-es": ">=4.17.23", - "lodash": ">=4.17.23", - "@babel/traverse": ">=7.23.2", - "ws": ">=7.5.10", - "http-proxy-middleware": ">=2.0.9", - "tar-fs": ">=2.1.4", - "webpack-dev-middleware": ">=5.3.4", - "braces": ">=3.0.3", - "axios": ">=0.30.2", - "webpack": ">=5.94.0", - "serve-static": ">=1.16.0", - "path-to-regexp": ">=0.1.12" + "prismjs": "1.30.0", + "js-yaml": "4.1.1", + "glob": "13.0.0", + "minimatch": "10.2.4", + "lodash": "4.17.23", + "ws": "8.19.0", + "braces": "3.0.3", + "axios": "1.13.6" }, "engines": { "node": ">=18.17.0", diff --git a/ui/litellm-dashboard/src/app/login/LoginPage.tsx b/ui/litellm-dashboard/src/app/login/LoginPage.tsx index e130dddc4a4..202820a11a2 100644 --- a/ui/litellm-dashboard/src/app/login/LoginPage.tsx +++ b/ui/litellm-dashboard/src/app/login/LoginPage.tsx @@ -48,6 +48,7 @@ function LoginPageContent() { const params = new URLSearchParams(window.location.search); const ssoCode = params.get("code"); if (ssoCode) { + // codeql[js/user-controlled-bypass] const workerUrl = localStorage.getItem("litellm_worker_url"); exchangeLoginCode(ssoCode, workerUrl).then(() => { params.delete("code"); diff --git a/ui/litellm-dashboard/src/app/page.tsx b/ui/litellm-dashboard/src/app/page.tsx index f957df12f35..44df1b5bd41 100644 --- a/ui/litellm-dashboard/src/app/page.tsx +++ b/ui/litellm-dashboard/src/app/page.tsx @@ -43,7 +43,7 @@ import SpendLogsTable from "@/components/view_logs"; import ViewUserDashboard from "@/components/view_users"; import { ThemeProvider } from "@/contexts/ThemeContext"; import { isJwtExpired } from "@/utils/jwtUtils"; -import { buildLoginUrlWithReturn, consumeReturnUrl, normalizeUrlForCompare, storeReturnUrl } from "@/utils/returnUrlUtils"; +import { buildLoginUrlWithReturn, consumeReturnUrl, isValidReturnUrl, normalizeUrlForCompare, storeReturnUrl } from "@/utils/returnUrlUtils"; import { formatUserRole, isAdminRole } from "@/utils/roles"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { jwtDecode } from "jwt-decode"; @@ -276,7 +276,7 @@ function CreateKeyPageContent() { // Check for a stored return URL const returnUrl = consumeReturnUrl(); - if (returnUrl) { + if (returnUrl && isValidReturnUrl(returnUrl)) { const currentUrl = window.location.href; const normalizedReturnUrl = normalizeUrlForCompare(returnUrl); const normalizedCurrentUrl = normalizeUrlForCompare(currentUrl); diff --git a/ui/litellm-dashboard/src/components/guardrails/TeamGuardrailsTab.tsx b/ui/litellm-dashboard/src/components/guardrails/TeamGuardrailsTab.tsx index a2246fd976d..8fbbae56124 100644 --- a/ui/litellm-dashboard/src/components/guardrails/TeamGuardrailsTab.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/TeamGuardrailsTab.tsx @@ -145,7 +145,7 @@ function buildEquivalentConfigYaml(g: TeamGuardrail): string { const lines: string[] = [ "litellm_settings:", " guardrails:", - ` - guardrail_name: "${g.name.replace(/"/g, '\\"')}"`, + ` - guardrail_name: "${g.name.replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"`, " litellm_params:", ` guardrail: ${g.guardrailType ?? "generic_guardrail_api"}`, ` mode: ${g.mode ?? "pre_call"} # or post_call, during_call`, @@ -160,7 +160,7 @@ function buildEquivalentConfigYaml(g: TeamGuardrail): string { if (g.customHeaders.length > 0) { lines.push(" headers: # static headers (sent with every request)"); for (const h of g.customHeaders) { - lines.push(` ${h.key}: "${String(h.value).replace(/"/g, '\\"')}"`); + lines.push(` ${h.key}: "${String(h.value).replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"`); } } if (g.extraHeaders.length > 0) { diff --git a/ui/litellm-dashboard/src/components/logging_settings_view.tsx b/ui/litellm-dashboard/src/components/logging_settings_view.tsx index 1f454f0b202..079711d196d 100644 --- a/ui/litellm-dashboard/src/components/logging_settings_view.tsx +++ b/ui/litellm-dashboard/src/components/logging_settings_view.tsx @@ -1,5 +1,5 @@ import React from "react"; -import { Text, Badge } from "@tremor/react"; +import { Tag } from "antd"; import { CogIcon, BanIcon } from "@heroicons/react/outline"; import { callbackInfo, callback_map, reverse_callback_map } from "./callback_info_helpers"; @@ -28,7 +28,7 @@ export function LoggingSettingsView({ return callbackDisplayName || callbackName; }; - const getEventTypeColor = (eventType: string) => { + const getEventTypeColor = (eventType: string): string | undefined => { switch (eventType) { case "success": return "green"; @@ -37,7 +37,7 @@ export function LoggingSettingsView({ case "success_and_failure": return "blue"; default: - return "gray"; + return undefined; } }; @@ -60,10 +60,10 @@ export function LoggingSettingsView({
- Logging Integrations - + Logging Integrations + {loggingConfigs.length} - +
{loggingConfigs.length > 0 ? ( @@ -84,15 +84,15 @@ export function LoggingSettingsView({ )}
- {displayName} - + {displayName} + {Object.keys(config.callback_vars).length} parameters configured - +
- + {getEventTypeLabel(config.callback_type)} - +
); })} @@ -100,7 +100,7 @@ export function LoggingSettingsView({ ) : (
- No logging integrations configured + No logging integrations configured
)}
@@ -109,10 +109,10 @@ export function LoggingSettingsView({
- Disabled Callbacks - + Disabled Callbacks + {disabledCallbacks.length} - +
{disabledCallbacks.length > 0 ? ( @@ -134,13 +134,13 @@ export function LoggingSettingsView({ )}
- {displayName} - Disabled for this key + {displayName} + Disabled for this key
- + Disabled - +
); })} @@ -148,7 +148,7 @@ export function LoggingSettingsView({ ) : (
- No callbacks disabled + No callbacks disabled
)}
@@ -160,10 +160,10 @@ export function LoggingSettingsView({
- Logging Settings - + Logging Settings + Active logging integrations and disabled callbacks for this key - +
{content} @@ -173,7 +173,7 @@ export function LoggingSettingsView({ return (
- Logging Settings + Logging Settings {content}
); diff --git a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx index e6845402893..aaddf53fd7e 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx @@ -94,6 +94,7 @@ const CreateMCPServer: React.FC = ({ } try { const values = form.getFieldsValue(true); + // codeql[js/clear-text-storage-of-sensitive-data] window.sessionStorage.setItem( CREATE_OAUTH_UI_STATE_KEY, JSON.stringify({ diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx index 04cce343038..118461f0043 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx @@ -73,6 +73,7 @@ const MCPServerEdit: React.FC = ({ } try { const values = form.getFieldsValue(true); + // codeql[js/clear-text-storage-of-sensitive-data] window.sessionStorage.setItem( EDIT_OAUTH_UI_STATE_KEY, JSON.stringify({ diff --git a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx index c0eb89a5980..e80c2b2f742 100644 --- a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx +++ b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx @@ -162,6 +162,7 @@ const ChatUI: React.FC = ({ clearChatHistory: clearChatHistoryHook, clearMCPEvents, } = useChatHistory({ simplified }); + // codeql[js/clear-text-storage-of-sensitive-data] const [apiKeySource, setApiKeySource] = useState<"session" | "custom">(() => { const saved = sessionStorage.getItem("apiKeySource"); if (saved) { @@ -173,6 +174,7 @@ const ChatUI: React.FC = ({ } return disabledPersonalKeyCreation ? "custom" : "session"; }); + // codeql[js/clear-text-storage-of-sensitive-data] const [apiKey, setApiKey] = useState(() => sessionStorage.getItem("apiKey") || ""); const [customProxyBaseUrl, setCustomProxyBaseUrl] = useState( () => sessionStorage.getItem("customProxyBaseUrl") || "", @@ -339,7 +341,9 @@ const ChatUI: React.FC = ({ ]); useEffect(() => { + // codeql[js/clear-text-storage-of-sensitive-data] sessionStorage.setItem("apiKeySource", JSON.stringify(apiKeySource)); + // codeql[js/clear-text-storage-of-sensitive-data] sessionStorage.setItem("apiKey", apiKey); sessionStorage.setItem("endpointType", endpointType); sessionStorage.setItem("selectedTags", JSON.stringify(selectedTags)); @@ -1837,7 +1841,7 @@ const ChatUI: React.FC = ({ @@ -1858,7 +1862,7 @@ const ChatUI: React.FC = ({ key={prompt} type="button" className="shrink-0 rounded-full border border-gray-200 px-3 py-1 text-xs font-medium text-gray-600 transition-colors hover:bg-blue-50 hover:border-blue-300 hover:text-blue-600 cursor-pointer" - onClick={() => setInputMessage(prompt)} + onClick={() => setInputMessage(prompt)} // lgtm[js/xss-through-dom] > {prompt} diff --git a/ui/litellm-dashboard/src/components/playground/chat_ui/CodeSnippets.tsx b/ui/litellm-dashboard/src/components/playground/chat_ui/CodeSnippets.tsx index 6998d542401..aa573c8210a 100644 --- a/ui/litellm-dashboard/src/components/playground/chat_ui/CodeSnippets.tsx +++ b/ui/litellm-dashboard/src/components/playground/chat_ui/CodeSnippets.tsx @@ -536,7 +536,7 @@ audio_file = open("path/to/your/audio/file.mp3", "rb") # Make the transcription request response = client.audio.transcriptions.create( model="${modelNameForCode}", - file=audio_file${inputMessage ? `,\n prompt="${inputMessage.replace(/"/g, '\\"')}"` : ""} + file=audio_file${inputMessage ? `,\n prompt="${inputMessage.replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"` : ""} ) print(response.text) diff --git a/ui/litellm-dashboard/src/components/public_model_hub.tsx b/ui/litellm-dashboard/src/components/public_model_hub.tsx index 21974ad9729..59c9c6d9cb3 100644 --- a/ui/litellm-dashboard/src/components/public_model_hub.tsx +++ b/ui/litellm-dashboard/src/components/public_model_hub.tsx @@ -1376,7 +1376,7 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded {selectedModel.model_group}, you can use any string ( - {selectedModel.model_group.replace("*", "my-custom-value")} + {selectedModel.model_group.replaceAll("*", "my-custom-value")} ) that matches this pattern. diff --git a/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.tsx b/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.tsx index 4e7153b64cd..8c88de49d0d 100644 --- a/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.tsx @@ -69,13 +69,7 @@ export function useLogFilterLogic({ ); const [filters, setFilters] = useState(defaultFilters); - const [backendFilteredLogs, setBackendFilteredLogs] = useState({ - data: [], - total: 0, - page: 1, - page_size: 50, - total_pages: 0, - }); + const [backendFilteredLogs, setBackendFilteredLogs] = useState(null); const lastSearchTimestamp = useRef(0); const performSearch = useCallback( async (filters: LogFilterState, page = 1) => { @@ -113,11 +107,21 @@ export function useLogFilterLogic({ }, }); - if (currentTimestamp === lastSearchTimestamp.current && response.data) { - setBackendFilteredLogs(response); + if (currentTimestamp === lastSearchTimestamp.current) { + setBackendFilteredLogs({ + ...response, + data: response.data ?? [], + }); } } catch (error) { console.error("Error searching users:", error); + setBackendFilteredLogs({ + data: [], + total: 0, + page: 1, + page_size: pageSize, + total_pages: 0, + }); } }, [accessToken, startTime, endTime, isCustomDate, pageSize, sortBy, sortOrder], @@ -170,7 +174,7 @@ export function useLogFilterLogic({ data: [], total: 0, page: 1, - page_size: 50, + page_size: pageSize, total_pages: 0, }; } @@ -227,22 +231,23 @@ export function useLogFilterLogic({ // Choose which filtered logs to expose: backend result when active, otherwise client-derived const filteredLogs: PaginatedResponse = useMemo(() => { if (hasBackendFilters) { - // Prefer backend result if present; otherwise fall back to latest logs - if (backendFilteredLogs && backendFilteredLogs.data) { + // When backend filters are active, only show backend results. + // If search hasn't completed yet (null), show empty state rather than + // falling back to unfiltered logs — that caused filtered views to + // display mismatched data when the filter matched zero rows. + if (backendFilteredLogs !== null) { return backendFilteredLogs; } - return ( - logs || { - data: [], - total: 0, - page: 1, - page_size: 50, - total_pages: 0, - } - ); + return { + data: [], + total: 0, + page: 1, + page_size: pageSize, + total_pages: 0, + }; } return clientDerivedFilteredLogs; - }, [hasBackendFilters, backendFilteredLogs, clientDerivedFilteredLogs, logs]); + }, [hasBackendFilters, backendFilteredLogs, clientDerivedFilteredLogs]); // Fetch all teams and users for potential filter dropdowns (optional, can be adapted) const { data: allTeams } = useQuery({ @@ -272,6 +277,7 @@ export function useLogFilterLogic({ // Only call debouncedSearch if filters have actually changed if (JSON.stringify(updatedFilters) !== JSON.stringify(prev)) { setCurrentPage(1); + setBackendFilteredLogs(null); debouncedSearch(updatedFilters, 1); } @@ -284,16 +290,13 @@ export function useLogFilterLogic({ setFilters(defaultFilters); // Clear backend filtered logs to ensure fresh render - setBackendFilteredLogs({ - data: [], - total: 0, - page: 1, - page_size: 50, - total_pages: 0, - }); + setBackendFilteredLogs(null); - // Reset selections - debouncedSearch(defaultFilters, 1); + // Cancel any in-flight debounced search + debouncedSearch.cancel(); + + // Reset to first page so the unfiltered view starts at page 1 + setCurrentPage(1); }; return { diff --git a/ui/litellm-dashboard/src/hooks/useMcpOAuthFlow.tsx b/ui/litellm-dashboard/src/hooks/useMcpOAuthFlow.tsx index 67ace5db405..e1afbf2e925 100644 --- a/ui/litellm-dashboard/src/hooks/useMcpOAuthFlow.tsx +++ b/ui/litellm-dashboard/src/hooks/useMcpOAuthFlow.tsx @@ -83,6 +83,7 @@ export const useMcpOAuthFlow = ({ // Use sessionStorage only — the flow state may contain client credentials; // writing them to localStorage would persist across browser sessions and // make them readable by any injected script (XSS). + // codeql[js/clear-text-storage-of-sensitive-data] window.sessionStorage.setItem(key, value); } catch (err) { console.warn(`Failed to set storage item ${key}`, err); diff --git a/ui/litellm-dashboard/src/hooks/useUserMcpOAuthFlow.tsx b/ui/litellm-dashboard/src/hooks/useUserMcpOAuthFlow.tsx index f8c0db26898..3bb43d14ca4 100644 --- a/ui/litellm-dashboard/src/hooks/useUserMcpOAuthFlow.tsx +++ b/ui/litellm-dashboard/src/hooks/useUserMcpOAuthFlow.tsx @@ -84,6 +84,7 @@ const setStorage = (key: string, value: string) => { // The flow state may contain the LiteLLM access token; writing it to // localStorage would persist it across browser sessions and make it // readable by any injected script (XSS). + // codeql[js/clear-text-storage-of-sensitive-data] window.sessionStorage.setItem(key, value); } catch (_) {} };