scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-05 08:07:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 137

fix: resolve merge conflicts with upstream/main

Browse source 
- Add redis_caching_coverage to coverage combine list (from upstream)
- Remove e2e_ui_testing job (deleted upstream)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
user 2026-04-09 18:04:29 +00:00
commit 7d507c4be6
No known key found for this signature in database
87 changed files with 6127 additions and 832 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

288
.circleci/config.yml
View file

@ -1375,6 +1375,57 @@ jobs:
paths:
- audio_coverage.xml
- audio_coverage
redis_caching_unit_tests:
docker:
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
steps:
- checkout
- setup_google_dns
- run:
name: Install Dependencies
command: |
python -m pip install --upgrade pip uv
uv pip install --system -r requirements.txt
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
pip install "pytest-cov==5.0.0"
pip install "pytest-asyncio==0.21.1"
pip install "pytest-xdist==3.6.1"
pip install "pytest-rerunfailures==14.0"
# Run pytest and generate JUnit XML report
- run:
name: Run tests
command: |
pwd
ls
python -m pytest -vv \
tests/local_testing/test_dual_cache.py \
tests/local_testing/test_redis_batch_optimizations.py \
tests/local_testing/test_router_utils.py \
--cov=litellm --cov-report=xml \
-x -s -v --junitxml=test-results/junit.xml \
--durations=5 -n 2 \
--reruns 2 --reruns-delay 1
no_output_timeout: 20m
- run:
name: Rename the coverage files
command: |
mv coverage.xml redis_caching_coverage.xml
mv .coverage redis_caching_coverage
# Store test results
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- redis_caching_coverage.xml
- redis_caching_coverage
installing_litellm_on_python:
docker:
- image: cimg/python:3.11
@ -2902,7 +2953,7 @@ jobs:
rm -f /tmp/uv-install.sh
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
export PATH="$HOME/.local/bin:$PATH"
uv run --with 'coverage[toml]==7.10.6' coverage combine realtime_translation_coverage ocr_coverage search_coverage mcp_coverage litellm_mcps_tests_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage
uv run --with 'coverage[toml]==7.10.6' coverage combine realtime_translation_coverage ocr_coverage search_coverage mcp_coverage litellm_mcps_tests_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage redis_caching_coverage
uv run --with 'coverage[toml]==7.10.6' coverage xml
- codecov/upload:
file: ./coverage.xml
@ -3065,6 +3116,117 @@ jobs:
CI=true npm run test -- --run \
--pool forks --poolOptions.forks.maxForks=8
e2e_ui_testing:
docker:
- image: cimg/python:3.12-browsers
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
- image: cimg/postgres:16.0
environment:
POSTGRES_USER: e2euser
POSTGRES_PASSWORD: e2epassword
POSTGRES_DB: litellm_e2e
resource_class: large
working_directory: ~/project
environment:
DATABASE_URL: "postgresql://e2euser:e2epassword@localhost:5432/litellm_e2e"
CI: "true"
steps:
- checkout
- setup_google_dns
- restore_cache:
keys:
- ui-e2e-py-deps-v1-{{ checksum "requirements.txt" }}
- run:
name: Install Python dependencies
command: |
python -m pip install --upgrade pip uv
uv pip install --system -r requirements.txt
pip install "prisma==0.11.0"
prisma generate --schema litellm/proxy/schema.prisma
- save_cache:
key: ui-e2e-py-deps-v1-{{ checksum "requirements.txt" }}
paths:
- ~/.local/lib
- ~/.local/bin
- restore_cache:
keys:
- ui-e2e-node-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
- run:
name: Install Node dependencies and Playwright
command: |
cd ui/litellm-dashboard
npm ci
npx playwright install chromium --with-deps
- save_cache:
key: ui-e2e-node-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
paths:
- ui/litellm-dashboard/node_modules
- run:
name: Build UI from source
command: |
cd ui/litellm-dashboard
npm run build
cp -r out/ ../../litellm/proxy/_experimental/out/
# Restructure HTML so extensionless routes work (login.html -> login/index.html)
find ../../litellm/proxy/_experimental/out -name '*.html' ! -name 'index.html' | while read -r f; do
d="${f%.html}"; mkdir -p "$d"; mv "$f" "$d/index.html"
done
- run:
name: Wait for PostgreSQL
command: dockerize -wait tcp://localhost:5432 -timeout 30s
- run:
name: Push Prisma schema
command: prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
- run:
name: Seed database
command: |
PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \
-f ui/litellm-dashboard/e2e_tests/fixtures/seed.sql
- run:
name: Start mock LLM server
command: python ui/litellm-dashboard/e2e_tests/fixtures/mock_llm_server/server.py
background: true
- run:
name: Start LiteLLM proxy
environment:
LITELLM_MASTER_KEY: "sk-1234"
MOCK_LLM_URL: "http://127.0.0.1:8090/v1"
DISABLE_SCHEMA_UPDATE: "true"
SERVER_ROOT_PATH: ""
PROXY_LOGOUT_URL: ""
command: |
python -m litellm.proxy.proxy_cli \
--config ui/litellm-dashboard/e2e_tests/fixtures/config.yml \
--port 4000
background: true
- run:
name: Wait for proxy to be ready
command: |
for i in $(seq 1 60); do
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:4000/health -H "Authorization: Bearer sk-1234" 2>/dev/null || true)
if [ "$HTTP_CODE" = "200" ]; then
echo "Proxy is ready"
exit 0
fi
sleep 2
done
echo "Proxy failed to start"
exit 1
- run:
name: Run Playwright E2E tests
command: |
cd ui/litellm-dashboard
npx playwright test --config e2e_tests/playwright.config.ts
no_output_timeout: 10m
- store_artifacts:
path: ui/litellm-dashboard/test-results
destination: e2e-test-results
- store_artifacts:
path: ui/litellm-dashboard/playwright-report
destination: e2e-playwright-report
build_docker_database_image:
machine:
image: ubuntu-2204:2024.04.1
@ -3090,91 +3252,6 @@ jobs:
paths:
- litellm-docker-database.tar.zst
e2e_ui_testing:
machine:
image: ubuntu-2204:2023.10.1
resource_class: large
working_directory: ~/project
parameters:
browser:
type: string
steps:
- checkout
- setup_google_dns
- attach_workspace:
at: ~/project
- run:
name: Load Docker Database Image
command: |
zstd -d litellm-docker-database.tar.zst --stdout | docker load
docker images | grep litellm-docker-database
- run:
name: Install Dependencies
command: |
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
rm -f /tmp/uv-install.sh
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
export PATH="$HOME/.local/bin:$PATH"
if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then
export PATH="$HOME/miniconda/bin:$PATH"
source "$HOME/miniconda/etc/profile.d/conda.sh"
conda activate myenv
fi
uv sync --frozen --all-groups --all-extras --python "$(which python)"
- run:
name: Install Playwright Browsers
command: |
npx playwright install
- run:
name: Run Docker container
command: |
docker run -d \
-p 4000:4000 \
-e DATABASE_URL=$E2E_UI_TEST_DATABASE_URL \
-e LITELLM_MASTER_KEY="sk-1234" \
-e OPENAI_API_KEY=$OPENAI_API_KEY \
-e UI_USERNAME="admin" \
-e UI_PASSWORD="gm" \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
--name litellm-docker-database-<< parameters.browser >> \
-v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug
- run:
name: Install curl and dockerize
command: |
sudo apt-get update
sudo apt-get install -y curl
sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
- run:
name: Start outputting logs
command: docker logs -f litellm-docker-database-<< parameters.browser >>
background: true
- run:
name: Wait for app to be ready
command: dockerize -wait http://localhost:4000 -timeout 5m
- run:
name: Run Playwright Tests
command: |
npx playwright test \
--project << parameters.browser >> \
--config ui/litellm-dashboard/e2e_tests/playwright.config.ts \
--reporter=html \
--output=test-results
no_output_timeout: 15m
- store_artifacts:
path: test-results
destination: playwright-results
- store_artifacts:
path: playwright-report
destination: playwright-report
prisma_schema_sync:
machine:
@ -3403,32 +3480,12 @@ workflows:
only:
- main
- /litellm_.*/
# - e2e_ui_testing:
# name: e2e_ui_testing_chromium
# browser: chromium
# context: e2e_ui_tests
# requires:
# - ui_build
# - build_docker_database_image
# - prisma_schema_sync
# filters:
# branches:
# only:
# - main
# - /litellm_.*/
# - e2e_ui_testing:
# name: e2e_ui_testing_firefox
# browser: firefox
# context: e2e_ui_tests
# requires:
# - ui_build
# - build_docker_database_image
# - prisma_schema_sync
# filters:
# branches:
# only:
# - main
# - /litellm_.*/
- e2e_ui_testing:
filters:
branches:
only:
- main
- /litellm_.*/
- build_and_test:
requires:
- build_docker_database_image
@ -3617,6 +3674,12 @@ workflows:
only:
- main
- /litellm_.*/
- redis_caching_unit_tests:
filters:
branches:
only:
- main
- /litellm_.*/
- upload-coverage:
requires:
- realtime_translation_testing
@ -3635,6 +3698,7 @@ workflows:
- image_gen_testing
- logging_testing
- audio_testing
- redis_caching_unit_tests
- langfuse_logging_unit_tests
- local_testing_part1
- local_testing_part2

17
.github/workflows/_test-unit-services-base.yml vendored
View file

@ -27,11 +27,6 @@ on:
required: false
type: number
default: 10
enable-redis:
description: "Pass Redis Cloud credentials to tests via REDIS_HOST/PORT/PASSWORD env vars"
required: false
type: boolean
default: false
enable-postgres:
description: "Start a local Postgres service container and run Prisma migrations"
required: false
@ -43,12 +38,6 @@ on:
type: string
default: "run"
secrets:
REDIS_HOST:
required: false
REDIS_PORT:
required: false
REDIS_PASSWORD:
required: false
DATABASE_URL:
required: false
POSTGRES_USER:
@ -66,11 +55,8 @@ jobs:
timeout-minutes: ${{ inputs.timeout-minutes }}
# Environment is derived from the enable-* flags, not caller-controllable.
# This prevents callers from passing arbitrary environment names to bypass secret scoping.
# Note: Postgres service container always starts (GHA limitation), so any Redis job
# also needs Postgres secrets → uses integration-redis-postgres, not integration-redis.
environment: >-
${{
inputs.enable-redis && 'integration-redis-postgres' ||
inputs.enable-postgres && 'integration-postgres' ||
''
}}
@ -139,9 +125,6 @@ jobs:
WORKERS: ${{ inputs.workers }}
RERUNS: ${{ inputs.reruns }}
DATABASE_URL: ${{ inputs.enable-postgres && secrets.DATABASE_URL || '' }}
REDIS_HOST: ${{ inputs.enable-redis && secrets.REDIS_HOST || '' }}
REDIS_PORT: ${{ inputs.enable-redis && secrets.REDIS_PORT || '' }}
REDIS_PASSWORD: ${{ inputs.enable-redis && secrets.REDIS_PASSWORD || '' }}
run: |
if [ "${WORKERS}" = "0" ]; then
uv run --no-sync pytest ${TEST_PATH:?} \

41
.github/workflows/test-unit-caching-redis.yml vendored
View file

@ -1,41 +0,0 @@
name: "Unit Tests: Caching (Redis)"
# Uses cloud Redis credentials — only runs on trusted branches, not PRs.
# This prevents external PRs from accessing Redis credentials.
on:
push:
branches: [main, "litellm_*"]
permissions:
contents: read
id-token: write
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
caching-redis:
uses: ./.github/workflows/_test-unit-services-base.yml
with:
# Redis-only tests that do NOT require provider API keys.
# Tests needing API keys (test_caching.py, test_caching_ssl.py, test_prometheus_service.py,
# test_router_caching.py) are in Phase 3 integration workflows.
test-path: >-
tests/local_testing/test_dual_cache.py
tests/local_testing/test_redis_batch_optimizations.py
tests/local_testing/test_router_utils.py
workers: 2
reruns: 2
timeout-minutes: 20
enable-redis: true
enable-postgres: false
artifact-name: caching-redis
secrets:
REDIS_HOST: ${{ secrets.REDIS_HOST }}
REDIS_PORT: ${{ secrets.REDIS_PORT }}
REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }}
DATABASE_URL: ${{ secrets.DATABASE_URL }}
POSTGRES_USER: ${{ secrets.POSTGRES_USER }}
POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}

1
.github/workflows/test-unit-proxy-db.yml vendored
View file

@ -41,7 +41,6 @@ jobs:
workers: ${{ matrix.workers }}
reruns: 2
timeout-minutes: ${{ matrix.timeout }}
enable-redis: false
enable-postgres: true
artifact-name: proxy-db-${{ matrix.test-group }}
secrets:

1
.github/workflows/test-unit-security.yml vendored
View file

@ -22,7 +22,6 @@ jobs:
workers: 1
reruns: 2
timeout-minutes: 20
enable-redis: false
enable-postgres: true
artifact-name: security
secrets:

8
docker/README.md
View file

@ -13,19 +13,19 @@ To build and run the application, you will use the `docker-compose.yml` file loc
### 1. Set the Master Key
The application requires a `MASTER_KEY` for signing and validating tokens. You must set this key as an environment variable before running the application.
The application requires a `LITELLM_MASTER_KEY` for signing and validating tokens. You must set this key as an environment variable before running the application.
Create a `.env` file in the root of the project and add the following line:
```
MASTER_KEY=your-secret-key
LITELLM_MASTER_KEY=your-secret-key
```
Replace `your-secret-key` with a strong, randomly generated secret.
### 2. Build and Run the Containers
Once you have set the `MASTER_KEY`, you can build and run the containers using the following command:
Once you have set the `LITELLM_MASTER_KEY`, you can build and run the containers using the following command:
```bash
docker compose up -d --build
@ -89,4 +89,4 @@ This command should succeed (showing engine versions) even with `--network none`
## Troubleshooting
- **`build_admin_ui.sh: not found`**: This error can occur if the Docker build context is not set correctly. Ensure that you are running the `docker-compose` command from the root of the project.
- **`Master key is not initialized`**: This error means the `MASTER_key` environment variable is not set. Make sure you have created a `.env` file in the project root with the `MASTER_KEY` defined.
- **`Master key is not initialized`**: This error means the `LITELLM_MASTER_KEY` environment variable is not set. Make sure you have created a `.env` file in the project root with the `LITELLM_MASTER_KEY` defined.

131
docs/my-website/docs/observability/ramp_integration.md Normal file
View file

@ -0,0 +1,131 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Ramp
Send AI usage and cost data to Ramp for automated spend tracking.
[Ramp](https://ramp.com/) is a finance automation platform that helps businesses manage expenses, corporate cards, and vendor payments. With the Ramp callback integration, your LiteLLM AI usage — including token counts, model costs, and request metadata — is automatically sent to Ramp for real-time spend visibility.
:::info
We want to learn how we can make the callbacks better! Meet the LiteLLM [founders](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) or
join our [discord](https://discord.gg/wuPM9dRgDw)
:::
## Pre-Requisites
1. Log in to [Ramp](https://app.ramp.com/) and search for **"LiteLLM"** using the search bar. Click the **LiteLLM** integration result.
> **Note:** Only business owners and admins can access and configure integrations.
2. On the LiteLLM integration page, click the **Connect** button in the top right.
3. In the Connect LiteLLM drawer, click **Generate API Key** to create an API key.
> **Important:** Copy the API key immediately — it won't be shown again. If you lose it, you can revoke the existing key and generate a new one from the integration settings.
```shell
pip install litellm
```
## Quick Start
Set your `RAMP_API_KEY` and add `"ramp"` to your callbacks to start logging LLM usage to Ramp.
<Tabs>
<TabItem value="python" label="SDK">
```python
litellm.callbacks = ["ramp"]
```
```python
import litellm
import os
# Ramp API Key
os.environ["RAMP_API_KEY"] = "your-ramp-api-key"
# LLM API Keys
os.environ['OPENAI_API_KEY'] = ""
# Set ramp as a callback
litellm.callbacks = ["ramp"]
# OpenAI call
response = litellm.completion(
model="gpt-3.5-turbo",
messages=[
{"role": "user", "content": "Hi - I'm testing Ramp integration"}
]
)
```
</TabItem>
<TabItem value="proxy" label="LiteLLM Proxy">
1. Setup config.yaml
```yaml
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: openai/gpt-3.5-turbo
api_key: os.environ/OPENAI_API_KEY
litellm_settings:
callbacks: ["ramp"]
environment_variables:
RAMP_API_KEY: os.environ/RAMP_API_KEY
```
2. Start LiteLLM Proxy
```bash
litellm --config /path/to/config.yaml
```
3. Test it!
```bash
curl -L -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-d '{
"model": "gpt-3.5-turbo",
"messages": [
{
"role": "user",
"content": "Hey, how are you?"
}
]
}'
```
</TabItem>
</Tabs>
## What Data is Logged?
LiteLLM sends the [Standard Logging Payload](https://docs.litellm.ai/docs/proxy/logging_spec) to Ramp on successful LLM API calls, which includes:
- **Request details**: Model, messages, parameters
- **Response details**: Completion text, token usage, latency
- **Metadata**: User ID, custom metadata, timestamps
- **Cost tracking**: Response cost based on token usage
## Authentication
Set the `RAMP_API_KEY` environment variable with your Ramp API key.
| Environment Variable | Description |
|---|---|
| `RAMP_API_KEY` | Your Ramp API key (required) |
## Support & Talk to Founders
- [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)
- [Community Discord 💭](https://discord.gg/wuPM9dRgDw)
- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬
- Our emails ✉️ ishaan@berri.ai / krrish@berri.ai

16
docs/my-website/docs/providers/gemini.md
View file

@ -65,14 +65,13 @@ response = completion(
- modalities
- reasoning_content
- audio (for TTS models only)
- service_tier
**Anthropic Params**
- thinking (used to set max budget tokens across anthropic/gemini models)
[**See Updated List**](https://github.com/BerriAI/litellm/blob/main/litellm/llms/gemini/chat/transformation.py#L70)
## Usage - Thinking / `reasoning_content`
LiteLLM translates OpenAI's `reasoning_effort` to Gemini's `thinking` parameter. [Code](https://github.com/BerriAI/litellm/blob/620664921902d7a9bfb29897a7b27c1a7ef4ddfb/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py#L362)
@ -298,6 +297,19 @@ curl http://0.0.0.0:4000/v1/chat/completions \
## Usage - `service_tier`
LiteLLM propagates OpenAI's `service_tier` parameter to Gemini, and also extracts it from the response headers (`x-gemini-service-tier`) into `model_response.service_tier`.
| OpenAI `service_tier` | Gemini `service_tier` | Notes |
| --------------------- | --------------------- | ----- |
| `"auto"` | `"priority"` | LiteLLM maps OpenAI's `"auto"` to Gemini's `"priority"` tier, as `priority` will fall back on Gemini. |
| `"flex"` | `"flex"` | Direct mapping. |
| `"priority"` | `"priority"` | Direct mapping. |
| `"default"` | `"standard"` | LiteLLM maps `"default"` to `"standard"`. |
| Any other value | Passed as-is (lowercased) | Values are case-insensitive and normalized to lowercase. |
On the response, LiteLLM maps `"standard"` back to `"default"` for the Gemini API.
## Text-to-Speech (TTS) Audio Output

267
docs/my-website/docs/providers/sap.md
View file

@ -55,24 +55,33 @@ uv add litellm
```
### Step 2: Set Your Credentials
Choose **one** of these authentication methods:
> **Breaking change**: credential resolution is "first-source-wins"
>
> Credential resolution no longer merges individual fields across sources.
>
> Resolution order is:
`kwargs` → `service key` → `env (AICORE_*)` → `config` → `VCAP service`
>
> **Important behavior:** once LiteLLM finds *any* credential value in a source, it takes **all** credentials from that source exclusively (except `resource_group`, which may still be resolved separately).
Choose **one** of these authentication methods:
<Tabs>
<TabItem value="service-key" label="Service Key JSON (Recommended)">
<Tabs>
<TabItem value="service-key" label="Service Key JSON (Recommended)">
The simplest approach - paste your entire service key as a single environment variable.
The simplest approach - paste your entire service key as a single environment variable. The service key must be wrapped in a `credentials` object:
> **Note:** the service key no more needs to be wrapped in a "credentials" key.
```bash
export AICORE_SERVICE_KEY='{
"credentials": {
"clientid": "your-client-id",
"clientsecret": "your-client-secret",
"url": "https://<your-instance>.authentication.sap.hana.ondemand.com",
"serviceurls": {
"AI_API_URL": "https://api.ai.<your-region>.aws.ml.hana.ondemand.com"
}
}
}'
export AICORE_RESOURCE_GROUP="default"
```
@ -220,6 +229,17 @@ model="sap/gemini-2.5-pro"
# Incorrect - missing prefix
model="gpt-4o" # ❌ Won't work
```
3. **Environment variables** - Set the following list of credentials in .env file
<pre>
AICORE_AUTH_URL = "https://* * * .authentication.sap.hana.ondemand.com/oauth/token",
AICORE_CLIENT_ID = " *** ",
AICORE_CLIENT_SECRET = " *** ",
AICORE_RESOURCE_GROUP = " *** ",
AICORE_BASE_URL = "https://api.ai.***.cfapps.sap.hana.ondemand.com/v2"
</pre>
Other credential configuration options are also available. For more information, see the [SAP AI Core Documentation](https://help.sap.com/doc/generative-ai-hub-sdk/CLOUD/en-US/_reference/README_sphynx.html#configuration).
## Usage - LiteLLM Python SDK
### Proxy Usage
@ -506,6 +526,241 @@ response = embedding(
print(response.data[0]["embedding"]) # Vector representation
```
### Additional Modules
The SAP Gen AI Hub includes additional modules for advanced use cases:
- [Grounding](https://help.sap.com/docs/sap-ai-core/generative-ai/grounding-035c455a5a424697b60f4a24b6d791fe?locale=en-US)
- [Translation](https://help.sap.com/docs/sap-ai-core/generative-ai/translation?locale=en-US)
- [Data Masking](https://help.sap.com/docs/sap-ai-core/generative-ai/data-masking-d9a54d9ca54b40beacbd24e1663ec3b4?locale=en-US)
- [Content Filtering](https://help.sap.com/docs/sap-ai-core/generative-ai/content-filtering?locale=en-US)
#### Grounding
Grounding is a service designed to handle data-related tasks, such as grounding and retrieval, using vector databases. It provides specialized data retrieval through these databases, grounding the retrieval process with your own external and context-relevant data. Grounding combines generative AI capabilities with the ability to use real-time, precise data to improve decision-making and business operations for specific AI-driven business solutions.
##### Prerequisites
To use the Grounding module in the orchestration pipeline, you need to prepare the knowledge base in advance.
Generative AI hub offers multiple options for users to provide data (prepare a knowledge base):
- For Option 1: Upload the documents to a supported data repository and run the data pipeline to vectorize the documents.
- For Option 2: Provide the chunks of document via Vector API directly.
To use grounding, choose from one of the following options.
Usage example:
```python showLineNumbers title="Grounding Example"
from litellm import completion
grounding_config = {
'type': 'document_grounding_service',
'config': {
'filters': [
{'id': 's3-docs',
'data_repository_type': 'vector',
'search_config': {'max_chunk_count': 2},
'data_repositories': ['012345-6789-0123-4567-890123456789']
}
],
'placeholders': {'input': ['user_query'], 'output': 'grounding_response'},
'metadata_params': ['source', 'webUrl', 'title', 'mimeType', 'fileSuffix']
}
}
response = completion(model="sap/gpt-4o",
messages=[
{"content":"""Facility Solutions Company provides services to luxury residential complexes,
apartments, individual homes, and commercial properties such as office buildings, retail
spaces, industrial facilities, and educational institutions. Customers are encouraged to
reach out with maintenance requests, service deficiencies, follow-ups, or any issues they
need by email.""", "role": "system"},
{"content":"""You are a helpful assistant for any queries for answering questions.
Answer the request by providing relevant answers that fit to the request.
Request: {{ ?user_query }}
Context:{{ ?grounding_response }}""", "role": "user"}
],
placeholder_values={"user_query": "Is there a complaint?"},
grounding=grounding_config
)
print(response.choices[0].message.content)
```
For more information about all available grounding configurations, see the [documentation](https://help.sap.com/docs/sap-ai-core/generative-ai/using-grounding-module-e1c4dd100dfb42ab890e1d95f3516187?locale=en-US).
#### Translation
The translation module allows you to translate LLM text prompts into a chosen target language.
```python showLineNumbers title="Translation Example"
from litellm import completion
translation_config = {
'input':
{'type': 'sap_document_translation',
'config':
{'source_language': 'en-US',
'target_language': 'de-DE'}
},
'output':
{'type': 'sap_document_translation',
'config':
{'source_language': 'de-DE',
'target_language': 'fr-FR'}
}
}
response = completion(model="sap/gpt-4o",
messages=[{"role": "user", "content": "Hello world!"}],
translation=translation_config)
print(response.choices[0].message.content)
```
For more information about all available translation configurations, see the [documentation](https://help.sap.com/docs/sap-ai-core/generative-ai/translation?locale=en-US)
#### Data Masking
The data masking module serves to anonymize or pseudonymize personally identifiable information from the input for selected entities.
```python showLineNumbers title="Data Masking Example"
from litellm import completion, embedding
masking_config = {
'providers':
[
{
'type': 'sap_data_privacy_integration',
'method': 'anonymization',
'entities': [
{'type': 'profile-address'},
{'type': 'profile-email'},
{'type': 'profile-phone'},
{'type': 'profile-person'},
{'type': 'profile-location'}
]
}
]
}
mock_cv = "some text with personal information"
response = completion(model="sap/gpt-4o",
messages=[{"role": "user", "content": "Give a one sentence summary of the CV. CV: {{?cv}}?"}],
placeholder_values={"cv": mock_cv},
masking=masking_config)
print(response.choices[0].message.content)
# Data masking module also available for embedding
response = embedding(model="sap/text-embedding-3-small",
input=mock_cv,
masking=masking_config)
print(response.data[0])
```
For more information about all available data masking configurations, see the [documentation](https://help.sap.com/docs/sap-ai-core/generative-ai/enhancing-model-consumption-with-data-masking-66ad6f469afc4c2cbaa91a27a33f7b21?locale=en-US)
#### Content Filtering
The content filtering module allows you to filter input and output based on content safety criteria.
The module supports two services:
* Azure Content Safety
* Llama Guard 3
```python showLineNumbers title="Content Filtering Example"
from litellm import completion
filtering_config_azure = {
'input':
{
'filters':
[
{'type': 'azure_content_safety',
'config':
{'hate': 0,
'sexual': 0,
'violence': 0,
'self_harm': 0
}
}
]
},
'output':
{
'filters':
[
{'type': 'azure_content_safety',
'config': {'hate': 0,
'sexual': 0,
'violence': 0,
'self_harm': 0
}
}
]
}
}
response = completion(model="sap/gpt-4o",
messages=[{"role": "user", "content": "Hello world!"}],
filtering=filtering_config_azure)
print(response.choices[0].message.content)
# The model responds normally because the content does not violate any safety rules.
try:
response = completion(model="sap/gpt-4o",
messages=[{"role": "user", "content": "I hate you"}],
filtering=filtering_config_azure)
except Exception as e:
print(e)
# The service raises an error:
# "Input Filter: Content filtered due to safety violations. Please modify the prompt and try again."
```
For more information about all available content filtering configurations, see the [documentation](https://help.sap.com/docs/sap-ai-core/generative-ai/content-filtering?locale=en-US)
#### List of modules configuration for fallback
SAP GEN AI Hub supports a fallback mechanism for handling errors. This mechanism allows you to specify a list of fallback modules to use in case of errors. The fallback modules should contain all parameters that are required for configuring the request.
Required parameters:
- `model`
- `messages`
Optional parameters:
- `filtering`
- `grounding`
- `translation`
- `masking`
- `tools`
- and any of model's specific parameters.
```python showLineNumbers title="Fallback Example"
from litellm import completion
translation_config = {
'input':
{'type': 'sap_document_translation',
'config':
{'source_language': 'en-US',
'target_language': 'de-DE'}
},
'output':
{'type': 'sap_document_translation',
'config':
{'source_language': 'de-DE',
'target_language': 'fr-FR'}
}
}
response = completion(model="sap/gpt-4o",
messages=[{"role": "user", "content": "Hello world!"}],
translation=translation_config,
fallback_sap_modules=[{
"model":"sap/gemini-2.5-flash",
"messages":[{"role": "user", "content": "Hello world!"}],
"translation":translation_config
}])
# In case of error with the first configuration (model gpt-4o), the fallback module is used.
print(response.choices[0].message.content)
```
## Reference
### Supported Parameters

1
docs/my-website/docs/proxy/config_settings.md
View file

@ -597,6 +597,7 @@ router_settings:
| LITELLM_MCP_TOOL_LISTING_TIMEOUT | Timeout in seconds for listing tools from an MCP server. Default is 30
| LITELLM_MCP_METADATA_TIMEOUT | HTTP client timeout in seconds for OAuth metadata fetching. Default is 10
| LITELLM_MCP_HEALTH_CHECK_TIMEOUT | Health check timeout in seconds for MCP servers. Default is 10
| LITELLM_MCP_STDIO_EXTRA_COMMANDS | Comma-separated extra command basenames allowed for MCP stdio transport beyond the built-in allowlist. Example: `my-mcp-bin`. Empty by default
| MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL | Default TTL in seconds for MCP OAuth2 token cache. Default is 3600
| MCP_OAUTH2_TOKEN_CACHE_MAX_SIZE | Maximum number of entries in MCP OAuth2 token cache. Default is 200
| MCP_OAUTH2_TOKEN_CACHE_MIN_TTL | Minimum TTL in seconds for MCP OAuth2 token cache. Default is 10

274
docs/my-website/docs/proxy/credential_routing.md Normal file
View file

@ -0,0 +1,274 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Per-Team/Project Credential Routing
Route the same model to different LLM provider endpoints (e.g. different Azure instances) based on which team or project makes the request.
## Overview
In multi-tenant deployments, different teams often need the same model name (e.g., `gpt-4`) to hit different provider endpoints — for example, separate Azure OpenAI instances per business unit for cost isolation, data residency, or rate limit separation.
**Credential routing** lets you configure this in team/project metadata using the existing [credentials table](./ui_credentials.md), without duplicating model definitions or creating separate model groups per team.
```
Hotel Team → gpt-4 → https://hotel-eastus.openai.azure.com/
Flight Team → gpt-4 → https://flight-centralus.openai.azure.com/
```
### Precedence Chain
When a request comes in, the system walks this precedence chain (first match wins):
1. **Clientside credentials** — `api_base`/`api_key` passed in the request body ([docs](./clientside_auth.md))
2. **Project model-specific** — override for this exact model in the project's `model_config`
3. **Project default** — `defaultconfig` in the project's `model_config`
4. **Team model-specific** — override for this exact model in the team's `model_config`
5. **Team default** — `defaultconfig` in the team's `model_config`
6. **Deployment default** — the model's `litellm_params` as configured in `config.yaml`
## Quick Start
### Step 1: Create Credentials
Store your Azure endpoint credentials in the credentials table. You can do this via the [UI](./ui_credentials.md) or API:
```bash showLineNumbers
# Create credential for Hotel team's Azure endpoint
curl -X POST 'http://0.0.0.0:4000/credentials' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"credential_name": "hotel-azure-eastus",
"credential_values": {
"api_base": "https://hotel-eastus.openai.azure.com/",
"api_key": "sk-azure-hotel-key-xxx"
}
}'
```
```bash showLineNumbers
# Create credential for Flight team's Azure endpoint
curl -X POST 'http://0.0.0.0:4000/credentials' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"credential_name": "flight-azure-centralus",
"credential_values": {
"api_base": "https://flight-centralus.openai.azure.com/",
"api_key": "sk-azure-flight-key-xxx"
}
}'
```
### Step 2: Set `model_config` on Teams
Add a `model_config` key to the team's metadata referencing the credential by name:
```bash showLineNumbers
# Hotel team — default Azure endpoint for all models
curl -X PATCH 'http://0.0.0.0:4000/team/update' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"team_id": "hotel-team-id",
"metadata": {
"model_config": {
"defaultconfig": {
"azure": {
"litellm_credentials": "hotel-azure-eastus"
}
}
}
}
}'
```
```bash showLineNumbers
# Flight team — default Azure endpoint for all models
curl -X PATCH 'http://0.0.0.0:4000/team/update' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"team_id": "flight-team-id",
"metadata": {
"model_config": {
"defaultconfig": {
"azure": {
"litellm_credentials": "flight-azure-centralus"
}
}
}
}
}'
```
### Step 3: Make Requests
Requests are automatically routed to the correct Azure endpoint based on the API key's team:
```bash showLineNumbers
# Request using Hotel team's API key → routes to hotel-eastus.openai.azure.com
curl http://localhost:4000/v1/chat/completions \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-hotel-team-key' \
-d '{"model": "gpt-4", "messages": [{"role": "user", "content": "Hello"}]}'
# Request using Flight team's API key → routes to flight-centralus.openai.azure.com
curl http://localhost:4000/v1/chat/completions \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-flight-team-key' \
-d '{"model": "gpt-4", "messages": [{"role": "user", "content": "Hello"}]}'
```
## Per-Model Overrides
You can set different credentials for specific models while keeping a default for everything else:
```bash showLineNumbers
curl -X PATCH 'http://0.0.0.0:4000/team/update' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"team_id": "hotel-team-id",
"metadata": {
"model_config": {
"defaultconfig": {
"azure": {
"litellm_credentials": "hotel-azure-eastus"
}
},
"gpt-4": {
"azure": {
"litellm_credentials": "hotel-azure-westus"
}
}
}
}
}'
```
With this config:
- `gpt-4` requests → `hotel-azure-westus` credential (model-specific)
- All other models → `hotel-azure-eastus` credential (default)
## Project-Level Overrides
Projects inherit their team's `model_config` but can override at the project level. Project overrides take precedence over team overrides.
```bash showLineNumbers
# Project overrides the team default for all models
curl -X PATCH 'http://0.0.0.0:4000/project/update' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"project_id": "hotel-rec-app-id",
"metadata": {
"model_config": {
"defaultconfig": {
"azure": {
"litellm_credentials": "hotel-rec-azure"
}
},
"gpt-4-vision": {
"azure": {
"litellm_credentials": "hotel-rec-vision"
}
}
}
}
}'
```
### Full Example: Hotel Team with Two Projects
**Setup:**
- **Hotel Team**: default `hotel-azure-eastus`, GPT-4 override to `hotel-azure-westus`
- **Hotel Rec App** (project): default `hotel-rec-azure`, GPT-4-Vision override to `hotel-rec-vision`
- **Hotel Review App** (project): no overrides — inherits team config
**Resolution:**
| Request | Resolved Credential | Why |
|---|---|---|
| Hotel Rec App → `gpt-4` | `hotel-rec-azure` | Project default (no project model-specific match for gpt-4) |
| Hotel Rec App → `gpt-4-vision` | `hotel-rec-vision` | Project model-specific |
| Hotel Review App → `gpt-3.5` | `hotel-azure-eastus` | Team default (no project config) |
| Hotel Review App → `gpt-4` | `hotel-azure-westus` | Team model-specific |
## `model_config` Schema
The `model_config` key is a JSON object in team/project `metadata`:
```json
{
"model_config": {
"defaultconfig": {
"<provider>": {
"litellm_credentials": "<credential-name>"
}
},
"<model-name>": {
"<provider>": {
"litellm_credentials": "<credential-name>"
}
}
}
}
```
| Field | Description |
|---|---|
| `defaultconfig` | Fallback credential for any model not explicitly listed |
| `<model-name>` | Model-specific override — must match the LiteLLM model group name |
| `<provider>` | Provider key (e.g. `azure`, `openai`, `bedrock`). When the model name includes a provider prefix (e.g. `azure/gpt-4`), the system prefers the matching provider key |
| `litellm_credentials` | Name of a credential in the [credentials table](./ui_credentials.md) |
### Credential Values
The referenced credential can contain any combination of:
| Key | Description |
|---|---|
| `api_base` | Provider endpoint URL |
| `api_key` | API key for the provider |
| `api_version` | API version (e.g. for Azure) |
Only keys present in the credential are applied. Keys already in the request (e.g. clientside `api_version`) are never overwritten.
## Enabling the Feature
This feature is **disabled by default** and must be explicitly enabled. To enable it:
<Tabs>
<TabItem value="config" label="config.yaml">
```yaml
litellm_settings:
enable_model_config_credential_overrides: true
```
</TabItem>
<TabItem value="env" label="Environment Variable">
```bash
export LITELLM_ENABLE_MODEL_CONFIG_CREDENTIAL_OVERRIDES=true
```
</TabItem>
</Tabs>
:::info
The feature flag must be enabled before `model_config` entries in team/project metadata take effect. Without it, credential routing is completely inert — no metadata is read, no credentials are resolved.
:::
## Related Documentation
- [Adding LLM Credentials](./ui_credentials.md) — Create and manage reusable credentials
- [Project Management](./project_management.md) — Project hierarchy and API
- [Team Budgets](./team_budgets.md) — Team-level budget management
- [Clientside LLM Credentials](./clientside_auth.md) — Passing credentials in the request body
- [Credential Usage Tracking](./credential_usage_tracking.md) — Track spend by credential

3
docs/my-website/sidebars.js
View file

@ -563,7 +563,8 @@ const sidebars = {
"proxy/model_access",
"proxy/model_access_groups",
"proxy/access_groups",
"proxy/team_model_add"
"proxy/team_model_add",
"proxy/credential_routing"
]
},
{

2
enterprise/litellm_enterprise/enterprise_callbacks/pagerduty/pagerduty.py
View file

@ -114,6 +114,7 @@ class PagerDutyAlerting(SlackAlerting):
user_api_key_max_budget=_meta.get("user_api_key_max_budget"),
user_api_key_budget_reset_at=_meta.get("user_api_key_budget_reset_at"),
user_api_key_org_id=_meta.get("user_api_key_org_id"),
user_api_key_org_alias=_meta.get("user_api_key_org_alias"),
user_api_key_team_id=_meta.get("user_api_key_team_id"),
user_api_key_project_id=_meta.get("user_api_key_project_id"),
user_api_key_project_alias=_meta.get("user_api_key_project_alias"),
@ -196,6 +197,7 @@ class PagerDutyAlerting(SlackAlerting):
else None
),
user_api_key_org_id=user_api_key_dict.org_id,
user_api_key_org_alias=user_api_key_dict.organization_alias,
user_api_key_team_id=user_api_key_dict.team_id,
user_api_key_project_id=user_api_key_dict.project_id,
user_api_key_project_alias=user_api_key_dict.project_alias,

1
litellm/__init__.py
View file

@ -318,6 +318,7 @@ return_response_headers: bool = (
False # get response headers from LLM Api providers - example x-remaining-requests,
)
enable_json_schema_validation: bool = False
enable_model_config_credential_overrides: bool = False
enable_key_alias_format_validation: bool = (
False # opt-in validation of key_alias format on /key/generate and /key/update
)

6
litellm/_logging.py
View file

@ -243,6 +243,12 @@ class JsonFormatter(Formatter):
if key not in _STANDARD_RECORD_ATTRS and key not in json_record:
json_record[key] = value
# Set component/logger only if not already supplied via extra={...}
if "component" not in json_record:
json_record["component"] = record.name
if "logger" not in json_record:
json_record["logger"] = f"{record.filename}:{record.lineno}"
if record.exc_info:
json_record["stacktrace"] = record.exc_text or self.formatException(
record.exc_info

6
litellm/a2a_protocol/providers/pydantic_ai_agents/config.py
View file

@ -21,11 +21,11 @@ class PydanticAIProviderConfig(BaseA2AProviderConfig):
request_id: str,
params: Dict[str, Any],
api_base: Optional[str] = None,
**kwargs,
**kwargs: Any,
) -> Dict[str, Any]:
"""Handle non-streaming request to Pydantic AI agent."""
if not api_base:
raise ValueError("api_base is required for Pydantic AI agents")
if api_base is None:
raise ValueError("api_base is required for PydanticAIProviderConfig")
return await PydanticAIHandler.handle_non_streaming(
request_id=request_id,
params=params,

11
litellm/constants.py
View file

@ -141,6 +141,17 @@ MCP_TOOL_LISTING_TIMEOUT = float(os.getenv("LITELLM_MCP_TOOL_LISTING_TIMEOUT", "
MCP_METADATA_TIMEOUT = float(os.getenv("LITELLM_MCP_METADATA_TIMEOUT", "10.0"))
MCP_HEALTH_CHECK_TIMEOUT = float(os.getenv("LITELLM_MCP_HEALTH_CHECK_TIMEOUT", "10.0"))
# Allowlist of commands permitted for MCP stdio transport.
# Prevents arbitrary command execution via /mcp-rest/test/* endpoints or server creation.
# Note: allowlisted runtimes can still execute code via args (e.g. python -c "...").
# This is an accepted residual risk since these endpoints require PROXY_ADMIN.
# Extend via LITELLM_MCP_STDIO_EXTRA_COMMANDS env var (comma-separated).
_MCP_STDIO_EXTRA_COMMANDS = os.getenv("LITELLM_MCP_STDIO_EXTRA_COMMANDS", "")
MCP_STDIO_ALLOWED_COMMANDS: frozenset = frozenset(
{"npx", "uvx", "python", "python3", "node", "docker", "deno"}
| (set(_MCP_STDIO_EXTRA_COMMANDS.split(",")) - {""})
)
LITELLM_UI_ALLOW_HEADERS = [
"x-litellm-semantic-filter",
"x-litellm-semantic-filter-tools",

9
litellm/integrations/generic_api/generic_api_compatible_callbacks.json
View file

@ -33,5 +33,14 @@
"X-Qualifire-API-Key": "{{environment_variables.QUALIFIRE_API_KEY}}"
},
"environment_variables": ["QUALIFIRE_API_KEY", "QUALIFIRE_WEBHOOK_URL"]
},
"ramp": {
"event_types": ["llm_api_success"],
"endpoint": "https://api.ramp.com/developer/v1/ai-usage/litellm",
"headers": {
"Content-Type": "application/json",
"Authorization": "Bearer {{environment_variables.RAMP_API_KEY}}"
},
"environment_variables": ["RAMP_API_KEY"]
}
}

7
litellm/integrations/prometheus.py
View file

@ -1031,6 +1031,9 @@ class PrometheusLogger(CustomLogger):
user_api_key_org_id = standard_logging_payload["metadata"].get(
"user_api_key_org_id"
)
user_api_key_org_alias = standard_logging_payload["metadata"].get(
"user_api_key_org_alias"
)
output_tokens = standard_logging_payload["completion_tokens"]
tokens_used = standard_logging_payload["total_tokens"]
response_cost = standard_logging_payload["response_cost"]
@ -1068,6 +1071,8 @@ class PrometheusLogger(CustomLogger):
model_group=standard_logging_payload["model_group"],
team=user_api_team,
team_alias=user_api_team_alias,
org_id=user_api_key_org_id,
org_alias=user_api_key_org_alias,
user=user_id,
user_email=standard_logging_payload["metadata"]["user_api_key_user_email"],
status_code="200",
@ -1746,6 +1751,8 @@ class PrometheusLogger(CustomLogger):
api_key_alias=user_api_key_dict.key_alias,
team=user_api_key_dict.team_id,
team_alias=user_api_key_dict.team_alias,
org_id=user_api_key_dict.org_id,
org_alias=user_api_key_dict.organization_alias,
requested_model=request_data.get("model", ""),
status_code=str(status_code),
exception_status=str(status_code),

2
litellm/litellm_core_utils/litellm_logging.py
View file

@ -4754,6 +4754,7 @@ class StandardLoggingPayloadSetup:
user_api_key_budget_reset_at=None,
user_api_key_team_id=None,
user_api_key_org_id=None,
user_api_key_org_alias=None,
user_api_key_project_id=None,
user_api_key_project_alias=None,
user_api_key_user_id=None,
@ -5586,6 +5587,7 @@ def get_standard_logging_metadata(
user_api_key_budget_reset_at=None,
user_api_key_team_id=None,
user_api_key_org_id=None,
user_api_key_org_alias=None,
user_api_key_project_id=None,
user_api_key_project_alias=None,
user_api_key_user_id=None,

12
litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py
View file

@ -322,9 +322,8 @@ class StandardBuiltInToolCostTracking:
)
if has_url_citations:
return True
# Fallback: Check usage object for providers that use usage instead of annotations
# (e.g., Vertex AI Gemini uses usage.prompt_tokens_details.web_search_requests)
if usage is not None:
# Vertex AI Gemini uses usage.prompt_tokens_details.web_search_requests
if (
hasattr(usage, "prompt_tokens_details")
and usage.prompt_tokens_details is not None
@ -335,6 +334,15 @@ class StandardBuiltInToolCostTracking:
and usage.prompt_tokens_details.web_search_requests is not None
):
return True
# Anthropic Claude (direct API and Vertex AI) uses server_tool_use.web_search_requests.
# Without this check, Claude ModelResponse always falls through to return False
# and _handle_web_search_cost() is never called.
if (
hasattr(usage, "server_tool_use")
and usage.server_tool_use is not None
and usage.server_tool_use.web_search_requests is not None
):
return True
return False
elif isinstance(response_object, ResponsesAPIResponse):
# response api explicitly includes web_search_call in the output

12
litellm/llms/__init__.py
View file

@ -34,6 +34,18 @@ def get_cost_for_web_search_request(
return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage)
elif custom_llm_provider.startswith("vertex_ai"):
# Anthropic Claude models on Vertex AI populate server_tool_use.web_search_requests
# (same as the direct Anthropic API), not prompt_tokens_details.web_search_requests
# (which is the Gemini field). Route claude-* models to the Anthropic calculator.
model_key: str = model_info.get("key", "") if model_info else ""
if "claude" in model_key.lower():
from .anthropic.cost_calculation import get_cost_for_anthropic_web_search
verbose_logger.debug(
"vertex_ai/claude model detected — routing web search cost to Anthropic calculator"
)
return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage)
from .vertex_ai.gemini.cost_calculator import (
cost_per_web_search_request as cost_per_web_search_request_vertex_ai,
)

21
litellm/llms/anthropic/chat/handler.py
View file

@ -89,7 +89,12 @@ async def make_call(
try:
response = await client.post(
api_base, headers=headers, data=data, stream=True, timeout=timeout
api_base,
headers=headers,
data=data,
stream=True,
timeout=timeout,
logging_obj=logging_obj,
)
except httpx.HTTPStatusError as e:
error_headers = getattr(e, "headers", None)
@ -142,7 +147,12 @@ def make_sync_call(
try:
response = client.post(
api_base, headers=headers, data=data, stream=True, timeout=timeout
api_base,
headers=headers,
data=data,
stream=True,
timeout=timeout,
logging_obj=logging_obj,
)
except httpx.HTTPStatusError as e:
error_headers = getattr(e, "headers", None)
@ -266,7 +276,11 @@ class AnthropicChatCompletion(BaseLLM):
try:
response = await async_handler.post(
api_base, headers=headers, json=data, timeout=timeout
api_base,
headers=headers,
json=data,
timeout=timeout,
logging_obj=logging_obj,
)
except Exception as e:
## LOGGING
@ -469,6 +483,7 @@ class AnthropicChatCompletion(BaseLLM):
headers=headers,
data=json.dumps(data),
timeout=timeout,
logging_obj=logging_obj,
)
except Exception as e:
status_code = getattr(e, "status_code", 500)

1
litellm/llms/gemini/chat/transformation.py
View file

@ -91,6 +91,7 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig):
"modalities",
"parallel_tool_calls",
"web_search_options",
"service_tier",
]
if supports_reasoning(model, custom_llm_provider="gemini"):
supported_params.append("reasoning_effort")

0
litellm/llms/sap/__init__.py Normal file
View file

623
litellm/llms/sap/chat/models.py
View file

@ -1,6 +1,8 @@
from typing import Union, Literal
from typing import Union, Literal, Optional
from enum import Enum
import warnings
from pydantic import BaseModel, Field, field_validator
from pydantic import BaseModel, Field, field_validator, model_validator
def validate_different_content(v: Union[str, dict, list]) -> str:
@ -20,7 +22,7 @@ def validate_different_content(v: Union[str, dict, list]) -> str:
elif isinstance(v, str):
return v
raise ValueError("Content must be a string")
return v
class TextContent(BaseModel):
@ -49,6 +51,10 @@ class FunctionTool(BaseModel):
parameters: dict = {"type": "object", "properties": {}}
strict: bool = False
def model_dump(self, **kwargs) -> dict:
kwargs["exclude_unset"] = False
return super().model_dump(**kwargs)
@field_validator("parameters", mode="before")
@classmethod
def ensure_object_type(cls, v: dict) -> dict:
@ -66,6 +72,10 @@ class ChatCompletionTool(BaseModel):
type_: Literal["function"] = Field(default="function", alias="type")
function: FunctionTool
def model_dump(self, **kwargs) -> dict:
kwargs["exclude_unset"] = False
return super().model_dump(**kwargs)
class MessageToolCall(BaseModel):
id: str
@ -114,6 +124,9 @@ class SAPToolChatMessage(BaseModel):
)
ChatMessage = Union[SAPMessage, SAPUserMessage, SAPAssistantMessage, SAPToolChatMessage]
class ResponseFormat(BaseModel):
type_: Literal["text", "json_object"] = Field(default="text", alias="type")
@ -128,3 +141,607 @@ class JSONResponseSchema(BaseModel):
class ResponseFormatJSONSchema(BaseModel):
type_: Literal["json_schema"] = Field(default="json_schema", alias="type")
json_schema: JSONResponseSchema
class KeyValueListPair(BaseModel):
key: str
value: list[str]
class DocumentMetadataKeyValueListPairs(KeyValueListPair):
select_mode: Optional[list[Literal["ignoreIfKeyAbsent"]]] = None
class GroundingSearchConfig(BaseModel):
max_chunk_count: Optional[int] = Field(default=None, ge=0)
max_document_count: Optional[int] = Field(default=None, ge=0)
@model_validator(mode="after")
def validate_max_chunk_count_and_max_document_count(self):
if self.max_chunk_count is not None and self.max_document_count is not None:
raise ValueError("Cannot specify both maxChunkCount and maxDocumentCount.")
return self
class DocumentGroundingFilter(BaseModel):
id_: Optional[str] = Field(default=None, alias="id")
data_repository_type: Literal["vector", "help.sap.com"]
search_config: Optional[GroundingSearchConfig] = None
data_repositories: Optional[list[str]] = None
data_repository_metadata: Optional[list[KeyValueListPair]] = None
document_metadata: Optional[list[DocumentMetadataKeyValueListPairs]] = None
chunk_metadata: Optional[list[KeyValueListPair]] = None
class DocumentGroundingPlaceholders(BaseModel):
input: list[str] = Field(min_length=1)
output: str
class DocumentGroundingConfig(BaseModel):
filters: Optional[list[DocumentGroundingFilter]] = None
placeholders: DocumentGroundingPlaceholders
metadata_params: Optional[list[str]] = None
class GroundingModuleConfig(BaseModel):
type_: Literal["document_grounding_service"] = Field(
default="document_grounding_service", alias="type"
)
config: DocumentGroundingConfig
class Template(BaseModel):
template: list[ChatMessage]
defaults: Optional[dict[str, str]] = None
response_format: Optional[Union[ResponseFormat, ResponseFormatJSONSchema]] = None
tools: Optional[list[ChatCompletionTool]] = None
class LLMModelDetails(BaseModel):
name: str
version: str = "latest"
params: Optional[dict] = None
class PromptTemplatingModuleConfig(BaseModel):
prompt: Template
model: LLMModelDetails
class SAPMaskingProfileEntity(str, Enum):
"""
Enumerates the entity categories that can be masked by the SAP Data Privacy Integration service.
This enum lists different types of personal or sensitive information (PII) that can be detected and masked
by the data masking module, such as personal details, organizational data, contact information, and identifiers.
Values:
PERSON: Represents personal names.
ORG: Represents organizational names.
UNIVERSITY: Represents educational institutions.
LOCATION: Represents geographical locations.
EMAIL: Represents email addresses.
PHONE: Represents phone numbers.
ADDRESS: Represents physical addresses.
SAP_IDS_INTERNAL: Represents internal SAP identifiers.
SAP_IDS_PUBLIC: Represents public SAP identifiers.
URL: Represents URLs.
USERNAME_PASSWORD: Represents usernames and passwords.
NATIONAL_ID: Represents national identification numbers.
IBAN: Represents International Bank Account Numbers.
SSN: Represents Social Security Numbers.
CREDIT_CARD_NUMBER: Represents credit card numbers.
PASSPORT: Represents passport numbers.
DRIVING_LICENSE: Represents driving license numbers.
NATIONALITY: Represents nationality information.
RELIGIOUS_GROUP: Represents religious group affiliation.
POLITICAL_GROUP: Represents political group affiliation.
PRONOUNS_GENDER: Represents pronouns and gender identity.
GENDER: Represents gender information.
SEXUAL_ORIENTATION: Represents sexual orientation.
TRADE_UNION: Represents trade union membership.
SENSITIVE_DATA: Represents any other sensitive information.
"""
PERSON = "profile-person"
ORG = "profile-org"
UNIVERSITY = "profile-university"
LOCATION = "profile-location"
EMAIL = "profile-email"
PHONE = "profile-phone"
ADDRESS = "profile-address"
SAP_IDS_INTERNAL = "profile-sapids-internal"
SAP_IDS_PUBLIC = "profile-sapids-public"
URL = "profile-url"
USERNAME_PASSWORD = "profile-username-password"
NATIONAL_ID = "profile-nationalid"
IBAN = "profile-iban"
SSN = "profile-ssn"
CREDIT_CARD_NUMBER = "profile-credit-card-number"
PASSPORT = "profile-passport"
DRIVING_LICENSE = "profile-driverlicense"
NATIONALITY = "profile-nationality"
RELIGIOUS_GROUP = "profile-religious-group"
POLITICAL_GROUP = "profile-political-group"
PRONOUNS_GENDER = "profile-pronouns-gender"
GENDER = "profile-gender"
SEXUAL_ORIENTATION = "profile-sexual-orientation"
TRADE_UNION = "profile-trade-union"
SENSITIVE_DATA = "profile-sensitive-data"
ETHNICITY = "profile-ethnicity"
class DPIMethodConstant(BaseModel):
"""
Replaces the entity with the specified value followed by an incrementing number
"""
method: Literal["constant"] = "constant"
value: str
class DPIMethodFabricatedData(BaseModel):
"""
Replaces the entity with a randomly generated value appropriate to its type.
"""
method: Literal["fabricated_data"] = "fabricated_data"
class DPICustomEntity(BaseModel):
"""
regex: Regular expression to match the entity
replacement_strategy: Replacement strategy to be used for the entity
"""
regex: str
replacement_strategy: DPIMethodConstant
class DPIStandardEntity(BaseModel):
"""
type: Standard entity type to be masked
replacement_strategy: Replacement strategy to be used for the entity
"""
type_: SAPMaskingProfileEntity = Field(..., alias="type")
replacement_strategy: Optional[
Union[DPIMethodConstant, DPIMethodFabricatedData]
] = None
class MaskGroundingInput(BaseModel):
"""
Controls whether the input to the grounding module will be masked with the configuration
supplied in the masking module
"""
enabled: bool = False
class MaskingProviderConfig(BaseModel):
"""
SAP Data Privacy Integration provider for data masking.
This class implements the SAP Data Privacy Integration service, which can anonymize or pseudonymize
specified entity categories in the input data. It supports masking sensitive information like personal names,
contact details, and identifiers.
Args:
method: The method of masking to apply (anonymization or pseudonymization).
entities: A list of entity categories to be masked, such as names, locations, or emails.
allowlist: A list of strings that should not be masked.
mask_grounding_input: A flag indicating whether to mask input to the grounding module.
"""
type_: Literal["sap_data_privacy_integration"] = Field(
default="sap_data_privacy_integration", alias="type"
)
method: Literal["anonymization", "pseudonymization"]
entities: list[Union[DPIStandardEntity, DPICustomEntity]]
allowlist: Optional[list[str]] = None
mask_grounding_input: Optional[MaskGroundingInput] = None
class MaskingModuleConfig(BaseModel):
"""
Configuration for the data masking module.
Args:
providers: list of masking service provider configurations
masking_providers: list of masking provider configurations
IMPORTANT: use exactly one of the parameters to set the list of masking provider configurations.
DEPRECATED: parameter 'masking_providers' will be removed Sept 15, 2026. Use 'providers' instead.
"""
providers: Optional[list[MaskingProviderConfig]] = Field(min_length=1, default=None)
masking_providers: Optional[list[MaskingProviderConfig]] = Field(
min_length=1, default=None
)
@model_validator(mode="after")
def enforce_exactly_one_provider_list(self):
has_providers = self.providers is not None
has_masking_providers = self.masking_providers is not None
if not has_providers and not has_masking_providers:
raise ValueError(
"For SAP Masking Module Config you must provide 'providers'."
)
if has_providers and has_masking_providers:
raise ValueError(
"For SAP Masking Module Config you must set exactly one of: 'providers' or 'masking_providers', not both."
)
if has_masking_providers:
warnings.warn(
"The 'masking_providers' parameter is deprecated and will be removed on Sept 15, 2026. "
"Use 'providers' instead.",
DeprecationWarning,
stacklevel=5,
)
return self
class AzureThreshold(int, Enum):
"""
Enumerates the threshold levels for the Azure Content Safety service.
This enum defines the various threshold levels that can be used to filter
content based on its safety score. Each threshold value represents a specific
level of content moderation.
Values:
ALLOW_SAFE: Allows only Safe content.
ALLOW_SAFE_LOW: Allows Safe and Low content.
ALLOW_SAFE_LOW_MEDIUM: Allows Safe, Low, and Medium content.
ALLOW_ALL: Allows all content (Safe, Low, Medium, and High).
"""
ALLOW_SAFE = 0
ALLOW_SAFE_LOW = 2
ALLOW_SAFE_LOW_MEDIUM = 4
ALLOW_ALL = 6
class AzureContentFilter(BaseModel):
"""
Specific filter configuration for Azure Content Safety.
This class configures content filtering based on Azure's categories and
severity levels. It allows setting thresholds for hate speech, sexual content,
violence, and self-harm content.
Values:
hate: Threshold for hate speech content.
sexual: Threshold for sexual content.
violence: Threshold for violent content.
self_harm: Threshold for self-harm content.
"""
hate: Optional[Union[AzureThreshold, Literal[0, 2, 4, 6]]] = None
sexual: Optional[Union[AzureThreshold, Literal[0, 2, 4, 6]]] = None
violence: Optional[Union[AzureThreshold, Literal[0, 2, 4, 6]]] = None
self_harm: Optional[Union[AzureThreshold, Literal[0, 2, 4, 6]]] = None
class AzureContentSafetyInput(AzureContentFilter):
"""
Filter configuration for Azure Content Safety Input
Values:
hate: Threshold for hate speech content.
sexual: Threshold for sexual content.
violence: Threshold for violent content.
self_harm: Threshold for self-harm content.
prompt_shield: A flag to use prompt shield
"""
prompt_shield: Optional[bool] = False
class AzureContentSafetyOutput(AzureContentFilter):
"""
Filter configuration for Azure Content Safety Output
Values:
hate: Threshold for hate speech content.
sexual: Threshold for sexual content.
violence: Threshold for violent content.
self_harm: Threshold for self-harm content.
protected_material_code: Detect protected code content from known GitHub repositories.
The scan includes software libraries, source code, algorithms,
and other proprietary programming content.
"""
protected_material_code: Optional[bool] = False
class LlamaGuard38bFilter(BaseModel):
"""
Specific implementation of ContentFilter for Llama Guard 3. Llama Guard 3 is a
Llama-3.1-8B pretrained model, fine-tuned for content safety classification.
Args:
violent_crimes: Responses that enable, encourage, or endorse the commission of violent crimes.
non_violent_crimes: Responses that enable, encourage, or endorse the commission of non-violent crimes.
sex_crimes: Responses that enable, encourage, or endorse the commission of sex-related crimes.
child_exploitation: Responses that contain, describe, enable or endorse the sexual abuse of children.
defamation: Responses that are both verifiably false and likely to injure a living person's reputation.
specialized_advice: Responses that contain specialized financial, medical or legal advice.
privacy: Responses that contain sensitive or nonpublic personal information.
intellectual_property: Responses that may violate the intellectual property rights of any third party.
indiscriminate_weapons: Responses that enable, encourage, or endorse the creation of indiscriminate weapons.
hate: Responses that demean or dehumanize people on the basis of their sensitive, personal characteristics.
self_harm: Responses that enable, encourage, or endorse acts of intentional self-harm.
sexual_content: Responses that contain erotica.
elections: Responses that contain factually incorrect information about electoral systems and processes.
code_interpreter_abuse: Responses that seek to abuse code interpreters.
"""
violent_crimes: bool = Field(default=False)
non_violent_crimes: bool = Field(default=False)
sex_crimes: bool = Field(default=False)
child_exploitation: bool = Field(default=False)
defamation: bool = Field(default=False)
specialized_advice: bool = Field(default=False)
privacy: bool = Field(default=False)
intellectual_property: bool = Field(default=False)
indiscriminate_weapons: bool = Field(default=False)
hate: bool = Field(default=False)
self_harm: bool = Field(default=False)
sexual_content: bool = Field(default=False)
elections: bool = Field(default=False)
code_interpreter_abuse: bool = Field(default=False)
class LlamaGuard38bFilterConfig(BaseModel):
type_: Literal["llama_guard_3_8b"] = Field(default="llama_guard_3_8b", alias="type")
config: LlamaGuard38bFilter
class AzureContentSafetyInputFilterConfig(BaseModel):
type_: Literal["azure_content_safety"] = Field(
default="azure_content_safety", alias="type"
)
config: Optional[AzureContentSafetyInput] = None
class AzureContentSafetyOutputFilterConfig(BaseModel):
type_: Literal["azure_content_safety"] = Field(
default="azure_content_safety", alias="type"
)
config: Optional[AzureContentSafetyOutput] = None
class FilteringStreamOptions(BaseModel):
"""
overlap: Number of characters that should be additionally sent to content filtering services
from previous chunks as additional context.
"""
overlap: Optional[int] = Field(default=0, ge=0, le=10000)
class InputFiltering(BaseModel):
"""Module for managing and applying input content filters.
Args:
filters: List of ContentFilter objects to be applied to input content.
"""
filters: list[
Union[AzureContentSafetyInputFilterConfig, LlamaGuard38bFilterConfig]
] = Field(min_length=1)
class OutputFiltering(BaseModel):
"""Module for managing and applying output content filters.
Args:
filters: List of ContentFilter objects to be applied to output content.
stream_options: Module-specific streaming options.
"""
filters: list[
Union[AzureContentSafetyOutputFilterConfig, LlamaGuard38bFilterConfig]
] = Field(min_length=1)
stream_options: Optional[FilteringStreamOptions] = None
class FilteringModuleConfig(BaseModel):
"""Module for managing and applying content filters.
Args:
input: Module for filtering and validating input content before processing.
output: Module for filtering and validating output content after generation.
"""
input: Optional[InputFiltering] = None
output: Optional[OutputFiltering] = None
@model_validator(mode="after")
def enforce_min_properties(self) -> "FilteringModuleConfig":
"""
Ensure at least one of input or output filtering is provided.
"""
if self.input is None and self.output is None:
raise ValueError(
"For using SAP Filtering Module you must provide at least one property: input or output filters."
)
return self
class SAPDocumentTranslationApplyToSelector(BaseModel):
"""
This selector allows you to define the scope of translation, such as specific placeholders or
messages with specific roles.
For example, {"category": "placeholders",
"items": ["user_input"],
"source_language": "de-DE"}
targets the value of "user_input" in placeholder_values specified in the request payload;
and considers the value to be in German.
"""
category: Literal["placeholders", "template_roles"]
items: list[str]
source_language: str
class InputTranslationConfig(BaseModel):
"""
Configuration for input translation.
Args:
source_language: Language of the text to be translated. Example: de-DE
target_language: Language to which the text should be translated. Example: en-US
apply_to: List of selectors that define the scope of translation.
"""
source_language: Optional[str] = None
target_language: str
apply_to: Optional[list[SAPDocumentTranslationApplyToSelector]] = None
class OutputTranslationConfig(BaseModel):
source_language: Optional[str] = None
target_language: Union[str, SAPDocumentTranslationApplyToSelector]
class SAPDocumentTranslationInput(BaseModel):
"""
Configuration for input translation
Args:
type: The type of translation module (e.g., 'sap_document_translation').
translate_messages_history: If true, the messages history will be translated as well.
config: Configuration object for the translation module.
"""
type_: Literal["sap_document_translation"] = Field(
default="sap_document_translation", alias="type"
)
translate_messages_history: Optional[bool] = None
config: InputTranslationConfig
class SAPDocumentTranslationOutput(BaseModel):
"""
Configuration for output translation
Args:
type: The type of translation module (e.g., 'sap_document_translation').
config: Configuration object for the translation module.
"""
type_: Literal["sap_document_translation"] = Field(
default="sap_document_translation", alias="type"
)
config: OutputTranslationConfig
class TranslationModuleConfig(BaseModel):
"""
Configuration for translation module
Args:
input: Configuration for input translation
output: Configuration for output translation
"""
input: Optional[SAPDocumentTranslationInput] = None
output: Optional[SAPDocumentTranslationOutput] = None
@model_validator(mode="after")
def enforce_min_properties(self) -> "TranslationModuleConfig":
if self.input is None and self.output is None:
raise ValueError(
"TranslationModuleConfig requires at least one of 'input' or 'output'."
)
return self
class ModuleConfig(BaseModel):
prompt_templating: PromptTemplatingModuleConfig
filtering: Optional[FilteringModuleConfig] = None
masking: Optional[MaskingModuleConfig] = None
grounding: Optional[GroundingModuleConfig] = None
translation: Optional[TranslationModuleConfig] = None
class GlobalStreamOptions(BaseModel):
enabled: bool = False
chunk_size: Optional[int] = Field(default=None, ge=1)
delimiters: Optional[list[str]] = None
class OrchestrationConfig(BaseModel):
modules: Union[ModuleConfig, list[ModuleConfig]]
stream: Optional[GlobalStreamOptions] = None
class OrchestrationRequest(BaseModel):
config: OrchestrationConfig
placeholder_values: Optional[dict[str, str]] = None

230
litellm/llms/sap/chat/transformation.py
View file

@ -11,6 +11,7 @@ from typing import (
TYPE_CHECKING,
Iterator,
AsyncIterator,
FrozenSet,
)
from functools import cached_property
import litellm
@ -31,12 +32,13 @@ else:
from ..credentials import get_token_creator
from .models import (
SAPMessage,
SAPAssistantMessage,
SAPToolChatMessage,
ChatCompletionTool,
ResponseFormatJSONSchema,
OrchestrationRequest,
ResponseFormat,
ResponseFormatJSONSchema,
SAPAssistantMessage,
SAPMessage,
SAPToolChatMessage,
SAPUserMessage,
)
from .handler import (
@ -45,9 +47,65 @@ from .handler import (
SAPStreamIterator,
)
# Keys routed outside SAP orchestration `model.params` (prompt, stream, fallbacks, etc.)
_SAP_MODEL_PARAMS_EXCLUDED_KEYS: FrozenSet[str] = frozenset(
{
"tools",
"tool_choice",
"stream_options",
"fallback_sap_modules",
"placeholder_values",
"model_version",
}
)
def validate_dict(data: dict, model) -> dict:
return model(**data).model_dump(by_alias=True)
return model(**data).model_dump(by_alias=True, exclude_unset=True)
def _messages_to_sap_template(messages: List[Dict[str, str]]) -> list: # type: ignore[type-arg]
template = []
for message in messages:
if message["role"] == "user":
template.append(validate_dict(message, SAPUserMessage))
elif message["role"] == "assistant":
template.append(validate_dict(message, SAPAssistantMessage))
elif message["role"] == "tool":
template.append(validate_dict(message, SAPToolChatMessage))
else:
template.append(validate_dict(message, SAPMessage))
return template
def _tools_response_format_and_stream(
optional_params: dict, model_params: dict
) -> Tuple[dict, dict, dict]:
tools_ = optional_params.pop("tools", [])
tools_ = [validate_dict(tool, ChatCompletionTool) for tool in tools_]
tools: dict = {"tools": tools_} if tools_ else {}
response_format = model_params.pop("response_format", {})
resp_type = response_format.get("type", None)
if resp_type:
if resp_type == "json_schema":
response_format = validate_dict(
response_format, ResponseFormatJSONSchema
)
else:
response_format = validate_dict(response_format, ResponseFormat)
response_format = {"response_format": response_format}
model_params.pop("stream", False)
stream_config: dict = {}
if "stream_options" in optional_params:
stream_options = optional_params.pop("stream_options", {})
if "chunk_size" in stream_options:
stream_config["chunk_size"] = stream_options.get("chunk_size")
if "delimiters" in stream_options:
stream_config["delimiters"] = stream_options.get("delimiters")
return tools, response_format, stream_config
class GenAIHubOrchestrationConfig(OpenAIGPTConfig):
@ -208,48 +266,25 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig):
api_base_ = f"{self.deployment_url}/v2/completion"
return api_base_
def transform_request(
def _build_prompt_module(
self,
model: str,
messages: List[Dict[str, str]], # type: ignore
optional_params: dict,
litellm_params: dict,
headers: dict,
model_name: str,
template_messages: List[Dict[str, str]],
params: dict,
) -> dict:
# Filter out parameters that are not valid model params for SAP Orchestration API
# - tools, model_version, deployment_url: handled separately
excluded_params = {"tools", "model_version", "deployment_url"}
# Filter strict for GPT models only - SAP AI Core doesn't accept it as a model param
# LangChain agents pass strict=true at top level, which fails for GPT models
# Anthropic models accept strict, so preserve it for them
if model.startswith("gpt"):
excluded_params.add("strict")
if model_name.startswith("gpt") and "strict" in params:
params.pop("strict")
model_params = {
k: v for k, v in optional_params.items() if k not in excluded_params
}
model_version = params.pop("model_version", "latest")
model_version = optional_params.pop("model_version", "latest")
template = []
for message in messages:
if message["role"] == "user":
template.append(validate_dict(message, SAPUserMessage))
elif message["role"] == "assistant":
template.append(validate_dict(message, SAPAssistantMessage))
elif message["role"] == "tool":
template.append(validate_dict(message, SAPToolChatMessage))
else:
template.append(validate_dict(message, SAPMessage))
tools_ = optional_params.pop("tools", [])
tools_ = params.pop("tools", [])
tools_ = [validate_dict(tool, ChatCompletionTool) for tool in tools_]
if tools_ != []:
tools = {"tools": tools_}
else:
tools = {}
tools = {"tools": tools_} if tools_ else {}
response_format = model_params.pop("response_format", {})
response_format = params.pop("response_format", {})
resp_type = response_format.get("type", None)
if resp_type:
if resp_type == "json_schema":
@ -259,33 +294,104 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig):
else:
response_format = validate_dict(response_format, ResponseFormat)
response_format = {"response_format": response_format}
model_params.pop("stream", False)
stream_config = {}
if "stream_options" in model_params:
# stream_config["enabled"] = True
stream_options = model_params.pop("stream_options", {})
stream_config["chunk_size"] = stream_options.get("chunk_size", 100)
if "delimiters" in stream_options:
stream_config["delimiters"] = stream_options.get("delimiters")
# else:
# stream_config["enabled"] = False
config = {
"config": {
"modules": {
"prompt_templating": {
"prompt": {"template": template, **tools, **response_format},
"model": {
"name": model,
"params": model_params,
"version": model_version,
},
},
else:
response_format = {}
placeholder_defaults = params.pop("placeholder_defaults", {})
placeholder_defaults = (
{"defaults": placeholder_defaults} if placeholder_defaults else {}
)
optional_modules = {}
optional_modules_lst = ["grounding", "masking", "filtering", "translation"]
for module in optional_modules_lst:
if params.get(module, None) is not None:
optional_modules[module] = params.pop(module)
return {
"prompt_templating": {
"prompt": {
"template": template_messages,
**placeholder_defaults,
**tools,
**response_format,
},
"stream": stream_config,
}
"model": {
"name": model_name,
"params": params,
"version": model_version,
},
},
**optional_modules,
}
return config
def transform_request(
self,
model: str,
messages: List[Dict[str, str]], # type: ignore
optional_params: dict,
litellm_params: dict,
headers: dict,
) -> dict:
optional_params = dict(optional_params)
optional_params.pop("deployment_url", None)
template = _messages_to_sap_template(messages)
placeholder_values = optional_params.pop("placeholder_values", None)
fallback_modules = optional_params.pop("fallback_sap_modules", [])
optional_params.pop("stream", None)
stream_config: dict = {}
if "stream_options" in optional_params:
stream_options = optional_params.pop("stream_options", {})
if "chunk_size" in stream_options:
stream_config["chunk_size"] = stream_options["chunk_size"]
if "delimiters" in stream_options:
stream_config["delimiters"] = stream_options["delimiters"]
optional_params.pop("tool_choice", None)
modules = [
self._build_prompt_module(
model_name=model,
template_messages=template,
params=dict(optional_params),
)
]
for modules_dict in fallback_modules:
modules_dict = dict(modules_dict)
fallback_model = modules_dict.pop("model", None)
if fallback_model is None:
raise ValueError(
"Each entry in `fallback_sap_modules` must include a 'model' key."
)
if fallback_model.startswith("sap/"):
fallback_model = fallback_model[4:]
fallback_template = modules_dict.pop("messages", [])
modules.append(
self._build_prompt_module(
model_name=fallback_model,
template_messages=fallback_template,
params=modules_dict,
)
)
config_payload: Dict[str, Any] = {
"modules": modules if len(modules) > 1 else modules[0],
}
if stream_config:
config_payload["stream"] = stream_config
request_body: Dict[str, Any] = {"config": config_payload}
if placeholder_values is not None:
request_body["placeholder_values"] = placeholder_values
body = validate_dict(request_body, OrchestrationRequest)
return body
def transform_response(
self,

379
litellm/llms/sap/credentials.py
View file

@ -1,5 +1,5 @@
from __future__ import annotations
from typing import Any, Callable, Dict, Final, List, Optional, Sequence, Tuple
from typing import Any, Callable, Dict, Final, List, Optional, Sequence, Tuple, Union
from datetime import datetime, timedelta, timezone
from threading import Lock
from pathlib import Path
@ -7,9 +7,11 @@ from dataclasses import dataclass
import json
import os
import tempfile
import httpx
from litellm import sap_service_key
from litellm.llms.custom_httpx.http_handler import _get_httpx_client
from litellm.llms.custom_httpx.http_handler import _get_httpx_client, HTTPHandler
from litellm._logging import verbose_logger
import litellm
AUTH_ENDPOINT_SUFFIX = "/oauth/token"
@ -28,11 +30,25 @@ def _get_home() -> str:
return os.getenv(HOME_PATH_ENV_VAR, DEFAULT_HOME_PATH)
def _get_nested(d: Dict[str, Any], path: Sequence[str]) -> Any:
def _get_nested(d: Union[Dict[str, Any], str], path: Sequence[str]) -> Any:
cur: Any = d
if isinstance(cur, str):
# This shouldn't happen if service keys are pre-parsed correctly
try:
cur = json.loads(cur)
except json.JSONDecodeError:
verbose_logger.warning(
"SAP service key or VCAP service is a string but not valid JSON."
)
return None
for k in path:
if not isinstance(cur, dict) or k not in cur:
raise KeyError(".".join(path))
if not isinstance(cur, dict):
verbose_logger.warning(
f"SAP service key or VCAP service traversal hit non-dict type '{type(cur).__name__}' at key '{k}'."
)
return None
if k not in cur:
return None
cur = cur[k]
return cur
@ -47,6 +63,13 @@ def _load_json_env(var_name: str) -> Optional[Dict[str, Any]]:
return None
def _str_or_none(value) -> Optional[str]:
try:
return str(value) if value is not None else None
except Exception:
return None
def _load_vcap() -> Dict[str, Any]:
return _load_json_env(VCAP_SERVICES_ENV_VAR) or {}
@ -59,6 +82,12 @@ def _get_vcap_service(label: str) -> Optional[Dict[str, Any]]:
return None
@dataclass
class Source:
name: str
get: Callable[[CredentialsValue], Optional[str]]
@dataclass(frozen=True)
class CredentialsValue:
name: str
@ -82,7 +111,6 @@ CREDENTIAL_VALUES: Final[List[CredentialsValue]] = [
transform_fn=lambda url: url.rstrip("/")
+ ("" if url.endswith("/v2") else "/v2"),
),
CredentialsValue("resource_group", default="default"),
CredentialsValue(
"cert_url",
("certurl",),
@ -145,81 +173,239 @@ def _env_name(name: str) -> str:
return f"AICORE_{name.upper()}"
def _resolve_value(
cred: CredentialsValue,
*,
kwargs: Dict[str, Any],
env: Dict[str, str],
config: Dict[str, Any],
service_like: Optional[Dict[str, Any]],
) -> Optional[str]:
# 1) explicit kwargs
if cred.name in kwargs and kwargs[cred.name] is not None:
return kwargs[cred.name]
def extract_credentials(source: Source) -> Dict[str, str]:
"""Extract all credentials from a source."""
credentials = {}
for cv in CREDENTIAL_VALUES:
value = source.get(cv)
if value is not None:
credentials[cv.name] = cv.transform_fn(value) if cv.transform_fn else value
return credentials
# 2) environment variables (primary name)
env_key = _env_name(cred.name)
if env_key in env and env[env_key] is not None:
return env[env_key]
# 3) config file (accept both prefixed and plain keys)
for key in (env_key, cred.name):
if key in config and config[key] is not None:
return config[key]
def resolve_credentials(sources: List[Source]) -> Dict[str, str]:
"""Extract credentials from the first source that has any defined."""
for source in sources:
credentials = extract_credentials(source)
if credentials:
verbose_logger.debug(f"Resolved SAP credentials from source {source.name}")
return credentials
raise ValueError("No credentials found in any source")
# 4) service-like source (AICORE_SERVICE_KEY first, else VCAP)
if service_like and cred.vcap_key:
def resolve_resource_group(sources: List[Source]) -> Optional[str]:
"""Find resource_group from the first source that defines it."""
rg_cred = CredentialsValue("resource_group", default="default")
for source in sources:
value = source.get(rg_cred)
if value is not None:
verbose_logger.debug(
f"Resolved GEN AI Hub resource_group from source {source.name}"
)
return value
return rg_cred.default
def _parse_service_key_once(
service_key: Optional[Union[str, dict]]
) -> Optional[Dict[str, Any]]:
"""
Pre-parse service_key if it's a string to avoid repeated JSON parsing.
Returns None if parsing fails (other credential sources may still work).
"""
if service_key is None:
return None
if isinstance(service_key, dict):
return service_key
if isinstance(service_key, str):
try:
val = _get_nested(service_like, ("credentials",) + cred.vcap_key)
if val is not None:
return val
except KeyError:
pass
return json.loads(service_key)
except json.JSONDecodeError:
verbose_logger.warning(
"SAP service key is a string but not valid JSON. Skipping this source."
)
return None
verbose_logger.warning(
f"SAP service key has unexpected type '{type(service_key).__name__}'. Expected str or dict. Ignoring."
)
return None
# 5) default
return cred.default
def _resolve_credential_from_service_key(
service_key: Optional[Union[str, dict]], cv: CredentialsValue
) -> Optional[str]:
if service_key is None:
return None
val = _str_or_none(
_get_nested(
service_key, (("credentials",) + cv.vcap_key) if cv.vcap_key else (cv.name,)
)
)
if val is None:
return _str_or_none(
_get_nested(service_key, cv.vcap_key if cv.vcap_key else (cv.name,))
)
return val
def fetch_credentials(
service_key: Optional[str] = None, profile: Optional[str] = None, **kwargs
service_key: Optional[Union[str, dict]] = None,
profile: Optional[str] = None,
**kwargs,
) -> Dict[str, str]:
"""
Resolution order per key:
Resolution order (first-source-wins):
Sources are checked in this order:
kwargs
> service key
> env (AICORE_<NAME>)
> config (AICORE_<NAME> or plain <name>)
> service-like source from JSON in $AICORE_SERVICE_KEY (same structure as a VCAP service object)
falling back to service entry in $VCAP_SERVICES with label 'aicore'
> vcap service key
> default
Important:
- Credentials are extracted from the FIRST source that provides any credential value.
- Values are NOT merged per key across sources. Except resource_group, which is merged.
Warning:
- This function does NOT validate the returned credentials just parsed it from the sources.
- Callers MUST explicitly call validate_credentials() on the returned dict
"""
config = init_conf(profile)
env = os.environ # snapshot for testability
service_like = None
if not config:
# Prefer AICORE_SERVICE_KEY if present; otherwise fall back to the VCAP service.
service_like = (
service_key
or sap_service_key
or _load_json_env(SERVICE_KEY_ENV_VAR)
or _get_vcap_service(VCAP_AICORE_SERVICE_NAME)
service_key = _parse_service_key_once(
service_key or litellm.sap_service_key or os.environ.get(SERVICE_KEY_ENV_VAR)
)
vcap_service = _get_vcap_service(VCAP_AICORE_SERVICE_NAME)
sources = [
Source("kwargs", lambda cv: _str_or_none(kwargs.get(cv.name))),
Source(
"service key",
lambda cv: _resolve_credential_from_service_key(service_key, cv),
),
Source(
"environment variables",
lambda cv: _str_or_none(os.environ.get(f"AICORE_{cv.name.upper()}")),
),
Source(
"config file",
lambda cv: _str_or_none(
config.get(f"AICORE_{cv.name.upper()}")
if config.get(f"AICORE_{cv.name.upper()}") is not None
else config.get(cv.name)
),
),
Source(
"VCAP service",
lambda cv: (
_str_or_none(
_get_nested(
vcap_service,
(("credentials",) + cv.vcap_key) if cv.vcap_key else (cv.name,),
)
)
if vcap_service
else None
),
), # type: ignore[arg-type]
]
credentials = resolve_credentials(sources)
resource_group = resolve_resource_group(sources)
if resource_group is not None:
credentials["resource_group"] = resource_group
if "cert_url" in credentials:
credentials["auth_url"] = credentials.pop("cert_url")
return credentials
def validate_credentials(
auth_url: Optional[str] = None,
base_url: Optional[str] = None,
client_id: Optional[str] = None,
client_secret: Optional[str] = None,
cert_str: Optional[str] = None,
key_str: Optional[str] = None,
cert_file_path: Optional[str] = None,
key_file_path: Optional[str] = None,
) -> None:
"""
Validate SAP AI Core credentials for completeness and consistency.
Args:
auth_url: OAuth2 token endpoint URL (required)
base_url: SAP AI Core API base URL (required)
client_id: OAuth2 client ID (required)
client_secret: OAuth2 client secret (for secret-based auth)
cert_str: PEM-encoded certificate string (for cert-based auth)
key_str: PEM-encoded private key string (for cert-based auth)
cert_file_path: Path to certificate file (for file-based cert auth)
key_file_path: Path to private key file (for file-based cert auth)
Raises:
ValueError: If required fields are missing or authentication mode is ambiguous.
Note:
- This function does NOT validate resource_group (resolved separately).
- Exactly one authentication method must be provided:
* client_secret, OR
* (cert_str AND key_str), OR
* (cert_file_path AND key_file_path)
"""
if not auth_url or not client_id or not base_url:
raise ValueError(
"SAP AI Core credentials not found. "
"Please provide credentials by setting appropriate environment variables "
"(e.g. AICORE_CLIENT_ID, AICORE_CLIENT_SECRET, etc.)"
)
out: Dict[str, str] = {}
for cred in CREDENTIAL_VALUES:
value = _resolve_value(cred, kwargs=kwargs, env=env, config=config, service_like=service_like) # type: ignore
if value is None:
continue
if cred.transform_fn:
value = cred.transform_fn(value)
out[cred.name] = value
if "cert_url" in out.keys():
out["auth_url"] = out.pop("cert_url")
return out
modes = [
bool(client_secret),
bool(cert_str) and bool(key_str),
bool(cert_file_path) and bool(key_file_path),
]
if sum(bool(m) for m in modes) != 1:
raise ValueError(
"SAP AI Core credentials are incomplete. "
"Invalid credentials: provide exactly one of client_secret, "
"(cert_str & key_str), or (cert_file_path & key_file_path)."
)
def _request_token(
client_id: str, auth_url: str, timeout: float, cert_pair=None, client_secret=None
) -> tuple[str, datetime]:
data = {"grant_type": "client_credentials", "client_id": client_id}
if client_secret:
data["client_secret"] = client_secret
resp: Optional[httpx.Response] = None
try:
if cert_pair:
with httpx.Client(cert=cert_pair) as raw_client:
handler = HTTPHandler(client=raw_client)
resp = handler.post(auth_url, data=data, timeout=timeout) # type: ignore[arg-type]
payload = resp.json()
else:
handler = _get_httpx_client()
resp = handler.post(auth_url, data=data, timeout=timeout) # type: ignore[arg-type]
payload = resp.json()
access_token = payload["access_token"]
expires_in = int(payload.get("expires_in", 3600))
expiry_date = datetime.now(timezone.utc) + timedelta(seconds=expires_in)
return f"Bearer {access_token}", expiry_date
except Exception as e:
msg = resp.text if resp is not None else getattr(e, "text", str(e))
raise RuntimeError(f"Token request failed: {msg}") from e
def get_token_creator(
service_key: Optional[str] = None,
service_key: Optional[Union[str, dict]] = None,
profile: Optional[str] = None,
*,
timeout: float = 30.0,
@ -237,7 +423,7 @@ def get_token_creator(
Args:
profile: Optional AICore profile name
timeout: HTTP request timeout in seconds (default 30s)
timeout: Timeout for HTTP requests
expiry_buffer_minutes: Refresh the token this many minutes before expiry
overrides: Any explicit credential overrides (client_id, client_secret, etc.)
@ -251,6 +437,7 @@ def get_token_creator(
)
auth_url = credentials.get("auth_url")
base_url = credentials.get("base_url")
client_id = credentials.get("client_id")
client_secret = credentials.get("client_secret")
cert_str = credentials.get("cert_str")
@ -259,49 +446,30 @@ def get_token_creator(
key_file_path = credentials.get("key_file_path")
# Sanity check
if not auth_url or not client_id:
raise ValueError(
"fetch_credentials did not return valid 'auth_url' or 'client_id'"
)
modes = [
client_secret is not None,
(cert_str is not None and key_str is not None),
(cert_file_path is not None and key_file_path is not None),
]
if sum(bool(m) for m in modes) != 1:
raise ValueError(
"Invalid credentials: provide exactly one of client_secret, "
"(cert_str & key_str), or (cert_file_path & key_file_path)."
)
validate_credentials(
auth_url,
base_url,
client_id,
client_secret,
cert_str,
key_str,
cert_file_path,
key_file_path,
)
lock = Lock()
token: Optional[str] = None
token_expiry: Optional[datetime] = None
def _request_token(cert_pair=None) -> tuple[str, datetime]:
data = {"grant_type": "client_credentials", "client_id": client_id}
if client_secret:
data["client_secret"] = client_secret
client = _get_httpx_client()
# with httpx.Client(cert=cert_pair, timeout=timeout) as client:
resp = client.post(auth_url, data=data)
try:
resp.raise_for_status()
payload = resp.json()
access_token = payload["access_token"]
expires_in = int(payload.get("expires_in", 3600))
expiry_date = datetime.now(timezone.utc) + timedelta(seconds=expires_in)
return f"Bearer {access_token}", expiry_date
except Exception as e:
msg = getattr(resp, "text", str(e))
raise RuntimeError(f"Token request failed: {msg}") from e
def _fetch_token() -> tuple[str, datetime]:
# Case 1: secret-based auth
if client_secret:
return _request_token()
return _request_token(
auth_url=auth_url, # type: ignore[arg-type]
client_id=client_id, # type: ignore[arg-type]
timeout=timeout,
client_secret=client_secret,
)
# Case 2: cert/key strings
if cert_str and key_str:
cert_str_fixed = cert_str.replace("\\n", "\n")
@ -313,9 +481,24 @@ def get_token_creator(
f.write(cert_str_fixed)
with open(key_path, "w") as f:
f.write(key_str_fixed)
return _request_token(cert_pair=(cert_path, key_path))
return _request_token(
auth_url=auth_url, # type: ignore[arg-type]
client_id=client_id, # type: ignore[arg-type]
timeout=timeout,
cert_pair=(cert_path, key_path),
)
# Case 3: file-based cert/key
return _request_token(cert_pair=(cert_file_path, key_file_path))
if cert_file_path is not None and key_file_path is not None:
return _request_token(
auth_url=auth_url, # type: ignore[arg-type]
client_id=client_id, # type: ignore[arg-type]
timeout=timeout,
cert_pair=(cert_file_path, key_file_path),
)
# Defensive guard: should never reach here due to validate_credentials()
raise ValueError(
"Invalid authentication configuration: no valid credentials found. "
)
def get_token() -> str:
nonlocal token, token_expiry

42
litellm/llms/sap/embed/transformation.py
View file

@ -5,6 +5,7 @@ Translates from OpenAI's `/v1/embeddings` to IBM's `/text/embeddings` route.
from typing import Optional, List, Dict, Literal, Union
from pydantic import BaseModel, Field
from functools import cached_property
from litellm.llms.sap.chat.models import MaskingModuleConfig
import httpx
@ -47,25 +48,36 @@ class EmbeddingsResponse(BaseModel):
class EmbeddingModel(BaseModel):
name: str
version: str = "latest"
params: dict = Field(default_factory=dict, validation_alias="parameters")
params: dict = Field(default_factory=dict)
timeout: Optional[int] = Field(default=None, ge=1, le=600)
max_retries: Optional[int] = Field(default=None, ge=0, le=5)
class EmbeddingsModelConfig(BaseModel):
model: EmbeddingModel
class EmbeddingsModules(BaseModel):
embeddings: EmbeddingModel
embeddings: EmbeddingsModelConfig
masking: Optional[MaskingModuleConfig] = None
class EmbeddingInput(BaseModel):
text: Union[str, List[str]]
type: Literal["text", "document", "query"] = "text"
type: Optional[Literal["text", "document", "query"]] = None
class EmbeddingConfig(BaseModel):
modules: EmbeddingsModules
class EmbeddingRequest(BaseModel):
config: EmbeddingsModules
config: EmbeddingConfig
input: EmbeddingInput
def validate_dict(data: dict, model) -> dict:
return model(**data).model_dump()
return model(**data).model_dump(exclude_unset=True, by_alias=True)
class GenAIHubEmbeddingConfig(BaseEmbeddingConfig):
@ -152,15 +164,23 @@ class GenAIHubEmbeddingConfig(BaseEmbeddingConfig):
model_dict["name"] = model
model_dict["version"] = optional_params.get("version", "latest")
model_dict["params"] = optional_params.get("parameters", {})
timeout = optional_params.get("timeout", None)
if timeout is not None:
model_dict["timeout"] = timeout
max_retries = optional_params.get("max_retries", None)
if max_retries is not None:
model_dict["max_retries"] = max_retries
input_dict = {"text": input}
input_type = optional_params.get("type")
if input_type is not None:
input_dict["type"] = input_type
masking = optional_params.get("masking")
masking = {"masking": masking} if masking is not None else {}
body = {
"config": {
"modules": {
"embeddings": {"model": validate_dict(model_dict, EmbeddingModel)}
}
},
"input": validate_dict(input_dict, EmbeddingInput),
"config": {"modules": {"embeddings": {"model": model_dict}, **masking}},
"input": input_dict,
}
body = validate_dict(body, EmbeddingRequest)
return body
def transform_embedding_response(

31
litellm/llms/triton/embedding/transformation.py
View file

@ -8,7 +8,8 @@ from litellm.llms.base_llm.embedding.transformation import (
LiteLLMLoggingObj,
)
from litellm.types.llms.openai import AllEmbeddingInputValues
from litellm.types.utils import EmbeddingResponse
from litellm.types.utils import EmbeddingResponse, Usage
from litellm.utils import token_counter
from ..common_utils import TritonError
@ -103,8 +104,36 @@ class TritonEmbeddingConfig(BaseEmbeddingConfig):
model_response.model = raw_response_json.get("model_name", "None")
model_response.data = _embedding_output
model_response.usage = self._build_embedding_usage(
model=model, request_data=request_data
)
return model_response
def _build_embedding_usage(self, model: str, request_data: dict) -> Usage:
input_data = request_data.get("inputs", [])
input_text_values: List[str] = []
for item in input_data:
if isinstance(item, dict) and item.get("name") == "input_text":
data_values = item.get("data", [])
if isinstance(data_values, list):
input_text_values = [str(value) for value in data_values]
break
prompt_tokens = 0
for text in input_text_values:
if not text:
continue
try:
prompt_tokens += token_counter(model=model, text=text)
except Exception:
prompt_tokens += len(text.split())
return Usage(
prompt_tokens=prompt_tokens,
completion_tokens=0,
total_tokens=prompt_tokens,
)
def get_error_class(
self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
) -> BaseLLMException:

10
litellm/llms/vertex_ai/gemini/transformation.py
View file

@ -763,6 +763,16 @@ def _transform_request_body( # noqa: PLR0915
data["generationConfig"] = generation_config
if cached_content is not None:
data["cachedContent"] = cached_content
if service_tier := optional_params.pop("service_tier", None):
if isinstance(service_tier, str):
if service_tier.lower() == "default":
data["serviceTier"] = "standard"
else:
data["serviceTier"] = service_tier.lower()
else:
data["serviceTier"] = service_tier
# Only add labels for Vertex AI endpoints (not Google GenAI/AI Studio) and only if non-empty
if labels and custom_llm_provider != LlmProviders.GEMINI:
data["labels"] = labels

230
litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
View file

@ -318,6 +318,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
"parallel_tool_calls",
"web_search_options",
"include_server_side_tool_invocations",
"service_tier",
]
# Add penalty parameters only for non-preview models
@ -362,6 +363,17 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
"""
return Tools(googleSearch={})
def _map_service_tier_param(self, value: str, optional_params: dict) -> None:
"""
Map OpenAI service_tier (string) to Gemini serviceTier.
'auto' maps to 'priority'.
Other values are passed lowercased.
"""
if value.lower() == "auto":
optional_params["service_tier"] = "priority"
else:
optional_params["service_tier"] = value.lower()
def _transform_computer_use_config(self, computer_use_config: dict) -> dict:
"""
Transform Computer Use configuration to Gemini API format.
@ -1121,6 +1133,8 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
optional_params = self._add_tools_to_optional_params(
optional_params, [_tools]
)
elif param == "service_tier" and isinstance(value, str):
self._map_service_tier_param(value, optional_params)
elif param == "include_server_side_tool_invocations" and value is True:
optional_params["include_server_side_tool_invocations"] = True
if litellm.vertex_ai_safety_settings is not None:
@ -2415,6 +2429,14 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
"provider_specific_fields", {}
)["traffic_type"] = traffic_type
## ADD SERVICE TIER ##
if getattr(raw_response, "headers", None):
if service_tier := raw_response.headers.get("x-gemini-service-tier"):
if service_tier.lower() == "standard":
setattr(model_response, "service_tier", "default")
else:
setattr(model_response, "service_tier", service_tier.lower())
except Exception as e:
raise VertexAIError(
message="Received={}, Error converting to valid response block={}. File an issue if litellm error - https://github.com/BerriAI/litellm/issues".format(
@ -2513,6 +2535,7 @@ async def make_call(
streaming_response=response.aiter_lines(),
sync_stream=False,
logging_obj=logging_obj,
response_headers=response.headers,
)
# LOGGING
logging_obj.post_call(
@ -2555,6 +2578,7 @@ def make_sync_call(
streaming_response=response.iter_lines(),
sync_stream=True,
logging_obj=logging_obj,
response_headers=response.headers,
)
# LOGGING
@ -3011,7 +3035,11 @@ class VertexLLM(VertexBase):
class ModelResponseIterator:
def __init__(
self, streaming_response, sync_stream: bool, logging_obj: LoggingClass
self,
streaming_response,
sync_stream: bool,
logging_obj: LoggingClass,
response_headers: Optional[Dict[str, str]] = None,
):
from litellm.litellm_core_utils.prompt_templates.common_utils import (
check_is_function_call,
@ -3022,10 +3050,120 @@ class ModelResponseIterator:
self.accumulated_json = ""
self.sent_first_chunk = False
self.logging_obj = logging_obj
self.response_headers = response_headers or {}
self.is_function_call = check_is_function_call(logging_obj)
self.cumulative_tool_call_index: int = 0
self.has_seen_tool_calls: bool = False
def _apply_stream_candidates(
self,
_candidates: List[Candidates],
model_response: Any,
) -> Tuple[List[dict], List[dict], List[dict], List[dict]]:
(
grounding_metadata,
url_context_metadata,
safety_ratings,
citation_metadata,
self.cumulative_tool_call_index,
) = VertexGeminiConfig._process_candidates(
_candidates,
model_response,
self.logging_obj.optional_params,
cumulative_tool_call_index=self.cumulative_tool_call_index,
)
# Track whether tool_calls have been seen across streaming chunks.
# Gemini sends tool_calls and finishReason in separate chunks,
# so we need to remember if earlier chunks contained tool_calls
# to correctly set finish_reason="tool_calls" per the OpenAI spec.
if not self.has_seen_tool_calls:
for choice in model_response.choices:
if (
hasattr(choice, "delta")
and choice.delta
and choice.delta.tool_calls
):
self.has_seen_tool_calls = True
break
# Handle final chunk with finishReason but no content.
# _process_candidates skips candidates without "content",
# so the finish_reason from the final chunk is lost.
if not model_response.choices and _candidates:
from litellm.types.utils import Delta, StreamingChoices
for candidate in _candidates:
finish_reason_str = candidate.get("finishReason")
if finish_reason_str is not None:
if self.has_seen_tool_calls:
mapped_finish_reason = "tool_calls"
else:
mapped_finish_reason = VertexGeminiConfig._check_finish_reason(
None, finish_reason_str
)
choice = StreamingChoices(
finish_reason=mapped_finish_reason,
index=candidate.get("index", 0),
delta=Delta(content=None, role=None),
logprobs=None,
enhancements=None,
)
model_response.choices.append(choice)
# Also handle the case where the final chunk has empty
# content (e.g. text:"") WITH finishReason. In this case
# _process_candidates DOES create a choice, but maps
# finishReason="STOP" to "stop" because the current chunk
# has no tool_calls. Override if we saw tool_calls earlier.
if self.has_seen_tool_calls:
for choice in model_response.choices:
if choice.finish_reason == "stop":
choice.finish_reason = "tool_calls"
setattr(model_response, "vertex_ai_grounding_metadata", grounding_metadata) # type: ignore
setattr(model_response, "vertex_ai_url_context_metadata", url_context_metadata) # type: ignore
setattr(model_response, "vertex_ai_safety_ratings", safety_ratings) # type: ignore
setattr(model_response, "vertex_ai_citation_metadata", citation_metadata) # type: ignore
return grounding_metadata, url_context_metadata, safety_ratings, citation_metadata
def _apply_stream_usage_metadata(
self,
processed_chunk: Any,
model_response: Any,
grounding_metadata: List[dict],
) -> Optional[Usage]:
if "usageMetadata" not in processed_chunk:
return None
usage = VertexGeminiConfig._calculate_usage(
completion_response=processed_chunk,
)
web_search_requests = VertexGeminiConfig._calculate_web_search_requests(
grounding_metadata
)
if web_search_requests is not None:
cast(
PromptTokensDetailsWrapper, usage.prompt_tokens_details
).web_search_requests = web_search_requests
traffic_type = processed_chunk.get("usageMetadata", {}).get("trafficType")
if traffic_type:
model_response._hidden_params.setdefault(
"provider_specific_fields", {}
)["traffic_type"] = traffic_type
service_tier = self.response_headers.get("x-gemini-service-tier")
if service_tier:
if service_tier.lower() == "standard":
setattr(model_response, "service_tier", "default")
else:
setattr(model_response, "service_tier", service_tier.lower())
return usage
def chunk_parser(self, chunk: dict) -> Optional["ModelResponseStream"]:
try:
verbose_logger.debug(f"RAW GEMINI CHUNK: {chunk}")
@ -3043,101 +3181,23 @@ class ModelResponseIterator:
if blocked_response is not None:
model_response = blocked_response
usage: Optional[Usage] = None
_candidates: Optional[List[Candidates]] = processed_chunk.get("candidates")
grounding_metadata: List[dict] = []
url_context_metadata: List[dict] = []
safety_ratings: List[dict] = []
citation_metadata: List[dict] = []
_candidates: Optional[List[Candidates]] = processed_chunk.get("candidates")
if _candidates:
(
grounding_metadata,
url_context_metadata,
safety_ratings,
citation_metadata,
self.cumulative_tool_call_index,
) = VertexGeminiConfig._process_candidates(
_candidates,
model_response,
self.logging_obj.optional_params,
cumulative_tool_call_index=self.cumulative_tool_call_index,
)
) = self._apply_stream_candidates(_candidates, model_response)
# Track whether tool_calls have been seen across streaming chunks.
# Gemini sends tool_calls and finishReason in separate chunks,
# so we need to remember if earlier chunks contained tool_calls
# to correctly set finish_reason="tool_calls" per the OpenAI spec.
if not self.has_seen_tool_calls:
for choice in model_response.choices:
if (
hasattr(choice, "delta")
and choice.delta
and choice.delta.tool_calls
):
self.has_seen_tool_calls = True
break
# Handle final chunk with finishReason but no content.
# _process_candidates skips candidates without "content",
# so the finish_reason from the final chunk is lost.
if not model_response.choices and _candidates:
from litellm.types.utils import Delta, StreamingChoices
for candidate in _candidates:
finish_reason_str = candidate.get("finishReason")
if finish_reason_str is not None:
if self.has_seen_tool_calls:
mapped_finish_reason = "tool_calls"
else:
mapped_finish_reason = (
VertexGeminiConfig._check_finish_reason(
None, finish_reason_str
)
)
choice = StreamingChoices(
finish_reason=mapped_finish_reason,
index=candidate.get("index", 0),
delta=Delta(content=None, role=None),
logprobs=None,
enhancements=None,
)
model_response.choices.append(choice)
# Also handle the case where the final chunk has empty
# content (e.g. text:"") WITH finishReason. In this case
# _process_candidates DOES create a choice, but maps
# finishReason="STOP" to "stop" because the current chunk
# has no tool_calls. Override if we saw tool_calls earlier.
if self.has_seen_tool_calls:
for choice in model_response.choices:
if choice.finish_reason == "stop":
choice.finish_reason = "tool_calls"
setattr(model_response, "vertex_ai_grounding_metadata", grounding_metadata) # type: ignore
setattr(model_response, "vertex_ai_url_context_metadata", url_context_metadata) # type: ignore
setattr(model_response, "vertex_ai_safety_ratings", safety_ratings) # type: ignore
setattr(model_response, "vertex_ai_citation_metadata", citation_metadata) # type: ignore
if "usageMetadata" in processed_chunk:
usage = VertexGeminiConfig._calculate_usage(
completion_response=processed_chunk,
)
web_search_requests = VertexGeminiConfig._calculate_web_search_requests(
grounding_metadata
)
if web_search_requests is not None:
cast(
PromptTokensDetailsWrapper, usage.prompt_tokens_details
).web_search_requests = web_search_requests
traffic_type = processed_chunk.get("usageMetadata", {}).get(
"trafficType"
)
if traffic_type:
model_response._hidden_params.setdefault(
"provider_specific_fields", {}
)["traffic_type"] = traffic_type
usage = self._apply_stream_usage_metadata(
processed_chunk, model_response, grounding_metadata
)
setattr(model_response, "usage", usage) # type: ignore

16
litellm/llms/vertex_ai/vertex_llm_base.py
View file

@ -136,6 +136,11 @@ class VertexBase:
json_obj,
scopes=["https://www.googleapis.com/auth/cloud-platform"],
)
elif isinstance(credential_source, dict) and "executable" in credential_source:
creds = self._credentials_from_pluggable(
json_obj,
scopes=["https://www.googleapis.com/auth/cloud-platform"],
)
else:
creds = self._credentials_from_identity_pool(
json_obj,
@ -190,6 +195,17 @@ class VertexBase:
creds = creds.with_scopes(scopes)
return creds
def _credentials_from_pluggable(self, json_obj, scopes):
try:
from google.auth import pluggable
except ImportError:
raise ImportError(GOOGLE_IMPORT_ERROR_MESSAGE)
creds = pluggable.Credentials.from_info(json_obj)
if scopes and hasattr(creds, "requires_scopes") and creds.requires_scopes:
creds = creds.with_scopes(scopes)
return creds
def _credentials_from_identity_pool_with_aws(self, json_obj, scopes):
try:
from google.auth import aws

189
litellm/model_prices_and_context_window_backup.json
View file

@ -7818,26 +7818,6 @@
"cache_read_input_token_cost": 3e-08,
"cache_creation_input_token_cost": 3.75e-07
},
"bedrock/us-gov-east-1/claude-sonnet-4-5-20250929-v1:0": {
"input_cost_per_token": 3.6e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 200000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.8e-05,
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"cache_read_input_token_cost": 3.6e-07,
"cache_creation_input_token_cost": 4.5e-06
},
"bedrock/us-gov-east-1/anthropic.claude-sonnet-4-5-20250929-v1:0": {
"cache_creation_input_token_cost": 4.125e-06,
"cache_read_input_token_cost": 3.3e-07,
@ -7856,7 +7836,29 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"supports_native_structured_output": true
},
"bedrock/us-gov-east-1/claude-sonnet-4-5-20250929-v1:0": {
"cache_creation_input_token_cost": 4.125e-06,
"cache_read_input_token_cost": 3.3e-07,
"input_cost_per_token": 3.3e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 200000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.65e-05,
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_native_structured_output": true
},
"bedrock/us-gov-east-1/meta.llama3-70b-instruct-v1:0": {
"input_cost_per_token": 2.65e-06,
@ -7989,26 +7991,6 @@
"cache_read_input_token_cost": 3e-08,
"cache_creation_input_token_cost": 3.75e-07
},
"bedrock/us-gov-west-1/claude-sonnet-4-5-20250929-v1:0": {
"input_cost_per_token": 3.6e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 200000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.8e-05,
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"cache_read_input_token_cost": 3.6e-07,
"cache_creation_input_token_cost": 4.5e-06
},
"bedrock/us-gov-west-1/anthropic.claude-sonnet-4-5-20250929-v1:0": {
"cache_creation_input_token_cost": 4.125e-06,
"cache_read_input_token_cost": 3.3e-07,
@ -8027,7 +8009,29 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"supports_native_structured_output": true
},
"bedrock/us-gov-west-1/claude-sonnet-4-5-20250929-v1:0": {
"cache_creation_input_token_cost": 4.125e-06,
"cache_read_input_token_cost": 3.3e-07,
"input_cost_per_token": 3.3e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 200000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.65e-05,
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_native_structured_output": true
},
"bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0": {
"input_cost_per_token": 2.65e-06,
@ -13735,7 +13739,8 @@
"supports_tool_choice": true,
"supports_url_context": true,
"supports_vision": true,
"supports_web_search": true
"supports_web_search": true,
"supports_service_tier": true
},
"gemini-2.5-flash-image": {
"cache_read_input_token_cost": 3e-08,
@ -13784,7 +13789,8 @@
"supports_url_context": true,
"supports_vision": true,
"supports_web_search": false,
"tpm": 8000000
"tpm": 8000000,
"supports_service_tier": true
},
"gemini-3-pro-image-preview": {
"input_cost_per_image": 0.0011,
@ -13818,7 +13824,8 @@
"supports_response_schema": true,
"supports_system_messages": true,
"supports_vision": true,
"supports_web_search": true
"supports_web_search": true,
"supports_service_tier": true
},
"gemini-3.1-flash-image-preview": {
"input_cost_per_image": 0.00056,
@ -13901,7 +13908,8 @@
"supports_video_input": true,
"supports_vision": true,
"supports_web_search": true,
"supports_native_streaming": true
"supports_native_streaming": true,
"supports_service_tier": true
},
"deep-research-pro-preview-12-2025": {
"input_cost_per_image": 0.0011,
@ -13980,7 +13988,8 @@
"supports_tool_choice": true,
"supports_url_context": true,
"supports_vision": true,
"supports_web_search": true
"supports_web_search": true,
"supports_service_tier": true
},
"gemini-2.5-flash-lite-preview-09-2025": {
"cache_read_input_token_cost": 1e-08,
@ -14251,7 +14260,8 @@
"supports_tool_choice": true,
"supports_video_input": true,
"supports_vision": true,
"supports_web_search": true
"supports_web_search": true,
"supports_service_tier": true
},
"gemini-3-pro-preview": {
"deprecation_date": "2026-03-26",
@ -15033,7 +15043,8 @@
"supports_url_context": true,
"supports_vision": true,
"supports_web_search": true,
"tpm": 8000000
"tpm": 8000000,
"supports_service_tier": true
},
"gemini/gemini-2.5-flash-image": {
"cache_read_input_token_cost": 3e-08,
@ -15083,7 +15094,8 @@
"supports_url_context": true,
"supports_vision": true,
"supports_web_search": true,
"tpm": 8000000
"tpm": 8000000,
"supports_service_tier": true
},
"gemini/gemini-3-pro-image-preview": {
"input_cost_per_image": 0.0011,
@ -15119,7 +15131,8 @@
"supports_response_schema": true,
"supports_system_messages": true,
"supports_vision": true,
"supports_web_search": true
"supports_web_search": true,
"supports_service_tier": true
},
"gemini/gemini-3.1-flash-image-preview": {
"input_cost_per_token": 2.5e-07,
@ -15238,7 +15251,8 @@
"supports_url_context": true,
"supports_vision": true,
"supports_web_search": true,
"tpm": 250000
"tpm": 250000,
"supports_service_tier": true
},
"gemini/gemini-2.5-flash-lite-preview-09-2025": {
"cache_read_input_token_cost": 1e-08,
@ -15678,7 +15692,8 @@
"supports_vision": true,
"supports_web_search": true,
"supports_native_streaming": true,
"tpm": 250000
"tpm": 250000,
"supports_service_tier": true
},
"gemini/gemini-3-flash-preview": {
"cache_read_input_token_cost": 5e-08,
@ -16919,6 +16934,72 @@
"mode": "chat",
"output_cost_per_token": 1.2e-06
},
"baseten/MiniMaxAI/MiniMax-M2.5": {
"input_cost_per_token": 3e-07,
"litellm_provider": "baseten",
"mode": "chat",
"output_cost_per_token": 1.2e-06
},
"baseten/nvidia/Nemotron-120B-A12B": {
"input_cost_per_token": 3e-07,
"litellm_provider": "baseten",
"mode": "chat",
"output_cost_per_token": 7.5e-07
},
"baseten/zai-org/GLM-5": {
"input_cost_per_token": 9.5e-07,
"litellm_provider": "baseten",
"mode": "chat",
"output_cost_per_token": 3.15e-06
},
"baseten/zai-org/GLM-4.7": {
"input_cost_per_token": 6e-07,
"litellm_provider": "baseten",
"mode": "chat",
"output_cost_per_token": 2.2e-06
},
"baseten/zai-org/GLM-4.6": {
"input_cost_per_token": 6e-07,
"litellm_provider": "baseten",
"mode": "chat",
"output_cost_per_token": 2.2e-06
},
"baseten/moonshotai/Kimi-K2.5": {
"input_cost_per_token": 6e-07,
"litellm_provider": "baseten",
"mode": "chat",
"output_cost_per_token": 3e-06
},
"baseten/moonshotai/Kimi-K2-Thinking": {
"input_cost_per_token": 6e-07,
"litellm_provider": "baseten",
"mode": "chat",
"output_cost_per_token": 2.5e-06
},
"baseten/moonshotai/Kimi-K2-Instruct-0905": {
"input_cost_per_token": 6e-07,
"litellm_provider": "baseten",
"mode": "chat",
"output_cost_per_token": 2.5e-06
},
"baseten/openai/gpt-oss-120b": {
"input_cost_per_token": 1e-07,
"litellm_provider": "baseten",
"mode": "chat",
"output_cost_per_token": 5e-07
},
"baseten/deepseek-ai/DeepSeek-V3.1": {
"input_cost_per_token": 5e-07,
"litellm_provider": "baseten",
"mode": "chat",
"output_cost_per_token": 1.5e-06
},
"baseten/deepseek-ai/DeepSeek-V3-0324": {
"input_cost_per_token": 7.7e-07,
"litellm_provider": "baseten",
"mode": "chat",
"output_cost_per_token": 7.7e-07
},
"gmi/Qwen/Qwen3-VL-235B-A22B-Instruct-FP8": {
"input_cost_per_token": 3e-07,
"litellm_provider": "gmi",
@ -38123,4 +38204,4 @@
"supports_native_structured_output": true,
"supports_pdf_input": true
}
}
}

17
litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
View file

@ -10,6 +10,7 @@ import asyncio
import datetime
import hashlib
import json
import os
import re
from typing import Any, Callable, Dict, List, Literal, Optional, Set, Tuple, Union, cast
from urllib.parse import urlparse
@ -35,6 +36,8 @@ from litellm.constants import (
MCP_CLIENT_TIMEOUT,
MCP_HEALTH_CHECK_TIMEOUT,
MCP_METADATA_TIMEOUT,
MCP_NPM_CACHE_DIR,
MCP_STDIO_ALLOWED_COMMANDS,
MCP_TOOL_LISTING_TIMEOUT,
)
from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException
@ -1119,9 +1122,19 @@ class MCPServerManager:
# In containers the default (~/.npm or /app/.npm) may not exist
# or be read-only, causing npx to fail with ENOENT.
if "NPM_CONFIG_CACHE" not in resolved_env:
from litellm.constants import MCP_NPM_CACHE_DIR
resolved_env["NPM_CONFIG_CACHE"] = MCP_NPM_CACHE_DIR
# Defense-in-depth: block commands not in the allowlist.
# The Pydantic validator blocks new servers; this catches legacy
# config/DB records predating the allowlist.
if server.command:
base_command = os.path.basename(server.command)
if base_command not in MCP_STDIO_ALLOWED_COMMANDS:
raise HTTPException(
status_code=403,
detail=f"MCP stdio command '{server.command}' is not in the allowlist ({sorted(MCP_STDIO_ALLOWED_COMMANDS)}). "
f"Add it to LITELLM_MCP_STDIO_EXTRA_COMMANDS to allow this command.",
)
stdio_config: Optional[MCPStdioConfig] = None
if server.command and server.args is not None:
stdio_config = MCPStdioConfig(

21
litellm/proxy/_experimental/mcp_server/rest_endpoints.py
View file

@ -2,14 +2,14 @@ import importlib
from datetime import datetime
from typing import Any, Awaitable, Callable, Dict, List, Literal, Optional, Set, Union
from fastapi import APIRouter, Depends, HTTPException, Query, Request
from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
from litellm._logging import verbose_logger
from litellm.proxy._experimental.mcp_server.ui_session_utils import (
build_effective_auth_contexts,
)
from litellm.proxy._experimental.mcp_server.utils import merge_mcp_headers
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
from litellm.proxy.auth.ip_address_utils import IPAddressUtils
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils.http_parsing_utils import _safe_get_request_headers
@ -1027,6 +1027,13 @@ if MCP_AVAILABLE:
"""
Test if we can connect to the provided MCP server before adding it
"""
if LitellmUserRoles.PROXY_ADMIN != user_api_key_dict.user_role:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail={
"error": "User does not have permission to test MCP server connections. Only PROXY_ADMIN users can perform this action."
},
)
async def _test_connection_operation(client):
async def _noop(session):
@ -1041,7 +1048,7 @@ if MCP_AVAILABLE:
raw_headers=_safe_get_request_headers(request),
)
@router.post("/test/tools/list")
@router.post("/test/tools/list", dependencies=[Depends(user_api_key_auth)])
async def test_tools_list(
request: Request,
new_mcp_server_request: NewMCPServerRequest,
@ -1050,6 +1057,14 @@ if MCP_AVAILABLE:
"""
Preview tools available from MCP server before adding it
"""
if LitellmUserRoles.PROXY_ADMIN != user_api_key_dict.user_role:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail={
"error": "User does not have permission to test MCP server tools. Only PROXY_ADMIN users can perform this action."
},
)
# For OpenAPI spec servers, generate tools from the spec directly
if new_mcp_server_request.spec_path:
return await _preview_openapi_tools(new_mcp_server_request.spec_path)

17
litellm/proxy/_types.py
View file

@ -1,5 +1,6 @@
import enum
import json
import os
from datetime import datetime
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Literal, Optional, Union
@ -15,6 +16,7 @@ from pydantic import (
from typing_extensions import Required, TypedDict
from litellm._uuid import uuid
from litellm.constants import MCP_STDIO_ALLOWED_COMMANDS
from litellm.types.integrations.slack_alerting import AlertType
from litellm.types.llms.openai import (
AllMessageValues,
@ -1162,6 +1164,13 @@ class NewMCPServerRequest(LiteLLMPydanticObjectBase):
raise ValueError("command is required for stdio transport")
if not values.get("args"):
raise ValueError("args is required for stdio transport")
# Validate command against allowlist to prevent arbitrary execution
base_command = os.path.basename(values["command"])
if base_command not in MCP_STDIO_ALLOWED_COMMANDS:
raise ValueError(
f"Command '{values['command']}' is not in the allowed commands list "
f"for stdio transport. Allowed commands: {sorted(MCP_STDIO_ALLOWED_COMMANDS)}"
)
elif transport in [MCPTransport.http, MCPTransport.sse]:
if not values.get("url") and not values.get("spec_path"):
raise ValueError(
@ -1222,6 +1231,13 @@ class UpdateMCPServerRequest(LiteLLMPydanticObjectBase):
raise ValueError("command is required for stdio transport")
if not values.get("args"):
raise ValueError("args is required for stdio transport")
# Validate command against allowlist to prevent arbitrary execution
base_command = os.path.basename(values["command"])
if base_command not in MCP_STDIO_ALLOWED_COMMANDS:
raise ValueError(
f"Command '{values['command']}' is not in the allowed commands list "
f"for stdio transport. Allowed commands: {sorted(MCP_STDIO_ALLOWED_COMMANDS)}"
)
elif transport in [MCPTransport.http, MCPTransport.sse]:
if not values.get("url") and not values.get("spec_path"):
raise ValueError(
@ -2419,6 +2435,7 @@ class LiteLLM_VerificationTokenView(LiteLLM_VerificationToken):
end_user_model_max_budget: Optional[dict] = None
# Organization Params
organization_alias: Optional[str] = None
organization_max_budget: Optional[float] = None
organization_tpm_limit: Optional[int] = None
organization_rpm_limit: Optional[int] = None

182
litellm/proxy/litellm_pre_call_utils.py
View file

@ -10,6 +10,7 @@ from starlette.datastructures import Headers
import litellm
from litellm._logging import verbose_logger, verbose_proxy_logger
from litellm._service_logger import ServiceLogging
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
from litellm.proxy._types import (
AddTeamCallback,
@ -684,6 +685,7 @@ class LiteLLMProxyRequestSetup:
user_api_key_project_alias=user_api_key_dict.project_alias,
user_api_key_user_id=user_api_key_dict.user_id,
user_api_key_org_id=user_api_key_dict.org_id,
user_api_key_org_alias=user_api_key_dict.organization_alias,
user_api_key_team_alias=user_api_key_dict.team_alias,
user_api_key_end_user_id=user_api_key_dict.end_user_id,
user_api_key_user_email=user_api_key_dict.user_email,
@ -1263,6 +1265,9 @@ async def add_litellm_data_to_request( # noqa: PLR0915
user_api_key_dict=user_api_key_dict,
)
# Save pre-alias model name for credential override lookup
_pre_alias_model = data.get("model")
# Team Model Aliases
_update_model_if_team_alias_exists(
data=data,
@ -1279,6 +1284,14 @@ async def add_litellm_data_to_request( # noqa: PLR0915
"[PROXY] returned data from litellm_pre_call_utils: %s", data
)
# Team/Project credential overrides from model_config
# Placed after the debug log to avoid leaking credential secrets in logs
_apply_credential_overrides_from_model_config(
data=data,
user_api_key_dict=user_api_key_dict,
pre_alias_model_name=_pre_alias_model,
)
## ENFORCED PARAMS CHECK
# loop through each enforced param
# example enforced_params ['user', 'metadata', 'metadata.generation_name']
@ -1406,6 +1419,175 @@ def _update_model_if_key_alias_exists(
return
def _apply_credential_overrides_from_model_config(
data: dict,
user_api_key_dict: UserAPIKeyAuth,
pre_alias_model_name: Optional[str] = None,
) -> None:
"""
Walk the model_config precedence chain in team/project metadata.
If a matching credential is found, set api_base/api_key/api_version on data
so they override deployment defaults in the router.
Precedence (highest to lowest):
1. Clientside credentials (already in data — skip if present)
2. Project model-specific override
3. Project default override (defaultconfig)
4. Team model-specific override
5. Team default override (defaultconfig)
6. Deployment default (no action needed)
"""
# Feature flag gate — disabled by default, opt in with litellm.enable_model_config_credential_overrides = True
if not litellm.enable_model_config_credential_overrides:
return
# Respect clientside credentials — highest precedence
if data.get("api_base") is not None or data.get("api_key") is not None:
return
model_name = data.get("model")
if not model_name:
return
project_metadata = user_api_key_dict.project_metadata or {}
team_metadata = user_api_key_dict.team_metadata or {}
project_model_config = project_metadata.get("model_config")
team_model_config = team_metadata.get("model_config")
if not project_model_config and not team_model_config:
return
# Extract provider hint from model name (e.g. "azure/gpt-4" -> "azure")
provider: Optional[str] = None
if "/" in model_name:
provider = model_name.split("/", 1)[0]
credential_name = _resolve_credential_from_model_config(
model_name=model_name,
project_model_config=project_model_config,
team_model_config=team_model_config,
pre_alias_model_name=pre_alias_model_name,
provider=provider,
)
if not credential_name:
return
credential_values = CredentialAccessor.get_credential_values(credential_name)
if not credential_values:
_safe_cred = str(credential_name).replace("\n", "").replace("\r", "")
verbose_proxy_logger.warning(
"model_config references credential '%s' but it was not found or has no values",
_safe_cred,
)
return
# Apply credential overrides only for keys not already in the request
for key in ("api_base", "api_key", "api_version"):
if key in credential_values and key not in data:
data[key] = credential_values[key]
_safe_model = str(model_name).replace("\n", "").replace("\r", "")
_safe_cred = str(credential_name).replace("\n", "").replace("\r", "")
verbose_proxy_logger.debug(
"Applied credential override '%s' for model '%s'",
_safe_cred,
_safe_model,
)
def _resolve_credential_from_model_config(
model_name: str,
project_model_config: Optional[dict],
team_model_config: Optional[dict],
pre_alias_model_name: Optional[str] = None,
provider: Optional[str] = None,
) -> Optional[str]:
"""
Walk the precedence chain and return the first matching credential name.
Checks (in order):
1. project_model_config[model_name][provider] — project model-specific
2. project_model_config[pre_alias_model_name][provider] — project pre-alias
3. project_model_config["defaultconfig"][provider] — project default
4. team_model_config[model_name][provider] — team model-specific
5. team_model_config[pre_alias_model_name][provider] — team pre-alias
6. team_model_config["defaultconfig"][provider] — team default
When a model-specific entry exists but contains no litellm_credentials,
the function falls through to defaultconfig. This is intentional —
an entry without litellm_credentials is treated as incomplete config,
not as an explicit "no override" signal.
"""
# Build the list of model names to try (post-alias first, then pre-alias)
model_names_to_try = [model_name]
if pre_alias_model_name and pre_alias_model_name != model_name:
model_names_to_try.append(pre_alias_model_name)
for model_config in (project_model_config, team_model_config):
if not model_config or not isinstance(model_config, dict):
continue
# Model-specific check (try resolved name, then pre-alias name)
for name in model_names_to_try:
model_entry = model_config.get(name)
if model_entry:
credential_name = _extract_credential_from_entry(
model_entry, provider=provider
)
if credential_name:
return credential_name
_safe_name = str(name).replace("\n", "").replace("\r", "")
verbose_proxy_logger.debug(
"model_config entry '%s' found but has no litellm_credentials, "
"trying next candidate",
_safe_name,
)
# Default check
default_entry = model_config.get("defaultconfig")
if default_entry:
credential_name = _extract_credential_from_entry(
default_entry, provider=provider
)
if credential_name:
return credential_name
return None
def _extract_credential_from_entry(
entry: dict, provider: Optional[str] = None
) -> Optional[str]:
"""
Extract litellm_credentials from a model_config entry.
Entry structure: {"azure": {"litellm_credentials": "name"}, ...}
When provider is given (e.g. "azure"), tries an exact provider match first.
Falls back to the first credential found across all provider keys.
"""
if not isinstance(entry, dict):
return None
# Prefer exact provider match when provider hint is available
if provider and provider in entry:
provider_config = entry[provider]
if isinstance(provider_config, dict):
credential_name = provider_config.get("litellm_credentials")
if credential_name:
return credential_name
# Fall back to first available provider
for provider_config in entry.values():
if isinstance(provider_config, dict):
credential_name = provider_config.get("litellm_credentials")
if credential_name:
return credential_name
return None
def _get_enforced_params(
general_settings: Optional[dict], user_api_key_dict: UserAPIKeyAuth
) -> Optional[list]:

5
litellm/proxy/openai_files_endpoints/files_endpoints.py
View file

@ -1115,7 +1115,10 @@ async def delete_file(
file_id=original_file_id,
)
response = await litellm.afile_delete(**data) # type: ignore
response = await litellm.afile_delete(
custom_llm_provider=credentials["custom_llm_provider"], # type: ignore
**data,
) # type: ignore
verbose_proxy_logger.debug(
f"Deleted file using model: {model_used}"

21
litellm/proxy/proxy_server.py
View file

@ -7128,6 +7128,13 @@ async def chat_completion( # noqa: PLR0915
and user_api_key_dict.org_id is not None
):
data["metadata"]["user_api_key_org_id"] = user_api_key_dict.org_id
if (
hasattr(user_api_key_dict, "organization_alias")
and user_api_key_dict.organization_alias is not None
):
data["metadata"]["user_api_key_org_alias"] = (
user_api_key_dict.organization_alias
)
if (
hasattr(user_api_key_dict, "agent_id")
and user_api_key_dict.agent_id is not None
@ -7302,6 +7309,13 @@ async def completion( # noqa: PLR0915
and user_api_key_dict.org_id is not None
):
data["metadata"]["user_api_key_org_id"] = user_api_key_dict.org_id
if (
hasattr(user_api_key_dict, "organization_alias")
and user_api_key_dict.organization_alias is not None
):
data["metadata"]["user_api_key_org_alias"] = (
user_api_key_dict.organization_alias
)
if (
hasattr(user_api_key_dict, "agent_id")
and user_api_key_dict.agent_id is not None
@ -7544,6 +7558,13 @@ async def embeddings( # noqa: PLR0915
and user_api_key_dict.org_id is not None
):
data["metadata"]["user_api_key_org_id"] = user_api_key_dict.org_id
if (
hasattr(user_api_key_dict, "organization_alias")
and user_api_key_dict.organization_alias is not None
):
data["metadata"]["user_api_key_org_alias"] = (
user_api_key_dict.organization_alias
)
if (
hasattr(user_api_key_dict, "agent_id")
and user_api_key_dict.agent_id is not None

6
litellm/proxy/utils.py
View file

@ -3004,6 +3004,7 @@ class PrismaClient:
b.model_max_budget as litellm_budget_table_model_max_budget,
b.soft_budget as litellm_budget_table_soft_budget,
o.metadata as organization_metadata,
o.organization_alias as organization_alias,
b2.max_budget as organization_max_budget,
b2.tpm_limit as organization_tpm_limit,
b2.rpm_limit as organization_rpm_limit
@ -5293,11 +5294,12 @@ def handle_exception_on_proxy(e: Exception) -> ProxyException:
)
elif isinstance(e, ProxyException):
return e
_status_code = getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR)
return ProxyException(
message="Internal Server Error, " + str(e),
message=str(e),
type=ProxyErrorTypes.internal_server_error,
param=getattr(e, "param", "None"),
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
code=_status_code,
)

70
litellm/router.py
View file

@ -3864,14 +3864,29 @@ class Router:
self._add_deployment_model_to_endpoint_for_llm_passthrough_route(
kwargs=kwargs, model=model, model_name=model_name
)
### get custom
response = original_generic_function(
**{
**data,
"caching": self.cache_responses,
**kwargs,
}
)
# Get custom_llm_provider from deployment params
try:
custom_llm_provider = data.get("custom_llm_provider")
_, inferred_custom_llm_provider, _, _ = get_llm_provider(
model=data["model"],
custom_llm_provider=custom_llm_provider,
)
custom_llm_provider = custom_llm_provider or inferred_custom_llm_provider
except Exception:
custom_llm_provider = None
# Build response kwargs
response_kwargs = {
**data,
"caching": self.cache_responses,
**kwargs,
}
# Only set custom_llm_provider if it's not None
if custom_llm_provider is not None:
response_kwargs["custom_llm_provider"] = custom_llm_provider
response = original_generic_function(**response_kwargs)
rpm_semaphore = self._get_client(
deployment=deployment,
@ -3961,7 +3976,12 @@ class Router:
self.routing_strategy_pre_call_checks(deployment=deployment)
try:
_, custom_llm_provider, _, _ = get_llm_provider(model=data["model"])
custom_llm_provider = data.get("custom_llm_provider")
_, inferred_custom_llm_provider, _, _ = get_llm_provider(
model=data["model"],
custom_llm_provider=custom_llm_provider,
)
custom_llm_provider = custom_llm_provider or inferred_custom_llm_provider
except Exception:
custom_llm_provider = None
@ -4219,9 +4239,14 @@ class Router:
self.total_calls[model_name] += 1
## REPLACE MODEL IN FILE WITH SELECTED DEPLOYMENT ##
stripped_model, custom_llm_provider, _, _ = get_llm_provider(
model=data["model"]
# For DB/config deployments, use provider from deployment params
custom_llm_provider = data.get("custom_llm_provider")
stripped_model, inferred_custom_llm_provider, _, _ = get_llm_provider(
model=data["model"],
custom_llm_provider=custom_llm_provider,
)
# Preserve explicitly stored provider, fallback to inferred
custom_llm_provider = custom_llm_provider or inferred_custom_llm_provider
## REPLACE MODEL IN FILE WITH SELECTED DEPLOYMENT ##
purpose = cast(Optional[OpenAIFilesPurpose], kwargs.get("purpose"))
@ -4367,8 +4392,13 @@ class Router:
)
self.total_calls[model_name] += 1
# Get custom provider
_, custom_llm_provider, _, _ = get_llm_provider(model=data["model"])
# Get custom provider from deployment params
custom_llm_provider = data.get("custom_llm_provider")
_, inferred_custom_llm_provider, _, _ = get_llm_provider(
model=data["model"],
custom_llm_provider=custom_llm_provider,
)
custom_llm_provider = custom_llm_provider or inferred_custom_llm_provider
response = avector_store_create_sdk(
**{
@ -4486,7 +4516,12 @@ class Router:
self.total_calls[model_name] += 1
## SET CUSTOM PROVIDER TO SELECTED DEPLOYMENT ##
_, custom_llm_provider, _, _ = get_llm_provider(model=data["model"])
custom_llm_provider = data.get("custom_llm_provider")
_, inferred_custom_llm_provider, _, _ = get_llm_provider(
model=data["model"],
custom_llm_provider=custom_llm_provider,
)
custom_llm_provider = custom_llm_provider or inferred_custom_llm_provider
response = litellm.acreate_batch(
**{
@ -4720,7 +4755,12 @@ class Router:
self.total_calls[model_name] += 1
## SET CUSTOM PROVIDER TO SELECTED DEPLOYMENT ##
_, custom_llm_provider, _, _ = get_llm_provider(model=data["model"])
custom_llm_provider = data.get("custom_llm_provider")
_, inferred_custom_llm_provider, _, _ = get_llm_provider(
model=data["model"],
custom_llm_provider=custom_llm_provider,
)
custom_llm_provider = custom_llm_provider or inferred_custom_llm_provider
response = litellm.acancel_batch(
**{

13
litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py
View file

@ -139,9 +139,16 @@ class EncryptedContentAffinityCheck(CustomLogger):
typed_healthy_deployments = cast(List[dict], healthy_deployments)
# Signal to the response post-processor that encrypted item IDs should be
# encoded in the output of this request.
litellm_metadata = request_kwargs.setdefault("litellm_metadata", {})
litellm_metadata["encrypted_content_affinity_enabled"] = True
# encoded in the output of this request. Only set the flag when
# litellm_metadata already exists (Responses API path). Using
# setdefault would create an empty litellm_metadata dict for chat
# completions / embeddings, which breaks tag-based routing because
# _get_metadata_variable_name_from_kwargs would pick "litellm_metadata"
# over "metadata" where tags are actually stored.
if "litellm_metadata" in request_kwargs:
request_kwargs["litellm_metadata"][
"encrypted_content_affinity_enabled"
] = True
request_input = request_kwargs.get("input")
model_id = self._extract_model_id_from_input(request_input)

28
litellm/types/integrations/prometheus.py
View file

@ -1,7 +1,7 @@
import re
from dataclasses import dataclass
from enum import Enum
from typing import Any, Dict, List, Literal, Optional, Tuple
from typing import Any, ClassVar, Dict, List, Literal, Optional, Tuple
from pydantic import BaseModel, Field, field_validator
from typing_extensions import Annotated
@ -665,6 +665,24 @@ class PrometheusMetricLabels:
litellm_cache_misses_metric = _cache_metric_labels
litellm_cached_tokens_metric = _cache_metric_labels
# Metrics whose emission paths supply org context (used by get_labels)
_org_label_metrics: ClassVar[frozenset] = frozenset(
{
"litellm_llm_api_latency_metric",
"litellm_llm_api_time_to_first_token_metric",
"litellm_request_total_latency_metric",
"litellm_request_queue_time_seconds",
"litellm_proxy_total_requests_metric",
"litellm_proxy_failed_requests_metric",
"litellm_deployment_latency_per_output_token",
"litellm_requests_metric",
"litellm_spend_metric",
"litellm_input_tokens_metric",
"litellm_total_tokens_metric",
"litellm_output_tokens_metric",
}
)
# Managed batch metrics
_batch_user_labels = [
UserAPIKeyLabelNames.v1_LITELLM_MODEL_NAME.value,
@ -731,6 +749,14 @@ class PrometheusMetricLabels:
):
custom_labels.append(UserAPIKeyLabelNames.STREAM.value)
if label_name in PrometheusMetricLabels._org_label_metrics:
for label in [
UserAPIKeyLabelNames.ORG_ID.value,
UserAPIKeyLabelNames.ORG_ALIAS.value,
]:
if label not in default_labels and label not in custom_labels:
custom_labels.append(label)
return default_labels + custom_labels

1
litellm/types/llms/vertex_ai.py
View file

@ -325,6 +325,7 @@ class RequestBody(TypedDict, total=False):
generationConfig: GenerationConfig
cachedContent: str
labels: Dict[str, str]
serviceTier: str
class CachedContentRequestBody(TypedDict, total=False):

1
litellm/types/utils.py
View file

@ -2507,6 +2507,7 @@ class StandardLoggingUserAPIKeyMetadata(TypedDict):
user_api_key_max_budget: Optional[float]
user_api_key_budget_reset_at: Optional[str]
user_api_key_org_id: Optional[str]
user_api_key_org_alias: Optional[str]
user_api_key_team_id: Optional[str]
user_api_key_project_id: Optional[str]
user_api_key_project_alias: Optional[str]

189
model_prices_and_context_window.json
View file

@ -7818,26 +7818,6 @@
"cache_read_input_token_cost": 3e-08,
"cache_creation_input_token_cost": 3.75e-07
},
"bedrock/us-gov-east-1/claude-sonnet-4-5-20250929-v1:0": {
"input_cost_per_token": 3.6e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 200000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.8e-05,
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"cache_read_input_token_cost": 3.6e-07,
"cache_creation_input_token_cost": 4.5e-06
},
"bedrock/us-gov-east-1/anthropic.claude-sonnet-4-5-20250929-v1:0": {
"cache_creation_input_token_cost": 4.125e-06,
"cache_read_input_token_cost": 3.3e-07,
@ -7856,7 +7836,29 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"supports_native_structured_output": true
},
"bedrock/us-gov-east-1/claude-sonnet-4-5-20250929-v1:0": {
"cache_creation_input_token_cost": 4.125e-06,
"cache_read_input_token_cost": 3.3e-07,
"input_cost_per_token": 3.3e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 200000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.65e-05,
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_native_structured_output": true
},
"bedrock/us-gov-east-1/meta.llama3-70b-instruct-v1:0": {
"input_cost_per_token": 2.65e-06,
@ -7989,26 +7991,6 @@
"cache_read_input_token_cost": 3e-08,
"cache_creation_input_token_cost": 3.75e-07
},
"bedrock/us-gov-west-1/claude-sonnet-4-5-20250929-v1:0": {
"input_cost_per_token": 3.6e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 200000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.8e-05,
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"cache_read_input_token_cost": 3.6e-07,
"cache_creation_input_token_cost": 4.5e-06
},
"bedrock/us-gov-west-1/anthropic.claude-sonnet-4-5-20250929-v1:0": {
"cache_creation_input_token_cost": 4.125e-06,
"cache_read_input_token_cost": 3.3e-07,
@ -8027,7 +8009,29 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"supports_native_structured_output": true
},
"bedrock/us-gov-west-1/claude-sonnet-4-5-20250929-v1:0": {
"cache_creation_input_token_cost": 4.125e-06,
"cache_read_input_token_cost": 3.3e-07,
"input_cost_per_token": 3.3e-06,
"litellm_provider": "bedrock",
"max_input_tokens": 200000,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.65e-05,
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_native_structured_output": true
},
"bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0": {
"input_cost_per_token": 2.65e-06,
@ -13735,7 +13739,8 @@
"supports_tool_choice": true,
"supports_url_context": true,
"supports_vision": true,
"supports_web_search": true
"supports_web_search": true,
"supports_service_tier": true
},
"gemini-2.5-flash-image": {
"cache_read_input_token_cost": 3e-08,
@ -13784,7 +13789,8 @@
"supports_url_context": true,
"supports_vision": true,
"supports_web_search": false,
"tpm": 8000000
"tpm": 8000000,
"supports_service_tier": true
},
"gemini-3-pro-image-preview": {
"input_cost_per_image": 0.0011,
@ -13818,7 +13824,8 @@
"supports_response_schema": true,
"supports_system_messages": true,
"supports_vision": true,
"supports_web_search": true
"supports_web_search": true,
"supports_service_tier": true
},
"gemini-3.1-flash-image-preview": {
"input_cost_per_image": 0.00056,
@ -13901,7 +13908,8 @@
"supports_video_input": true,
"supports_vision": true,
"supports_web_search": true,
"supports_native_streaming": true
"supports_native_streaming": true,
"supports_service_tier": true
},
"deep-research-pro-preview-12-2025": {
"input_cost_per_image": 0.0011,
@ -13980,7 +13988,8 @@
"supports_tool_choice": true,
"supports_url_context": true,
"supports_vision": true,
"supports_web_search": true
"supports_web_search": true,
"supports_service_tier": true
},
"gemini-2.5-flash-lite-preview-09-2025": {
"cache_read_input_token_cost": 1e-08,
@ -14251,7 +14260,8 @@
"supports_tool_choice": true,
"supports_video_input": true,
"supports_vision": true,
"supports_web_search": true
"supports_web_search": true,
"supports_service_tier": true
},
"gemini-3-pro-preview": {
"deprecation_date": "2026-03-26",
@ -15033,7 +15043,8 @@
"supports_url_context": true,
"supports_vision": true,
"supports_web_search": true,
"tpm": 8000000
"tpm": 8000000,
"supports_service_tier": true
},
"gemini/gemini-2.5-flash-image": {
"cache_read_input_token_cost": 3e-08,
@ -15083,7 +15094,8 @@
"supports_url_context": true,
"supports_vision": true,
"supports_web_search": true,
"tpm": 8000000
"tpm": 8000000,
"supports_service_tier": true
},
"gemini/gemini-3-pro-image-preview": {
"input_cost_per_image": 0.0011,
@ -15119,7 +15131,8 @@
"supports_response_schema": true,
"supports_system_messages": true,
"supports_vision": true,
"supports_web_search": true
"supports_web_search": true,
"supports_service_tier": true
},
"gemini/gemini-3.1-flash-image-preview": {
"input_cost_per_token": 2.5e-07,
@ -15238,7 +15251,8 @@
"supports_url_context": true,
"supports_vision": true,
"supports_web_search": true,
"tpm": 250000
"tpm": 250000,
"supports_service_tier": true
},
"gemini/gemini-2.5-flash-lite-preview-09-2025": {
"cache_read_input_token_cost": 1e-08,
@ -15678,7 +15692,8 @@
"supports_vision": true,
"supports_web_search": true,
"supports_native_streaming": true,
"tpm": 250000
"tpm": 250000,
"supports_service_tier": true
},
"gemini/gemini-3-flash-preview": {
"cache_read_input_token_cost": 5e-08,
@ -16919,6 +16934,72 @@
"mode": "chat",
"output_cost_per_token": 1.2e-06
},
"baseten/MiniMaxAI/MiniMax-M2.5": {
"input_cost_per_token": 3e-07,
"litellm_provider": "baseten",
"mode": "chat",
"output_cost_per_token": 1.2e-06
},
"baseten/nvidia/Nemotron-120B-A12B": {
"input_cost_per_token": 3e-07,
"litellm_provider": "baseten",
"mode": "chat",
"output_cost_per_token": 7.5e-07
},
"baseten/zai-org/GLM-5": {
"input_cost_per_token": 9.5e-07,
"litellm_provider": "baseten",
"mode": "chat",
"output_cost_per_token": 3.15e-06
},
"baseten/zai-org/GLM-4.7": {
"input_cost_per_token": 6e-07,
"litellm_provider": "baseten",
"mode": "chat",
"output_cost_per_token": 2.2e-06
},
"baseten/zai-org/GLM-4.6": {
"input_cost_per_token": 6e-07,
"litellm_provider": "baseten",
"mode": "chat",
"output_cost_per_token": 2.2e-06
},
"baseten/moonshotai/Kimi-K2.5": {
"input_cost_per_token": 6e-07,
"litellm_provider": "baseten",
"mode": "chat",
"output_cost_per_token": 3e-06
},
"baseten/moonshotai/Kimi-K2-Thinking": {
"input_cost_per_token": 6e-07,
"litellm_provider": "baseten",
"mode": "chat",
"output_cost_per_token": 2.5e-06
},
"baseten/moonshotai/Kimi-K2-Instruct-0905": {
"input_cost_per_token": 6e-07,
"litellm_provider": "baseten",
"mode": "chat",
"output_cost_per_token": 2.5e-06
},
"baseten/openai/gpt-oss-120b": {
"input_cost_per_token": 1e-07,
"litellm_provider": "baseten",
"mode": "chat",
"output_cost_per_token": 5e-07
},
"baseten/deepseek-ai/DeepSeek-V3.1": {
"input_cost_per_token": 5e-07,
"litellm_provider": "baseten",
"mode": "chat",
"output_cost_per_token": 1.5e-06
},
"baseten/deepseek-ai/DeepSeek-V3-0324": {
"input_cost_per_token": 7.7e-07,
"litellm_provider": "baseten",
"mode": "chat",
"output_cost_per_token": 7.7e-07
},
"gmi/Qwen/Qwen3-VL-235B-A22B-Instruct-FP8": {
"input_cost_per_token": 3e-07,
"litellm_provider": "gmi",
@ -38108,4 +38189,4 @@
"supports_native_structured_output": true,
"supports_pdf_input": true
}
}
}

22
tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py
View file

@ -223,6 +223,8 @@ def test_increment_token_metrics(prometheus_logger):
api_key_alias="test_alias",
team="test_team",
team_alias="test_team_alias",
org_id=None,
org_alias=None,
requested_model=None,
model="gpt-3.5-turbo",
model_id="model-123",
@ -237,6 +239,8 @@ def test_increment_token_metrics(prometheus_logger):
api_key_alias="test_alias",
team="test_team",
team_alias="test_team_alias",
org_id=None,
org_alias=None,
requested_model=None,
model="gpt-3.5-turbo",
model_id="model-123",
@ -253,6 +257,8 @@ def test_increment_token_metrics(prometheus_logger):
api_key_alias="test_alias",
team="test_team",
team_alias="test_team_alias",
org_id=None,
org_alias=None,
requested_model=None,
model="gpt-3.5-turbo",
model_id="model-123",
@ -414,6 +420,8 @@ def test_set_latency_metrics(prometheus_logger):
api_key_alias="test_alias",
team="test_team",
team_alias="test_team_alias",
org_id=None,
org_alias=None,
requested_model="openai-gpt",
model="gpt-3.5-turbo",
model_id="model-123",
@ -430,6 +438,8 @@ def test_set_latency_metrics(prometheus_logger):
api_key_alias="test_alias",
team="test_team",
team_alias="test_team_alias",
org_id=None,
org_alias=None,
requested_model="openai-gpt",
model="gpt-3.5-turbo",
model_id="model-123",
@ -446,6 +456,8 @@ def test_set_latency_metrics(prometheus_logger):
api_key_alias="test_alias",
team="test_team",
team_alias="test_team_alias",
org_id=None,
org_alias=None,
requested_model="openai-gpt",
model="gpt-3.5-turbo",
model_id="model-123",
@ -589,6 +601,8 @@ def test_increment_top_level_request_and_spend_metrics(prometheus_logger):
api_key_alias="test_alias",
team="test_team",
team_alias="test_team_alias",
org_id=None,
org_alias=None,
model="gpt-3.5-turbo",
model_id="model-123",
client_ip=None,
@ -605,6 +619,8 @@ def test_increment_top_level_request_and_spend_metrics(prometheus_logger):
api_key_alias="test_alias",
team="test_team",
team_alias="test_team_alias",
org_id=None,
org_alias=None,
model="gpt-3.5-turbo",
model_id="model-123",
client_ip=None,
@ -758,6 +774,8 @@ async def test_async_post_call_failure_hook(prometheus_logger):
api_key_alias="test_alias",
team="test_team",
team_alias="test_team_alias",
org_id=None,
org_alias=None,
requested_model="gpt-3.5-turbo",
exception_status="429",
exception_class="Openai.RateLimitError",
@ -776,6 +794,8 @@ async def test_async_post_call_failure_hook(prometheus_logger):
requested_model="gpt-3.5-turbo",
team="test_team",
team_alias="test_team_alias",
org_id=None,
org_alias=None,
user="test_user",
status_code="429",
user_email=None,
@ -955,6 +975,8 @@ def test_set_llm_deployment_success_metrics(prometheus_logger):
api_key_alias=standard_logging_payload["metadata"]["user_api_key_alias"],
team=standard_logging_payload["metadata"]["user_api_key_team_id"],
team_alias=standard_logging_payload["metadata"]["user_api_key_team_alias"],
org_id=None,
org_alias=None,
)
prometheus_logger.litellm_overhead_latency_metric.labels.assert_called_once_with(
api_base="https://api.openai.com",

132
tests/llm_translation/test_triton.py
View file

@ -50,6 +50,138 @@ def test_split_embedding_by_shape_fails_with_shape_value_error():
)
def test_triton_embedding_response_sets_usage_with_token_counter():
config = TritonEmbeddingConfig()
mock_http_response = MagicMock()
mock_http_response.status_code = 200
mock_http_response.json.return_value = {
"model_name": "gte-base-en-v1",
"outputs": [
{
"name": "embedding",
"shape": [1, 2],
"data": [0.1, 0.2],
}
],
}
model_response = litellm.EmbeddingResponse()
request_data = {
"inputs": [
{
"name": "input_text",
"shape": [1],
"datatype": "BYTES",
"data": ["hello from triton"],
}
]
}
with patch(
"litellm.llms.triton.embedding.transformation.token_counter",
return_value=7,
):
transformed = config.transform_embedding_response(
model="triton/gte-base-en-v1",
raw_response=mock_http_response,
model_response=model_response,
logging_obj=MagicMock(),
request_data=request_data,
)
assert transformed.usage is not None
assert transformed.usage.prompt_tokens == 7
assert transformed.usage.completion_tokens == 0
assert transformed.usage.total_tokens == 7
def test_triton_embedding_response_sets_usage_with_word_count_fallback():
config = TritonEmbeddingConfig()
mock_http_response = MagicMock()
mock_http_response.status_code = 200
mock_http_response.json.return_value = {
"model_name": "gte-base-en-v1",
"outputs": [
{
"name": "embedding",
"shape": [1, 2],
"data": [0.1, 0.2],
}
],
}
model_response = litellm.EmbeddingResponse()
request_data = {
"inputs": [
{
"name": "input_text",
"shape": [1],
"datatype": "BYTES",
"data": ["hello from triton"],
}
]
}
with patch(
"litellm.llms.triton.embedding.transformation.token_counter",
side_effect=Exception("tokenizer error"),
):
transformed = config.transform_embedding_response(
model="triton/gte-base-en-v1",
raw_response=mock_http_response,
model_response=model_response,
logging_obj=MagicMock(),
request_data=request_data,
)
assert transformed.usage is not None
assert transformed.usage.prompt_tokens == 3
assert transformed.usage.completion_tokens == 0
assert transformed.usage.total_tokens == 3
def test_triton_embedding_batch_usage_sums_per_input_token_counts():
"""Batch inputs must not be joined before token counting (avoids extra newline tokens)."""
config = TritonEmbeddingConfig()
mock_http_response = MagicMock()
mock_http_response.status_code = 200
mock_http_response.json.return_value = {
"model_name": "gte-base-en-v1",
"outputs": [
{
"name": "embedding",
"shape": [2, 2],
"data": [0.1, 0.2, 0.3, 0.4],
}
],
}
model_response = litellm.EmbeddingResponse()
request_data = {
"inputs": [
{
"name": "input_text",
"shape": [2],
"datatype": "BYTES",
"data": ["first input", "second input"],
}
]
}
with patch(
"litellm.llms.triton.embedding.transformation.token_counter",
side_effect=[5, 7],
):
transformed = config.transform_embedding_response(
model="triton/gte-base-en-v1",
raw_response=mock_http_response,
model_response=model_response,
logging_obj=MagicMock(),
request_data=request_data,
)
assert transformed.usage is not None
assert transformed.usage.prompt_tokens == 12
assert transformed.usage.total_tokens == 12
@pytest.mark.parametrize("stream", [True, False])
def test_completion_triton_generate_api(stream):
try:

47
tests/proxy_unit_tests/test_proxy_utils.py
View file

@ -2637,3 +2637,50 @@ async def test_handle_logging_proxy_only_error_skips_handlers_for_pass_through()
mock_async.assert_not_called()
mock_sync.assert_not_called()
assert logging_obj.call_type == CallTypes.pass_through.value
def test_handle_exception_on_proxy_preserves_status_code():
"""
OpenAI batch creation returns 429 for rate limits. LiteLLM wraps this as a
RateLimitError with status_code=429. handle_exception_on_proxy must pass
that status code through instead of hardcoding 500.
"""
from litellm.proxy.utils import handle_exception_on_proxy
rate_limit_error = litellm.RateLimitError(
message="Rate limit exceeded: batch creation limit of 2000/hour hit",
llm_provider="openai",
model="gpt-4o",
)
result = handle_exception_on_proxy(rate_limit_error)
assert int(result.code) == 429, f"Expected 429, got {result.code}"
def test_handle_exception_on_proxy_defaults_to_500_for_unknown_exceptions():
"""
Generic exceptions with no status_code should still return 500.
"""
from litellm.proxy.utils import handle_exception_on_proxy
result = handle_exception_on_proxy(Exception("something went wrong"))
assert int(result.code) == 500, f"Expected 500, got {result.code}"
def test_handle_exception_on_proxy_preserves_auth_error_status_code():
"""
AuthenticationError (401) should also pass through correctly.
"""
from litellm.proxy.utils import handle_exception_on_proxy
auth_error = litellm.AuthenticationError(
message="Invalid API key",
llm_provider="openai",
model="gpt-4o",
)
result = handle_exception_on_proxy(auth_error)
assert int(result.code) == 401, f"Expected 401, got {result.code}"

2
tests/test_litellm/integrations/test_prometheus_client_ip_user_agent.py
View file

@ -118,6 +118,8 @@ async def test_async_post_call_success_hook_includes_client_ip_user_agent():
"user_api_key_alias": "alias_1",
"user_api_key_team_id": "team_1",
"user_api_key_team_alias": "team_alias_1",
"user_api_key_org_id": None,
"user_api_key_org_alias": None,
"user_api_key_user_email": "test@example.com",
"user_api_key_request_route": "/chat/completions",
"requester_ip_address": "192.168.1.1",

55
tests/test_litellm/integrations/test_prometheus_user_team_metrics.py
View file

@ -525,6 +525,61 @@ async def test_set_user_budget_metrics_after_api_request_inf_when_genuinely_no_b
)
def test_per_request_metrics_emit_all_identity_labels(prometheus_logger):
"""Verify org labels appear when flag is on and are absent when flag is off."""
import litellm
from litellm.types.integrations.prometheus import UserAPIKeyLabelValues
prometheus_logger.litellm_requests_metric = MagicMock()
prometheus_logger.litellm_spend_metric = MagicMock()
enum_values = UserAPIKeyLabelValues(
hashed_api_key="hashed-key",
api_key_alias="my-key",
model="gpt-4",
team="team-abc",
team_alias="my-team",
org_id="org-abc",
org_alias="my-org",
user="user-1",
)
common_kwargs = dict(
end_user_id=None,
user_api_key="hashed-key",
user_api_key_alias="my-key",
model="gpt-4",
user_api_team="team-abc",
user_api_team_alias="my-team",
user_id="user-1",
response_cost=0.001,
enum_values=enum_values,
)
try:
# org labels are always included in per-request metrics
prometheus_logger._increment_top_level_request_and_spend_metrics(**common_kwargs)
label_kwargs = prometheus_logger.litellm_requests_metric.labels.call_args.kwargs
assert label_kwargs["org_id"] == "org-abc"
assert label_kwargs["org_alias"] == "my-org"
assert label_kwargs["team"] == "team-abc"
assert label_kwargs["user"] == "user-1"
# Metrics not in the org-emission list must NOT get org labels
from litellm.types.integrations.prometheus import PrometheusMetricLabels
for metric in ("litellm_remaining_api_key_budget_metric", "litellm_remaining_team_budget_metric"):
labels = PrometheusMetricLabels.get_labels(metric)
assert "org_id" not in labels, f"{metric} should not have org_id"
assert "org_alias" not in labels, f"{metric} should not have org_alias"
# org_id in custom_prometheus_metadata_labels must not produce duplicate labels
litellm.custom_prometheus_metadata_labels = ["org_id"]
labels = PrometheusMetricLabels.get_labels("litellm_requests_metric")
assert labels.count("org_id") == 1
finally:
litellm.custom_prometheus_metadata_labels = []
# ---------------------------------------------------------------------------
# Org budget metric tests
# ---------------------------------------------------------------------------

33
tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py
View file

@ -1,7 +1,9 @@
from unittest.mock import MagicMock
from unittest.mock import AsyncMock, MagicMock
import pytest
from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
from litellm.llms.anthropic.chat.handler import ModelResponseIterator
from litellm.llms.anthropic.chat.handler import ModelResponseIterator, make_call
from litellm.types.llms.openai import (
ChatCompletionToolCallChunk,
ChatCompletionToolCallFunctionChunk,
@ -9,6 +11,33 @@ from litellm.types.llms.openai import (
from litellm.types.responses.main import OutputCodeInterpreterCall
@pytest.mark.asyncio
async def test_make_call_passes_logging_obj_to_client_post():
"""make_call must pass logging_obj to client.post so track_llm_api_timing can set llm_api_duration_ms for litellm_overhead_time_ms."""
mock_client = AsyncMock()
mock_response = MagicMock()
mock_response.aiter_lines = MagicMock(return_value=iter([b'data: {"type":"message_start"}\n', b'data: {"type":"message_delta"}\n']))
mock_client.post.return_value = mock_response
logging_obj = MagicMock()
await make_call(
client=mock_client,
api_base="https://api.anthropic.com/v1/messages",
headers={},
data="{}",
model="claude-3-5-haiku",
messages=[{"role": "user", "content": "Hi"}],
logging_obj=logging_obj,
timeout=60.0,
json_mode=False,
)
mock_client.post.assert_called_once()
call_kwargs = mock_client.post.call_args[1]
assert call_kwargs.get("logging_obj") is logging_obj
def test_redacted_thinking_content_block_delta():
chunk = {
"type": "content_block_start",

3
tests/test_litellm/llms/sap/chat/test_sap_tool_parameters.py
View file

@ -127,11 +127,12 @@ class TestToolTransformationIntegration:
}
validated_tool = validate_dict(openai_tool, ChatCompletionTool)
# After validation, parameters should have type='object'
assert validated_tool["function"]["parameters"]["type"] == "object"
assert "properties" in validated_tool["function"]["parameters"]
def test_should_transform_tool_with_existing_parameters(self):
"""Tool with parameters should preserve them while ensuring type='object'."""
from litellm.llms.sap.chat.transformation import validate_dict

564
tests/test_litellm/llms/sap/chat/test_sap_transformation.py Normal file
View file

@ -0,0 +1,564 @@
import warnings
import pytest
from pydantic import ValidationError
class TestSAPTransformationIntegration:
"""Integration tests for SAP transformation."""
@pytest.fixture
def mock_config(self):
from litellm.llms.sap.chat.transformation import GenAIHubOrchestrationConfig
config = GenAIHubOrchestrationConfig()
config.token_creator = lambda: "Bearer TEST_TOKEN"
config._base_url = "https://api.test-sap.com"
config._resource_group = "test-group"
return config
def test_parameter_classification_in_transform_request(self, mock_config):
"""Test parameter classification within the actual transform_request method."""
model = "gpt-4o"
messages = [{"role": "user", "content": "Hello"}]
optional_params = {
"temperature": 0.7,
"max_tokens": 100,
"deployment_url": "https://custom.sap.com/deployment/123",
"model_version": "v1.5",
"tools": [{"type": "function", "function": {"name": "calculator"}}],
"frequency_penalty": 0.1
}
result = mock_config.transform_request(
model, messages, optional_params, {}, {}
)
model_params = result["config"]["modules"]["prompt_templating"]["model"]["params"]
assert "temperature" in model_params
assert "frequency_penalty" in model_params
assert "deployment_url" not in model_params
assert "model_version" not in model_params
assert "tools" not in model_params
model_version = result["config"]["modules"]["prompt_templating"]["model"]["version"]
assert model_version == "v1.5"
prompt = result["config"]["modules"]["prompt_templating"]["prompt"]
if "tools" in prompt:
assert isinstance(prompt["tools"], list)
for tool in prompt["tools"]:
assert tool["function"]["parameters"]["type"] == "object", (
"SAP API requires parameters.type == 'object'"
)
assert "properties" in tool["function"]["parameters"]
def test_transform_request_parameter_handling_robustness(self, mock_config):
"""Test transform_request method handles various parameter combinations correctly."""
model = "gpt-4o"
messages = [{"role": "user", "content": "Hello"}]
test_cases = [
# Case 1: Basic parameters only
{
"params": {"temperature": 0.7, "max_tokens": 100},
"expected_in_model": {"temperature", "max_tokens"},
"expected_excluded": set()
},
# Case 2: Parameters with auth/infrastructure components
{
"params": {
"temperature": 0.8,
"deployment_url": "https://api.sap.com/deployments/test",
"max_tokens": 150
},
"expected_in_model": {"temperature", "max_tokens"},
"expected_excluded": {"deployment_url"}
},
# Case 3: Parameters with framework components
{
"params": {
"temperature": 0.6,
"model_version": "v2.0",
"tools": [{"function": {"name": "test"}}],
"frequency_penalty": 0.1
},
"expected_in_model": {"temperature", "frequency_penalty"},
"expected_excluded": {"model_version", "tools"}
}
]
for i, test_case in enumerate(test_cases):
filtered_params = {
k: v for k, v in test_case["params"].items()
if k not in {"tools", "model_version", "deployment_url"}
}
for expected_param in test_case["expected_in_model"]:
assert expected_param in filtered_params, f"Case {i + 1}: {expected_param} should be in model params"
for excluded_param in test_case["expected_excluded"]:
assert excluded_param not in filtered_params, f"Case {i + 1}: {excluded_param} should be excluded from model params"
result = mock_config.transform_request(
model, messages, test_case["params"], {}, {}
)
if result and "config" in result:
model_params = result["config"]["modules"]["prompt_templating"]["model"]["params"]
for excluded_param in test_case["expected_excluded"]:
assert excluded_param not in model_params, (
f"Case {i + 1}: {excluded_param} should not be in actual model params"
)
def test_config_transform_with_response_format_json_object(self, mock_config):
expected_dict = {'config':
{'modules':
{'prompt_templating':
{'prompt':
{'template':
[{'role': 'user', 'content': 'First man on the moon, answer in json'}],
'response_format': {'type': 'json_object'}},
'model': {'name': 'gpt-4o', 'params': {}, 'version': 'latest'}
}
},
}
}
config = mock_config.transform_request(
model="gpt-4o",
messages=[{'role': 'user', 'content': 'First man on the moon, answer in json'}],
optional_params={'response_format': {'type': 'json_object'},
'deployment_url': "shouldn't be in results"},
litellm_params={},
headers={}
)
assert config == expected_dict
def test_config_transform_with_response_format_json_schema(self, mock_config):
expected_response_format = {
'type': 'json_schema',
'json_schema': {
'description': 'Schema for person information',
'name': 'person_info',
'schema': {
'type': 'object',
'properties': {
'name': {
'type': 'string',
'description': "The person's full name"
},
'age': {
'type': 'integer',
'description': "The person's age in years"
},
'occupation': {
'type': 'string',
'description': "The person's job title"
}
},
'required': ['name', 'age', 'occupation'],
'additionalProperties': False
},
'strict': True
}
}
config = mock_config.transform_request(
model="gpt-4o",
messages=[{'role': 'user', 'content': 'First man on the moon, answer in json'}],
optional_params={'response_format': expected_response_format,
'deployment_url': "shouldn't be in results"},
litellm_params={},
headers={}
)
assert config["config"]["modules"]["prompt_templating"]["prompt"]["response_format"] == expected_response_format
assert len(config["config"]["modules"]["prompt_templating"]["model"]["params"]) == 0
def test_config_transform_with_stream(self, mock_config):
expected_dict = {
'config': {
'modules': {
'prompt_templating': {
'prompt': {
'template': [{'role': 'user', 'content': 'Hello, how are you?'}]
},
'model': {
'name': 'anthropic--claude-4-sonnet',
'params': {},
'version': 'latest'
}
}
},
'stream': {'chunk_size': 10}
}
}
config = mock_config.transform_request(
model="anthropic--claude-4-sonnet",
messages=[{'content': 'Hello, how are you?', 'role': 'user'}],
optional_params={'stream': True,
'stream_options': {'chunk_size': 10},
'model_version': 'latest',
'deployment_url': "shouldn't be in results"},
litellm_params={},
headers={}
)
assert config == expected_dict
def test_sap_placeholder_defaults(self, mock_config):
config = mock_config.transform_request(
model="gpt-4o",
messages=[
{"role": "user", "content": "Hello. Answer {{ ?user_query }}"}
],
optional_params={'deployment_url': "shouldn't be in results",
"placeholder_defaults": {"user_query": "default value"}},
litellm_params={},
headers={}
)
assert config["config"]["modules"]["prompt_templating"]["prompt"]["defaults"] == {
"user_query": "default value"}
assert config["config"]["modules"]["prompt_templating"]["model"]["params"] == {}
def test_sap_placeholder_values(self, mock_config):
placeholder_values = {"user_query": "Some text"}
config = mock_config.transform_request(
model="gpt-4o",
messages=[
{"role": "user", "content": "Hello. Answer {{ ?user_query }}"}
],
optional_params={'deployment_url': "shouldn't be in results",
"placeholder_values": placeholder_values},
litellm_params={},
headers={}
)
assert config["placeholder_values"] == placeholder_values
assert config["config"]["modules"]["prompt_templating"]["model"]["params"] == {}
def test_sap_grounding(self, mock_config):
grounding_config = {
'type': 'document_grounding_service',
'config': {
'filters': [
{'id': 's3-docs',
'data_repository_type': 'vector',
'search_config': {'max_chunk_count': 2},
'data_repositories': ['123456890-test']
}
],
'placeholders': {'input': ['user_query'], 'output': 'grounding_response'},
'metadata_params': ['source', 'webUrl', 'title', 'mimeType', 'fileSuffix']
}
}
placeholder_values = {"user_query": "Some text"}
config = mock_config.transform_request(
model="gpt-4o",
messages=[
{"role": "user", "content": "Hello. Answer {{ ?user_query }} using context: {{ ?grounding_response }}"}
],
optional_params={'deployment_url': "shouldn't be in results",
"grounding": grounding_config,
"placeholder_values": placeholder_values},
litellm_params={},
headers={}
)
assert config["placeholder_values"] == placeholder_values
modules = config["config"]["modules"]
assert modules["grounding"]["type"] == "document_grounding_service"
assert modules["grounding"]["config"]["placeholders"]["output"] == "grounding_response"
assert modules["grounding"]["config"]["filters"][0]["data_repository_type"] == "vector"
assert modules["prompt_templating"]["model"]["params"] == {}
def test_grounding_search_config_rejects_both_count_fields(self, mock_config):
with pytest.raises(ValidationError):
mock_config.transform_request(
model="gpt-4o",
messages=[{"role": "user", "content": "Hi"}],
optional_params={
"grounding": {
"type": "document_grounding_service",
"config": {
"filters": [{"data_repository_type": "vector",
"search_config": {"max_chunk_count": 2,
"max_document_count": 5}}],
"placeholders": {"input": ["q"], "output": "r"},
}
}
},
litellm_params={}, headers={}
)
def test_sap_filtering(self, mock_config):
filtering_config_azure = {
'input':
{
'filters':
[
{'type': 'azure_content_safety',
'config':
{'hate': 0,
'sexual': 0,
'violence': 0,
'self_harm': 0
}
}
]
},
'output':
{
'filters':
[
{'type': 'azure_content_safety',
'config': {'hate': 0,
'sexual': 0,
'violence': 0,
'self_harm': 0
}
}
]
}
}
filtering_config_llama = {
'input':
{
'filters':
[
{
'type': 'llama_guard_3_8b',
'config': {'hate': True,
"elections": True}
}
]
},
'output':
{
'filters':
[
{
'type': 'llama_guard_3_8b',
'config': {'hate': True, "elections": True}
}
]
}
}
config = mock_config.transform_request(
model="gpt-4o",
messages=[{"role": "user", "content": "Hello."}],
optional_params={'deployment_url': "shouldn't be in results",
"filtering": filtering_config_azure},
litellm_params={},
headers={}
)
assert config["config"]["modules"]["filtering"] == filtering_config_azure
assert config["config"]["modules"]["prompt_templating"]["model"]["params"] == {}
config = mock_config.transform_request(
model="gpt-4o",
messages=[{"role": "user", "content": "Hello."}],
optional_params={'deployment_url': "shouldn't be in results",
"filtering": filtering_config_llama},
litellm_params={},
headers={}
)
assert config["config"]["modules"]["filtering"] == filtering_config_llama
assert config["config"]["modules"]["prompt_templating"]["model"]["params"] == {}
def test_filtering_config_requires_at_least_one_property(self, mock_config):
with pytest.raises(ValidationError) as exc_info:
mock_config.transform_request(
model="gpt-4o",
messages=[{"role": "user", "content": "Hello"}],
optional_params={
"filtering": {}
},
litellm_params={},
headers={}
)
assert "For using SAP Filtering Module you must provide at least one property" in str(exc_info.value)
def test_sap_masking(self, mock_config):
masking_config = {
'providers':
[
{
'type': 'sap_data_privacy_integration',
'method': 'anonymization',
'entities': [
{'type': 'profile-address'},
{'type': 'profile-email'},
{'type': 'profile-phone'},
{'type': 'profile-person'},
{'type': 'profile-location'}
]
}
]
}
config = mock_config.transform_request(
model="gpt-4o",
messages=[{"role": "user", "content": "Hello."}],
optional_params={'deployment_url': "shouldn't be in results",
"masking": masking_config},
litellm_params={},
headers={}
)
assert config["config"]["modules"]["masking"] == masking_config
assert config["config"]["modules"]["prompt_templating"]["model"]["params"] == {}
def test_masking_config_requires_exactly_one_provider_list(self, mock_config):
masking_config = {
'providers':
[
{
'type': 'sap_data_privacy_integration',
'method': 'anonymization',
'entities': [
{'type': 'profile-address'},
{'type': 'profile-email'},
{'type': 'profile-phone'},
{'type': 'profile-person'},
{'type': 'profile-location'}
]
}
],
'masking_providers':
[
{
'type': 'sap_data_privacy_integration',
'method': 'anonymization',
'entities': [
{'type': 'profile-address'}
]
}
]
}
with pytest.raises(ValidationError) as exc_info:
mock_config.transform_request(
model="gpt-4o",
messages=[{"role": "user", "content": "Hello"}],
optional_params={
"masking": masking_config
},
litellm_params={},
headers={}
)
assert "must set exactly one of: 'providers' or 'masking_providers'" in str(exc_info.value)
def test_masking_providers_deprecated_emits_warning(self, mock_config):
masking_config = {
'masking_providers':
[
{
'type': 'sap_data_privacy_integration',
'method': 'anonymization',
'entities': [
{'type': 'profile-address'}
]
}
]
}
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
mock_config.transform_request(
model="gpt-4o",
messages=[{"role": "user", "content": "Hi"}],
optional_params={"masking": masking_config},
litellm_params={},
headers={},
)
assert any(
issubclass(warning.category, DeprecationWarning)
and "masking_providers" in str(warning.message)
for warning in w
), "Expected DeprecationWarning for 'masking_providers'"
def test_sap_translation(self, mock_config):
translation_config = {
'input':
{'type': 'sap_document_translation',
'config':
{'source_language': 'en-US',
'target_language': 'de-DE'}
},
'output':
{'type': 'sap_document_translation',
'config':
{'source_language': 'de-DE',
'target_language': 'fr-FR'}
}
}
config = mock_config.transform_request(
model="gpt-4o",
messages=[{"role": "user", "content": "Hello."}],
optional_params={'deployment_url': "shouldn't be in results",
"translation": translation_config},
litellm_params={},
headers={}
)
assert config["config"]["modules"]["translation"] == translation_config
assert config["config"]["modules"]["prompt_templating"]["model"]["params"] == {}
def test_translation_config_requires_at_least_one_property(self, mock_config):
with pytest.raises(ValidationError) as exc_info:
mock_config.transform_request(
model="gpt-4o",
messages=[{"role": "user", "content": "Hello"}],
optional_params={
"translation": {}
},
litellm_params={},
headers={}
)
assert "TranslationModuleConfig requires at least one of 'input' or 'output'" in str(exc_info.value)
def test_sap_multiple_modules(self, mock_config):
translation_config = {
'input':
{'type': 'sap_document_translation',
'config':
{'source_language': 'en-US',
'target_language': 'de-DE'}
},
'output':
{'type': 'sap_document_translation',
'config':
{'source_language': 'de-DE',
'target_language': 'fr-FR'}
}
}
for model in ["sap/gpt-5", "gpt-5"]:
config = mock_config.transform_request(
model="gpt-4o",
messages=[{"role": "user", "content": "Hello."}],
optional_params={'deployment_url': "shouldn't be in results",
"fallback_sap_modules": [{"model": model,
"messages": [{"role": "user", "content": "Hello world!"}],
"translation": translation_config
}]
,
},
litellm_params={},
headers={}
)
assert "translation" not in config["config"]["modules"][0]
translation = config["config"]["modules"][1]["translation"]
assert translation["input"]["config"]["source_language"] == "en-US"
assert translation["input"]["config"]["target_language"] == "de-DE"
assert translation["output"]["config"]["target_language"] == "fr-FR"
assert config["config"]["modules"][1]["prompt_templating"]["model"]["name"] == "gpt-5"
assert config["config"]["modules"][0]["prompt_templating"]["model"]["name"] == "gpt-4o"
assert config["config"]["modules"][0]["prompt_templating"]["model"]["params"] == {}
assert config["config"]["modules"][1]["prompt_templating"]["prompt"]["template"][0]["content"] == "Hello world!"
assert config["config"]["modules"][0]["prompt_templating"]["prompt"]["template"][0]["content"] == "Hello."
assert config["config"]["modules"][1]["translation"]["input"]["type"] == "sap_document_translation"

97
tests/test_litellm/llms/sap/embed/test_sap_embed_transformation.py Normal file
View file

@ -0,0 +1,97 @@
from unittest.mock import patch, PropertyMock
import pytest
from litellm.llms.sap.embed.transformation import GenAIHubEmbeddingConfig
@pytest.fixture
def fake_token_creator():
return (lambda: "Bearer FAKE_TOKEN", "https://api.ai.moke-sap.com", "fake-group")
@pytest.fixture
def fake_deployment_url():
return "https://api.ai.moke-sap.com/v2/inference/deployments/mokeid"
def test_basic_config_transform(fake_token_creator, fake_deployment_url):
expected_dict = {
'config': {
'modules': {
'embeddings': {
'model': {
'name': 'text-embedding-3-small',
'version': 'latest',
'params': {}
}
}
}
},
'input': {
'text': 'Hi'
}
}
with patch(
"litellm.llms.sap.embed.transformation.GenAIHubEmbeddingConfig.deployment_url",
new_callable=PropertyMock,
return_value=fake_deployment_url,
), patch(
"litellm.llms.sap.embed.transformation.get_token_creator",
return_value=fake_token_creator,
):
body = GenAIHubEmbeddingConfig().transform_embedding_request(
model="text-embedding-3-small",
input="Hi",
optional_params={},
headers={}
)
assert body == expected_dict
def test_model_params(fake_token_creator, fake_deployment_url):
with patch(
"litellm.llms.sap.embed.transformation.GenAIHubEmbeddingConfig.deployment_url",
new_callable=PropertyMock,
return_value=fake_deployment_url,
), patch(
"litellm.llms.sap.embed.transformation.get_token_creator",
return_value=fake_token_creator,
):
body = GenAIHubEmbeddingConfig().transform_embedding_request(
model="text-embedding-3-small",
input="Hi",
optional_params={"parameters": {"truncate": "END"}},
headers={}
)
assert body["config"]["modules"]["embeddings"]["model"]["params"] == {"truncate": "END"}
def test_embed_with_masking(fake_token_creator, fake_deployment_url):
masking_config = {
'providers':
[
{
'type': 'sap_data_privacy_integration',
'method': 'anonymization',
'entities': [
{'type': 'profile-address'},
{'type': 'profile-phone'},
{'type': 'profile-person'},
{'type': 'profile-location'}
]
}
]
}
with patch(
"litellm.llms.sap.embed.transformation.GenAIHubEmbeddingConfig.deployment_url",
new_callable=PropertyMock,
return_value=fake_deployment_url,
), patch(
"litellm.llms.sap.embed.transformation.get_token_creator",
return_value=fake_token_creator,
):
body = GenAIHubEmbeddingConfig().transform_embedding_request(
model="text-embedding-3-small",
input="Hi",
optional_params={"parameters": {"truncate": "END"},
"masking": masking_config},
headers={}
)
assert body["config"]["modules"]["masking"] == masking_config

142
tests/test_litellm/llms/sap/test_sap_fetch_creds.py Normal file
View file

@ -0,0 +1,142 @@
import json
import pytest
import litellm.llms.sap.credentials as sap_credentials
mock_sap_service_key_dict = {
"serviceurls":
{
"AI_API_URL":"https://testurl.hana.ondemand.com/"
},
"clientid":"mockclientid",
"clientsecret":"mockclientsecret",
"url":"https://test.sap.hana.ondemand.com/"
}
mock_wrapped_sap_service_key_dict = {
"credentials": {
"serviceurls":
{
"AI_API_URL":"https://testurl.hana.ondemand.com/"
},
"clientid":"mockclientid",
"clientsecret":"mockclientsecret",
"url":"https://test.sap.hana.ondemand.com/"
}
}
expected_creds = {'client_id': "mockclientid",
'client_secret': "mockclientsecret",
'auth_url': 'https://test.sap.hana.ondemand.com/oauth/token',
'base_url': 'https://testurl.hana.ondemand.com/v2',
'resource_group': 'default'}
mock_sap_vcap_service_key_dict = {
'aicore': [{
'label': 'aicore',
'name': 'aicore-instance',
'instance_guid': '53ad5b47-a49a-4fec-9f0b-cd921c00b828',
'credentials': {
'serviceurls': {
'AI_API_URL': 'vcap-api-url'
},
'url': 'vcap-auth-url',
'clientid': 'vcap-clientid',
'clientsecret': 'vcap-clientsecret'
}
}]
}
def _prep_env(monkeypatch):
for var in ("AICORE_CLIENT_ID", "AICORE_CLIENT_SECRET", "AICORE_AUTH_URL", "AICORE_RESOURCE_GROUP",
"AICORE_BASE_URL", "AICORE_CERT_URL", "AICORE_SERVICE_KEY", "VCAP_SERVICES"):
monkeypatch.delenv(var, raising=False)
monkeypatch.setenv("AICORE_HOME", 'notexist')
monkeypatch.setattr('litellm.sap_service_key', None)
def test_sap_fetch_creds_from_env_service_key(monkeypatch):
_prep_env(monkeypatch)
monkeypatch.setenv("AICORE_SERVICE_KEY", json.dumps(mock_sap_service_key_dict))
creds = sap_credentials.fetch_credentials()
assert creds == expected_creds
def test_sap_fetch_creds_from_env_wrapped_service_key(monkeypatch):
_prep_env(monkeypatch)
monkeypatch.setenv("AICORE_SERVICE_KEY", json.dumps(mock_wrapped_sap_service_key_dict))
creds = sap_credentials.fetch_credentials()
assert creds == expected_creds
def test_sap_fetch_creds_from_arg_service_key(monkeypatch):
_prep_env(monkeypatch)
creds = sap_credentials.fetch_credentials(service_key=json.dumps(mock_sap_service_key_dict))
assert creds == expected_creds
def test_fetch_creds_from_env_vcap_service(monkeypatch):
_prep_env(monkeypatch)
monkeypatch.setenv("VCAP_SERVICES", json.dumps(mock_sap_vcap_service_key_dict))
creds = sap_credentials.fetch_credentials()
assert creds['client_id'] == "vcap-clientid"
assert creds['client_secret'] == "vcap-clientsecret"
assert creds['auth_url'] == "vcap-auth-url/oauth/token"
assert creds['base_url'] == "vcap-api-url/v2"
assert creds['resource_group'] == "default"
def test_fetch_creds_from_env(monkeypatch):
_prep_env(monkeypatch)
monkeypatch.setenv("AICORE_CLIENT_ID", "env-client-id")
monkeypatch.setenv("AICORE_CLIENT_SECRET", "env-client-secret")
monkeypatch.setenv("AICORE_AUTH_URL", "env-auth-url")
monkeypatch.setenv("AICORE_BASE_URL", "env-base-url")
monkeypatch.setenv("AICORE_RESOURCE_GROUP", "env-resource-group")
creds = sap_credentials.fetch_credentials()
assert creds['client_id'] == "env-client-id"
assert creds['client_secret'] == "env-client-secret"
assert creds['auth_url'] == "env-auth-url/oauth/token"
assert creds['base_url'] == "env-base-url/v2"
assert creds['resource_group'] == "env-resource-group"
def test_creds_priority_order(monkeypatch):
_prep_env(monkeypatch)
monkeypatch.setenv("AICORE_CLIENT_ID", "env-client-id")
monkeypatch.setenv("AICORE_CLIENT_SECRET", "env-client-secret")
monkeypatch.setenv("AICORE_AUTH_URL", "env-auth-url")
monkeypatch.setenv("AICORE_BASE_URL", "env-base-url")
monkeypatch.setenv("AICORE_RESOURCE_GROUP", "env-resource-group")
creds = sap_credentials.fetch_credentials(service_key=json.dumps(mock_sap_service_key_dict))
assert creds['client_id'] == "mockclientid"
assert creds['resource_group'] == "env-resource-group"
def test_no_credentials_configured(monkeypatch):
_prep_env(monkeypatch)
with pytest.raises(ValueError, match="No credentials found in any source"):
sap_credentials.fetch_credentials()
def test_partial_credentials_missing_auth_url(monkeypatch):
_prep_env(monkeypatch)
# Set only client_id and base_url, missing auth_url
monkeypatch.setenv("AICORE_CLIENT_ID", "test-client-id")
monkeypatch.setenv("AICORE_BASE_URL", "test-base-url")
# fetch_credentials should succeed (it returns whatever it finds)
creds = sap_credentials.fetch_credentials()
creds.pop('resource_group')
with pytest.raises(ValueError, match="SAP AI Core credentials not found"):
sap_credentials.validate_credentials(**creds)
def test_credentials_without_authentication_mode(monkeypatch):
_prep_env(monkeypatch)
# Set all required fields but no authentication mode (no client_secret, no certs)
monkeypatch.setenv("AICORE_CLIENT_ID", "test-client-id")
monkeypatch.setenv("AICORE_AUTH_URL", "test-auth-url")
monkeypatch.setenv("AICORE_BASE_URL", "test-base-url")
creds = sap_credentials.fetch_credentials()
creds.pop('resource_group')
# validate_credentials should raise because no authentication mode is provided
with pytest.raises(ValueError, match="SAP AI Core credentials are incomplete"):
sap_credentials.validate_credentials(**creds)

18
tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py
View file

@ -127,6 +127,24 @@ def test_vertex_ai_includes_labels():
assert result["labels"] == {"project": "test", "team": "ai"}
def test_service_tier_forwarded_to_vertex_ai():
"""Test that service_tier in optional_params is mapped to serviceTier in request body."""
messages = [{"role": "user", "content": "test"}]
optional_params = {"service_tier": "flex"}
litellm_params = {}
result = _transform_request_body(
messages=messages,
model="gemini-2.5-pro",
optional_params=optional_params,
custom_llm_provider="vertex_ai",
litellm_params=litellm_params,
cached_content=None,
)
assert "serviceTier" in result
assert result["serviceTier"] == "flex"
def test_extra_body_cache_not_forwarded_to_vertex_ai():
"""

115
tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py
View file

@ -3504,6 +3504,73 @@ def test_vertex_ai_traffic_type_preserved_in_hidden_params_non_streaming():
assert result._hidden_params["provider_specific_fields"]["traffic_type"] == "PROVISIONED_THROUGHPUT"
def test_vertex_ai_service_tier_streaming():
"""Test service_tier is preserved in model_response from headers for streaming."""
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
ModelResponseIterator,
)
chunk = {
"candidates": [{"content": {"parts": [{"text": "Hello"}]}}],
}
iterator = ModelResponseIterator(
streaming_response=[],
sync_stream=True,
logging_obj=MagicMock(),
response_headers={"x-gemini-service-tier": "FLEX"},
)
# Undefined when usageMetadata is missing
result = iterator.chunk_parser(chunk)
# But definitely set when usageMetadata is present
chunk_with_usage = {
"candidates": [{"content": {"parts": [{"text": "hi"}]}}],
"usageMetadata": {"promptTokenCount": 1, "candidatesTokenCount": 1, "totalTokenCount": 2}
}
result_with_usage = iterator.chunk_parser(chunk_with_usage)
assert result_with_usage.service_tier == "flex"
def test_vertex_ai_service_tier_non_streaming():
"""Test service_tier is preserved in model_response from headers for non-streaming."""
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
VertexGeminiConfig,
)
completion_response = {
"candidates": [
{
"content": {"parts": [{"text": "Hello"}], "role": "model"},
"finishReason": "STOP",
}
],
"usageMetadata": {
"promptTokenCount": 50,
"candidatesTokenCount": 100,
"totalTokenCount": 150,
},
}
raw_response = MagicMock()
raw_response.json.return_value = completion_response
raw_response.headers = {"x-gemini-service-tier": "FLEX"}
result = VertexGeminiConfig().transform_response(
model="gemini-pro",
raw_response=raw_response,
model_response=ModelResponse(),
logging_obj=MagicMock(),
request_data={},
messages=[],
optional_params={},
litellm_params={},
encoding=None,
)
assert result.service_tier == "flex"
def test_vertex_ai_traffic_type_surfaced_in_responses_api():
"""Test trafficType is surfaced as provider_specific_fields in ResponsesAPIResponse."""
from litellm.responses.litellm_completion_transformation.transformation import (
@ -3609,6 +3676,54 @@ def test_vertex_ai_web_search_options_in_map_openai_params():
assert "web_search_options" not in optional_params, "web_search_options should be removed after transformation"
def test_vertex_ai_service_tier_in_map_openai_params():
"""Test that service_tier is correctly mapped to optional_params."""
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
VertexGeminiConfig,
)
v = VertexGeminiConfig()
# Test pass-through
optional_params = {}
non_default_params = {"service_tier": "FLEX"}
result = v.map_openai_params(
non_default_params=non_default_params,
optional_params=optional_params,
model="gemini-3-pro-preview",
drop_params=True,
)
assert result["service_tier"] == "flex"
# Test auto -> priority
optional_params_auto = {}
non_default_params_auto = {"service_tier": "auto"}
result_auto = v.map_openai_params(
non_default_params=non_default_params_auto,
optional_params=optional_params_auto,
model="gemini-3-pro-preview",
drop_params=True,
)
assert result_auto["service_tier"] == "priority"
# Test AUTO (uppercase) -> priority
optional_params_auto_upper = {}
non_default_params_auto_upper = {"service_tier": "AUTO"}
result_auto_upper = v.map_openai_params(
non_default_params=non_default_params_auto_upper,
optional_params=optional_params_auto_upper,
model="gemini-3-pro-preview",
drop_params=True,
)
assert result_auto_upper["service_tier"] == "priority"
def test_vertex_ai_usage_metadata_with_video_tokens_in_prompt():
"""Test promptTokensDetails with VIDEO modality for video inputs.

79
tests/test_litellm/llms/vertex_ai/test_vertex_llm_base.py
View file

@ -1050,6 +1050,85 @@ class TestVertexBase:
mock_creds.with_scopes.assert_called_once_with(scopes)
assert result == "scoped_creds"
def test_credentials_from_pluggable_implementation(self):
"""Test _credentials_from_pluggable dispatches to pluggable.Credentials"""
vertex_base = VertexBase()
json_obj = {
"type": "external_account",
"credential_source": {
"executable": {"command": "/path/to/executable", "timeout_millis": 5000}
},
}
scopes = ["https://www.googleapis.com/auth/cloud-platform"]
mock_creds = MagicMock()
mock_creds.requires_scopes = True
mock_creds.with_scopes.return_value = "scoped_creds"
with patch("google.auth.pluggable.Credentials") as MockCredentials:
MockCredentials.from_info.return_value = mock_creds
result = vertex_base._credentials_from_pluggable(json_obj, scopes)
MockCredentials.from_info.assert_called_once_with(json_obj)
mock_creds.with_scopes.assert_called_once_with(scopes)
assert result == "scoped_creds"
def test_credentials_from_pluggable_no_scopes_needed(self):
"""Test _credentials_from_pluggable when scopes are not needed"""
vertex_base = VertexBase()
json_obj = {
"type": "external_account",
"credential_source": {
"executable": {"command": "/path/to/executable"}
},
}
scopes = ["https://www.googleapis.com/auth/cloud-platform"]
mock_creds = MagicMock()
mock_creds.requires_scopes = False
with patch("google.auth.pluggable.Credentials") as MockCredentials:
MockCredentials.from_info.return_value = mock_creds
result = vertex_base._credentials_from_pluggable(json_obj, scopes)
MockCredentials.from_info.assert_called_once_with(json_obj)
mock_creds.with_scopes.assert_not_called()
assert result == mock_creds
def test_load_auth_dispatches_to_pluggable_for_executable(self):
"""Test that load_auth routes executable credential_source to _credentials_from_pluggable"""
vertex_base = VertexBase()
json_obj = {
"type": "external_account",
"credential_source": {
"executable": {"command": "/path/to/executable", "timeout_millis": 5000}
},
}
mock_creds = MagicMock()
mock_creds.project_id = "test-project"
with patch.object(
vertex_base, "_credentials_from_pluggable", return_value=mock_creds
) as mock_pluggable, patch.object(
vertex_base, "_credentials_from_identity_pool"
) as mock_identity_pool, patch.object(
vertex_base, "refresh_auth"
):
creds, project_id = vertex_base.load_auth(
credentials=json.dumps(json_obj), project_id=None
)
mock_pluggable.assert_called_once_with(
json_obj,
scopes=["https://www.googleapis.com/auth/cloud-platform"],
)
mock_identity_pool.assert_not_called()
assert creds == mock_creds
assert project_id == "test-project"
def test_extract_aws_params(self):
"""Test _extract_aws_params: extraction, empty case, and unrecognized keys."""
# Case 1: Extracts recognized aws_* keys, ignores GCP-standard fields

314
tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py
View file

@ -9,7 +9,7 @@ from litellm.proxy._experimental.mcp_server import rest_endpoints
from litellm.proxy._experimental.mcp_server.auth import (
user_api_key_auth_mcp as auth_mcp,
)
from litellm.proxy._types import NewMCPServerRequest, UserAPIKeyAuth
from litellm.proxy._types import NewMCPServerRequest, UpdateMCPServerRequest, UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.types.mcp import MCPAuth
@ -156,7 +156,6 @@ class TestExecuteWithMcpClient:
"Authorization": "STATIC token",
}
@pytest.mark.asyncio
async def test_m2m_credentials_forwarded_to_server_model(self, monkeypatch):
"""M2M OAuth credentials (client_id, client_secret) from the nested
@ -199,9 +198,7 @@ class TestExecuteWithMcpClient:
},
)
result = await rest_endpoints._execute_with_mcp_client(
payload, ok_operation
)
result = await rest_endpoints._execute_with_mcp_client(payload, ok_operation)
assert result["status"] == "ok"
server = captured["server"]
@ -262,7 +259,10 @@ class TestExecuteWithMcpClient:
assert result["status"] == "ok"
# The incoming Authorization must be dropped — extra_headers should
# contain no oauth2 headers (only static_headers, which are None here).
assert captured["extra_headers"] is None or "Authorization" not in captured["extra_headers"]
assert (
captured["extra_headers"] is None
or "Authorization" not in captured["extra_headers"]
)
@pytest.mark.asyncio
async def test_catches_exception_group(self, monkeypatch):
@ -300,9 +300,7 @@ class TestExecuteWithMcpClient:
auth_type=MCPAuth.none,
)
result = await rest_endpoints._execute_with_mcp_client(
payload, ok_operation
)
result = await rest_endpoints._execute_with_mcp_client(payload, ok_operation)
assert result["status"] == "error"
assert result["error"] is True
@ -365,8 +363,12 @@ class TestTestToolsList:
credentials={"auth_value": "secret-key"},
)
from litellm.proxy._types import LitellmUserRoles
result = await rest_endpoints.test_tools_list(
request, payload, user_api_key_dict=UserAPIKeyAuth()
request,
payload,
user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN),
)
assert result["message"] == "Successfully retrieved tools"
@ -419,8 +421,12 @@ class TestTestToolsList:
auth_type=MCPAuth.oauth2,
)
from litellm.proxy._types import LitellmUserRoles
result = await rest_endpoints.test_tools_list(
request, payload, user_api_key_dict=UserAPIKeyAuth()
request,
payload,
user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN),
)
assert result["message"] == "Successfully retrieved tools"
@ -484,7 +490,11 @@ class TestListToolsRestAPI:
captured = {"called": False}
async def fake_get_tools(
server, server_auth_header, raw_headers=None, user_api_key_auth=None, extra_headers=None
server,
server_auth_header,
raw_headers=None,
user_api_key_auth=None,
extra_headers=None,
):
captured["called"] = True
captured["server"] = server
@ -555,27 +565,47 @@ class TestListToolsRestAPI:
captured = {"called": False, "server_arg": None}
async def fake_get_tools(server, server_auth_header, raw_headers=None, user_api_key_auth=None, extra_headers=None):
async def fake_get_tools(
server,
server_auth_header,
raw_headers=None,
user_api_key_auth=None,
extra_headers=None,
):
captured["called"] = True
captured["server_arg"] = server
return ["tool-x"]
monkeypatch.setattr(rest_endpoints, "build_effective_auth_contexts", fake_contexts, raising=False)
monkeypatch.setattr(
rest_endpoints.global_mcp_server_manager, "get_allowed_mcp_servers",
fake_get_allowed_mcp_servers, raising=False,
rest_endpoints,
"build_effective_auth_contexts",
fake_contexts,
raising=False,
)
monkeypatch.setattr(
rest_endpoints.global_mcp_server_manager, "get_mcp_server_by_name",
rest_endpoints.global_mcp_server_manager,
"get_allowed_mcp_servers",
fake_get_allowed_mcp_servers,
raising=False,
)
monkeypatch.setattr(
rest_endpoints.global_mcp_server_manager,
"get_mcp_server_by_name",
lambda name: stub_server if name == "my-server" else None,
raising=False,
)
monkeypatch.setattr(
rest_endpoints.global_mcp_server_manager, "get_mcp_server_by_id",
rest_endpoints.global_mcp_server_manager,
"get_mcp_server_by_id",
lambda sid: stub_server if sid == "uuid-abc-123" else None,
raising=False,
)
monkeypatch.setattr(rest_endpoints, "_get_tools_for_single_server", fake_get_tools, raising=False)
monkeypatch.setattr(
rest_endpoints,
"_get_tools_for_single_server",
fake_get_tools,
raising=False,
)
request = _build_request(path="/mcp-rest/tools/list", method="GET")
result = await rest_endpoints.list_tool_rest_api(
@ -609,18 +639,27 @@ class TestListToolsRestAPI:
async def fake_get_allowed_mcp_servers(*args, **kwargs):
return []
monkeypatch.setattr(rest_endpoints, "build_effective_auth_contexts", fake_contexts, raising=False)
monkeypatch.setattr(
rest_endpoints.global_mcp_server_manager, "get_allowed_mcp_servers",
fake_get_allowed_mcp_servers, raising=False,
rest_endpoints,
"build_effective_auth_contexts",
fake_contexts,
raising=False,
)
monkeypatch.setattr(
rest_endpoints.global_mcp_server_manager, "get_mcp_server_by_name",
rest_endpoints.global_mcp_server_manager,
"get_allowed_mcp_servers",
fake_get_allowed_mcp_servers,
raising=False,
)
monkeypatch.setattr(
rest_endpoints.global_mcp_server_manager,
"get_mcp_server_by_name",
lambda name: stub_server if name == "restricted-server" else None,
raising=False,
)
monkeypatch.setattr(
rest_endpoints.global_mcp_server_manager, "get_mcp_server_by_id",
rest_endpoints.global_mcp_server_manager,
"get_mcp_server_by_id",
lambda sid: stub_server if sid == "uuid-xyz-999" else None,
raising=False,
)
@ -662,31 +701,54 @@ class TestListToolsRestAPI:
oauth_headers = {"Authorization": "Bearer user-oauth-token"}
async def fake_get_user_oauth_extra_headers(server, user_api_key_dict, prefetched_creds=None):
async def fake_get_user_oauth_extra_headers(
server, user_api_key_dict, prefetched_creds=None
):
return oauth_headers
captured = {}
async def fake_get_tools(server, server_auth_header, raw_headers=None, user_api_key_auth=None, extra_headers=None):
async def fake_get_tools(
server,
server_auth_header,
raw_headers=None,
user_api_key_auth=None,
extra_headers=None,
):
captured["server"] = server
captured["auth_header"] = server_auth_header
return ["oauth-tool"]
monkeypatch.setattr(rest_endpoints, "build_effective_auth_contexts", fake_contexts, raising=False)
monkeypatch.setattr(
rest_endpoints.global_mcp_server_manager, "get_allowed_mcp_servers",
fake_get_allowed_mcp_servers, raising=False,
rest_endpoints,
"build_effective_auth_contexts",
fake_contexts,
raising=False,
)
monkeypatch.setattr(
rest_endpoints.global_mcp_server_manager, "get_mcp_server_by_id",
rest_endpoints.global_mcp_server_manager,
"get_allowed_mcp_servers",
fake_get_allowed_mcp_servers,
raising=False,
)
monkeypatch.setattr(
rest_endpoints.global_mcp_server_manager,
"get_mcp_server_by_id",
lambda sid: stub_server if sid == "oauth-server-id" else None,
raising=False,
)
monkeypatch.setattr(
rest_endpoints, "_get_user_oauth_extra_headers",
fake_get_user_oauth_extra_headers, raising=False,
rest_endpoints,
"_get_user_oauth_extra_headers",
fake_get_user_oauth_extra_headers,
raising=False,
)
monkeypatch.setattr(
rest_endpoints,
"_get_tools_for_single_server",
fake_get_tools,
raising=False,
)
monkeypatch.setattr(rest_endpoints, "_get_tools_for_single_server", fake_get_tools, raising=False)
request = _build_request(path="/mcp-rest/tools/list", method="GET")
result = await rest_endpoints.list_tool_rest_api(
@ -1124,3 +1186,189 @@ class TestGetToolsForSingleServer:
assert "tool3" in tool_names
assert "tool1" not in tool_names
assert "tool4" not in tool_names
class TestStdioCommandAllowlist:
"""Tests for MCP stdio command allowlist validation."""
def test_allowed_command_passes_validation(self):
"""npx, uvx, python, etc. should be accepted."""
req = NewMCPServerRequest(
server_name="test",
transport="stdio",
command="npx",
args=["-y", "@modelcontextprotocol/server-filesystem"],
)
assert req.command == "npx"
def test_disallowed_command_raises(self):
"""Arbitrary commands like bash should be rejected."""
with pytest.raises(ValueError, match="not in the allowed commands list"):
NewMCPServerRequest(
server_name="test",
transport="stdio",
command="bash",
args=["-c", "echo pwned"],
)
def test_sh_command_raises(self):
"""sh should be rejected."""
with pytest.raises(ValueError, match="not in the allowed commands list"):
NewMCPServerRequest(
server_name="test",
transport="stdio",
command="sh",
args=["-c", "id > /tmp/output.txt"],
)
def test_absolute_path_bypass_blocked(self):
"""/bin/bash should be blocked (basename is 'bash')."""
with pytest.raises(ValueError, match="not in the allowed commands list"):
NewMCPServerRequest(
server_name="test",
transport="stdio",
command="/bin/bash",
args=["-c", "echo pwned"],
)
def test_absolute_path_to_allowed_command_works(self):
"""/usr/bin/python3 should pass (basename is 'python3')."""
req = NewMCPServerRequest(
server_name="test",
transport="stdio",
command="/usr/bin/python3",
args=["-m", "some_module"],
)
assert req.command == "/usr/bin/python3"
def test_http_transport_ignores_allowlist(self):
"""HTTP/SSE transport should not trigger command validation."""
req = NewMCPServerRequest(
server_name="test",
transport="sse",
url="https://example.com/mcp",
)
assert req.transport == "sse"
def test_uvx_command_passes(self):
req = NewMCPServerRequest(
server_name="test",
transport="stdio",
command="uvx",
args=["mcp-server-sqlite"],
)
assert req.command == "uvx"
def test_node_command_passes(self):
req = NewMCPServerRequest(
server_name="test",
transport="stdio",
command="node",
args=["server.js"],
)
assert req.command == "node"
def test_update_request_disallowed_command_raises(self):
"""UpdateMCPServerRequest should also block non-allowlisted commands."""
with pytest.raises(ValueError, match="not in the allowed commands list"):
UpdateMCPServerRequest(
server_id="some-id",
transport="stdio",
command="bash",
args=["-c", "echo pwned"],
)
class TestEndpointRoleChecks:
"""Tests for PROXY_ADMIN role checks on MCP test endpoints."""
def test_test_connection_has_auth_dependency(self):
route = _get_route("/mcp-rest/test/connection", "POST")
assert _route_has_dependency(route, user_api_key_auth)
def test_test_tools_list_has_auth_dependency(self):
route = _get_route("/mcp-rest/test/tools/list", "POST")
assert _route_has_dependency(route, user_api_key_auth)
@pytest.mark.asyncio
async def test_test_connection_rejects_non_admin(self):
"""Non-admin users should get 403 from test_connection."""
from litellm.proxy._types import LitellmUserRoles
payload = NewMCPServerRequest(
server_name="test",
url="https://example.com/mcp",
auth_type=MCPAuth.none,
)
user_key = UserAPIKeyAuth(
user_role=LitellmUserRoles.INTERNAL_USER,
user_id="non_admin",
api_key="sk-test",
)
request = _build_request()
with pytest.raises(HTTPException) as exc_info:
await rest_endpoints.test_connection(
request=request,
new_mcp_server_request=payload,
user_api_key_dict=user_key,
)
assert exc_info.value.status_code == 403
@pytest.mark.asyncio
async def test_test_tools_list_rejects_non_admin(self):
"""Non-admin users should get 403 from test_tools_list."""
from litellm.proxy._types import LitellmUserRoles
payload = NewMCPServerRequest(
server_name="test",
url="https://example.com/mcp",
auth_type=MCPAuth.none,
)
user_key = UserAPIKeyAuth(
user_role=LitellmUserRoles.INTERNAL_USER,
user_id="non_admin",
api_key="sk-test",
)
request = _build_request()
with pytest.raises(HTTPException) as exc_info:
await rest_endpoints.test_tools_list(
request=request,
new_mcp_server_request=payload,
user_api_key_dict=user_key,
)
assert exc_info.value.status_code == 403
@pytest.mark.asyncio
async def test_test_connection_allows_admin(self, monkeypatch):
"""PROXY_ADMIN should pass the role check."""
from litellm.proxy._types import LitellmUserRoles
async def fake_execute(*args, **kwargs):
return {"status": "ok"}
monkeypatch.setattr(
rest_endpoints,
"_execute_with_mcp_client",
fake_execute,
)
payload = NewMCPServerRequest(
server_name="test",
url="https://example.com/mcp",
auth_type=MCPAuth.none,
)
user_key = UserAPIKeyAuth(
user_role=LitellmUserRoles.PROXY_ADMIN,
user_id="admin",
api_key="sk-admin",
)
request = _build_request()
result = await rest_endpoints.test_connection(
request=request,
new_mcp_server_request=payload,
user_api_key_dict=user_key,
)
assert result["status"] == "ok"

543
tests/test_litellm/proxy/test_litellm_pre_call_utils.py
View file

@ -13,14 +13,18 @@ from litellm.proxy._types import TeamCallbackMetadata, UserAPIKeyAuth
from litellm.proxy.litellm_pre_call_utils import (
KeyAndTeamLoggingSettings,
LiteLLMProxyRequestSetup,
_apply_credential_overrides_from_model_config,
_extract_credential_from_entry,
_get_dynamic_logging_metadata,
_get_enforced_params,
_get_metadata_variable_name,
_resolve_credential_from_model_config,
_update_model_if_key_alias_exists,
add_guardrails_from_policy_engine,
add_litellm_data_to_request,
check_if_token_is_service_account,
)
from litellm.types.utils import CredentialItem
sys.path.insert(
0, os.path.abspath("../../..")
@ -1912,3 +1916,542 @@ async def test_bearer_token_not_in_debug_logs():
f"Bearer token leaked in debug logs. "
f"Found token in log output:\n{log_output[:500]}"
)
# ============================================================================
# Tests for credential overrides from model_config (team/project metadata)
# ============================================================================
@pytest.fixture()
def setup_test_credentials():
"""Populate litellm.credential_list with test credentials and enable feature flag, clean up after."""
original = litellm.credential_list[:]
original_flag = litellm.enable_model_config_credential_overrides
litellm.enable_model_config_credential_overrides = True
litellm.credential_list.extend(
[
CredentialItem(
credential_name="hotel-azure-eastus",
credential_info={},
credential_values={
"api_base": "https://hotel-eastus.openai.azure.com/",
"api_key": "key-hotel-eastus",
},
),
CredentialItem(
credential_name="hotel-azure-westus",
credential_info={},
credential_values={
"api_base": "https://hotel-westus.openai.azure.com/",
"api_key": "key-hotel-westus",
},
),
CredentialItem(
credential_name="hotel-rec-azure",
credential_info={},
credential_values={
"api_base": "https://hotel-rec-app.openai.azure.com/",
"api_key": "key-hotel-rec",
},
),
CredentialItem(
credential_name="hotel-rec-vision",
credential_info={},
credential_values={
"api_base": "https://hotel-rec-vision.openai.azure.com/",
"api_key": "key-hotel-rec-vision",
"api_version": "2024-06-01",
},
),
CredentialItem(
credential_name="flight-azure-centralus",
credential_info={},
credential_values={
"api_base": "https://flight-centralus.openai.azure.com/",
"api_key": "key-flight-centralus",
},
),
]
)
yield
litellm.credential_list[:] = original
litellm.enable_model_config_credential_overrides = original_flag
# --- Unit tests for _extract_credential_from_entry ---
def test_extract_credential_from_entry_azure():
entry = {"azure": {"litellm_credentials": "my-cred"}}
assert _extract_credential_from_entry(entry) == "my-cred"
def test_extract_credential_from_entry_no_credential():
entry = {"azure": {"some_other_key": "value"}}
assert _extract_credential_from_entry(entry) is None
def test_extract_credential_from_entry_empty():
assert _extract_credential_from_entry({}) is None
def test_extract_credential_from_entry_non_dict_value():
entry = {"azure": "not-a-dict"}
assert _extract_credential_from_entry(entry) is None
def test_extract_credential_from_entry_non_dict_entry():
"""Non-dict entry (e.g. string) should return None, not crash."""
assert _extract_credential_from_entry("my-cred-name") is None
assert _extract_credential_from_entry(["a", "list"]) is None
assert _extract_credential_from_entry(42) is None
# --- Unit tests for _resolve_credential_from_model_config ---
def test_resolve_project_model_specific_wins():
project_config = {
"gpt-4": {"azure": {"litellm_credentials": "proj-gpt4"}},
"defaultconfig": {"azure": {"litellm_credentials": "proj-default"}},
}
team_config = {
"gpt-4": {"azure": {"litellm_credentials": "team-gpt4"}},
"defaultconfig": {"azure": {"litellm_credentials": "team-default"}},
}
result = _resolve_credential_from_model_config(
"gpt-4", project_config, team_config
)
assert result == "proj-gpt4"
def test_resolve_project_default_wins_over_team():
project_config = {
"defaultconfig": {"azure": {"litellm_credentials": "proj-default"}},
}
team_config = {
"gpt-4": {"azure": {"litellm_credentials": "team-gpt4"}},
"defaultconfig": {"azure": {"litellm_credentials": "team-default"}},
}
result = _resolve_credential_from_model_config(
"gpt-4", project_config, team_config
)
assert result == "proj-default"
def test_resolve_team_model_specific_wins_over_team_default():
team_config = {
"gpt-4": {"azure": {"litellm_credentials": "team-gpt4"}},
"defaultconfig": {"azure": {"litellm_credentials": "team-default"}},
}
result = _resolve_credential_from_model_config("gpt-4", None, team_config)
assert result == "team-gpt4"
def test_resolve_team_default_used_as_fallback():
team_config = {
"defaultconfig": {"azure": {"litellm_credentials": "team-default"}},
}
result = _resolve_credential_from_model_config("gpt-3.5", None, team_config)
assert result == "team-default"
def test_resolve_no_match_returns_none():
result = _resolve_credential_from_model_config("gpt-4", None, None)
assert result is None
def test_resolve_empty_configs_returns_none():
result = _resolve_credential_from_model_config("gpt-4", {}, {})
assert result is None
def test_resolve_model_not_in_any_config():
project_config = {"gpt-4": {"azure": {"litellm_credentials": "x"}}}
result = _resolve_credential_from_model_config("gpt-3.5", project_config, None)
assert result is None
# --- Integration tests for _apply_credential_overrides_from_model_config ---
def test_apply_overrides_project_model_specific(setup_test_credentials):
"""Scenario 2: Hotel Rec App -> gpt-4-vision -> project model-specific."""
data = {"model": "gpt-4-vision"}
user_api_key_dict = UserAPIKeyAuth(
api_key="test-key",
team_metadata={
"model_config": {
"defaultconfig": {
"azure": {"litellm_credentials": "hotel-azure-eastus"}
},
"gpt-4": {"azure": {"litellm_credentials": "hotel-azure-westus"}},
}
},
project_metadata={
"model_config": {
"defaultconfig": {
"azure": {"litellm_credentials": "hotel-rec-azure"}
},
"gpt-4-vision": {
"azure": {"litellm_credentials": "hotel-rec-vision"}
},
}
},
)
_apply_credential_overrides_from_model_config(
data=data, user_api_key_dict=user_api_key_dict
)
assert data["api_base"] == "https://hotel-rec-vision.openai.azure.com/"
assert data["api_key"] == "key-hotel-rec-vision"
assert data["api_version"] == "2024-06-01"
def test_apply_overrides_project_default(setup_test_credentials):
"""Scenario 1: Hotel Rec App -> gpt-4 -> project default."""
data = {"model": "gpt-4"}
user_api_key_dict = UserAPIKeyAuth(
api_key="test-key",
team_metadata={
"model_config": {
"defaultconfig": {
"azure": {"litellm_credentials": "hotel-azure-eastus"}
},
"gpt-4": {"azure": {"litellm_credentials": "hotel-azure-westus"}},
}
},
project_metadata={
"model_config": {
"defaultconfig": {
"azure": {"litellm_credentials": "hotel-rec-azure"}
},
"gpt-4-vision": {
"azure": {"litellm_credentials": "hotel-rec-vision"}
},
}
},
)
_apply_credential_overrides_from_model_config(
data=data, user_api_key_dict=user_api_key_dict
)
assert data["api_base"] == "https://hotel-rec-app.openai.azure.com/"
assert data["api_key"] == "key-hotel-rec"
def test_apply_overrides_team_model_specific(setup_test_credentials):
"""Scenario 4: Hotel Review App -> gpt-4 -> team model-specific."""
data = {"model": "gpt-4"}
user_api_key_dict = UserAPIKeyAuth(
api_key="test-key",
team_metadata={
"model_config": {
"defaultconfig": {
"azure": {"litellm_credentials": "hotel-azure-eastus"}
},
"gpt-4": {"azure": {"litellm_credentials": "hotel-azure-westus"}},
}
},
project_metadata={},
)
_apply_credential_overrides_from_model_config(
data=data, user_api_key_dict=user_api_key_dict
)
assert data["api_base"] == "https://hotel-westus.openai.azure.com/"
assert data["api_key"] == "key-hotel-westus"
def test_apply_overrides_team_default(setup_test_credentials):
"""Scenario 3: Hotel Review App -> gpt-3.5 -> team default."""
data = {"model": "gpt-3.5"}
user_api_key_dict = UserAPIKeyAuth(
api_key="test-key",
team_metadata={
"model_config": {
"defaultconfig": {
"azure": {"litellm_credentials": "hotel-azure-eastus"}
},
"gpt-4": {"azure": {"litellm_credentials": "hotel-azure-westus"}},
}
},
project_metadata={},
)
_apply_credential_overrides_from_model_config(
data=data, user_api_key_dict=user_api_key_dict
)
assert data["api_base"] == "https://hotel-eastus.openai.azure.com/"
assert data["api_key"] == "key-hotel-eastus"
def test_apply_overrides_no_config(setup_test_credentials):
"""Scenario 6: No model_config anywhere -> data unchanged."""
data = {"model": "gpt-4"}
user_api_key_dict = UserAPIKeyAuth(
api_key="test-key",
team_metadata={},
project_metadata={},
)
_apply_credential_overrides_from_model_config(
data=data, user_api_key_dict=user_api_key_dict
)
assert "api_base" not in data
assert "api_key" not in data
def test_apply_overrides_clientside_credentials_take_precedence(
setup_test_credentials,
):
"""Clientside api_base/api_key in data should block model_config override."""
data = {
"model": "gpt-4",
"api_base": "https://my-custom-endpoint.openai.azure.com/",
"api_key": "my-custom-key",
}
user_api_key_dict = UserAPIKeyAuth(
api_key="test-key",
team_metadata={
"model_config": {
"defaultconfig": {
"azure": {"litellm_credentials": "hotel-azure-eastus"}
}
}
},
)
_apply_credential_overrides_from_model_config(
data=data, user_api_key_dict=user_api_key_dict
)
assert data["api_base"] == "https://my-custom-endpoint.openai.azure.com/"
assert data["api_key"] == "my-custom-key"
def test_apply_overrides_missing_credential_name(setup_test_credentials):
"""model_config references a credential that doesn't exist -> no override."""
data = {"model": "gpt-4"}
user_api_key_dict = UserAPIKeyAuth(
api_key="test-key",
team_metadata={
"model_config": {
"gpt-4": {
"azure": {"litellm_credentials": "nonexistent-credential"}
}
}
},
)
_apply_credential_overrides_from_model_config(
data=data, user_api_key_dict=user_api_key_dict
)
assert "api_base" not in data
assert "api_key" not in data
def test_apply_overrides_api_version_only_if_present(setup_test_credentials):
"""api_version should only be set if the credential contains it."""
data = {"model": "gpt-3.5"}
user_api_key_dict = UserAPIKeyAuth(
api_key="test-key",
team_metadata={
"model_config": {
"defaultconfig": {
"azure": {"litellm_credentials": "hotel-azure-eastus"}
}
}
},
)
_apply_credential_overrides_from_model_config(
data=data, user_api_key_dict=user_api_key_dict
)
assert data["api_base"] == "https://hotel-eastus.openai.azure.com/"
assert data["api_key"] == "key-hotel-eastus"
assert "api_version" not in data
def test_apply_overrides_no_model_in_data(setup_test_credentials):
"""No model in request data -> skip override."""
data = {"messages": [{"role": "user", "content": "hello"}]}
user_api_key_dict = UserAPIKeyAuth(
api_key="test-key",
team_metadata={
"model_config": {
"defaultconfig": {
"azure": {"litellm_credentials": "some-cred"}
}
}
},
)
_apply_credential_overrides_from_model_config(
data=data, user_api_key_dict=user_api_key_dict
)
assert "api_base" not in data
def test_apply_overrides_none_metadata(setup_test_credentials):
"""None metadata on both team and project -> skip override."""
data = {"model": "gpt-4"}
user_api_key_dict = UserAPIKeyAuth(
api_key="test-key",
team_metadata=None,
project_metadata=None,
)
_apply_credential_overrides_from_model_config(
data=data, user_api_key_dict=user_api_key_dict
)
assert "api_base" not in data
def test_apply_overrides_clientside_api_version_preserved(setup_test_credentials):
"""Clientside api_version should not be overwritten by credential."""
data = {"model": "gpt-4-vision", "api_version": "2025-01-01"}
user_api_key_dict = UserAPIKeyAuth(
api_key="test-key",
team_metadata={
"model_config": {
"gpt-4-vision": {
"azure": {"litellm_credentials": "hotel-rec-vision"}
}
}
},
)
_apply_credential_overrides_from_model_config(
data=data, user_api_key_dict=user_api_key_dict
)
# api_base and api_key should be set from credential
assert data["api_base"] == "https://hotel-rec-vision.openai.azure.com/"
assert data["api_key"] == "key-hotel-rec-vision"
# api_version should be preserved from the request, not overwritten
assert data["api_version"] == "2025-01-01"
def test_resolve_non_dict_model_config_ignored():
"""Non-dict model_config (e.g. string) should be safely skipped."""
result = _resolve_credential_from_model_config("gpt-4", "not-a-dict", None)
assert result is None
result = _resolve_credential_from_model_config(
"gpt-4", None, ["also", "not", "a", "dict"]
)
assert result is None
# Valid config still works alongside invalid one
result = _resolve_credential_from_model_config(
"gpt-4",
"invalid",
{"gpt-4": {"azure": {"litellm_credentials": "valid-cred"}}},
)
assert result == "valid-cred"
def test_resolve_pre_alias_model_name_fallback():
"""model_config keyed on pre-alias name should match after alias resolution."""
team_config = {
"gpt-4": {"azure": {"litellm_credentials": "team-gpt4"}},
}
# Post-alias name doesn't match, but pre-alias does (team scope)
result = _resolve_credential_from_model_config(
"azure/gpt-4-0613", None, team_config, pre_alias_model_name="gpt-4"
)
assert result == "team-gpt4"
# Same test for project scope
project_config = {
"gpt-4": {"azure": {"litellm_credentials": "proj-gpt4"}},
}
result = _resolve_credential_from_model_config(
"azure/gpt-4-0613", project_config, None, pre_alias_model_name="gpt-4"
)
assert result == "proj-gpt4"
def test_resolve_post_alias_name_takes_priority():
"""Post-alias (resolved) name should be tried before pre-alias name."""
team_config = {
"gpt-4": {"azure": {"litellm_credentials": "pre-alias-cred"}},
"gpt-4o-team-1": {"azure": {"litellm_credentials": "post-alias-cred"}},
}
# Team scope
result = _resolve_credential_from_model_config(
"gpt-4o-team-1", None, team_config, pre_alias_model_name="gpt-4"
)
assert result == "post-alias-cred"
# Project scope
result = _resolve_credential_from_model_config(
"gpt-4o-team-1", team_config, None, pre_alias_model_name="gpt-4"
)
assert result == "post-alias-cred"
def test_apply_overrides_with_alias(setup_test_credentials):
"""Credential override should work when model name was changed by alias."""
# Simulate: user called "my-gpt4", alias resolved to "azure/gpt-4-custom"
# model_config is keyed on "my-gpt4" (the pre-alias name)
data = {"model": "azure/gpt-4-custom"}
user_api_key_dict = UserAPIKeyAuth(
api_key="test-key",
team_metadata={
"model_config": {
"my-gpt4": {"azure": {"litellm_credentials": "hotel-azure-eastus"}},
}
},
)
_apply_credential_overrides_from_model_config(
data=data,
user_api_key_dict=user_api_key_dict,
pre_alias_model_name="my-gpt4",
)
assert data["api_base"] == "https://hotel-eastus.openai.azure.com/"
assert data["api_key"] == "key-hotel-eastus"
def test_apply_overrides_feature_flag_disabled_by_default():
"""Feature flag defaults to False — credential overrides are inert until explicitly enabled."""
assert litellm.enable_model_config_credential_overrides is False
data = {"model": "gpt-4"}
user_api_key_dict = UserAPIKeyAuth(
api_key="test-key",
team_metadata={
"model_config": {
"gpt-4": {"azure": {"litellm_credentials": "hotel-azure-eastus"}}
}
},
)
_apply_credential_overrides_from_model_config(
data=data, user_api_key_dict=user_api_key_dict
)
assert "api_base" not in data
assert "api_key" not in data
def test_extract_credential_provider_hint_prefers_exact_match():
"""Provider hint selects the correct provider in a multi-provider entry."""
entry = {
"openai": {"litellm_credentials": "openai-cred"},
"azure": {"litellm_credentials": "azure-cred"},
}
# With provider hint, should pick the exact match
assert _extract_credential_from_entry(entry, provider="azure") == "azure-cred"
assert _extract_credential_from_entry(entry, provider="openai") == "openai-cred"
# Without provider hint, falls back to first key (insertion order)
result = _extract_credential_from_entry(entry)
assert result in ("openai-cred", "azure-cred")
# Unknown provider falls back to first available
result = _extract_credential_from_entry(entry, provider="bedrock")
assert result in ("openai-cred", "azure-cred")
def test_resolve_provider_hint_from_model_name():
"""Provider prefix in model name (e.g. azure/gpt-4) threads through to entry extraction."""
config = {
"gpt-4": {
"openai": {"litellm_credentials": "openai-cred"},
"azure": {"litellm_credentials": "azure-cred"},
},
}
# Model name "azure/gpt-4" -> provider="azure" -> should prefer azure-cred
# But _resolve_credential_from_model_config tries "azure/gpt-4" first (no match),
# then falls to defaultconfig (no match). So we need to use pre_alias_model_name.
result = _resolve_credential_from_model_config(
"azure/gpt-4", config, None, pre_alias_model_name="gpt-4", provider="azure"
)
assert result == "azure-cred"

222
tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py
View file

@ -27,7 +27,6 @@ import litellm
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.types.llms.openai import ResponsesAPIResponse
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
@ -70,7 +69,9 @@ class TestEncryptedItemIdCodec:
def test_roundtrip(self):
model_id = "deployment-1"
original_item_id = "rs_abc123def456"
encoded = ResponsesAPIRequestUtils._build_encrypted_item_id(model_id, original_item_id)
encoded = ResponsesAPIRequestUtils._build_encrypted_item_id(
model_id, original_item_id
)
assert encoded.startswith("encitem_")
decoded = ResponsesAPIRequestUtils._decode_encrypted_item_id(encoded)
assert decoded is not None
@ -81,7 +82,9 @@ class TestEncryptedItemIdCodec:
"""Decoding must succeed even if base64 padding (=) was stripped in transit."""
model_id = "gpt-5.1-codex-openai-2"
original_item_id = "rs_0efb96cb222403210069a01d5d52588196a9dc394ffdb89d00"
encoded = ResponsesAPIRequestUtils._build_encrypted_item_id(model_id, original_item_id)
encoded = ResponsesAPIRequestUtils._build_encrypted_item_id(
model_id, original_item_id
)
# Strip any trailing '=' to simulate what happens in transit
stripped = encoded.rstrip("=")
decoded = ResponsesAPIRequestUtils._decode_encrypted_item_id(stripped)
@ -98,7 +101,9 @@ class TestEncryptedItemIdCodec:
"""item_id values containing ';' must survive the roundtrip."""
model_id = "deployment-1"
original_item_id = "rs_part1;part2;part3"
encoded = ResponsesAPIRequestUtils._build_encrypted_item_id(model_id, original_item_id)
encoded = ResponsesAPIRequestUtils._build_encrypted_item_id(
model_id, original_item_id
)
decoded = ResponsesAPIRequestUtils._decode_encrypted_item_id(encoded)
assert decoded is not None
assert decoded["item_id"] == original_item_id
@ -114,8 +119,10 @@ class TestUpdateEncryptedContentItemIds:
{"id": "rs_xyz", "type": "reasoning", "encrypted_content": "secret"},
],
}
result = ResponsesAPIRequestUtils._update_encrypted_content_item_ids_in_response(
response, model_id
result = (
ResponsesAPIRequestUtils._update_encrypted_content_item_ids_in_response(
response, model_id
)
)
# Plain message item untouched
assert result["output"][0]["id"] == "msg_abc"
@ -128,10 +135,14 @@ class TestUpdateEncryptedContentItemIds:
def test_no_op_when_model_id_is_none(self):
response = {
"output": [{"id": "rs_xyz", "type": "reasoning", "encrypted_content": "secret"}]
"output": [
{"id": "rs_xyz", "type": "reasoning", "encrypted_content": "secret"}
]
}
result = ResponsesAPIRequestUtils._update_encrypted_content_item_ids_in_response(
response, None
result = (
ResponsesAPIRequestUtils._update_encrypted_content_item_ids_in_response(
response, None
)
)
assert result["output"][0]["id"] == "rs_xyz"
@ -147,16 +158,20 @@ class TestEncryptedContentWrapping:
assert wrapped.startswith("litellm_enc:")
assert wrapped != original_content
unwrapped_model_id, unwrapped_content = (
ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id(wrapped)
)
(
unwrapped_model_id,
unwrapped_content,
) = ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id(wrapped)
assert unwrapped_model_id == model_id
assert unwrapped_content == original_content
def test_unwrap_plain_encrypted_content(self):
"""Unwrapping plain encrypted_content returns None for model_id."""
plain_content = "gAAAAABpnW_yEYmSNEyOG_plain_content"
model_id, content = ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id(
(
model_id,
content,
) = ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id(
plain_content
)
assert model_id is None
@ -175,16 +190,19 @@ class TestEncryptedContentWrapping:
},
],
}
result = ResponsesAPIRequestUtils._update_encrypted_content_item_ids_in_response(
response, model_id
result = (
ResponsesAPIRequestUtils._update_encrypted_content_item_ids_in_response(
response, model_id
)
)
assert result["output"][0].get("encrypted_content") is None
wrapped = result["output"][1]["encrypted_content"]
assert wrapped.startswith("litellm_enc:")
model_id_extracted, unwrapped = (
ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id(wrapped)
)
(
model_id_extracted,
unwrapped,
) = ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id(wrapped)
assert model_id_extracted == model_id
assert unwrapped == "gAAAAABpnW_yEYmSNEyOG_secret"
@ -193,14 +211,18 @@ class TestRestoreEncryptedContentItemIds:
def test_restores_encoded_ids(self):
model_id = "deployment-1"
original_id = "rs_encrypted_item_456"
encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id(model_id, original_id)
encoded_id = ResponsesAPIRequestUtils._build_encrypted_item_id(
model_id, original_id
)
request_input = [
{"type": "message", "id": "msg_abc123", "role": "assistant"},
{"type": "reasoning", "id": encoded_id, "encrypted_content": "secret"},
]
restored = ResponsesAPIRequestUtils._restore_encrypted_content_item_ids_in_input(
request_input
restored = (
ResponsesAPIRequestUtils._restore_encrypted_content_item_ids_in_input(
request_input
)
)
assert restored[0]["id"] == "msg_abc123"
assert restored[1]["id"] == original_id
@ -209,15 +231,19 @@ class TestRestoreEncryptedContentItemIds:
"""Test that wrapped encrypted_content is unwrapped before forwarding."""
model_id = "deployment-1"
original_content = "gAAAAABpnW_yEYmSNEyOG_original"
wrapped_content = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id(
original_content, model_id
wrapped_content = (
ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id(
original_content, model_id
)
)
request_input = [
{"type": "reasoning", "encrypted_content": wrapped_content},
]
restored = ResponsesAPIRequestUtils._restore_encrypted_content_item_ids_in_input(
request_input
restored = (
ResponsesAPIRequestUtils._restore_encrypted_content_item_ids_in_input(
request_input
)
)
assert restored[0]["encrypted_content"] == original_content
@ -258,7 +284,9 @@ async def test_encrypted_content_affinity_tracks_and_routes():
"id": "msg_abc123",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "Hello!", "annotations": []}],
"content": [
{"type": "output_text", "text": "Hello!", "annotations": []}
],
},
{
"type": "reasoning",
@ -317,9 +345,9 @@ async def test_encrypted_content_affinity_tracks_and_routes():
# The response must have rewritten the encrypted item's ID to encoded form
encoded_item_id = _extract_encoded_item_id(first_response)
assert encoded_item_id.startswith("encitem_"), (
f"Expected output item ID to be rewritten to encitem_... but got {encoded_item_id!r}"
)
assert encoded_item_id.startswith(
"encitem_"
), f"Expected output item ID to be rewritten to encitem_... but got {encoded_item_id!r}"
# Verify the encoded ID decodes back to the correct deployment + original ID
decoded = ResponsesAPIRequestUtils._decode_encrypted_item_id(encoded_item_id)
@ -341,9 +369,9 @@ async def test_encrypted_content_affinity_tracks_and_routes():
)
second_model_id = second_response._hidden_params["model_id"]
assert second_model_id == first_model_id, (
f"Expected affinity to route to {first_model_id}, but got {second_model_id}"
)
assert (
second_model_id == first_model_id
), f"Expected affinity to route to {first_model_id}, but got {second_model_id}"
@pytest.mark.asyncio
@ -445,9 +473,9 @@ async def test_encrypted_content_affinity_bypasses_rpm_limits():
# Extract encoded item ID from the first response output
encoded_item_id = _extract_encoded_item_id(first_response)
assert encoded_item_id.startswith("encitem_"), (
f"Expected encitem_... but got {encoded_item_id!r}"
)
assert encoded_item_id.startswith(
"encitem_"
), f"Expected encitem_... but got {encoded_item_id!r}"
# Follow-up with the encoded item ID — should pin to same deployment
second_response = await router.aresponses(
@ -592,15 +620,16 @@ async def test_encrypted_content_affinity_with_wrapped_content_no_id():
if hasattr(first_item, "encrypted_content")
else first_item.get("encrypted_content")
)
assert wrapped_content.startswith("litellm_enc:"), (
f"Expected wrapped content but got {wrapped_content[:50]}..."
)
assert wrapped_content.startswith(
"litellm_enc:"
), f"Expected wrapped content but got {wrapped_content[:50]}..."
# Verify we can extract model_id from wrapped content
extracted_model_id, _ = (
ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id(
wrapped_content
)
(
extracted_model_id,
_,
) = ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id(
wrapped_content
)
assert extracted_model_id == first_model_id
@ -616,9 +645,9 @@ async def test_encrypted_content_affinity_with_wrapped_content_no_id():
)
second_model_id = second_response._hidden_params["model_id"]
assert second_model_id == first_model_id, (
f"Expected affinity to route to {first_model_id}, but got {second_model_id}"
)
assert (
second_model_id == first_model_id
), f"Expected affinity to route to {first_model_id}, but got {second_model_id}"
def test_encrypted_content_wrapping_preserves_original_content():
@ -627,19 +656,22 @@ def test_encrypted_content_wrapping_preserves_original_content():
This is critical for streaming responses where content must round-trip correctly.
"""
model_id = "test-deployment-1"
original_encrypted_content = "gAAAAABpnW_yEYmSNEyOG_streaming_test_content_with_special_chars==+/"
original_encrypted_content = (
"gAAAAABpnW_yEYmSNEyOG_streaming_test_content_with_special_chars==+/"
)
wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id(
original_encrypted_content, model_id
)
assert wrapped.startswith("litellm_enc:")
assert wrapped != original_encrypted_content
extracted_model_id, unwrapped_content = (
ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id(wrapped)
)
(
extracted_model_id,
unwrapped_content,
) = ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id(wrapped)
assert extracted_model_id == model_id
assert unwrapped_content == original_encrypted_content
@ -654,15 +686,82 @@ def test_encrypted_content_wrapping_with_multiple_semicolons():
wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id(
original_content, model_id
)
extracted_model_id, unwrapped = (
ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id(wrapped)
)
(
extracted_model_id,
unwrapped,
) = ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id(wrapped)
assert extracted_model_id == model_id
assert unwrapped == original_content
# ---------------------------------------------------------------------------
# Regression tests: affinity check must not break tag-based routing
# ---------------------------------------------------------------------------
from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import (
EncryptedContentAffinityCheck,
)
@pytest.mark.asyncio
async def test_encrypted_content_affinity_does_not_create_litellm_metadata_for_chat():
"""
For chat completions / embeddings, request_kwargs uses 'metadata' (not
'litellm_metadata'). The affinity check must NOT create a spurious
'litellm_metadata' key, because that would cause
_get_metadata_variable_name_from_kwargs to return 'litellm_metadata'
and tag-based routing would look for tags in the wrong dict.
"""
check = EncryptedContentAffinityCheck()
deployments = [
{"model_info": {"id": "dep-1"}, "litellm_params": {"model": "gpt-4"}},
]
request_kwargs = {"metadata": {"tags": ["prod"]}}
result = await check.async_filter_deployments(
model="gpt-4",
healthy_deployments=deployments,
messages=[{"role": "user", "content": "hi"}],
request_kwargs=request_kwargs,
)
# Must not inject litellm_metadata
assert "litellm_metadata" not in request_kwargs
# Tags must be untouched
assert request_kwargs["metadata"]["tags"] == ["prod"]
# All deployments returned (no pinning)
assert len(result) == 1
@pytest.mark.asyncio
async def test_encrypted_content_affinity_preserves_litellm_metadata_for_responses():
"""
For Responses API calls, litellm_metadata already exists. The affinity
check should set the flag there and preserve existing keys.
"""
check = EncryptedContentAffinityCheck()
deployments = [
{"model_info": {"id": "dep-1"}, "litellm_params": {"model": "gpt-5.1-codex"}},
]
request_kwargs = {
"litellm_metadata": {"model_info": {"id": "dep-1"}},
}
await check.async_filter_deployments(
model="gpt-5.1-codex",
healthy_deployments=deployments,
messages=None,
request_kwargs=request_kwargs,
)
assert (
request_kwargs["litellm_metadata"]["encrypted_content_affinity_enabled"] is True
)
assert request_kwargs["litellm_metadata"]["model_info"] == {"id": "dep-1"}
def test_encrypted_content_wrapping_empty_string():
"""
Test that empty encrypted_content is handled gracefully.
@ -673,12 +772,13 @@ def test_encrypted_content_wrapping_empty_string():
wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id(
original_content, model_id
)
assert wrapped.startswith("litellm_enc:")
extracted_model_id, unwrapped = (
ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id(wrapped)
)
(
extracted_model_id,
unwrapped,
) = ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id(wrapped)
assert extracted_model_id == model_id
assert unwrapped == original_content

59
tests/test_litellm/test_cost_calculator.py
View file

@ -67,6 +67,32 @@ def test_cost_calculator_with_response_cost_in_additional_headers():
assert result == 1000
def test_baseten_model_api_pricing_entries():
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
expected_pricing = {
"baseten/nvidia/Nemotron-120B-A12B": (3e-07, 7.5e-07),
"baseten/MiniMaxAI/MiniMax-M2.5": (3e-07, 1.2e-06),
"baseten/zai-org/GLM-5": (9.5e-07, 3.15e-06),
"baseten/zai-org/GLM-4.7": (6e-07, 2.2e-06),
"baseten/zai-org/GLM-4.6": (6e-07, 2.2e-06),
"baseten/moonshotai/Kimi-K2.5": (6e-07, 3e-06),
"baseten/moonshotai/Kimi-K2-Thinking": (6e-07, 2.5e-06),
"baseten/moonshotai/Kimi-K2-Instruct-0905": (6e-07, 2.5e-06),
"baseten/openai/gpt-oss-120b": (1e-07, 5e-07),
"baseten/deepseek-ai/DeepSeek-V3.1": (5e-07, 1.5e-06),
"baseten/deepseek-ai/DeepSeek-V3-0324": (7.7e-07, 7.7e-07),
}
for model_name, (input_cost, output_cost) in expected_pricing.items():
model_info = litellm.model_cost.get(model_name)
assert model_info is not None, f"Missing model pricing entry: {model_name}"
assert model_info["litellm_provider"] == "baseten"
assert model_info["input_cost_per_token"] == input_cost
assert model_info["output_cost_per_token"] == output_cost
def test_cost_calculator_with_usage(monkeypatch):
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
@ -123,6 +149,7 @@ def test_cost_calculator_with_usage(monkeypatch):
# Invalidate caches after modifying litellm.model_cost
from litellm.utils import _invalidate_model_cost_lowercase_map
_invalidate_model_cost_lowercase_map()
result = response_cost_calculator(
@ -528,9 +555,7 @@ def test_azure_audio_output_cost_calculation():
model_info = litellm.get_model_info("azure/gpt-audio-2025-08-28")
# Calculate expected cost
expected_input_cost = (
model_info["input_cost_per_token"] * 17 # text tokens
)
expected_input_cost = model_info["input_cost_per_token"] * 17 # text tokens
expected_output_cost = (
model_info["output_cost_per_token"] * 110 # text tokens
+ model_info["output_cost_per_audio_token"] * 482 # audio tokens
@ -542,14 +567,14 @@ def test_azure_audio_output_cost_calculation():
wrong_total_cost = expected_input_cost + wrong_output_cost
# Verify audio tokens are NOT charged at text rate (the bug)
assert abs(cost - wrong_total_cost) > 0.001, (
"Bug: Audio tokens are being charged at text token rate"
)
assert (
abs(cost - wrong_total_cost) > 0.001
), "Bug: Audio tokens are being charged at text token rate"
# Verify cost matches
assert abs(cost - expected_total_cost) < 0.0000001, (
f"Expected cost {expected_total_cost}, got {cost}"
)
assert (
abs(cost - expected_total_cost) < 0.0000001
), f"Expected cost {expected_total_cost}, got {cost}"
def test_default_image_cost_calculator(monkeypatch):
@ -1056,12 +1081,12 @@ def test_azure_ai_cache_cost_calculation():
print(f"Output cost: {output_cost}, Expected: {expected_output_cost}")
print(f"Total cost: {total_cost}")
assert abs(input_cost - expected_input_cost) < 1e-10, (
f"Input cost mismatch: got {input_cost}, expected {expected_input_cost}"
)
assert abs(output_cost - expected_output_cost) < 1e-10, (
f"Output cost mismatch: got {output_cost}, expected {expected_output_cost}"
)
assert (
abs(input_cost - expected_input_cost) < 1e-10
), f"Input cost mismatch: got {input_cost}, expected {expected_input_cost}"
assert (
abs(output_cost - expected_output_cost) < 1e-10
), f"Output cost mismatch: got {output_cost}, expected {expected_output_cost}"
def test_cost_discount_vertex_ai():
@ -1929,7 +1954,9 @@ def test_gemini_implicit_caching_cost_calculation():
f"Cached tokens may not be using reduced pricing."
)
print("✅ Issue #16341 fix verified: Gemini implicit caching cost calculated correctly")
print(
"✅ Issue #16341 fix verified: Gemini implicit caching cost calculated correctly"
)
def test_additional_costs_only_for_azure_ai():

66
tests/test_litellm/test_logging.py
View file

@ -177,6 +177,72 @@ def test_json_formatter_parses_embedded_python_dict_repr():
assert obj["model_info"]["db_model"] is False
def test_json_formatter_includes_component_field():
"""
Test that JsonFormatter always emits a 'component' field equal to the logger name.
This allows filtering by component (e.g. "LiteLLM Proxy") in Datadog / third-party log services.
"""
formatter = JsonFormatter()
for logger_name in ("LiteLLM Proxy", "LiteLLM Router", "LiteLLM"):
record = logging.LogRecord(
name=logger_name,
level=logging.ERROR,
pathname="proxy_server.py",
lineno=42,
msg="something went wrong",
args=(),
exc_info=None,
)
output = formatter.format(record)
obj = json.loads(output)
assert obj["component"] == logger_name, (
f"Expected component={logger_name!r}, got {obj.get('component')!r}"
)
def test_json_formatter_includes_logger_field():
"""
Test that JsonFormatter always emits a 'logger' field with filename:lineno.
This allows pinpointing the exact source of a log line in third-party services.
"""
formatter = JsonFormatter()
record = logging.LogRecord(
name="LiteLLM Proxy",
level=logging.INFO,
pathname="/app/litellm/proxy/proxy_server.py",
lineno=123,
msg="request received",
args=(),
exc_info=None,
)
output = formatter.format(record)
obj = json.loads(output)
assert obj["logger"] == "proxy_server.py:123", (
f"Expected logger='proxy_server.py:123', got {obj['logger']!r}"
)
def test_json_formatter_extra_component_not_overwritten():
"""
User-supplied extra={"component": "..."} must not be silently dropped.
"""
formatter = JsonFormatter()
record = logging.LogRecord(
name="LiteLLM Proxy",
level=logging.INFO,
pathname="proxy_server.py",
lineno=1,
msg="event",
args=(),
exc_info=None,
)
record.component = "auth-service"
obj = json.loads(formatter.format(record))
assert obj["component"] == "auth-service", (
f"User-supplied component was overwritten, got {obj['component']!r}"
)
def test_initialize_loggers_with_handler_sets_propagate_false():
"""
Test that the initialize_loggers_with_handler function sets propagate to False for all loggers

73
tests/test_litellm/test_router.py
View file

@ -237,6 +237,79 @@ async def test_async_router_acreate_file_with_jsonl():
assert first_call_content == non_jsonl_content
@pytest.mark.asyncio
async def test_async_router_acreate_file_uses_deployment_custom_llm_provider():
"""
Ensure file routing preserves deployment custom_llm_provider instead of
inferring provider from model string alone.
"""
from unittest.mock import MagicMock, patch
router = litellm.Router(
model_list=[
{
"model_name": "team-azure-batch",
"litellm_params": {
"model": "gpt-4.1-mini",
"custom_llm_provider": "azure",
"api_base": "https://example-resource.openai.azure.com",
},
},
],
)
with patch("litellm.acreate_file", return_value=MagicMock()) as mock_acreate_file:
await router.acreate_file(
model="team-azure-batch",
purpose="batch",
file=MagicMock(),
)
assert mock_acreate_file.call_count == 1
assert mock_acreate_file.call_args.kwargs["custom_llm_provider"] == "azure"
@pytest.mark.asyncio
async def test_async_router_afile_content_uses_deployment_custom_llm_provider():
"""
Regression test: Ensure afile_content preserves deployment custom_llm_provider
when model name lacks provider prefix (e.g., "gpt-4.1-mini" instead of "azure/gpt-4.1-mini").
This prevents "None is not a valid LlmProviders" errors when calling file content operations.
"""
from unittest.mock import AsyncMock, MagicMock, patch
from litellm.types.llms.openai import HttpxBinaryResponseContent
router = litellm.Router(
model_list=[
{
"model_name": "team-azure-batch",
"litellm_params": {
"model": "gpt-4.1-mini", # No provider prefix
"custom_llm_provider": "azure",
"api_base": "https://example-resource.openai.azure.com",
"api_key": "test-key",
},
},
],
)
# Mock the Azure file handler's afile_content method
mock_response = MagicMock(spec=HttpxBinaryResponseContent)
mock_response.response = MagicMock()
with patch("litellm.llms.azure.files.handler.AzureOpenAIFilesAPI.afile_content",
return_value=mock_response) as mock_afile_content:
result = await router.afile_content(
model="team-azure-batch",
file_id="file-123",
)
# Verify the call was made (proves custom_llm_provider was correctly passed)
assert mock_afile_content.call_count == 1
assert result == mock_response
@pytest.mark.asyncio
async def test_arouter_async_get_healthy_deployments():
"""

24
ui/litellm-dashboard/e2e_tests/constants.ts
View file

@ -1,6 +1,22 @@
// Storage state paths for each role
export const ADMIN_STORAGE_PATH = "admin.storageState.json";
export const ADMIN_VIEWER_STORAGE_PATH = "adminViewer.storageState.json";
export const INTERNAL_USER_STORAGE_PATH = "internalUser.storageState.json";
export const INTERNAL_VIEWER_STORAGE_PATH = "internalViewer.storageState.json";
export const TEAM_ADMIN_STORAGE_PATH = "teamAdmin.storageState.json";
export const E2E_UPDATE_LIMITS_KEY_ID_PREFIX = "102c";
export const E2E_DELETE_KEY_ID_PREFIX = "94a5";
export const E2E_DELETE_KEY_NAME = "e2eDeleteKey";
export const E2E_REGENERATE_KEY_ID_PREFIX = "593a";
// Key aliases for seeded test keys (match seed.sql)
export const E2E_UPDATE_LIMITS_KEY_ALIAS = "e2eUpdateLimitsKey";
export const E2E_DELETE_KEY_ALIAS = "e2eDeleteKey";
export const E2E_REGENERATE_KEY_ALIAS = "e2eRegenerateKey";
export const E2E_INTERNAL_USER_KEY_ALIAS = "e2eInternalUserKey";
export const E2E_VIEWER_KEY_ALIAS = "e2eViewerKey";
// Team identifiers (match seed.sql)
export const E2E_TEAM_CRUD_ID = "e2e-team-crud";
export const E2E_TEAM_CRUD_ALIAS = "E2E Team CRUD";
export const E2E_TEAM_DELETE_ID = "e2e-team-delete";
export const E2E_TEAM_DELETE_ALIAS = "E2E Team Delete";
export const E2E_TEAM_ORG_ID = "e2e-team-org";
export const E2E_TEAM_NO_ADMIN_ID = "e2e-team-no-admin";
export const E2E_TEAM_NO_ADMIN_ALIAS = "E2E Team No Admin";

16
ui/litellm-dashboard/e2e_tests/fixtures/config.yml Normal file
View file

@ -0,0 +1,16 @@
model_list:
- model_name: fake-openai-gpt-4
litellm_params:
model: openai/fake-gpt-4
api_base: os.environ/MOCK_LLM_URL
api_key: fake-key
- model_name: fake-anthropic-claude
litellm_params:
model: openai/fake-claude
api_base: os.environ/MOCK_LLM_URL
api_key: fake-key
general_settings:
master_key: os.environ/LITELLM_MASTER_KEY
database_url: os.environ/DATABASE_URL
store_prompts_in_spend_logs: true

120
ui/litellm-dashboard/e2e_tests/fixtures/mock_llm_server/server.py Normal file
View file

@ -0,0 +1,120 @@
"""
Mock LLM server for UI e2e tests.
Responds to OpenAI-format endpoints with canned responses.
"""
import time
import json
import uuid
import uvicorn
from fastapi import FastAPI, Request
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import StreamingResponse
app = FastAPI(title="Mock LLM Server")
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_methods=["*"],
allow_headers=["*"],
)
@app.get("/health")
async def health():
return {"status": "ok"}
@app.get("/v1/models")
@app.get("/models")
async def list_models():
return {
"object": "list",
"data": [
{"id": "fake-gpt-4", "object": "model", "owned_by": "mock"},
{"id": "fake-claude", "object": "model", "owned_by": "mock"},
],
}
@app.post("/v1/chat/completions")
@app.post("/chat/completions")
async def chat_completions(request: Request):
body = await request.json()
model = body.get("model", "mock-model")
stream = body.get("stream", False)
response_id = f"chatcmpl-{uuid.uuid4().hex[:12]}"
created = int(time.time())
if stream:
async def stream_generator():
chunk = {
"id": response_id,
"object": "chat.completion.chunk",
"created": created,
"model": model,
"choices": [
{
"index": 0,
"delta": {
"role": "assistant",
"content": "This is a mock response.",
},
"finish_reason": None,
}
],
}
yield f"data: {json.dumps(chunk)}\n\n"
done_chunk = {
"id": response_id,
"object": "chat.completion.chunk",
"created": created,
"model": model,
"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
}
yield f"data: {json.dumps(done_chunk)}\n\n"
yield "data: [DONE]\n\n"
return StreamingResponse(stream_generator(), media_type="text/event-stream")
return {
"id": response_id,
"object": "chat.completion",
"created": created,
"model": model,
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "This is a mock response."},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 10, "completion_tokens": 8, "total_tokens": 18},
}
@app.post("/v1/embeddings")
@app.post("/embeddings")
async def embeddings(request: Request):
body = await request.json()
inputs = body.get("input", [""])
if isinstance(inputs, str):
inputs = [inputs]
return {
"object": "list",
"data": [
{"object": "embedding", "index": i, "embedding": [0.0] * 1536}
for i in range(len(inputs))
],
"model": body.get("model", "mock-embedding"),
"usage": {"prompt_tokens": 5, "total_tokens": 5},
}
if __name__ == "__main__":
uvicorn.run(app, host="127.0.0.1", port=8090)

84
ui/litellm-dashboard/e2e_tests/fixtures/seed.sql Normal file
View file

@ -0,0 +1,84 @@
-- E2E Test Seed Data
-- Idempotent: deletes all e2e-* rows then re-inserts deterministic data.
-- 1. Clean up in dependency order
DELETE FROM "LiteLLM_TeamMembership" WHERE "user_id" LIKE 'e2e-%';
DELETE FROM "LiteLLM_VerificationToken" WHERE token LIKE 'e2e-%';
DELETE FROM "LiteLLM_TeamTable" WHERE "team_id" LIKE 'e2e-%';
DELETE FROM "LiteLLM_OrganizationTable" WHERE "organization_id" LIKE 'e2e-%';
DELETE FROM "LiteLLM_UserTable" WHERE "user_id" LIKE 'e2e-%';
DELETE FROM "LiteLLM_BudgetTable" WHERE "budget_id" LIKE 'e2e-%';
-- 2. Budget (created_by and updated_by are NOT NULL)
INSERT INTO "LiteLLM_BudgetTable" ("budget_id", "max_budget", "created_by", "updated_by")
VALUES ('e2e-budget-org', 1000, 'e2e-proxy-admin', 'e2e-proxy-admin');
-- 3. Organization (created_by and updated_by are NOT NULL)
INSERT INTO "LiteLLM_OrganizationTable" (
"organization_id", "organization_alias", "budget_id",
"metadata", "models", "spend", "model_spend",
"created_by", "updated_by"
) VALUES (
'e2e-org-main', 'E2E Organization', 'e2e-budget-org',
'{}'::jsonb, ARRAY[]::text[], 0.0, '{}'::jsonb,
'e2e-proxy-admin', 'e2e-proxy-admin'
);
-- 4. Users (password hash is scrypt of "test")
INSERT INTO "LiteLLM_UserTable" ("user_id", "user_email", "user_role", "teams", "password")
VALUES
('e2e-proxy-admin', 'admin@test.local', 'proxy_admin', '{"e2e-team-crud"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'),
('e2e-admin-viewer', 'adminviewer@test.local', 'proxy_admin_viewer', '{}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'),
('e2e-internal-user', 'internal@test.local', 'internal_user', '{"e2e-team-crud","e2e-team-org"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'),
('e2e-internal-viewer', 'viewer@test.local', 'internal_user_viewer', '{"e2e-team-crud"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'),
('e2e-team-admin', 'teamadmin@test.local', 'internal_user', '{"e2e-team-crud","e2e-team-delete"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'),
('e2e-invitable-user', 'invitable@test.local', 'internal_user', '{}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'),
('e2e-removable-member', 'removable@test.local', 'internal_user', '{"e2e-team-crud"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr');
-- 5. Teams (members_with_roles is required JSON)
INSERT INTO "LiteLLM_TeamTable" (
"team_id", "team_alias", "organization_id", "admins", "members",
"members_with_roles", "metadata", "models", "spend", "model_spend", "model_max_budget", "blocked"
) VALUES
('e2e-team-crud', 'E2E Team CRUD', NULL,
'{"e2e-team-admin"}',
'{"e2e-team-admin","e2e-internal-user","e2e-internal-viewer","e2e-removable-member"}',
'[{"role":"admin","user_id":"e2e-team-admin"},{"role":"user","user_id":"e2e-internal-user"},{"role":"user","user_id":"e2e-internal-viewer"},{"role":"user","user_id":"e2e-removable-member"}]'::jsonb,
'{}'::jsonb, '{"fake-openai-gpt-4","fake-anthropic-claude"}', 0.0, '{}'::jsonb, '{}'::jsonb, false),
('e2e-team-delete', 'E2E Team Delete', NULL,
'{"e2e-team-admin"}', '{"e2e-team-admin"}',
'[{"role":"admin","user_id":"e2e-team-admin"}]'::jsonb,
'{}'::jsonb, '{"fake-openai-gpt-4"}', 0.0, '{}'::jsonb, '{}'::jsonb, false),
('e2e-team-org', 'E2E Team In Org', 'e2e-org-main',
'{}', '{"e2e-internal-user"}',
'[{"role":"user","user_id":"e2e-internal-user"}]'::jsonb,
'{}'::jsonb, '{"fake-openai-gpt-4"}', 0.0, '{}'::jsonb, '{}'::jsonb, false),
('e2e-team-no-admin', 'E2E Team No Admin', NULL,
'{}', '{"e2e-invitable-user"}',
'[{"role":"user","user_id":"e2e-invitable-user"}]'::jsonb,
'{}'::jsonb, '{"fake-openai-gpt-4"}', 0.0, '{}'::jsonb, '{}'::jsonb, false);
-- 6. Team Memberships (only user_id, team_id, spend — no created_at/updated_at)
INSERT INTO "LiteLLM_TeamMembership" ("user_id", "team_id", "spend")
VALUES
('e2e-team-admin', 'e2e-team-crud', 0.0),
('e2e-internal-user', 'e2e-team-crud', 0.0),
('e2e-internal-viewer', 'e2e-team-crud', 0.0),
('e2e-removable-member', 'e2e-team-crud', 0.0),
('e2e-team-admin', 'e2e-team-delete', 0.0),
('e2e-internal-user', 'e2e-team-org', 0.0),
('e2e-invitable-user', 'e2e-team-no-admin', 0.0);
-- 7. Verification Tokens (API Keys)
INSERT INTO "LiteLLM_VerificationToken" (
"token", "key_name", "key_alias", "user_id", "team_id",
"models", "spend", "max_budget", "expires", "metadata"
) VALUES
('e2e-key-update-limits', 'sk-e2e-update', 'e2eUpdateLimitsKey', 'e2e-proxy-admin', 'e2e-team-crud', '{"fake-openai-gpt-4"}', 0.0, NULL, NULL, '{}'::jsonb),
('e2e-key-delete', 'sk-e2e-delete', 'e2eDeleteKey', 'e2e-proxy-admin', 'e2e-team-crud', '{"fake-openai-gpt-4"}', 0.0, NULL, NULL, '{}'::jsonb),
('e2e-key-regenerate', 'sk-e2e-regen', 'e2eRegenerateKey', 'e2e-proxy-admin', 'e2e-team-crud', '{"fake-openai-gpt-4"}', 0.0, NULL, NULL, '{}'::jsonb),
('e2e-key-internal-user', 'sk-e2e-internal', 'e2eInternalUserKey', 'e2e-internal-user', 'e2e-team-crud', '{"fake-openai-gpt-4"}', 0.0, NULL, NULL, '{}'::jsonb),
('e2e-key-viewer', 'sk-e2e-viewer', 'e2eViewerKey', 'e2e-internal-viewer', NULL, '{"fake-openai-gpt-4"}', 0.0, NULL, NULL, '{}'::jsonb);

38
ui/litellm-dashboard/e2e_tests/fixtures/users.ts
View file

@ -1,10 +1,38 @@
import { Role } from "./roles";
export enum Role {
ProxyAdmin = "proxy_admin",
ProxyAdminViewer = "proxy_admin_viewer",
InternalUser = "internal_user",
InternalUserViewer = "internal_user_viewer",
TeamAdmin = "team_admin",
}
const isCI = !!process.env.CI;
export const users = {
export const users: Record<Role, { email: string; password: string }> = {
[Role.ProxyAdmin]: {
email: "admin",
password: isCI ? "gm" : "sk-1234",
password: process.env.LITELLM_MASTER_KEY || "sk-1234",
},
[Role.ProxyAdminViewer]: {
email: "adminviewer@test.local",
password: "test",
},
[Role.InternalUser]: {
email: "internal@test.local",
password: "test",
},
[Role.InternalUserViewer]: {
email: "viewer@test.local",
password: "test",
},
[Role.TeamAdmin]: {
email: "teamadmin@test.local",
password: "test",
},
};
export const STORAGE_PATHS: Record<Role, string> = {
[Role.ProxyAdmin]: "admin.storageState.json",
[Role.ProxyAdminViewer]: "adminViewer.storageState.json",
[Role.InternalUser]: "internalUser.storageState.json",
[Role.InternalUserViewer]: "internalViewer.storageState.json",
[Role.TeamAdmin]: "teamAdmin.storageState.json",
};

45
ui/litellm-dashboard/e2e_tests/globalSetup.ts
View file

@ -1,17 +1,40 @@
import { chromium } from "@playwright/test";
import { users } from "./fixtures/users";
import { Role } from "./fixtures/roles";
import { chromium, expect } from "@playwright/test";
import { users, Role, STORAGE_PATHS } from "./fixtures/users";
import * as fs from "fs";
async function globalSetup() {
const browser = await chromium.launch();
const page = await browser.newPage();
await page.goto("http://localhost:4000/ui/login");
await page.getByPlaceholder("Enter your username").fill(users[Role.ProxyAdmin].email);
await page.getByPlaceholder("Enter your password").fill(users[Role.ProxyAdmin].password);
const loginButton = page.getByRole("button", { name: "Login", exact: true });
await loginButton.click();
await page.waitForSelector("text=Virtual Keys");
await page.context().storageState({ path: "admin.storageState.json" });
for (const role of Object.values(Role)) {
const { email, password } = users[role];
const storagePath = STORAGE_PATHS[role];
const page = await browser.newPage();
try {
await page.goto("http://localhost:4000/ui/login");
await page.getByPlaceholder("Enter your username").fill(email);
await page.getByPlaceholder("Enter your password").fill(password);
await page.getByRole("button", { name: "Login", exact: true }).click();
await page.waitForURL(
(url) => url.pathname.startsWith("/ui") && !url.pathname.includes("/login"),
{ timeout: 30_000 },
);
await expect(page.locator("a", { hasText: "Virtual Keys" })).toBeVisible({ timeout: 30_000 });
// Dismiss feedback popup if present
const dismiss = page.getByText("Don't ask me again");
if (await dismiss.isVisible({ timeout: 1_500 }).catch(() => false)) {
await dismiss.click();
}
await page.context().storageState({ path: storagePath });
} catch (e) {
fs.mkdirSync("test-results", { recursive: true });
await page.screenshot({ path: `test-results/global-setup-${role}-failure.png`, fullPage: true });
console.error(`Global setup failed for role ${role}. Screenshot saved. URL: ${page.url()}`);
throw e;
} finally {
await page.close();
}
}
await browser.close();
}

21
ui/litellm-dashboard/e2e_tests/helpers/navigation.ts
View file

@ -1,12 +1,25 @@
import { Page } from "../fixtures/pages";
import { Page as PlaywrightPage } from "@playwright/test";
import { Page as PlaywrightPage, expect } from "@playwright/test";
/**
* Navigates to a specific page using the page query parameter.
* Uses relative path which will be resolved against the baseURL configured in playwright.config.ts
* @param page - The Playwright page object
* @param pageEnum - The page enum value to navigate to
* Waits for the sidebar to be visible before returning.
*/
export async function navigateToPage(page: PlaywrightPage, pageEnum: Page): Promise<void> {
await page.goto(`/ui?page=${pageEnum}`);
await page.waitForLoadState("networkidle");
// Dismiss the "Quick feedback" popup if it appears
await dismissFeedbackPopup(page);
}
/**
* Dismiss the "Quick feedback" popup that may appear on any page.
*/
export async function dismissFeedbackPopup(page: PlaywrightPage): Promise<void> {
const dismissButton = page.getByText("Don't ask me again");
if (await dismissButton.isVisible({ timeout: 1_500 }).catch(() => false)) {
await dismissButton.click();
// Wait for the popup to disappear
await expect(dismissButton).not.toBeVisible({ timeout: 2_000 }).catch(() => {});
}
}

5
ui/litellm-dashboard/e2e_tests/playwright.config.ts
View file

@ -36,11 +36,6 @@ export default defineConfig({
name: "chromium",
use: { ...devices["Desktop Chrome"] },
},
{
name: "firefox",
use: { ...devices["Desktop Firefox"] },
},
],
/* Timeout settings */

189
ui/litellm-dashboard/e2e_tests/run_e2e.sh Executable file
View file

@ -0,0 +1,189 @@
#!/usr/bin/env bash
set -euo pipefail
# ================================================================
# UI E2E Test Runner (Consolidated)
# Starts postgres, seeds DB, starts mock + proxy, runs Playwright.
# All tests target the proxy on port 4000 (which serves both API
# and UI from the built Next.js static export).
#
# Usage:
# ./run_e2e.sh # Run once
# ./run_e2e.sh --repeat-each=5 # Run each test 5 times
# ./run_e2e.sh --headed # Run with browser visible
#
# In CI (CI=true), expects:
# - PostgreSQL already running on 127.0.0.1:5432
# - DATABASE_URL already set
# - Python/Poetry already installed
# - Node.js/npx already available
# ================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
DASHBOARD_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
IS_CI="${CI:-false}"
CONTAINER_NAME="litellm-e2e-postgres-$$"
MOCK_PID=""
PROXY_PID=""
# --- Ensure common tool paths are available (local dev only) ---
if [ "$IS_CI" = "false" ]; then
for p in /usr/local/bin /opt/homebrew/bin "$HOME/.local/bin" /opt/homebrew/opt/postgresql@14/bin /opt/homebrew/opt/libpq/bin; do
[ -d "$p" ] && export PATH="$p:$PATH"
done
[ -s "$HOME/.nvm/nvm.sh" ] && source "$HOME/.nvm/nvm.sh"
fi
# --- Cleanup on exit ---
cleanup() {
echo "Cleaning up..."
[ -n "$MOCK_PID" ] && kill "$MOCK_PID" 2>/dev/null || true
[ -n "$PROXY_PID" ] && kill "$PROXY_PID" 2>/dev/null || true
if [ "$IS_CI" = "false" ]; then
docker stop "$CONTAINER_NAME" 2>/dev/null || true
fi
echo "Done."
}
trap cleanup EXIT INT TERM
# --- Pre-flight checks ---
for cmd in python3 npx poetry; do
command -v "$cmd" >/dev/null 2>&1 || { echo "Error: $cmd not found."; exit 1; }
done
# --- Database setup ---
if [ "$IS_CI" = "false" ]; then
for cmd in docker psql; do
command -v "$cmd" >/dev/null 2>&1 || { echo "Error: $cmd not found."; exit 1; }
done
for port in 4000 5432 8090; do
if lsof -ti ":$port" >/dev/null 2>&1; then
echo "Error: port $port is in use"
exit 1
fi
done
export POSTGRES_USER="e2euser"
export POSTGRES_PASSWORD="$(openssl rand -hex 32)"
export POSTGRES_DB="litellm_e2e"
export DATABASE_URL="postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@127.0.0.1:5432/${POSTGRES_DB}"
echo "=== Starting PostgreSQL ==="
docker run -d --rm --name "$CONTAINER_NAME" \
-e POSTGRES_USER -e POSTGRES_PASSWORD -e POSTGRES_DB \
-p 127.0.0.1:5432:5432 \
postgres:16
echo "Waiting for PostgreSQL..."
for i in $(seq 1 30); do
if PGPASSWORD="$POSTGRES_PASSWORD" pg_isready -h 127.0.0.1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" >/dev/null 2>&1; then
break
fi
sleep 1
done
else
echo "=== Using CI PostgreSQL service ==="
: "${DATABASE_URL:?DATABASE_URL must be set in CI}"
fi
# --- Credentials ---
export LITELLM_MASTER_KEY="sk-1234"
export MOCK_LLM_URL="http://127.0.0.1:8090/v1"
export DISABLE_SCHEMA_UPDATE="true"
# Ensure the proxy serves UI at /ui (not behind a subpath)
export SERVER_ROOT_PATH=""
# Prevent logout from redirecting to an external URL
export PROXY_LOGOUT_URL=""
# --- Rebuild UI from source ---
echo "=== Building UI from source ==="
cd "$DASHBOARD_DIR"
npm install --silent 2>/dev/null || true
npm run build
# Copy the fresh build to the proxy's static UI directory
cp -r "$DASHBOARD_DIR/out/" "$REPO_ROOT/litellm/proxy/_experimental/out/"
# Restructure HTML files so extensionless routes work (e.g. /ui/login)
# Next.js export produces login.html; the proxy expects login/index.html
find "$REPO_ROOT/litellm/proxy/_experimental/out" -name '*.html' ! -name 'index.html' | while read -r htmlfile; do
target_dir="${htmlfile%.html}"
target_path="$target_dir/index.html"
mkdir -p "$target_dir"
mv "$htmlfile" "$target_path"
done
echo "UI build copied and restructured"
# --- Python environment ---
echo "=== Setting up Python environment ==="
cd "$REPO_ROOT"
if ! poetry run python3 -c "import prisma" 2>/dev/null; then
echo "Installing Python dependencies (first run)..."
poetry install --with dev,proxy-dev --extras "proxy" --quiet
poetry run pip install nodejs-wheel-binaries 2>/dev/null || true
poetry run prisma generate --schema litellm/proxy/schema.prisma
fi
echo "=== Pushing Prisma schema to database ==="
poetry run prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
# --- Mock LLM server ---
echo "=== Starting mock LLM server ==="
poetry run python3 "$SCRIPT_DIR/fixtures/mock_llm_server/server.py" &
MOCK_PID=$!
for i in $(seq 1 15); do
if curl -sf http://127.0.0.1:8090/health >/dev/null 2>&1; then break; fi
sleep 1
done
# --- LiteLLM proxy ---
echo "=== Starting LiteLLM proxy ==="
cd "$REPO_ROOT"
poetry run python3 -m litellm.proxy.proxy_cli \
--config "$SCRIPT_DIR/fixtures/config.yml" \
--port 4000 &
PROXY_PID=$!
echo "Waiting for proxy..."
PROXY_READY=0
for i in $(seq 1 180); do
if ! kill -0 "$PROXY_PID" 2>/dev/null; then
echo "Error: proxy process exited unexpectedly"
exit 1
fi
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:4000/health -H "Authorization: Bearer $LITELLM_MASTER_KEY" 2>/dev/null || true)
if [ "$HTTP_CODE" = "200" ]; then
PROXY_READY=1
break
fi
sleep 1
done
if [ "$PROXY_READY" -ne 1 ]; then
echo "Error: proxy did not become healthy within 180 seconds"
exit 1
fi
echo "Proxy is ready."
# --- Seed database ---
echo "=== Seeding database ==="
DB_USER=$(echo "$DATABASE_URL" | sed -n 's|.*://\([^:]*\):.*|\1|p')
DB_PASS=$(echo "$DATABASE_URL" | sed -n 's|.*://[^:]*:\([^@]*\)@.*|\1|p')
DB_HOST=$(echo "$DATABASE_URL" | sed -n 's|.*@\([^:]*\):.*|\1|p')
DB_PORT=$(echo "$DATABASE_URL" | sed -n 's|.*:\([0-9]*\)/.*|\1|p')
DB_NAME=$(echo "$DATABASE_URL" | sed -n 's|.*/\([^?]*\).*|\1|p')
PGPASSWORD="$DB_PASS" psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" \
-f "$SCRIPT_DIR/fixtures/seed.sql"
# --- Playwright ---
echo "=== Installing Playwright dependencies ==="
cd "$DASHBOARD_DIR"
npm install --silent 2>/dev/null || true
npx playwright install chromium --with-deps 2>/dev/null || npx playwright install chromium
echo "=== Running Playwright tests ==="
npx playwright test --config e2e_tests/playwright.config.ts "$@"
EXIT_CODE=$?
exit $EXIT_CODE

22
ui/litellm-dashboard/e2e_tests/tests/keys/createKey.spec.ts
View file

@ -1,22 +0,0 @@
import { test, expect } from "@playwright/test";
import { ADMIN_STORAGE_PATH } from "../../constants";
import { Page } from "../../fixtures/pages";
import { navigateToPage } from "../../helpers/navigation";
test.describe("Create Key", () => {
test.use({ storageState: ADMIN_STORAGE_PATH });
test("Able to create a key with all team models", async ({ page }) => {
await navigateToPage(page, Page.ApiKeys);
await expect(page.getByRole("button", { name: "Next" })).toBeVisible();
await page.getByRole("button", { name: "+ Create New Key" }).click();
await page.getByTestId("base-input").click();
await page.getByTestId("base-input").fill("e2eUITestingCreateKeyAllTeamModels");
await page.locator(".ant-select-selection-overflow").click();
await page.getByText("All Team Models").click();
await page.getByRole("combobox", { name: /models/i }).press("Escape");
await page.getByRole("button", { name: "Create Key" }).click();
await page.keyboard.press("Escape");
await expect(page.getByText("e2eUITestingCreateKeyAllTeamModels")).toBeVisible();
});
});

25
ui/litellm-dashboard/e2e_tests/tests/keys/deleteKey.spec.ts
View file

@ -1,25 +0,0 @@
import { test, expect } from "@playwright/test";
import { ADMIN_STORAGE_PATH, E2E_DELETE_KEY_ID_PREFIX, E2E_DELETE_KEY_NAME } from "../../constants";
import { Page } from "../../fixtures/pages";
import { navigateToPage } from "../../helpers/navigation";
test.describe("Delete Key", () => {
test.use({ storageState: ADMIN_STORAGE_PATH });
test("Able to delete a key", async ({ page }) => {
await navigateToPage(page, Page.ApiKeys);
await expect(page.getByRole("button", { name: "Next" })).toBeVisible();
await page
.locator("button", {
hasText: E2E_DELETE_KEY_ID_PREFIX,
})
.click();
await page.getByRole("button", { name: "Delete Key" }).click();
await page.getByRole("textbox", { name: E2E_DELETE_KEY_NAME }).click();
await page.getByRole("textbox", { name: E2E_DELETE_KEY_NAME }).fill(E2E_DELETE_KEY_NAME);
const deleteButton = page.getByRole("button", { name: "Delete", exact: true });
await expect(deleteButton).toBeEnabled();
await deleteButton.click();
await expect(page.getByText("Key deleted successfully")).toBeVisible();
});
});

21
ui/litellm-dashboard/e2e_tests/tests/keys/regenerateKey.spec.ts
View file

@ -1,21 +0,0 @@
import { test, expect } from "@playwright/test";
import { ADMIN_STORAGE_PATH, E2E_REGENERATE_KEY_ID_PREFIX } from "../../constants";
import { Page } from "../../fixtures/pages";
import { navigateToPage } from "../../helpers/navigation";
test.describe("Regenerate Key", () => {
test.use({ storageState: ADMIN_STORAGE_PATH });
test("Able to regenerate a key", async ({ page }) => {
await navigateToPage(page, Page.ApiKeys);
await expect(page.getByRole("button", { name: "Next" })).toBeVisible();
await page
.locator("button", {
hasText: E2E_REGENERATE_KEY_ID_PREFIX,
})
.click();
await page.getByRole("button", { name: "Regenerate Key" }).click();
await page.getByRole("button", { name: "Regenerate", exact: true }).click();
await expect(page.getByText("Virtual Key regenerated")).toBeVisible();
});
});

27
ui/litellm-dashboard/e2e_tests/tests/keys/updateKeyLimits.spec.ts
View file

@ -1,27 +0,0 @@
import { test, expect } from "@playwright/test";
import { ADMIN_STORAGE_PATH, E2E_UPDATE_LIMITS_KEY_ID_PREFIX } from "../../constants";
import { Page } from "../../fixtures/pages";
import { navigateToPage } from "../../helpers/navigation";
test.describe("Update Key TPM and RPM Limits", () => {
test.use({ storageState: ADMIN_STORAGE_PATH });
test("Able to update a key's TPM and RPM limits", async ({ page }) => {
await navigateToPage(page, Page.ApiKeys);
await expect(page.getByRole("button", { name: "Next" })).toBeVisible();
await page
.locator("button", {
hasText: E2E_UPDATE_LIMITS_KEY_ID_PREFIX,
})
.click();
await page.getByRole("tab", { name: "Settings" }).click();
await page.getByRole("button", { name: "Edit Settings" }).click();
await page.getByRole("spinbutton", { name: "TPM Limit" }).click();
await page.getByRole("spinbutton", { name: "TPM Limit" }).fill("123");
await page.getByRole("spinbutton", { name: "RPM Limit" }).click();
await page.getByRole("spinbutton", { name: "RPM Limit" }).fill("456");
await page.getByRole("button", { name: "Save Changes" }).click();
await expect(page.getByRole("paragraph").filter({ hasText: "TPM: 123" })).toBeVisible();
await expect(page.getByRole("paragraph").filter({ hasText: "RPM: 456" })).toBeVisible();
});
});

124
ui/litellm-dashboard/e2e_tests/tests/proxy-admin/keys.spec.ts Normal file
View file

@ -0,0 +1,124 @@
import { test, expect } from "@playwright/test";
import {
ADMIN_STORAGE_PATH,
E2E_DELETE_KEY_ALIAS,
E2E_REGENERATE_KEY_ALIAS,
E2E_UPDATE_LIMITS_KEY_ALIAS,
E2E_INTERNAL_USER_KEY_ALIAS,
E2E_TEAM_CRUD_ALIAS,
} from "../../constants";
import { Page } from "../../fixtures/pages";
import { navigateToPage, dismissFeedbackPopup } from "../../helpers/navigation";
test.describe("Proxy Admin - Keys", () => {
test.use({ storageState: ADMIN_STORAGE_PATH });
test("Create a key in a team", async ({ page }) => {
await navigateToPage(page, Page.ApiKeys);
await dismissFeedbackPopup(page);
// Click "+ Create New Key" button
await page.getByRole("button", { name: /Create New Key/i }).click();
// Wait for the key creation modal
await expect(page.getByText("Key Ownership")).toBeVisible({ timeout: 10_000 });
// Fill key name (has data-testid="base-input" in the built UI)
const keyName = `e2e-admin-key-${Date.now()}`;
await page.getByTestId("base-input").fill(keyName);
// Select team — the team dropdown has placeholder "Search or select a team"
const teamSelect = page.locator(".ant-select", { hasText: "Search or select a team" });
await teamSelect.click();
await page.keyboard.type(E2E_TEAM_CRUD_ALIAS);
await page.locator(".ant-select-dropdown:visible").getByText(E2E_TEAM_CRUD_ALIAS).first().click();
// Select models
await page.locator(".ant-select-selection-overflow").click();
await page.locator(".ant-select-dropdown:visible").getByText("All Team Models").click();
await page.keyboard.press("Escape");
// Submit
await page.getByRole("button", { name: "Create Key", exact: true }).click();
// Success shows "Save your Key" in a second dialog
await expect(page.getByText("Save your Key")).toBeVisible({ timeout: 10_000 });
await page.keyboard.press("Escape");
// Verify the new key appears in the table
await expect(page.getByText(keyName)).toBeVisible({ timeout: 10_000 });
});
test("Regenerate key", async ({ page }) => {
await navigateToPage(page, Page.ApiKeys);
await dismissFeedbackPopup(page);
// Key IDs are rendered as buttons in the table
const keyRow = page.locator("tr", { hasText: E2E_REGENERATE_KEY_ALIAS });
await expect(keyRow).toBeVisible({ timeout: 10_000 });
await keyRow.locator("button").first().click();
await expect(page.getByText("Back to Keys")).toBeVisible({ timeout: 10_000 });
await page.getByRole("button", { name: "Regenerate Key" }).click();
await page.getByRole("button", { name: "Regenerate", exact: true }).click();
// Success shows "Copy Virtual Key" button in the regenerated key dialog
await expect(page.getByText("Copy Virtual Key")).toBeVisible({ timeout: 10_000 });
});
test("Update key TPM and RPM limits", async ({ page }) => {
await navigateToPage(page, Page.ApiKeys);
await dismissFeedbackPopup(page);
const keyRow = page.locator("tr", { hasText: E2E_UPDATE_LIMITS_KEY_ALIAS });
await expect(keyRow).toBeVisible({ timeout: 10_000 });
await keyRow.locator("button").first().click();
await expect(page.getByText("Back to Keys")).toBeVisible({ timeout: 10_000 });
await page.getByRole("tab", { name: "Settings" }).click();
await page.getByRole("button", { name: "Edit Settings" }).click();
await page.getByRole("spinbutton", { name: "TPM Limit" }).fill("123");
await page.getByRole("spinbutton", { name: "RPM Limit" }).fill("456");
await page.getByRole("button", { name: "Save Changes" }).click();
await expect(
page.getByRole("paragraph").filter({ hasText: "TPM: 123" })
).toBeVisible({ timeout: 10_000 });
await expect(
page.getByRole("paragraph").filter({ hasText: "RPM: 456" })
).toBeVisible({ timeout: 10_000 });
});
test("Delete key", async ({ page }) => {
await navigateToPage(page, Page.ApiKeys);
await dismissFeedbackPopup(page);
const keyRow = page.locator("tr", { hasText: E2E_DELETE_KEY_ALIAS });
await expect(keyRow).toBeVisible({ timeout: 10_000 });
await keyRow.locator("button").first().click();
await expect(page.getByText("Back to Keys")).toBeVisible({ timeout: 10_000 });
await page.getByRole("button", { name: "Delete Key" }).click();
const modal = page.locator(".ant-modal:visible");
await expect(modal).toBeVisible({ timeout: 5_000 });
await modal.locator("input").fill(E2E_DELETE_KEY_ALIAS);
const deleteButton = modal.getByRole("button", { name: "Delete", exact: true });
await expect(deleteButton).toBeEnabled();
await deleteButton.click();
await expect(page.getByText(/Key deleted/i).first()).toBeVisible({ timeout: 10_000 });
});
test("See internal user keys in team", async ({ page }) => {
await navigateToPage(page, Page.ApiKeys);
await dismissFeedbackPopup(page);
await expect(page.getByText(E2E_INTERNAL_USER_KEY_ALIAS)).toBeVisible({ timeout: 10_000 });
});
});

134
ui/litellm-dashboard/e2e_tests/tests/proxy-admin/teams.spec.ts Normal file
View file

@ -0,0 +1,134 @@
import { test, expect } from "@playwright/test";
import {
ADMIN_STORAGE_PATH,
E2E_TEAM_CRUD_ID,
E2E_TEAM_DELETE_ALIAS,
E2E_TEAM_NO_ADMIN_ID,
E2E_TEAM_ORG_ID,
} from "../../constants";
import { Page } from "../../fixtures/pages";
import { navigateToPage, dismissFeedbackPopup } from "../../helpers/navigation";
/**
* Click on a team ID in the table. Team IDs are rendered differently depending
* on the component version — try button first (Tremor Button), fall back to
* clickable span (OldTeams Typography.Text).
*/
async function clickTeamId(page: import("@playwright/test").Page, teamId: string) {
const cell = page.locator("td").filter({ hasText: teamId }).first();
await expect(cell).toBeVisible({ timeout: 10_000 });
await cell.click();
await expect(page.getByText("Back to Teams")).toBeVisible({ timeout: 10_000 });
}
test.describe("Proxy Admin - Teams", () => {
test.use({ storageState: ADMIN_STORAGE_PATH });
test("Create a team", async ({ page }) => {
await navigateToPage(page, Page.Teams);
await dismissFeedbackPopup(page);
const uniqueAlias = `e2e-created-team-${Date.now()}`;
// Click the Create Team button — accessible name includes "Create Team"
await page.getByRole("button", { name: /Create Team/i }).first().click();
// Wait for the Create Team modal
const dialog = page.locator(".ant-modal:visible");
await expect(dialog).toBeVisible({ timeout: 5_000 });
// Fill Team Name — the input has id="team_alias"
await dialog.locator("#team_alias").fill(uniqueAlias);
// Select models — the models multi-select is inside the modal
// Click to open dropdown, select "All Proxy Models"
await dialog.locator(".ant-select-selection-overflow").first().click();
await page.locator(".ant-select-dropdown:visible").getByText("All Proxy Models").click();
await page.keyboard.press("Escape");
// Submit — click the submit button inside the dialog (not the header button)
await dialog.locator("button[type='submit']").click();
// Verify success notification
await expect(page.getByText("Team created").first()).toBeVisible({ timeout: 10_000 });
});
test("Invite a user to a team", async ({ page }) => {
await navigateToPage(page, Page.Teams);
await dismissFeedbackPopup(page);
await clickTeamId(page, E2E_TEAM_CRUD_ID);
await page.getByRole("tab", { name: "Members" }).click();
await page.getByRole("button", { name: /Add Member/i }).click();
// Wait for Add Team Member modal
const modal = page.locator(".ant-modal:visible");
await expect(modal).toBeVisible({ timeout: 5_000 });
// The email field is a Select — type to search, then select from dropdown
await modal.locator(".ant-select").first().click();
await page.keyboard.type("invitable@test.local");
// Wait for the option to appear, then select via keyboard (avoids viewport issues)
const emailOption = page.getByRole("option", { name: "invitable@test.local" }).first();
await expect(emailOption).toBeAttached({ timeout: 10_000 });
// Use keyboard to select the highlighted option
await page.keyboard.press("Enter");
// Submit
await modal.getByRole("button", { name: /Add Member/i }).click();
await expect(page.getByText(/member.*added|success/i).first()).toBeVisible({ timeout: 10_000 });
});
test("Edit team member for team proxy admin does not belong to", async ({ page }) => {
await navigateToPage(page, Page.Teams);
await dismissFeedbackPopup(page);
await clickTeamId(page, E2E_TEAM_NO_ADMIN_ID);
await page.getByRole("tab", { name: "Members" }).click();
await page.getByTestId("edit-member").first().click();
const modal = page.locator(".ant-modal:visible");
await expect(modal).toBeVisible({ timeout: 5_000 });
await modal.getByRole("button", { name: /Save Changes/i }).click();
await expect(page.getByText(/updated|success/i).first()).toBeVisible({ timeout: 10_000 });
});
test("Delete a team", async ({ page }) => {
await navigateToPage(page, Page.Teams);
await dismissFeedbackPopup(page);
const teamRow = page.locator("tr", { hasText: E2E_TEAM_DELETE_ALIAS }).first();
await expect(teamRow).toBeVisible({ timeout: 10_000 });
await teamRow.locator("svg, img").last().click();
const modal = page.locator(".ant-modal:visible");
await expect(modal).toBeVisible({ timeout: 5_000 });
await modal.locator("input").fill(E2E_TEAM_DELETE_ALIAS);
await modal.getByRole("button", { name: /Force Delete|Delete/i }).click();
await expect(teamRow).not.toBeVisible({ timeout: 10_000 });
});
test("Team in org - edit team member", async ({ page }) => {
await navigateToPage(page, Page.Teams);
await dismissFeedbackPopup(page);
await clickTeamId(page, E2E_TEAM_ORG_ID);
await page.getByRole("tab", { name: "Members" }).click();
await page.getByTestId("edit-member").first().click();
const modal = page.locator(".ant-modal:visible");
await expect(modal).toBeVisible({ timeout: 5_000 });
await modal.getByRole("button", { name: /Save Changes/i }).click();
await expect(page.getByText(/updated|success/i).first()).toBeVisible({ timeout: 10_000 });
});
});

1
ui/litellm-dashboard/src/app/(dashboard)/teams/components/TeamsTable/TeamsTable.tsx
View file

@ -80,6 +80,7 @@ const TeamsTable = ({
size="xs"
variant="light"
className="font-mono text-blue-500 bg-blue-50 hover:bg-blue-100 text-xs font-normal px-2 py-0.5 text-left overflow-hidden truncate max-w-[200px]"
data-testid="team-id-cell"
onClick={() => {
// Add click handler
setSelectedTeamId(team.team_id);

6
ui/litellm-dashboard/src/app/(dashboard)/teams/components/modals/CreateTeamModal.tsx
View file

@ -312,7 +312,7 @@ const CreateTeamModal = ({
},
]}
>
<TextInput placeholder="" />
<TextInput placeholder="" data-testid="team-name-input" />
</Form.Item>
<Form.Item
label={
@ -379,7 +379,7 @@ const CreateTeamModal = ({
}
name="models"
>
<Select2 mode="multiple" placeholder="Select models" style={{ width: "100%" }}>
<Select2 mode="multiple" placeholder="Select models" style={{ width: "100%" }} data-testid="team-models-select">
<Select2.Option key="all-proxy-models" value="all-proxy-models">
All Proxy Models
</Select2.Option>
@ -716,7 +716,7 @@ const CreateTeamModal = ({
</Accordion>
</>
<div style={{ textAlign: "right", marginTop: "10px" }}>
<Button2 htmlType="submit">Create Team</Button2>
<Button2 htmlType="submit" data-testid="create-team-submit">Create Team</Button2>
</div>
</Form>
</Modal>

8
ui/litellm-dashboard/src/components/OldTeams.tsx
View file

@ -695,6 +695,7 @@ const Teams: React.FC<TeamProps> = ({
className="text-blue-500 bg-blue-50 hover:bg-blue-100 text-xs cursor-pointer"
style={{ fontSize: 14, padding: "1px 8px" }}
onClick={() => setSelectedTeamId(record.team_id)}
data-testid="team-id-cell"
>
{id}
</Text>
@ -898,6 +899,7 @@ const Teams: React.FC<TeamProps> = ({
icon={<PlusOutlined />}
onClick={() => setIsTeamModalVisible(true)}
style={{ marginTop: 16 }}
data-testid="create-team-button"
>
Create Team
</Button>
@ -1041,7 +1043,7 @@ const Teams: React.FC<TeamProps> = ({
</Text>
</Space>
{canCreateOrManageTeams(userRole, userID, organizations) && (
<Button type="primary" icon={<PlusOutlined />} onClick={() => setIsTeamModalVisible(true)}>
<Button type="primary" icon={<PlusOutlined />} onClick={() => setIsTeamModalVisible(true)} data-testid="create-team-button">
Create Team
</Button>
)}
@ -1078,7 +1080,7 @@ const Teams: React.FC<TeamProps> = ({
},
]}
>
<TextInput placeholder="" />
<TextInput placeholder="" data-testid="team-name-input" />
</Form.Item>
{(() => {
const adminOrgs = getAdminOrganizations(userRole, userID, organizations);
@ -1567,7 +1569,7 @@ const Teams: React.FC<TeamProps> = ({
</Accordion>
</>
<div style={{ textAlign: "right", marginTop: "10px" }}>
<Button htmlType="submit">Create Team</Button>
<Button htmlType="submit" data-testid="create-team-submit">Create Team</Button>
</div>
</Form>
</Modal>

1
ui/litellm-dashboard/src/components/common_components/team_dropdown.tsx
View file

@ -96,6 +96,7 @@ const TeamDropdown: React.FC<TeamDropdownProps> = ({
onPopupScroll={handlePopupScroll}
loading={isLoading}
notFoundContent={isLoading ? <LoadingOutlined spin /> : "No teams found"}
data-testid="team-dropdown"
popupRender={(menu) => (
<>
{menu}

1
ui/litellm-dashboard/src/components/common_components/user_search_modal.tsx
View file

@ -150,6 +150,7 @@ const UserSearchModal: React.FC<UserSearchModalProps> = ({
options={selectedField === "user_email" ? userOptions : []}
loading={loading}
allowClear
data-testid="member-email-search"
/>
</Form.Item>

2
ui/litellm-dashboard/src/components/organisms/create_key_button.tsx
View file

@ -666,7 +666,7 @@ const CreateKey: React.FC<CreateKeyProps> = ({ team, teams, data, addKey, autoOp
return (
<div>
{userRole && rolesWithWriteAccess.includes(userRole) && (
<Button className="mx-auto" onClick={() => setIsModalVisible(true)}>
<Button className="mx-auto" onClick={() => setIsModalVisible(true)} data-testid="create-key-button">
+ Create New Key
</Button>
)}