scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-17 23:51:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 66

Merge branch 'main' into newrelic

Browse source
This commit is contained in:
Josh Bonczkowski 2025-12-31 10:56:56 -05:00
commit 2f84ae8530
304 changed files with 19057 additions and 5460 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
.circleci/config.yml
View file

@ -3476,7 +3476,6 @@ jobs:
name: Install Playwright Browsers
command: |
npx playwright install
- run:
name: Build Docker image
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
@ -3515,7 +3514,10 @@ jobs:
- run:
name: Run Playwright Tests
command: |
npx playwright test e2e_ui_tests/ --reporter=html --output=test-results
npx playwright test \
--config ui/litellm-dashboard/e2e_tests/playwright.config.ts \
--reporter=html \
--output=test-results
no_output_timeout: 120m
- store_artifacts:
path: test-results
@ -3973,4 +3975,4 @@ workflows:
- proxy_pass_through_endpoint_tests
- check_code_and_doc_quality
- publish_proxy_extras
- guardrails_testing
- guardrails_testing

104
.gitguardian.yaml Normal file
View file

@ -0,0 +1,104 @@
version: 2
secret:
# Exclude files and paths by globbing
ignored_paths:
- "**/*.whl"
- "**/*.pyc"
- "**/__pycache__/**"
- "**/node_modules/**"
- "**/dist/**"
- "**/build/**"
- "**/.git/**"
- "**/venv/**"
- "**/.venv/**"
# Large data/metadata files that don't need scanning
- "**/model_prices_and_context_window*.json"
- "**/*_metadata/*.txt"
- "**/tokenizers/*.json"
- "**/tokenizers/*"
- "miniconda.sh"
# Build outputs and static assets
- "litellm/proxy/_experimental/out/**"
- "ui/litellm-dashboard/public/**"
- "**/swagger/*.js"
- "**/*.woff"
- "**/*.woff2"
- "**/*.avif"
- "**/*.webp"
# Test data files
- "**/tests/**/data_map.txt"
- "tests/**/*.txt"
# Documentation and other non-code files
- "docs/**"
- "**/*.md"
- "**/*.lock"
- "poetry.lock"
- "package-lock.json"
# Ignore security incidents with the SHA256 of the occurrence (false positives)
ignored_matches:
# === Current detected false positives (SHA-based) ===
# gcs_pub_sub_body - folder name, not a password
- name: GCS pub/sub test folder name
match: 75f377c456eede69e5f6e47399ccee6016a2a93cc5dd11db09cc5b1359ae569a
# os.environ/APORIA_API_KEY_1 - environment variable reference
- name: Environment variable reference APORIA_API_KEY_1
match: e2ddeb8b88eca97a402559a2be2117764e11c074d86159ef9ad2375dea188094
# os.environ/APORIA_API_KEY_2 - environment variable reference
- name: Environment variable reference APORIA_API_KEY_2
match: 09aa39a29e050b86603aa55138af1ff08fb86a4582aa965c1bd0672e1575e052
# oidc/circleci_v2/ - test authentication path, not a secret
- name: OIDC CircleCI test path
match: feb3475e1f89a65b7b7815ac4ec597e18a9ec1847742ad445c36ca617b536e15
# text-davinci-003 - OpenAI model identifier, not a secret
- name: OpenAI model identifier text-davinci-003
match: c489000cf6c7600cee0eefb80ad0965f82921cfb47ece880930eb7e7635cf1f1
# Base64 Basic Auth in test_pass_through_endpoints.py - test fixture, not a real secret
- name: Test Base64 Basic Auth header in pass_through_endpoints test
match: 61bac0491f395040617df7ef6d06029eac4d92a4457ac784978db80d97be1ae0
# PostgreSQL password "postgres" in CI configs - standard test database password
- name: Test PostgreSQL password in CI configurations
match: 6e0d657eb1f0fbc40cf0b8f3c3873ef627cc9cb7c4108d1c07d979c04bc8a4bb
# Bearer token in locustfile.py - test/example API key for load testing
- name: Test Bearer token in locustfile load test
match: 2a0abc2b0c3c1760a51ffcdf8d6b1d384cef69af740504b1cfa82dd70cdc7ff9
# Inkeep API key in docusaurus.config.js - public documentation site key
- name: Inkeep API key in documentation config
match: c366657791bfb5fc69045ec11d49452f09a0aebbc8648f94e2469b4025e29a75
# Langfuse credentials in test_completion.py - test credentials for integration test
- name: Langfuse test credentials in test_completion
match: c39310f68cc3d3e22f7b298bb6353c4f45759adcc37080d8b7f4e535d3cfd7f4
# === Preventive patterns for test keys (pattern-based) ===
# Test API keys (124 instances across 45 files)
- name: Test API keys with sk-test prefix
match: sk-test-
# Mock API keys
- name: Mock API keys with sk-mock prefix
match: sk-mock-
# Fake API keys
- name: Fake API keys with sk-fake prefix
match: sk-fake-
# Generic test API key patterns
- name: Test API key patterns
match: test-api-key

4
.gitignore vendored
View file

@ -100,3 +100,7 @@ update_model_cost_map.py
tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
litellm/proxy/_experimental/out/guardrails/index.html
scripts/test_vertex_ai_search.py
LAZY_LOADING_IMPROVEMENTS.md
**/test-results
**/playwright-report
**/*.storageState.json

21
AGENTS.md
View file

@ -49,6 +49,27 @@ LiteLLM is a unified interface for 100+ LLMs that:
- Test provider-specific functionality thoroughly
- Consider adding load tests for performance-critical changes
### MAKING CODE CHANGES FOR THE UI (IGNORE FOR BACKEND)
1. **Use Common Components as much as possible**:
- These are usually defined in the `common_components` directory
- Use these components as much as possible and avoid building new components unless needed
- Tremor components are deprecated; prefer using Ant Design (AntD) as much as possible
2. **Testing**:
- The codebase uses **Vitest** and **React Testing Library**
- **Query Priority Order**: Use query methods in this order: `getByRole`, `getByLabelText`, `getByPlaceholderText`, `getByText`, `getByTestId`
- **Always use `screen`** instead of destructuring from `render()` (e.g., use `screen.getByText()` not `getByText`)
- **Wrap user interactions in `act()`**: Always wrap `fireEvent` calls with `act()` to ensure React state updates are properly handled
- **Use `query` methods for absence checks**: Use `queryBy*` methods (not `getBy*`) when expecting an element to NOT be present
- **Test names must start with "should"**: All test names should follow the pattern `it("should ...")`
- **Mock external dependencies**: Check `setupTests.ts` for global mocks and mock child components/networking calls as needed
- **Structure tests properly**:
- First test should verify the component renders successfully
- Subsequent tests should focus on functionality and user interactions
- Use `waitFor` for async operations that aren't already awaited
- **Avoid using `querySelector`**: Prefer React Testing Library queries over direct DOM manipulation
### IMPORTANT PATTERNS
1. **Function/Tool Calling**:

40
ci_cd/TEST_KEY_PATTERNS.md Normal file
View file

@ -0,0 +1,40 @@
# Test Key Patterns Standard
Standard patterns for test/mock keys and credentials in the LiteLLM codebase to avoid triggering secret detection.
## How GitGuardian Works
GitGuardian uses **machine learning and entropy analysis**, not just pattern matching:
- **Low entropy** values (like `sk-1234`, `postgres`) are automatically ignored
- **High entropy** values (realistic-looking secrets) trigger detection
- **Context-aware** detection understands code syntax like `os.environ["KEY"]`
## Recommended Test Key Patterns
### Option 1: Low Entropy Values (Simplest)
These won't trigger GitGuardian's ML detector:
```python
api_key = "sk-1234"
api_key = "sk-12345"
database_password = "postgres"
token = "test123"
```
### Option 2: High Entropy with Test Prefixes
If you need realistic-looking test keys with high entropy, use these prefixes:
```python
api_key = "sk-test-abc123def456ghi789..." # OpenAI-style test key
api_key = "sk-mock-1234567890abcdef1234..." # Mock key
api_key = "sk-fake-xyz789uvw456rst123..." # Fake key
token = "test-api-key-with-high-entropy"
```
## Configured Ignore Patterns
These patterns are in `.gitguardian.yaml` for high-entropy test keys:
- `sk-test-*` - OpenAI-style test keys
- `sk-mock-*` - Mock API keys
- `sk-fake-*` - Fake API keys
- `test-api-key` - Generic test tokens

28
ci_cd/security_scans.sh
View file

@ -58,20 +58,20 @@ run_secret_detection() {
# Use --recursive for directory scanning and auto-confirm if prompted
# .gitguardian.yaml will automatically exclude binary files, wheel files, etc.
# GITGUARDIAN_API_KEY environment variable will be used for authentication
# echo y | ggshield secret scan path . --recursive || {
# echo ""
# echo "=========================================="
# echo "ERROR: Secret Detection Failed"
# echo "=========================================="
# echo "ggshield has detected secrets in the codebase."
# echo "Please review discovered secrets above, revoke any actively used secrets"
# echo "from underlying systems and make changes to inject secrets dynamically at runtime."
# echo ""
# echo "For more information, see: https://docs.gitguardian.com/secrets-detection/"
# echo "=========================================="
# echo ""
# exit 1
# }
echo y | ggshield secret scan path . --recursive || {
echo ""
echo "=========================================="
echo "ERROR: Secret Detection Failed"
echo "=========================================="
echo "ggshield has detected secrets in the codebase."
echo "Please review discovered secrets above, revoke any actively used secrets"
echo "from underlying systems and make changes to inject secrets dynamically at runtime."
echo ""
echo "For more information, see: https://docs.gitguardian.com/secrets-detection/"
echo "=========================================="
echo ""
exit 1
}
echo "Secret detection scans completed successfully"
}

2
cookbook/LiteLLM_PromptLayer.ipynb vendored
View file

@ -39,7 +39,7 @@
"import os\n",
"os.environ['OPENAI_API_KEY'] = \"\"\n",
"os.environ['REPLICATE_API_TOKEN'] = \"\"\n",
"os.environ['PROMPTLAYER_API_KEY'] = \"pl_4ea2bb00a4dca1b8a70cebf2e9e11564\"\n",
"os.environ['PROMPTLAYER_API_KEY'] = \"test-promptlayer-key-123\"\n",
"\n",
"# Set Promptlayer as a success callback\n",
"litellm.success_callback =['promptlayer']\n",

278
cookbook/Migrating_to_LiteLLM_Proxy_from_OpenAI_Azure_OpenAI.ipynb vendored
View file

@ -1,21 +1,10 @@
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
},
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "kccfk0mHZ4Ad"
},
"source": [
"# Migrating to LiteLLM Proxy from OpenAI/Azure OpenAI\n",
"\n",
@ -32,29 +21,26 @@
"To pass provider-specific args, [go here](https://docs.litellm.ai/docs/completion/provider_specific_params#proxy-usage)\n",
"\n",
"To drop unsupported params (E.g. frequency_penalty for bedrock with librechat), [go here](https://docs.litellm.ai/docs/completion/drop_params#openai-proxy-usage)\n"
],
"metadata": {
"id": "kccfk0mHZ4Ad"
}
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "nmSClzCPaGH6"
},
"source": [
"## /chat/completion\n",
"\n"
],
"metadata": {
"id": "nmSClzCPaGH6"
}
]
},
{
"cell_type": "markdown",
"source": [
"### OpenAI Python SDK"
],
"metadata": {
"id": "_vqcjwOVaKpO"
}
},
"source": [
"### OpenAI Python SDK"
]
},
{
"cell_type": "code",
@ -94,15 +80,20 @@
},
{
"cell_type": "markdown",
"source": [
"## Function Calling"
],
"metadata": {
"id": "AqkyKk9Scxgj"
}
},
"source": [
"## Function Calling"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "wDg10VqLczE1"
},
"outputs": [],
"source": [
"from openai import OpenAI\n",
"client = OpenAI(\n",
@ -139,24 +130,24 @@
")\n",
"\n",
"print(completion)\n"
],
"metadata": {
"id": "wDg10VqLczE1"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
"source": [
"### Azure OpenAI Python SDK"
],
"metadata": {
"id": "YYoxLloSaNWW"
}
},
"source": [
"### Azure OpenAI Python SDK"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "yA1XcgowaSRy"
},
"outputs": [],
"source": [
"import openai\n",
"client = openai.AzureOpenAI(\n",
@ -184,24 +175,24 @@
")\n",
"\n",
"print(response)"
],
"metadata": {
"id": "yA1XcgowaSRy"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
"source": [
"### Langchain Python"
],
"metadata": {
"id": "yl9qhDvnaTpL"
}
},
"source": [
"### Langchain Python"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "5MUZgSquaW5t"
},
"outputs": [],
"source": [
"from langchain.chat_models import ChatOpenAI\n",
"from langchain.prompts.chat import (\n",
@ -239,24 +230,22 @@
"response = chat(messages)\n",
"\n",
"print(response)"
],
"metadata": {
"id": "5MUZgSquaW5t"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
"source": [
"### Curl"
],
"metadata": {
"id": "B9eMgnULbRaz"
}
},
"source": [
"### Curl"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "VWCCk5PFcmhS"
},
"source": [
"\n",
"\n",
@ -280,22 +269,24 @@
"}'\n",
"```\n",
"\n"
],
"metadata": {
"id": "VWCCk5PFcmhS"
}
]
},
{
"cell_type": "markdown",
"source": [
"### LlamaIndex"
],
"metadata": {
"id": "drBAm2e1b6xe"
}
},
"source": [
"### LlamaIndex"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "d0bZcv8fb9mL"
},
"outputs": [],
"source": [
"import os, dotenv\n",
"\n",
@ -326,24 +317,24 @@
"query_engine = index.as_query_engine()\n",
"response = query_engine.query(\"What did the author do growing up?\")\n",
"print(response)\n"
],
"metadata": {
"id": "d0bZcv8fb9mL"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
"source": [
"### Langchain JS"
],
"metadata": {
"id": "xypvNdHnb-Yy"
}
},
"source": [
"### Langchain JS"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "R55mK2vCcBN2"
},
"outputs": [],
"source": [
"import { ChatOpenAI } from \"@langchain/openai\";\n",
"\n",
@ -359,24 +350,24 @@
"const message = await model.invoke(\"Hi there!\");\n",
"\n",
"console.log(message);\n"
],
"metadata": {
"id": "R55mK2vCcBN2"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
"source": [
"### OpenAI JS"
],
"metadata": {
"id": "nC4bLifCcCiW"
}
},
"source": [
"### OpenAI JS"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "MICH8kIMcFpg"
},
"outputs": [],
"source": [
"const { OpenAI } = require('openai');\n",
"\n",
@ -398,24 +389,24 @@
"}\n",
"\n",
"main();\n"
],
"metadata": {
"id": "MICH8kIMcFpg"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
"source": [
"### Anthropic SDK"
],
"metadata": {
"id": "D1Q07pEAcGTb"
}
},
"source": [
"### Anthropic SDK"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "qBjFcAvgcI3t"
},
"outputs": [],
"source": [
"import os\n",
"\n",
@ -423,7 +414,7 @@
"\n",
"client = Anthropic(\n",
" base_url=\"http://localhost:4000\", # proxy endpoint\n",
" api_key=\"sk-s4xN1IiLTCytwtZFJaYQrA\", # litellm proxy virtual key\n",
" api_key=\"sk-test-proxy-key-123\", # litellm proxy virtual key (example)\n",
")\n",
"\n",
"message = client.messages.create(\n",
@ -437,33 +428,33 @@
" model=\"claude-3-opus-20240229\",\n",
")\n",
"print(message.content)"
],
"metadata": {
"id": "qBjFcAvgcI3t"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
"source": [
"## /embeddings"
],
"metadata": {
"id": "dFAR4AJGcONI"
}
},
"source": [
"## /embeddings"
]
},
{
"cell_type": "markdown",
"source": [
"### OpenAI Python SDK"
],
"metadata": {
"id": "lgNoM281cRzR"
}
},
"source": [
"### OpenAI Python SDK"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "NY3DJhPfcQhA"
},
"outputs": [],
"source": [
"import openai\n",
"from openai import OpenAI\n",
@ -478,24 +469,24 @@
")\n",
"\n",
"print(response)\n"
],
"metadata": {
"id": "NY3DJhPfcQhA"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
"source": [
"### Langchain Embeddings"
],
"metadata": {
"id": "hmbg-DW6cUZs"
}
},
"source": [
"### Langchain Embeddings"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "lX2S8Nl1cWVP"
},
"outputs": [],
"source": [
"from langchain.embeddings import OpenAIEmbeddings\n",
"\n",
@ -526,24 +517,22 @@
"\n",
"print(f\"TITAN EMBEDDINGS\")\n",
"print(query_result[:5])"
],
"metadata": {
"id": "lX2S8Nl1cWVP"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
"source": [
"### Curl Request"
],
"metadata": {
"id": "oqGbWBCQcYfd"
}
},
"source": [
"### Curl Request"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "7rkIMV9LcdwQ"
},
"source": [
"\n",
"\n",
@ -556,10 +545,21 @@
" }'\n",
"```\n",
"\n"
],
"metadata": {
"id": "7rkIMV9LcdwQ"
}
]
}
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 0
}

24
docker/Dockerfile.non_root
View file

@ -40,7 +40,7 @@ COPY . .
ENV LITELLM_NON_ROOT=true
# Build Admin UI using the upstream command order while keeping a single RUN layer
RUN mkdir -p /tmp/litellm_ui && \
RUN mkdir -p /var/lib/litellm/ui && \
npm install -g npm@latest && npm cache clean --force && \
cd /app/ui/litellm-dashboard && \
if [ -f "/app/enterprise/enterprise_ui/enterprise_colors.json" ]; then \
@ -49,10 +49,10 @@ RUN mkdir -p /tmp/litellm_ui && \
rm -f package-lock.json && \
npm install --legacy-peer-deps && \
npm run build && \
cp -r /app/ui/litellm-dashboard/out/* /tmp/litellm_ui/ && \
mkdir -p /tmp/litellm_assets && \
cp /app/litellm/proxy/logo.jpg /tmp/litellm_assets/logo.jpg && \
( cd /tmp/litellm_ui && \
cp -r /app/ui/litellm-dashboard/out/* /var/lib/litellm/ui/ && \
mkdir -p /var/lib/litellm/assets && \
cp /app/litellm/proxy/logo.jpg /var/lib/litellm/assets/logo.jpg && \
( cd /var/lib/litellm/ui && \
for html_file in *.html; do \
if [ "$html_file" != "index.html" ] && [ -f "$html_file" ]; then \
folder_name="${html_file%.html}" && \
@ -111,8 +111,8 @@ COPY --from=builder /app/docker/entrypoint.sh /app/docker/prod_entrypoint.sh /ap
COPY --from=builder /app/docker/supervisord.conf /etc/supervisord.conf
COPY --from=builder /app/schema.prisma /app/
COPY --from=builder /wheels/ /wheels/
COPY --from=builder /tmp/litellm_ui /tmp/litellm_ui
COPY --from=builder /tmp/litellm_assets /tmp/litellm_assets
COPY --from=builder /var/lib/litellm/ui /var/lib/litellm/ui
COPY --from=builder /var/lib/litellm/assets /var/lib/litellm/assets
COPY --from=builder /app/.cache /app/.cache
COPY --from=builder /app/litellm-proxy-extras /app/litellm-proxy-extras
COPY --from=builder \
@ -145,8 +145,8 @@ RUN pip install --no-index --find-links=/wheels/ -r requirements.txt && \
# Permissions, cleanup, and Prisma prep
RUN chmod +x docker/entrypoint.sh docker/prod_entrypoint.sh && \
mkdir -p /nonexistent /.npm /tmp/litellm_assets /tmp/litellm_ui && \
chown -R nobody:nogroup /app /tmp/litellm_ui /tmp/litellm_assets /nonexistent /.npm && \
mkdir -p /nonexistent /.npm /var/lib/litellm/assets /var/lib/litellm/ui && \
chown -R nobody:nogroup /app /var/lib/litellm/ui /var/lib/litellm/assets /nonexistent /.npm && \
pip uninstall jwt -y || true && \
pip uninstall PyJWT -y || true && \
pip install --no-index --find-links=/wheels/ PyJWT==2.10.1 --no-cache-dir && \
@ -156,11 +156,11 @@ RUN chmod +x docker/entrypoint.sh docker/prod_entrypoint.sh && \
LITELLM_PKG_MIGRATIONS_PATH="$(python -c 'import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))' 2>/dev/null || echo '')/migrations" && \
[ -n "$LITELLM_PKG_MIGRATIONS_PATH" ] && chown -R nobody:nogroup $LITELLM_PKG_MIGRATIONS_PATH && \
LITELLM_PROXY_EXTRAS_PATH=$(python -c "import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))" 2>/dev/null || echo "") && \
chgrp -R 0 $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \
chgrp -R 0 $PRISMA_PATH /var/lib/litellm/ui /var/lib/litellm/assets && \
[ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chgrp -R 0 $LITELLM_PROXY_EXTRAS_PATH || true && \
chmod -R g=u $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \
chmod -R g=u $PRISMA_PATH /var/lib/litellm/ui /var/lib/litellm/assets && \
[ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g=u $LITELLM_PROXY_EXTRAS_PATH || true && \
chmod -R g+w $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \
chmod -R g+w $PRISMA_PATH /var/lib/litellm/ui /var/lib/litellm/assets && \
[ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g+w $LITELLM_PROXY_EXTRAS_PATH || true && \
chmod -R g+rX $PRISMA_PATH && \
chmod -R g+rX /app/.cache && \

57
docs/my-website/docs/interactions.md
View file

@ -8,7 +8,7 @@ import TabItem from '@theme/TabItem';
| Logging | ✅ | Works across all integrations |
| Streaming | ✅ | |
| Loadbalancing | ✅ | Between supported models |
| Supported Providers | `gemini` | [Google Interactions API](https://ai.google.dev/gemini-api/docs/interactions) |
| Supported LLM providers | **All LiteLLM supported providers** | `openai`, `anthropic`, `bedrock`, `vertex_ai`, `gemini`, `azure`, `azure_ai` etc. |
## **LiteLLM Python SDK Usage**
@ -207,8 +207,63 @@ for chunk in client.interactions.create_stream(
}
```
## **Calling non-Interactions API endpoints (`/interactions` to `/responses` Bridge)**
LiteLLM allows you to call non-Interactions API models via a bridge to LiteLLM's `/responses` endpoint. This is useful for calling OpenAI, Anthropic, and other providers that don't natively support the Interactions API.
#### Python SDK Usage
```python showLineNumbers title="SDK Usage"
import litellm
import os
# Set API key
os.environ["OPENAI_API_KEY"] = "your-openai-api-key"
# Non-streaming interaction
response = litellm.interactions.create(
model="gpt-4o",
input="Tell me a short joke about programming."
)
print(response.outputs[-1].text)
```
#### LiteLLM Proxy Usage
**Setup Config:**
```yaml showLineNumbers title="Example Configuration"
model_list:
- model_name: openai-model
litellm_params:
model: gpt-4o
api_key: os.environ/OPENAI_API_KEY
```
**Start Proxy:**
```bash showLineNumbers title="Start LiteLLM Proxy"
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
**Make Request:**
```bash showLineNumbers title="non-Interactions API Model Request"
curl http://localhost:4000/v1beta/interactions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "openai-model",
"input": "Tell me a short joke about programming."
}'
```
## **Supported Providers**
| Provider | Link to Usage |
|----------|---------------|
| Google AI Studio | [Usage](#quick-start) |
| All other LiteLLM providers | [Bridge Usage](#calling-non-interactions-api-endpoints-interactions-to-responses-bridge) |

27
docs/my-website/docs/mcp.md
View file

@ -746,8 +746,33 @@ curl --location 'http://localhost:4000/github_mcp/mcp' \
3. **Header Forwarding**: LiteLLM automatically forwards matching headers to the backend MCP server
4. **Authentication**: The backend MCP server receives both the configured auth headers and the custom headers
---
### Passing Request Headers to STDIO env Vars
If your stdio MCP server needs per-request credentials, you can map HTTP headers from the client request directly into the environment for the launched stdio process. Reference the header name in the env value using the `${X-HEADER_NAME}` syntax. LiteLLM will read that header from the incoming request and set the env var before starting the command.
```json title="Forward X-GITHUB_PERSONAL_ACCESS_TOKEN header to stdio env" showLineNumbers
{
"mcpServers": {
"github": {
"command": "docker",
"args": [
"run",
"-i",
"--rm",
"-e",
"GITHUB_PERSONAL_ACCESS_TOKEN",
"ghcr.io/github/github-mcp-server"
],
"env": {
"GITHUB_PERSONAL_ACCESS_TOKEN": "${X-GITHUB_PERSONAL_ACCESS_TOKEN}"
}
}
}
}
```
In this example, when a client makes a request with the `X-GITHUB_PERSONAL_ACCESS_TOKEN` header, the proxy forwards that value into the stdio process as the `GITHUB_PERSONAL_ACCESS_TOKEN` environment variable.
## Using your MCP with client side credentials

46
docs/my-website/docs/observability/cloudzero.md
View file

@ -65,6 +65,52 @@ Start your LiteLLM proxy with the configuration:
litellm --config /path/to/config.yaml
```
## Setup on UI
1\. Click "Settings"
![](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-22/5ac36280-c688-41a3-8d0e-23e19c6a470b/ascreenshot.jpeg?tl_px=0,332&br_px=1308,1064&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=119,444)
2\. Click "Logging & Alerts"
![](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-22/13f76b09-e0c4-4738-ba05-2d5111c6ad3e/ascreenshot.jpeg?tl_px=0,332&br_px=1308,1064&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=58,507)
3\. Click "CloudZero Cost Tracking"
![](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-22/f96cc1e5-7bc0-4d7c-9aeb-5cbbec549b12/ascreenshot.jpeg?tl_px=0,0&br_px=1308,731&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=389,56)
4\. Click "Add CloudZero Integration"
![](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-22/04fbc748-0e6f-43bb-8a57-dd2e83dbfcb5/ascreenshot.jpeg?tl_px=0,90&br_px=1308,821&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=616,277)
5\. Enter your CloudZero API Key.
![](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-22/080e82f1-f94f-4ed7-8014-e495380336f3/ascreenshot.jpeg?tl_px=0,0&br_px=1308,731&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=506,129)
6\. Enter your CloudZero Connection ID.
![](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-22/af417aa2-67a8-4dee-a014-84b1892dc07e/ascreenshot.jpeg?tl_px=0,0&br_px=1308,731&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=488,213)
7\. Click "Create"
![](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-22/647e672f-9a4a-4754-a7b0-abf1397abad4/ascreenshot.jpeg?tl_px=0,88&br_px=1308,819&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=711,277)
8\. Test your payload with "Run Dry Run Simulation"
![](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-22/7447cbe0-3450-4be5-bdc4-37fb8280aa58/ascreenshot.jpeg?tl_px=0,125&br_px=1308,856&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=334,277)
10\. Click "Export Data Now" to export to CLoudZero
![](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-22/7be9bd48-6e27-4c68-bc75-946f3ab593d9/ascreenshot.jpeg?tl_px=0,130&br_px=1308,861&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=518,277)
## Testing Your Setup
### Dry Run Export

2
docs/my-website/docs/oidc.md
View file

@ -106,7 +106,7 @@ model_list:
aws_region_name: us-west-2
aws_session_name: "my-test-session"
aws_role_name: "arn:aws:iam::335785316107:role/litellm-github-unit-tests-circleci"
aws_web_identity_token: "oidc/circleci_v2/"
aws_web_identity_token: "oidc/example-provider/"
```
#### Amazon IAM Role Configuration for CircleCI v2 -> Bedrock

129
docs/my-website/docs/providers/apertis.md Normal file
View file

@ -0,0 +1,129 @@
# Apertis AI (Stima API)
## Overview
| Property | Details |
|-------|-------|
| Description | Apertis AI (formerly Stima API) is a unified API platform providing access to 430+ AI models through a single interface, with cost savings of up to 50%. |
| Provider Route on LiteLLM | `apertis/` |
| Link to Provider Doc | [Apertis AI Website ↗](https://api.stima.tech) |
| Base URL | `https://api.stima.tech/v1` |
| Supported Operations | [`/chat/completions`](#sample-usage) |
<br />
## What is Apertis AI?
Apertis AI is a unified API platform that lets developers:
- **Access 430+ AI Models**: All models through a single API
- **Save 50% on Costs**: Competitive pricing with significant discounts
- **Unified Billing**: Single bill for all model usage
- **Quick Setup**: Start with just $2 registration
- **GitHub Integration**: Link with your GitHub account
## Required Variables
```python showLineNumbers title="Environment Variables"
os.environ["STIMA_API_KEY"] = "" # your Apertis AI API key
```
Get your Apertis AI API key from [api.stima.tech](https://api.stima.tech).
## Usage - LiteLLM Python SDK
### Non-streaming
```python showLineNumbers title="Apertis AI Non-streaming Completion"
import os
import litellm
from litellm import completion
os.environ["STIMA_API_KEY"] = "" # your Apertis AI API key
messages = [{"content": "What is the capital of France?", "role": "user"}]
# Apertis AI call
response = completion(
model="apertis/model-name", # Replace with actual model name
messages=messages
)
print(response)
```
### Streaming
```python showLineNumbers title="Apertis AI Streaming Completion"
import os
import litellm
from litellm import completion
os.environ["STIMA_API_KEY"] = "" # your Apertis AI API key
messages = [{"content": "Write a short poem about AI", "role": "user"}]
# Apertis AI call with streaming
response = completion(
model="apertis/model-name", # Replace with actual model name
messages=messages,
stream=True
)
for chunk in response:
print(chunk)
```
## Usage - LiteLLM Proxy Server
### 1. Save key in your environment
```bash
export STIMA_API_KEY=""
```
### 2. Start the proxy
```yaml
model_list:
- model_name: apertis-model
litellm_params:
model: apertis/model-name # Replace with actual model name
api_key: os.environ/STIMA_API_KEY
```
## Supported OpenAI Parameters
Apertis AI supports all standard OpenAI-compatible parameters:
| Parameter | Type | Description |
|-----------|------|-------------|
| `messages` | array | **Required**. Array of message objects with 'role' and 'content' |
| `model` | string | **Required**. Model ID from 430+ available models |
| `stream` | boolean | Optional. Enable streaming responses |
| `temperature` | float | Optional. Sampling temperature |
| `top_p` | float | Optional. Nucleus sampling parameter |
| `max_tokens` | integer | Optional. Maximum tokens to generate |
| `frequency_penalty` | float | Optional. Penalize frequent tokens |
| `presence_penalty` | float | Optional. Penalize tokens based on presence |
| `stop` | string/array | Optional. Stop sequences |
| `tools` | array | Optional. List of available tools/functions |
| `tool_choice` | string/object | Optional. Control tool/function calling |
## Cost Benefits
Apertis AI offers significant cost advantages:
- **50% Cost Savings**: Save money compared to direct provider costs
- **Unified Billing**: Single invoice for all your AI model usage
- **Low Entry**: Start with just $2 registration
## Model Availability
With access to 430+ AI models, Apertis AI provides:
- Multiple providers through one API
- Latest model releases
- Various model types (text, image, video)
## Additional Resources
- [Apertis AI Website](https://api.stima.tech)
- [Apertis AI Enterprise](https://api.stima.tech/enterprise)

172
docs/my-website/docs/providers/chutes.md Normal file
View file

@ -0,0 +1,172 @@
# Chutes
## Overview
| Property | Details |
|-------|-------|
| Description | Chutes is a cloud-native AI deployment platform that allows you to deploy, run, and scale LLM applications with OpenAI-compatible APIs using pre-built templates for popular frameworks like vLLM and SGLang. |
| Provider Route on LiteLLM | `chutes/` |
| Link to Provider Doc | [Chutes Website ↗](https://chutes.ai) |
| Base URL | `https://llm.chutes.ai/v1/` |
| Supported Operations | [`/chat/completions`](#sample-usage), Embeddings |
<br />
## What is Chutes?
Chutes is a powerful AI deployment and serving platform that provides:
- **Pre-built Templates**: Ready-to-use configurations for vLLM, SGLang, diffusion models, and embeddings
- **OpenAI-Compatible APIs**: Use standard OpenAI SDKs and clients
- **Multi-GPU Scaling**: Support for large models across multiple GPUs
- **Streaming Responses**: Real-time model outputs
- **Custom Configurations**: Override any parameter for your specific needs
- **Performance Optimization**: Pre-configured optimization settings
## Required Variables
```python showLineNumbers title="Environment Variables"
os.environ["CHUTES_API_KEY"] = "" # your Chutes API key
```
Get your Chutes API key from [chutes.ai](https://chutes.ai).
## Usage - LiteLLM Python SDK
### Non-streaming
```python showLineNumbers title="Chutes Non-streaming Completion"
import os
import litellm
from litellm import completion
os.environ["CHUTES_API_KEY"] = "" # your Chutes API key
messages = [{"content": "What is the capital of France?", "role": "user"}]
# Chutes call
response = completion(
model="chutes/model-name", # Replace with actual model name
messages=messages
)
print(response)
```
### Streaming
```python showLineNumbers title="Chutes Streaming Completion"
import os
import litellm
from litellm import completion
os.environ["CHUTES_API_KEY"] = "" # your Chutes API key
messages = [{"content": "Write a short poem about AI", "role": "user"}]
# Chutes call with streaming
response = completion(
model="chutes/model-name", # Replace with actual model name
messages=messages,
stream=True
)
for chunk in response:
print(chunk)
```
## Usage - LiteLLM Proxy Server
### 1. Save key in your environment
```bash
export CHUTES_API_KEY=""
```
### 2. Start the proxy
```yaml
model_list:
- model_name: chutes-model
litellm_params:
model: chutes/model-name # Replace with actual model name
api_key: os.environ/CHUTES_API_KEY
```
## Supported OpenAI Parameters
Chutes supports all standard OpenAI-compatible parameters:
| Parameter | Type | Description |
|-----------|------|-------------|
| `messages` | array | **Required**. Array of message objects with 'role' and 'content' |
| `model` | string | **Required**. Model ID or HuggingFace model identifier |
| `stream` | boolean | Optional. Enable streaming responses |
| `temperature` | float | Optional. Sampling temperature |
| `top_p` | float | Optional. Nucleus sampling parameter |
| `max_tokens` | integer | Optional. Maximum tokens to generate |
| `frequency_penalty` | float | Optional. Penalize frequent tokens |
| `presence_penalty` | float | Optional. Penalize tokens based on presence |
| `stop` | string/array | Optional. Stop sequences |
| `tools` | array | Optional. List of available tools/functions |
| `tool_choice` | string/object | Optional. Control tool/function calling |
| `response_format` | object | Optional. Response format specification |
## Support Frameworks
Chutes provides optimized templates for popular AI frameworks:
### vLLM (High-Performance LLM Serving)
- OpenAI-compatible endpoints
- Multi-GPU scaling support
- Advanced optimization settings
- Best for production workloads
### SGLang (Advanced LLM Serving)
- Structured generation capabilities
- Advanced features and controls
- Custom configuration options
- Best for complex use cases
### Diffusion Models (Image Generation)
- Pre-configured image generation templates
- Optimized settings for best results
- Support for popular diffusion models
### Embedding Models
- Text embedding templates
- Vector search optimization
- Support for popular embedding models
## Authentication
Chutes supports multiple authentication methods:
- API Key via `X-API-Key` header
- Bearer token via `Authorization` header
Example for LiteLLM (uses environment variable):
```python
os.environ["CHUTES_API_KEY"] = "your-api-key"
```
## Performance Optimization
Chutes offers hardware selection and optimization:
- **Small Models (7B-13B)**: 1 GPU with 24GB VRAM
- **Medium Models (30B-70B)**: 4 GPUs with 80GB VRAM each
- **Large Models (100B+)**: 8 GPUs with 140GB+ VRAM each
Engine optimization parameters available for fine-tuning performance.
## Deployment Options
Chutes provides flexible deployment:
- **Quick Setup**: Use pre-built templates for instant deployment
- **Custom Images**: Deploy with custom Docker images
- **Scaling**: Configure max instances and auto-scaling thresholds
- **Hardware**: Choose specific GPU types and configurations
## Additional Resources
- [Chutes Documentation](https://chutes.ai/docs)
- [Chutes Getting Started](https://chutes.ai/docs/getting-started/running-a-chute)
- [Chutes API Reference](https://chutes.ai/docs/sdk-reference)

94
docs/my-website/docs/providers/databricks.md
View file

@ -11,6 +11,99 @@ LiteLLM supports all models on Databricks
:::
## Authentication
LiteLLM supports multiple authentication methods for Databricks, listed in order of preference:
### OAuth M2M (Recommended for Production)
OAuth Machine-to-Machine authentication using Service Principal credentials is the **recommended method for production** deployments per Databricks Partner requirements.
```python
import os
from litellm import completion
# Set OAuth credentials (Service Principal)
os.environ["DATABRICKS_CLIENT_ID"] = "your-service-principal-application-id"
os.environ["DATABRICKS_CLIENT_SECRET"] = "your-service-principal-secret"
os.environ["DATABRICKS_API_BASE"] = "https://adb-xxx.azuredatabricks.net/serving-endpoints"
response = completion(
model="databricks/databricks-dbrx-instruct",
messages=[{"role": "user", "content": "Hello!"}],
)
```
### Personal Access Token (PAT)
PAT authentication is supported for development and testing scenarios.
```python
import os
from litellm import completion
os.environ["DATABRICKS_API_KEY"] = "dapi..." # Your Personal Access Token
os.environ["DATABRICKS_API_BASE"] = "https://adb-xxx.azuredatabricks.net/serving-endpoints"
response = completion(
model="databricks/databricks-dbrx-instruct",
messages=[{"role": "user", "content": "Hello!"}],
)
```
### Databricks SDK Authentication (Automatic)
If no credentials are provided, LiteLLM will use the Databricks SDK for automatic authentication. This supports OAuth, Azure AD, and other unified auth methods configured in your environment.
```python
from litellm import completion
# No environment variables needed - uses Databricks SDK unified auth
# Requires: pip install databricks-sdk
response = completion(
model="databricks/databricks-dbrx-instruct",
messages=[{"role": "user", "content": "Hello!"}],
)
```
## Custom User-Agent for Partner Attribution
If you're building a product on top of LiteLLM that integrates with Databricks, you can pass your own partner identifier for proper attribution in Databricks telemetry.
The partner name will be prefixed to the LiteLLM user agent:
```python
# Via parameter
response = completion(
model="databricks/databricks-dbrx-instruct",
messages=[{"role": "user", "content": "Hello!"}],
user_agent="mycompany/1.0.0",
)
# Resulting User-Agent: mycompany_litellm/1.79.1
# Via environment variable
os.environ["DATABRICKS_USER_AGENT"] = "mycompany/1.0.0"
# Resulting User-Agent: mycompany_litellm/1.79.1
```
| Input | Resulting User-Agent |
|-------|---------------------|
| (none) | `litellm/1.79.1` |
| `mycompany/1.0.0` | `mycompany_litellm/1.79.1` |
| `partner_product/2.5.0` | `partner_product_litellm/1.79.1` |
| `acme` | `acme_litellm/1.79.1` |
**Note:** The version from your custom user agent is ignored; LiteLLM's version is always used.
## Security
LiteLLM automatically redacts sensitive information (tokens, secrets, API keys) from all debug logs to prevent credential leakage. This includes:
- Authorization headers
- API keys and tokens
- Client secrets
- Personal access tokens (PATs)
## Usage
<Tabs>
@ -51,6 +144,7 @@ response = completion(
model: databricks/databricks-dbrx-instruct
api_key: os.environ/DATABRICKS_API_KEY
api_base: os.environ/DATABRICKS_API_BASE
user_agent: "mycompany/1.0.0" # Optional: for partner attribution
```

31
docs/my-website/docs/providers/groq.md
View file

@ -150,15 +150,15 @@ We support ALL Groq models, just set `groq/` as a prefix when sending completion
| Model Name | Usage |
|--------------------|---------------------------------------------------------|
| llama-3.1-8b-instant | `completion(model="groq/llama-3.1-8b-instant", messages)` |
| llama-3.1-70b-versatile | `completion(model="groq/llama-3.1-70b-versatile", messages)` |
| llama3-8b-8192 | `completion(model="groq/llama3-8b-8192", messages)` |
| llama3-70b-8192 | `completion(model="groq/llama3-70b-8192", messages)` |
| llama2-70b-4096 | `completion(model="groq/llama2-70b-4096", messages)` |
| mixtral-8x7b-32768 | `completion(model="groq/mixtral-8x7b-32768", messages)` |
| gemma-7b-it | `completion(model="groq/gemma-7b-it", messages)` |
| moonshotai/kimi-k2-instruct | `completion(model="groq/moonshotai/kimi-k2-instruct", messages)` |
| qwen3-32b | `completion(model="groq/qwen/qwen3-32b", messages)` |
| llama-3.3-70b-versatile | `completion(model="groq/llama-3.3-70b-versatile", messages)` |
| llama-3.1-8b-instant | `completion(model="groq/llama-3.1-8b-instant", messages)` |
| meta-llama/llama-4-scout-17b-16e-instruct | `completion(model="groq/meta-llama/llama-4-scout-17b-16e-instruct", messages)` |
| meta-llama/llama-4-maverick-17b-128e-instruct | `completion(model="groq/meta-llama/llama-4-maverick-17b-128e-instruct", messages)` |
| meta-llama/llama-guard-4-12b | `completion(model="groq/meta-llama/llama-guard-4-12b", messages)` |
| qwen/qwen3-32b | `completion(model="groq/qwen/qwen3-32b", messages)` |
| moonshotai/kimi-k2-instruct-0905 | `completion(model="groq/moonshotai/kimi-k2-instruct-0905", messages)` |
| openai/gpt-oss-120b | `completion(model="groq/openai/gpt-oss-120b", messages)` |
| openai/gpt-oss-20b | `completion(model="groq/openai/gpt-oss-20b", messages)` |
## Groq - Tool / Function Calling Example
@ -261,31 +261,28 @@ if tool_calls:
print("second response\n", second_response)
```
## Groq - Vision Example
## Groq - Vision Example
Select Groq models support vision. Check out their [model list](https://console.groq.com/docs/vision) for more details.
Groq's Llama 4 models support vision. Check out their [model list](https://console.groq.com/docs/vision) for more details.
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
import os
import os
from litellm import completion
os.environ["GROQ_API_KEY"] = "your-api-key"
# openai call
response = completion(
model = "groq/llama-3.2-11b-vision-preview",
model = "groq/meta-llama/llama-4-scout-17b-16e-instruct",
messages=[
{
"role": "user",
"content": [
{
"type": "text",
"text": "What’s in this image?"
"text": "What's in this image?"
},
{
"type": "image_url",

639
docs/my-website/docs/providers/minimax.md Normal file
View file

@ -0,0 +1,639 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# MiniMax
# MiniMax - v1/messages
## Overview
Litellm provides anthropic specs compatible support for minmax
## Supported Models
MiniMax offers three models through their Anthropic-compatible API:
| Model | Description | Input Cost | Output Cost | Prompt Caching Read | Prompt Caching Write |
|-------|-------------|------------|-------------|---------------------|----------------------|
| **MiniMax-M2.1** | Powerful Multi-Language Programming with Enhanced Programming Experience (~60 tps) | $0.3/M tokens | $1.2/M tokens | $0.03/M tokens | $0.375/M tokens |
| **MiniMax-M2.1-lightning** | Faster and More Agile (~100 tps) | $0.3/M tokens | $2.4/M tokens | $0.03/M tokens | $0.375/M tokens |
| **MiniMax-M2** | Agentic capabilities, Advanced reasoning | $0.3/M tokens | $1.2/M tokens | $0.03/M tokens | $0.375/M tokens |
## Usage Examples
### Basic Chat Completion
```python
import litellm
response = litellm.anthropic.messages.acreate(
model="minimax/MiniMax-M2.1",
messages=[{"role": "user", "content": "Hello, how are you?"}],
api_key="your-minimax-api-key",
api_base="https://api.minimax.io/anthropic/v1/messages",
max_tokens=1000
)
print(response.choices[0].message.content)
```
### Using Environment Variables
```bash
export MINIMAX_API_KEY="your-minimax-api-key"
export MINIMAX_API_BASE="https://api.minimax.io/anthropic/v1/messages"
```
```python
import litellm
response = litellm.anthropic.messages.acreate(
model="minimax/MiniMax-M2.1",
messages=[{"role": "user", "content": "Hello!"}],
max_tokens=1000
)
```
### With Thinking (M2.1 Feature)
```python
response = litellm.anthropic.messages.acreate(
model="minimax/MiniMax-M2.1",
messages=[{"role": "user", "content": "Solve: 2+2=?"}],
thinking={"type": "enabled", "budget_tokens": 1000},
api_key="your-minimax-api-key"
)
# Access thinking content
for block in response.choices[0].message.content:
if hasattr(block, 'type') and block.type == 'thinking':
print(f"Thinking: {block.thinking}")
```
### With Tool Calling
```python
tools = [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get current weather",
"parameters": {
"type": "object",
"properties": {
"location": {"type": "string"}
},
"required": ["location"]
}
}
}
]
response = litellm.anthropic.messages.acreate(
model="minimax/MiniMax-M2.1",
messages=[{"role": "user", "content": "What's the weather in SF?"}],
tools=tools,
api_key="your-minimax-api-key",
max_tokens=1000
)
```
## Usage with LiteLLM Proxy
You can use MiniMax models with the Anthropic SDK by routing through LiteLLM Proxy:
| Step | Description |
|------|-------------|
| **1. Start LiteLLM Proxy** | Configure proxy with MiniMax models in `config.yaml` |
| **2. Set Environment Variables** | Point Anthropic SDK to proxy endpoint |
| **3. Use Anthropic SDK** | Call MiniMax models using native Anthropic SDK |
### Step 1: Configure LiteLLM Proxy
Create a `config.yaml`:
```yaml
model_list:
- model_name: minimax/MiniMax-M2.1
litellm_params:
model: minimax/MiniMax-M2.1
api_key: os.environ/MINIMAX_API_KEY
api_base: https://api.minimax.io/anthropic/v1/messages
```
Start the proxy:
```bash
litellm --config config.yaml
```
### Step 2: Use with Anthropic SDK
```python
import os
os.environ["ANTHROPIC_BASE_URL"] = "http://localhost:4000"
os.environ["ANTHROPIC_API_KEY"] = "sk-1234" # Your LiteLLM proxy key
import anthropic
client = anthropic.Anthropic()
message = client.messages.create(
model="minimax/MiniMax-M2.1",
max_tokens=1000,
system="You are a helpful assistant.",
messages=[
{
"role": "user",
"content": [
{
"type": "text",
"text": "Hi, how are you?"
}
]
}
]
)
for block in message.content:
if block.type == "thinking":
print(f"Thinking:\n{block.thinking}\n")
elif block.type == "text":
print(f"Text:\n{block.text}\n")
```
# MiniMax - v1/chat/completions
## Usage with LiteLLM SDK
You can use MiniMax's OpenAI-compatible API directly with LiteLLM:
### Basic Chat Completion
```python
import litellm
response = litellm.completion(
model="minimax/MiniMax-M2.1",
messages=[
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "Hello, how are you?"}
],
api_key="your-minimax-api-key",
api_base="https://api.minimax.io/v1"
)
print(response.choices[0].message.content)
```
### Using Environment Variables
```bash
export MINIMAX_API_KEY="your-minimax-api-key"
export MINIMAX_API_BASE="https://api.minimax.io/v1"
```
```python
import litellm
response = litellm.completion(
model="minimax/MiniMax-M2.1",
messages=[{"role": "user", "content": "Hello!"}]
)
```
### With Reasoning Split
```python
response = litellm.completion(
model="minimax/MiniMax-M2.1",
messages=[
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "Solve: 2+2=?"}
],
extra_body={"reasoning_split": True},
api_key="your-minimax-api-key",
api_base="https://api.minimax.io/v1"
)
# Access reasoning details if available
if hasattr(response.choices[0].message, 'reasoning_details'):
print(f"Thinking: {response.choices[0].message.reasoning_details}")
print(f"Response: {response.choices[0].message.content}")
```
### With Tool Calling
```python
tools = [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get current weather",
"parameters": {
"type": "object",
"properties": {
"location": {"type": "string"}
},
"required": ["location"]
}
}
}
]
response = litellm.completion(
model="minimax/MiniMax-M2.1",
messages=[{"role": "user", "content": "What's the weather in SF?"}],
tools=tools,
api_key="your-minimax-api-key",
api_base="https://api.minimax.io/v1"
)
```
### Streaming
```python
response = litellm.completion(
model="minimax/MiniMax-M2.1",
messages=[{"role": "user", "content": "Tell me a story"}],
stream=True,
api_key="your-minimax-api-key",
api_base="https://api.minimax.io/v1"
)
for chunk in response:
if chunk.choices[0].delta.content:
print(chunk.choices[0].delta.content, end="")
```
## Usage with OpenAI SDK via LiteLLM Proxy
You can also use MiniMax models with the OpenAI SDK by routing through LiteLLM Proxy:
| Step | Description |
|------|-------------|
| **1. Start LiteLLM Proxy** | Configure proxy with MiniMax models in `config.yaml` |
| **2. Set Environment Variables** | Point OpenAI SDK to proxy endpoint |
| **3. Use OpenAI SDK** | Call MiniMax models using native OpenAI SDK |
### Step 1: Configure LiteLLM Proxy
Create a `config.yaml`:
```yaml
model_list:
- model_name: minimax/MiniMax-M2.1
litellm_params:
model: minimax/MiniMax-M2.1
api_key: os.environ/MINIMAX_API_KEY
api_base: https://api.minimax.io/v1
```
Start the proxy:
```bash
litellm --config config.yaml
```
### Step 2: Use with OpenAI SDK
```python
import os
os.environ["OPENAI_BASE_URL"] = "http://localhost:4000"
os.environ["OPENAI_API_KEY"] = "sk-1234" # Your LiteLLM proxy key
from openai import OpenAI
client = OpenAI()
response = client.chat.completions.create(
model="minimax/MiniMax-M2.1",
messages=[
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "Hi, how are you?"},
],
# Set reasoning_split=True to separate thinking content
extra_body={"reasoning_split": True},
)
# Access thinking and response
if hasattr(response.choices[0].message, 'reasoning_details'):
print(f"Thinking:\n{response.choices[0].message.reasoning_details[0]['text']}\n")
print(f"Text:\n{response.choices[0].message.content}\n")
```
### Streaming with OpenAI SDK
```python
from openai import OpenAI
client = OpenAI()
stream = client.chat.completions.create(
model="minimax/MiniMax-M2.1",
messages=[
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "Tell me a story"},
],
extra_body={"reasoning_split": True},
stream=True,
)
reasoning_buffer = ""
text_buffer = ""
for chunk in stream:
if hasattr(chunk.choices[0].delta, "reasoning_details") and chunk.choices[0].delta.reasoning_details:
for detail in chunk.choices[0].delta.reasoning_details:
if "text" in detail:
reasoning_text = detail["text"]
new_reasoning = reasoning_text[len(reasoning_buffer):]
if new_reasoning:
print(new_reasoning, end="", flush=True)
reasoning_buffer = reasoning_text
if chunk.choices[0].delta.content:
content_text = chunk.choices[0].delta.content
new_text = content_text[len(text_buffer):] if text_buffer else content_text
if new_text:
print(new_text, end="", flush=True)
text_buffer = content_text
```
## Cost Calculation
Cost calculation works automatically using the pricing information in `model_prices_and_context_window.json`.
Example:
```python
response = litellm.completion(
model="minimax/MiniMax-M2.1",
messages=[{"role": "user", "content": "Hello!"}],
api_key="your-minimax-api-key"
)
# Access cost information
print(f"Cost: ${response._hidden_params.get('response_cost', 0)}")
```
# MiniMax - Text-to-Speech
## Quick Start
## **LiteLLM Python SDK Usage**
### Basic Usage
```python
from pathlib import Path
from litellm import speech
import os
os.environ["MINIMAX_API_KEY"] = "your-api-key"
speech_file_path = Path(__file__).parent / "speech.mp3"
response = speech(
model="minimax/speech-2.6-hd",
voice="alloy",
input="The quick brown fox jumped over the lazy dogs",
)
response.stream_to_file(speech_file_path)
```
### Async Usage
```python
from litellm import aspeech
from pathlib import Path
import os, asyncio
os.environ["MINIMAX_API_KEY"] = "your-api-key"
async def test_async_speech():
speech_file_path = Path(__file__).parent / "speech.mp3"
response = await aspeech(
model="minimax/speech-2.6-hd",
voice="alloy",
input="The quick brown fox jumped over the lazy dogs",
)
response.stream_to_file(speech_file_path)
asyncio.run(test_async_speech())
```
### Voice Selection
MiniMax supports many voices. LiteLLM provides OpenAI-compatible voice names that map to MiniMax voices:
```python
from litellm import speech
# OpenAI-compatible voice names
voices = ["alloy", "echo", "fable", "onyx", "nova", "shimmer"]
for voice in voices:
response = speech(
model="minimax/speech-2.6-hd",
voice=voice,
input=f"This is the {voice} voice",
)
response.stream_to_file(f"speech_{voice}.mp3")
```
You can also use MiniMax-native voice IDs directly:
```python
response = speech(
model="minimax/speech-2.6-hd",
voice="male-qn-qingse", # MiniMax native voice ID
input="Using native MiniMax voice ID",
)
```
### Custom Parameters
MiniMax TTS supports additional parameters for fine-tuning audio output:
```python
from litellm import speech
response = speech(
model="minimax/speech-2.6-hd",
voice="alloy",
input="Custom audio parameters",
speed=1.5, # Speed: 0.5 to 2.0
response_format="mp3", # Format: mp3, pcm, wav, flac
extra_body={
"vol": 1.2, # Volume: 0.1 to 10
"pitch": 2, # Pitch adjustment: -12 to 12
"sample_rate": 32000, # 16000, 24000, or 32000
"bitrate": 128000, # For MP3: 64000, 128000, 192000, 256000
"channel": 1, # 1 for mono, 2 for stereo
}
)
response.stream_to_file("custom_speech.mp3")
```
### Response Formats
```python
from litellm import speech
# MP3 format (default)
response = speech(
model="minimax/speech-2.6-hd",
voice="alloy",
input="MP3 format audio",
response_format="mp3",
)
# PCM format
response = speech(
model="minimax/speech-2.6-hd",
voice="alloy",
input="PCM format audio",
response_format="pcm",
)
# WAV format
response = speech(
model="minimax/speech-2.6-hd",
voice="alloy",
input="WAV format audio",
response_format="wav",
)
# FLAC format
response = speech(
model="minimax/speech-2.6-hd",
voice="alloy",
input="FLAC format audio",
response_format="flac",
)
```
## **LiteLLM Proxy Usage**
LiteLLM provides an OpenAI-compatible `/audio/speech` endpoint for MiniMax TTS.
### Setup
Add MiniMax to your proxy configuration:
```yaml
model_list:
- model_name: tts
litellm_params:
model: minimax/speech-2.6-hd
api_key: os.environ/MINIMAX_API_KEY
- model_name: tts-turbo
litellm_params:
model: minimax/speech-2.6-turbo
api_key: os.environ/MINIMAX_API_KEY
```
Start the proxy:
```bash
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
### Making Requests
```bash
curl http://0.0.0.0:4000/v1/audio/speech \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "tts",
"input": "The quick brown fox jumped over the lazy dog.",
"voice": "alloy"
}' \
--output speech.mp3
```
With custom parameters:
```bash
curl http://0.0.0.0:4000/v1/audio/speech \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "tts",
"input": "Custom parameters example.",
"voice": "nova",
"speed": 1.5,
"response_format": "mp3",
"extra_body": {
"vol": 1.2,
"pitch": 1,
"sample_rate": 32000
}
}' \
--output custom_speech.mp3
```
## Voice Mappings
LiteLLM maps OpenAI-compatible voice names to MiniMax voice IDs:
| OpenAI Voice | MiniMax Voice ID | Description |
|--------------|------------------|-------------|
| alloy | male-qn-qingse | Male voice |
| echo | male-qn-jingying | Male voice |
| fable | female-shaonv | Female voice |
| onyx | male-qn-badao | Male voice |
| nova | female-yujie | Female voice |
| shimmer | female-tianmei | Female voice |
You can also use any MiniMax-native voice ID directly by passing it as the `voice` parameter.
### Streaming (WebSocket)
:::note
The current implementation uses MiniMax's HTTP endpoint. For WebSocket streaming support, please refer to MiniMax's official documentation at [https://platform.minimax.io/docs](https://platform.minimax.io/docs).
:::
## Error Handling
```python
from litellm import speech
import litellm
try:
response = speech(
model="minimax/speech-2.6-hd",
voice="alloy",
input="Test input",
)
response.stream_to_file("output.mp3")
except litellm.exceptions.BadRequestError as e:
print(f"Bad request: {e}")
except litellm.exceptions.AuthenticationError as e:
print(f"Authentication failed: {e}")
except Exception as e:
print(f"Error: {e}")
```
### Extra Body Parameters
Pass these via `extra_body`:
| Parameter | Type | Description | Default |
|-----------|------|-------------|---------|
| vol | float | Volume (0.1 to 10) | 1.0 |
| pitch | int | Pitch adjustment (-12 to 12) | 0 |
| sample_rate | int | Sample rate: 16000, 24000, 32000 | 32000 |
| bitrate | int | Bitrate for MP3: 64000, 128000, 192000, 256000 | 128000 |
| channel | int | Audio channels: 1 (mono) or 2 (stereo) | 1 |
| output_format | string | Output format: "hex" or "url" (url returns a URL valid for 24 hours) | hex |

170
docs/my-website/docs/providers/nano-gpt.md Normal file
View file

@ -0,0 +1,170 @@
# NanoGPT
## Overview
| Property | Details |
|-------|-------|
| Description | NanoGPT is a pay-per-prompt and subscription based AI service providing instant access to over 200+ powerful AI models with no subscriptions or registration required. |
| Provider Route on LiteLLM | `nano-gpt/` |
| Link to Provider Doc | [NanoGPT Website ↗](https://nano-gpt.com) |
| Base URL | `https://nano-gpt.com/api/v1` |
| Supported Operations | [`/chat/completions`](#sample-usage), [`/completions`](#text-completion), [`/embeddings`](#embeddings) |
<br />
## What is NanoGPT?
NanoGPT is a flexible AI API service that offers:
- **Pay-Per-Prompt Pricing**: No subscriptions, pay only for what you use
- **200+ AI Models**: Access to text, image, and video generation models
- **No Registration Required**: Get started instantly
- **OpenAI-Compatible API**: Easy integration with existing code
- **Streaming Support**: Real-time response streaming
- **Tool Calling**: Support for function calling
## Required Variables
```python showLineNumbers title="Environment Variables"
os.environ["NANOGPT_API_KEY"] = "" # your NanoGPT API key
```
Get your NanoGPT API key from [nano-gpt.com](https://nano-gpt.com).
## Usage - LiteLLM Python SDK
### Non-streaming
```python showLineNumbers title="NanoGPT Non-streaming Completion"
import os
import litellm
from litellm import completion
os.environ["NANOGPT_API_KEY"] = "" # your NanoGPT API key
messages = [{"content": "What is the capital of France?", "role": "user"}]
# NanoGPT call
response = completion(
model="nano-gpt/model-name", # Replace with actual model name
messages=messages
)
print(response)
```
### Streaming
```python showLineNumbers title="NanoGPT Streaming Completion"
import os
import litellm
from litellm import completion
os.environ["NANOGPT_API_KEY"] = "" # your NanoGPT API key
messages = [{"content": "Write a short poem about AI", "role": "user"}]
# NanoGPT call with streaming
response = completion(
model="nano-gpt/model-name", # Replace with actual model name
messages=messages,
stream=True
)
for chunk in response:
print(chunk)
```
### Tool Calling
```python showLineNumbers title="NanoGPT Tool Calling"
import os
import litellm
os.environ["NANOGPT_API_KEY"] = ""
tools = [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get current weather",
"parameters": {
"type": "object",
"properties": {
"location": {"type": "string"}
}
}
}
}
]
response = litellm.completion(
model="nano-gpt/model-name",
messages=[{"role": "user", "content": "What's the weather in Paris?"}],
tools=tools
)
```
## Usage - LiteLLM Proxy Server
### 1. Save key in your environment
```bash
export NANOGPT_API_KEY=""
```
### 2. Start the proxy
```yaml
model_list:
- model_name: nano-gpt-model
litellm_params:
model: nano-gpt/model-name # Replace with actual model name
api_key: os.environ/NANOGPT_API_KEY
```
## Supported OpenAI Parameters
NanoGPT supports all standard OpenAI-compatible parameters:
| Parameter | Type | Description |
|-----------|------|-------------|
| `messages` | array | **Required**. Array of message objects with 'role' and 'content' |
| `model` | string | **Required**. Model ID from 200+ available models |
| `stream` | boolean | Optional. Enable streaming responses |
| `temperature` | float | Optional. Sampling temperature |
| `top_p` | float | Optional. Nucleus sampling parameter |
| `max_tokens` | integer | Optional. Maximum tokens to generate |
| `frequency_penalty` | float | Optional. Penalize frequent tokens |
| `presence_penalty` | float | Optional. Penalize tokens based on presence |
| `stop` | string/array | Optional. Stop sequences |
| `n` | integer | Optional. Number of completions to generate |
| `tools` | array | Optional. List of available tools/functions |
| `tool_choice` | string/object | Optional. Control tool/function calling |
| `response_format` | object | Optional. Response format specification |
| `user` | string | Optional. User identifier |
## Model Categories
NanoGPT provides access to multiple model categories:
- **Text Generation**: 200+ LLMs for chat, completion, and analysis
- **Image Generation**: AI models for creating images
- **Video Generation**: AI models for video creation
- **Embedding Models**: Text embedding models for vector search
## Pricing Model
NanoGPT offers a flexible pricing structure:
- **Pay-Per-Prompt**: No subscription required
- **No Registration**: Get started immediately
- **Transparent Pricing**: Pay only for what you use
## API Documentation
For detailed API documentation, visit [docs.nano-gpt.com](https://docs.nano-gpt.com).
## Additional Resources
- [NanoGPT Website](https://nano-gpt.com)
- [NanoGPT API Documentation](https://nano-gpt.com/api)
- [NanoGPT Model List](https://docs.nano-gpt.com/api-reference/endpoint/models)

2
docs/my-website/docs/providers/openai.md
View file

@ -495,7 +495,7 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
|-------|----------------------|------------------|
| `gpt-5.1` | `none` | `none`, `low`, `medium`, `high` |
| `gpt-5` | `medium` | `minimal`, `low`, `medium`, `high` |
| `gpt-5-mini` | `medium` | `none`, `minimal`, `low`, `medium`, `high` |
| `gpt-5-mini` | `medium` | `minimal`, `low`, `medium`, `high` |
| `gpt-5-nano` | `none` | `none`, `low`, `medium`, `high` |
| `gpt-5-codex` | `adaptive` | `low`, `medium`, `high` (no `minimal`) |
| `gpt-5.1-codex` | `adaptive` | `low`, `medium`, `high` (no `minimal`) |

139
docs/my-website/docs/providers/poe.md Normal file
View file

@ -0,0 +1,139 @@
# Poe
## Overview
| Property | Details |
|-------|-------|
| Description | Poe is Quora's AI platform that provides access to more than 100 models across text, image, video, and voice modalities through a developer-friendly API. |
| Provider Route on LiteLLM | `poe/` |
| Link to Provider Doc | [Poe Website ↗](https://poe.com) |
| Base URL | `https://api.poe.com/v1` |
| Supported Operations | [`/chat/completions`](#sample-usage) |
<br />
## What is Poe?
Poe is Quora's comprehensive AI platform that offers:
- **100+ Models**: Access to a wide variety of AI models
- **Multiple Modalities**: Text, image, video, and voice AI
- **Popular Models**: Including OpenAI's GPT series and Anthropic's Claude
- **Developer API**: Easy integration for applications
- **Extensive Reach**: Benefits from Quora's 400M monthly unique visitors
## Required Variables
```python showLineNumbers title="Environment Variables"
os.environ["POE_API_KEY"] = "" # your Poe API key
```
Get your Poe API key from the [Poe platform](https://poe.com).
## Usage - LiteLLM Python SDK
### Non-streaming
```python showLineNumbers title="Poe Non-streaming Completion"
import os
import litellm
from litellm import completion
os.environ["POE_API_KEY"] = "" # your Poe API key
messages = [{"content": "What is the capital of France?", "role": "user"}]
# Poe call
response = completion(
model="poe/model-name", # Replace with actual model name
messages=messages
)
print(response)
```
### Streaming
```python showLineNumbers title="Poe Streaming Completion"
import os
import litellm
from litellm import completion
os.environ["POE_API_KEY"] = "" # your Poe API key
messages = [{"content": "Write a short poem about AI", "role": "user"}]
# Poe call with streaming
response = completion(
model="poe/model-name", # Replace with actual model name
messages=messages,
stream=True
)
for chunk in response:
print(chunk)
```
## Usage - LiteLLM Proxy Server
### 1. Save key in your environment
```bash
export POE_API_KEY=""
```
### 2. Start the proxy
```yaml
model_list:
- model_name: poe-model
litellm_params:
model: poe/model-name # Replace with actual model name
api_key: os.environ/POE_API_KEY
```
## Supported OpenAI Parameters
Poe supports all standard OpenAI-compatible parameters:
| Parameter | Type | Description |
|-----------|------|-------------|
| `messages` | array | **Required**. Array of message objects with 'role' and 'content' |
| `model` | string | **Required**. Model ID from 100+ available models |
| `stream` | boolean | Optional. Enable streaming responses |
| `temperature` | float | Optional. Sampling temperature |
| `top_p` | float | Optional. Nucleus sampling parameter |
| `max_tokens` | integer | Optional. Maximum tokens to generate |
| `frequency_penalty` | float | Optional. Penalize frequent tokens |
| `presence_penalty` | float | Optional. Penalize tokens based on presence |
| `stop` | string/array | Optional. Stop sequences |
| `tools` | array | Optional. List of available tools/functions |
| `tool_choice` | string/object | Optional. Control tool/function calling |
| `response_format` | object | Optional. Response format specification |
| `user` | string | Optional. User identifier |
## Available Model Categories
Poe provides access to models across multiple providers:
- **OpenAI Models**: Including GPT-4, GPT-4 Turbo, GPT-3.5 Turbo
- **Anthropic Models**: Including Claude 3 Opus, Sonnet, Haiku
- **Other Popular Models**: Various provider models available
- **Multi-Modal**: Text, image, video, and voice models
## Platform Benefits
Using Poe through LiteLLM offers several advantages:
- **Unified Access**: Single API for many different models
- **Quora Integration**: Access to large user base and content ecosystem
- **Content Sharing**: Capabilities to share model outputs with followers
- **Content Distribution**: Best AI content distributed to all users
- **Model Discovery**: Efficient way to explore new AI models
## Developer Resources
Poe is actively building developer features and welcomes early access requests for API integration.
## Additional Resources
- [Poe Website](https://poe.com)
- [Poe AI Quora Space](https://poeai.quora.com)
- [Quora Blog Post about Poe](https://quorablog.quora.com/Poe)

119
docs/my-website/docs/providers/synthetic.md Normal file
View file

@ -0,0 +1,119 @@
# Synthetic
## Overview
| Property | Details |
|-------|-------|
| Description | Synthetic runs open-source AI models in secure datacenters within the US and EU, with a focus on privacy. They never train on your data and auto-delete API data within 14 days. |
| Provider Route on LiteLLM | `synthetic/` |
| Link to Provider Doc | [Synthetic Website ↗](https://synthetic.new) |
| Base URL | `https://api.synthetic.new/openai/v1` |
| Supported Operations | [`/chat/completions`](#sample-usage) |
<br />
## What is Synthetic?
Synthetic is a privacy-focused AI platform that provides access to open-source LLMs with the following guarantees:
- **Privacy-First**: Data never used for training
- **Secure Hosting**: Models run in secure datacenters in US and EU
- **Auto-Deletion**: API data automatically deleted within 14 days
- **Open Source**: Runs open-source AI models
## Required Variables
```python showLineNumbers title="Environment Variables"
os.environ["SYNTHETIC_API_KEY"] = "" # your Synthetic API key
```
Get your Synthetic API key from [synthetic.new](https://synthetic.new).
## Usage - LiteLLM Python SDK
### Non-streaming
```python showLineNumbers title="Synthetic Non-streaming Completion"
import os
import litellm
from litellm import completion
os.environ["SYNTHETIC_API_KEY"] = "" # your Synthetic API key
messages = [{"content": "What is the capital of France?", "role": "user"}]
# Synthetic call
response = completion(
model="synthetic/model-name", # Replace with actual model name
messages=messages
)
print(response)
```
### Streaming
```python showLineNumbers title="Synthetic Streaming Completion"
import os
import litellm
from litellm import completion
os.environ["SYNTHETIC_API_KEY"] = "" # your Synthetic API key
messages = [{"content": "Write a short poem about AI", "role": "user"}]
# Synthetic call with streaming
response = completion(
model="synthetic/model-name", # Replace with actual model name
messages=messages,
stream=True
)
for chunk in response:
print(chunk)
```
## Usage - LiteLLM Proxy Server
### 1. Save key in your environment
```bash
export SYNTHETIC_API_KEY=""
```
### 2. Start the proxy
```yaml
model_list:
- model_name: synthetic-model
litellm_params:
model: synthetic/model-name # Replace with actual model name
api_key: os.environ/SYNTHETIC_API_KEY
```
## Supported OpenAI Parameters
Synthetic supports all standard OpenAI-compatible parameters:
| Parameter | Type | Description |
|-----------|------|-------------|
| `messages` | array | **Required**. Array of message objects with 'role' and 'content' |
| `model` | string | **Required**. Model ID |
| `stream` | boolean | Optional. Enable streaming responses |
| `temperature` | float | Optional. Sampling temperature |
| `top_p` | float | Optional. Nucleus sampling parameter |
| `max_tokens` | integer | Optional. Maximum tokens to generate |
| `frequency_penalty` | float | Optional. Penalize frequent tokens |
| `presence_penalty` | float | Optional. Penalize tokens based on presence |
| `stop` | string/array | Optional. Stop sequences |
## Privacy & Security
Synthetic provides enterprise-grade privacy protections:
- Data auto-deleted within 14 days
- No data used for model training
- Secure hosting in US and EU datacenters
- Compliance-friendly architecture
## Additional Resources
- [Synthetic Website](https://synthetic.new)

22
docs/my-website/docs/proxy/alerting.md
View file

@ -215,16 +215,16 @@ general_settings:
alerting: ["slack"]
alerting_threshold: 0.0001 # (Seconds) set an artificially low threshold for testing alerting
alert_to_webhook_url: {
"llm_exceptions": "https://hooks.slack.com/services/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",
"llm_too_slow": "https://hooks.slack.com/services/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",
"llm_requests_hanging": "https://hooks.slack.com/services/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",
"budget_alerts": "https://hooks.slack.com/services/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",
"db_exceptions": "https://hooks.slack.com/services/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",
"daily_reports": "https://hooks.slack.com/services/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",
"spend_reports": "https://hooks.slack.com/services/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",
"cooldown_deployment": "https://hooks.slack.com/services/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",
"new_model_added": "https://hooks.slack.com/services/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",
"outage_alerts": "https://hooks.slack.com/services/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",
"llm_exceptions": "example-slack-webhook-url",
"llm_too_slow": "example-slack-webhook-url",
"llm_requests_hanging": "example-slack-webhook-url",
"budget_alerts": "example-slack-webhook-url",
"db_exceptions": "example-slack-webhook-url",
"daily_reports": "example-slack-webhook-url",
"spend_reports": "example-slack-webhook-url",
"cooldown_deployment": "example-slack-webhook-url",
"new_model_added": "example-slack-webhook-url",
"outage_alerts": "example-slack-webhook-url",
}
litellm_settings:
@ -399,7 +399,7 @@ curl -X GET --location 'http://0.0.0.0:4000/health/services?service=webhook' \
{
"spend": 1, # the spend for the 'event_group'
"max_budget": 0, # the 'max_budget' set for the 'event_group'
"token": "88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b",
"token": "example-api-key-123",
"user_id": "default_user_id",
"team_id": null,
"user_email": null,

52
docs/my-website/docs/proxy/call_hooks.md
View file

@ -17,6 +17,7 @@ import Image from '@theme/IdealImage';
| `async_pre_call_hook` | Modify incoming request before it's sent to model | Before the LLM API call is made |
| `async_moderation_hook` | Run checks on input in parallel to LLM API call | In parallel with the LLM API call |
| `async_post_call_success_hook` | Modify outgoing response (non-streaming) | After successful LLM API call, for non-streaming responses |
| `async_post_call_failure_hook` | Transform error responses sent to clients | After failed LLM API call |
| `async_post_call_streaming_hook` | Modify outgoing response (streaming) | After successful LLM API call, for streaming responses |
See a complete example with our [parallel request rate limiter](https://github.com/BerriAI/litellm/blob/main/litellm/proxy/hooks/parallel_request_limiter.py)
@ -60,7 +61,21 @@ class MyCustomHandler(CustomLogger): # https://docs.litellm.ai/docs/observabilit
original_exception: Exception,
user_api_key_dict: UserAPIKeyAuth,
traceback_str: Optional[str] = None,
):
) -> Optional[HTTPException]:
"""
Transform error responses sent to clients.
Return an HTTPException to replace the original error with a user-friendly message.
Return None to use the original exception.
Example:
if isinstance(original_exception, litellm.ContextWindowExceededError):
return HTTPException(
status_code=400,
detail="Your prompt is too long. Please reduce the length and try again."
)
return None # Use original exception
"""
pass
async def async_post_call_success_hook(
@ -339,3 +354,38 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
"usage": {}
}
```
## Advanced - Transform Error Responses
Transform technical API errors into user-friendly messages using `async_post_call_failure_hook`. Return an `HTTPException` to replace the original error, or `None` to use the original exception.
```python
from litellm.integrations.custom_logger import CustomLogger
from fastapi import HTTPException
from typing import Optional
import litellm
class MyErrorTransformer(CustomLogger):
async def async_post_call_failure_hook(
self,
request_data: dict,
original_exception: Exception,
user_api_key_dict: UserAPIKeyAuth,
traceback_str: Optional[str] = None,
) -> Optional[HTTPException]:
if isinstance(original_exception, litellm.ContextWindowExceededError):
return HTTPException(
status_code=400,
detail="Your prompt is too long. Please reduce the length and try again."
)
if isinstance(original_exception, litellm.RateLimitError):
return HTTPException(
status_code=429,
detail="Rate limit exceeded. Please try again in a moment."
)
return None # Use original exception
proxy_handler_instance = MyErrorTransformer()
```
**Result:** Clients receive `"Your prompt is too long..."` instead of `"ContextWindowExceededError: Prompt exceeds context window"`.

2
docs/my-website/docs/proxy/configs.md
View file

@ -116,7 +116,7 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
"role": "user",
"content": "what llm are you"
}
],
]
}
'
```

6
docs/my-website/docs/proxy/cost_tracking.md
View file

@ -722,7 +722,7 @@ curl -X GET 'http://localhost:4000/global/spend/report?start_date=2024-04-01&end
```shell
[
{
"api_key": "88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b",
"api_key": "example-api-key-123",
"total_cost": 0.3201286305151999,
"total_input_tokens": 36.0,
"total_output_tokens": 1593.0,
@ -766,7 +766,7 @@ curl -X GET 'http://localhost:4000/global/spend/report?start_date=2024-04-01&end
```shell
[
{
"api_key": "88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b",
"api_key": "example-api-key-123",
"total_cost": 0.00013132,
"total_input_tokens": 105.0,
"total_output_tokens": 872.0,
@ -1151,7 +1151,7 @@ curl -X GET "http://0.0.0.0:4000/spend/logs?request_id=<your-call-id" \ # e.g.:
"request_id": "chatcmpl-9ZKMURhVYSi9D6r6PJ9vLcayIK0Vm",
"call_type": "acompletion",
"metadata": {
"user_api_key": "88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b",
"user_api_key": "example-api-key-123",
"user_api_key_alias": null,
"spend_logs_metadata": { # 👈 LOGGED CUSTOM METADATA
"hello": "world"

55
docs/my-website/docs/proxy/custom_pricing.md
View file

@ -9,7 +9,8 @@ LiteLLM provides flexible cost tracking and pricing customization for all LLM pr
- **Custom Pricing** - Override default model costs or set pricing for custom models
- **Cost Per Token** - Track costs based on input/output tokens (most common)
- **Cost Per Second** - Track costs based on runtime (e.g., Sagemaker)
- **Provider Discounts** - Apply percentage-based discounts to specific providers
- **[Provider Discounts](./provider_discounts.md)** - Apply percentage-based discounts to specific providers
- **[Provider Margins](./provider_margins.md)** - Add fees/margins to LLM costs for internal billing
- **Base Model Mapping** - Ensure accurate cost tracking for Azure deployments
By default, the response cost is accessible in the logging object via `kwargs["response_cost"]` on success (sync + async). [**Learn More**](../observability/custom_callback.md)
@ -66,58 +67,6 @@ model_list:
output_cost_per_token: 0.000520 # 👈 ONLY to track cost per token
```
## Provider-Specific Cost Discounts
Apply percentage-based discounts to specific providers (e.g., negotiated enterprise pricing).
#### Usage with LiteLLM Proxy Server
**Step 1: Add discount config to config.yaml**
```yaml
# Apply 5% discount to all Vertex AI and Gemini costs
cost_discount_config:
vertex_ai: 0.05 # 5% discount
gemini: 0.05 # 5% discount
openrouter: 0.05 # 5% discount
# openai: 0.10 # 10% discount (example)
```
**Step 2: Start proxy**
```bash
litellm /path/to/config.yaml
```
The discount will be automatically applied to all cost calculations for the configured providers.
#### How Discounts Work
- Discounts are applied **after** all other cost calculations (tokens, caching, tools, etc.)
- The discount is a percentage (0.05 = 5%, 0.10 = 10%, etc.)
- Discounts only apply to the configured providers
- Original cost, discount amount, and final cost are tracked in cost breakdown logs
- Discount information is returned in response headers:
- `x-litellm-response-cost` - Final cost after discount
- `x-litellm-response-cost-original` - Cost before discount
- `x-litellm-response-cost-discount-amount` - Discount amount in USD
#### Supported Providers
You can apply discounts to all LiteLLM supported providers. Common examples:
- `vertex_ai` - Google Vertex AI
- `gemini` - Google Gemini
- `openai` - OpenAI
- `anthropic` - Anthropic
- `azure` - Azure OpenAI
- `bedrock` - AWS Bedrock
- `cohere` - Cohere
- `openrouter` - OpenRouter
See the full list of providers in the [LlmProviders](https://github.com/BerriAI/litellm/blob/main/litellm/types/utils.py) enum.
## Override Model Cost Map
You can override [our model cost map](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json) with your own custom pricing for a mapped model.

2
docs/my-website/docs/proxy/customers.md
View file

@ -103,7 +103,7 @@ Expected Response
{
"spend": 0.0011120000000000001, # 👈 SPEND
"max_budget": null,
"token": "88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b",
"token": "example-api-key-123",
"customer_id": "krrish12", # 👈 CUSTOMER ID
"user_id": null,
"team_id": null,

2
docs/my-website/docs/proxy/guardrails/litellm_content_filter.md
View file

@ -257,7 +257,7 @@ Contact me at [EMAIL_REDACTED]
| `amex` | American Express cards | `3782-822463-10005` |
| `aws_access_key` | AWS access keys | `AKIAIOSFODNN7EXAMPLE` |
| `aws_secret_key` | AWS secret keys | `wJalrXUtnFEMI/K7MDENG/bPxRfi...` |
| `github_token` | GitHub tokens | `ghp_16C7e42F292c6912E7710c838347Ae178B4a` |
| `github_token` | GitHub tokens | `example-github-token-123` |
### Using Prebuilt Patterns

4
docs/my-website/docs/proxy/guardrails/pillar_security.md
View file

@ -790,7 +790,7 @@ curl -X POST "http://localhost:4000/v1/chat/completions" \
"messages": [
{
"role": "user",
"content": "Generate python code that accesses my Github repo using this PAT: ghp_A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8"
"content": "Generate python code that accesses my Github repo using this PAT: example-github-token-123"
}
],
"max_tokens": 50
@ -815,7 +815,7 @@ curl -X POST "http://localhost:4000/v1/chat/completions" \
"type": "github_token",
"start_idx": 66,
"end_idx": 106,
"evidence": "ghp_A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8",
"evidence": "example-github-token-123",
}
]
}

2
docs/my-website/docs/proxy/multiple_admins.md
View file

@ -89,7 +89,7 @@ curl -X POST 'http://0.0.0.0:4000/team/update' \
"id": "bd136c28-edd0-4cb6-b963-f35464cf6f5a",
"updated_at": "2024-06-08 23:41:14.793",
"changed_by": "krrish@berri.ai", # 👈 CHANGED BY
"changed_by_api_key": "88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b",
"changed_by_api_key": "example-api-key-123",
"action": "updated",
"table_name": "LiteLLM_TeamTable",
"object_id": "8bf18b11-7f52-4717-8e1f-7c65f9d01e52",

2
docs/my-website/docs/proxy/prod.md
View file

@ -33,7 +33,7 @@ litellm_settings:
Set slack webhook url in your env
```shell
export SLACK_WEBHOOK_URL="https://hooks.slack.com/services/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"
export SLACK_WEBHOOK_URL="example-slack-webhook-url"
```
Turn off FASTAPI's default info logs

52
docs/my-website/docs/proxy/provider_discounts.md Normal file
View file

@ -0,0 +1,52 @@
# Provider Discounts
Apply percentage-based discounts to specific providers. This is useful for negotiated enterprise pricing with providers.
## Usage with LiteLLM Proxy Server
**Step 1: Add discount config to config.yaml**
```yaml
# Apply 5% discount to all Vertex AI and Gemini costs
cost_discount_config:
vertex_ai: 0.05 # 5% discount
gemini: 0.05 # 5% discount
openrouter: 0.05 # 5% discount
# openai: 0.10 # 10% discount (example)
```
**Step 2: Start proxy**
```bash
litellm /path/to/config.yaml
```
The discount will be automatically applied to all cost calculations for the configured providers.
## How Discounts Work
- Discounts are applied **after** all other cost calculations (tokens, caching, tools, etc.)
- The discount is a percentage (0.05 = 5%, 0.10 = 10%, etc.)
- Discounts only apply to the configured providers
- Original cost, discount amount, and final cost are tracked in cost breakdown logs
- Discount information is returned in response headers:
- `x-litellm-response-cost` - Final cost after discount
- `x-litellm-response-cost-original` - Cost before discount
- `x-litellm-response-cost-discount-amount` - Discount amount in USD
## Supported Providers
You can apply discounts to all LiteLLM supported providers. Common examples:
- `vertex_ai` - Google Vertex AI
- `gemini` - Google Gemini
- `openai` - OpenAI
- `anthropic` - Anthropic
- `azure` - Azure OpenAI
- `bedrock` - AWS Bedrock
- `cohere` - Cohere
- `openrouter` - OpenRouter
See the full list of providers in the [LlmProviders](https://github.com/BerriAI/litellm/blob/main/litellm/types/utils.py) enum.

214
docs/my-website/docs/proxy/provider_margins.md Normal file
View file

@ -0,0 +1,214 @@
# Fee/Price Margin on LLM Costs
Apply percentage-based or fixed-amount margins to specific providers or globally. This is useful for enterprises that need to add operational overhead costs to bill internal consumers.
## When to Use This Feature
If your Generative AI platform involves various operational and architectural overheads, along with infrastructure costs, you may need the capability to apply an additional fee or margin to the total LLM costs.
**Common use cases:**
- **Internal chargebacks** - Add operational overhead costs when billing internal teams
- **Cost recovery** - Recover infrastructure, support, and platform maintenance costs
## Setup Margins via UI
This walkthrough shows how to add a provider margin and view the cost breakdown in the LiteLLM UI.
### Step 1: Navigate to Settings
From the LiteLLM dashboard, click on **Settings** in the left sidebar.
![Click Settings](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-25/a9a42382-1c93-4338-8c7e-c0ebc4ee239f/ascreenshot.jpeg?tl_px=0,730&br_px=2064,1884&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=47,292)
### Step 2: Open Cost Tracking
Click on **Cost Tracking** to access the cost configuration options.
![Click Cost Tracking](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-25/c3ad52c0-1c8d-4be5-bd04-1e37ce186c8e/ascreenshot.jpeg?tl_px=0,730&br_px=2064,1884&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=65,403)
### Step 3: Select Fee/Price Margin
Click on **Fee/Price Margin** - this section allows you to add fees or margins to LLM costs for internal billing and cost recovery.
![Click Fee/Price Margin](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-25/0810c7bf-e927-4ab6-a55d-37c51d8c17af/ascreenshot.jpeg?tl_px=553,0&br_px=2618,1153&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=551,220)
### Step 4: Add Provider Margin
Click **+ Add Provider Margin** to create a new margin configuration.
![Click Add Provider Margin](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-25/8762b7d9-74e5-45eb-acc3-be0d9c5b799d/ascreenshot.jpeg?tl_px=553,2&br_px=2618,1155&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=929,277)
### Step 5: Select Provider
Click the search field to select which provider to apply the margin to.
![Click search field](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-25/7ff01cdc-2749-43f3-a46f-4fd5543446e3/ascreenshot.jpeg?tl_px=507,0&br_px=2572,1153&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=524,177)
You can select **Global (All Providers)** to apply the margin to all providers, or choose a specific provider like Bedrock, OpenAI, or Anthropic.
![Select Global](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-25/c9efe187-0995-45ae-9366-290cb20835a2/ascreenshot.jpeg?tl_px=0,0&br_px=2064,1153&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=485,182)
In this example, we'll select **Bedrock** as the provider.
![Select Bedrock](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-25/ea1524ed-7217-4ee6-9beb-797e3ff08b3a/ascreenshot.jpeg?tl_px=0,0&br_px=2617,1462&force_format=jpeg&q=100&width=1120.0)
### Step 6: Choose Margin Type
Select the margin type. You can choose between **Percentage-based** (e.g., 10% markup) or **Fixed Amount** (e.g., $0.001 per request).
![Click Percentage-based](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-25/137ffea5-0a5e-445a-809f-a85d20701c87/ascreenshot.jpeg?tl_px=0,0&br_px=2064,1153&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=355,259)
For this example, we'll select **Fixed Amount** to add a flat fee per request.
![Click Fixed Amount](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-25/56828562-2bae-4f69-b68e-13b1b6a03aa6/ascreenshot.jpeg?tl_px=0,0&br_px=2064,1153&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=493,252)
### Step 7: Enter Margin Value
Enter the margin value. In this example, we're adding a $25 fixed fee per request.
![Enter margin value](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-25/80018d4b-0205-43a3-a534-9a0e39ddf139/ascreenshot.jpeg?tl_px=0,0&br_px=2618,1462&force_format=jpeg&q=100&width=1120.0)
### Step 8: Save the Margin
Click **Add Provider Margin** to save your configuration.
![Click Add Provider Margin](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-25/84a5bcb8-f475-4aef-83ec-f0b3b620613f/ascreenshot.jpeg?tl_px=553,206&br_px=2618,1359&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=636,276)
### Step 9: Test the Margin in Playground
Navigate to **Playground** to test your margin configuration by making a request.
![Click Playground](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-25/cda7293a-2439-4301-bc44-211e6d6833a6/ascreenshot.jpeg?tl_px=0,0&br_px=2064,1153&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=37,106)
Select a model and send a test message.
![Send test message](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-25/48c3e28e-a01a-483c-838d-2d1643f44be7/ascreenshot.jpeg?tl_px=0,0&br_px=2617,1462&force_format=jpeg&q=100&width=1120.0)
Enter your prompt in the message field and submit.
![Enter prompt](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-25/88963dbe-6bad-4aac-8bd3-7f4eac0dd995/ascreenshot.jpeg?tl_px=243,730&br_px=2308,1884&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=524,451)
You'll receive a response from the model.
![View response](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-25/1d69ef9c-cc22-40ad-8f10-f14a359d2fb6/ascreenshot.jpeg?tl_px=553,17&br_px=2618,1170&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=549,276)
### Step 10: View Cost Breakdown in Logs
Navigate to **Logs** to view the detailed cost breakdown for your request.
![Click Logs](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-25/5cf6dd8b-0783-41ee-b23a-32f3424c2092/ascreenshot.jpeg?tl_px=0,99&br_px=2064,1252&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=32,276)
Click on the expand icon to view the request details.
![Click expand icon](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-25/3ae2900f-1515-4bb9-a4aa-328b43f13b61/ascreenshot.jpeg?tl_px=0,12&br_px=2064,1165&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=187,277)
### Step 11: View Cost Breakdown Details
Click on **Cost Breakdown** to see how the total cost was calculated, including the margin.
![Click Cost Breakdown](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-25/8bce9050-58ca-4860-9e18-1b704e086cf4/ascreenshot.jpeg?tl_px=392,575&br_px=2457,1728&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=524,276)
The cost breakdown shows the margin amount that was added. In this example, you can see the **+$25.00** margin clearly displayed.
![View margin amount](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-25/c4a65d38-a47a-4634-baf2-608447a7d711/ascreenshot.jpeg?tl_px=0,730&br_px=2064,1884&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=388,282)
The total cost reflects the base LLM cost plus the margin, giving you full transparency into your cost structure.
![View total cost](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-25/3b13550d-5255-4818-b3ee-3d4391991c13/ascreenshot.jpeg?tl_px=0,730&br_px=2064,1884&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=384,323)
## Setup Margins via Config
You can also configure margins directly in your `config.yaml` file.
**Step 1: Add margin config to config.yaml**
```yaml
# Apply margins to providers
cost_margin_config:
global: 0.05 # 5% global margin on all providers
openai: 0.10 # 10% margin for OpenAI (overrides global)
anthropic:
fixed_amount: 0.001 # $0.001 fixed fee per request
```
**Step 2: Start proxy**
```bash
litellm /path/to/config.yaml
```
The margin will be automatically applied to all cost calculations for the configured providers.
## How Margins Work
- Margins are applied **after** discounts (if configured)
- Margins are calculated independently from discounts
- You can use:
- **Percentage-based**: `{"openai": 0.10}` = 10% margin
- **Fixed amount**: `{"openai": {"fixed_amount": 0.001}}` = $0.001 per request
- **Global**: `{"global": 0.05}` = 5% margin on all providers (unless provider-specific margin exists)
- Provider-specific margins override global margins
- Margin information is tracked in cost breakdown logs
- Margin information is returned in response headers:
- `x-litellm-response-cost-margin-amount` - Total margin added in USD
- `x-litellm-response-cost-margin-percent` - Margin percentage applied
## Margin Calculation Examples
**Example 1: Percentage-only margin**
```yaml
cost_margin_config:
openai: 0.10 # 10% margin
```
If base cost is $1.00, final cost = $1.00 x 1.10 = $1.10
**Example 2: Fixed amount only**
```yaml
cost_margin_config:
anthropic:
fixed_amount: 0.001 # $0.001 per request
```
If base cost is $1.00, final cost = $1.00 + $0.001 = $1.001
**Example 3: Global margin with provider override**
```yaml
cost_margin_config:
global: 0.05 # 5% global margin
openai: 0.10 # 10% margin for OpenAI (overrides global)
```
- OpenAI requests: 10% margin applied
- All other providers: 5% margin applied
## Margins with Discounts
Margins and discounts are calculated independently:
1. Base cost is calculated
2. Discount is applied (if configured)
3. Margin is applied to the discounted cost
**Example:**
```yaml
cost_discount_config:
openai: 0.05 # 5% discount
cost_margin_config:
openai: 0.10 # 10% margin
```
If base cost is $1.00:
- After discount: $1.00 x 0.95 = $0.95
- After margin: $0.95 x 1.10 = $1.045
## Supported Providers
You can apply margins to all LiteLLM supported providers, or use `global` to apply to all providers. Common examples:
- `global` - Applies to all providers (unless provider-specific margin exists)
- `openai` - OpenAI
- `anthropic` - Anthropic
- `vertex_ai` - Google Vertex AI
- `gemini` - Google Gemini
- `azure` - Azure OpenAI
- `bedrock` - AWS Bedrock
See the full list of providers in the [LlmProviders](https://github.com/BerriAI/litellm/blob/main/litellm/types/utils.py) enum.

2
docs/my-website/docs/proxy/quick_start.md
View file

@ -400,7 +400,7 @@ from anthropic import Anthropic
client = Anthropic(
base_url="http://localhost:4000", # proxy endpoint
api_key="sk-s4xN1IiLTCytwtZFJaYQrA", # litellm proxy virtual key
api_key="sk-test-proxy-key-123", # litellm proxy virtual key (example)
)
message = client.messages.create(

2
docs/my-website/docs/proxy/user_keys.md
View file

@ -285,7 +285,7 @@ from anthropic import Anthropic
client = Anthropic(
base_url="http://localhost:4000", # proxy endpoint
api_key="sk-s4xN1IiLTCytwtZFJaYQrA", # litellm proxy virtual key
api_key="sk-test-proxy-key-123", # litellm proxy virtual key (example)
)
message = client.messages.create(

34
docs/my-website/docs/rag_ingest.md
View file

@ -4,9 +4,13 @@ All-in-one document ingestion pipeline: **Upload → Chunk → Embed → Vector
| Feature | Supported |
|---------|-----------|
| Logging | ✅ |
| Logging | Yes |
| Supported Providers | `openai`, `bedrock`, `vertex_ai`, `gemini` |
:::tip
After ingesting documents, use [/rag/query](./rag_query.md) to search and generate responses with your ingested content.
:::
## Quick Start
### OpenAI
@ -82,9 +86,33 @@ curl -X POST "http://localhost:4000/v1/rag/ingest" \
}
```
## Query the Vector Store
## Query with RAG
After ingestion, query with `/vector_stores/{vector_store_id}/search`:
After ingestion, use the [/rag/query](./rag_query.md) endpoint to search and generate LLM responses:
```bash showLineNumbers title="RAG Query"
curl -X POST "http://localhost:4000/v1/rag/query" \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "gpt-4o-mini",
"messages": [{"role": "user", "content": "What is the main topic?"}],
"retrieval_config": {
"vector_store_id": "vs_xyz789",
"custom_llm_provider": "openai",
"top_k": 5
}
}'
```
This will:
1. Search the vector store for relevant context
2. Prepend the context to your messages
3. Generate an LLM response
### Direct Vector Store Search
Alternatively, search the vector store directly with `/vector_stores/{vector_store_id}/search`:
```bash showLineNumbers title="Search the vector store"
curl -X POST "http://localhost:4000/v1/vector_stores/vs_xyz789/search" \

273
docs/my-website/docs/rag_query.md Normal file
View file

@ -0,0 +1,273 @@
# /rag/query
RAG Query endpoint: **Search Vector Store → (Rerank) → LLM Completion**
| Feature | Supported |
|---------|-----------|
| Logging | Yes |
| Streaming | Yes |
| Reranking | Yes (optional) |
| Supported Providers | `openai`, `bedrock`, `vertex_ai` |
## Quick Start
```bash showLineNumbers title="RAG Query with OpenAI"
curl -X POST "http://localhost:4000/v1/rag/query" \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "gpt-4o-mini",
"messages": [{"role": "user", "content": "What is LiteLLM?"}],
"retrieval_config": {
"vector_store_id": "vs_abc123",
"custom_llm_provider": "openai",
"top_k": 5
}
}'
```
## How It Works
The RAG query endpoint performs the following steps:
1. **Extract Query**: Extracts the query text from the last user message
2. **Search Vector Store**: Searches the specified vector store for relevant context
3. **Rerank (Optional)**: Reranks the search results using a reranking model
4. **Generate Response**: Calls the LLM with the retrieved context prepended to the messages
## Response
The response follows the standard OpenAI chat completion format, with additional search metadata:
```json
{
"id": "chatcmpl-abc123",
"object": "chat.completion",
"created": 1703123456,
"model": "gpt-4o-mini",
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": "LiteLLM is a unified interface for 100+ LLMs..."
},
"finish_reason": "stop"
}
],
"usage": {
"prompt_tokens": 150,
"completion_tokens": 50,
"total_tokens": 200
},
"_hidden_params": {
"search_results": {...},
"rerank_results": {...}
}
}
```
## With Reranking
Add a `rerank` configuration to improve result quality:
```bash showLineNumbers title="RAG Query with Reranking"
curl -X POST "http://localhost:4000/v1/rag/query" \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "gpt-4o-mini",
"messages": [{"role": "user", "content": "What is LiteLLM?"}],
"retrieval_config": {
"vector_store_id": "vs_abc123",
"custom_llm_provider": "openai",
"top_k": 10
},
"rerank": {
"enabled": true,
"model": "cohere/rerank-english-v3.0",
"top_n": 3
}
}'
```
## Streaming
Enable streaming for real-time responses:
```bash showLineNumbers title="RAG Query with Streaming"
curl -X POST "http://localhost:4000/v1/rag/query" \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "gpt-4o-mini",
"messages": [{"role": "user", "content": "What is LiteLLM?"}],
"retrieval_config": {
"vector_store_id": "vs_abc123",
"custom_llm_provider": "openai"
},
"stream": true
}'
```
## Request Parameters
### Top-Level
| Parameter | Type | Required | Description |
|-----------|------|----------|-------------|
| `model` | string | Yes | The LLM model to use for generation |
| `messages` | array | Yes | Array of chat messages (OpenAI format) |
| `retrieval_config` | object | Yes | Vector store search configuration |
| `rerank` | object | No | Reranking configuration |
| `stream` | boolean | No | Enable streaming (default: `false`) |
### retrieval_config
| Parameter | Type | Default | Description |
|-----------|------|---------|-------------|
| `vector_store_id` | string | **required** | ID of the vector store to search |
| `custom_llm_provider` | string | `"openai"` | Vector store provider |
| `top_k` | integer | `10` | Number of results to retrieve |
### rerank
| Parameter | Type | Default | Description |
|-----------|------|---------|-------------|
| `enabled` | boolean | `false` | Enable reranking |
| `model` | string | - | Reranking model (e.g., `cohere/rerank-english-v3.0`) |
| `top_n` | integer | `5` | Number of results after reranking |
## End-to-End Example
### 1. Ingest a Document
First, ingest a document using the [/rag/ingest](./rag_ingest.md) endpoint:
```bash showLineNumbers title="Step 1: Ingest"
curl -X POST "http://localhost:4000/v1/rag/ingest" \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d "{
\"file\": {
\"filename\": \"company_docs.txt\",
\"content\": \"$(base64 -i company_docs.txt)\",
\"content_type\": \"text/plain\"
},
\"ingest_options\": {
\"vector_store\": {
\"custom_llm_provider\": \"openai\"
}
}
}"
```
Response:
```json
{
"id": "ingest_abc123",
"status": "completed",
"vector_store_id": "vs_xyz789",
"file_id": "file-123"
}
```
### 2. Query with RAG
Now query the ingested documents:
```bash showLineNumbers title="Step 2: Query"
curl -X POST "http://localhost:4000/v1/rag/query" \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "gpt-4o-mini",
"messages": [
{"role": "user", "content": "What products does the company offer?"}
],
"retrieval_config": {
"vector_store_id": "vs_xyz789",
"custom_llm_provider": "openai",
"top_k": 5
}
}'
```
Response:
```json
{
"id": "chatcmpl-abc123",
"object": "chat.completion",
"model": "gpt-4o-mini",
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": "Based on the company documents, the company offers..."
},
"finish_reason": "stop"
}
]
}
```
## Provider Examples
### Bedrock
```bash showLineNumbers title="RAG Query with Bedrock"
curl -X POST "http://localhost:4000/v1/rag/query" \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
"messages": [{"role": "user", "content": "What is LiteLLM?"}],
"retrieval_config": {
"vector_store_id": "KNOWLEDGE_BASE_ID",
"custom_llm_provider": "bedrock",
"top_k": 5
}
}'
```
### Vertex AI
```bash showLineNumbers title="RAG Query with Vertex AI"
curl -X POST "http://localhost:4000/v1/rag/query" \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "vertex_ai/gemini-1.5-pro",
"messages": [{"role": "user", "content": "What is LiteLLM?"}],
"retrieval_config": {
"vector_store_id": "your-corpus-id",
"custom_llm_provider": "vertex_ai",
"top_k": 5
}
}'
```
## Python SDK
```python showLineNumbers title="Using litellm.aquery()"
import litellm
response = await litellm.aquery(
model="gpt-4o-mini",
messages=[{"role": "user", "content": "What is LiteLLM?"}],
retrieval_config={
"vector_store_id": "vs_abc123",
"custom_llm_provider": "openai",
"top_k": 5,
},
rerank={
"enabled": True,
"model": "cohere/rerank-english-v3.0",
"top_n": 3,
},
)
print(response.choices[0].message.content)
```

3
docs/my-website/docs/text_to_speech.md
View file

@ -14,7 +14,7 @@ import TabItem from '@theme/TabItem';
| Fallbacks | ✅ | Works between supported models |
| Loadbalancing | ✅ | Works between supported models |
| Guardrails | ✅ | Applies to input text (non-streaming only) |
| Supported Providers | OpenAI, Azure OpenAI, Vertex AI, AWS Polly, ElevenLabs | |
| Supported Providers | OpenAI, Azure OpenAI, Vertex AI, AWS Polly, ElevenLabs , MiniMax |
## **LiteLLM Python SDK Usage**
### Quick Start
@ -105,6 +105,7 @@ litellm --config /path/to/config.yaml
| Vertex AI | [Usage](../docs/providers/vertex#text-to-speech-apis) |
| Gemini | [Usage](#gemini-text-to-speech) |
| ElevenLabs | [Usage](../docs/providers/elevenlabs#text-to-speech-tts) |
| MiniMax | [Usage](../docs/providers/minimax#minimax---text-to-speech) |
## `/audio/speech` to `/chat/completions` Bridge

18
docs/my-website/sidebars.js
View file

@ -390,6 +390,8 @@ const sidebars = {
items: [
"proxy/cost_tracking",
"proxy/custom_pricing",
"proxy/provider_margins",
"proxy/provider_discounts",
"proxy/sync_models_github",
"proxy/billing",
],
@ -528,7 +530,14 @@ const sidebars = {
"proxy/pass_through_guardrails"
]
},
"rag_ingest",
{
type: "category",
label: "/rag",
items: [
"rag_ingest",
"rag_query",
]
},
"realtime",
"rerank",
"response_api",
@ -674,9 +683,11 @@ const sidebars = {
"providers/aleph_alpha",
"providers/amazon_nova",
"providers/anyscale",
"providers/apertis",
"providers/baseten",
"providers/bytez",
"providers/cerebras",
"providers/chutes",
"providers/clarifai",
"providers/cloudflare_workers",
"providers/codestral",
@ -722,10 +733,12 @@ const sidebars = {
"providers/meta_llama",
"providers/milvus_vector_stores",
"providers/mistral",
"providers/minimax",
"providers/moonshot",
"providers/morph",
"providers/nebius",
"providers/nlp_cloud",
"providers/nano-gpt",
"providers/novita",
{ type: "doc", id: "providers/nscale", label: "Nscale (EU Sovereign)" },
{
@ -742,6 +755,7 @@ const sidebars = {
"providers/ovhcloud",
"providers/perplexity",
"providers/petals",
"providers/poe",
"providers/publicai",
"providers/predibase",
"providers/pydantic_ai_agent",
@ -758,6 +772,8 @@ const sidebars = {
},
"providers/sambanova",
"providers/sap",
"providers/stability",
"providers/synthetic",
"providers/snowflake",
"providers/togetherai",
"providers/topaz",

678
litellm/__init__.py
View file

@ -75,39 +75,24 @@ from litellm.constants import (
DEFAULT_SOFT_BUDGET,
DEFAULT_ALLOWED_FAILS,
)
from litellm.types.secret_managers.main import (
KeyManagementSystem,
KeyManagementSettings,
)
from litellm.types.proxy.management_endpoints.ui_sso import (
DefaultTeamSSOParams,
LiteLLM_UpperboundKeyGenerateParams,
)
from litellm.types.utils import LlmProviders
from litellm.types.utils import PriorityReservationSettings
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.logging_callback_manager import LoggingCallbackManager
import httpx
import dotenv
from litellm.llms.custom_httpx.async_client_cleanup import register_async_client_cleanup
# register_async_client_cleanup is lazy-loaded and called on first access
litellm_mode = os.getenv("LITELLM_MODE", "DEV") # "PRODUCTION", "DEV"
if litellm_mode == "DEV":
dotenv.load_dotenv()
# Register async client cleanup to prevent resource leaks
register_async_client_cleanup()
####################################################
if set_verbose:
_turn_on_debug()
####################################################
### Callbacks /Logging / Success / Failure Handlers #####
CALLBACK_TYPES = Union[str, Callable, CustomLogger]
CALLBACK_TYPES = Union[str, Callable, "CustomLogger"] # CustomLogger is lazy-loaded
input_callback: List[CALLBACK_TYPES] = []
success_callback: List[CALLBACK_TYPES] = []
failure_callback: List[CALLBACK_TYPES] = []
service_callback: List[CALLBACK_TYPES] = []
logging_callback_manager = LoggingCallbackManager()
# logging_callback_manager is lazy-loaded via __getattr__
_custom_logger_compatible_callbacks_literal = Literal[
"lago",
"openmeter",
@ -160,7 +145,7 @@ _known_custom_logger_compatible_callbacks: List = list(
get_args(_custom_logger_compatible_callbacks_literal)
)
callbacks: List[
Union[Callable, _custom_logger_compatible_callbacks_literal, CustomLogger]
Union[Callable, _custom_logger_compatible_callbacks_literal, "CustomLogger"] # CustomLogger is lazy-loaded
] = []
callback_settings: Dict[str, Dict[str, Any]] = {}
initialized_langfuse_clients: int = 0
@ -177,13 +162,13 @@ generic_api_use_v1: Optional[bool] = (
False # if you want to use v1 generic api logged payload
)
argilla_transformation_object: Optional[Dict[str, Any]] = None
_async_input_callback: List[Union[str, Callable, CustomLogger]] = (
_async_input_callback: List[Union[str, Callable, "CustomLogger"]] = ( # CustomLogger is lazy-loaded
[]
) # internal variable - async custom callbacks are routed here.
_async_success_callback: List[Union[str, Callable, CustomLogger]] = (
_async_success_callback: List[Union[str, Callable, "CustomLogger"]] = ( # CustomLogger is lazy-loaded
[]
) # internal variable - async custom callbacks are routed here.
_async_failure_callback: List[Union[str, Callable, CustomLogger]] = (
_async_failure_callback: List[Union[str, Callable, "CustomLogger"]] = ( # CustomLogger is lazy-loaded
[]
) # internal variable - async custom callbacks are routed here.
pre_call_rules: List[Callable] = []
@ -391,9 +376,7 @@ public_model_groups_links: Dict[str, Union[str, Dict[str, Any]]] = {}
priority_reservation: Optional[
Dict[str, Union[float, "PriorityReservationDict"]]
] = None
priority_reservation_settings: "PriorityReservationSettings" = (
PriorityReservationSettings()
)
# priority_reservation_settings is lazy-loaded via __getattr__
######## Networking Settings ########
@ -426,8 +409,11 @@ secret_manager_client: Optional[Any] = (
None # list of instantiated key management clients - e.g. azure kv, infisical, etc.
)
_google_kms_resource_name: Optional[str] = None
_key_management_system: Optional[KeyManagementSystem] = None
_key_management_settings: KeyManagementSettings = KeyManagementSettings()
_key_management_system: Optional["KeyManagementSystem"] = None
# Note: KeyManagementSettings must be eagerly imported because _key_management_settings
# is accessed during import time in secret_managers/main.py
# We'll import it after the lazy import system is set up
# We can't define it here because KeyManagementSettings is lazy-loaded
#### PII MASKING ####
output_parse_pii: bool = False
#############################################
@ -437,6 +423,13 @@ model_cost = get_model_cost_map(url=model_cost_map_url)
cost_discount_config: Dict[str, float] = (
{}
) # Provider-specific cost discounts {"vertex_ai": 0.05} = 5% discount
cost_margin_config: Dict[str, Union[float, Dict[str, float]]] = (
{}
) # Provider-specific or global cost margins. Examples:
# Percentage: {"openai": 0.10} = 10% margin
# Fixed: {"openai": {"fixed_amount": 0.001}} = $0.001 per request
# Global: {"global": 0.05} = 5% global margin on all providers
# Combined: {"vertex_ai": {"percentage": 0.08, "fixed_amount": 0.0005}}
custom_prompt_dict: Dict[str, dict] = {}
check_provider_endpoint = False
@ -923,7 +916,7 @@ model_list = list(
model_list_set = set(model_list)
provider_list: List[Union[LlmProviders, str]] = list(LlmProviders)
# provider_list is lazy-loaded via __getattr__ to avoid importing LlmProviders at import time
models_by_provider: dict = {
@ -1058,9 +1051,15 @@ openai_image_generation_models = ["dall-e-2", "dall-e-3"]
####### VIDEO GENERATION MODELS ###################
openai_video_generation_models = ["sora-2"]
from .timeout import timeout
# timeout is lazy-loaded via __getattr__
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.litellm_core_utils.core_helpers import remove_index_from_tool_calls
# Import KeyManagementSettings here (before utils import) because _key_management_settings
# is accessed during import time in secret_managers/main.py (via dd_tracing -> datadog -> _service_logger -> utils)
from litellm.types.secret_managers.main import KeyManagementSettings
_key_management_settings: KeyManagementSettings = KeyManagementSettings()
# client must be imported immediately as it's used as a decorator at function definition time
from .utils import client
# Note: Most other utils imports are lazy-loaded via __getattr__ to avoid loading utils.py
@ -1069,32 +1068,11 @@ from .utils import client
from .llms.custom_llm import CustomLLM
from .llms.anthropic.common_utils import AnthropicModelInfo
from .llms.ai21.chat.transformation import AI21ChatConfig, AI21ChatConfig as AI21Config
from .llms.meta_llama.chat.transformation import LlamaAPIConfig
from .llms.anthropic.experimental_pass_through.messages.transformation import (
AnthropicMessagesConfig,
)
from .llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import (
AmazonAnthropicClaudeMessagesConfig,
)
from .llms.together_ai.chat import TogetherAIConfig
from .llms.together_ai.completion.transformation import TogetherAITextCompletionConfig
from .llms.cloudflare.chat.transformation import CloudflareChatConfig
from .llms.novita.chat.transformation import NovitaConfig
from .llms.deprecated_providers.palm import (
PalmConfig,
) # here to prevent breaking changes
from .llms.nlp_cloud.chat.handler import NLPCloudConfig
from .llms.petals.completion.transformation import PetalsConfig
from .llms.deprecated_providers.aleph_alpha import AlephAlphaConfig
from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
VertexGeminiConfig,
VertexGeminiConfig as VertexAIConfig,
)
from .llms.gemini.common_utils import GeminiModelInfo
from .llms.gemini.chat.transformation import (
GoogleAIStudioGeminiConfig,
GoogleAIStudioGeminiConfig as GeminiConfig, # aliased to maintain backwards compatibility
)
from .llms.vertex_ai.vertex_embeddings.transformation import (
@ -1103,227 +1081,21 @@ from .llms.vertex_ai.vertex_embeddings.transformation import (
vertexAITextEmbeddingConfig = VertexAITextEmbeddingConfig()
from .llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import (
VertexAIAnthropicConfig,
)
from .llms.vertex_ai.vertex_ai_partner_models.llama3.transformation import (
VertexAILlama3Config,
)
from .llms.vertex_ai.vertex_ai_partner_models.ai21.transformation import (
VertexAIAi21Config,
)
from .llms.ollama.chat.transformation import OllamaChatConfig
from .llms.ollama.completion.transformation import OllamaConfig
from .llms.sagemaker.completion.transformation import SagemakerConfig
from .llms.sagemaker.chat.transformation import SagemakerChatConfig
from .llms.bedrock.chat.invoke_handler import (
AmazonCohereChatConfig,
bedrock_tool_name_mappings,
)
from .llms.bedrock.common_utils import (
AmazonBedrockGlobalConfig,
)
from .llms.bedrock.chat.invoke_transformations.amazon_ai21_transformation import (
AmazonAI21Config,
)
from .llms.bedrock.chat.invoke_transformations.amazon_nova_transformation import (
AmazonInvokeNovaConfig,
)
from .llms.bedrock.chat.invoke_transformations.amazon_qwen2_transformation import (
AmazonQwen2Config,
)
from .llms.bedrock.chat.invoke_transformations.amazon_qwen3_transformation import (
AmazonQwen3Config,
)
from .llms.bedrock.chat.invoke_transformations.anthropic_claude2_transformation import (
AmazonAnthropicConfig,
)
from .llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import (
AmazonAnthropicClaudeConfig,
)
from .llms.bedrock.chat.invoke_transformations.amazon_cohere_transformation import (
AmazonCohereConfig,
)
from .llms.bedrock.chat.invoke_transformations.amazon_llama_transformation import (
AmazonLlamaConfig,
)
from .llms.bedrock.chat.invoke_transformations.amazon_deepseek_transformation import (
AmazonDeepSeekR1Config,
)
from .llms.bedrock.chat.invoke_transformations.amazon_mistral_transformation import (
AmazonMistralConfig,
)
from .llms.bedrock.chat.invoke_transformations.amazon_titan_transformation import (
AmazonTitanConfig,
)
from .llms.bedrock.chat.invoke_transformations.amazon_twelvelabs_pegasus_transformation import (
AmazonTwelveLabsPegasusConfig,
)
from .llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
AmazonInvokeConfig,
)
from .llms.bedrock.chat.invoke_transformations.amazon_openai_transformation import (
AmazonBedrockOpenAIConfig,
)
from .llms.bedrock.image_generation.amazon_stability1_transformation import AmazonStabilityConfig
from .llms.bedrock.image_generation.amazon_stability3_transformation import AmazonStability3Config
from .llms.bedrock.image_generation.amazon_nova_canvas_transformation import AmazonNovaCanvasConfig
from .llms.bedrock.embed.amazon_titan_g1_transformation import AmazonTitanG1Config
from .llms.bedrock.embed.amazon_titan_multimodal_transformation import (
AmazonTitanMultimodalEmbeddingG1Config,
)
from .llms.bedrock.embed.amazon_titan_v2_transformation import (
AmazonTitanV2Config,
)
from .llms.cohere.chat.transformation import CohereChatConfig
from .llms.cohere.chat.v2_transformation import CohereV2ChatConfig
from .llms.bedrock.embed.cohere_transformation import BedrockCohereEmbeddingConfig
from .llms.bedrock.embed.twelvelabs_marengo_transformation import (
TwelveLabsMarengoEmbeddingConfig,
)
from .llms.bedrock.embed.amazon_nova_transformation import (
AmazonNovaEmbeddingConfig,
)
from .llms.openai.openai import OpenAIConfig, MistralEmbeddingConfig
from .llms.openai.image_variations.transformation import OpenAIImageVariationConfig
from .llms.deepinfra.chat.transformation import DeepInfraConfig
from .llms.deepgram.audio_transcription.transformation import (
DeepgramAudioTranscriptionConfig,
)
from .llms.topaz.common_utils import TopazModelInfo
from .llms.topaz.image_variations.transformation import TopazImageVariationConfig
from litellm.llms.openai.completion.transformation import OpenAITextCompletionConfig
from .llms.groq.chat.transformation import GroqChatConfig
from .llms.sap.chat.transformation import GenAIHubOrchestrationConfig
from .llms.voyage.embedding.transformation import VoyageEmbeddingConfig
from .llms.voyage.embedding.transformation_contextual import (
VoyageContextualEmbeddingConfig,
)
from .llms.infinity.embedding.transformation import InfinityEmbeddingConfig
from .llms.azure_ai.chat.transformation import AzureAIStudioConfig
from .llms.mistral.chat.transformation import MistralConfig
from .llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from .llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig
from .llms.azure.responses.o_series_transformation import (
AzureOpenAIOSeriesResponsesAPIConfig,
)
from .llms.xai.responses.transformation import XAIResponsesAPIConfig
from .llms.litellm_proxy.responses.transformation import (
LiteLLMProxyResponsesAPIConfig,
)
from .llms.gemini.interactions.transformation import GoogleAIStudioInteractionsConfig
from .llms.openai.chat.o_series_transformation import (
OpenAIOSeriesConfig as OpenAIO1Config, # maintain backwards compatibility
OpenAIOSeriesConfig,
)
from .llms.anthropic.skills.transformation import AnthropicSkillsConfig
from .llms.base_llm.skills.transformation import BaseSkillsAPIConfig
from .llms.gradient_ai.chat.transformation import GradientAIConfig
openaiOSeriesConfig = OpenAIOSeriesConfig()
from .llms.openai.chat.gpt_transformation import (
OpenAIGPTConfig,
)
from .llms.openai.chat.gpt_5_transformation import (
OpenAIGPT5Config,
)
from .llms.openai.transcriptions.whisper_transformation import (
OpenAIWhisperAudioTranscriptionConfig,
)
from .llms.openai.transcriptions.gpt_transformation import (
OpenAIGPTAudioTranscriptionConfig,
)
openAIGPTConfig = OpenAIGPTConfig()
from .llms.openai.chat.gpt_audio_transformation import (
OpenAIGPTAudioConfig,
)
openAIGPTAudioConfig = OpenAIGPTAudioConfig()
openAIGPT5Config = OpenAIGPT5Config()
from .llms.nvidia_nim.chat.transformation import NvidiaNimConfig
from .llms.nvidia_nim.embed import NvidiaNimEmbeddingConfig
nvidiaNimConfig = NvidiaNimConfig()
nvidiaNimEmbeddingConfig = NvidiaNimEmbeddingConfig()
from .llms.featherless_ai.chat.transformation import FeatherlessAIConfig
from .llms.cerebras.chat import CerebrasConfig
from .llms.baseten.chat import BasetenConfig
from .llms.sambanova.chat import SambanovaConfig
from .llms.sambanova.embedding.transformation import SambaNovaEmbeddingConfig
from .llms.fireworks_ai.chat.transformation import FireworksAIConfig
from .llms.fireworks_ai.completion.transformation import FireworksAITextCompletionConfig
from .llms.fireworks_ai.audio_transcription.transformation import (
FireworksAIAudioTranscriptionConfig,
)
from .llms.fireworks_ai.embed.fireworks_ai_transformation import (
FireworksAIEmbeddingConfig,
)
from .llms.friendliai.chat.transformation import FriendliaiChatConfig
from .llms.jina_ai.embedding.transformation import JinaAIEmbeddingConfig
from .llms.xai.chat.transformation import XAIChatConfig
# OpenAIOSeriesConfig is lazy loaded - openaiOSeriesConfig will be created on first access
# OpenAIGPTConfig, OpenAIGPT5Config, etc. are lazy loaded - instances will be created on first access
from .llms.xai.common_utils import XAIModelInfo
from .llms.zai.chat.transformation import ZAIChatConfig
from .llms.aiml.chat.transformation import AIMLChatConfig
from .llms.volcengine.chat.transformation import (
VolcEngineChatConfig as VolcEngineConfig,
)
from .llms.codestral.completion.transformation import CodestralTextCompletionConfig
from .llms.azure.azure import (
AzureOpenAIError,
AzureOpenAIAssistantsAPIConfig,
)
from .llms.heroku.chat.transformation import HerokuChatConfig
from .llms.cometapi.chat.transformation import CometAPIConfig
from .llms.azure.chat.gpt_transformation import AzureOpenAIConfig
from .llms.azure.chat.gpt_5_transformation import AzureOpenAIGPT5Config
from .llms.azure.completion.transformation import AzureOpenAITextConfig
from .llms.hosted_vllm.chat.transformation import HostedVLLMChatConfig
from .llms.llamafile.chat.transformation import LlamafileChatConfig
from .llms.litellm_proxy.chat.transformation import LiteLLMProxyChatConfig
from .llms.vllm.completion.transformation import VLLMConfig
from .llms.deepseek.chat.transformation import DeepSeekChatConfig
from .llms.lm_studio.chat.transformation import LMStudioChatConfig
from .llms.lm_studio.embed.transformation import LmStudioEmbeddingConfig
from .llms.nscale.chat.transformation import NscaleConfig
from .llms.perplexity.chat.transformation import PerplexityChatConfig
from .llms.azure.chat.o_series_transformation import AzureOpenAIO1Config
from .llms.watsonx.completion.transformation import IBMWatsonXAIConfig
from .llms.watsonx.chat.transformation import IBMWatsonXChatConfig
from .llms.watsonx.embed.transformation import IBMWatsonXEmbeddingConfig
from .llms.sap.embed.transformation import GenAIHubEmbeddingConfig
from .llms.watsonx.audio_transcription.transformation import (
IBMWatsonXAudioTranscriptionConfig,
)
from .llms.github_copilot.chat.transformation import GithubCopilotConfig
from .llms.github_copilot.responses.transformation import (
GithubCopilotResponsesAPIConfig,
)
from .llms.github_copilot.embedding.transformation import GithubCopilotEmbeddingConfig
from .llms.nebius.chat.transformation import NebiusConfig
from .llms.wandb.chat.transformation import WandbConfig
from .llms.dashscope.chat.transformation import DashScopeChatConfig
from .llms.moonshot.chat.transformation import MoonshotChatConfig
# PublicAI now uses JSON-based configuration (see litellm/llms/openai_like/providers.json)
from .llms.docker_model_runner.chat.transformation import DockerModelRunnerChatConfig
from .llms.v0.chat.transformation import V0ChatConfig
from .llms.oci.chat.transformation import OCIChatConfig
from .llms.morph.chat.transformation import MorphChatConfig
from .llms.ragflow.chat.transformation import RAGFlowConfig
from .llms.lambda_ai.chat.transformation import LambdaAIChatConfig
from .llms.hyperbolic.chat.transformation import HyperbolicChatConfig
from .llms.vercel_ai_gateway.chat.transformation import VercelAIGatewayConfig
from .llms.ovhcloud.chat.transformation import OVHCloudChatConfig
from .llms.ovhcloud.embedding.transformation import OVHCloudEmbeddingConfig
from .llms.cometapi.embed.transformation import CometAPIEmbeddingConfig
from .llms.lemonade.chat.transformation import LemonadeChatConfig
from .llms.snowflake.embedding.transformation import SnowflakeEmbeddingConfig
from .llms.amazon_nova.chat.transformation import AmazonNovaChatConfig
# All remaining configs are now lazy loaded - see _lazy_imports_registry.py
# Import LlmProviders here (before main import) because it's imported during import time
# in multiple places including openai.py (via main import)
from litellm.types.utils import LlmProviders
## Lazy loading this is not straightforward, will leave it here for now.
from .main import * # type: ignore
@ -1485,6 +1257,7 @@ if TYPE_CHECKING:
from .llms.bytez.chat.transformation import BytezChatConfig as BytezChatConfig
from .llms.compactifai.chat.transformation import CompactifAIChatConfig as CompactifAIChatConfig
from .llms.empower.chat.transformation import EmpowerChatConfig as EmpowerChatConfig
from .llms.minimax.chat.transformation import MinimaxChatConfig as MinimaxChatConfig
from .llms.aiohttp_openai.chat.transformation import AiohttpOpenAIChatConfig as AiohttpOpenAIChatConfig
from .llms.huggingface.chat.transformation import HuggingFaceChatConfig as HuggingFaceChatConfig
from .llms.huggingface.embedding.transformation import HuggingFaceEmbeddingConfig as HuggingFaceEmbeddingConfig
@ -1519,6 +1292,167 @@ if TYPE_CHECKING:
from .llms.voyage.rerank.transformation import VoyageRerankConfig as VoyageRerankConfig
from .llms.clarifai.chat.transformation import ClarifaiConfig as ClarifaiConfig
from .llms.ai21.chat.transformation import AI21ChatConfig as AI21ChatConfig
from .llms.meta_llama.chat.transformation import LlamaAPIConfig as LlamaAPIConfig
from .llms.together_ai.completion.transformation import TogetherAITextCompletionConfig as TogetherAITextCompletionConfig
from .llms.cloudflare.chat.transformation import CloudflareChatConfig as CloudflareChatConfig
from .llms.novita.chat.transformation import NovitaConfig as NovitaConfig
from .llms.petals.completion.transformation import PetalsConfig as PetalsConfig
from .llms.ollama.chat.transformation import OllamaChatConfig as OllamaChatConfig
from .llms.ollama.completion.transformation import OllamaConfig as OllamaConfig
from .llms.sagemaker.completion.transformation import SagemakerConfig as SagemakerConfig
from .llms.sagemaker.chat.transformation import SagemakerChatConfig as SagemakerChatConfig
from .llms.cohere.chat.transformation import CohereChatConfig as CohereChatConfig
from .llms.anthropic.experimental_pass_through.messages.transformation import AnthropicMessagesConfig as AnthropicMessagesConfig
from .llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import AmazonAnthropicClaudeMessagesConfig as AmazonAnthropicClaudeMessagesConfig
from .llms.together_ai.chat import TogetherAIConfig as TogetherAIConfig
from .llms.nlp_cloud.chat.handler import NLPCloudConfig as NLPCloudConfig
from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig as VertexGeminiConfig
from .llms.gemini.chat.transformation import GoogleAIStudioGeminiConfig as GoogleAIStudioGeminiConfig
from .llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import VertexAIAnthropicConfig as VertexAIAnthropicConfig
from .llms.vertex_ai.vertex_ai_partner_models.llama3.transformation import VertexAILlama3Config as VertexAILlama3Config
from .llms.vertex_ai.vertex_ai_partner_models.ai21.transformation import VertexAIAi21Config as VertexAIAi21Config
from .llms.bedrock.chat.invoke_handler import AmazonCohereChatConfig as AmazonCohereChatConfig
from .llms.bedrock.common_utils import AmazonBedrockGlobalConfig as AmazonBedrockGlobalConfig
from .llms.bedrock.chat.invoke_transformations.amazon_ai21_transformation import AmazonAI21Config as AmazonAI21Config
from .llms.bedrock.chat.invoke_transformations.amazon_nova_transformation import AmazonInvokeNovaConfig as AmazonInvokeNovaConfig
from .llms.bedrock.chat.invoke_transformations.amazon_qwen2_transformation import AmazonQwen2Config as AmazonQwen2Config
from .llms.bedrock.chat.invoke_transformations.amazon_qwen3_transformation import AmazonQwen3Config as AmazonQwen3Config
from .llms.bedrock.chat.invoke_transformations.anthropic_claude2_transformation import AmazonAnthropicConfig as AmazonAnthropicConfig
from .llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import AmazonAnthropicClaudeConfig as AmazonAnthropicClaudeConfig
from .llms.bedrock.chat.invoke_transformations.amazon_cohere_transformation import AmazonCohereConfig as AmazonCohereConfig
from .llms.bedrock.chat.invoke_transformations.amazon_llama_transformation import AmazonLlamaConfig as AmazonLlamaConfig
from .llms.bedrock.chat.invoke_transformations.amazon_deepseek_transformation import AmazonDeepSeekR1Config as AmazonDeepSeekR1Config
from .llms.bedrock.chat.invoke_transformations.amazon_mistral_transformation import AmazonMistralConfig as AmazonMistralConfig
from .llms.bedrock.chat.invoke_transformations.amazon_titan_transformation import AmazonTitanConfig as AmazonTitanConfig
from .llms.bedrock.chat.invoke_transformations.amazon_twelvelabs_pegasus_transformation import AmazonTwelveLabsPegasusConfig as AmazonTwelveLabsPegasusConfig
from .llms.bedrock.chat.invoke_transformations.base_invoke_transformation import AmazonInvokeConfig as AmazonInvokeConfig
from .llms.bedrock.chat.invoke_transformations.amazon_openai_transformation import AmazonBedrockOpenAIConfig as AmazonBedrockOpenAIConfig
from .llms.bedrock.image_generation.amazon_stability1_transformation import AmazonStabilityConfig as AmazonStabilityConfig
from .llms.bedrock.image_generation.amazon_stability3_transformation import AmazonStability3Config as AmazonStability3Config
from .llms.bedrock.image_generation.amazon_nova_canvas_transformation import AmazonNovaCanvasConfig as AmazonNovaCanvasConfig
from .llms.bedrock.embed.amazon_titan_g1_transformation import AmazonTitanG1Config as AmazonTitanG1Config
from .llms.bedrock.embed.amazon_titan_multimodal_transformation import AmazonTitanMultimodalEmbeddingG1Config as AmazonTitanMultimodalEmbeddingG1Config
from .llms.cohere.chat.v2_transformation import CohereV2ChatConfig as CohereV2ChatConfig
from .llms.bedrock.embed.cohere_transformation import BedrockCohereEmbeddingConfig as BedrockCohereEmbeddingConfig
from .llms.bedrock.embed.twelvelabs_marengo_transformation import TwelveLabsMarengoEmbeddingConfig as TwelveLabsMarengoEmbeddingConfig
from .llms.bedrock.embed.amazon_nova_transformation import AmazonNovaEmbeddingConfig as AmazonNovaEmbeddingConfig
from .llms.openai.openai import OpenAIConfig as OpenAIConfig, MistralEmbeddingConfig as MistralEmbeddingConfig
from .llms.openai.image_variations.transformation import OpenAIImageVariationConfig as OpenAIImageVariationConfig
from .llms.deepgram.audio_transcription.transformation import DeepgramAudioTranscriptionConfig as DeepgramAudioTranscriptionConfig
from .llms.topaz.image_variations.transformation import TopazImageVariationConfig as TopazImageVariationConfig
from litellm.llms.openai.completion.transformation import OpenAITextCompletionConfig as OpenAITextCompletionConfig
from .llms.groq.chat.transformation import GroqChatConfig as GroqChatConfig
from .llms.voyage.embedding.transformation import VoyageEmbeddingConfig as VoyageEmbeddingConfig
from .llms.voyage.embedding.transformation_contextual import VoyageContextualEmbeddingConfig as VoyageContextualEmbeddingConfig
from .llms.infinity.embedding.transformation import InfinityEmbeddingConfig as InfinityEmbeddingConfig
from .llms.azure_ai.chat.transformation import AzureAIStudioConfig as AzureAIStudioConfig
from .llms.mistral.chat.transformation import MistralConfig as MistralConfig
from .llms.openai.responses.transformation import OpenAIResponsesAPIConfig as OpenAIResponsesAPIConfig
from .llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig as AzureOpenAIResponsesAPIConfig
from .llms.azure.responses.o_series_transformation import AzureOpenAIOSeriesResponsesAPIConfig as AzureOpenAIOSeriesResponsesAPIConfig
from .llms.xai.responses.transformation import XAIResponsesAPIConfig as XAIResponsesAPIConfig
from .llms.litellm_proxy.responses.transformation import LiteLLMProxyResponsesAPIConfig as LiteLLMProxyResponsesAPIConfig
from .llms.gemini.interactions.transformation import GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig
from .llms.openai.chat.o_series_transformation import OpenAIOSeriesConfig as OpenAIOSeriesConfig, OpenAIOSeriesConfig as OpenAIO1Config
from .llms.anthropic.skills.transformation import AnthropicSkillsConfig as AnthropicSkillsConfig
from .llms.base_llm.skills.transformation import BaseSkillsAPIConfig as BaseSkillsAPIConfig
from .llms.gradient_ai.chat.transformation import GradientAIConfig as GradientAIConfig
from .llms.openai.chat.gpt_transformation import OpenAIGPTConfig as OpenAIGPTConfig
from .llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config as OpenAIGPT5Config
from .llms.openai.transcriptions.whisper_transformation import OpenAIWhisperAudioTranscriptionConfig as OpenAIWhisperAudioTranscriptionConfig
from .llms.openai.transcriptions.gpt_transformation import OpenAIGPTAudioTranscriptionConfig as OpenAIGPTAudioTranscriptionConfig
from .llms.openai.chat.gpt_audio_transformation import OpenAIGPTAudioConfig as OpenAIGPTAudioConfig
from .llms.nvidia_nim.chat.transformation import NvidiaNimConfig as NvidiaNimConfig
from .llms.nvidia_nim.embed import NvidiaNimEmbeddingConfig as NvidiaNimEmbeddingConfig
# Type stubs for lazy-loaded config instances
openaiOSeriesConfig: OpenAIOSeriesConfig
openAIGPTConfig: OpenAIGPTConfig
openAIGPTAudioConfig: OpenAIGPTAudioConfig
openAIGPT5Config: OpenAIGPT5Config
nvidiaNimConfig: NvidiaNimConfig
nvidiaNimEmbeddingConfig: NvidiaNimEmbeddingConfig
# Import config classes that need type stubs (for mypy) - import with _ prefix to avoid circular reference
from .llms.vllm.completion.transformation import VLLMConfig as _VLLMConfig
from .llms.deepseek.chat.transformation import DeepSeekChatConfig as _DeepSeekChatConfig
from .llms.sap.chat.transformation import GenAIHubOrchestrationConfig as _GenAIHubOrchestrationConfig
from .llms.sap.embed.transformation import GenAIHubEmbeddingConfig as _GenAIHubEmbeddingConfig
from .llms.azure.chat.o_series_transformation import AzureOpenAIO1Config as _AzureOpenAIO1Config
from .llms.perplexity.chat.transformation import PerplexityChatConfig as _PerplexityChatConfig
from .llms.nscale.chat.transformation import NscaleConfig as _NscaleConfig
from .llms.watsonx.chat.transformation import IBMWatsonXChatConfig as _IBMWatsonXChatConfig
from .llms.watsonx.completion.transformation import IBMWatsonXAIConfig as _IBMWatsonXAIConfig
from .llms.litellm_proxy.chat.transformation import LiteLLMProxyChatConfig as _LiteLLMProxyChatConfig
from .llms.deepinfra.chat.transformation import DeepInfraConfig as _DeepInfraConfig
from .llms.llamafile.chat.transformation import LlamafileChatConfig as _LlamafileChatConfig
from .llms.lm_studio.chat.transformation import LMStudioChatConfig as _LMStudioChatConfig
from .llms.lm_studio.embed.transformation import LmStudioEmbeddingConfig as _LmStudioEmbeddingConfig
from .llms.watsonx.embed.transformation import IBMWatsonXEmbeddingConfig as _IBMWatsonXEmbeddingConfig
from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig as _VertexGeminiConfig
# Type stubs for lazy-loaded config classes (to help mypy understand types)
VLLMConfig: Type[_VLLMConfig]
DeepSeekChatConfig: Type[_DeepSeekChatConfig]
GenAIHubOrchestrationConfig: Type[_GenAIHubOrchestrationConfig]
GenAIHubEmbeddingConfig: Type[_GenAIHubEmbeddingConfig]
AzureOpenAIO1Config: Type[_AzureOpenAIO1Config]
PerplexityChatConfig: Type[_PerplexityChatConfig]
NscaleConfig: Type[_NscaleConfig]
IBMWatsonXChatConfig: Type[_IBMWatsonXChatConfig]
IBMWatsonXAIConfig: Type[_IBMWatsonXAIConfig]
LiteLLMProxyChatConfig: Type[_LiteLLMProxyChatConfig]
DeepInfraConfig: Type[_DeepInfraConfig]
LlamafileChatConfig: Type[_LlamafileChatConfig]
LMStudioChatConfig: Type[_LMStudioChatConfig]
LmStudioEmbeddingConfig: Type[_LmStudioEmbeddingConfig]
IBMWatsonXEmbeddingConfig: Type[_IBMWatsonXEmbeddingConfig]
VertexAIConfig: Type[_VertexGeminiConfig] # Alias for VertexGeminiConfig
from .llms.featherless_ai.chat.transformation import FeatherlessAIConfig as FeatherlessAIConfig
from .llms.cerebras.chat import CerebrasConfig as CerebrasConfig
from .llms.baseten.chat import BasetenConfig as BasetenConfig
from .llms.sambanova.chat import SambanovaConfig as SambanovaConfig
from .llms.sambanova.embedding.transformation import SambaNovaEmbeddingConfig as SambaNovaEmbeddingConfig
from .llms.fireworks_ai.chat.transformation import FireworksAIConfig as FireworksAIConfig
from .llms.fireworks_ai.completion.transformation import FireworksAITextCompletionConfig as FireworksAITextCompletionConfig
from .llms.fireworks_ai.audio_transcription.transformation import FireworksAIAudioTranscriptionConfig as FireworksAIAudioTranscriptionConfig
from .llms.fireworks_ai.embed.fireworks_ai_transformation import FireworksAIEmbeddingConfig as FireworksAIEmbeddingConfig
from .llms.friendliai.chat.transformation import FriendliaiChatConfig as FriendliaiChatConfig
from .llms.jina_ai.embedding.transformation import JinaAIEmbeddingConfig as JinaAIEmbeddingConfig
from .llms.xai.chat.transformation import XAIChatConfig as XAIChatConfig
from .llms.zai.chat.transformation import ZAIChatConfig as ZAIChatConfig
from .llms.aiml.chat.transformation import AIMLChatConfig as AIMLChatConfig
from .llms.volcengine.chat.transformation import VolcEngineChatConfig as VolcEngineChatConfig, VolcEngineChatConfig as VolcEngineConfig
from .llms.codestral.completion.transformation import CodestralTextCompletionConfig as CodestralTextCompletionConfig
from .llms.azure.azure import AzureOpenAIAssistantsAPIConfig as AzureOpenAIAssistantsAPIConfig
from .llms.heroku.chat.transformation import HerokuChatConfig as HerokuChatConfig
from .llms.cometapi.chat.transformation import CometAPIConfig as CometAPIConfig
from .llms.azure.chat.gpt_transformation import AzureOpenAIConfig as AzureOpenAIConfig
from .llms.azure.chat.gpt_5_transformation import AzureOpenAIGPT5Config as AzureOpenAIGPT5Config
from .llms.azure.completion.transformation import AzureOpenAITextConfig as AzureOpenAITextConfig
from .llms.hosted_vllm.chat.transformation import HostedVLLMChatConfig as HostedVLLMChatConfig
from .llms.github_copilot.chat.transformation import GithubCopilotConfig as GithubCopilotConfig
from .llms.github_copilot.responses.transformation import GithubCopilotResponsesAPIConfig as GithubCopilotResponsesAPIConfig
from .llms.github_copilot.embedding.transformation import GithubCopilotEmbeddingConfig as GithubCopilotEmbeddingConfig
from .llms.nebius.chat.transformation import NebiusConfig as NebiusConfig
from .llms.wandb.chat.transformation import WandbConfig as WandbConfig
from .llms.dashscope.chat.transformation import DashScopeChatConfig as DashScopeChatConfig
from .llms.moonshot.chat.transformation import MoonshotChatConfig as MoonshotChatConfig
from .llms.docker_model_runner.chat.transformation import DockerModelRunnerChatConfig as DockerModelRunnerChatConfig
from .llms.v0.chat.transformation import V0ChatConfig as V0ChatConfig
from .llms.oci.chat.transformation import OCIChatConfig as OCIChatConfig
from .llms.morph.chat.transformation import MorphChatConfig as MorphChatConfig
from .llms.ragflow.chat.transformation import RAGFlowConfig as RAGFlowConfig
from .llms.lambda_ai.chat.transformation import LambdaAIChatConfig as LambdaAIChatConfig
from .llms.hyperbolic.chat.transformation import HyperbolicChatConfig as HyperbolicChatConfig
from .llms.vercel_ai_gateway.chat.transformation import VercelAIGatewayConfig as VercelAIGatewayConfig
from .llms.ovhcloud.chat.transformation import OVHCloudChatConfig as OVHCloudChatConfig
from .llms.ovhcloud.embedding.transformation import OVHCloudEmbeddingConfig as OVHCloudEmbeddingConfig
from .llms.cometapi.embed.transformation import CometAPIEmbeddingConfig as CometAPIEmbeddingConfig
from .llms.lemonade.chat.transformation import LemonadeChatConfig as LemonadeChatConfig
from .llms.snowflake.embedding.transformation import SnowflakeEmbeddingConfig as SnowflakeEmbeddingConfig
from .llms.amazon_nova.chat.transformation import AmazonNovaChatConfig as AmazonNovaChatConfig
from litellm.caching.llm_caching_handler import LLMClientCache
from litellm.types.llms.bedrock import COHERE_EMBEDDING_INPUT_TYPES
from litellm.types.utils import (
@ -1528,6 +1462,10 @@ if TYPE_CHECKING:
StandardKeyGenerationConfig,
)
from litellm.types.guardrails import GuardrailItem
from litellm.types.proxy.management_endpoints.ui_sso import (
DefaultTeamSSOParams,
LiteLLM_UpperboundKeyGenerateParams,
)
# Cost calculator functions
cost_per_token: Callable[..., Tuple[float, float]]
@ -1572,97 +1510,149 @@ if TYPE_CHECKING:
module_level_aclient: AsyncHTTPHandler
module_level_client: HTTPHandler
# Bedrock tool name mappings instance (lazy-loaded)
from litellm.caching.caching import InMemoryCache
bedrock_tool_name_mappings: InMemoryCache
# Azure exception class (lazy-loaded)
from litellm.llms.azure.common_utils import AzureOpenAIError
# Secret manager types (lazy-loaded)
from litellm.types.secret_managers.main import (
KeyManagementSystem,
KeyManagementSettings, # Not lazy-loaded - needed for _key_management_settings initialization
)
# Custom logger class (lazy-loaded)
from litellm.integrations.custom_logger import CustomLogger
# Logging callback manager class and instance (lazy-loaded)
from litellm.litellm_core_utils.logging_callback_manager import LoggingCallbackManager
logging_callback_manager: LoggingCallbackManager
# provider_list is lazy-loaded
from litellm.types.utils import LlmProviders
provider_list: List[Union[LlmProviders, str]]
# Note: AmazonConverseConfig and OpenAILikeChatConfig are imported above in TYPE_CHECKING block
# Track if async client cleanup has been registered (for lazy loading)
_async_client_cleanup_registered = False
def __getattr__(name: str) -> Any:
"""Lazy import handler"""
from ._lazy_imports import (
COST_CALCULATOR_NAMES,
LITELLM_LOGGING_NAMES,
UTILS_NAMES,
TOKEN_COUNTER_NAMES,
LLM_CLIENT_CACHE_NAMES,
BEDROCK_TYPES_NAMES,
TYPES_UTILS_NAMES,
CACHING_NAMES,
HTTP_HANDLER_NAMES,
DOTPROMPT_NAMES,
LLM_CONFIG_NAMES,
TYPES_NAMES,
)
"""Lazy import handler with cached registry for improved performance."""
global _async_client_cleanup_registered
# Register async client cleanup on first access (only once)
if not _async_client_cleanup_registered:
from litellm.llms.custom_httpx.async_client_cleanup import register_async_client_cleanup
register_async_client_cleanup()
_async_client_cleanup_registered = True
# Lazy load cost_calculator functions
if name in COST_CALCULATOR_NAMES:
from ._lazy_imports import _lazy_import_cost_calculator
return _lazy_import_cost_calculator(name)
# Lazy load litellm_logging functions
if name in LITELLM_LOGGING_NAMES:
from ._lazy_imports import _lazy_import_litellm_logging
return _lazy_import_litellm_logging(name)
# Lazy load utils functions
if name in UTILS_NAMES:
from ._lazy_imports import _lazy_import_utils
return _lazy_import_utils(name)
# Use cached registry from _lazy_imports instead of importing tuples every time
from ._lazy_imports import _get_lazy_import_registry
# Lazy load token counter utilities
if name in TOKEN_COUNTER_NAMES:
from ._lazy_imports import _lazy_import_token_counter
return _lazy_import_token_counter(name)
registry = _get_lazy_import_registry()
# Lazy load Bedrock type aliases
if name in BEDROCK_TYPES_NAMES:
from ._lazy_imports import _lazy_import_bedrock_types
return _lazy_import_bedrock_types(name)
# Lazy load common types.utils symbols
if name in TYPES_UTILS_NAMES:
from ._lazy_imports import _lazy_import_types_utils
return _lazy_import_types_utils(name)
# Lazy load LLM client cache and its singleton
if name in LLM_CLIENT_CACHE_NAMES:
from ._lazy_imports import _lazy_import_llm_client_cache
return _lazy_import_llm_client_cache(name)
# Lazy load caching classes
if name in CACHING_NAMES:
from ._lazy_imports import _lazy_import_caching
return _lazy_import_caching(name)
# Lazy-load HTTP handler singletons used across the codebase
if name in HTTP_HANDLER_NAMES:
from ._lazy_imports import _lazy_import_http_handlers
return _lazy_import_http_handlers(name)
# Lazy load dotprompt integration globals
if name in DOTPROMPT_NAMES:
from ._lazy_imports import _lazy_import_dotprompt
return _lazy_import_dotprompt(name)
# Lazy load LLM config classes
if name in LLM_CONFIG_NAMES:
from ._lazy_imports import _lazy_import_llm_configs
return _lazy_import_llm_configs(name)
# Lazy load types
if name in TYPES_NAMES:
from ._lazy_imports import _lazy_import_types
return _lazy_import_types(name)
# Check if name is in registry and call the cached handler function
if name in registry:
handler_func = registry[name]
return handler_func(name)
# Lazy load encoding from main.py to avoid heavy tiktoken import
if name == "encoding":
from .main import encoding as _encoding
# Cache it in the module's __dict__ for subsequent accesses
import sys
sys.modules[__name__].__dict__["encoding"] = _encoding
return _encoding
from ._lazy_imports import _get_litellm_globals
_globals = _get_litellm_globals()
# Check if already cached
if "encoding" not in _globals:
from .main import encoding as _encoding
_globals["encoding"] = _encoding
return _globals["encoding"]
# Lazy load bedrock_tool_name_mappings instance
if name == "bedrock_tool_name_mappings":
from ._lazy_imports import _get_litellm_globals
_globals = _get_litellm_globals()
# Check if already cached
if "bedrock_tool_name_mappings" not in _globals:
from .llms.bedrock.chat.invoke_handler import bedrock_tool_name_mappings as _bedrock_tool_name_mappings
_globals["bedrock_tool_name_mappings"] = _bedrock_tool_name_mappings
return _globals["bedrock_tool_name_mappings"]
# Lazy load AzureOpenAIError exception class
if name == "AzureOpenAIError":
from ._lazy_imports import _get_litellm_globals
_globals = _get_litellm_globals()
# Check if already cached
if "AzureOpenAIError" not in _globals:
from .llms.azure.common_utils import AzureOpenAIError as _AzureOpenAIError
_globals["AzureOpenAIError"] = _AzureOpenAIError
return _globals["AzureOpenAIError"]
# Lazy load openaiOSeriesConfig instance
if name == "openaiOSeriesConfig":
from ._lazy_imports import _get_litellm_globals
_globals = _get_litellm_globals()
if "openaiOSeriesConfig" not in _globals:
# Import the config class and instantiate it
config_class = __getattr__("OpenAIOSeriesConfig")
_globals["openaiOSeriesConfig"] = config_class()
return _globals["openaiOSeriesConfig"]
# Lazy load other config instances
_config_instances = {
"openAIGPTConfig": "OpenAIGPTConfig",
"openAIGPTAudioConfig": "OpenAIGPTAudioConfig",
"openAIGPT5Config": "OpenAIGPT5Config",
"nvidiaNimConfig": "NvidiaNimConfig",
"nvidiaNimEmbeddingConfig": "NvidiaNimEmbeddingConfig",
}
if name in _config_instances:
from ._lazy_imports import _get_litellm_globals
_globals = _get_litellm_globals()
if name not in _globals:
# Import the config class and instantiate it
config_class = __getattr__(_config_instances[name])
_globals[name] = config_class()
return _globals[name]
# Handle OpenAIO1Config alias
if name == "OpenAIO1Config":
return __getattr__("OpenAIOSeriesConfig")
# Lazy load provider_list
if name == "provider_list":
from ._lazy_imports import _get_litellm_globals
_globals = _get_litellm_globals()
# Check if already cached
if "provider_list" not in _globals:
# LlmProviders is eagerly imported above, so we can import it directly
from litellm.types.utils import LlmProviders
_globals["provider_list"] = list(LlmProviders)
return _globals["provider_list"]
# Lazy load priority_reservation_settings instance
if name == "priority_reservation_settings":
from ._lazy_imports import _get_litellm_globals
_globals = _get_litellm_globals()
# Check if already cached
if "priority_reservation_settings" not in _globals:
# Import the class and instantiate it
PriorityReservationSettings = __getattr__("PriorityReservationSettings")
_globals["priority_reservation_settings"] = PriorityReservationSettings()
return _globals["priority_reservation_settings"]
# Lazy load logging_callback_manager instance
if name == "logging_callback_manager":
from ._lazy_imports import _get_litellm_globals
_globals = _get_litellm_globals()
# Check if already cached
if "logging_callback_manager" not in _globals:
# Import the class and instantiate it
LoggingCallbackManager = __getattr__("LoggingCallbackManager")
_globals["logging_callback_manager"] = LoggingCallbackManager()
return _globals["logging_callback_manager"]
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

1100
litellm/_lazy_imports.py
View file

File diff suppressed because it is too large Load diff

602
litellm/_lazy_imports_registry.py Normal file
View file

@ -0,0 +1,602 @@
"""
Registry data for lazy imports.
This module contains all the name tuples and import maps used by the lazy import system.
Separated from the handler functions for better organization.
"""
# Cost calculator names that support lazy loading via _lazy_import_cost_calculator
COST_CALCULATOR_NAMES = (
"completion_cost",
"cost_per_token",
"response_cost_calculator",
)
# Litellm logging names that support lazy loading via _lazy_import_litellm_logging
LITELLM_LOGGING_NAMES = (
"Logging",
"modify_integration",
)
# Utils names that support lazy loading via _lazy_import_utils
UTILS_NAMES = (
"exception_type", "get_optional_params", "get_response_string", "token_counter",
"create_pretrained_tokenizer", "create_tokenizer", "supports_function_calling",
"supports_web_search", "supports_url_context", "supports_response_schema",
"supports_parallel_function_calling", "supports_vision", "supports_audio_input",
"supports_audio_output", "supports_system_messages", "supports_reasoning",
"get_litellm_params", "acreate", "get_max_tokens", "get_model_info",
"register_prompt_template", "validate_environment", "check_valid_key",
"register_model", "encode", "decode", "_calculate_retry_after", "_should_retry",
"get_supported_openai_params", "get_api_base", "get_first_chars_messages",
"ModelResponse", "ModelResponseStream", "EmbeddingResponse", "ImageResponse",
"TranscriptionResponse", "TextCompletionResponse", "get_provider_fields",
"ModelResponseListIterator", "get_valid_models", "timeout",
)
# Token counter names that support lazy loading via _lazy_import_token_counter
TOKEN_COUNTER_NAMES = (
"get_modified_max_tokens",
)
# LLM client cache names that support lazy loading via _lazy_import_llm_client_cache
LLM_CLIENT_CACHE_NAMES = (
"LLMClientCache",
"in_memory_llm_clients_cache",
)
# Bedrock type names that support lazy loading via _lazy_import_bedrock_types
BEDROCK_TYPES_NAMES = (
"COHERE_EMBEDDING_INPUT_TYPES",
)
# Common types from litellm.types.utils that support lazy loading via
# _lazy_import_types_utils
TYPES_UTILS_NAMES = (
"ImageObject",
"BudgetConfig",
"all_litellm_params",
"_litellm_completion_params",
"CredentialItem",
"PriorityReservationDict",
"StandardKeyGenerationConfig",
"SearchProviders",
"GenericStreamingChunk",
)
# Caching / cache classes that support lazy loading via _lazy_import_caching
CACHING_NAMES = (
"Cache",
"DualCache",
"RedisCache",
"InMemoryCache",
)
# HTTP handler names that support lazy loading via _lazy_import_http_handlers
HTTP_HANDLER_NAMES = (
"module_level_aclient",
"module_level_client",
)
# Dotprompt integration names that support lazy loading via _lazy_import_dotprompt
DOTPROMPT_NAMES = (
"global_prompt_manager",
"global_prompt_directory",
"set_global_prompt_directory",
)
# LLM config classes that support lazy loading via _lazy_import_llm_configs
LLM_CONFIG_NAMES = (
"AmazonConverseConfig",
"OpenAILikeChatConfig",
"GaladrielChatConfig",
"GithubChatConfig",
"AzureAnthropicConfig",
"BytezChatConfig",
"CompactifAIChatConfig",
"EmpowerChatConfig",
"MinimaxChatConfig",
"AiohttpOpenAIChatConfig",
"HuggingFaceChatConfig",
"HuggingFaceEmbeddingConfig",
"OobaboogaConfig",
"MaritalkConfig",
"OpenrouterConfig",
"DataRobotConfig",
"AnthropicConfig",
"AnthropicTextConfig",
"GroqSTTConfig",
"TritonConfig",
"TritonGenerateConfig",
"TritonInferConfig",
"TritonEmbeddingConfig",
"HuggingFaceRerankConfig",
"DatabricksConfig",
"DatabricksEmbeddingConfig",
"PredibaseConfig",
"ReplicateConfig",
"SnowflakeConfig",
"CohereRerankConfig",
"CohereRerankV2Config",
"AzureAIRerankConfig",
"InfinityRerankConfig",
"JinaAIRerankConfig",
"DeepinfraRerankConfig",
"HostedVLLMRerankConfig",
"NvidiaNimRerankConfig",
"NvidiaNimRankingConfig",
"VertexAIRerankConfig",
"FireworksAIRerankConfig",
"VoyageRerankConfig",
"ClarifaiConfig",
"AI21ChatConfig",
"LlamaAPIConfig",
"TogetherAITextCompletionConfig",
"CloudflareChatConfig",
"NovitaConfig",
"PetalsConfig",
"OllamaChatConfig",
"OllamaConfig",
"SagemakerConfig",
"SagemakerChatConfig",
"CohereChatConfig",
"AnthropicMessagesConfig",
"AmazonAnthropicClaudeMessagesConfig",
"TogetherAIConfig",
"NLPCloudConfig",
"VertexGeminiConfig",
"GoogleAIStudioGeminiConfig",
"VertexAIAnthropicConfig",
"VertexAILlama3Config",
"VertexAIAi21Config",
"AmazonCohereChatConfig",
"AmazonBedrockGlobalConfig",
"AmazonAI21Config",
"AmazonInvokeNovaConfig",
"AmazonQwen2Config",
"AmazonQwen3Config",
# Aliases for backwards compatibility
"VertexAIConfig", # Alias for VertexGeminiConfig
"GeminiConfig", # Alias for GoogleAIStudioGeminiConfig
"AmazonAnthropicConfig",
"AmazonAnthropicClaudeConfig",
"AmazonCohereConfig",
"AmazonLlamaConfig",
"AmazonDeepSeekR1Config",
"AmazonMistralConfig",
"AmazonTitanConfig",
"AmazonTwelveLabsPegasusConfig",
"AmazonInvokeConfig",
"AmazonBedrockOpenAIConfig",
"AmazonStabilityConfig",
"AmazonStability3Config",
"AmazonNovaCanvasConfig",
"AmazonTitanG1Config",
"AmazonTitanMultimodalEmbeddingG1Config",
"CohereV2ChatConfig",
"BedrockCohereEmbeddingConfig",
"TwelveLabsMarengoEmbeddingConfig",
"AmazonNovaEmbeddingConfig",
"OpenAIConfig",
"MistralEmbeddingConfig",
"OpenAIImageVariationConfig",
"DeepInfraConfig",
"DeepgramAudioTranscriptionConfig",
"TopazImageVariationConfig",
"OpenAITextCompletionConfig",
"GroqChatConfig",
"GenAIHubOrchestrationConfig",
"VoyageEmbeddingConfig",
"VoyageContextualEmbeddingConfig",
"InfinityEmbeddingConfig",
"AzureAIStudioConfig",
"MistralConfig",
"OpenAIResponsesAPIConfig",
"AzureOpenAIResponsesAPIConfig",
"AzureOpenAIOSeriesResponsesAPIConfig",
"XAIResponsesAPIConfig",
"LiteLLMProxyResponsesAPIConfig",
"GoogleAIStudioInteractionsConfig",
"OpenAIOSeriesConfig",
"AnthropicSkillsConfig",
"BaseSkillsAPIConfig",
"GradientAIConfig",
# Alias for backwards compatibility
"OpenAIO1Config", # Alias for OpenAIOSeriesConfig
"OpenAIGPTConfig",
"OpenAIGPT5Config",
"OpenAIWhisperAudioTranscriptionConfig",
"OpenAIGPTAudioTranscriptionConfig",
"OpenAIGPTAudioConfig",
"NvidiaNimConfig",
"NvidiaNimEmbeddingConfig",
"FeatherlessAIConfig",
"CerebrasConfig",
"BasetenConfig",
"SambanovaConfig",
"SambaNovaEmbeddingConfig",
"FireworksAIConfig",
"FireworksAITextCompletionConfig",
"FireworksAIAudioTranscriptionConfig",
"FireworksAIEmbeddingConfig",
"FriendliaiChatConfig",
"JinaAIEmbeddingConfig",
"XAIChatConfig",
"ZAIChatConfig",
"AIMLChatConfig",
"VolcEngineChatConfig",
"CodestralTextCompletionConfig",
"AzureOpenAIAssistantsAPIConfig",
"HerokuChatConfig",
"CometAPIConfig",
"AzureOpenAIConfig",
"AzureOpenAIGPT5Config",
"AzureOpenAITextConfig",
"HostedVLLMChatConfig",
# Alias for backwards compatibility
"VolcEngineConfig", # Alias for VolcEngineChatConfig
"LlamafileChatConfig",
"LiteLLMProxyChatConfig",
"VLLMConfig",
"DeepSeekChatConfig",
"LMStudioChatConfig",
"LmStudioEmbeddingConfig",
"NscaleConfig",
"PerplexityChatConfig",
"AzureOpenAIO1Config",
"IBMWatsonXAIConfig",
"IBMWatsonXChatConfig",
"IBMWatsonXEmbeddingConfig",
"GenAIHubEmbeddingConfig",
"IBMWatsonXAudioTranscriptionConfig",
"GithubCopilotConfig",
"GithubCopilotResponsesAPIConfig",
"GithubCopilotEmbeddingConfig",
"NebiusConfig",
"WandbConfig",
"DashScopeChatConfig",
"MoonshotChatConfig",
"DockerModelRunnerChatConfig",
"V0ChatConfig",
"OCIChatConfig",
"MorphChatConfig",
"RAGFlowConfig",
"LambdaAIChatConfig",
"HyperbolicChatConfig",
"VercelAIGatewayConfig",
"OVHCloudChatConfig",
"OVHCloudEmbeddingConfig",
"CometAPIEmbeddingConfig",
"LemonadeChatConfig",
"SnowflakeEmbeddingConfig",
"AmazonNovaChatConfig",
)
# Types that support lazy loading via _lazy_import_types
TYPES_NAMES = (
"GuardrailItem",
"DefaultTeamSSOParams",
"LiteLLM_UpperboundKeyGenerateParams",
"KeyManagementSystem",
"PriorityReservationSettings",
"CustomLogger",
"LoggingCallbackManager",
# Note: LlmProviders is NOT lazy-loaded because it's imported during import time
# in multiple places including openai.py (via main import)
# Note: KeyManagementSettings is NOT lazy-loaded because _key_management_settings
# is accessed during import time in secret_managers/main.py
)
# Import maps for registry pattern - reduces repetition
_UTILS_IMPORT_MAP = {
"exception_type": (".utils", "exception_type"),
"get_optional_params": (".utils", "get_optional_params"),
"get_response_string": (".utils", "get_response_string"),
"token_counter": (".utils", "token_counter"),
"create_pretrained_tokenizer": (".utils", "create_pretrained_tokenizer"),
"create_tokenizer": (".utils", "create_tokenizer"),
"supports_function_calling": (".utils", "supports_function_calling"),
"supports_web_search": (".utils", "supports_web_search"),
"supports_url_context": (".utils", "supports_url_context"),
"supports_response_schema": (".utils", "supports_response_schema"),
"supports_parallel_function_calling": (".utils", "supports_parallel_function_calling"),
"supports_vision": (".utils", "supports_vision"),
"supports_audio_input": (".utils", "supports_audio_input"),
"supports_audio_output": (".utils", "supports_audio_output"),
"supports_system_messages": (".utils", "supports_system_messages"),
"supports_reasoning": (".utils", "supports_reasoning"),
"get_litellm_params": (".utils", "get_litellm_params"),
"acreate": (".utils", "acreate"),
"get_max_tokens": (".utils", "get_max_tokens"),
"get_model_info": (".utils", "get_model_info"),
"register_prompt_template": (".utils", "register_prompt_template"),
"validate_environment": (".utils", "validate_environment"),
"check_valid_key": (".utils", "check_valid_key"),
"register_model": (".utils", "register_model"),
"encode": (".utils", "encode"),
"decode": (".utils", "decode"),
"_calculate_retry_after": (".utils", "_calculate_retry_after"),
"_should_retry": (".utils", "_should_retry"),
"get_supported_openai_params": (".utils", "get_supported_openai_params"),
"get_api_base": (".utils", "get_api_base"),
"get_first_chars_messages": (".utils", "get_first_chars_messages"),
"ModelResponse": (".utils", "ModelResponse"),
"ModelResponseStream": (".utils", "ModelResponseStream"),
"EmbeddingResponse": (".utils", "EmbeddingResponse"),
"ImageResponse": (".utils", "ImageResponse"),
"TranscriptionResponse": (".utils", "TranscriptionResponse"),
"TextCompletionResponse": (".utils", "TextCompletionResponse"),
"get_provider_fields": (".utils", "get_provider_fields"),
"ModelResponseListIterator": (".utils", "ModelResponseListIterator"),
"get_valid_models": (".utils", "get_valid_models"),
"timeout": (".timeout", "timeout"),
}
_COST_CALCULATOR_IMPORT_MAP = {
"completion_cost": (".cost_calculator", "completion_cost"),
"cost_per_token": (".cost_calculator", "cost_per_token"),
"response_cost_calculator": (".cost_calculator", "response_cost_calculator"),
}
_TYPES_UTILS_IMPORT_MAP = {
"ImageObject": (".types.utils", "ImageObject"),
"BudgetConfig": (".types.utils", "BudgetConfig"),
"all_litellm_params": (".types.utils", "all_litellm_params"),
"_litellm_completion_params": (".types.utils", "all_litellm_params"), # Alias
"CredentialItem": (".types.utils", "CredentialItem"),
"PriorityReservationDict": (".types.utils", "PriorityReservationDict"),
"StandardKeyGenerationConfig": (".types.utils", "StandardKeyGenerationConfig"),
"SearchProviders": (".types.utils", "SearchProviders"),
"GenericStreamingChunk": (".types.utils", "GenericStreamingChunk"),
}
_TOKEN_COUNTER_IMPORT_MAP = {
"get_modified_max_tokens": ("litellm.litellm_core_utils.token_counter", "get_modified_max_tokens"),
}
_BEDROCK_TYPES_IMPORT_MAP = {
"COHERE_EMBEDDING_INPUT_TYPES": ("litellm.types.llms.bedrock", "COHERE_EMBEDDING_INPUT_TYPES"),
}
_CACHING_IMPORT_MAP = {
"Cache": ("litellm.caching.caching", "Cache"),
"DualCache": ("litellm.caching.caching", "DualCache"),
"RedisCache": ("litellm.caching.caching", "RedisCache"),
"InMemoryCache": ("litellm.caching.caching", "InMemoryCache"),
}
_LITELLM_LOGGING_IMPORT_MAP = {
"Logging": ("litellm.litellm_core_utils.litellm_logging", "Logging"),
"modify_integration": ("litellm.litellm_core_utils.litellm_logging", "modify_integration"),
}
_DOTPROMPT_IMPORT_MAP = {
"global_prompt_manager": ("litellm.integrations.dotprompt", "global_prompt_manager"),
"global_prompt_directory": ("litellm.integrations.dotprompt", "global_prompt_directory"),
"set_global_prompt_directory": ("litellm.integrations.dotprompt", "set_global_prompt_directory"),
}
_TYPES_IMPORT_MAP = {
"GuardrailItem": ("litellm.types.guardrails", "GuardrailItem"),
"DefaultTeamSSOParams": ("litellm.types.proxy.management_endpoints.ui_sso", "DefaultTeamSSOParams"),
"LiteLLM_UpperboundKeyGenerateParams": ("litellm.types.proxy.management_endpoints.ui_sso", "LiteLLM_UpperboundKeyGenerateParams"),
"KeyManagementSystem": ("litellm.types.secret_managers.main", "KeyManagementSystem"),
"PriorityReservationSettings": ("litellm.types.utils", "PriorityReservationSettings"),
"CustomLogger": ("litellm.integrations.custom_logger", "CustomLogger"),
"LoggingCallbackManager": ("litellm.litellm_core_utils.logging_callback_manager", "LoggingCallbackManager"),
}
_LLM_CONFIGS_IMPORT_MAP = {
"AmazonConverseConfig": (".llms.bedrock.chat.converse_transformation", "AmazonConverseConfig"),
"OpenAILikeChatConfig": (".llms.openai_like.chat.handler", "OpenAILikeChatConfig"),
"GaladrielChatConfig": (".llms.galadriel.chat.transformation", "GaladrielChatConfig"),
"GithubChatConfig": (".llms.github.chat.transformation", "GithubChatConfig"),
"AzureAnthropicConfig": (".llms.azure_ai.anthropic.transformation", "AzureAnthropicConfig"),
"BytezChatConfig": (".llms.bytez.chat.transformation", "BytezChatConfig"),
"CompactifAIChatConfig": (".llms.compactifai.chat.transformation", "CompactifAIChatConfig"),
"EmpowerChatConfig": (".llms.empower.chat.transformation", "EmpowerChatConfig"),
"MinimaxChatConfig": (".llms.minimax.chat.transformation", "MinimaxChatConfig"),
"AiohttpOpenAIChatConfig": (".llms.aiohttp_openai.chat.transformation", "AiohttpOpenAIChatConfig"),
"HuggingFaceChatConfig": (".llms.huggingface.chat.transformation", "HuggingFaceChatConfig"),
"HuggingFaceEmbeddingConfig": (".llms.huggingface.embedding.transformation", "HuggingFaceEmbeddingConfig"),
"OobaboogaConfig": (".llms.oobabooga.chat.transformation", "OobaboogaConfig"),
"MaritalkConfig": (".llms.maritalk", "MaritalkConfig"),
"OpenrouterConfig": (".llms.openrouter.chat.transformation", "OpenrouterConfig"),
"DataRobotConfig": (".llms.datarobot.chat.transformation", "DataRobotConfig"),
"AnthropicConfig": (".llms.anthropic.chat.transformation", "AnthropicConfig"),
"AnthropicTextConfig": (".llms.anthropic.completion.transformation", "AnthropicTextConfig"),
"GroqSTTConfig": (".llms.groq.stt.transformation", "GroqSTTConfig"),
"TritonConfig": (".llms.triton.completion.transformation", "TritonConfig"),
"TritonGenerateConfig": (".llms.triton.completion.transformation", "TritonGenerateConfig"),
"TritonInferConfig": (".llms.triton.completion.transformation", "TritonInferConfig"),
"TritonEmbeddingConfig": (".llms.triton.embedding.transformation", "TritonEmbeddingConfig"),
"HuggingFaceRerankConfig": (".llms.huggingface.rerank.transformation", "HuggingFaceRerankConfig"),
"DatabricksConfig": (".llms.databricks.chat.transformation", "DatabricksConfig"),
"DatabricksEmbeddingConfig": (".llms.databricks.embed.transformation", "DatabricksEmbeddingConfig"),
"PredibaseConfig": (".llms.predibase.chat.transformation", "PredibaseConfig"),
"ReplicateConfig": (".llms.replicate.chat.transformation", "ReplicateConfig"),
"SnowflakeConfig": (".llms.snowflake.chat.transformation", "SnowflakeConfig"),
"CohereRerankConfig": (".llms.cohere.rerank.transformation", "CohereRerankConfig"),
"CohereRerankV2Config": (".llms.cohere.rerank_v2.transformation", "CohereRerankV2Config"),
"AzureAIRerankConfig": (".llms.azure_ai.rerank.transformation", "AzureAIRerankConfig"),
"InfinityRerankConfig": (".llms.infinity.rerank.transformation", "InfinityRerankConfig"),
"JinaAIRerankConfig": (".llms.jina_ai.rerank.transformation", "JinaAIRerankConfig"),
"DeepinfraRerankConfig": (".llms.deepinfra.rerank.transformation", "DeepinfraRerankConfig"),
"HostedVLLMRerankConfig": (".llms.hosted_vllm.rerank.transformation", "HostedVLLMRerankConfig"),
"NvidiaNimRerankConfig": (".llms.nvidia_nim.rerank.transformation", "NvidiaNimRerankConfig"),
"NvidiaNimRankingConfig": (".llms.nvidia_nim.rerank.ranking_transformation", "NvidiaNimRankingConfig"),
"VertexAIRerankConfig": (".llms.vertex_ai.rerank.transformation", "VertexAIRerankConfig"),
"FireworksAIRerankConfig": (".llms.fireworks_ai.rerank.transformation", "FireworksAIRerankConfig"),
"VoyageRerankConfig": (".llms.voyage.rerank.transformation", "VoyageRerankConfig"),
"ClarifaiConfig": (".llms.clarifai.chat.transformation", "ClarifaiConfig"),
"AI21ChatConfig": (".llms.ai21.chat.transformation", "AI21ChatConfig"),
"LlamaAPIConfig": (".llms.meta_llama.chat.transformation", "LlamaAPIConfig"),
"TogetherAITextCompletionConfig": (".llms.together_ai.completion.transformation", "TogetherAITextCompletionConfig"),
"CloudflareChatConfig": (".llms.cloudflare.chat.transformation", "CloudflareChatConfig"),
"NovitaConfig": (".llms.novita.chat.transformation", "NovitaConfig"),
"PetalsConfig": (".llms.petals.completion.transformation", "PetalsConfig"),
"OllamaChatConfig": (".llms.ollama.chat.transformation", "OllamaChatConfig"),
"OllamaConfig": (".llms.ollama.completion.transformation", "OllamaConfig"),
"SagemakerConfig": (".llms.sagemaker.completion.transformation", "SagemakerConfig"),
"SagemakerChatConfig": (".llms.sagemaker.chat.transformation", "SagemakerChatConfig"),
"CohereChatConfig": (".llms.cohere.chat.transformation", "CohereChatConfig"),
"AnthropicMessagesConfig": (".llms.anthropic.experimental_pass_through.messages.transformation", "AnthropicMessagesConfig"),
"AmazonAnthropicClaudeMessagesConfig": (".llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation", "AmazonAnthropicClaudeMessagesConfig"),
"TogetherAIConfig": (".llms.together_ai.chat", "TogetherAIConfig"),
"NLPCloudConfig": (".llms.nlp_cloud.chat.handler", "NLPCloudConfig"),
"VertexGeminiConfig": (".llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini", "VertexGeminiConfig"),
"GoogleAIStudioGeminiConfig": (".llms.gemini.chat.transformation", "GoogleAIStudioGeminiConfig"),
"VertexAIAnthropicConfig": (".llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation", "VertexAIAnthropicConfig"),
"VertexAILlama3Config": (".llms.vertex_ai.vertex_ai_partner_models.llama3.transformation", "VertexAILlama3Config"),
"VertexAIAi21Config": (".llms.vertex_ai.vertex_ai_partner_models.ai21.transformation", "VertexAIAi21Config"),
"AmazonCohereChatConfig": (".llms.bedrock.chat.invoke_handler", "AmazonCohereChatConfig"),
"AmazonBedrockGlobalConfig": (".llms.bedrock.common_utils", "AmazonBedrockGlobalConfig"),
"AmazonAI21Config": (".llms.bedrock.chat.invoke_transformations.amazon_ai21_transformation", "AmazonAI21Config"),
"AmazonInvokeNovaConfig": (".llms.bedrock.chat.invoke_transformations.amazon_nova_transformation", "AmazonInvokeNovaConfig"),
"AmazonQwen2Config": (".llms.bedrock.chat.invoke_transformations.amazon_qwen2_transformation", "AmazonQwen2Config"),
"AmazonQwen3Config": (".llms.bedrock.chat.invoke_transformations.amazon_qwen3_transformation", "AmazonQwen3Config"),
# Aliases for backwards compatibility
"VertexAIConfig": (".llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini", "VertexGeminiConfig"), # Alias
"GeminiConfig": (".llms.gemini.chat.transformation", "GoogleAIStudioGeminiConfig"), # Alias
"AmazonAnthropicConfig": (".llms.bedrock.chat.invoke_transformations.anthropic_claude2_transformation", "AmazonAnthropicConfig"),
"AmazonAnthropicClaudeConfig": (".llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation", "AmazonAnthropicClaudeConfig"),
"AmazonCohereConfig": (".llms.bedrock.chat.invoke_transformations.amazon_cohere_transformation", "AmazonCohereConfig"),
"AmazonLlamaConfig": (".llms.bedrock.chat.invoke_transformations.amazon_llama_transformation", "AmazonLlamaConfig"),
"AmazonDeepSeekR1Config": (".llms.bedrock.chat.invoke_transformations.amazon_deepseek_transformation", "AmazonDeepSeekR1Config"),
"AmazonMistralConfig": (".llms.bedrock.chat.invoke_transformations.amazon_mistral_transformation", "AmazonMistralConfig"),
"AmazonTitanConfig": (".llms.bedrock.chat.invoke_transformations.amazon_titan_transformation", "AmazonTitanConfig"),
"AmazonTwelveLabsPegasusConfig": (".llms.bedrock.chat.invoke_transformations.amazon_twelvelabs_pegasus_transformation", "AmazonTwelveLabsPegasusConfig"),
"AmazonInvokeConfig": (".llms.bedrock.chat.invoke_transformations.base_invoke_transformation", "AmazonInvokeConfig"),
"AmazonBedrockOpenAIConfig": (".llms.bedrock.chat.invoke_transformations.amazon_openai_transformation", "AmazonBedrockOpenAIConfig"),
"AmazonStabilityConfig": (".llms.bedrock.image_generation.amazon_stability1_transformation", "AmazonStabilityConfig"),
"AmazonStability3Config": (".llms.bedrock.image_generation.amazon_stability3_transformation", "AmazonStability3Config"),
"AmazonNovaCanvasConfig": (".llms.bedrock.image_generation.amazon_nova_canvas_transformation", "AmazonNovaCanvasConfig"),
"AmazonTitanG1Config": (".llms.bedrock.embed.amazon_titan_g1_transformation", "AmazonTitanG1Config"),
"AmazonTitanMultimodalEmbeddingG1Config": (".llms.bedrock.embed.amazon_titan_multimodal_transformation", "AmazonTitanMultimodalEmbeddingG1Config"),
"CohereV2ChatConfig": (".llms.cohere.chat.v2_transformation", "CohereV2ChatConfig"),
"BedrockCohereEmbeddingConfig": (".llms.bedrock.embed.cohere_transformation", "BedrockCohereEmbeddingConfig"),
"TwelveLabsMarengoEmbeddingConfig": (".llms.bedrock.embed.twelvelabs_marengo_transformation", "TwelveLabsMarengoEmbeddingConfig"),
"AmazonNovaEmbeddingConfig": (".llms.bedrock.embed.amazon_nova_transformation", "AmazonNovaEmbeddingConfig"),
"OpenAIConfig": (".llms.openai.openai", "OpenAIConfig"),
"MistralEmbeddingConfig": (".llms.openai.openai", "MistralEmbeddingConfig"),
"OpenAIImageVariationConfig": (".llms.openai.image_variations.transformation", "OpenAIImageVariationConfig"),
"DeepInfraConfig": (".llms.deepinfra.chat.transformation", "DeepInfraConfig"),
"DeepgramAudioTranscriptionConfig": (".llms.deepgram.audio_transcription.transformation", "DeepgramAudioTranscriptionConfig"),
"TopazImageVariationConfig": (".llms.topaz.image_variations.transformation", "TopazImageVariationConfig"),
"OpenAITextCompletionConfig": ("litellm.llms.openai.completion.transformation", "OpenAITextCompletionConfig"),
"GroqChatConfig": (".llms.groq.chat.transformation", "GroqChatConfig"),
"GenAIHubOrchestrationConfig": (".llms.sap.chat.transformation", "GenAIHubOrchestrationConfig"),
"VoyageEmbeddingConfig": (".llms.voyage.embedding.transformation", "VoyageEmbeddingConfig"),
"VoyageContextualEmbeddingConfig": (".llms.voyage.embedding.transformation_contextual", "VoyageContextualEmbeddingConfig"),
"InfinityEmbeddingConfig": (".llms.infinity.embedding.transformation", "InfinityEmbeddingConfig"),
"AzureAIStudioConfig": (".llms.azure_ai.chat.transformation", "AzureAIStudioConfig"),
"MistralConfig": (".llms.mistral.chat.transformation", "MistralConfig"),
"OpenAIResponsesAPIConfig": (".llms.openai.responses.transformation", "OpenAIResponsesAPIConfig"),
"AzureOpenAIResponsesAPIConfig": (".llms.azure.responses.transformation", "AzureOpenAIResponsesAPIConfig"),
"AzureOpenAIOSeriesResponsesAPIConfig": (".llms.azure.responses.o_series_transformation", "AzureOpenAIOSeriesResponsesAPIConfig"),
"XAIResponsesAPIConfig": (".llms.xai.responses.transformation", "XAIResponsesAPIConfig"),
"LiteLLMProxyResponsesAPIConfig": (".llms.litellm_proxy.responses.transformation", "LiteLLMProxyResponsesAPIConfig"),
"GoogleAIStudioInteractionsConfig": (".llms.gemini.interactions.transformation", "GoogleAIStudioInteractionsConfig"),
"OpenAIOSeriesConfig": (".llms.openai.chat.o_series_transformation", "OpenAIOSeriesConfig"),
"AnthropicSkillsConfig": (".llms.anthropic.skills.transformation", "AnthropicSkillsConfig"),
"BaseSkillsAPIConfig": (".llms.base_llm.skills.transformation", "BaseSkillsAPIConfig"),
"GradientAIConfig": (".llms.gradient_ai.chat.transformation", "GradientAIConfig"),
# Alias for backwards compatibility
"OpenAIO1Config": (".llms.openai.chat.o_series_transformation", "OpenAIOSeriesConfig"), # Alias
"OpenAIGPTConfig": (".llms.openai.chat.gpt_transformation", "OpenAIGPTConfig"),
"OpenAIGPT5Config": (".llms.openai.chat.gpt_5_transformation", "OpenAIGPT5Config"),
"OpenAIWhisperAudioTranscriptionConfig": (".llms.openai.transcriptions.whisper_transformation", "OpenAIWhisperAudioTranscriptionConfig"),
"OpenAIGPTAudioTranscriptionConfig": (".llms.openai.transcriptions.gpt_transformation", "OpenAIGPTAudioTranscriptionConfig"),
"OpenAIGPTAudioConfig": (".llms.openai.chat.gpt_audio_transformation", "OpenAIGPTAudioConfig"),
"NvidiaNimConfig": (".llms.nvidia_nim.chat.transformation", "NvidiaNimConfig"),
"NvidiaNimEmbeddingConfig": (".llms.nvidia_nim.embed", "NvidiaNimEmbeddingConfig"),
"FeatherlessAIConfig": (".llms.featherless_ai.chat.transformation", "FeatherlessAIConfig"),
"CerebrasConfig": (".llms.cerebras.chat", "CerebrasConfig"),
"BasetenConfig": (".llms.baseten.chat", "BasetenConfig"),
"SambanovaConfig": (".llms.sambanova.chat", "SambanovaConfig"),
"SambaNovaEmbeddingConfig": (".llms.sambanova.embedding.transformation", "SambaNovaEmbeddingConfig"),
"FireworksAIConfig": (".llms.fireworks_ai.chat.transformation", "FireworksAIConfig"),
"FireworksAITextCompletionConfig": (".llms.fireworks_ai.completion.transformation", "FireworksAITextCompletionConfig"),
"FireworksAIAudioTranscriptionConfig": (".llms.fireworks_ai.audio_transcription.transformation", "FireworksAIAudioTranscriptionConfig"),
"FireworksAIEmbeddingConfig": (".llms.fireworks_ai.embed.fireworks_ai_transformation", "FireworksAIEmbeddingConfig"),
"FriendliaiChatConfig": (".llms.friendliai.chat.transformation", "FriendliaiChatConfig"),
"JinaAIEmbeddingConfig": (".llms.jina_ai.embedding.transformation", "JinaAIEmbeddingConfig"),
"XAIChatConfig": (".llms.xai.chat.transformation", "XAIChatConfig"),
"ZAIChatConfig": (".llms.zai.chat.transformation", "ZAIChatConfig"),
"AIMLChatConfig": (".llms.aiml.chat.transformation", "AIMLChatConfig"),
"VolcEngineChatConfig": (".llms.volcengine.chat.transformation", "VolcEngineChatConfig"),
"CodestralTextCompletionConfig": (".llms.codestral.completion.transformation", "CodestralTextCompletionConfig"),
"AzureOpenAIAssistantsAPIConfig": (".llms.azure.azure", "AzureOpenAIAssistantsAPIConfig"),
"HerokuChatConfig": (".llms.heroku.chat.transformation", "HerokuChatConfig"),
"CometAPIConfig": (".llms.cometapi.chat.transformation", "CometAPIConfig"),
"AzureOpenAIConfig": (".llms.azure.chat.gpt_transformation", "AzureOpenAIConfig"),
"AzureOpenAIGPT5Config": (".llms.azure.chat.gpt_5_transformation", "AzureOpenAIGPT5Config"),
"AzureOpenAITextConfig": (".llms.azure.completion.transformation", "AzureOpenAITextConfig"),
"HostedVLLMChatConfig": (".llms.hosted_vllm.chat.transformation", "HostedVLLMChatConfig"),
# Alias for backwards compatibility
"VolcEngineConfig": (".llms.volcengine.chat.transformation", "VolcEngineChatConfig"), # Alias
"LlamafileChatConfig": (".llms.llamafile.chat.transformation", "LlamafileChatConfig"),
"LiteLLMProxyChatConfig": (".llms.litellm_proxy.chat.transformation", "LiteLLMProxyChatConfig"),
"VLLMConfig": (".llms.vllm.completion.transformation", "VLLMConfig"),
"DeepSeekChatConfig": (".llms.deepseek.chat.transformation", "DeepSeekChatConfig"),
"LMStudioChatConfig": (".llms.lm_studio.chat.transformation", "LMStudioChatConfig"),
"LmStudioEmbeddingConfig": (".llms.lm_studio.embed.transformation", "LmStudioEmbeddingConfig"),
"NscaleConfig": (".llms.nscale.chat.transformation", "NscaleConfig"),
"PerplexityChatConfig": (".llms.perplexity.chat.transformation", "PerplexityChatConfig"),
"AzureOpenAIO1Config": (".llms.azure.chat.o_series_transformation", "AzureOpenAIO1Config"),
"IBMWatsonXAIConfig": (".llms.watsonx.completion.transformation", "IBMWatsonXAIConfig"),
"IBMWatsonXChatConfig": (".llms.watsonx.chat.transformation", "IBMWatsonXChatConfig"),
"IBMWatsonXEmbeddingConfig": (".llms.watsonx.embed.transformation", "IBMWatsonXEmbeddingConfig"),
"GenAIHubEmbeddingConfig": (".llms.sap.embed.transformation", "GenAIHubEmbeddingConfig"),
"IBMWatsonXAudioTranscriptionConfig": (".llms.watsonx.audio_transcription.transformation", "IBMWatsonXAudioTranscriptionConfig"),
"GithubCopilotConfig": (".llms.github_copilot.chat.transformation", "GithubCopilotConfig"),
"GithubCopilotResponsesAPIConfig": (".llms.github_copilot.responses.transformation", "GithubCopilotResponsesAPIConfig"),
"GithubCopilotEmbeddingConfig": (".llms.github_copilot.embedding.transformation", "GithubCopilotEmbeddingConfig"),
"NebiusConfig": (".llms.nebius.chat.transformation", "NebiusConfig"),
"WandbConfig": (".llms.wandb.chat.transformation", "WandbConfig"),
"DashScopeChatConfig": (".llms.dashscope.chat.transformation", "DashScopeChatConfig"),
"MoonshotChatConfig": (".llms.moonshot.chat.transformation", "MoonshotChatConfig"),
"DockerModelRunnerChatConfig": (".llms.docker_model_runner.chat.transformation", "DockerModelRunnerChatConfig"),
"V0ChatConfig": (".llms.v0.chat.transformation", "V0ChatConfig"),
"OCIChatConfig": (".llms.oci.chat.transformation", "OCIChatConfig"),
"MorphChatConfig": (".llms.morph.chat.transformation", "MorphChatConfig"),
"RAGFlowConfig": (".llms.ragflow.chat.transformation", "RAGFlowConfig"),
"LambdaAIChatConfig": (".llms.lambda_ai.chat.transformation", "LambdaAIChatConfig"),
"HyperbolicChatConfig": (".llms.hyperbolic.chat.transformation", "HyperbolicChatConfig"),
"VercelAIGatewayConfig": (".llms.vercel_ai_gateway.chat.transformation", "VercelAIGatewayConfig"),
"OVHCloudChatConfig": (".llms.ovhcloud.chat.transformation", "OVHCloudChatConfig"),
"OVHCloudEmbeddingConfig": (".llms.ovhcloud.embedding.transformation", "OVHCloudEmbeddingConfig"),
"CometAPIEmbeddingConfig": (".llms.cometapi.embed.transformation", "CometAPIEmbeddingConfig"),
"LemonadeChatConfig": (".llms.lemonade.chat.transformation", "LemonadeChatConfig"),
"SnowflakeEmbeddingConfig": (".llms.snowflake.embedding.transformation", "SnowflakeEmbeddingConfig"),
"AmazonNovaChatConfig": (".llms.amazon_nova.chat.transformation", "AmazonNovaChatConfig"),
}
# Export all name tuples and import maps for use in _lazy_imports.py
__all__ = [
# Name tuples
"COST_CALCULATOR_NAMES",
"LITELLM_LOGGING_NAMES",
"UTILS_NAMES",
"TOKEN_COUNTER_NAMES",
"LLM_CLIENT_CACHE_NAMES",
"BEDROCK_TYPES_NAMES",
"TYPES_UTILS_NAMES",
"CACHING_NAMES",
"HTTP_HANDLER_NAMES",
"DOTPROMPT_NAMES",
"LLM_CONFIG_NAMES",
"TYPES_NAMES",
# Import maps
"_UTILS_IMPORT_MAP",
"_COST_CALCULATOR_IMPORT_MAP",
"_TYPES_UTILS_IMPORT_MAP",
"_TOKEN_COUNTER_IMPORT_MAP",
"_BEDROCK_TYPES_IMPORT_MAP",
"_CACHING_IMPORT_MAP",
"_LITELLM_LOGGING_IMPORT_MAP",
"_DOTPROMPT_IMPORT_MAP",
"_TYPES_IMPORT_MAP",
"_LLM_CONFIGS_IMPORT_MAP",
]

18
litellm/constants.py
View file

@ -556,6 +556,11 @@ openai_compatible_endpoints: List = [
"https://dashscope-intl.aliyuncs.com/compatible-mode/v1",
"https://api.moonshot.ai/v1",
"https://api.publicai.co/v1",
"https://api.synthetic.new/openai/v1",
"https://api.stima.tech/v1",
"https://nano-gpt.com/api/v1",
"https://api.poe.com/v1",
"https://llm.chutes.ai/v1/",
"https://api.v0.dev/v1",
"https://api.morphllm.com/v1",
"https://api.lambda.ai/v1",
@ -599,12 +604,16 @@ openai_compatible_providers: List = [
"novita",
"meta_llama",
"publicai", # PublicAI - JSON-configured provider
"synthetic", # Synthetic - JSON-configured provider
"apertis", # Apertis - JSON-configured provider
"nano-gpt", # Nano-GPT - JSON-configured provider
"poe", # Poe - JSON-configured provider
"chutes", # Chutes - JSON-configured provider
"featherless_ai",
"nscale",
"nebius",
"dashscope",
"moonshot",
"publicai",
"v0",
"helicone",
"morph",
@ -630,6 +639,11 @@ openai_text_completion_compatible_providers: List = (
"dashscope",
"moonshot",
"publicai",
"synthetic",
"apertis",
"nano-gpt",
"poe",
"chutes",
"v0",
"lambda_ai",
"hyperbolic",
@ -1186,6 +1200,8 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES = [
"public_agent_groups",
"public_model_groups",
"public_model_groups_links",
"cost_discount_config",
"cost_margin_config",
]
SPECIAL_LITELLM_AUTH_TOKEN = ["ui-token"]
DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL = int(

100
litellm/cost_calculator.py
View file

@ -708,6 +708,69 @@ def _apply_cost_discount(
return base_cost, discount_percent, discount_amount
def _apply_cost_margin(
base_cost: float,
custom_llm_provider: Optional[str],
) -> Tuple[float, float, float, float]:
"""
Apply provider-specific or global cost margin from module-level config.
Args:
base_cost: The base cost before margin (after discount if applicable)
custom_llm_provider: The LLM provider name
Returns:
Tuple of (final_cost, margin_percent, margin_fixed_amount, margin_total_amount)
"""
original_cost = base_cost
margin_percent = 0.0
margin_fixed_amount = 0.0
margin_total_amount = 0.0
# Get margin config - check provider-specific first, then global
margin_config = None
if custom_llm_provider and custom_llm_provider in litellm.cost_margin_config:
margin_config = litellm.cost_margin_config[custom_llm_provider]
verbose_logger.debug(
f"Found provider-specific margin config for {custom_llm_provider}: {margin_config}"
)
elif "global" in litellm.cost_margin_config:
margin_config = litellm.cost_margin_config["global"]
verbose_logger.debug(f"Using global margin config: {margin_config}")
else:
verbose_logger.debug(
f"No margin config found. Provider: {custom_llm_provider}, "
f"Available configs: {list(litellm.cost_margin_config.keys())}"
)
if margin_config is not None:
# Handle different margin config formats
if isinstance(margin_config, (int, float)):
# Simple percentage: {"openai": 0.10}
margin_percent = float(margin_config)
margin_total_amount = original_cost * margin_percent
elif isinstance(margin_config, dict):
# Complex config: {"percentage": 0.08, "fixed_amount": 0.0005}
if "percentage" in margin_config:
margin_percent = float(margin_config["percentage"])
margin_total_amount += original_cost * margin_percent
if "fixed_amount" in margin_config:
margin_fixed_amount = float(margin_config["fixed_amount"])
margin_total_amount += margin_fixed_amount
final_cost = original_cost + margin_total_amount
verbose_logger.debug(
f"Applied margin to {custom_llm_provider or 'global'}: "
f"${original_cost:.6f} -> ${final_cost:.6f} "
f"(margin: {margin_percent*100 if margin_percent > 0 else 0}% + ${margin_fixed_amount:.6f} = ${margin_total_amount:.6f})"
)
return final_cost, margin_percent, margin_fixed_amount, margin_total_amount
return base_cost, margin_percent, margin_fixed_amount, margin_total_amount
def _store_cost_breakdown_in_logging_obj(
litellm_logging_obj: Optional[LitellmLoggingObject],
prompt_tokens_cost_usd_dollar: float,
@ -717,6 +780,9 @@ def _store_cost_breakdown_in_logging_obj(
original_cost: Optional[float] = None,
discount_percent: Optional[float] = None,
discount_amount: Optional[float] = None,
margin_percent: Optional[float] = None,
margin_fixed_amount: Optional[float] = None,
margin_total_amount: Optional[float] = None,
) -> None:
"""
Helper function to store cost breakdown in the logging object.
@ -730,6 +796,9 @@ def _store_cost_breakdown_in_logging_obj(
original_cost: Cost before discount
discount_percent: Discount percentage applied (0.05 = 5%)
discount_amount: Discount amount in USD
margin_percent: Margin percentage applied (0.10 = 10%)
margin_fixed_amount: Fixed margin amount in USD
margin_total_amount: Total margin added in USD
"""
if litellm_logging_obj is None:
return
@ -744,6 +813,9 @@ def _store_cost_breakdown_in_logging_obj(
original_cost=original_cost,
discount_percent=discount_percent,
discount_amount=discount_amount,
margin_percent=margin_percent,
margin_fixed_amount=margin_fixed_amount,
margin_total_amount=margin_total_amount,
)
except Exception as breakdown_error:
@ -1106,6 +1178,17 @@ def completion_cost( # noqa: PLR0915
custom_llm_provider=custom_llm_provider,
)
# Apply margin from module-level config if configured
(
_final_cost,
margin_percent,
margin_fixed_amount,
margin_total_amount,
) = _apply_cost_margin(
base_cost=_final_cost,
custom_llm_provider=custom_llm_provider,
)
# Store cost breakdown in logging object if available
_store_cost_breakdown_in_logging_obj(
litellm_logging_obj=litellm_logging_obj,
@ -1116,6 +1199,9 @@ def completion_cost( # noqa: PLR0915
original_cost=original_cost,
discount_percent=discount_percent,
discount_amount=discount_amount,
margin_percent=margin_percent,
margin_fixed_amount=margin_fixed_amount,
margin_total_amount=margin_total_amount,
)
return _final_cost
@ -1239,6 +1325,17 @@ def completion_cost( # noqa: PLR0915
custom_llm_provider=custom_llm_provider,
)
# Apply margin from module-level config if configured
(
_final_cost,
margin_percent,
margin_fixed_amount,
margin_total_amount,
) = _apply_cost_margin(
base_cost=_final_cost,
custom_llm_provider=custom_llm_provider,
)
# Store cost breakdown in logging object if available
_store_cost_breakdown_in_logging_obj(
litellm_logging_obj=litellm_logging_obj,
@ -1249,6 +1346,9 @@ def completion_cost( # noqa: PLR0915
original_cost=original_cost,
discount_percent=discount_percent,
discount_amount=discount_amount,
margin_percent=margin_percent,
margin_fixed_amount=margin_fixed_amount,
margin_total_amount=margin_total_amount,
)
return _final_cost

52
litellm/images/main.py
View file

@ -2,7 +2,18 @@ import asyncio
import contextvars
import importlib
from functools import partial
from typing import TYPE_CHECKING, Any, Coroutine, Dict, List, Literal, Optional, Union, cast, overload
from typing import (
TYPE_CHECKING,
Any,
Coroutine,
Dict,
List,
Literal,
Optional,
Union,
cast,
overload,
)
if TYPE_CHECKING:
from litellm.images.utils import ImageEditRequestUtils
@ -10,7 +21,7 @@ if TYPE_CHECKING:
import httpx
import litellm
from litellm.utils import exception_type, get_litellm_params
# client is imported from litellm as it's a decorator
from litellm import client
from litellm.constants import DEFAULT_IMAGE_ENDPOINT_MODEL
@ -23,6 +34,7 @@ from litellm.llms.base_llm import BaseImageEditConfig, BaseImageGenerationConfig
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.llms.custom_llm import CustomLLM
from litellm.utils import exception_type, get_litellm_params
#################### Initialize provider clients ####################
llm_http_handler: BaseLLMHTTPHandler = BaseLLMHTTPHandler()
@ -32,8 +44,8 @@ from litellm.main import (
azure_chat_completions,
base_llm_aiohttp_handler,
base_llm_http_handler,
bedrock_image_generation,
bedrock_image_edit,
bedrock_image_generation,
openai_chat_completions,
openai_image_variations,
)
@ -330,11 +342,36 @@ def image_generation( # noqa: PLR0915
azure_ad_token = optional_params.pop(
"azure_ad_token", None
) or get_secret_str("AZURE_AD_TOKEN")
# Create azure_ad_token_provider from tenant_id, client_id, client_secret if not already provided
if azure_ad_token_provider is None:
from litellm.llms.azure.common_utils import (
get_azure_ad_token_from_entra_id,
)
# Extract Azure AD credentials from litellm_params
tenant_id = litellm_params_dict.get("tenant_id")
client_id = litellm_params_dict.get("client_id")
client_secret = litellm_params_dict.get("client_secret")
azure_scope = litellm_params_dict.get("azure_scope") or "https://cognitiveservices.azure.com/.default"
# Create token provider if credentials are available
if tenant_id and client_id and client_secret:
azure_ad_token_provider = get_azure_ad_token_from_entra_id(
tenant_id=tenant_id,
client_id=client_id,
client_secret=client_secret,
scope=azure_scope,
)
default_headers = {
"Content-Type": "application/json",
"api-key": api_key,
}
# Only add api-key header if api_key is not None
# Azure AD authentication will use Authorization header instead
if api_key is not None:
default_headers["api-key"] = api_key
for k, v in default_headers.items():
if k not in headers:
headers[k] = v
@ -399,8 +436,12 @@ def image_generation( # noqa: PLR0915
default_headers = {
"Content-Type": "application/json",
"api-key": api_key,
}
# Only add api-key header if api_key is not None
# Azure AD authentication will use Authorization header instead
if api_key is not None:
default_headers["api-key"] = api_key
for k, v in default_headers.items():
if k not in headers:
headers[k] = v
@ -983,6 +1024,7 @@ def __getattr__(name: str) -> Any:
if name == "ImageEditRequestUtils":
# Lazy load ImageEditRequestUtils to avoid heavy import from images.utils at module load time
from .utils import ImageEditRequestUtils as _ImageEditRequestUtils
# Cache it in the module's __dict__ for subsequent accesses
module = importlib.import_module(__name__)
module.__dict__["ImageEditRequestUtils"] = _ImageEditRequestUtils

47
litellm/integrations/custom_guardrail.py
View file

@ -243,14 +243,14 @@ class CustomGuardrail(CustomLogger):
def _is_valid_response_type(self, result: Any) -> bool:
"""
Check if result is a valid LLMResponseTypes instance.
Safely handles TypedDict types which don't support isinstance checks.
For non-LiteLLM responses (like passthrough httpx.Response), returns True
to allow them through.
"""
if result is None:
return False
try:
# Try isinstance check on valid types that support it
response_types = get_args(LLMResponseTypes)
@ -506,6 +506,7 @@ class CustomGuardrail(CustomLogger):
duration: Optional[float] = None,
masked_entity_count: Optional[Dict[str, int]] = None,
guardrail_provider: Optional[str] = None,
event_type: Optional[GuardrailEventHooks] = None,
) -> None:
"""
Builds `StandardLoggingGuardrailInformation` and adds it to the request metadata so it can be used for logging to DataDog, Langfuse, etc.
@ -514,14 +515,19 @@ class CustomGuardrail(CustomLogger):
guardrail_json_response = str(guardrail_json_response)
from litellm.types.utils import GuardrailMode
# Use event_type if provided, otherwise fall back to self.event_hook
guardrail_mode: Union[GuardrailEventHooks, GuardrailMode, List[GuardrailEventHooks]]
if event_type is not None:
guardrail_mode = event_type
elif isinstance(self.event_hook, Mode):
guardrail_mode = GuardrailMode(**dict(self.event_hook.model_dump())) # type: ignore[typeddict-item]
else:
guardrail_mode = self.event_hook # type: ignore[assignment]
slg = StandardLoggingGuardrailInformation(
guardrail_name=self.guardrail_name,
guardrail_provider=guardrail_provider,
guardrail_mode=(
GuardrailMode(**self.event_hook.model_dump()) # type: ignore
if isinstance(self.event_hook, Mode)
else self.event_hook
),
guardrail_mode=guardrail_mode,
guardrail_response=guardrail_json_response,
guardrail_status=guardrail_status,
start_time=start_time,
@ -589,6 +595,7 @@ class CustomGuardrail(CustomLogger):
start_time: Optional[float] = None,
end_time: Optional[float] = None,
duration: Optional[float] = None,
event_type: Optional[GuardrailEventHooks] = None,
):
"""
Add StandardLoggingGuardrailInformation to the request data
@ -605,6 +612,7 @@ class CustomGuardrail(CustomLogger):
duration=duration,
start_time=start_time,
end_time=end_time,
event_type=event_type,
)
return response
@ -615,6 +623,7 @@ class CustomGuardrail(CustomLogger):
start_time: Optional[float] = None,
end_time: Optional[float] = None,
duration: Optional[float] = None,
event_type: Optional[GuardrailEventHooks] = None,
):
"""
Add StandardLoggingGuardrailInformation to the request data
@ -628,6 +637,7 @@ class CustomGuardrail(CustomLogger):
duration=duration,
start_time=start_time,
end_time=end_time,
event_type=event_type,
)
raise e
@ -712,16 +722,32 @@ def log_guardrail_information(func):
Logs for:
- pre_call
- during_call
- TODO: log post_call. This is more involved since the logs are sent to DD, s3 before the guardrail is even run
- post_call
"""
import asyncio
import functools
def _infer_event_type_from_function_name(
func_name: str,
) -> Optional[GuardrailEventHooks]:
"""Infer the actual event type from the function name"""
if func_name == "async_pre_call_hook":
return GuardrailEventHooks.pre_call
elif func_name == "async_moderation_hook":
return GuardrailEventHooks.during_call
elif func_name in (
"async_post_call_success_hook",
"async_post_call_streaming_hook",
):
return GuardrailEventHooks.post_call
return None
@functools.wraps(func)
async def async_wrapper(*args, **kwargs):
start_time = datetime.now() # Move start_time inside the wrapper
self: CustomGuardrail = args[0]
request_data: dict = kwargs.get("data") or kwargs.get("request_data") or {}
event_type = _infer_event_type_from_function_name(func.__name__)
try:
response = await func(*args, **kwargs)
return self._process_response(
@ -730,6 +756,7 @@ def log_guardrail_information(func):
start_time=start_time.timestamp(),
end_time=datetime.now().timestamp(),
duration=(datetime.now() - start_time).total_seconds(),
event_type=event_type,
)
except Exception as e:
return self._process_error(
@ -738,6 +765,7 @@ def log_guardrail_information(func):
start_time=start_time.timestamp(),
end_time=datetime.now().timestamp(),
duration=(datetime.now() - start_time).total_seconds(),
event_type=event_type,
)
@functools.wraps(func)
@ -745,18 +773,21 @@ def log_guardrail_information(func):
start_time = datetime.now() # Move start_time inside the wrapper
self: CustomGuardrail = args[0]
request_data: dict = kwargs.get("data") or kwargs.get("request_data") or {}
event_type = _infer_event_type_from_function_name(func.__name__)
try:
response = func(*args, **kwargs)
return self._process_response(
response=response,
request_data=request_data,
duration=(datetime.now() - start_time).total_seconds(),
event_type=event_type,
)
except Exception as e:
return self._process_error(
e=e,
request_data=request_data,
duration=(datetime.now() - start_time).total_seconds(),
event_type=event_type,
)
@functools.wraps(func)

17
litellm/integrations/custom_logger.py
View file

@ -32,6 +32,8 @@ from litellm.types.utils import (
)
if TYPE_CHECKING:
from fastapi import HTTPException
from litellm.caching.caching import DualCache
from opentelemetry.trace import Span as _Span
@ -348,7 +350,20 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
original_exception: Exception,
user_api_key_dict: UserAPIKeyAuth,
traceback_str: Optional[str] = None,
):
) -> Optional["HTTPException"]:
"""
Called after an LLM API call fails. Can return or raise HTTPException to transform error responses.
Args:
- request_data: dict - The request data.
- original_exception: Exception - The original exception that occurred.
- user_api_key_dict: UserAPIKeyAuth - The user API key dictionary.
- traceback_str: Optional[str] - The traceback string.
Returns:
- Optional[HTTPException]: Return an HTTPException to transform the error response sent to the client.
Return None to use the original exception.
"""
pass
async def async_post_call_success_hook(

20
litellm/integrations/datadog/datadog_llm_obs.py
View file

@ -217,8 +217,14 @@ class DataDogLLMObsLogger(CustomBatchLogger):
error_info = self._assemble_error_info(standard_logging_payload)
metadata_parent_id: Optional[str] = None
if isinstance(metadata, dict):
metadata_parent_id = metadata.get("parent_id")
meta = Meta(
kind=self._get_datadog_span_kind(standard_logging_payload.get("call_type")),
kind=self._get_datadog_span_kind(
standard_logging_payload.get("call_type"), metadata_parent_id
),
input=input_meta,
output=output_meta,
metadata=self._get_dd_llm_obs_payload_metadata(standard_logging_payload),
@ -237,7 +243,7 @@ class DataDogLLMObsLogger(CustomBatchLogger):
)
payload: LLMObsPayload = LLMObsPayload(
parent_id=metadata.get("parent_id", "undefined"),
parent_id=metadata_parent_id if metadata_parent_id else "undefined",
trace_id=standard_logging_payload.get("trace_id", str(uuid.uuid4())),
span_id=metadata.get("span_id", str(uuid.uuid4())),
name=metadata.get("name", "litellm_llm_call"),
@ -367,14 +373,16 @@ class DataDogLLMObsLogger(CustomBatchLogger):
return []
def _get_datadog_span_kind(
self, call_type: Optional[str]
self, call_type: Optional[str], parent_id: Optional[str] = None
) -> Literal["llm", "tool", "task", "embedding", "retrieval"]:
"""
Map liteLLM call_type to appropriate DataDog LLM Observability span kind.
Available DataDog span kinds: "llm", "tool", "task", "embedding", "retrieval"
see: https://docs.datadoghq.com/ja/llm_observability/terms/
"""
if call_type is None:
# Non llm/workflow/agent kinds cannot be root spans, so fallback to "llm" when parent metadata is missing
if call_type is None or parent_id is None:
return "llm"
# Embedding operations
@ -392,6 +400,8 @@ class DataDogLLMObsLogger(CustomBatchLogger):
CallTypes.generate_content_stream.value,
CallTypes.agenerate_content_stream.value,
CallTypes.anthropic_messages.value,
CallTypes.responses.value,
CallTypes.aresponses.value,
]:
return "llm"
@ -417,8 +427,6 @@ class DataDogLLMObsLogger(CustomBatchLogger):
CallTypes.aretrieve_batch.value,
CallTypes.retrieve_fine_tuning_job.value,
CallTypes.aretrieve_fine_tuning_job.value,
CallTypes.responses.value,
CallTypes.aresponses.value,
CallTypes.alist_input_items.value,
]:
return "retrieval"

16
litellm/interactions/litellm_responses_transformation/__init__.py Normal file
View file

@ -0,0 +1,16 @@
"""
Bridge module for connecting Interactions API to Responses API via litellm.responses().
"""
from litellm.interactions.litellm_responses_transformation.handler import (
LiteLLMResponsesInteractionsHandler,
)
from litellm.interactions.litellm_responses_transformation.transformation import (
LiteLLMResponsesInteractionsConfig,
)
__all__ = [
"LiteLLMResponsesInteractionsHandler",
"LiteLLMResponsesInteractionsConfig", # Transformation config class (not BaseInteractionsAPIConfig)
]

156
litellm/interactions/litellm_responses_transformation/handler.py Normal file
View file

@ -0,0 +1,156 @@
"""
Handler for transforming interactions API requests to litellm.responses requests.
"""
from typing import (
Any,
AsyncIterator,
Coroutine,
Dict,
Iterator,
Optional,
Union,
cast,
)
import litellm
from litellm.interactions.litellm_responses_transformation.streaming_iterator import (
LiteLLMResponsesInteractionsStreamingIterator,
)
from litellm.interactions.litellm_responses_transformation.transformation import (
LiteLLMResponsesInteractionsConfig,
)
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
from litellm.types.interactions import (
InteractionInput,
InteractionsAPIOptionalRequestParams,
InteractionsAPIResponse,
InteractionsAPIStreamingResponse,
)
from litellm.types.llms.openai import ResponsesAPIResponse
class LiteLLMResponsesInteractionsHandler:
"""Handler for bridging Interactions API to Responses API via litellm.responses()."""
def interactions_api_handler(
self,
model: str,
input: Optional[InteractionInput],
optional_params: InteractionsAPIOptionalRequestParams,
custom_llm_provider: Optional[str] = None,
_is_async: bool = False,
stream: Optional[bool] = None,
**kwargs,
) -> Union[
InteractionsAPIResponse,
Iterator[InteractionsAPIStreamingResponse],
Coroutine[
Any,
Any,
Union[
InteractionsAPIResponse,
AsyncIterator[InteractionsAPIStreamingResponse],
],
],
]:
"""
Handle Interactions API request by calling litellm.responses().
Args:
model: The model to use
input: The input content
optional_params: Optional parameters for the request
custom_llm_provider: Override LLM provider
_is_async: Whether this is an async call
stream: Whether to stream the response
**kwargs: Additional parameters
Returns:
InteractionsAPIResponse or streaming iterator
"""
# Transform interactions request to responses request
responses_request = (
LiteLLMResponsesInteractionsConfig.transform_interactions_request_to_responses_request(
model=model,
input=input,
optional_params=optional_params,
custom_llm_provider=custom_llm_provider,
stream=stream,
**kwargs,
)
)
if _is_async:
return self.async_interactions_api_handler(
responses_request=responses_request,
model=model,
input=input,
optional_params=optional_params,
**kwargs,
)
# Call litellm.responses()
# Note: litellm.responses() returns Union[ResponsesAPIResponse, BaseResponsesAPIStreamingIterator]
# but the type checker may see it as a coroutine in some contexts
responses_response = litellm.responses(
**responses_request,
)
# Handle streaming response
if isinstance(responses_response, BaseResponsesAPIStreamingIterator):
return LiteLLMResponsesInteractionsStreamingIterator(
model=model,
litellm_custom_stream_wrapper=responses_response,
request_input=input,
optional_params=optional_params,
custom_llm_provider=custom_llm_provider,
litellm_metadata=kwargs.get("litellm_metadata", {}),
)
# At this point, responses_response must be ResponsesAPIResponse (not streaming)
# Cast to satisfy type checker since we've already checked it's not a streaming iterator
responses_api_response = cast(ResponsesAPIResponse, responses_response)
# Transform responses response to interactions response
return LiteLLMResponsesInteractionsConfig.transform_responses_response_to_interactions_response(
responses_response=responses_api_response,
model=model,
)
async def async_interactions_api_handler(
self,
responses_request: Dict[str, Any],
model: str,
input: Optional[InteractionInput],
optional_params: InteractionsAPIOptionalRequestParams,
**kwargs,
) -> Union[InteractionsAPIResponse, AsyncIterator[InteractionsAPIStreamingResponse]]:
"""Async handler for interactions API requests."""
# Call litellm.aresponses()
# Note: litellm.aresponses() returns Union[ResponsesAPIResponse, BaseResponsesAPIStreamingIterator]
responses_response = await litellm.aresponses(
**responses_request,
)
# Handle streaming response
if isinstance(responses_response, BaseResponsesAPIStreamingIterator):
return LiteLLMResponsesInteractionsStreamingIterator(
model=model,
litellm_custom_stream_wrapper=responses_response,
request_input=input,
optional_params=optional_params,
custom_llm_provider=responses_request.get("custom_llm_provider"),
litellm_metadata=kwargs.get("litellm_metadata", {}),
)
# At this point, responses_response must be ResponsesAPIResponse (not streaming)
# Cast to satisfy type checker since we've already checked it's not a streaming iterator
responses_api_response = cast(ResponsesAPIResponse, responses_response)
# Transform responses response to interactions response
return LiteLLMResponsesInteractionsConfig.transform_responses_response_to_interactions_response(
responses_response=responses_api_response,
model=model,
)

260
litellm/interactions/litellm_responses_transformation/streaming_iterator.py Normal file
View file

@ -0,0 +1,260 @@
"""
Streaming iterator for transforming Responses API stream to Interactions API stream.
"""
from typing import Any, AsyncIterator, Dict, Iterator, Optional, cast
from litellm.responses.streaming_iterator import (
BaseResponsesAPIStreamingIterator,
ResponsesAPIStreamingIterator,
SyncResponsesAPIStreamingIterator,
)
from litellm.types.interactions import (
InteractionInput,
InteractionsAPIOptionalRequestParams,
InteractionsAPIStreamingResponse,
)
from litellm.types.llms.openai import (
OutputTextDeltaEvent,
ResponseCompletedEvent,
ResponseCreatedEvent,
ResponseInProgressEvent,
ResponsesAPIStreamingResponse,
)
class LiteLLMResponsesInteractionsStreamingIterator:
"""
Iterator that wraps Responses API streaming and transforms chunks to Interactions API format.
This class handles both sync and async iteration, transforming Responses API
streaming events (output.text.delta, response.completed, etc.) to Interactions
API streaming events (content.delta, interaction.complete, etc.).
"""
def __init__(
self,
model: str,
litellm_custom_stream_wrapper: BaseResponsesAPIStreamingIterator,
request_input: Optional[InteractionInput],
optional_params: InteractionsAPIOptionalRequestParams,
custom_llm_provider: Optional[str] = None,
litellm_metadata: Optional[Dict[str, Any]] = None,
):
self.model = model
self.responses_stream_iterator = litellm_custom_stream_wrapper
self.request_input = request_input
self.optional_params = optional_params
self.custom_llm_provider = custom_llm_provider
self.litellm_metadata = litellm_metadata or {}
self.finished = False
self.collected_text = ""
self.sent_interaction_start = False
self.sent_content_start = False
def _transform_responses_chunk_to_interactions_chunk(
self,
responses_chunk: ResponsesAPIStreamingResponse,
) -> Optional[InteractionsAPIStreamingResponse]:
"""
Transform a Responses API streaming chunk to an Interactions API streaming chunk.
Responses API events:
- output.text.delta -> content.delta
- response.completed -> interaction.complete
Interactions API events:
- interaction.start
- content.start
- content.delta
- content.stop
- interaction.complete
"""
if not responses_chunk:
return None
# Handle OutputTextDeltaEvent -> content.delta
if isinstance(responses_chunk, OutputTextDeltaEvent):
delta_text = responses_chunk.delta if isinstance(responses_chunk.delta, str) else ""
self.collected_text += delta_text
# Send interaction.start if not sent
if not self.sent_interaction_start:
self.sent_interaction_start = True
return InteractionsAPIStreamingResponse(
event_type="interaction.start",
id=getattr(responses_chunk, "item_id", None) or f"interaction_{id(self)}",
object="interaction",
status="in_progress",
model=self.model,
)
# Send content.start if not sent
if not self.sent_content_start:
self.sent_content_start = True
return InteractionsAPIStreamingResponse(
event_type="content.start",
id=getattr(responses_chunk, "item_id", None),
object="content",
delta={"type": "text", "text": ""},
)
# Send content.delta
return InteractionsAPIStreamingResponse(
event_type="content.delta",
id=getattr(responses_chunk, "item_id", None),
object="content",
delta={"text": delta_text},
)
# Handle ResponseCreatedEvent or ResponseInProgressEvent -> interaction.start
if isinstance(responses_chunk, (ResponseCreatedEvent, ResponseInProgressEvent)):
if not self.sent_interaction_start:
self.sent_interaction_start = True
response_id = getattr(responses_chunk.response, "id", None) if hasattr(responses_chunk, "response") else None
return InteractionsAPIStreamingResponse(
event_type="interaction.start",
id=response_id or f"interaction_{id(self)}",
object="interaction",
status="in_progress",
model=self.model,
)
# Handle ResponseCompletedEvent -> interaction.complete
if isinstance(responses_chunk, ResponseCompletedEvent):
self.finished = True
response = responses_chunk.response
# Send content.stop first if content was started
if self.sent_content_start:
# Note: We'll send this in the iterator, not here
pass
# Send interaction.complete
return InteractionsAPIStreamingResponse(
event_type="interaction.complete",
id=getattr(response, "id", None) or f"interaction_{id(self)}",
object="interaction",
status="completed",
model=self.model,
outputs=[
{
"type": "text",
"text": self.collected_text,
}
],
)
# For other event types, return None (skip)
return None
def __iter__(self) -> Iterator[InteractionsAPIStreamingResponse]:
"""Sync iterator implementation."""
return self
def __next__(self) -> InteractionsAPIStreamingResponse:
"""Get next chunk in sync mode."""
if self.finished:
raise StopIteration
# Check if we have a pending interaction.complete to send
if hasattr(self, "_pending_interaction_complete"):
pending: InteractionsAPIStreamingResponse = getattr(self, "_pending_interaction_complete")
delattr(self, "_pending_interaction_complete")
return pending
# Use a loop instead of recursion to avoid stack overflow
sync_iterator = cast(SyncResponsesAPIStreamingIterator, self.responses_stream_iterator)
while True:
try:
# Get next chunk from responses API stream
chunk = next(sync_iterator)
# Transform chunk (chunk is already a ResponsesAPIStreamingResponse)
transformed = self._transform_responses_chunk_to_interactions_chunk(chunk)
if transformed:
# If we finished and content was started, send content.stop before interaction.complete
if self.finished and self.sent_content_start and transformed.event_type == "interaction.complete":
# Send content.stop first
content_stop = InteractionsAPIStreamingResponse(
event_type="content.stop",
id=transformed.id,
object="content",
delta={"type": "text", "text": self.collected_text},
)
# Store the interaction.complete to send next
self._pending_interaction_complete = transformed
return content_stop
return transformed
# If no transformation, continue to next chunk (loop continues)
except StopIteration:
self.finished = True
# Send final events if needed
if self.sent_content_start:
return InteractionsAPIStreamingResponse(
event_type="content.stop",
object="content",
delta={"type": "text", "text": self.collected_text},
)
raise StopIteration
def __aiter__(self) -> AsyncIterator[InteractionsAPIStreamingResponse]:
"""Async iterator implementation."""
return self
async def __anext__(self) -> InteractionsAPIStreamingResponse:
"""Get next chunk in async mode."""
if self.finished:
raise StopAsyncIteration
# Check if we have a pending interaction.complete to send
if hasattr(self, "_pending_interaction_complete"):
pending: InteractionsAPIStreamingResponse = getattr(self, "_pending_interaction_complete")
delattr(self, "_pending_interaction_complete")
return pending
# Use a loop instead of recursion to avoid stack overflow
async_iterator = cast(ResponsesAPIStreamingIterator, self.responses_stream_iterator)
while True:
try:
# Get next chunk from responses API stream
chunk = await async_iterator.__anext__()
# Transform chunk (chunk is already a ResponsesAPIStreamingResponse)
transformed = self._transform_responses_chunk_to_interactions_chunk(chunk)
if transformed:
# If we finished and content was started, send content.stop before interaction.complete
if self.finished and self.sent_content_start and transformed.event_type == "interaction.complete":
# Send content.stop first
content_stop = InteractionsAPIStreamingResponse(
event_type="content.stop",
id=transformed.id,
object="content",
delta={"type": "text", "text": self.collected_text},
)
# Store the interaction.complete to send next
self._pending_interaction_complete = transformed
return content_stop
return transformed
# If no transformation, continue to next chunk (loop continues)
except StopAsyncIteration:
self.finished = True
# Send final events if needed
if self.sent_content_start:
return InteractionsAPIStreamingResponse(
event_type="content.stop",
object="content",
delta={"type": "text", "text": self.collected_text},
)
raise StopAsyncIteration

277
litellm/interactions/litellm_responses_transformation/transformation.py Normal file
View file

@ -0,0 +1,277 @@
"""
Transformation utilities for bridging Interactions API to Responses API.
This module handles transforming between:
- Interactions API format (Google's format with Turn[], system_instruction, etc.)
- Responses API format (OpenAI's format with input[], instructions, etc.)
"""
from typing import Any, Dict, List, Optional, cast
from litellm.types.interactions import (
InteractionInput,
InteractionsAPIOptionalRequestParams,
InteractionsAPIResponse,
Turn,
)
from litellm.types.llms.openai import (
ResponseInputParam,
ResponsesAPIResponse,
)
class LiteLLMResponsesInteractionsConfig:
"""Configuration class for transforming between Interactions API and Responses API."""
@staticmethod
def transform_interactions_request_to_responses_request(
model: str,
input: Optional[InteractionInput],
optional_params: InteractionsAPIOptionalRequestParams,
**kwargs,
) -> Dict[str, Any]:
"""
Transform an Interactions API request to a Responses API request.
Key transformations:
- system_instruction -> instructions
- input (string | Turn[]) -> input (ResponseInputParam)
- tools -> tools (similar format)
- generation_config -> temperature, top_p, etc.
"""
responses_request: Dict[str, Any] = {
"model": model,
}
# Transform input
if input is not None:
responses_request["input"] = (
LiteLLMResponsesInteractionsConfig._transform_interactions_input_to_responses_input(
input
)
)
# Transform system_instruction -> instructions
if optional_params.get("system_instruction"):
responses_request["instructions"] = optional_params["system_instruction"]
# Transform tools (similar format, pass through for now)
if optional_params.get("tools"):
responses_request["tools"] = optional_params["tools"]
# Transform generation_config to temperature, top_p, etc.
generation_config = optional_params.get("generation_config")
if generation_config:
if isinstance(generation_config, dict):
if "temperature" in generation_config:
responses_request["temperature"] = generation_config["temperature"]
if "top_p" in generation_config:
responses_request["top_p"] = generation_config["top_p"]
if "top_k" in generation_config:
# Responses API doesn't have top_k, skip it
pass
if "max_output_tokens" in generation_config:
responses_request["max_output_tokens"] = generation_config["max_output_tokens"]
# Pass through other optional params that match
passthrough_params = ["stream", "store", "metadata", "user"]
for param in passthrough_params:
if param in optional_params and optional_params[param] is not None:
responses_request[param] = optional_params[param]
# Add any extra kwargs
responses_request.update(kwargs)
return responses_request
@staticmethod
def _transform_interactions_input_to_responses_input(
input: InteractionInput,
) -> ResponseInputParam:
"""
Transform Interactions API input to Responses API input format.
Interactions API input can be:
- string: "Hello"
- Turn[]: [{"role": "user", "content": [...]}]
- Content object
Responses API input is:
- string: "Hello"
- Message[]: [{"role": "user", "content": [...]}]
"""
if isinstance(input, str):
# ResponseInputParam accepts str
return cast(ResponseInputParam, input)
if isinstance(input, list):
# Turn[] format - convert to Responses API Message[] format
messages = []
for turn in input:
if isinstance(turn, dict):
role = turn.get("role", "user")
content = turn.get("content", [])
# Transform content array
transformed_content = (
LiteLLMResponsesInteractionsConfig._transform_content_array(content)
)
messages.append({
"role": role,
"content": transformed_content,
})
elif isinstance(turn, Turn):
# Pydantic model
role = turn.role if hasattr(turn, "role") else "user"
content = turn.content if hasattr(turn, "content") else []
# Ensure content is a list for _transform_content_array
# Cast to List[Any] to handle various content types
if isinstance(content, list):
content_list: List[Any] = list(content)
elif content is not None:
content_list = [content]
else:
content_list = []
transformed_content = (
LiteLLMResponsesInteractionsConfig._transform_content_array(content_list)
)
messages.append({
"role": role,
"content": transformed_content,
})
return cast(ResponseInputParam, messages)
# Single content object - wrap in message
if isinstance(input, dict):
return cast(ResponseInputParam, [{
"role": "user",
"content": LiteLLMResponsesInteractionsConfig._transform_content_array(
input.get("content", []) if isinstance(input.get("content"), list) else [input]
),
}])
# Fallback: convert to string
return cast(ResponseInputParam, str(input))
@staticmethod
def _transform_content_array(content: List[Any]) -> List[Dict[str, Any]]:
"""Transform Interactions API content array to Responses API format."""
if not isinstance(content, list):
# Single content item - wrap in array
content = [content]
transformed: List[Dict[str, Any]] = []
for item in content:
if isinstance(item, dict):
# Already in dict format, pass through
transformed.append(item)
elif isinstance(item, str):
# Plain string - wrap in text format
transformed.append({"type": "text", "text": item})
else:
# Pydantic model or other - convert to dict
if hasattr(item, "model_dump"):
dumped = item.model_dump()
if isinstance(dumped, dict):
transformed.append(dumped)
else:
# Fallback: wrap in text format
transformed.append({"type": "text", "text": str(dumped)})
elif hasattr(item, "dict"):
dumped = item.dict()
if isinstance(dumped, dict):
transformed.append(dumped)
else:
# Fallback: wrap in text format
transformed.append({"type": "text", "text": str(dumped)})
else:
# Fallback: wrap in text format
transformed.append({"type": "text", "text": str(item)})
return transformed
@staticmethod
def transform_responses_response_to_interactions_response(
responses_response: ResponsesAPIResponse,
model: Optional[str] = None,
) -> InteractionsAPIResponse:
"""
Transform a Responses API response to an Interactions API response.
Key transformations:
- Extract text from output[].content[].text
- Convert created_at (int) to created (ISO string)
- Map status
- Extract usage
"""
# Extract text from outputs
outputs = []
if hasattr(responses_response, "output") and responses_response.output:
for output_item in responses_response.output:
# Use getattr with None default to safely access content
content = getattr(output_item, "content", None)
if content is not None:
content_items = content if isinstance(content, list) else [content]
for content_item in content_items:
# Check if content_item has text attribute
text = getattr(content_item, "text", None)
if text is not None:
outputs.append({
"type": "text",
"text": text,
})
elif isinstance(content_item, dict) and content_item.get("type") == "text":
outputs.append(content_item)
# Convert created_at to ISO string
created_at = getattr(responses_response, "created_at", None)
if isinstance(created_at, int):
from datetime import datetime
created = datetime.fromtimestamp(created_at).isoformat()
elif created_at is not None and hasattr(created_at, "isoformat"):
created = created_at.isoformat()
else:
created = None
# Map status
status = getattr(responses_response, "status", "completed")
if status == "completed":
interactions_status = "completed"
elif status == "in_progress":
interactions_status = "in_progress"
else:
interactions_status = status
# Build interactions response
interactions_response_dict: Dict[str, Any] = {
"id": getattr(responses_response, "id", ""),
"object": "interaction",
"status": interactions_status,
"outputs": outputs,
"model": model or getattr(responses_response, "model", ""),
"created": created,
}
# Add usage if available
# Map Responses API usage (input_tokens, output_tokens) to Interactions API spec format
# (total_input_tokens, total_output_tokens)
usage = getattr(responses_response, "usage", None)
if usage:
interactions_response_dict["usage"] = {
"total_input_tokens": getattr(usage, "input_tokens", 0),
"total_output_tokens": getattr(usage, "output_tokens", 0),
}
# Add role
interactions_response_dict["role"] = "model"
# Add updated (same as created for now)
interactions_response_dict["updated"] = created
return InteractionsAPIResponse(**interactions_response_dict)

24
litellm/interactions/main.py
View file

@ -272,18 +272,30 @@ def create(
model=model,
)
if interactions_api_config is None:
raise ValueError(
f"Interactions API is not supported for provider: {custom_llm_provider}. "
"Currently only 'gemini' is supported."
)
# Get optional params using utility (similar to responses API pattern)
local_vars.update(kwargs)
optional_params = InteractionsAPIRequestUtils.get_requested_interactions_api_optional_params(
local_vars
)
# Check if this is a bridge provider (litellm_responses) - similar to responses API
# Either provider is explicitly "litellm_responses" or no config found (bridge to responses)
if custom_llm_provider == "litellm_responses" or interactions_api_config is None:
# Bridge to litellm.responses() for non-native providers
from litellm.interactions.litellm_responses_transformation.handler import (
LiteLLMResponsesInteractionsHandler,
)
handler = LiteLLMResponsesInteractionsHandler()
return handler.interactions_api_handler(
model=model or "",
input=input,
optional_params=optional_params,
custom_llm_provider=custom_llm_provider,
_is_async=_is_async,
stream=stream,
**kwargs,
)
litellm_logging_obj.update_environment_variables(
model=model,
optional_params=dict(optional_params),

19
litellm/litellm_core_utils/default_encoding.py
View file

@ -19,5 +19,22 @@ os.environ["TIKTOKEN_CACHE_DIR"] = os.getenv(
"CUSTOM_TIKTOKEN_CACHE_DIR", filename
) # use local copy of tiktoken b/c of - https://github.com/BerriAI/litellm/issues/1071
import tiktoken
import time
import random
encoding = tiktoken.get_encoding("cl100k_base")
# Retry logic to handle race conditions when multiple processes try to create
# the tiktoken cache file simultaneously (common in parallel test execution on Windows)
_max_retries = 5
_retry_delay = 0.1 # Start with 100ms
for attempt in range(_max_retries):
try:
encoding = tiktoken.get_encoding("cl100k_base")
break
except (FileExistsError, OSError):
if attempt == _max_retries - 1:
# Last attempt, re-raise the exception
raise
# Exponential backoff with jitter to reduce collision probability
delay = _retry_delay * (2 ** attempt) + random.uniform(0, 0.1)
time.sleep(delay)

12
litellm/litellm_core_utils/exception_mapping_utils.py
View file

@ -78,9 +78,7 @@ class ExceptionCheckers:
"is longer than the model's context length",
"input tokens exceed the configured limit",
"`inputs` tokens + `max_new_tokens` must be",
# Gemini pattern: "The input token count exceeds the maximum number of tokens allowed"
# See: https://github.com/BerriAI/litellm/issues/XXXX
"input token count exceeds the maximum number of tokens allowed",
"exceeds the maximum number of tokens allowed", # Gemini
]
for substring in known_exception_substrings:
if substring in _error_str_lowercase:
@ -1262,6 +1260,14 @@ def exception_type( # type: ignore # noqa: PLR0915
model=model,
llm_provider=custom_llm_provider,
)
elif ExceptionCheckers.is_error_str_context_window_exceeded(error_str):
exception_mapping_worked = True
raise ContextWindowExceededError(
message=f"ContextWindowExceededError: {custom_llm_provider.capitalize()}Exception - {error_str}",
model=model,
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
)
elif (
"None Unknown Error." in error_str
or "Content has no parts." in error_str

23
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -4,8 +4,8 @@ import httpx
import litellm
from litellm.constants import REPLICATE_MODEL_NAME_WITH_ID_LENGTH
from litellm.secret_managers.main import get_secret, get_secret_str
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
from litellm.secret_managers.main import get_secret, get_secret_str
from ..types.router import LiteLLM_Params
@ -267,9 +267,30 @@ def get_llm_provider( # noqa: PLR0915
elif endpoint == "api.moonshot.ai/v1":
custom_llm_provider = "moonshot"
dynamic_api_key = get_secret_str("MOONSHOT_API_KEY")
elif endpoint == "api.minimax.io/anthropic" or endpoint == "api.minimaxi.com/anthropic":
custom_llm_provider = "minimax"
dynamic_api_key = get_secret_str("MINIMAX_API_KEY")
elif endpoint == "api.minimax.io/v1" or endpoint == "api.minimaxi.com/v1":
custom_llm_provider = "minimax"
dynamic_api_key = get_secret_str("MINIMAX_API_KEY")
elif endpoint == "platform.publicai.co/v1":
custom_llm_provider = "publicai"
dynamic_api_key = get_secret_str("PUBLICAI_API_KEY")
elif endpoint == "https://api.synthetic.new/openai/v1":
custom_llm_provider = "synthetic"
dynamic_api_key = get_secret_str("SYNTHETIC_API_KEY")
elif endpoint == "https://api.stima.tech/v1":
custom_llm_provider = "apertis"
dynamic_api_key = get_secret_str("STIMA_API_KEY")
elif endpoint == "https://nano-gpt.com/api/v1":
custom_llm_provider = "nano-gpt"
dynamic_api_key = get_secret_str("NANOGPT_API_KEY")
elif endpoint == "https://api.poe.com/v1":
custom_llm_provider = "poe"
dynamic_api_key = get_secret_str("POE_API_KEY")
elif endpoint == "https://llm.chutes.ai/v1/":
custom_llm_provider = "chutes"
dynamic_api_key = get_secret_str("CHUTES_API_KEY")
elif endpoint == "https://api.v0.dev/v1":
custom_llm_provider = "v0"
dynamic_api_key = get_secret_str("V0_API_KEY")

14
litellm/litellm_core_utils/litellm_logging.py
View file

@ -1291,6 +1291,9 @@ class Logging(LiteLLMLoggingBaseClass):
original_cost: Optional[float] = None,
discount_percent: Optional[float] = None,
discount_amount: Optional[float] = None,
margin_percent: Optional[float] = None,
margin_fixed_amount: Optional[float] = None,
margin_total_amount: Optional[float] = None,
) -> None:
"""
Helper method to store cost breakdown in the logging object.
@ -1303,6 +1306,9 @@ class Logging(LiteLLMLoggingBaseClass):
original_cost: Cost before discount
discount_percent: Discount percentage (0.05 = 5%)
discount_amount: Discount amount in USD
margin_percent: Margin percentage applied (0.10 = 10%)
margin_fixed_amount: Fixed margin amount in USD
margin_total_amount: Total margin added in USD
"""
self.cost_breakdown = CostBreakdown(
@ -1320,6 +1326,14 @@ class Logging(LiteLLMLoggingBaseClass):
if discount_amount is not None:
self.cost_breakdown["discount_amount"] = discount_amount
# Store margin information if provided
if margin_percent is not None:
self.cost_breakdown["margin_percent"] = margin_percent
if margin_fixed_amount is not None:
self.cost_breakdown["margin_fixed_amount"] = margin_fixed_amount
if margin_total_amount is not None:
self.cost_breakdown["margin_total_amount"] = margin_total_amount
def _response_cost_calculator(
self,
result: Union[

8
litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py
View file

@ -740,9 +740,7 @@ class LiteLLMAnthropicMessagesAdapter:
from litellm.types.llms.anthropic import TextBlock, ToolUseBlock
for choice in choices:
if choice.delta.content is not None and len(choice.delta.content) > 0:
return "text", TextBlock(type="text", text="")
elif (
if (
choice.delta.tool_calls is not None
and len(choice.delta.tool_calls) > 0
and choice.delta.tool_calls[0].function is not None
@ -753,6 +751,8 @@ class LiteLLMAnthropicMessagesAdapter:
name=choice.delta.tool_calls[0].function.name or "",
input={}, # type: ignore[typeddict-item]
)
elif choice.delta.content is not None and len(choice.delta.content) > 0:
return "text", TextBlock(type="text", text="")
elif isinstance(choice, StreamingChoices) and hasattr(
choice.delta, "thinking_blocks"
):
@ -796,7 +796,7 @@ class LiteLLMAnthropicMessagesAdapter:
for choice in choices:
if choice.delta.content is not None and len(choice.delta.content) > 0:
text += choice.delta.content
elif choice.delta.tool_calls is not None:
if choice.delta.tool_calls is not None:
partial_json = ""
for tool in choice.delta.tool_calls:
if (

5
litellm/llms/bedrock/image_generation/image_handler.py
View file

@ -12,6 +12,9 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging
from litellm.llms.bedrock.image_generation.amazon_nova_canvas_transformation import (
AmazonNovaCanvasConfig,
)
from litellm.llms.bedrock.image_generation.amazon_stability1_transformation import (
AmazonStabilityConfig,
)
from litellm.llms.bedrock.image_generation.amazon_stability3_transformation import (
AmazonStability3Config,
)
@ -50,7 +53,7 @@ BedrockImageConfigClass = Union[
type[AmazonTitanImageGenerationConfig],
type[AmazonNovaCanvasConfig],
type[AmazonStability3Config],
type[litellm.AmazonStabilityConfig],
type[AmazonStabilityConfig],
]

45
litellm/llms/databricks/chat/transformation.py
View file

@ -2,6 +2,7 @@
Translates from OpenAI's `/v1/chat/completions` to Databricks' `/chat/completions`
"""
import os
from typing import (
TYPE_CHECKING,
Any,
@ -26,7 +27,7 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo
_should_convert_tool_call_to_json_mode,
)
from litellm.litellm_core_utils.prompt_templates.common_utils import (
strip_name_from_message
strip_name_from_message,
)
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.types.llms.anthropic import AllAnthropicToolsValues
@ -124,12 +125,24 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
api_key: Optional[str] = None,
api_base: Optional[str] = None,
) -> dict:
# Check for custom user agent in optional_params or environment
# This allows partners building on LiteLLM to set their own telemetry
# Use pop() to remove these keys so they don't get sent to the API
custom_user_agent = (
optional_params.pop("user_agent", None)
or optional_params.pop("databricks_user_agent", None)
or litellm_params.get("user_agent")
or os.getenv("LITELLM_USER_AGENT")
or os.getenv("DATABRICKS_USER_AGENT")
)
api_base, headers = self.databricks_validate_environment(
api_base=api_base,
api_key=api_key,
endpoint_type="chat_completions",
custom_endpoint=False,
headers=headers,
custom_user_agent=custom_user_agent,
)
# Ensure Content-Type header is set
headers["Content-Type"] = "application/json"
@ -173,9 +186,9 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
# Build DatabricksFunction explicitly to avoid parameter conflicts
function_params: DatabricksFunction = {
"name": tool["name"],
"parameters": cast(dict, tool.get("input_schema") or {})
"parameters": cast(dict, tool.get("input_schema") or {}),
}
# Only add description if it exists
description = tool.get("description")
if description is not None:
@ -229,7 +242,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
Databricks supports Anthropic-style cache control for Claude models.
Databricks ignores the cache_control flag with other models.
"""
# TODO: Think about how to best design the request transformation so that
# TODO: Think about how to best design the request transformation so that
# every request doesn't have to be transformed for to OpenAI and Anthropic request formats.
return messages, tools
@ -347,15 +360,17 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
messages=new_messages, model=model, is_async=cast(Literal[False], False)
)
def _move_cache_control_into_string_content_block(self, message: AllMessageValues) -> AllMessageValues:
def _move_cache_control_into_string_content_block(
self, message: AllMessageValues
) -> AllMessageValues:
"""
Moves message-level cache_control into a content block when content is a string.
Transforms:
{"role": "user", "content": "text", "cache_control": {...}}
Into:
{"role": "user", "content": [{"type": "text", "text": "text", "cache_control": {...}}]}
This is required for Anthropic's prompt caching API when cache_control is specified
at the message level but content is a simple string (not already an array of content blocks).
"""
@ -371,7 +386,6 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
}
]
return cast(AllMessageValues, transformed_message)
@staticmethod
def extract_content_str(
@ -509,9 +523,9 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
reasoning_content=reasoning_content,
thinking_blocks=thinking_blocks,
tool_calls=choice["message"].get("tool_calls"),
provider_specific_fields={"citations": citations}
if citations is not None
else None,
provider_specific_fields=(
{"citations": citations} if citations is not None else None
),
)
if finish_reason is None:
@ -543,12 +557,15 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
api_key: Optional[str] = None,
json_mode: Optional[bool] = None,
) -> ModelResponse:
## LOGGING
# Redact sensitive data before logging to prevent credential leakage
redacted_request_data = self.redact_sensitive_data(request_data)
## LOGGING - Never log actual API keys
logging_obj.post_call(
input=messages,
api_key=api_key,
api_key="[REDACTED]",
original_response=raw_response.text,
additional_args={"complete_input_dict": request_data},
additional_args={"complete_input_dict": redacted_request_data},
)
## RESPONSE OBJECT

311
litellm/llms/databricks/common_utils.py
View file

@ -1,4 +1,18 @@
from typing import Literal, Optional, Tuple
"""
Databricks integration utilities for LiteLLM.
This module provides authentication, telemetry, and security utilities
for the Databricks LLM provider integration.
Authentication priority:
1. OAuth M2M (DATABRICKS_CLIENT_ID + DATABRICKS_CLIENT_SECRET) - Recommended for production
2. PAT (DATABRICKS_API_KEY) - Supported for development
3. Databricks SDK automatic auth - Fallback (uses unified auth)
"""
import os
import re
from typing import Any, Dict, Literal, Optional, Tuple
from litellm.llms.base_llm.chat.transformation import BaseLLMException
@ -8,17 +22,175 @@ class DatabricksException(BaseLLMException):
class DatabricksBase:
"""
Base class for Databricks integration with authentication,
telemetry, and security utilities.
"""
# Patterns to redact in logs
SENSITIVE_PATTERNS = [
(re.compile(r"(Bearer\s+)[A-Za-z0-9\-_\.]+", re.IGNORECASE), r"\1[REDACTED]"),
(re.compile(r"(Authorization:\s*)[^\s,}]+", re.IGNORECASE), r"\1[REDACTED]"),
(
re.compile(r'(api[_-]?key["\s:=]+)[^\s,}"\']+', re.IGNORECASE),
r"\1[REDACTED]",
),
(
re.compile(r'(client[_-]?secret["\s:=]+)[^\s,}"\']+', re.IGNORECASE),
r"\1[REDACTED]",
),
(re.compile(r"(dapi[a-zA-Z0-9]{32,})", re.IGNORECASE), r"[REDACTED_PAT]"),
(
re.compile(r'(access[_-]?token["\s:=]+)[^\s,}"\']+', re.IGNORECASE),
r"\1[REDACTED]",
),
]
@classmethod
def redact_sensitive_data(cls, data: Any) -> Any:
"""
Redact sensitive information (tokens, secrets) from data before logging.
Handles strings, dicts, and lists recursively. Keys containing sensitive
terms (authorization, api_key, token, secret, password, credential) are
fully redacted.
Args:
data: String, dict, or other data structure to redact
Returns:
Redacted version of the data safe for logging
"""
if data is None:
return None
if isinstance(data, str):
result = data
for pattern, replacement in cls.SENSITIVE_PATTERNS:
result = pattern.sub(replacement, result)
return result
if isinstance(data, dict):
redacted = {}
for key, value in data.items():
lower_key = key.lower()
if any(
sensitive in lower_key
for sensitive in [
"authorization",
"api_key",
"apikey",
"token",
"secret",
"password",
"credential",
]
):
redacted[key] = "[REDACTED]"
else:
redacted[key] = cls.redact_sensitive_data(value)
return redacted
if isinstance(data, list):
return [cls.redact_sensitive_data(item) for item in data]
return data
@classmethod
def redact_headers_for_logging(cls, headers: Dict[str, str]) -> Dict[str, str]:
"""
Create a copy of headers with sensitive values redacted for safe logging.
Shows first 8 characters of sensitive values for debugging purposes,
with the rest redacted.
Args:
headers: HTTP headers dictionary
Returns:
New dictionary with sensitive headers redacted
"""
if not headers:
return {}
redacted = {}
sensitive_headers = {
"authorization",
"x-api-key",
"api-key",
"x-databricks-token",
}
for key, value in headers.items():
if key.lower() in sensitive_headers:
if len(value) > 10:
redacted[key] = f"{value[:8]}...[REDACTED]"
else:
redacted[key] = "[REDACTED]"
else:
redacted[key] = value
return redacted
@staticmethod
def _build_user_agent(custom_user_agent: Optional[str] = None) -> str:
"""
Build the User-Agent string for Databricks API calls.
If a custom user agent is provided, the partner name (part before /)
is extracted and prefixed to the litellm user agent with an underscore.
The custom version is ignored; LiteLLM's version is always used.
Args:
custom_user_agent: Optional custom user agent string (e.g., "mycompany/1.0.0")
Returns:
User-Agent string in format:
- Default: "litellm/{version}"
- With custom: "{partner}_litellm/{version}"
Examples:
- None -> "litellm/1.79.1"
- "mycompany/1.0.0" -> "mycompany_litellm/1.79.1"
- "partner_product/2.0.0" -> "partner_product_litellm/1.79.1"
- "acme" -> "acme_litellm/1.79.1"
"""
try:
from litellm._version import version
except Exception:
version = "0.0.0"
if custom_user_agent:
custom_user_agent = custom_user_agent.strip()
# Extract partner name (part before / if present)
if "/" in custom_user_agent:
partner_name = custom_user_agent.split("/")[0].strip()
else:
partner_name = custom_user_agent
# Validate partner name: alphanumeric, underscore, hyphen only
if (
partner_name
and partner_name.replace("_", "").replace("-", "").isalnum()
):
return f"{partner_name}_litellm/{version}"
# Default: just litellm
return f"litellm/{version}"
def _get_api_base(self, api_base: Optional[str]) -> str:
"""
Get the Databricks API base URL.
If not provided, attempts to get it from the Databricks SDK.
"""
if api_base is None:
try:
from databricks.sdk import WorkspaceClient
databricks_client = WorkspaceClient()
api_base = (
api_base or f"{databricks_client.config.host}/serving-endpoints"
)
api_base = f"{databricks_client.config.host}/serving-endpoints"
return api_base
except ImportError:
raise DatabricksException(
@ -30,12 +202,87 @@ class DatabricksBase:
)
return api_base
def _get_oauth_m2m_token(
self,
api_base: str,
client_id: str,
client_secret: str,
) -> str:
"""
Obtain an OAuth M2M access token using client credentials flow.
This is the recommended authentication method for production integrations
per Databricks Partner requirements.
Args:
api_base: Databricks workspace URL
client_id: OAuth client ID (Service Principal application ID)
client_secret: OAuth client secret
Returns:
Access token string
Raises:
DatabricksException: If token request fails
"""
import requests
# Extract workspace URL from api_base
workspace_url = api_base.rstrip("/")
if "/serving-endpoints" in workspace_url:
workspace_url = workspace_url.replace("/serving-endpoints", "")
token_url = f"{workspace_url}/oidc/v1/token"
try:
response = requests.post(
token_url,
data={
"grant_type": "client_credentials",
"scope": "all-apis",
},
auth=(client_id, client_secret),
headers={"Content-Type": "application/x-www-form-urlencoded"},
timeout=30,
)
except requests.RequestException as e:
raise DatabricksException(
status_code=500,
message=f"OAuth M2M token request failed: {str(e)}",
)
if response.status_code != 200:
raise DatabricksException(
status_code=response.status_code,
message=f"OAuth M2M token request failed: {response.text}",
)
token_data = response.json()
return token_data["access_token"]
def _get_databricks_credentials(
self, api_key: Optional[str], api_base: Optional[str], headers: Optional[dict]
) -> Tuple[str, dict]:
"""
Get Databricks credentials using the Databricks SDK.
Also registers LiteLLM as a partner for proper telemetry attribution
in Databricks system.access.audit table.
Args:
api_key: Optional API key (PAT)
api_base: Optional API base URL
headers: Optional existing headers
Returns:
Tuple of (api_base, headers)
"""
headers = headers or {"Content-Type": "application/json"}
try:
from databricks.sdk import WorkspaceClient
from databricks.sdk import WorkspaceClient, useragent
# Register LiteLLM as partner for Databricks telemetry attribution
useragent.with_partner("litellm")
databricks_client = WorkspaceClient()
@ -66,14 +313,53 @@ class DatabricksBase:
endpoint_type: Literal["chat_completions", "embeddings"],
custom_endpoint: Optional[bool],
headers: Optional[dict],
custom_user_agent: Optional[str] = None,
) -> Tuple[str, dict]:
if api_key is None and not headers: # handle empty headers
"""
Validate and configure the Databricks environment.
Authentication priority:
1. OAuth M2M (DATABRICKS_CLIENT_ID + DATABRICKS_CLIENT_SECRET) - Recommended
2. PAT (DATABRICKS_API_KEY) - Supported for development
3. Databricks SDK automatic auth - Fallback (uses unified auth)
Args:
api_key: Personal access token (PAT)
api_base: Databricks workspace URL with /serving-endpoints
endpoint_type: Type of endpoint (chat_completions or embeddings)
custom_endpoint: Whether using a custom endpoint URL
headers: Existing headers dict
custom_user_agent: Optional custom user agent to prefix
Returns:
Tuple of (api_base, headers) with authentication configured
"""
from litellm._logging import verbose_logger
# Check for OAuth M2M credentials (recommended for production)
client_id = os.getenv("DATABRICKS_CLIENT_ID")
client_secret = os.getenv("DATABRICKS_CLIENT_SECRET")
# Determine api_base first
if api_base is None:
api_base = os.getenv("DATABRICKS_API_BASE")
if client_id and client_secret and api_base:
# Use OAuth M2M flow (preferred for production)
verbose_logger.debug("Using OAuth M2M authentication for Databricks")
access_token = self._get_oauth_m2m_token(api_base, client_id, client_secret)
headers = headers or {}
headers["Authorization"] = f"Bearer {access_token}"
headers["Content-Type"] = "application/json"
elif api_key is None and not headers:
if custom_endpoint is True:
raise DatabricksException(
status_code=400,
message="Missing API Key - A call is being made to LLM Provider but no key is set either in the environment variables ({LLM_PROVIDER}_API_KEY) or via params",
)
else:
# Fallback to Databricks SDK (registers partner telemetry)
verbose_logger.debug("Using Databricks SDK for authentication")
api_base, headers = self._get_databricks_credentials(
api_base=api_base, api_key=api_key, headers=headers
)
@ -101,8 +387,17 @@ class DatabricksBase:
if api_key is not None:
headers["Authorization"] = f"Bearer {api_key}"
# Set User-Agent with optional custom prefix
headers["User-Agent"] = self._build_user_agent(custom_user_agent)
# Debug logging with redaction (never log actual tokens)
verbose_logger.debug(
f"Databricks request headers: {self.redact_headers_for_logging(headers)}"
)
if endpoint_type == "chat_completions" and custom_endpoint is not True:
api_base = "{}/chat/completions".format(api_base)
elif endpoint_type == "embeddings" and custom_endpoint is not True:
api_base = "{}/embeddings".format(api_base)
return api_base, headers

12
litellm/llms/databricks/embed/handler.py
View file

@ -2,6 +2,7 @@
Calling logic for Databricks embeddings
"""
import os
from typing import Optional
from litellm.utils import EmbeddingResponse
@ -26,12 +27,23 @@ class DatabricksEmbeddingHandler(OpenAILikeEmbeddingHandler, DatabricksBase):
custom_endpoint: Optional[bool] = None,
headers: Optional[dict] = None,
) -> EmbeddingResponse:
# Check for custom user agent in optional_params or environment
# This allows partners building on LiteLLM to set their own telemetry
# Use pop() to remove these keys so they don't get sent to the API
custom_user_agent = (
optional_params.pop("user_agent", None)
or optional_params.pop("databricks_user_agent", None)
or os.getenv("LITELLM_USER_AGENT")
or os.getenv("DATABRICKS_USER_AGENT")
)
api_base, headers = self.databricks_validate_environment(
api_base=api_base,
api_key=api_key,
endpoint_type="embeddings",
custom_endpoint=custom_endpoint,
headers=headers,
custom_user_agent=custom_user_agent,
)
return super().embedding(
model=model,

14
litellm/llms/minimax/__init__.py Normal file
View file

@ -0,0 +1,14 @@
"""
MiniMax LLM Provider
"""
from .text_to_speech.transformation import (
MinimaxException,
MinimaxTextToSpeechConfig,
)
__all__ = [
"MinimaxTextToSpeechConfig",
"MinimaxException",
]

4
litellm/llms/minimax/chat/__init__.py Normal file
View file

@ -0,0 +1,4 @@
"""
MiniMax OpenAI-compatible chat API
"""

83
litellm/llms/minimax/chat/transformation.py Normal file
View file

@ -0,0 +1,83 @@
"""
MiniMax OpenAI transformation config - extends OpenAI chat config for MiniMax's OpenAI-compatible API
"""
from typing import Optional
import litellm
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
from litellm.secret_managers.main import get_secret_str
class MinimaxChatConfig(OpenAIGPTConfig):
"""
MiniMax OpenAI configuration that extends OpenAIGPTConfig.
MiniMax provides an OpenAI-compatible API at:
- International: https://api.minimax.io/v1
- China: https://api.minimaxi.com/v1
Supported models:
- MiniMax-M2.1
- MiniMax-M2.1-lightning
- MiniMax-M2
"""
@staticmethod
def get_api_key(api_key: Optional[str] = None) -> Optional[str]:
"""
Get MiniMax API key from environment or parameters.
"""
return (
api_key
or get_secret_str("MINIMAX_API_KEY")
or litellm.api_key
)
@staticmethod
def get_api_base(
api_base: Optional[str] = None,
) -> str:
"""
Get MiniMax API base URL.
Defaults to international endpoint: https://api.minimax.io/v1
For China, set to: https://api.minimaxi.com/v1
"""
return (
api_base
or get_secret_str("MINIMAX_API_BASE")
or "https://api.minimax.io/v1"
)
def get_complete_url(
self,
api_base: Optional[str],
api_key: Optional[str],
model: str,
optional_params: dict,
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
"""
Get the complete URL for MiniMax OpenAI API.
Override to ensure we use MiniMax's endpoint.
"""
# Get the base URL (either provided or default MiniMax endpoint)
base_url = self.get_api_base(api_base=api_base)
# Ensure it ends with /chat/completions
if base_url.endswith("/chat/completions"):
return base_url
elif base_url.endswith("/v1"):
return f"{base_url}/chat/completions"
elif base_url.endswith("/"):
return f"{base_url}v1/chat/completions"
else:
return f"{base_url}/v1/chat/completions"
def get_supported_openai_params(self, model: str) -> list:
"""
Get supported OpenAI parameters for MiniMax.
Adds reasoning_split to the list of supported params.
"""
base_params = super().get_supported_openai_params(model=model)
return base_params + ["reasoning_split"]

81
litellm/llms/minimax/messages/transformation.py Normal file
View file

@ -0,0 +1,81 @@
"""
MiniMax Anthropic transformation config - extends AnthropicConfig for MiniMax's Anthropic-compatible API
"""
from typing import Optional
import litellm
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
AnthropicMessagesConfig,
)
from litellm.secret_managers.main import get_secret_str
class MinimaxMessagesConfig(AnthropicMessagesConfig):
"""
MiniMax Anthropic configuration that extends AnthropicConfig.
MiniMax provides an Anthropic-compatible API at:
- International: https://api.minimax.io/anthropic
- China: https://api.minimaxi.com/anthropic
Supported models:
- MiniMax-M2.1
- MiniMax-M2.1-lightning
- MiniMax-M2
"""
@property
def custom_llm_provider(self) -> Optional[str]:
return "minimax"
@staticmethod
def get_api_key(api_key: Optional[str] = None) -> Optional[str]:
"""
Get MiniMax API key from environment or parameters.
"""
return (
api_key
or get_secret_str("MINIMAX_API_KEY")
or litellm.api_key
)
@staticmethod
def get_api_base(
api_base: Optional[str] = None,
) -> str:
"""
Get MiniMax API base URL.
Defaults to international endpoint: https://api.minimax.io/anthropic
For China, set to: https://api.minimaxi.com/anthropic
"""
return (
api_base
or get_secret_str("MINIMAX_API_BASE")
or "https://api.minimax.io/anthropic/v1/messages"
)
def get_complete_url(
self,
api_base: Optional[str],
api_key: Optional[str],
model: str,
optional_params: dict,
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
"""
Get the complete URL for MiniMax API.
Override to ensure we use MiniMax's endpoint, not Anthropic's.
"""
# Get the base URL (either provided or default MiniMax endpoint)
base_url = self.get_api_base(api_base=api_base)
# If the base URL already includes the full path, return it
if base_url.endswith("/v1/messages"):
return base_url
# Otherwise append the messages endpoint
if base_url.endswith("/"):
return f"{base_url}v1/messages"
else:
return f"{base_url}/v1/messages"

8
litellm/llms/minimax/text_to_speech/__init__.py Normal file
View file

@ -0,0 +1,8 @@
"""
MiniMax Text-to-Speech module
"""
from .transformation import MinimaxException, MinimaxTextToSpeechConfig
__all__ = ["MinimaxTextToSpeechConfig", "MinimaxException"]

421
litellm/llms/minimax/text_to_speech/transformation.py Normal file
View file

@ -0,0 +1,421 @@
"""
MiniMax Text-to-Speech transformation
Maps OpenAI TTS spec to MiniMax TTS API (WebSocket-based HTTP API)
Reference: https://platform.minimax.io/docs
"""
from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union
import httpx
from httpx import Headers
import litellm
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.text_to_speech.transformation import (
BaseTextToSpeechConfig,
TextToSpeechRequestData,
)
from litellm.secret_managers.main import get_secret_str
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.types.llms.openai import HttpxBinaryResponseContent
else:
LiteLLMLoggingObj = Any
HttpxBinaryResponseContent = Any
class MinimaxException(BaseLLMException):
"""Custom exception for MiniMax API errors"""
def __init__(
self,
status_code: int,
message: str,
headers: Optional[Union[dict, Headers]] = None,
):
super().__init__(status_code=status_code, message=message, headers=headers)
class MinimaxTextToSpeechConfig(BaseTextToSpeechConfig):
"""
Configuration for MiniMax Text-to-Speech
Reference: https://platform.minimax.io/docs
MiniMax TTS API supports both WebSocket and HTTP endpoints.
This implementation uses the HTTP endpoint for simplicity.
"""
TTS_BASE_URL = "https://api.minimax.io"
TTS_ENDPOINT_PATH = "/v1/t2a_v2"
# Voice mappings from OpenAI-style voices to MiniMax voice IDs
# MiniMax supports many voices, these are common mappings
VOICE_MAPPINGS = {
"alloy": "male-qn-qingse",
"echo": "male-qn-jingying",
"fable": "female-shaonv",
"onyx": "male-qn-badao",
"nova": "female-yujie",
"shimmer": "female-tianmei",
}
# Response format mappings from OpenAI to MiniMax
FORMAT_MAPPINGS = {
"mp3": "mp3",
"pcm": "pcm",
"wav": "wav",
"flac": "flac",
}
def get_supported_openai_params(self, model: str) -> list:
"""
MiniMax TTS supports these OpenAI parameters
"""
return ["voice", "response_format", "speed"]
def _extract_voice_id(self, voice: str) -> str:
"""
Normalize the provided voice information into a MiniMax voice_id.
"""
normalized_voice = voice.strip()
mapped_voice = self.VOICE_MAPPINGS.get(normalized_voice.lower())
return mapped_voice or normalized_voice
def _resolve_voice_id(
self,
voice: Optional[Union[str, Dict[str, Any]]],
params: Dict[str, Any],
) -> str:
"""
Determine the MiniMax voice_id based on provided voice input or parameters.
"""
mapped_voice: Optional[str] = None
if isinstance(voice, str) and voice.strip():
mapped_voice = self._extract_voice_id(voice)
elif isinstance(voice, dict):
for key in ("voice_id", "id", "name"):
candidate = voice.get(key)
if isinstance(candidate, str) and candidate.strip():
mapped_voice = self._extract_voice_id(candidate)
break
elif voice is not None:
mapped_voice = self._extract_voice_id(str(voice))
if mapped_voice is None:
voice_override = params.pop("voice_id", None)
if isinstance(voice_override, str) and voice_override.strip():
mapped_voice = self._extract_voice_id(voice_override)
if mapped_voice is None:
# Default to a common voice if not specified
mapped_voice = "male-qn-qingse"
return mapped_voice
def map_openai_params(
self,
model: str,
optional_params: Dict,
voice: Optional[Union[str, Dict]] = None,
drop_params: bool = False,
kwargs: Optional[Dict[str, Any]] = None,
) -> Tuple[Optional[str], Dict]:
"""
Map OpenAI parameters to MiniMax TTS parameters
"""
mapped_params: Dict[str, Any] = {}
# Work on a copy so we don't mutate the caller's dictionary
params = dict(optional_params) if optional_params else {}
# Extract voice identifier
mapped_voice = self._resolve_voice_id(voice, params)
# Response/output format
response_format = params.pop("response_format", None)
if isinstance(response_format, str):
mapped_format = self.FORMAT_MAPPINGS.get(response_format, "mp3")
mapped_params["format"] = mapped_format
else:
mapped_params["format"] = "mp3" # Default format
# Speed parameter (MiniMax supports speed from 0.5 to 2.0)
speed = params.pop("speed", None)
if speed is not None:
try:
speed_value = float(speed)
# Clamp speed to MiniMax's supported range
speed_value = max(0.5, min(2.0, speed_value))
mapped_params["speed"] = speed_value
except (TypeError, ValueError):
mapped_params["speed"] = 1.0
else:
mapped_params["speed"] = 1.0
# Instructions parameter is OpenAI-specific; omit to prevent API errors
params.pop("instructions", None)
# Store voice_id for later use in request construction
mapped_params["voice_id"] = mapped_voice
# Handle extra_body for additional MiniMax-specific parameters
extra_body = params.pop("extra_body", None)
if isinstance(extra_body, dict):
for key, value in extra_body.items():
if value is not None:
mapped_params[key] = value
# Pass through any remaining parameters
for key, value in params.items():
if value is not None:
mapped_params[key] = value
return mapped_voice, mapped_params
def validate_environment(
self,
headers: dict,
model: str,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
) -> dict:
"""
Validate MiniMax environment and set up authentication headers
"""
api_key = (
api_key
or litellm.api_key
or get_secret_str("MINIMAX_API_KEY")
)
if api_key is None:
raise ValueError(
"MiniMax API key is required. Set MINIMAX_API_KEY environment variable or pass api_key parameter."
)
headers.update(
{
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json",
}
)
return headers
def get_error_class(
self, error_message: str, status_code: int, headers: Union[dict, Headers]
) -> BaseLLMException:
return MinimaxException(
message=error_message, status_code=status_code, headers=headers
)
def transform_text_to_speech_request(
self,
model: str,
input: str,
voice: Optional[str],
optional_params: Dict,
litellm_params: Dict,
headers: dict,
) -> TextToSpeechRequestData:
"""
Build the MiniMax TTS request payload.
MiniMax uses a different structure than OpenAI:
- model: The TTS model to use
- text: The input text
- voice_setting: Voice configuration
- audio_setting: Audio output configuration
"""
params = dict(optional_params) if optional_params else {}
# Extract parameters
voice_id = params.pop("voice_id", voice or "male-qn-qingse")
speed = params.pop("speed", 1.0)
audio_format = params.pop("format", "mp3")
# Extract additional voice settings
vol = params.pop("vol", 1.0) # Volume (0.1 to 10)
pitch = params.pop("pitch", 0) # Pitch adjustment (-12 to 12)
# Extract audio settings
sample_rate = params.pop("sample_rate", 32000) # 16000, 24000, 32000
bitrate = params.pop("bitrate", 128000) # For MP3: 64000, 128000, 192000, 256000
channel = params.pop("channel", 1) # 1 for mono, 2 for stereo
# Output format: 'url' or 'hex' (default is 'hex')
output_format = params.pop("output_format", "hex")
request_body: Dict[str, Any] = {
"model": model,
"text": input,
"stream": False, # HTTP endpoint doesn't support streaming
"output_format": output_format, # 'url' or 'hex'
"voice_setting": {
"voice_id": voice_id,
"speed": speed,
"vol": vol,
"pitch": pitch,
},
"audio_setting": {
"sample_rate": sample_rate,
"bitrate": bitrate,
"format": audio_format,
"channel": channel,
},
}
# Handle any remaining parameters from extra_body
extra_body = params.pop("extra_body", None)
if isinstance(extra_body, dict):
for key, value in extra_body.items():
if value is not None and key not in request_body:
request_body[key] = value
return TextToSpeechRequestData(
dict_body=request_body,
headers={"Content-Type": "application/json"},
)
def transform_text_to_speech_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
) -> "HttpxBinaryResponseContent":
"""
Transform MiniMax response to standard format.
MiniMax returns JSON with base64-encoded audio data:
{
"base_resp": {"status_code": 0, "status_msg": "success"},
"audio_file": "<base64_encoded_audio>",
"extra_info": {...}
}
We need to decode the base64 audio and return it as binary content.
"""
import base64
import json
from litellm.types.llms.openai import HttpxBinaryResponseContent
try:
# Parse JSON response
response_json = raw_response.json()
# MiniMax API response format check
# The API can return different structures:
# 1. {"data": {"audio": "..."}, "status": 0, ...} for HTTP endpoint
# 2. {"base_resp": {"status_code": 0, ...}, "audio_file": "..."} for older versions
# Check for errors - MiniMax uses "status" field in HTTP endpoint response
# status: 0 = success, 2 = invalid api key, etc.
status = response_json.get("status")
if status is not None and status != 0:
ced = response_json.get("ced", "Unknown error")
error_detail = ced if ced else f"API returned status {status}"
raise MinimaxException(
status_code=raw_response.status_code,
message=f"MiniMax TTS error: {error_detail}",
headers=dict(raw_response.headers),
)
# Extract audio data
# MiniMax returns audio in "data" field
data = response_json.get("data", {})
# Check if response contains a URL (output_format='url')
audio_url = data.get("audio_url", None)
if audio_url:
# If URL format is used, we need to fetch the audio from the URL
# For now, return a response indicating URL mode (TODO: fetch audio from URL)
raise MinimaxException(
status_code=500,
message=f"URL output format is not yet supported. Use 'hex' format or fetch from URL: {audio_url}",
headers=dict(raw_response.headers),
)
# Get hex-encoded audio data
audio_hex = data.get("audio", "") or response_json.get("audio_file", "")
if not audio_hex:
raise MinimaxException(
status_code=500,
message=f"No audio data in MiniMax response. Response keys: {list(response_json.keys())}",
headers=dict(raw_response.headers),
)
# MiniMax returns hex-encoded audio by default
# Try hex decoding first, fall back to base64 if that fails
try:
audio_bytes = bytes.fromhex(audio_hex)
except ValueError:
# If hex decoding fails, try base64 (for older API versions)
try:
audio_bytes = base64.b64decode(audio_hex)
except Exception as e:
raise MinimaxException(
status_code=500,
message=f"Failed to decode audio data: {str(e)}",
headers=dict(raw_response.headers),
)
# Create a new response with binary audio content
# We need to create a response that contains the decoded audio bytes
# Remove gzip encoding headers to avoid decompression issues
clean_headers = dict(raw_response.headers)
clean_headers.pop('content-encoding', None)
clean_headers.pop('transfer-encoding', None)
clean_headers['content-length'] = str(len(audio_bytes))
# Create a new response object with the binary content
binary_response = httpx.Response(
status_code=200,
headers=clean_headers,
content=audio_bytes,
request=raw_response.request,
)
return HttpxBinaryResponseContent(binary_response)
except json.JSONDecodeError as e:
raise MinimaxException(
status_code=500,
message=f"Failed to parse MiniMax response: {str(e)}",
headers=dict(raw_response.headers),
)
except Exception as e:
if isinstance(e, MinimaxException):
raise
raise MinimaxException(
status_code=500,
message=f"Error processing MiniMax response: {str(e)}",
headers=dict(raw_response.headers),
)
def get_complete_url(
self,
model: str,
api_base: Optional[str],
litellm_params: dict,
) -> str:
"""
Construct the MiniMax endpoint URL.
"""
base_url = (
api_base
or get_secret_str("MINIMAX_API_BASE")
or self.TTS_BASE_URL
)
base_url = base_url.rstrip("/")
# MiniMax uses a simple endpoint path
url = f"{base_url}{self.TTS_ENDPOINT_PATH}"
return url

37
litellm/llms/openai_like/providers.json
View file

@ -25,5 +25,40 @@
"param_mappings": {
"max_completion_tokens": "max_tokens"
}
},
"synthetic": {
"base_url": "https://api.synthetic.new/openai/v1",
"api_key_env": "SYNTHETIC_API_KEY",
"param_mappings": {
"max_completion_tokens": "max_tokens"
}
},
"apertis": {
"base_url": "https://api.stima.tech/v1",
"api_key_env": "STIMA_API_KEY",
"param_mappings": {
"max_completion_tokens": "max_tokens"
}
},
"nano-gpt": {
"base_url": "https://nano-gpt.com/api/v1",
"api_key_env": "NANOGPT_API_KEY",
"param_mappings": {
"max_completion_tokens": "max_tokens"
}
},
"poe": {
"base_url": "https://api.poe.com/v1",
"api_key_env": "POE_API_KEY",
"param_mappings": {
"max_completion_tokens": "max_tokens"
}
},
"chutes": {
"base_url": "https://llm.chutes.ai/v1/",
"api_key_env": "CHUTES_API_KEY",
"param_mappings": {
"max_completion_tokens": "max_tokens"
}
}
}
}

4
litellm/llms/vertex_ai/agent_engine/transformation.py
View file

@ -23,6 +23,7 @@ from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMExcepti
from litellm.llms.vertex_ai.agent_engine.sse_iterator import (
VertexAgentEngineResponseIterator,
)
from litellm.llms.vertex_ai.common_utils import get_vertex_base_url
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import Choices, Message, ModelResponse, Usage
@ -130,8 +131,7 @@ class VertexAgentEngineConfig(BaseConfig, VertexBase):
)
resource_path = f"projects/{vertex_project}/locations/{vertex_location}/reasoningEngines/{engine_id}"
# Build the base URL
base_url = f"https://{vertex_location}-aiplatform.googleapis.com"
base_url = get_vertex_base_url(vertex_location)
# Always use :streamQuery endpoint for actual queries
# The :query endpoint only supports session management methods

4
litellm/llms/vertex_ai/batches/handler.py
View file

@ -8,6 +8,7 @@ from litellm.llms.custom_httpx.http_handler import (
_get_httpx_client,
get_async_httpx_client,
)
from litellm.llms.vertex_ai.common_utils import get_vertex_base_url
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM
from litellm.types.llms.openai import CreateBatchRequest
from litellm.types.llms.vertex_ai import (
@ -128,7 +129,8 @@ class VertexAIBatchPrediction(VertexLLM):
) -> str:
"""Return the base url for the vertex garden models"""
# POST https://LOCATION-aiplatform.googleapis.com/v1/projects/PROJECT_ID/locations/LOCATION/batchPredictionJobs
return f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}/batchPredictionJobs"
base_url = get_vertex_base_url(vertex_location)
return f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}/batchPredictionJobs"
def retrieve_batch(
self,

65
litellm/llms/vertex_ai/common_utils.py
View file

@ -193,6 +193,18 @@ def get_vertex_base_model_name(model: str) -> str:
return model
def get_vertex_base_url(
vertex_location: Optional[str],
) -> str:
"""
Get the base URL for Vertex AI API calls.
"""
if vertex_location == "global":
return "https://aiplatform.googleapis.com"
else:
return f"https://{vertex_location}-aiplatform.googleapis.com"
def _get_embedding_url(
model: str,
vertex_project: Optional[str],
@ -212,10 +224,18 @@ def _get_embedding_url(
# Strip routing prefixes (bge/, gemma/, etc.) for endpoint URL construction
model = get_vertex_base_model_name(model=model)
url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model}:{endpoint}"
# Get base URL (handles global vs regional)
base_url = get_vertex_base_url(vertex_location)
if model.isdigit():
# https://us-central1-aiplatform.googleapis.com/v1/projects/$PROJECT_ID/locations/us-central1/endpoints/$ENDPOINT_ID:predict
url = f"https://{vertex_location}-aiplatform.googleapis.com/{vertex_api_version}/projects/{vertex_project}/locations/{vertex_location}/endpoints/{model}:{endpoint}"
# https://aiplatform.googleapis.com/v1/projects/$PROJECT_ID/locations/global/endpoints/$ENDPOINT_ID:predict
url = f"{base_url}/{vertex_api_version}/projects/{vertex_project}/locations/{vertex_location}/endpoints/{model}:{endpoint}"
else:
# Regular model -> publisher model
# https://us-central1-aiplatform.googleapis.com/v1/projects/$PROJECT_ID/locations/us-central1/publishers/google/models/{model}:predict
# https://aiplatform.googleapis.com/v1/projects/$PROJECT_ID/locations/global/publishers/google/models/{model}:predict
url = f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model}:{endpoint}"
return url, endpoint
@ -236,26 +256,23 @@ def _get_vertex_url(
if mode == "chat":
### SET RUNTIME ENDPOINT ###
endpoint = "generateContent"
base_url = get_vertex_base_url(vertex_location)
if stream is True:
endpoint = "streamGenerateContent"
if vertex_location == "global":
url = f"https://aiplatform.googleapis.com/{vertex_api_version}/projects/{vertex_project}/locations/global/publishers/google/models/{model}:{endpoint}?alt=sse"
else:
url = f"https://{vertex_location}-aiplatform.googleapis.com/{vertex_api_version}/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model}:{endpoint}?alt=sse"
else:
if vertex_location == "global":
url = f"https://aiplatform.googleapis.com/{vertex_api_version}/projects/{vertex_project}/locations/global/publishers/google/models/{model}:{endpoint}"
else:
url = f"https://{vertex_location}-aiplatform.googleapis.com/{vertex_api_version}/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model}:{endpoint}"
# if model is only numeric chars then it's a fine tuned gemini model
# model = 4965075652664360960
# send to this url: url = f"https://{vertex_location}-aiplatform.googleapis.com/{version}/projects/{vertex_project}/locations/{vertex_location}/endpoints/{model}:{endpoint}"
# send to this url: url = f"{base_url}/{version}/projects/{vertex_project}/locations/{vertex_location}/endpoints/{model}:{endpoint}"
if model.isdigit():
# It's a fine-tuned Gemini model
url = f"https://{vertex_location}-aiplatform.googleapis.com/{vertex_api_version}/projects/{vertex_project}/locations/{vertex_location}/endpoints/{model}:{endpoint}"
if stream is True:
url += "?alt=sse"
# It's a fine-tuned Gemini model - use endpoints/ path
url = f"{base_url}/{vertex_api_version}/projects/{vertex_project}/locations/{vertex_location}/endpoints/{model}:{endpoint}"
else:
# Regular model - use publishers/google/models/ path
url = f"{base_url}/{vertex_api_version}/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model}:{endpoint}"
if stream is True:
url += "?alt=sse"
elif mode == "embedding":
return _get_embedding_url(
model=model,
@ -265,15 +282,17 @@ def _get_vertex_url(
)
elif mode == "image_generation":
endpoint = "predict"
url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model}:{endpoint}"
base_url = get_vertex_base_url(vertex_location)
if model.isdigit():
url = f"https://{vertex_location}-aiplatform.googleapis.com/{vertex_api_version}/projects/{vertex_project}/locations/{vertex_location}/endpoints/{model}:{endpoint}"
# Numeric model -> custom endpoint
url = f"{base_url}/{vertex_api_version}/projects/{vertex_project}/locations/{vertex_location}/endpoints/{model}:{endpoint}"
else:
# Regular model -> publisher model
url = f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model}:{endpoint}"
elif mode == "count_tokens":
endpoint = "countTokens"
if vertex_location == "global":
url = f"https://aiplatform.googleapis.com/{vertex_api_version}/projects/{vertex_project}/locations/global/publishers/google/models/{model}:{endpoint}"
else:
url = f"https://{vertex_location}-aiplatform.googleapis.com/{vertex_api_version}/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model}:{endpoint}"
base_url = get_vertex_base_url(vertex_location)
url = f"{base_url}/{vertex_api_version}/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model}:{endpoint}"
if not url or not endpoint:
raise ValueError(f"Unable to get vertex url/endpoint for mode: {mode}")
return url, endpoint

20
litellm/llms/vertex_ai/fine_tuning/handler.py
View file

@ -8,6 +8,7 @@ import httpx
import litellm
from litellm._logging import verbose_logger
from litellm.llms.custom_httpx.http_handler import HTTPHandler, get_async_httpx_client
from litellm.llms.vertex_ai.common_utils import get_vertex_base_url
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM
from litellm.types.fine_tuning import OpenAIFineTuningHyperparameters
from litellm.types.llms.openai import FineTuningJobCreate
@ -261,7 +262,8 @@ class VertexFineTuningAPI(VertexLLM):
original_hyperparameters=original_hyperparameters or {},
)
fine_tuning_url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}/tuningJobs"
base_url = get_vertex_base_url(vertex_location)
fine_tuning_url = f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}/tuningJobs"
if _is_async is True:
return self.acreate_fine_tuning_job( # type: ignore
fine_tuning_url=fine_tuning_url,
@ -329,19 +331,21 @@ class VertexFineTuningAPI(VertexLLM):
"Content-Type": "application/json",
}
base_url = get_vertex_base_url(vertex_location)
url = None
if request_route == "/tuningJobs":
url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}/tuningJobs"
url = f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}/tuningJobs"
elif "/tuningJobs/" in request_route and "cancel" in request_route:
url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}/tuningJobs{request_route}"
url = f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}/tuningJobs{request_route}"
elif "generateContent" in request_route:
url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}{request_route}"
url = f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}{request_route}"
elif "predict" in request_route:
url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}{request_route}"
url = f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}{request_route}"
elif "/batchPredictionJobs" in request_route:
url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}{request_route}"
url = f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}{request_route}"
elif "countTokens" in request_route:
url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}{request_route}"
url = f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}{request_route}"
elif "cachedContents" in request_route:
_model = request_data.get("model")
if _model is not None and "/publishers/google/models/" not in _model:
@ -349,7 +353,7 @@ class VertexFineTuningAPI(VertexLLM):
f"projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{_model}"
)
url = f"https://{vertex_location}-aiplatform.googleapis.com/v1beta1/projects/{vertex_project}/locations/{vertex_location}{request_route}"
url = f"{base_url}/v1beta1/projects/{vertex_project}/locations/{vertex_location}{request_route}"
else:
raise ValueError(f"Unsupported Vertex AI request route: {request_route}")
if self.async_handler is None:

12
litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
View file

@ -1318,13 +1318,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
_tool_response_chunk["provider_specific_fields"] = { # type: ignore
"thought_signature": thought_signature
}
# Only embed in ID if preview features are enabled
if litellm.enable_preview_features:
_tool_response_chunk[
"id"
] = _encode_tool_call_id_with_signature(
_tool_response_chunk["id"] or "", thought_signature
)
_tool_response_chunk[
"id"
] = _encode_tool_call_id_with_signature(
_tool_response_chunk["id"] or "", thought_signature
)
_tools.append(_tool_response_chunk)
cumulative_tool_call_idx += 1
if len(_tools) == 0:

7
litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py
View file

@ -10,6 +10,7 @@ from httpx._types import RequestFiles
import litellm
from litellm.images.utils import ImageEditRequestUtils
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
from litellm.llms.vertex_ai.common_utils import get_vertex_base_url
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM
from litellm.secret_managers.main import get_secret_str
from litellm.types.images.main import ImageEditOptionalRequestParams
@ -143,11 +144,7 @@ class VertexAIGeminiImageEditConfig(BaseImageEditConfig, VertexLLM):
if not vertex_project or not vertex_location:
raise ValueError("vertex_project and vertex_location are required for Vertex AI")
# Handle global location differently (no region prefix in URL)
if vertex_location == "global":
base_url = "https://aiplatform.googleapis.com"
else:
base_url = f"https://{vertex_location}-aiplatform.googleapis.com"
base_url = get_vertex_base_url(vertex_location)
return f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model_name}:generateContent"

4
litellm/llms/vertex_ai/image_edit/vertex_imagen_transformation.py
View file

@ -9,9 +9,9 @@ import httpx
from httpx._types import RequestFiles
import litellm
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
from litellm.llms.vertex_ai.common_utils import get_vertex_base_url
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM
from litellm.secret_managers.main import get_secret_str
from litellm.types.images.main import ImageEditOptionalRequestParams
@ -136,7 +136,7 @@ class VertexAIImagenImageEditConfig(BaseImageEditConfig, VertexLLM):
if api_base:
base_url = api_base.rstrip("/")
else:
base_url = f"https://{vertex_location}-aiplatform.googleapis.com"
base_url = get_vertex_base_url(vertex_location)
return f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model_name}:predict"

14
litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py
View file

@ -7,13 +7,19 @@ import litellm
from litellm.llms.base_llm.image_generation.transformation import (
BaseImageGenerationConfig,
)
from litellm.llms.vertex_ai.common_utils import get_vertex_base_url
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import (
AllMessageValues,
OpenAIImageGenerationOptionalParams,
)
from litellm.types.utils import ImageObject, ImageResponse, ImageUsage, ImageUsageInputTokensDetails
from litellm.types.utils import (
ImageObject,
ImageResponse,
ImageUsage,
ImageUsageInputTokensDetails,
)
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@ -140,11 +146,7 @@ class VertexAIGeminiImageGenerationConfig(BaseImageGenerationConfig, VertexLLM):
if not vertex_project or not vertex_location:
raise ValueError("vertex_project and vertex_location are required for Vertex AI")
# Handle global location differently (no region prefix in URL)
if vertex_location == "global":
base_url = "https://aiplatform.googleapis.com"
else:
base_url = f"https://{vertex_location}-aiplatform.googleapis.com"
base_url = get_vertex_base_url(vertex_location)
return f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model_name}:generateContent"

3
litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py
View file

@ -7,6 +7,7 @@ import litellm
from litellm.llms.base_llm.image_generation.transformation import (
BaseImageGenerationConfig,
)
from litellm.llms.vertex_ai.common_utils import get_vertex_base_url
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import (
@ -140,7 +141,7 @@ class VertexAIImagenImageGenerationConfig(BaseImageGenerationConfig, VertexLLM):
if not vertex_project or not vertex_location:
raise ValueError("vertex_project and vertex_location are required for Vertex AI")
base_url = f"https://{vertex_location}-aiplatform.googleapis.com"
base_url = get_vertex_base_url(vertex_location)
return f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model_name}:predict"

3
litellm/llms/vertex_ai/ocr/transformation.py
View file

@ -10,6 +10,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import (
)
from litellm.llms.base_llm.ocr.transformation import DocumentType, OCRRequestData
from litellm.llms.mistral.ocr.transformation import MistralOCRConfig
from litellm.llms.vertex_ai.common_utils import get_vertex_base_url
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
@ -104,7 +105,7 @@ class VertexAIOCRConfig(MistralOCRConfig):
# Get API base URL
if api_base is None:
api_base = f"https://{vertex_location}-aiplatform.googleapis.com"
api_base = get_vertex_base_url(vertex_location)
# Ensure no trailing slash
api_base = api_base.rstrip("/")

9
litellm/llms/vertex_ai/rag_engine/transformation.py
View file

@ -8,6 +8,7 @@ from typing import Any, Dict, Optional
from litellm._logging import verbose_logger
from litellm.constants import DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE
from litellm.llms.vertex_ai.common_utils import get_vertex_base_url
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
from litellm.types.rag import RAGChunkingStrategy
@ -37,8 +38,8 @@ class VertexAIRAGTransformation(VertexBase):
Note: The REST endpoint for importRagFiles may not be publicly available.
Vertex AI RAG Engine primarily uses gRPC-based SDK.
"""
base_url = f"https://{vertex_location}-aiplatform.googleapis.com/v1"
return f"{base_url}/projects/{vertex_project}/locations/{vertex_location}/ragCorpora/{corpus_id}:importRagFiles"
base_url = get_vertex_base_url(vertex_location)
return f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}/ragCorpora/{corpus_id}:importRagFiles"
def get_retrieve_contexts_url(
self,
@ -46,8 +47,8 @@ class VertexAIRAGTransformation(VertexBase):
vertex_location: str,
) -> str:
"""Get the URL for retrieving contexts (search)."""
base_url = f"https://{vertex_location}-aiplatform.googleapis.com/v1"
return f"{base_url}/projects/{vertex_project}/locations/{vertex_location}:retrieveContexts"
base_url = get_vertex_base_url(vertex_location)
return f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}:retrieveContexts"
def transform_chunking_strategy_to_vertex_format(
self,

4
litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py
View file

@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
import httpx
from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig
from litellm.llms.vertex_ai.common_utils import get_vertex_base_url
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
from litellm.types.router import GenericLiteLLMParams
from litellm.types.vector_stores import (
@ -88,7 +89,8 @@ class VertexVectorStoreConfig(BaseVectorStoreConfig, VertexBase):
return api_base.rstrip("/")
# Vertex AI RAG API endpoint for retrieveContexts
return f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}"
base_url = get_vertex_base_url(vertex_location)
return f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}"
def transform_search_vector_store_request(
self,

5
litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py
View file

@ -8,6 +8,7 @@ their respective publisher-specific count-tokens endpoints.
from typing import Any, Dict, Optional
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.llms.vertex_ai.common_utils import get_vertex_base_url
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
@ -65,10 +66,8 @@ class VertexAIPartnerModelsTokenCounter(VertexBase):
# Use custom api_base if provided, otherwise construct default
if api_base:
base_url = api_base
elif vertex_location == "global":
base_url = "https://aiplatform.googleapis.com"
else:
base_url = f"https://{vertex_location}-aiplatform.googleapis.com"
base_url = get_vertex_base_url(vertex_location)
# Construct the count-tokens endpoint
# Format: /v1/projects/{project}/locations/{location}/publishers/{publisher}/models/count-tokens:rawPredict

5
litellm/llms/vertex_ai/vertex_model_garden/main.py
View file

@ -20,6 +20,7 @@ from typing import Callable, Optional, Union
import httpx # type: ignore
from litellm.llms.vertex_ai.common_utils import get_vertex_base_url
from litellm.utils import ModelResponse
from ..common_utils import VertexAIError, get_vertex_base_model_name
@ -34,8 +35,8 @@ def create_vertex_url(
api_base: Optional[str] = None,
) -> str:
"""Return the base url for the vertex garden models"""
# f"https://{self.endpoint.location}-aiplatform.googleapis.com/v1beta1/projects/{PROJECT_ID}/locations/{self.endpoint.location}"
return f"https://{vertex_location}-aiplatform.googleapis.com/v1beta1/projects/{vertex_project}/locations/{vertex_location}/endpoints/{model}"
base_url = get_vertex_base_url(vertex_location)
return f"{base_url}/v1beta1/projects/{vertex_project}/locations/{vertex_location}/endpoints/{model}"
class VertexAIModelGardenModels(VertexBase):

5
litellm/llms/vertex_ai/videos/transformation.py
View file

@ -17,6 +17,7 @@ from litellm.images.utils import ImageEditRequestUtils
from litellm.llms.base_llm.videos.transformation import BaseVideoConfig
from litellm.llms.vertex_ai.common_utils import (
_convert_vertex_datetime_to_openai_datetime,
get_vertex_base_url,
)
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
from litellm.types.router import GenericLiteLLMParams
@ -222,10 +223,8 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase):
# Construct the URL
if api_base:
base_url = api_base.rstrip("/")
elif vertex_location == "global":
base_url = "https://aiplatform.googleapis.com"
else:
base_url = f"https://{vertex_location}-aiplatform.googleapis.com"
base_url = get_vertex_base_url(vertex_location)
url = f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model_name}"

79
litellm/main.py
View file

@ -68,7 +68,6 @@ from litellm.constants import (
DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT,
)
from litellm.exceptions import LiteLLMUnknownProvider
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.audio_utils.utils import (
@ -98,6 +97,7 @@ from litellm.llms.base_llm.base_model_iterator import (
from litellm.llms.bedrock.common_utils import BedrockModelInfo
from litellm.llms.cohere.common_utils import CohereModelInfo
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
from litellm.llms.vertex_ai.common_utils import (
VertexAIModelRoute,
get_vertex_ai_model_route,
@ -110,6 +110,7 @@ from litellm.types.utils import (
RawRequestTypedDict,
StreamingChoices,
)
from litellm.utils import (
Choices,
CustomStreamWrapper,
@ -2247,6 +2248,42 @@ def completion( # type: ignore # noqa: PLR0915
logging.post_call(
input=messages, api_key=api_key, original_response=response
)
elif custom_llm_provider == "minimax":
api_key = (
api_key
or get_secret_str("MINIMAX_API_KEY")
or litellm.api_key
)
api_base = (
api_base
or litellm.api_base
or get_secret_str("MINIMAX_API_BASE")
or "https://api.minimax.io/v1"
)
response = base_llm_http_handler.completion(
model=model,
messages=messages,
api_base=api_base,
custom_llm_provider=custom_llm_provider,
model_response=model_response,
encoding=_get_encoding(),
logging_obj=logging,
optional_params=optional_params,
timeout=timeout,
litellm_params=litellm_params,
shared_session=shared_session,
acompletion=acompletion,
stream=stream,
api_key=api_key,
headers=headers,
client=client,
provider_config=provider_config,
)
logging.post_call(
input=messages, api_key=api_key, original_response=response
)
elif (
model in litellm.open_ai_chat_completion_models
or custom_llm_provider == "custom_openai"
@ -6471,6 +6508,46 @@ def speech( # noqa: PLR0915
api_key=api_key,
**kwargs,
)
elif custom_llm_provider == "minimax":
from litellm.llms.minimax.text_to_speech.transformation import (
MinimaxTextToSpeechConfig,
)
# MiniMax Text-to-Speech
if text_to_speech_provider_config is None:
text_to_speech_provider_config = MinimaxTextToSpeechConfig()
minimax_config = cast(
MinimaxTextToSpeechConfig, text_to_speech_provider_config
)
if api_base is not None:
litellm_params_dict["api_base"] = api_base
if api_key is not None:
litellm_params_dict["api_key"] = api_key
# Convert voice to string if it's a dict (minimax handler expects Optional[str])
voice_str: Optional[str] = None
if isinstance(voice, str):
voice_str = voice
elif isinstance(voice, dict):
# Extract voice_id from dict if needed
voice_str = voice.get("voice_id") or voice.get("id") or voice.get("name")
response = base_llm_http_handler.text_to_speech_handler(
model=model,
input=input,
voice=voice_str,
text_to_speech_provider_config=minimax_config,
text_to_speech_optional_params=optional_params,
custom_llm_provider=custom_llm_provider,
litellm_params=litellm_params_dict,
logging_obj=logging_obj,
timeout=timeout,
extra_headers=extra_headers,
client=client,
_is_async=aspeech or False,
)
elif custom_llm_provider == "aws_polly":
from litellm.llms.aws_polly.text_to_speech.transformation import (
AWSPollyTextToSpeechConfig,

372
litellm/model_prices_and_context_window_backup.json
View file

@ -1357,6 +1357,20 @@
"litellm_provider": "azure",
"mode": "chat"
},
"azure_ai/gpt-oss-120b": {
"input_cost_per_token": 1.5e-7,
"output_cost_per_token": 6e-7,
"litellm_provider": "azure_ai",
"max_input_tokens": 131072,
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"azure/eu/gpt-4o-2024-08-06": {
"deprecation_date": "2026-02-27",
"cache_read_input_token_cost": 1.375e-06,
@ -3707,6 +3721,32 @@
"/v1/images/generations"
]
},
"azure/gpt-image-1.5": {
"cache_read_input_image_token_cost": 2e-06,
"cache_read_input_token_cost": 1.25e-06,
"input_cost_per_token": 5e-06,
"input_cost_per_image_token": 8e-06,
"litellm_provider": "azure",
"mode": "image_generation",
"output_cost_per_image_token": 3.2e-05,
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
]
},
"azure/gpt-image-1.5-2025-12-16": {
"cache_read_input_image_token_cost": 2e-06,
"cache_read_input_token_cost": 1.25e-06,
"input_cost_per_token": 5e-06,
"input_cost_per_image_token": 8e-06,
"litellm_provider": "azure",
"mode": "image_generation",
"output_cost_per_image_token": 3.2e-05,
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
]
},
"azure/low/1024-x-1024/gpt-image-1-mini": {
"input_cost_per_pixel": 2.0751953125e-09,
"litellm_provider": "azure",
@ -18053,75 +18093,6 @@
"supports_response_schema": true,
"supports_vision": true
},
"groq/deepseek-r1-distill-llama-70b": {
"input_cost_per_token": 7.5e-07,
"litellm_provider": "groq",
"max_input_tokens": 128000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 9.9e-07,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": false,
"supports_tool_choice": true
},
"groq/distil-whisper-large-v3-en": {
"input_cost_per_second": 5.56e-06,
"litellm_provider": "groq",
"mode": "audio_transcription",
"output_cost_per_second": 0.0
},
"groq/gemma-7b-it": {
"deprecation_date": "2024-12-18",
"input_cost_per_token": 7e-08,
"litellm_provider": "groq",
"max_input_tokens": 8192,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 7e-08,
"supports_function_calling": true,
"supports_response_schema": false,
"supports_tool_choice": true
},
"groq/gemma2-9b-it": {
"input_cost_per_token": 2e-07,
"litellm_provider": "groq",
"max_input_tokens": 8192,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 2e-07,
"supports_function_calling": false,
"supports_response_schema": false,
"supports_tool_choice": false
},
"groq/llama-3.1-405b-reasoning": {
"input_cost_per_token": 5.9e-07,
"litellm_provider": "groq",
"max_input_tokens": 8192,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 7.9e-07,
"supports_function_calling": true,
"supports_response_schema": false,
"supports_tool_choice": true
},
"groq/llama-3.1-70b-versatile": {
"deprecation_date": "2025-01-24",
"input_cost_per_token": 5.9e-07,
"litellm_provider": "groq",
"max_input_tokens": 8192,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 7.9e-07,
"supports_function_calling": true,
"supports_response_schema": false,
"supports_tool_choice": true
},
"groq/llama-3.1-8b-instant": {
"input_cost_per_token": 5e-08,
"litellm_provider": "groq",
@ -18134,97 +18105,6 @@
"supports_response_schema": false,
"supports_tool_choice": true
},
"groq/llama-3.2-11b-text-preview": {
"deprecation_date": "2024-10-28",
"input_cost_per_token": 1.8e-07,
"litellm_provider": "groq",
"max_input_tokens": 8192,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.8e-07,
"supports_function_calling": true,
"supports_response_schema": false,
"supports_tool_choice": true
},
"groq/llama-3.2-11b-vision-preview": {
"deprecation_date": "2025-04-14",
"input_cost_per_token": 1.8e-07,
"litellm_provider": "groq",
"max_input_tokens": 8192,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.8e-07,
"supports_function_calling": true,
"supports_response_schema": false,
"supports_tool_choice": true,
"supports_vision": true
},
"groq/llama-3.2-1b-preview": {
"deprecation_date": "2025-04-14",
"input_cost_per_token": 4e-08,
"litellm_provider": "groq",
"max_input_tokens": 8192,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 4e-08,
"supports_function_calling": true,
"supports_response_schema": false,
"supports_tool_choice": true
},
"groq/llama-3.2-3b-preview": {
"deprecation_date": "2025-04-14",
"input_cost_per_token": 6e-08,
"litellm_provider": "groq",
"max_input_tokens": 8192,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 6e-08,
"supports_function_calling": true,
"supports_response_schema": false,
"supports_tool_choice": true
},
"groq/llama-3.2-90b-text-preview": {
"deprecation_date": "2024-11-25",
"input_cost_per_token": 9e-07,
"litellm_provider": "groq",
"max_input_tokens": 8192,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 9e-07,
"supports_function_calling": true,
"supports_response_schema": false,
"supports_tool_choice": true
},
"groq/llama-3.2-90b-vision-preview": {
"deprecation_date": "2025-04-14",
"input_cost_per_token": 9e-07,
"litellm_provider": "groq",
"max_input_tokens": 8192,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 9e-07,
"supports_function_calling": true,
"supports_response_schema": false,
"supports_tool_choice": true,
"supports_vision": true
},
"groq/llama-3.3-70b-specdec": {
"deprecation_date": "2025-04-14",
"input_cost_per_token": 5.9e-07,
"litellm_provider": "groq",
"max_input_tokens": 8192,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 9.9e-07,
"supports_tool_choice": true
},
"groq/llama-3.3-70b-versatile": {
"input_cost_per_token": 5.9e-07,
"litellm_provider": "groq",
@ -18237,7 +18117,7 @@
"supports_response_schema": false,
"supports_tool_choice": true
},
"groq/llama-guard-3-8b": {
"groq/meta-llama/llama-guard-4-12b": {
"input_cost_per_token": 2e-07,
"litellm_provider": "groq",
"max_input_tokens": 8192,
@ -18246,44 +18126,6 @@
"mode": "chat",
"output_cost_per_token": 2e-07
},
"groq/llama2-70b-4096": {
"input_cost_per_token": 7e-07,
"litellm_provider": "groq",
"max_input_tokens": 4096,
"max_output_tokens": 4096,
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 8e-07,
"supports_function_calling": true,
"supports_response_schema": false,
"supports_tool_choice": true
},
"groq/llama3-groq-70b-8192-tool-use-preview": {
"deprecation_date": "2025-01-06",
"input_cost_per_token": 8.9e-07,
"litellm_provider": "groq",
"max_input_tokens": 8192,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 8.9e-07,
"supports_function_calling": true,
"supports_response_schema": false,
"supports_tool_choice": true
},
"groq/llama3-groq-8b-8192-tool-use-preview": {
"deprecation_date": "2025-01-06",
"input_cost_per_token": 1.9e-07,
"litellm_provider": "groq",
"max_input_tokens": 8192,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.9e-07,
"supports_function_calling": true,
"supports_response_schema": false,
"supports_tool_choice": true
},
"groq/meta-llama/llama-4-maverick-17b-128e-instruct": {
"input_cost_per_token": 2e-07,
"litellm_provider": "groq",
@ -18294,7 +18136,8 @@
"output_cost_per_token": 6e-07,
"supports_function_calling": true,
"supports_response_schema": true,
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_vision": true
},
"groq/meta-llama/llama-4-scout-17b-16e-instruct": {
"input_cost_per_token": 1.1e-07,
@ -18306,41 +18149,8 @@
"output_cost_per_token": 3.4e-07,
"supports_function_calling": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"groq/mistral-saba-24b": {
"input_cost_per_token": 7.9e-07,
"litellm_provider": "groq",
"max_input_tokens": 32000,
"max_output_tokens": 32000,
"max_tokens": 32000,
"mode": "chat",
"output_cost_per_token": 7.9e-07
},
"groq/mixtral-8x7b-32768": {
"deprecation_date": "2025-03-20",
"input_cost_per_token": 2.4e-07,
"litellm_provider": "groq",
"max_input_tokens": 32768,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 2.4e-07,
"supports_function_calling": true,
"supports_response_schema": false,
"supports_tool_choice": true
},
"groq/moonshotai/kimi-k2-instruct": {
"input_cost_per_token": 1e-06,
"litellm_provider": "groq",
"max_input_tokens": 131072,
"max_output_tokens": 16384,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 3e-06,
"supports_function_calling": true,
"supports_response_schema": true,
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_vision": true
},
"groq/moonshotai/kimi-k2-instruct-0905": {
"input_cost_per_token": 1e-06,
@ -19580,6 +19390,80 @@
"output_cost_per_token": 1.2e-06,
"supports_system_messages": true
},
"minimax/speech-02-hd": {
"input_cost_per_character": 0.0001,
"litellm_provider": "minimax",
"mode": "audio_speech",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"minimax/speech-02-turbo": {
"input_cost_per_character": 0.00006,
"litellm_provider": "minimax",
"mode": "audio_speech",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"minimax/speech-2.6-hd": {
"input_cost_per_character": 0.0001,
"litellm_provider": "minimax",
"mode": "audio_speech",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"minimax/speech-2.6-turbo": {
"input_cost_per_character": 0.00006,
"litellm_provider": "minimax",
"mode": "audio_speech",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"minimax/MiniMax-M2.1": {
"input_cost_per_token": 3e-07,
"output_cost_per_token": 1.2e-06,
"cache_read_input_token_cost": 3e-08,
"cache_creation_input_token_cost": 3.75e-07,
"litellm_provider": "minimax",
"mode": "chat",
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"max_input_tokens": 1000000,
"max_output_tokens": 8192
},
"minimax/MiniMax-M2.1-lightning": {
"input_cost_per_token": 3e-07,
"output_cost_per_token": 2.4e-06,
"cache_read_input_token_cost": 3e-08,
"cache_creation_input_token_cost": 3.75e-07,
"litellm_provider": "minimax",
"mode": "chat",
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"max_input_tokens": 1000000,
"max_output_tokens": 8192
},
"minimax/MiniMax-M2": {
"input_cost_per_token": 3e-07,
"output_cost_per_token": 1.2e-06,
"cache_read_input_token_cost": 3e-08,
"cache_creation_input_token_cost": 3.75e-07,
"litellm_provider": "minimax",
"mode": "chat",
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_prompt_caching": true,
"supports_system_messages": true,
"max_input_tokens": 200000,
"max_output_tokens": 8192
},
"mistral.magistral-small-2509": {
"input_cost_per_token": 5e-07,
"litellm_provider": "bedrock_converse",
@ -25111,6 +24995,7 @@
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"together_ai/Qwen/Qwen2.5-7B-Instruct-Turbo": {
@ -25118,6 +25003,7 @@
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"together_ai/Qwen/Qwen3-235B-A22B-Instruct-2507-tput": {
@ -25129,6 +25015,7 @@
"source": "https://www.together.ai/models/qwen3-235b-a22b-instruct-2507-fp8",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"together_ai/Qwen/Qwen3-235B-A22B-Thinking-2507": {
@ -25140,6 +25027,7 @@
"source": "https://www.together.ai/models/qwen3-235b-a22b-thinking-2507",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"together_ai/Qwen/Qwen3-235B-A22B-fp8-tput": {
@ -25162,6 +25050,7 @@
"source": "https://www.together.ai/models/qwen3-coder-480b-a35b-instruct",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"together_ai/deepseek-ai/DeepSeek-R1": {
@ -25174,6 +25063,7 @@
"output_cost_per_token": 7e-06,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"together_ai/deepseek-ai/DeepSeek-R1-0528-tput": {
@ -25185,6 +25075,7 @@
"source": "https://www.together.ai/models/deepseek-r1-0528-throughput",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"together_ai/deepseek-ai/DeepSeek-V3": {
@ -25197,6 +25088,7 @@
"output_cost_per_token": 1.25e-06,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"together_ai/deepseek-ai/DeepSeek-V3.1": {
@ -25216,6 +25108,7 @@
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo": {
@ -25245,6 +25138,7 @@
"output_cost_per_token": 8.5e-07,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"together_ai/meta-llama/Llama-4-Scout-17B-16E-Instruct": {
@ -25254,6 +25148,7 @@
"output_cost_per_token": 5.9e-07,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"together_ai/meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo": {
@ -25263,6 +25158,7 @@
"output_cost_per_token": 3.5e-06,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"together_ai/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo": {
@ -25318,6 +25214,7 @@
"source": "https://www.together.ai/models/kimi-k2-instruct",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"together_ai/openai/gpt-oss-120b": {
@ -25329,6 +25226,7 @@
"source": "https://www.together.ai/models/gpt-oss-120b",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"together_ai/openai/gpt-oss-20b": {
@ -25340,6 +25238,7 @@
"source": "https://www.together.ai/models/gpt-oss-20b",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"together_ai/togethercomputer/CodeLlama-34b-Instruct": {
@ -25358,6 +25257,7 @@
"source": "https://www.together.ai/models/glm-4-5-air",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"together_ai/zai-org/GLM-4.6": {
@ -25394,6 +25294,7 @@
"source": "https://www.together.ai/models/qwen3-next-80b-a3b-instruct",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"together_ai/Qwen/Qwen3-Next-80B-A3B-Thinking": {
@ -25405,6 +25306,7 @@
"source": "https://www.together.ai/models/qwen3-next-80b-a3b-thinking",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"tts-1": {

372
litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
View file

@ -11,7 +11,7 @@ import datetime
import hashlib
import json
import re
from typing import Any, Dict, List, Optional, Set, Tuple, Union, cast
from typing import Any, Dict, List, Literal, Optional, Set, Tuple, Union, cast
from urllib.parse import urlparse
from fastapi import HTTPException
@ -84,6 +84,8 @@ def _deserialize_json_dict(data: Any) -> Optional[Dict[str, str]]:
class MCPServerManager:
_STDIO_ENV_TEMPLATE_PATTERN = re.compile(r"^\$\{(X-[^}]+)\}$")
def __init__(self):
self.registry: Dict[str, MCPServer] = {}
self.config_mcp_servers: Dict[str, MCPServer] = {}
@ -671,11 +673,39 @@ class MCPServerManager:
#########################################################
# Methods that call the upstream MCP servers
#########################################################
def _build_stdio_env(
self,
server: MCPServer,
raw_headers: Optional[Dict[str, str]] = None,
) -> Optional[Dict[str, str]]:
"""Resolve stdio env values, supporting header-driven placeholders."""
if server.transport != MCPTransport.stdio or not server.env:
return None
resolved_env: Dict[str, str] = {}
normalized_headers = {k.lower(): v for k, v in (raw_headers or {}).items()}
for env_key, env_value in server.env.items():
stripped_value = env_value.strip()
match = self._STDIO_ENV_TEMPLATE_PATTERN.match(stripped_value)
if match:
header_name = match.group(1)
header_value = normalized_headers.get(header_name.lower())
if header_value is None:
continue
resolved_env[env_key] = header_value
else:
resolved_env[env_key] = env_value
return resolved_env
def _create_mcp_client(
self,
server: MCPServer,
mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None,
extra_headers: Optional[Dict[str, str]] = None,
stdio_env: Optional[Dict[str, str]] = None,
) -> MCPClient:
"""
Create an MCPClient instance for the given server.
@ -692,10 +722,13 @@ class MCPServerManager:
# Handle stdio transport
if transport == MCPTransport.stdio:
# For stdio, we need to get the stdio config from the server
resolved_env = stdio_env if stdio_env is not None else server.env or {}
stdio_config: Optional[MCPStdioConfig] = None
if server.command and server.args is not None:
stdio_config = MCPStdioConfig(
command=server.command, args=server.args, env=server.env or {}
command=server.command,
args=server.args,
env=resolved_env,
)
return MCPClient(
@ -725,6 +758,7 @@ class MCPServerManager:
mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None,
extra_headers: Optional[Dict[str, str]] = None,
add_prefix: bool = True,
raw_headers: Optional[Dict[str, str]] = None,
) -> List[MCPTool]:
"""
Helper method to get tools from a single MCP server with prefixed names.
@ -751,10 +785,13 @@ class MCPServerManager:
extra_headers = {}
extra_headers.update(server.static_headers)
stdio_env = self._build_stdio_env(server, raw_headers)
client = self._create_mcp_client(
server=server,
mcp_auth_header=mcp_auth_header,
extra_headers=extra_headers,
stdio_env=stdio_env,
)
## HANDLE OPENAPI TOOLS
@ -784,6 +821,7 @@ class MCPServerManager:
mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None,
extra_headers: Optional[Dict[str, str]] = None,
add_prefix: bool = True,
raw_headers: Optional[Dict[str, str]] = None,
) -> List[Prompt]:
"""
Helper method to get prompts from a single MCP server with prefixed names.
@ -807,10 +845,13 @@ class MCPServerManager:
extra_headers = {}
extra_headers.update(server.static_headers)
stdio_env = self._build_stdio_env(server, raw_headers)
client = self._create_mcp_client(
server=server,
mcp_auth_header=mcp_auth_header,
extra_headers=extra_headers,
stdio_env=stdio_env,
)
prompts = await client.list_prompts()
@ -833,6 +874,7 @@ class MCPServerManager:
mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None,
extra_headers: Optional[Dict[str, str]] = None,
add_prefix: bool = True,
raw_headers: Optional[Dict[str, str]] = None,
) -> List[Resource]:
"""Fetch available resources from a single MCP server."""
@ -847,10 +889,13 @@ class MCPServerManager:
extra_headers = {}
extra_headers.update(server.static_headers)
stdio_env = self._build_stdio_env(server, raw_headers)
client = self._create_mcp_client(
server=server,
mcp_auth_header=mcp_auth_header,
extra_headers=extra_headers,
stdio_env=stdio_env,
)
resources = await client.list_resources()
@ -873,6 +918,7 @@ class MCPServerManager:
mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None,
extra_headers: Optional[Dict[str, str]] = None,
add_prefix: bool = True,
raw_headers: Optional[Dict[str, str]] = None,
) -> List[ResourceTemplate]:
"""Fetch available resource templates from a single MCP server."""
@ -887,10 +933,13 @@ class MCPServerManager:
extra_headers = {}
extra_headers.update(server.static_headers)
stdio_env = self._build_stdio_env(server, raw_headers)
client = self._create_mcp_client(
server=server,
mcp_auth_header=mcp_auth_header,
extra_headers=extra_headers,
stdio_env=stdio_env,
)
resource_templates = await client.list_resource_templates()
@ -913,6 +962,7 @@ class MCPServerManager:
url: AnyUrl,
mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None,
extra_headers: Optional[Dict[str, str]] = None,
raw_headers: Optional[Dict[str, str]] = None,
) -> ReadResourceResult:
"""Read resource contents from a specific MCP server."""
@ -924,10 +974,13 @@ class MCPServerManager:
extra_headers = {}
extra_headers.update(server.static_headers)
stdio_env = self._build_stdio_env(server, raw_headers)
client = self._create_mcp_client(
server=server,
mcp_auth_header=mcp_auth_header,
extra_headers=extra_headers,
stdio_env=stdio_env,
)
return await client.read_resource(url)
@ -939,6 +992,7 @@ class MCPServerManager:
arguments: Optional[Dict[str, Any]] = None,
mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None,
extra_headers: Optional[Dict[str, str]] = None,
raw_headers: Optional[Dict[str, str]] = None,
) -> GetPromptResult:
"""Fetch a specific prompt definition from a single MCP server."""
@ -950,10 +1004,13 @@ class MCPServerManager:
extra_headers = {}
extra_headers.update(server.static_headers)
stdio_env = self._build_stdio_env(server, raw_headers)
client = self._create_mcp_client(
server=server,
mcp_auth_header=mcp_auth_header,
extra_headers=extra_headers,
stdio_env=stdio_env,
)
get_prompt_request_params = GetPromptRequestParams(
@ -1742,10 +1799,13 @@ class MCPServerManager:
extra_headers = {}
extra_headers.update(mcp_server.static_headers)
stdio_env = self._build_stdio_env(mcp_server, raw_headers)
client = self._create_mcp_client(
server=mcp_server,
mcp_auth_header=server_auth_header,
extra_headers=extra_headers,
stdio_env=stdio_env,
)
call_tool_params = MCPCallToolRequestParams(
@ -2067,7 +2127,7 @@ class MCPServerManager:
async def health_check_server(
self, server_id: str, mcp_auth_header: Optional[str] = None
) -> Dict[str, Any]:
) -> LiteLLM_MCPServerTable:
"""
Perform a health check on a specific MCP server.
@ -2078,206 +2138,180 @@ class MCPServerManager:
Returns:
Dict containing health check results
"""
import time
from datetime import datetime
server = self.get_mcp_server_by_id(server_id)
if not server:
return {
"server_id": server_id,
"server_name": None,
"status": "unknown",
"error": "Server not found",
"last_health_check": datetime.now().isoformat(),
"response_time_ms": None,
}
start_time = time.time()
try:
# Try to get tools from the server as a health check
tools = await self._get_tools_from_server(server, mcp_auth_header)
response_time = (time.time() - start_time) * 1000
return {
"server_id": server_id,
"server_name": server.name,
"status": "healthy",
"tools_count": len(tools),
"last_health_check": datetime.now().isoformat(),
"response_time_ms": round(response_time, 2),
"error": None,
}
except Exception as e:
response_time = (time.time() - start_time) * 1000
error_message = str(e)
return {
"server_id": server_id,
"server_name": server.name,
"status": "unhealthy",
"last_health_check": datetime.now().isoformat(),
"response_time_ms": round(response_time, 2),
"error": error_message,
}
async def health_check_all_servers(
self, mcp_auth_header: Optional[str] = None
) -> Dict[str, Any]:
"""
Perform health checks on all MCP servers.
Args:
mcp_auth_header: Optional authentication header for the MCP servers
Returns:
Dict containing health check results for all servers
"""
all_servers = self.get_registry()
results = {}
for server_id, server in all_servers.items():
results[server_id] = await self.health_check_server(
server_id, mcp_auth_header
verbose_logger.warning(f"MCP Server {server_id} not found")
return LiteLLM_MCPServerTable(
server_id=server_id,
server_name=None,
transport=MCPTransport.http, # Default transport for not found servers
status="unknown",
health_check_error="Server not found",
last_health_check=datetime.now(),
)
return results
status: Literal["healthy", "unhealthy", "unknown"] = "unknown"
health_check_error = None
async def health_check_allowed_servers(
self,
user_api_key_auth: Optional[UserAPIKeyAuth] = None,
mcp_auth_header: Optional[str] = None,
) -> Dict[str, Any]:
"""
Perform health checks on all MCP servers that the user has access to.
# Check if we should skip health check based on auth configuration
should_skip_health_check = False
Args:
user_api_key_auth: User authentication info for access control
mcp_auth_header: Optional authentication header for the MCP servers
# Skip if auth_type is oauth2
if server.auth_type == MCPAuth.oauth2:
should_skip_health_check = True
# Skip if auth_type is not none and authentication_token is missing
elif (
server.auth_type
and server.auth_type != MCPAuth.none
and not server.authentication_token
):
should_skip_health_check = True
Returns:
Dict containing health check results for accessible servers
"""
# Get allowed servers for the user
allowed_server_ids = await self.get_allowed_mcp_servers(user_api_key_auth)
if not should_skip_health_check:
extra_headers = {}
if server.static_headers:
extra_headers.update(server.static_headers)
# Perform health checks on allowed servers
results = {}
for server_id in allowed_server_ids:
results[server_id] = await self.health_check_server(
server_id, mcp_auth_header
client = self._create_mcp_client(
server=server,
mcp_auth_header=None,
extra_headers=extra_headers,
stdio_env=None,
)
return results
try:
async def _noop(session):
return "ok"
# Add timeout wrapper to prevent hanging
await asyncio.wait_for(client.run_with_session(_noop), timeout=10.0)
status = "healthy"
except asyncio.TimeoutError:
health_check_error = "Health check timed out after 10 seconds"
status = "unhealthy"
except Exception as e:
health_check_error = str(e)
status = "unhealthy"
return LiteLLM_MCPServerTable(
server_id=server.server_id,
server_name=server.server_name,
alias=server.alias,
description=(
server.mcp_info.get("description") if server.mcp_info else None
),
url=server.url,
transport=server.transport,
auth_type=server.auth_type,
created_at=datetime.now(),
updated_at=datetime.now(),
teams=[],
mcp_access_groups=server.access_groups or [],
allowed_tools=server.allowed_tools or [],
extra_headers=server.extra_headers or [],
mcp_info=server.mcp_info,
static_headers=server.static_headers,
status=status,
last_health_check=datetime.now(),
health_check_error=health_check_error,
command=getattr(server, "command", None),
args=getattr(server, "args", None) or [],
env=getattr(server, "env", None) or {},
)
async def get_all_mcp_servers_with_health_and_teams(
self,
user_api_key_auth: Optional[UserAPIKeyAuth] = None,
include_health: bool = True,
server_ids: Optional[List[str]] = None,
) -> List[LiteLLM_MCPServerTable]:
"""
Get all MCP servers that the user has access to, with health status and team information.
Args:
user_api_key_auth: User authentication info for access control
include_health: Whether to include health check information
server_ids: Optional list of server IDs to filter. If provided, only these servers
will be checked (subject to access control). If None, all accessible servers are checked.
Returns:
List of MCP server objects with health and team data
"""
from litellm.proxy._experimental.mcp_server.db import (
get_all_mcp_servers,
get_mcp_servers,
)
from litellm.proxy.management_endpoints.common_utils import _user_has_admin_view
from litellm.proxy.proxy_server import prisma_client
# Get allowed server IDs
allowed_server_ids = await self.get_allowed_mcp_servers(user_api_key_auth)
# Get servers from database
# Filter by requested server_ids if provided
if server_ids:
# Only check servers that are both requested AND accessible
target_server_ids = [sid for sid in server_ids if sid in allowed_server_ids]
else:
# Check all accessible servers
target_server_ids = allowed_server_ids
# Run health checks concurrently
tasks = [self.health_check_server(server_id) for server_id in target_server_ids]
results = await asyncio.gather(*tasks)
# Filter out None results (servers that were not found)
list_mcp_servers = [server for server in results if server is not None]
return list_mcp_servers
async def get_all_allowed_mcp_servers(
self,
user_api_key_auth: Optional[UserAPIKeyAuth] = None,
) -> List[LiteLLM_MCPServerTable]:
"""
Get all MCP servers that the user has access to.
Args:
user_api_key_auth: User authentication info for access control
Returns:
List of MCP server objects without health status
"""
from datetime import datetime
# Get allowed server IDs
allowed_server_ids = await self.get_allowed_mcp_servers(user_api_key_auth)
list_mcp_servers: List[LiteLLM_MCPServerTable] = []
if prisma_client is not None:
list_mcp_servers = await get_mcp_servers(prisma_client, allowed_server_ids)
# If admin, also get all servers from database
if user_api_key_auth and _user_has_admin_view(user_api_key_auth):
all_mcp_servers = await get_all_mcp_servers(prisma_client)
for server in all_mcp_servers:
if server.server_id not in allowed_server_ids:
list_mcp_servers.append(server)
for server_id in allowed_server_ids:
server = self.get_mcp_server_by_id(server_id)
if not server:
verbose_logger.warning(f"MCP Server {server_id} not found in registry")
continue
# Add config.yaml servers
for _server_id, _server_config in self.config_mcp_servers.items():
if _server_id in allowed_server_ids:
list_mcp_servers.append(
LiteLLM_MCPServerTable(
**{
**_server_config.model_dump(),
"created_at": datetime.datetime.now(),
"updated_at": datetime.datetime.now(),
"description": (
_server_config.mcp_info.get("description")
if _server_config.mcp_info
else None
),
"allowed_tools": _server_config.allowed_tools or [],
"mcp_info": _server_config.mcp_info,
"mcp_access_groups": _server_config.access_groups or [],
"extra_headers": _server_config.extra_headers or [],
"command": getattr(_server_config, "command", None),
"args": getattr(_server_config, "args", None) or [],
"env": getattr(_server_config, "env", None) or {},
}
)
)
# Get team information for non-admin users
server_to_teams_map: Dict[str, List[Dict[str, str]]] = {}
if (
user_api_key_auth
and not _user_has_admin_view(user_api_key_auth)
and prisma_client is not None
):
teams = await prisma_client.db.litellm_teamtable.find_many(
include={"object_permission": True}
# Build LiteLLM_MCPServerTable without health check
mcp_server_table = LiteLLM_MCPServerTable(
server_id=server.server_id,
server_name=server.server_name,
alias=server.alias,
description=(
server.mcp_info.get("description") if server.mcp_info else None
),
url=server.url,
transport=server.transport,
auth_type=server.auth_type,
created_at=datetime.now(),
updated_at=datetime.now(),
teams=[],
mcp_access_groups=server.access_groups or [],
allowed_tools=server.allowed_tools or [],
extra_headers=server.extra_headers or [],
mcp_info=server.mcp_info,
static_headers=server.static_headers,
status=None, # No health check performed
last_health_check=None, # No health check performed
health_check_error=None,
command=getattr(server, "command", None),
args=getattr(server, "args", None) or [],
env=getattr(server, "env", None) or {},
)
user_teams = []
for team in teams:
if team.members_with_roles:
for member in team.members_with_roles:
if (
"user_id" in member
and member["user_id"] is not None
and member["user_id"] == user_api_key_auth.user_id
):
user_teams.append(team)
# Create a mapping of server_id to teams that have access to it
for team in user_teams:
if team.object_permission and team.object_permission.mcp_servers:
for server_id in team.object_permission.mcp_servers:
if server_id not in server_to_teams_map:
server_to_teams_map[server_id] = []
server_to_teams_map[server_id].append(
{
"team_id": team.team_id,
"team_alias": team.team_alias,
"organization_id": team.organization_id,
}
)
## mark invalid servers w/ reason for being invalid
valid_server_ids = self.get_all_mcp_server_ids()
for server in list_mcp_servers:
if server.server_id not in valid_server_ids:
server.status = "unhealthy"
## try adding server to registry to get error
try:
await self.add_update_server(server)
except Exception as e:
server.health_check_error = str(e)
server.health_check_error = "Server is not in in memory registry yet. This could be a temporary sync issue."
list_mcp_servers.append(mcp_server_table)
return list_mcp_servers

66
litellm/proxy/_experimental/mcp_server/rest_endpoints.py
View file

@ -1,5 +1,4 @@
import importlib
import traceback
from typing import Dict, List, Optional, Union
from fastapi import APIRouter, Depends, Query, Request
@ -71,12 +70,17 @@ if MCP_AVAILABLE:
for tool in tools
]
async def _get_tools_for_single_server(server, server_auth_header):
async def _get_tools_for_single_server(
server,
server_auth_header,
raw_headers: Optional[Dict[str, str]] = None,
):
"""Helper function to get tools for a single server."""
tools = await global_mcp_server_manager._get_tools_from_server(
server=server,
mcp_auth_header=server_auth_header,
add_prefix=False,
raw_headers=raw_headers,
)
# Filter tools based on allowed_tools configuration
@ -122,6 +126,7 @@ if MCP_AVAILABLE:
try:
# Extract auth headers from request
headers = request.headers
raw_headers_from_request = dict(headers)
mcp_auth_header = MCPRequestHandler._get_mcp_auth_header_from_headers(
headers
)
@ -148,7 +153,7 @@ if MCP_AVAILABLE:
try:
list_tools_result = await _get_tools_for_single_server(
server, server_auth_header
server, server_auth_header, raw_headers_from_request
)
except Exception as e:
verbose_logger.exception(
@ -169,7 +174,7 @@ if MCP_AVAILABLE:
try:
tools_result = await _get_tools_for_single_server(
server, server_auth_header
server, server_auth_header, raw_headers_from_request
)
list_tools_result.extend(tools_result)
except Exception as e:
@ -232,13 +237,13 @@ if MCP_AVAILABLE:
# but they weren't being extracted and passed to call_mcp_tool.
# This fix ensures auth headers are properly extracted from the HTTP request
# and passed through to the MCP server for authentication.
headers = request.headers
raw_headers_from_request = dict(headers)
mcp_auth_header = MCPRequestHandler._get_mcp_auth_header_from_headers(
request.headers
headers
)
mcp_server_auth_headers = (
MCPRequestHandler._get_mcp_server_auth_headers_from_headers(
request.headers
)
MCPRequestHandler._get_mcp_server_auth_headers_from_headers(headers)
)
# Add extracted headers to data dict to pass to call_mcp_tool
@ -246,6 +251,7 @@ if MCP_AVAILABLE:
data["mcp_auth_header"] = mcp_auth_header
if mcp_server_auth_headers:
data["mcp_server_auth_headers"] = mcp_server_auth_headers
data["raw_headers"] = raw_headers_from_request
result = await call_mcp_tool(**data)
return result
@ -300,6 +306,7 @@ if MCP_AVAILABLE:
operation,
mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None,
oauth2_headers: Optional[Dict[str, str]] = None,
raw_headers: Optional[Dict[str, str]] = None,
):
"""
Common helper to create MCP client, execute operation, and ensure proper cleanup.
@ -312,33 +319,43 @@ if MCP_AVAILABLE:
Operation result or error response
"""
try:
server_model = MCPServer(
server_id=request.server_id or "",
name=request.alias or request.server_name or "",
url=request.url,
transport=request.transport,
auth_type=request.auth_type,
mcp_info=request.mcp_info,
command=request.command,
args=request.args,
env=request.env,
)
stdio_env = global_mcp_server_manager._build_stdio_env(
server_model, raw_headers
)
client = global_mcp_server_manager._create_mcp_client(
server=MCPServer(
server_id=request.server_id or "",
name=request.alias or request.server_name or "",
url=request.url,
transport=request.transport,
auth_type=request.auth_type,
mcp_info=request.mcp_info,
),
server=server_model,
mcp_auth_header=mcp_auth_header,
extra_headers=oauth2_headers,
stdio_env=stdio_env,
)
return await operation(client)
except Exception as e:
verbose_logger.error(f"Error in MCP operation: {e}", exc_info=True)
stack_trace = traceback.format_exc()
return {
"status": "error",
"message": f"An internal error has occurred: {str(e)}",
"stack_trace": stack_trace,
"message": "An internal error has occurred while testing the MCP server.",
}
@router.post("/test/connection")
@router.post("/test/connection", dependencies=[Depends(user_api_key_auth)])
async def test_connection(
request: NewMCPServerRequest,
request: Request,
new_mcp_server_request: NewMCPServerRequest,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
):
"""
Test if we can connect to the provided MCP server before adding it
@ -351,7 +368,11 @@ if MCP_AVAILABLE:
await client.run_with_session(_noop)
return {"status": "ok"}
return await _execute_with_mcp_client(request, _test_connection_operation)
return await _execute_with_mcp_client(
new_mcp_server_request,
_test_connection_operation,
raw_headers=dict(request.headers),
)
@router.post("/test/tools/list")
async def test_tools_list(
@ -405,4 +426,5 @@ if MCP_AVAILABLE:
_list_tools_operation,
mcp_auth_header=mcp_auth_header,
oauth2_headers=oauth2_headers,
raw_headers=dict(request.headers),
)

6
litellm/proxy/_experimental/mcp_server/server.py
View file

@ -775,6 +775,7 @@ if MCP_AVAILABLE:
mcp_auth_header=server_auth_header,
extra_headers=extra_headers,
add_prefix=add_prefix,
raw_headers=raw_headers,
)
filtered_tools = filter_tools_by_allowed_tools(tools, server)
@ -854,6 +855,7 @@ if MCP_AVAILABLE:
mcp_auth_header=server_auth_header,
extra_headers=extra_headers,
add_prefix=add_prefix,
raw_headers=raw_headers,
)
all_prompts.extend(prompts)
@ -912,6 +914,7 @@ if MCP_AVAILABLE:
mcp_auth_header=server_auth_header,
extra_headers=extra_headers,
add_prefix=add_prefix,
raw_headers=raw_headers,
)
all_resources.extend(resources)
@ -969,6 +972,7 @@ if MCP_AVAILABLE:
mcp_auth_header=server_auth_header,
extra_headers=extra_headers,
add_prefix=add_prefix,
raw_headers=raw_headers,
)
)
all_resource_templates.extend(resource_templates)
@ -1392,6 +1396,7 @@ if MCP_AVAILABLE:
arguments=arguments,
mcp_auth_header=server_auth_header,
extra_headers=extra_headers,
raw_headers=raw_headers,
)
async def mcp_read_resource(
@ -1440,6 +1445,7 @@ if MCP_AVAILABLE:
url=url,
mcp_auth_header=server_auth_header,
extra_headers=extra_headers,
raw_headers=raw_headers,
)
def _get_standard_logging_mcp_tool_call(

7
litellm/proxy/_types.py
View file

@ -33,6 +33,7 @@ from litellm.types.router import RouterErrors, UpdateRouterConfig
from litellm.types.secret_managers.main import KeyManagementSystem
from litellm.types.utils import (
CallTypes,
CostBreakdown,
EmbeddingResponse,
GenericBudgetConfigType,
ImageResponse,
@ -388,6 +389,8 @@ class LiteLLMRoutes(enum.Enum):
litellm_native_routes = [
"/rag/ingest",
"/v1/rag/ingest",
"/rag/query",
"/v1/rag/query",
]
anthropic_routes = [
@ -2149,6 +2152,7 @@ class UserAPIKeyAuth(
user_rpm_limit: Optional[int] = None
user_email: Optional[str] = None
request_route: Optional[str] = None
user: Optional[Any] = None # Expanded user object when expand=user is used
model_config = ConfigDict(arbitrary_types_allowed=True)
@ -2736,6 +2740,9 @@ class SpendLogsMetadata(TypedDict):
litellm_overhead_time_ms: Optional[
float
] # LiteLLM overhead time in milliseconds
cost_breakdown: Optional[
CostBreakdown
] # Detailed cost breakdown (input_cost, output_cost, margin, discount, etc.)
class SpendLogsPayload(TypedDict):

19
litellm/proxy/auth/auth_exception_handler.py
View file

@ -2,7 +2,6 @@
Handles Authentication Errors
"""
import asyncio
from typing import TYPE_CHECKING, Any, Optional, Union
from fastapi import HTTPException, Request, status
@ -90,15 +89,17 @@ class UserAPIKeyAuthExceptionHandler:
api_key=api_key,
request_route=route,
)
asyncio.create_task(
proxy_logging_obj.post_call_failure_hook(
request_data=request_data,
original_exception=e,
user_api_key_dict=user_api_key_dict,
error_type=ProxyErrorTypes.auth_error,
route=route,
)
# Allow callbacks to transform the error response
transformed_exception = await proxy_logging_obj.post_call_failure_hook(
request_data=request_data,
original_exception=e,
user_api_key_dict=user_api_key_dict,
error_type=ProxyErrorTypes.auth_error,
route=route,
)
# Use transformed exception if callback returned one, otherwise use original
if transformed_exception is not None:
e = transformed_exception
if isinstance(e, litellm.BudgetExceededError):
raise ProxyException(

20
litellm/proxy/auth/route_checks.py
View file

@ -293,6 +293,9 @@ class RouteChecks:
if route in LiteLLMRoutes.anthropic_routes.value:
return True
if route in LiteLLMRoutes.google_routes.value:
return True
if RouteChecks.check_route_access(
route=route, allowed_routes=LiteLLMRoutes.mcp_routes.value
@ -315,13 +318,28 @@ class RouteChecks:
):
return True
# Check for Google routes with placeholders like "/v1beta/models/{model_name}:generateContent"
for google_route in LiteLLMRoutes.google_routes.value:
if "{" in google_route:
if RouteChecks._route_matches_pattern(
route=route, pattern=google_route
):
return True
# Check for Anthropic routes with placeholders
for anthropic_route in LiteLLMRoutes.anthropic_routes.value:
if "{" in anthropic_route:
if RouteChecks._route_matches_pattern(
route=route, pattern=anthropic_route
):
return True
if RouteChecks._is_azure_openai_route(route=route):
return True
for _llm_passthrough_route in LiteLLMRoutes.mapped_pass_through_routes.value:
if _llm_passthrough_route in route:
return True
return False
@staticmethod

36
litellm/proxy/common_request_processing.py
View file

@ -179,24 +179,26 @@ async def create_streaming_response(
def _get_cost_breakdown_from_logging_obj(
litellm_logging_obj: Optional[LiteLLMLoggingObj],
) -> Tuple[Optional[float], Optional[float]]:
) -> Tuple[Optional[float], Optional[float], Optional[float], Optional[float]]:
"""
Extract discount information from logging object's cost breakdown.
Extract discount and margin information from logging object's cost breakdown.
Returns:
Tuple of (original_cost, discount_amount)
Tuple of (original_cost, discount_amount, margin_total_amount, margin_percent)
"""
if not litellm_logging_obj or not hasattr(litellm_logging_obj, "cost_breakdown"):
return None, None
return None, None, None, None
cost_breakdown = litellm_logging_obj.cost_breakdown
if not cost_breakdown:
return None, None
return None, None, None, None
original_cost = cost_breakdown.get("original_cost")
discount_amount = cost_breakdown.get("discount_amount")
margin_total_amount = cost_breakdown.get("margin_total_amount")
margin_percent = cost_breakdown.get("margin_percent")
return original_cost, discount_amount
return original_cost, discount_amount, margin_total_amount, margin_percent
class ProxyBaseLLMRequestProcessing:
@ -224,8 +226,8 @@ class ProxyBaseLLMRequestProcessing:
exclude_values = {"", None, "None"}
hidden_params = hidden_params or {}
# Extract discount info from cost_breakdown if available
original_cost, discount_amount = _get_cost_breakdown_from_logging_obj(
# Extract discount and margin info from cost_breakdown if available
original_cost, discount_amount, margin_total_amount, margin_percent = _get_cost_breakdown_from_logging_obj(
litellm_logging_obj=litellm_logging_obj
)
@ -258,6 +260,12 @@ class ProxyBaseLLMRequestProcessing:
"x-litellm-response-cost-discount-amount": (
str(discount_amount) if discount_amount is not None else None
),
"x-litellm-response-cost-margin-amount": (
str(margin_total_amount) if margin_total_amount is not None else None
),
"x-litellm-response-cost-margin-percent": (
str(margin_percent) if margin_percent is not None else None
),
"x-litellm-key-tpm-limit": str(user_api_key_dict.tpm_limit),
"x-litellm-key-rpm-limit": str(user_api_key_dict.rpm_limit),
"x-litellm-key-max-budget": str(user_api_key_dict.max_budget),
@ -786,11 +794,15 @@ class ProxyBaseLLMRequestProcessing:
verbose_proxy_logger.exception(
f"litellm.proxy.proxy_server._handle_llm_api_exception(): Exception occured - {str(e)}"
)
await proxy_logging_obj.post_call_failure_hook(
# Allow callbacks to transform the error response
transformed_exception = await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict,
original_exception=e,
request_data=self.data,
)
# Use transformed exception if callback returned one, otherwise use original
if transformed_exception is not None:
e = transformed_exception
litellm_debug_info = getattr(e, "litellm_debug_info", "")
verbose_proxy_logger.debug(
"\033[1;31mAn error occurred: %s %s\n\n Debug this by setting `--debug`, e.g. `litellm --model gpt-3.5-turbo --debug`",
@ -970,11 +982,15 @@ class ProxyBaseLLMRequestProcessing:
str(e)
)
)
await proxy_logging_obj.post_call_failure_hook(
# Allow callbacks to transform the error response
transformed_exception = await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict,
original_exception=e,
request_data=request_data,
)
# Use transformed exception if callback returned one, otherwise use original
if transformed_exception is not None:
e = transformed_exception
verbose_proxy_logger.debug(
f"\033[1;31mAn error occurred: {e}\n\n Debug this by setting `--debug`, e.g. `litellm --model gpt-3.5-turbo --debug`"
)

13
litellm/proxy/common_utils/http_parsing_utils.py
View file

@ -329,8 +329,8 @@ def populate_request_with_path_params(
request_data: dict, request: Request
) -> dict:
"""
Copy FastAPI path params into the request payload so downstream checks
(e.g. vector store RBAC) see them the same way as body params.
Copy FastAPI path params and query params into the request payload so downstream checks
(e.g. vector store RBAC, organization RBAC) see them the same way as body params.
Since path_params may not be available during dependency injection,
we parse the URL path directly for known patterns.
@ -340,8 +340,15 @@ def populate_request_with_path_params(
request: The FastAPI Request object
Returns:
dict: Updated request_data with path parameters added
dict: Updated request_data with path parameters and query parameters added
"""
# Add query parameters to request_data (for GET requests, etc.)
query_params = _safe_get_request_query_params(request)
if query_params:
for key, value in query_params.items():
# Don't overwrite existing values from request body
request_data.setdefault(key, value)
# Try to get path_params if available (sometimes populated by FastAPI)
path_params = getattr(request, "path_params", None)
if isinstance(path_params, dict) and path_params:

2
litellm/proxy/discovery_endpoints/ui_discovery_endpoints.py
View file

@ -18,9 +18,11 @@ async def get_ui_config():
from litellm.proxy.auth.auth_utils import _has_user_setup_sso
auto_redirect_ui_login_to_sso = os.getenv("AUTO_REDIRECT_UI_LOGIN_TO_SSO", "true").lower() == "true"
admin_ui_disabled = os.getenv("DISABLE_ADMIN_UI", "false").lower() == "true"
return UiDiscoveryEndpoints(
server_root_path=get_server_root_path(),
proxy_base_url=get_proxy_base_url(),
auto_redirect_to_sso=_has_user_setup_sso() and auto_redirect_ui_login_to_sso,
admin_ui_disabled=admin_ui_disabled,
)

26
litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py
View file

@ -449,6 +449,12 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
prepared_request.headers,
)
event_type = (
GuardrailEventHooks.pre_call
if source == "INPUT"
else GuardrailEventHooks.post_call
)
try:
httpx_response = await self.async_handler.post(
url=prepared_request.url,
@ -469,6 +475,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
start_time=start_time.timestamp(),
end_time=datetime.now().timestamp(),
duration=(datetime.now() - start_time).total_seconds(),
event_type=event_type,
)
# Re-raise the exception to maintain existing behavior
raise
@ -486,6 +493,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
start_time=start_time.timestamp(),
end_time=datetime.now().timestamp(),
duration=(datetime.now() - start_time).total_seconds(),
event_type=event_type,
)
#########################################################
if httpx_response.status_code == 200:
@ -605,10 +613,10 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
"""
Only raise exception for "BLOCKED" actions, not for "ANONYMIZED" actions.
If `self.mask_request_content` or `self.mask_response_content` is set to `True`,
If `self.mask_request_content` or `self.mask_response_content` is set to `True`,
then use the output from the guardrail to mask the request or response content.
However, even with masking enabled, content with action="BLOCKED" should still
However, even with masking enabled, content with action="BLOCKED" should still
raise an exception, only content with action="ANONYMIZED" should be masked.
"""
@ -731,9 +739,9 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
#########################################################
########## 1. Make the Bedrock API request ##########
#########################################################
bedrock_guardrail_response: Optional[Union[BedrockGuardrailResponse, str]] = (
None
)
bedrock_guardrail_response: Optional[
Union[BedrockGuardrailResponse, str]
] = None
try:
bedrock_guardrail_response = await self.make_bedrock_api_request(
source="INPUT", messages=filtered_messages, request_data=data
@ -803,9 +811,9 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
#########################################################
########## 1. Make the Bedrock API request ##########
#########################################################
bedrock_guardrail_response: Optional[Union[BedrockGuardrailResponse, str]] = (
None
)
bedrock_guardrail_response: Optional[
Union[BedrockGuardrailResponse, str]
] = None
try:
bedrock_guardrail_response = await self.make_bedrock_api_request(
source="INPUT", messages=filtered_messages, request_data=data

6
litellm/proxy/guardrails/guardrail_hooks/dynamoai/dynamoai.py
View file

@ -97,6 +97,7 @@ class DynamoAIGuardrails(CustomGuardrail):
async def _call_dynamoai_guardrails(
self,
messages: List[Dict[str, Any]],
event_type: GuardrailEventHooks,
text_type: str = "input",
request_data: Optional[dict] = None,
) -> DynamoAIResponse:
@ -157,6 +158,7 @@ class DynamoAIGuardrails(CustomGuardrail):
start_time=start_time.timestamp(),
end_time=end_time.timestamp(),
duration=duration,
event_type=event_type,
)
return response_json
@ -177,6 +179,7 @@ class DynamoAIGuardrails(CustomGuardrail):
start_time=start_time.timestamp(),
end_time=end_time.timestamp(),
duration=duration,
event_type=event_type,
)
raise
@ -332,6 +335,7 @@ class DynamoAIGuardrails(CustomGuardrail):
messages=_messages,
text_type="input",
request_data=data,
event_type=GuardrailEventHooks.pre_call,
)
verbose_proxy_logger.debug(
@ -380,6 +384,7 @@ class DynamoAIGuardrails(CustomGuardrail):
messages=_messages,
text_type="input",
request_data=data,
event_type=GuardrailEventHooks.during_call,
)
verbose_proxy_logger.debug(
@ -460,6 +465,7 @@ class DynamoAIGuardrails(CustomGuardrail):
messages=dynamoai_messages,
text_type="output",
request_data=data,
event_type=GuardrailEventHooks.post_call,
)
verbose_proxy_logger.debug(

13
litellm/proxy/guardrails/guardrail_hooks/ibm_guardrails/ibm_detector.py
View file

@ -108,6 +108,7 @@ class IBMGuardrailDetector(CustomGuardrail):
async def _call_detector_server(
self,
contents: List[str],
event_type: GuardrailEventHooks,
request_data: Optional[dict] = None,
) -> List[List[IBMDetectorDetection]]:
"""
@ -142,7 +143,6 @@ class IBMGuardrailDetector(CustomGuardrail):
)
try:
response = await self.async_handler.post(
url=self.api_url,
json=payload,
@ -172,6 +172,7 @@ class IBMGuardrailDetector(CustomGuardrail):
start_time=start_time.timestamp(),
end_time=end_time.timestamp(),
duration=duration,
event_type=event_type,
)
return response_json
@ -192,6 +193,7 @@ class IBMGuardrailDetector(CustomGuardrail):
start_time=start_time.timestamp(),
end_time=end_time.timestamp(),
duration=duration,
event_type=event_type,
)
raise
@ -199,6 +201,7 @@ class IBMGuardrailDetector(CustomGuardrail):
async def _call_orchestrator(
self,
content: str,
event_type: GuardrailEventHooks,
request_data: Optional[dict] = None,
) -> List[IBMDetectorDetection]:
"""
@ -258,6 +261,7 @@ class IBMGuardrailDetector(CustomGuardrail):
start_time=start_time.timestamp(),
end_time=end_time.timestamp(),
duration=duration,
event_type=event_type,
)
return response_json.get("detections", [])
@ -278,6 +282,7 @@ class IBMGuardrailDetector(CustomGuardrail):
start_time=start_time.timestamp(),
end_time=end_time.timestamp(),
duration=duration,
event_type=event_type,
)
raise
@ -472,6 +477,7 @@ class IBMGuardrailDetector(CustomGuardrail):
result = await self._call_detector_server(
contents=contents_to_check,
request_data=data,
event_type=GuardrailEventHooks.pre_call,
)
verbose_proxy_logger.debug(
@ -500,6 +506,7 @@ class IBMGuardrailDetector(CustomGuardrail):
orchestrator_result = await self._call_orchestrator(
content=content,
request_data=data,
event_type=GuardrailEventHooks.pre_call,
)
verbose_proxy_logger.debug(
@ -557,6 +564,7 @@ class IBMGuardrailDetector(CustomGuardrail):
result = await self._call_detector_server(
contents=contents_to_check,
request_data=data,
event_type=GuardrailEventHooks.during_call,
)
verbose_proxy_logger.debug(
@ -585,6 +593,7 @@ class IBMGuardrailDetector(CustomGuardrail):
orchestrator_result = await self._call_orchestrator(
content=content,
request_data=data,
event_type=GuardrailEventHooks.during_call,
)
verbose_proxy_logger.debug(
@ -673,6 +682,7 @@ class IBMGuardrailDetector(CustomGuardrail):
result = await self._call_detector_server(
contents=contents_to_check,
request_data=data,
event_type=GuardrailEventHooks.post_call,
)
verbose_proxy_logger.debug(
@ -702,6 +712,7 @@ class IBMGuardrailDetector(CustomGuardrail):
orchestrator_result = await self._call_orchestrator(
content=content,
request_data=data,
event_type=GuardrailEventHooks.post_call,
)
verbose_proxy_logger.debug(

6
litellm/proxy/guardrails/guardrail_hooks/javelin/javelin.py
View file

@ -83,6 +83,7 @@ class JavelinGuardrail(CustomGuardrail):
async def call_javelin_guard(
self,
request: JavelinGuardRequest,
event_type: GuardrailEventHooks,
) -> JavelinGuardResponse:
"""
Call the Javelin guard API.
@ -158,6 +159,7 @@ class JavelinGuardrail(CustomGuardrail):
start_time=start_time.timestamp(),
end_time=datetime.now().timestamp(),
duration=(datetime.now() - start_time).total_seconds(),
event_type=event_type,
)
async def async_pre_call_hook(
@ -208,7 +210,9 @@ class JavelinGuardrail(CustomGuardrail):
config=self.config if self.config else {},
)
javelin_response = await self.call_javelin_guard(request=javelin_guard_request)
javelin_response = await self.call_javelin_guard(
request=javelin_guard_request, event_type=GuardrailEventHooks.pre_call
)
assessments = javelin_response.get("assessments", [])
reject_prompt = ""

Some files were not shown because too many files have changed in this diff Show more