scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-07 08:26:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 36

Merge branch 'main' into main

Browse source
This commit is contained in:
Krish Dholakia 2025-08-27 22:13:29 -07:00 • committed by GitHub
commit c692f19346
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
504 changed files with 25781 additions and 7795 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

112
.circleci/config.yml
View file

@ -83,8 +83,8 @@ jobs:
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install pyarrow
pip install "boto3==1.34.34"
pip install "aioboto3==12.3.0"
pip install "boto3==1.36.0"
pip install "aioboto3==13.4.0"
pip install langchain
pip install lunary==0.2.5
pip install "azure-identity==1.16.1"
@ -95,7 +95,7 @@ jobs:
pip install opentelemetry-api==1.25.0
pip install opentelemetry-sdk==1.25.0
pip install opentelemetry-exporter-otlp==1.25.0
pip install openai==1.99.5
pip install openai==1.100.1
pip install prisma==0.11.0
pip install "detect_secrets==1.5.0"
pip install "httpx==0.24.1"
@ -208,8 +208,8 @@ jobs:
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install pyarrow
pip install "boto3==1.34.34"
pip install "aioboto3==12.3.0"
pip install "boto3==1.36.0"
pip install "aioboto3==13.4.0"
pip install langchain
pip install lunary==0.2.5
pip install "azure-identity==1.16.1"
@ -220,7 +220,7 @@ jobs:
pip install opentelemetry-api==1.25.0
pip install opentelemetry-sdk==1.25.0
pip install opentelemetry-exporter-otlp==1.25.0
pip install openai==1.99.5
pip install openai==1.100.1
pip install prisma==0.11.0
pip install "detect_secrets==1.5.0"
pip install "httpx==0.24.1"
@ -315,8 +315,8 @@ jobs:
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install pyarrow
pip install "boto3==1.34.34"
pip install "aioboto3==12.3.0"
pip install "boto3==1.36.0"
pip install "aioboto3==13.4.0"
pip install langchain
pip install lunary==0.2.5
pip install "azure-identity==1.16.1"
@ -327,7 +327,7 @@ jobs:
pip install opentelemetry-api==1.25.0
pip install opentelemetry-sdk==1.25.0
pip install opentelemetry-exporter-otlp==1.25.0
pip install openai==1.99.5
pip install openai==1.100.1
pip install prisma==0.11.0
pip install "detect_secrets==1.5.0"
pip install "httpx==0.24.1"
@ -439,6 +439,7 @@ jobs:
paths:
- auth_ui_unit_tests_coverage.xml
- auth_ui_unit_tests_coverage
litellm_router_testing: # Runs all tests with the "router" keyword
docker:
- image: cimg/python:3.11
@ -469,7 +470,55 @@ jobs:
command: |
pwd
ls
python -m pytest tests/local_testing tests/router_unit_tests --cov=litellm --cov-report=xml -vv -k "router" -x -v --junitxml=test-results/junit.xml --durations=5
python -m pytest tests/local_testing --cov=litellm --cov-report=xml -vv -k "router" -x -v --junitxml=test-results/junit.xml --durations=5
no_output_timeout: 120m
- run:
name: Rename the coverage files
command: |
mv coverage.xml litellm_router_coverage.xml
mv .coverage litellm_router_coverage
# Store test results
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- litellm_router_coverage.xml
- litellm_router_coverage
litellm_router_unit_testing: # Runs all tests with the "router" keyword
docker:
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
steps:
- checkout
- setup_google_dns
- run:
name: Install Dependencies
command: |
python -m pip install --upgrade pip
python -m pip install -r requirements.txt
pip install "pytest==7.3.1"
pip install "respx==0.22.0"
pip install "pytest-cov==5.0.0"
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
pip install semantic_router --no-deps
pip install aurelio_sdk --no-deps
pip install "pytest-xdist==3.6.1"
# Run pytest and generate JUnit XML report
- setup_litellm_enterprise_pip
- run:
name: Run tests
command: |
pwd
ls
python -m pytest -vv tests/router_unit_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
no_output_timeout: 120m
- run:
name: Rename the coverage files
@ -590,8 +639,8 @@ jobs:
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install pyarrow
pip install "boto3==1.34.34"
pip install "aioboto3==12.3.0"
pip install "boto3==1.36.0"
pip install "aioboto3==13.4.0"
pip install langchain
pip install lunary==0.2.5
pip install "azure-identity==1.16.1"
@ -602,7 +651,7 @@ jobs:
pip install opentelemetry-api==1.25.0
pip install opentelemetry-sdk==1.25.0
pip install opentelemetry-exporter-otlp==1.25.0
pip install openai==1.99.5
pip install openai==1.100.1
pip install prisma==0.11.0
pip install "detect_secrets==1.5.0"
pip install "httpx==0.24.1"
@ -816,7 +865,7 @@ jobs:
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.22.0"
pip install "pydantic==2.10.2"
pip install "boto3==1.34.34"
pip install "boto3==1.36.0"
# Run pytest and generate JUnit XML report
- run:
name: Run tests
@ -1238,10 +1287,11 @@ jobs:
pip install aiohttp
pip install openai
pip install click
pip install "boto3==1.34.34"
pip install "boto3==1.36.0"
pip install jinja2
pip install "tokenizers==0.20.0"
pip install "uvloop==0.21.0"
pip install "fastuuid==0.12.0"
pip install jsonschema
- setup_litellm_enterprise_pip
- run:
@ -1507,8 +1557,8 @@ jobs:
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install pyarrow
pip install "boto3==1.34.34"
pip install "aioboto3==12.3.0"
pip install "boto3==1.36.0"
pip install "aioboto3==13.4.0"
pip install langchain
pip install "langfuse>=2.0.0"
pip install "logfire==0.29.0"
@ -1522,7 +1572,7 @@ jobs:
pip install "aiodynamo==23.10.1"
pip install "asyncio==3.4.3"
pip install "PyGithub==1.59.1"
pip install "openai==1.99.5"
pip install "openai==1.100.1"
- run:
name: Install dockerize
command: |
@ -1663,8 +1713,8 @@ jobs:
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install pyarrow
pip install "boto3==1.34.34"
pip install "aioboto3==12.3.0"
pip install "boto3==1.36.0"
pip install "aioboto3==13.4.0"
pip install langchain
pip install "langchain_mcp_adapters==0.0.5"
pip install "langfuse>=2.0.0"
@ -1679,7 +1729,7 @@ jobs:
pip install "aiodynamo==23.10.1"
pip install "asyncio==3.4.3"
pip install "PyGithub==1.59.1"
pip install "openai==1.99.5"
pip install "openai==1.100.1"
# Run pytest and generate JUnit XML report
- run:
name: Install dockerize
@ -1804,8 +1854,8 @@ jobs:
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install pyarrow
pip install "boto3==1.34.34"
pip install "aioboto3==12.3.0"
pip install "boto3==1.36.0"
pip install "aioboto3==13.4.0"
pip install langchain
pip install "langfuse>=2.0.0"
pip install "logfire==0.29.0"
@ -1819,7 +1869,7 @@ jobs:
pip install "aiodynamo==23.10.1"
pip install "asyncio==3.4.3"
pip install "PyGithub==1.59.1"
pip install "openai==1.99.5"
pip install "openai==1.100.1"
- run:
name: Install dockerize
command: |
@ -2399,14 +2449,14 @@ jobs:
pip install "pytest-asyncio==0.21.1"
pip install "google-cloud-aiplatform==1.43.0"
pip install aiohttp
pip install "openai==1.99.5"
pip install "openai==1.100.1"
pip install "assemblyai==0.37.0"
python -m pip install --upgrade pip
pip install "pydantic==2.10.2"
pip install "pytest==7.3.1"
pip install "pytest-mock==3.12.0"
pip install "pytest-asyncio==0.21.1"
pip install "boto3==1.34.34"
pip install "boto3==1.36.0"
pip install mypy
pip install pyarrow
pip install numpydoc
@ -2790,7 +2840,7 @@ jobs:
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
pip install aiohttp
pip install "openai==1.99.5"
pip install "openai==1.100.1"
python -m pip install --upgrade pip
pip install "pydantic==2.10.2"
pip install "pytest==7.3.1"
@ -2982,6 +3032,12 @@ workflows:
only:
- main
- /litellm_.*/
- litellm_router_unit_testing:
filters:
branches:
only:
- main
- /litellm_.*/
- check_code_and_doc_quality:
filters:
branches:
@ -3128,6 +3184,7 @@ workflows:
- image_gen_testing
- logging_testing
- litellm_router_testing
- litellm_router_unit_testing
- caching_unit_tests
- litellm_proxy_unit_testing
- litellm_security_tests
@ -3187,6 +3244,7 @@ workflows:
- image_gen_testing
- logging_testing
- litellm_router_testing
- litellm_router_unit_testing
- caching_unit_tests
- langfuse_logging_unit_tests
- litellm_assistants_api_testing

5
.circleci/requirements.txt
View file

@ -1,5 +1,5 @@
# used by CI/CD testing
openai==1.99.5
openai==1.100.1
python-dotenv
tiktoken
importlib_metadata
@ -14,4 +14,5 @@ google-cloud-iam==2.19.1
fastapi-sso==0.16.0
uvloop==0.21.0
mcp==1.10.1 # for MCP server
semantic_router==0.1.10 # for auto-routing with litellm
semantic_router==0.1.10 # for auto-routing with litellm
fastuuid==0.12.0

9
.github/workflows/test-linting.yml vendored
View file

@ -22,11 +22,8 @@ jobs:
- name: Install dependencies
run: |
pip install openai==1.99.5
poetry install --with dev
pip install openai==1.99.5
poetry run pip install openai==1.100.1
- name: Run Black formatting
run: |
@ -40,6 +37,10 @@ jobs:
poetry run ruff check .
cd ..
- name: Print OpenAI version
run: |
poetry run python -c "import openai; print(f'OpenAI version: {openai.__version__}')"
- name: Run MyPy type checking
run: |
cd litellm

2
.github/workflows/test-litellm.yml vendored
View file

@ -7,7 +7,7 @@ on:
jobs:
test:
runs-on: ubuntu-latest
timeout-minutes: 20
timeout-minutes: 25
steps:
- uses: actions/checkout@v4

4
.gitignore vendored
View file

@ -86,6 +86,7 @@ litellm/proxy/db/migrations/0_init/migration.sql
litellm/proxy/db/migrations/*
litellm/proxy/migrations/*config.yaml
litellm/proxy/migrations/*
litellm/proxy/to_delete_loadtest_work/*
config.yaml
tests/litellm/litellm_core_utils/llm_cost_calc/log.txt
tests/test_custom_dir/*
@ -93,4 +94,5 @@ test.py
litellm_config.yaml
.cursor
.vscode/launch.json
.vscode/launch.json
litellm/proxy/to_delete_loadtest_work/*

4
Dockerfile
View file

@ -65,8 +65,8 @@ COPY --from=builder /wheels/ /wheels/
# Install the built wheel using pip; again using a wildcard if it's the only file
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
# Install semantic_router without dependencies
RUN pip install semantic_router --no-deps
# Install semantic_router and aurelio-sdk using script
RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
# Generate prisma client
RUN prisma generate

10
README.md
View file

@ -374,6 +374,8 @@ We welcome contributions to LiteLLM! Whether you're fixing bugs, adding features
## Quick Start for Contributors
This requires poetry to be installed.
```bash
git clone https://github.com/BerriAI/litellm.git
cd litellm
@ -381,6 +383,7 @@ make install-dev # Install development dependencies
make format # Format your code
make lint # Run all linting checks
make test-unit # Run unit tests
make format-check # Check formatting only
```
For detailed contributing guidelines, see [CONTRIBUTING.md](CONTRIBUTING.md).
@ -396,11 +399,6 @@ Our automated checks include:
- **Circular import detection**
- **Import safety checks**
Run all checks locally:
```bash
make lint # Run all linting (matches CI)
make format-check # Check formatting only
```
All these checks must pass before your PR can be merged.
@ -442,7 +440,7 @@ All these checks must pass before your PR can be merged.
1. (In root) create virtual environment `python -m venv .venv`
2. Activate virtual environment `source .venv/bin/activate`
3. Install dependencies `pip install -e ".[all]"`
4. Start proxy backend `uvicorn litellm.proxy.proxy_server:app --host localhost --port 4000 --reload`
4. Start proxy backend `python3 /path/to/litellm/proxy_cli.py`
### Frontend
1. Navigate to `ui/litellm-dashboard`

213
cookbook/liteLLM_Baseten.ipynb vendored
View file

@ -6,19 +6,21 @@
"id": "gZx-wHJapG5w"
},
"source": [
"# Use liteLLM to call Falcon, Wizard, MPT 7B using OpenAI chatGPT Input/output\n",
"# LiteLLM with Baseten Model APIs\n",
"\n",
"* Falcon 7B: https://app.baseten.co/explore/falcon_7b\n",
"* Wizard LM: https://app.baseten.co/explore/wizardlm\n",
"* MPT 7B Base: https://app.baseten.co/explore/mpt_7b_instruct\n",
"This notebook demonstrates how to use LiteLLM with Baseten's Model APIs instead of dedicated deployments.\n",
"\n",
"\n",
"## Call all baseten llm models using OpenAI chatGPT Input/Output using liteLLM\n",
"Example call\n",
"## Example Usage\n",
"```python\n",
"model = \"q841o8w\" # baseten model version ID\n",
"response = completion(model=model, messages=messages, custom_llm_provider=\"baseten\")\n",
"```"
"response = completion(\n",
" model=\"baseten/openai/gpt-oss-120b\",\n",
" messages=[{\"role\": \"user\", \"content\": \"Hello!\"}],\n",
" max_tokens=1000,\n",
" temperature=0.7\n",
")\n",
"```\n",
"\n",
"## Setup"
]
},
{
@ -29,20 +31,25 @@
},
"outputs": [],
"source": [
"!pip install litellm==0.1.399\n",
"!pip install baseten urllib3"
"%pip install litellm"
]
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": null,
"metadata": {
"id": "VEukLhDzo4vw"
},
"outputs": [],
"source": [
"import os\n",
"from litellm import completion"
"from litellm import completion\n",
"\n",
"# Set your Baseten API key\n",
"os.environ['BASETEN_API_KEY'] = \"\" #@param {type:\"string\"}\n",
"\n",
"# Test message\n",
"messages = [{\"role\": \"user\", \"content\": \"What is AGI?\"}]"
]
},
{
@ -51,19 +58,31 @@
"id": "4STYM2OHFNlc"
},
"source": [
"## Setup"
"## Example 1: Basic Completion\n",
"\n",
"Simple completion with the GPT-OSS 120B model"
]
},
{
"cell_type": "code",
"execution_count": 21,
"execution_count": null,
"metadata": {
"id": "DorpLxw1FHbC"
},
"outputs": [],
"source": [
"os.environ['BASETEN_API_KEY'] = \"\" #@param\n",
"messages = [{ \"content\": \"what does Baseten do? \",\"role\": \"user\"}]"
"print(\"=== Basic Completion ===\")\n",
"response = completion(\n",
" model=\"baseten/openai/gpt-oss-120b\",\n",
" messages=messages,\n",
" max_tokens=1000,\n",
" temperature=0.7,\n",
" top_p=0.9,\n",
" presence_penalty=0.1,\n",
" frequency_penalty=0.1,\n",
")\n",
"print(f\"Response: {response.choices[0].message.content}\")\n",
"print(f\"Usage: {response.usage}\")"
]
},
{
@ -72,13 +91,14 @@
"id": "syF3dTdKFSQQ"
},
"source": [
"## Calling Falcon 7B: https://app.baseten.co/explore/falcon_7b\n",
"### Pass Your Baseten model `Version ID` as `model`"
"## Example 2: Streaming Completion\n",
"\n",
"Streaming completion with usage statistics"
]
},
{
"cell_type": "code",
"execution_count": 18,
"execution_count": null,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
@ -86,137 +106,26 @@
"id": "rPgSoMlsojz0",
"outputId": "81d6dc7b-1681-4ae4-e4c8-5684eb1bd050"
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[32mINFO\u001b[0m API key set.\n",
"INFO:baseten:API key set.\n"
]
},
{
"data": {
"text/plain": [
"{'choices': [{'finish_reason': 'stop',\n",
" 'index': 0,\n",
" 'message': {'role': 'assistant',\n",
" 'content': \"what does Baseten do? \\nI'm sorry, I cannot provide a specific answer as\"}}],\n",
" 'created': 1692135883.699066,\n",
" 'model': 'qvv0xeq'}"
]
},
"execution_count": 18,
"metadata": {},
"output_type": "execute_result"
}
],
"outputs": [],
"source": [
"model = \"qvv0xeq\"\n",
"response = completion(model=model, messages=messages, custom_llm_provider=\"baseten\")\n",
"response"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "7n21UroEGCGa"
},
"source": [
"## Calling Wizard LM https://app.baseten.co/explore/wizardlm\n",
"### Pass Your Baseten model `Version ID` as `model`"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "uLVWFH899lAF",
"outputId": "61c2bc74-673b-413e-bb40-179cf408523d"
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[32mINFO\u001b[0m API key set.\n",
"INFO:baseten:API key set.\n"
]
},
{
"data": {
"text/plain": [
"{'choices': [{'finish_reason': 'stop',\n",
" 'index': 0,\n",
" 'message': {'role': 'assistant',\n",
" 'content': 'As an AI language model, I do not have personal beliefs or practices, but based on the information available online, Baseten is a popular name for a traditional Ethiopian dish made with injera, a spongy flatbread, and wat, a spicy stew made with meat or vegetables. It is typically served for breakfast or dinner and is a staple in Ethiopian cuisine. The name Baseten is also used to refer to a traditional Ethiopian coffee ceremony, where coffee is brewed and served in a special ceremony with music and food.'}}],\n",
" 'created': 1692135900.2806294,\n",
" 'model': 'q841o8w'}"
]
},
"execution_count": 19,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model = \"q841o8w\"\n",
"response = completion(model=model, messages=messages, custom_llm_provider=\"baseten\")\n",
"response"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "6-TFwmPAGPXq"
},
"source": [
"## Calling mosaicml/mpt-7b https://app.baseten.co/explore/mpt_7b_instruct\n",
"### Pass Your Baseten model `Version ID` as `model`"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "gbeYZOrUE_Bp",
"outputId": "838d86ea-2143-4cb3-bc80-2acc2346c37a"
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[32mINFO\u001b[0m API key set.\n",
"INFO:baseten:API key set.\n"
]
},
{
"data": {
"text/plain": [
"{'choices': [{'finish_reason': 'stop',\n",
" 'index': 0,\n",
" 'message': {'role': 'assistant',\n",
" 'content': \"\\n===================\\n\\nIt's a tool to build a local version of a game on your own machine to host\\non your website.\\n\\nIt's used to make game demos and show them on Twitter, Tumblr, and Facebook.\\n\\n\\n\\n## What's built\\n\\n- A directory of all your game directories, named with a version name and build number, with images linked to.\\n- Includes HTML to include in another site.\\n- Includes images for your icons and\"}}],\n",
" 'created': 1692135914.7472186,\n",
" 'model': '31dxrj3'}"
]
},
"execution_count": 20,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model = \"31dxrj3\"\n",
"response = completion(model=model, messages=messages, custom_llm_provider=\"baseten\")\n",
"response"
"print(\"=== Streaming Completion ===\")\n",
"response = completion(\n",
" model=\"baseten/openai/gpt-oss-120b\",\n",
" messages=[{\"role\": \"user\", \"content\": \"Write a short poem about AI\"}],\n",
" stream=True,\n",
" max_tokens=500,\n",
" temperature=0.8,\n",
" stream_options={\n",
" \"include_usage\": True,\n",
" \"continuous_usage_stats\": True\n",
" },\n",
")\n",
"\n",
"print(\"Streaming response:\")\n",
"for chunk in response:\n",
" if chunk.choices and chunk.choices[0].delta.content:\n",
" print(chunk.choices[0].delta.content, end=\"\", flush=True)\n",
"print(\"\\n\")"
]
}
],
@ -234,4 +143,4 @@
},
"nbformat": 4,
"nbformat_minor": 0
}
}

2
deploy/charts/litellm-helm/Chart.yaml
View file

@ -18,7 +18,7 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.4.4
version: 0.4.5
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to

4
deploy/charts/litellm-helm/README.md
View file

@ -24,7 +24,7 @@ If `db.useStackgresOperator` is used (not yet implemented):
| `replicaCount` | The number of LiteLLM Proxy pods to be deployed | `1` |
| `masterkeySecretName` | The name of the Kubernetes Secret that contains the Master API Key for LiteLLM. If not specified, use the generated secret name. | N/A |
| `masterkeySecretKey` | The key within the Kubernetes Secret that contains the Master API Key for LiteLLM. If not specified, use `masterkey` as the key. | N/A |
| `masterkey` | The Master API Key for LiteLLM. If not specified, a random key is generated. | N/A |
| `masterkey` | The Master API Key for LiteLLM. If not specified, a random key in the `sk-...` format is generated. | N/A |
| `environmentSecrets` | An optional array of Secret object names. The keys and values in these secrets will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` |
| `environmentConfigMaps` | An optional array of ConfigMap object names. The keys and values in these configmaps will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` |
| `image.repository` | LiteLLM Proxy image repository | `ghcr.io/berriai/litellm` |
@ -135,7 +135,7 @@ service, the **Proxy Endpoint** should be set to `http://<RELEASE>-litellm:4000`
The **Proxy Key** is the value specified for `masterkey` or, if a `masterkey`
was not provided to the helm command line, the `masterkey` is a randomly
generated string stored in the `<RELEASE>-litellm-masterkey` Kubernetes Secret.
generated string in the `sk-...` format stored in the `<RELEASE>-litellm-masterkey` Kubernetes Secret.
```bash
kubectl -n litellm get secret <RELEASE>-litellm-masterkey -o jsonpath="{.data.masterkey}"

7
deploy/charts/litellm-helm/templates/deployment.yaml
View file

@ -71,7 +71,14 @@ spec:
name: {{ .Values.db.secret.name }}
key: {{ .Values.db.secret.passwordKey }}
- name: DATABASE_HOST
{{- if .Values.db.secret.endpointKey }}
valueFrom:
secretKeyRef:
name: {{ .Values.db.secret.name }}
key: {{ .Values.db.secret.endpointKey }}
{{- else }}
value: {{ .Values.db.endpoint }}
{{- end }}
- name: DATABASE_NAME
value: {{ .Values.db.database }}
- name: DATABASE_URL

11
deploy/charts/litellm-helm/templates/migrations-job.yaml
View file

@ -49,7 +49,14 @@ spec:
name: {{ .Values.db.secret.name }}
key: {{ .Values.db.secret.passwordKey }}
- name: DATABASE_HOST
{{- if .Values.db.secret.endpointKey }}
valueFrom:
secretKeyRef:
name: {{ .Values.db.secret.name }}
key: {{ .Values.db.secret.endpointKey }}
{{- else }}
value: {{ .Values.db.endpoint }}
{{- end }}
- name: DATABASE_NAME
value: {{ .Values.db.database }}
- name: DATABASE_URL
@ -73,6 +80,10 @@ spec:
volumeMounts:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.migrationJob.resources }}
resources:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.migrationJob.extraContainers }}
{{- toYaml . | nindent 8 }}
{{- end }}

2
deploy/charts/litellm-helm/templates/secret-masterkey.yaml
View file

@ -1,5 +1,5 @@
{{- if not .Values.masterkeySecretName }}
{{ $masterkey := (.Values.masterkey | default (randAlphaNum 17)) }}
{{ $masterkey := (.Values.masterkey | default (printf "sk-%s" (randAlphaNum 18))) }}
apiVersion: v1
kind: Secret
metadata:

8
deploy/charts/litellm-helm/tests/masterkey-secret_tests.yaml
View file

@ -2,13 +2,19 @@ suite: test masterkey secret
templates:
- secret-masterkey.yaml
tests:
- it: should create a secret if masterkeySecretName is not set
- it: should create a secret if masterkeySecretName is not set. should start with sk-xxxx (base64 encoded as c2st*)
template: secret-masterkey.yaml
set:
masterkeySecretName: ""
asserts:
- isKind:
of: Secret
- matchRegex:
path: data.masterkey
pattern: ^c2st
# Note: The masterkey is generated as "sk-<18-random-chars>" in plain text,
# but stored as base64 encoded in Kubernetes secret (requirement).
# "sk-" base64 encodes to "c2st", so we check for "^c2st" pattern.
- it: should not create a secret if masterkeySecretName is set
template: secret-masterkey.yaml
set:

6
deploy/charts/litellm-helm/values.yaml
View file

@ -161,6 +161,8 @@ db:
name: postgres
usernameKey: username
passwordKey: password
# Optional: when set, DATABASE_HOST will be sourced from this secret key instead of db.endpoint
endpointKey: ""
# Use the Stackgres Helm chart to deploy an instance of a Stackgres cluster.
# The Stackgres Operator must already be installed within the target
@ -206,6 +208,10 @@ migrationJob:
disableSchemaUpdate: false # Skip schema migrations for specific environments. When True, the job will exit with code 0.
annotations: {}
ttlSecondsAfterFinished: 120
resources: {}
# requests:
# cpu: 100m
# memory: 100Mi
extraContainers: []
# Hook configuration

4
docker/Dockerfile.database
View file

@ -57,8 +57,8 @@ COPY --from=builder /wheels/ /wheels/
# Install the built wheel using pip; again using a wildcard if it's the only file
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
# Install semantic_router without dependencies
RUN pip install semantic_router --no-deps
# Install semantic_router and aurelio-sdk using script
RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
# ensure pyjwt is used, not jwt
RUN pip uninstall jwt -y

11
docker/Dockerfile.non_root
View file

@ -33,11 +33,12 @@ WORKDIR /app
# Install runtime dependencies
USER root
RUN apk upgrade --no-cache && \
apk add --no-cache bash libstdc++ ca-certificates openssl
apk add --no-cache bash libstdc++ ca-certificates openssl supervisor
# Copy only necessary artifacts from builder stage for runtime
COPY . .
COPY --from=builder /app/docker/entrypoint.sh /app/docker/prod_entrypoint.sh /app/docker/
COPY --from=builder /app/docker/supervisord.conf /etc/supervisord.conf
COPY --from=builder /app/schema.prisma /app/schema.prisma
COPY --from=builder /app/dist/*.whl .
COPY --from=builder /wheels/ /wheels/
@ -47,8 +48,8 @@ RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ \
&& rm -f *.whl \
&& rm -rf /wheels
# Install semantic_router without dependencies
RUN pip install semantic_router --no-deps
# Install semantic_router and aurelio-sdk using script
RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
# Ensure correct JWT library is used (pyjwt not jwt)
RUN pip uninstall jwt -y && \
@ -70,7 +71,9 @@ RUN mkdir -p /nonexistent /.npm && \
chown -R nobody:nogroup /app && \
chown -R nobody:nogroup /nonexistent /.npm && \
PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \
chown -R nobody:nogroup $PRISMA_PATH
chown -R nobody:nogroup $PRISMA_PATH && \
LITELLM_PKG_MIGRATIONS_PATH="$(python -c 'import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))' 2>/dev/null || echo '')/migrations" && \
[ -n "$LITELLM_PKG_MIGRATIONS_PATH" ] && chown -R nobody:nogroup $LITELLM_PKG_MIGRATIONS_PATH
# --- OpenShift Compatibility: Apply Red Hat recommended pattern ---
# Get paths for directories that need write access at runtime

1
docker/build_from_pip/requirements.txt
View file

@ -2,4 +2,5 @@ litellm[proxy]==1.67.4.dev1 # Specify the litellm version you want to use
prometheus_client
langfuse
prisma
openai==1.99.9
ddtrace==2.19.0 # for advanced DD tracing / profiling

3
docker/install_auto_router.sh Executable file
View file

@ -0,0 +1,3 @@
#!/bin/bash
pip install semantic_router==0.1.11 --no-deps
pip install aurelio-sdk==0.0.19

66
docs/my-website/docs/completion/document_understanding.md
View file

@ -10,6 +10,7 @@ Works for:
- Bedrock Models
- Anthropic API Models
- OpenAI API Models
- Mistral (Only using file ID of already uploaded file, similar to OpenAI file_id input)
## Quick Start
@ -279,6 +280,71 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
</Tabs>
## Mistral Example
Here is a sample payload for using the Mistral model for document understanding:
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm.utils import completion
# pdf file_id received from files endpoint
file_id = "fa778e5e-46ec-4562-8418-36623fe25a71"
# model
model = "mistral/mistral-large-latest"
file_content = [
{"type": "text", "text": "What's this file about?"},
{
"type": "file",
"file": {
"file_id": file_id,
}
},
]
response = completion(
model=model,
messages=[{"role": "user", "content": file_content}],
)
assert response is not None
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```bash
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-d '{
"model": "mistral/mistral-large-latest",
"messages": [
{
"role": "user",
"content": [
{
"type": "text",
"text": "What is the content of the file?"
},
{
"type": "file",
"file": {
"file_id": "fa778e5e-46ec-4562-8418-36623fe25a71"
}
}
]
}
]
}
```
</TabItem>
</Tabs>
## Checking if a model supports pdf input
<Tabs>

232
docs/my-website/docs/completion/image_generation_chat.md Normal file
View file

@ -0,0 +1,232 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Image Generation in Chat Completions, Responses API
This guide covers how to generate images when using the `chat/completions`. Note - if you want this on Responses API please file a Feature Request [here](https://github.com/BerriAI/litellm/issues/new).
:::info
Requires LiteLLM v1.76.1+
:::
Supported Providers:
- Google AI Studio (`gemini`)
- Vertex AI (`vertex_ai/`)
LiteLLM will standardize the `image` response in the assistant message for models that support image generation during chat completions.
```python title="Example response from litellm"
"message": {
...
"content": "Here's the image you requested:",
"image": {
"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...",
"detail": "auto"
}
}
```
## Quick Start
<Tabs>
<TabItem value="sdk" label="SDK">
```python showLineNumbers title="Image generation with chat completion"
from litellm import completion
import os
os.environ["GEMINI_API_KEY"] = "your-api-key"
response = completion(
model="gemini/gemini-2.5-flash-image-preview",
messages=[
{"role": "user", "content": "Generate an image of a banana wearing a costume that says LiteLLM"}
],
)
print(response.choices[0].message.content) # Text response
print(response.choices[0].message.image) # Image data
```
</TabItem>
<TabItem value="proxy" label="PROXY">
1. Setup config.yaml
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: gemini-image-gen
litellm_params:
model: gemini/gemini-2.5-flash-image-preview
api_key: os.environ/GEMINI_API_KEY
```
2. Run proxy server
```bash showLineNumbers title="Start the proxy"
litellm --config config.yaml
# RUNNING on http://0.0.0.0:4000
```
3. Test it!
```bash showLineNumbers title="Make request"
curl http://0.0.0.0:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $LITELLM_KEY" \
-d '{
"model": "gemini-image-gen",
"messages": [
{
"role": "user",
"content": "Generate an image of a banana wearing a costume that says LiteLLM"
}
]
}'
```
</TabItem>
</Tabs>
**Expected Response**
```bash
{
"id": "chatcmpl-3b66124d79a708e10c603496b363574c",
"choices": [
{
"finish_reason": "stop",
"index": 0,
"message": {
"content": "Here's the image you requested:",
"role": "assistant",
"image": {
"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...",
"detail": "auto"
}
}
}
],
"created": 1723323084,
"model": "gemini/gemini-2.5-flash-image-preview",
"object": "chat.completion",
"usage": {
"completion_tokens": 12,
"prompt_tokens": 16,
"total_tokens": 28
}
}
```
## Streaming Support
<Tabs>
<TabItem value="sdk" label="SDK">
```python showLineNumbers title="Streaming image generation"
from litellm import completion
import os
os.environ["GEMINI_API_KEY"] = "your-api-key"
response = completion(
model="gemini/gemini-2.5-flash-image-preview",
messages=[
{"role": "user", "content": "Generate an image of a banana wearing a costume that says LiteLLM"}
],
stream=True,
)
for chunk in response:
if hasattr(chunk.choices[0].delta, "image") and chunk.choices[0].delta.image is not None:
print("Generated image:", chunk.choices[0].delta.image["url"])
break
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```bash showLineNumbers title="Streaming request"
curl http://0.0.0.0:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $LITELLM_KEY" \
-d '{
"model": "gemini-image-gen",
"messages": [
{
"role": "user",
"content": "Generate an image of a banana wearing a costume that says LiteLLM"
}
],
"stream": true
}'
```
</TabItem>
</Tabs>
**Expected Streaming Response**
```bash
data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1723323084,"model":"gemini/gemini-2.5-flash-image-preview","choices":[{"index":0,"delta":{"role":"assistant"},"finish_reason":null}]}
data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1723323084,"model":"gemini/gemini-2.5-flash-image-preview","choices":[{"index":0,"delta":{"content":"Here's the image you requested:"},"finish_reason":null}]}
data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1723323084,"model":"gemini/gemini-2.5-flash-image-preview","choices":[{"index":0,"delta":{"image":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...","detail":"auto"}},"finish_reason":null}]}
data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1723323084,"model":"gemini/gemini-2.5-flash-image-preview","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}
data: [DONE]
```
## Async Support
```python showLineNumbers title="Async image generation"
from litellm import acompletion
import asyncio
import os
os.environ["GEMINI_API_KEY"] = "your-api-key"
async def generate_image():
response = await acompletion(
model="gemini/gemini-2.5-flash-image-preview",
messages=[
{"role": "user", "content": "Generate an image of a banana wearing a costume that says LiteLLM"}
],
)
print(response.choices[0].message.content) # Text response
print(response.choices[0].message.image) # Image data
return response
# Run the async function
asyncio.run(generate_image())
```
## Supported Models
| Provider | Model |
|----------|--------|
| Google AI Studio | `gemini/gemini-2.5-flash-image-preview` |
| Vertex AI | `vertex_ai/gemini-2.5-flash-image-preview` |
## Spec
The `image` field in the response follows this structure:
```python
"image": {
"url": "data:image/png;base64,<base64_encoded_image>",
"detail": "auto"
}
```
- `url` - str: Base64 encoded image data in data URI format
- `detail` - str: Image detail level (always "auto" for generated images)
The image is returned as a base64-encoded data URI that can be directly used in HTML `<img>` tags or saved to a file.

201
docs/my-website/docs/extras/gemini_img_migration.md Normal file
View file

@ -0,0 +1,201 @@
# Gemini Image Generation Migration Guide
## Who is impacted by this change?
Anyone using the following models with /chat/completions:
- `gemini/gemini-2.0-flash-exp-image-generation`
- `vertex_ai/gemini-2.5-flash-image-preview`
## Key Change
Gemini models now support image generation through chat completions. Images are returned in `response.choices[0].message.image` with base64 data URLs.
## Before and After
### Before
```python
from litellm import completion
response = completion(
model="gemini/gemini-2.0-flash-exp-image-generation",
messages=[{"role": "user", "content": "Generate an image of a cat"}],
modalities=["image", "text"],
)
base_64_image_data = response.choices[0].message.content
```
### After
```python
from litellm import completion
response = completion(
model="gemini/gemini-2.0-flash-exp-image-generation",
messages=[{"role": "user", "content": "Generate an image of a cat"}],
modalities=["image", "text"],
)
# Image is now available in the response
image_url = response.choices[0].message.image["url"] # "data:image/png;base64,..."
```
## Usage
### Using the Python SDK
**Key Change:**
```diff
# Before
-- base_64_image_data = response.choices[0].message.content
# After
++ image_url = response.choices[0].message.image["url"]
```
#### Basic Image Generation
```python
from litellm import completion
import os
# Set your API key
os.environ["GEMINI_API_KEY"] = "your-api-key"
# Generate an image
response = completion(
model="gemini/gemini-2.0-flash-exp-image-generation",
messages=[{"role": "user", "content": "Generate an image of a cat"}],
modalities=["image", "text"],
)
# Access the generated image
print(response.choices[0].message.content) # Text response (if any)
print(response.choices[0].message.image) # Image data
```
#### Response Format
The image is returned in the `message.image` field:
```python
{
"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...",
"detail": "auto"
}
```
### Using the LiteLLM Proxy Server
**Key Change:**
```diff
# Before
-- "content": "base64-image-data..."
# After
++ "image": {
++ "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...",
++ "detail": "auto"
++ }
```
#### Configuration Setup
1. **Configure your models in `config.yaml`:**
```yaml
model_list:
- model_name: gemini-image-gen
litellm_params:
model: gemini/gemini-2.0-flash-exp-image-generation
api_key: os.environ/GEMINI_API_KEY
- model_name: vertex-image-gen
litellm_params:
model: vertex_ai/gemini-2.5-flash-image-preview
vertex_project: your-project-id
vertex_location: us-central1
general_settings:
master_key: sk-1234 # Your proxy API key
```
2. **Start the proxy server:**
```bash
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
#### Making Requests
**Using OpenAI SDK:**
```python
from openai import OpenAI
# Point to your proxy server
client = OpenAI(
api_key="sk-1234", # Your proxy API key
base_url="http://0.0.0.0:4000"
)
response = client.chat.completions.create(
model="gemini-image-gen",
messages=[{"role": "user", "content": "Generate an image of a cat"}],
extra_body={"modalities": ["image", "text"]}
)
# Access the generated image
print(response.choices[0].message.content) # Text response (if any)
print(response.choices[0].message.image) # Image data
```
**Using curl:**
```bash
curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-d '{
"model": "gemini-image-gen",
"messages": [
{
"role": "user",
"content": "Generate an image of a cat"
}
],
"modalities": ["image", "text"]
}'
```
**Response format from proxy:**
```json
{
"id": "chatcmpl-123",
"object": "chat.completion",
"created": 1704089632,
"model": "gemini-image-gen",
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": "Here's an image of a cat for you!",
"image": {
"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...",
"detail": "auto"
}
},
"finish_reason": "stop"
}
],
"usage": {
"prompt_tokens": 10,
"completion_tokens": 8,
"total_tokens": 18
}
}
```

62
docs/my-website/docs/image_edits.md
View file

@ -4,7 +4,7 @@ import TabItem from '@theme/TabItem';
# /images/edits
LiteLLM provides image editing functionality that maps to OpenAI's `/images/edits` API endpoint.
LiteLLM provides image editing functionality that maps to OpenAI's `/images/edits` API endpoint. Now supports both single and multiple image editing.
| Feature | Supported | Notes |
|---------|-----------|--------|
@ -13,7 +13,7 @@ LiteLLM provides image editing functionality that maps to OpenAI's `/images/edit
| End-user Tracking | ✅ | |
| Fallbacks | ✅ | Works between supported models |
| Loadbalancing | ✅ | Works between supported models |
| Supported operations | Create image edits | |
| Supported operations | Create image edits | Single and multiple images supported |
| Supported LiteLLM SDK Versions | 1.63.8+ | |
| Supported LiteLLM Proxy Versions | 1.71.1+ | |
| Supported LLM providers | **OpenAI** | Currently only `openai` is supported |
@ -41,6 +41,26 @@ response = litellm.image_edit(
print(response)
```
#### Multiple Images Edit
```python showLineNumbers title="OpenAI Multiple Images Edit"
import litellm
# Edit multiple images with a prompt
response = litellm.image_edit(
model="gpt-image-1",
image=[
open("image1.png", "rb"),
open("image2.png", "rb"),
open("image3.png", "rb")
],
prompt="Apply vintage filter to all images",
n=1,
size="1024x1024"
)
print(response)
```
#### Image Edit with Mask
```python showLineNumbers title="OpenAI Image Edit with Mask"
import litellm
@ -80,6 +100,30 @@ response = asyncio.run(edit_image())
print(response)
```
#### Async Multiple Images Edit
```python showLineNumbers title="Async OpenAI Multiple Images Edit"
import litellm
import asyncio
async def edit_multiple_images():
response = await litellm.aimage_edit(
model="gpt-image-1",
image=[
open("portrait1.png", "rb"),
open("portrait2.png", "rb")
],
prompt="Add professional lighting to the portraits",
n=1,
size="1024x1024",
response_format="url"
)
return response
# Run the async function
response = asyncio.run(edit_multiple_images())
print(response)
```
#### Image Edit with Custom Parameters
```python showLineNumbers title="OpenAI Image Edit with Custom Parameters"
import litellm
@ -163,6 +207,20 @@ curl -X POST "http://localhost:4000/v1/images/edits" \
-F "response_format=url"
```
#### cURL Multiple Images Example
```bash showLineNumbers title="cURL Multiple Images Edit Request"
curl -X POST "http://localhost:4000/v1/images/edits" \
-H "Authorization: Bearer your-api-key" \
-F "model=gpt-image-1" \
-F "image=@image1.png" \
-F "image=@image2.png" \
-F "image=@image3.png" \
-F "prompt=Apply artistic filter to all images" \
-F "n=1" \
-F "size=1024x1024" \
-F "response_format=url"
```
</TabItem>
</Tabs>

19
docs/my-website/docs/observability/braintrust.md
View file

@ -71,6 +71,10 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
It is recommended that you include the `project_id` or `project_name` to ensure your traces are being written out to the correct Braintrust project.
### Custom Span Names
You can customize the span name in Braintrust logging by passing `span_name` in the metadata. By default, the span name is set to "Chat Completion".
<Tabs>
<TabItem value="sdk" label="SDK">
@ -84,7 +88,9 @@ response = litellm.completion(
"project_id": "1234",
# passing project_name will try to find a project with that name, or create one if it doesn't exist
# if both project_id and project_name are passed, project_id will be used
# "project_name": "my-special-project"
# "project_name": "my-special-project",
# custom span name for this operation (default: "Chat Completion")
"span_name": "User Greeting Handler"
}
)
```
@ -99,6 +105,7 @@ response = litellm.completion(
],
metadata={
"project_id": "1234",
"span_name": "Custom Operation",
"item1": "an item",
"item2": "another item"
}
@ -121,7 +128,8 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
{ "role": "user", "content": "What time is it now? Use your tool"}
],
"metadata": {
"project_id": "my-special-project"
"project_id": "my-special-project",
"span_name": "Tool Usage Request"
}
}'
```
@ -146,7 +154,8 @@ response = client.chat.completions.create(
],
extra_body={ # pass in any provider-specific param, if not supported by openai, https://docs.litellm.ai/docs/completion/input#provider-specific-params
"metadata": { # 👈 use for logging additional params (e.g. to braintrust)
"project_id": "my-special-project"
"project_id": "my-special-project",
"span_name": "Poetry Generation"
}
}
)
@ -168,3 +177,7 @@ Here's everything you can pass in metadata for a braintrust request
`braintrust_*` - If you are adding metadata from _proxy request headers_, any metadata field starting with `braintrust_` will be passed as metadata to the logging request. If you are using the SDK, just pass your metadata like normal (e.g., `metadata={"project_name": "my-test-project", "item1": "an item", "item2": "another item"}`)
`project_id` - Set the project id for a braintrust call. Default is `litellm`.
`project_name` - Set the project name for a braintrust call. Will try to find a project with that name, or create one if it doesn't exist. If both `project_id` and `project_name` are passed, `project_id` will be used.
`span_name` - Set a custom span name for the operation. Default is `"Chat Completion"`. Use this to provide more descriptive names for different types of operations in your application (e.g., "User Query", "Document Summary", "Code Generation").

23
docs/my-website/docs/observability/langfuse_otel_integration.md
View file

@ -35,14 +35,14 @@ The Langfuse OpenTelemetry integration allows you to send LiteLLM traces and obs
|----------|----------|-------------|---------|
| `LANGFUSE_PUBLIC_KEY` | Yes | Your Langfuse public key | `pk-lf-...` |
| `LANGFUSE_SECRET_KEY` | Yes | Your Langfuse secret key | `sk-lf-...` |
| `LANGFUSE_HOST` | No | Langfuse host URL | `https://us.cloud.langfuse.com` (default) |
| `LANGFUSE_OTEL_HOST` | No | OTEL endpoint host | `https://otel.my-langfuse.com` |
### Endpoint Resolution
The integration automatically constructs the OTEL endpoint from the `LANGFUSE_HOST`:
The integration automatically constructs the OTEL endpoint from `LANGFUSE_OTEL_HOST`
- **Default (US)**: `https://us.cloud.langfuse.com/api/public/otel`
- **EU Region**: `https://cloud.langfuse.com/api/public/otel`
- **Self-hosted**: `{LANGFUSE_HOST}/api/public/otel`
- **Self-hosted**: `{LANGFUSE_OTEL_HOST}/api/public/otel`
## Usage
@ -77,11 +77,11 @@ os.environ["LANGFUSE_PUBLIC_KEY"] = "pk-lf-..."
os.environ["LANGFUSE_SECRET_KEY"] = "sk-lf-..."
# Use EU region
os.environ["LANGFUSE_HOST"] = "https://cloud.langfuse.com" # EU region
# os.environ["LANGFUSE_HOST"] = "https://us.cloud.langfuse.com" # US region (default)
os.environ["LANGFUSE_OTEL_HOST"] = "https://cloud.langfuse.com" # EU region
# os.environ["LANGFUSE_OTEL_HOST"] = "https://otel.my-langfuse.company.com" # custom OTEL endpoint
# Or use self-hosted instance
# os.environ["LANGFUSE_HOST"] = "https://my-langfuse.company.com"
# os.environ["LANGFUSE_OTEL_HOST"] = "https://my-langfuse.company.com"
litellm.callbacks = ["langfuse_otel"]
```
@ -98,14 +98,16 @@ import litellm
# Get keys for your project from the project settings page: https://cloud.langfuse.com
os.environ["LANGFUSE_PUBLIC_KEY"] = "pk-lf-..."
os.environ["LANGFUSE_SECRET_KEY"] = "sk-lf-..."
os.environ["LANGFUSE_HOST"] = "https://cloud.langfuse.com" # EU region
# os.environ["LANGFUSE_HOST"] = "https://us.cloud.langfuse.com" # US region
os.environ["LANGFUSE_OTEL_HOST"] = "https://cloud.langfuse.com" # EU region
# os.environ["LANGFUSE_OTEL_HOST"] = "https://us.cloud.langfuse.com" # US region
# os.environ["LANGFUSE_OTEL_HOST"] = "https://otel.my-langfuse.company.com" # custom OTEL endpoint
LANGFUSE_AUTH = base64.b64encode(
f"{os.environ.get('LANGFUSE_PUBLIC_KEY')}:{os.environ.get('LANGFUSE_SECRET_KEY')}".encode()
).decode()
os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = os.environ.get("LANGFUSE_HOST") + "/api/public/otel"
host = os.environ.get("LANGFUSE_OTEL_HOST")
os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = host + "/api/public/otel"
os.environ["OTEL_EXPORTER_OTLP_HEADERS"] = f"Authorization=Basic {LANGFUSE_AUTH}"
litellm.callbacks = ["langfuse_otel"]
@ -120,7 +122,8 @@ Add the integration to your proxy configuration:
```bash
export LANGFUSE_PUBLIC_KEY="pk-lf-..."
export LANGFUSE_SECRET_KEY="sk-lf-..."
export LANGFUSE_HOST="https://us.cloud.langfuse.com" # Default US region
export LANGFUSE_OTEL_HOST="https://us.cloud.langfuse.com" # Default US region
# export LANGFUSE_OTEL_HOST="https://otel.my-langfuse.company.com" # custom OTEL endpoint
```
2. Setup config.yaml

30
docs/my-website/docs/providers/aiml.md
View file

@ -1,5 +1,23 @@
# AI/ML API
https://aimlapi.com/
## Overview
| Property | Details |
|-------|-------|
| Description | AI/ML API provides access to state-of-the-art AI models including flux-pro/v1.1 for high-quality image generation. |
| Provider Route on LiteLLM | `aiml/` |
| Link to Provider Doc | [AI/ML API ↗](https://docs.aimlapi.com/) |
| Supported Operations | [`/chat/completions`], [`/images/generations`](#image-generation) |
LiteLLM supports AI/ML API Image Generation calls.
## API Base, Key
```python
# env variable
os.environ['AIML_API_KEY'] = "your-api-key"
os.environ['AIML_API_BASE'] = "https://api.aimlapi.com" # [optional]
```
Getting started with the AI/ML API is simple. Follow these steps to set up your integration:
### 1. Get Your API Key
@ -24,7 +42,7 @@ You can choose from LLama, Qwen, Flux, and 200+ other open and closed-source mod
import litellm
response = litellm.completion(
model="openai/meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo", # The model name must include prefix "openai" + the model name from ai/ml api
model="aiml/meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo", # The model name must include prefix "openai" + the model name from ai/ml api
api_key="", # your aiml api-key
api_base="https://api.aimlapi.com/v2",
messages=[
@ -42,7 +60,7 @@ response = litellm.completion(
import litellm
response = litellm.completion(
model="openai/Qwen/Qwen2-72B-Instruct", # The model name must include prefix "openai" + the model name from ai/ml api
model="aiml/Qwen/Qwen2-72B-Instruct", # The model name must include prefix "openai" + the model name from ai/ml api
api_key="", # your aiml api-key
api_base="https://api.aimlapi.com/v2",
messages=[
@ -67,7 +85,7 @@ import litellm
async def main():
response = await litellm.acompletion(
model="openai/anthropic/claude-3-5-haiku", # The model name must include prefix "openai" + the model name from ai/ml api
model="aiml/anthropic/claude-3-5-haiku", # The model name must include prefix "openai" + the model name from ai/ml api
api_key="", # your aiml api-key
api_base="https://api.aimlapi.com/v2",
messages=[
@ -97,7 +115,7 @@ async def main():
try:
print("test acompletion + streaming")
response = await litellm.acompletion(
model="openai/nvidia/Llama-3.1-Nemotron-70B-Instruct-HF", # The model name must include prefix "openai" + the model name from ai/ml api
model="aiml/nvidia/Llama-3.1-Nemotron-70B-Instruct-HF", # The model name must include prefix "openai" + the model name from ai/ml api
api_key="", # your aiml api-key
api_base="https://api.aimlapi.com/v2",
messages=[{"content": "Hey, how's it going?", "role": "user"}],
@ -125,7 +143,7 @@ import litellm
async def main():
response = await litellm.aembedding(
model="openai/text-embedding-3-small", # The model name must include prefix "openai" + the model name from ai/ml api
model="aiml/text-embedding-3-small", # The model name must include prefix "openai" + the model name from ai/ml api
api_key="", # your aiml api-key
api_base="https://api.aimlapi.com/v1", # 👈 the URL has changed from v2 to v1
input="Your text string",
@ -147,7 +165,7 @@ import litellm
async def main():
response = await litellm.aimage_generation(
model="openai/dall-e-3", # The model name must include prefix "openai" + the model name from ai/ml api
model="aiml/dall-e-3", # The model name must include prefix "openai" + the model name from ai/ml api
api_key="", # your aiml api-key
api_base="https://api.aimlapi.com/v1", # 👈 the URL has changed from v2 to v1
prompt="A cute baby sea otter",

266
docs/my-website/docs/providers/azure_ai_img.md Normal file
View file

@ -0,0 +1,266 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Azure AI Image Generation
Azure AI provides powerful image generation capabilities using FLUX models from Black Forest Labs to create high-quality images from text descriptions.
## Overview
| Property | Details |
|----------|---------|
| Description | Azure AI Image Generation uses FLUX models to generate high-quality images from text descriptions. |
| Provider Route on LiteLLM | `azure_ai/` |
| Provider Doc | [Azure AI FLUX Models ↗](https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/black-forest-labs-flux-1-kontext-pro-and-flux1-1-pro-now-available-in-azure-ai-f/4434659) |
| Supported Operations | [`/images/generations`](#image-generation) |
## Setup
### API Key & Base URL
```python showLineNumbers
# Set your Azure AI API credentials
import os
os.environ["AZURE_AI_API_KEY"] = "your-api-key-here"
os.environ["AZURE_AI_API_BASE"] = "your-azure-ai-endpoint" # e.g., https://your-endpoint.eastus2.inference.ai.azure.com/
```
Get your API key and endpoint from [Azure AI Studio](https://ai.azure.com/).
## Supported Models
| Model Name | Description | Cost per Image |
|------------|-------------|----------------|
| `azure_ai/FLUX-1.1-pro` | Latest FLUX 1.1 Pro model for high-quality image generation | $0.04 |
| `azure_ai/FLUX.1-Kontext-pro` | FLUX 1 Kontext Pro model with enhanced context understanding | $0.04 |
## Image Generation
### Usage - LiteLLM Python SDK
<Tabs>
<TabItem value="basic" label="Basic Usage">
```python showLineNumbers title="Basic Image Generation"
import litellm
import os
# Set your API credentials
os.environ["AZURE_AI_API_KEY"] = "your-api-key-here"
os.environ["AZURE_AI_API_BASE"] = "your-azure-ai-endpoint"
# Generate a single image
response = litellm.image_generation(
model="azure_ai/FLUX.1-Kontext-pro",
prompt="A cute baby sea otter swimming in crystal clear water",
api_base=os.environ["AZURE_AI_API_BASE"],
api_key=os.environ["AZURE_AI_API_KEY"]
)
print(response.data[0].url)
```
</TabItem>
<TabItem value="flux11" label="FLUX 1.1 Pro">
```python showLineNumbers title="FLUX 1.1 Pro Image Generation"
import litellm
import os
# Set your API credentials
os.environ["AZURE_AI_API_KEY"] = "your-api-key-here"
os.environ["AZURE_AI_API_BASE"] = "your-azure-ai-endpoint"
# Generate image with FLUX 1.1 Pro
response = litellm.image_generation(
model="azure_ai/FLUX-1.1-pro",
prompt="A futuristic cityscape at night with neon lights and flying cars",
api_base=os.environ["AZURE_AI_API_BASE"],
api_key=os.environ["AZURE_AI_API_KEY"]
)
print(response.data[0].url)
```
</TabItem>
<TabItem value="async" label="Async Usage">
```python showLineNumbers title="Async Image Generation"
import litellm
import asyncio
import os
async def generate_image():
# Set your API credentials
os.environ["AZURE_AI_API_KEY"] = "your-api-key-here"
os.environ["AZURE_AI_API_BASE"] = "your-azure-ai-endpoint"
# Generate image asynchronously
response = await litellm.aimage_generation(
model="azure_ai/FLUX.1-Kontext-pro",
prompt="A beautiful sunset over mountains with vibrant colors",
api_base=os.environ["AZURE_AI_API_BASE"],
api_key=os.environ["AZURE_AI_API_KEY"],
n=1,
)
print(response.data[0].url)
return response
# Run the async function
asyncio.run(generate_image())
```
</TabItem>
<TabItem value="advanced" label="Advanced Parameters">
```python showLineNumbers title="Advanced Image Generation with Parameters"
import litellm
import os
# Set your API credentials
os.environ["AZURE_AI_API_KEY"] = "your-api-key-here"
os.environ["AZURE_AI_API_BASE"] = "your-azure-ai-endpoint"
# Generate image with additional parameters
response = litellm.image_generation(
model="azure_ai/FLUX-1.1-pro",
prompt="A majestic dragon soaring over a medieval castle at dawn",
api_base=os.environ["AZURE_AI_API_BASE"],
api_key=os.environ["AZURE_AI_API_KEY"],
n=1,
size="1024x1024",
quality="standard"
)
for image in response.data:
print(f"Generated image URL: {image.url}")
```
</TabItem>
</Tabs>
### Usage - LiteLLM Proxy Server
#### 1. Configure your config.yaml
```yaml showLineNumbers title="Azure AI Image Generation Configuration"
model_list:
- model_name: azure-flux-kontext
litellm_params:
model: azure_ai/FLUX.1-Kontext-pro
api_key: os.environ/AZURE_AI_API_KEY
api_base: os.environ/AZURE_AI_API_BASE
model_info:
mode: image_generation
- model_name: azure-flux-11-pro
litellm_params:
model: azure_ai/FLUX-1.1-pro
api_key: os.environ/AZURE_AI_API_KEY
api_base: os.environ/AZURE_AI_API_BASE
model_info:
mode: image_generation
general_settings:
master_key: sk-1234
```
#### 2. Start LiteLLM Proxy Server
```bash showLineNumbers title="Start LiteLLM Proxy Server"
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
#### 3. Make requests with OpenAI Python SDK
<Tabs>
<TabItem value="openai-sdk" label="OpenAI SDK">
```python showLineNumbers title="Azure AI Image Generation via Proxy - OpenAI SDK"
from openai import OpenAI
# Initialize client with your proxy URL
client = OpenAI(
base_url="http://localhost:4000", # Your proxy URL
api_key="sk-1234" # Your proxy API key
)
# Generate image with FLUX Kontext Pro
response = client.images.generate(
model="azure-flux-kontext",
prompt="A serene Japanese garden with cherry blossoms and a peaceful pond",
n=1,
size="1024x1024"
)
print(response.data[0].url)
```
</TabItem>
<TabItem value="litellm-sdk" label="LiteLLM SDK">
```python showLineNumbers title="Azure AI Image Generation via Proxy - LiteLLM SDK"
import litellm
# Configure LiteLLM to use your proxy
response = litellm.image_generation(
model="litellm_proxy/azure-flux-11-pro",
prompt="A cyberpunk warrior in a neon-lit alleyway",
api_base="http://localhost:4000",
api_key="sk-1234"
)
print(response.data[0].url)
```
</TabItem>
<TabItem value="curl" label="cURL">
```bash showLineNumbers title="Azure AI Image Generation via Proxy - cURL"
curl --location 'http://localhost:4000/v1/images/generations' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer sk-1234' \
--data '{
"model": "azure-flux-kontext",
"prompt": "A cozy coffee shop interior with warm lighting and rustic wooden furniture",
"n": 1,
"size": "1024x1024"
}'
```
</TabItem>
</Tabs>
## Supported Parameters
Azure AI Image Generation supports the following OpenAI-compatible parameters:
| Parameter | Type | Description | Default | Example |
|-----------|------|-------------|---------|---------|
| `prompt` | string | Text description of the image to generate | Required | `"A sunset over the ocean"` |
| `model` | string | The FLUX model to use for generation | Required | `"azure_ai/FLUX.1-Kontext-pro"` |
| `n` | integer | Number of images to generate (1-4) | `1` | `2` |
| `size` | string | Image dimensions | `"1024x1024"` | `"512x512"`, `"1024x1024"` |
| `api_base` | string | Your Azure AI endpoint URL | Required | `"https://your-endpoint.eastus2.inference.ai.azure.com/"` |
| `api_key` | string | Your Azure AI API key | Required | Environment variable or direct value |
## Getting Started
1. Create an account at [Azure AI Studio](https://ai.azure.com/)
2. Deploy a FLUX model in your Azure AI Studio workspace
3. Get your API key and endpoint from the deployment details
4. Set your `AZURE_AI_API_KEY` and `AZURE_AI_API_BASE` environment variables
5. Start generating images using LiteLLM
## Additional Resources
- [Azure AI Studio Documentation](https://docs.microsoft.com/en-us/azure/ai-services/)
- [FLUX Models Announcement](https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/black-forest-labs-flux-1-kontext-pro-and-flux1-1-pro-now-available-in-azure-ai-f/4434659)

113
docs/my-website/docs/providers/baseten.md
View file

@ -1,23 +1,106 @@
# Baseten
LiteLLM supports any Text-Gen-Interface models on Baseten.
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
[Here's a tutorial on deploying a huggingface TGI model (Llama2, CodeLlama, WizardCoder, Falcon, etc.) on Baseten](https://truss.baseten.co/examples/performance/tgi-server)
# Baseten
LiteLLM supports both Baseten Model APIs and dedicated deployments with automatic routing.
## API Types
### Model API (Default)
- **URL**: `https://inference.baseten.co/v1`
- **Format**: `baseten/<model-name>` (e.g., `baseten/openai/gpt-oss-120b`)
- **Best for**: Quick access to popular models
### Dedicated Deployments
- **URL**: `https://model-{id}.api.baseten.co/environments/production/sync/v1`
- **Format**: `baseten/{8-digit-alphanumeric-code}` (e.g., `baseten/abcd1234`)
- **Best for**: Custom models, latency SLAs
:::tip
**Automatic Routing**: LiteLLM detects the type based on model format:
- 8-digit alphanumeric codes → Dedicated deployment
- All other formats → Model API
:::
## Quick Start
### API KEYS
```python
import os
os.environ["BASETEN_API_KEY"] = ""
import os
from litellm import completion
os.environ['BASETEN_API_KEY'] = "your-api-key"
# Model API (default)
response = completion(
model="baseten/openai/gpt-oss-120b",
messages=[{"role": "user", "content": "Hello!"}]
)
# Dedicated deployment (8-digit ID)
response = completion(
model="baseten/abcd1234",
messages=[{"role": "user", "content": "Hello!"}]
)
```
### Baseten Models
Baseten provides infrastructure to deploy and serve ML models https://www.baseten.co/. Use liteLLM to easily call models deployed on Baseten.
## Examples
Example Baseten Usage - Note: liteLLM supports all models deployed on Baseten
### Basic Usage
```python
# Model API
response = completion(
model="baseten/openai/gpt-oss-120b",
messages=[{"role": "user", "content": "Explain quantum computing"}],
max_tokens=500,
temperature=0.7
)
Usage: Pass `model=baseten/<Model ID>`
# Dedicated deployment
response = completion(
model="baseten/abcd1234",
messages=[{"role": "user", "content": "Explain quantum computing"}],
max_tokens=500,
temperature=0.7
)
```
| Model Name | Function Call | Required OS Variables |
|------------------|--------------------------------------------|------------------------------------|
| Falcon 7B | `completion(model='baseten/qvv0xeq', messages=messages)` | `os.environ['BASETEN_API_KEY']` |
| Wizard LM | `completion(model='baseten/q841o8w', messages=messages)` | `os.environ['BASETEN_API_KEY']` |
| MPT 7B Base | `completion(model='baseten/31dxrj3', messages=messages)` | `os.environ['BASETEN_API_KEY']` |
### Streaming (Model API only)
```python
response = completion(
model="baseten/openai/gpt-oss-120b",
messages=[{"role": "user", "content": "Write a poem"}],
stream=True,
stream_options={"include_usage": True}
)
for chunk in response:
if chunk.choices and chunk.choices[0].delta.content:
print(chunk.choices[0].delta.content, end="")
```
## Usage with LiteLLM Proxy
1. **Config**:
```yaml
model_list:
- model_name: baseten-model
litellm_params:
model: baseten/openai/gpt-oss-120b
api_key: your-baseten-api-key
```
2. **Request**:
```python
import openai
client = openai.OpenAI(
api_key="sk-1234",
base_url="http://0.0.0.0:4000"
)
response = client.chat.completions.create(
model="baseten-model",
messages=[{"role": "user", "content": "Hello!"}]
)
```

144
docs/my-website/docs/providers/bedrock.md
View file

@ -584,6 +584,150 @@ Same as [Anthropic API response](../providers/anthropic#usage---thinking--reason
Same as [Anthropic API response](../providers/anthropic#usage---thinking--reasoning_content).
## Usage - Anthropic Beta Features
LiteLLM supports Anthropic's beta features on AWS Bedrock through the `anthropic-beta` header. This enables access to experimental features like:
- **1M Context Window** - Up to 1 million tokens of context (Claude Sonnet 4)
- **Computer Use Tools** - AI that can interact with computer interfaces
- **Token-Efficient Tools** - More efficient tool usage patterns
- **Extended Output** - Up to 128K output tokens
- **Enhanced Thinking** - Advanced reasoning capabilities
### Supported Beta Features
| Beta Feature | Header Value | Compatible Models | Description |
|--------------|-------------|------------------|-------------|
| 1M Context Window | `context-1m-2025-08-07` | Claude Sonnet 4 | Enable 1 million token context window |
| Computer Use (Latest) | `computer-use-2025-01-24` | Claude 3.7 Sonnet | Latest computer use tools |
| Computer Use (Legacy) | `computer-use-2024-10-22` | Claude 3.5 Sonnet v2 | Computer use tools for Claude 3.5 |
| Token-Efficient Tools | `token-efficient-tools-2025-02-19` | Claude 3.7 Sonnet | More efficient tool usage |
| Interleaved Thinking | `interleaved-thinking-2025-05-14` | Claude 4 models | Enhanced thinking capabilities |
| Extended Output | `output-128k-2025-02-19` | Claude 3.7 Sonnet | Up to 128K output tokens |
| Developer Thinking | `dev-full-thinking-2025-05-14` | Claude 4 models | Raw thinking mode for developers |
<Tabs>
<TabItem value="sdk" label="SDK">
**Single Beta Feature**
```python
from litellm import completion
import os
# set env
os.environ["AWS_ACCESS_KEY_ID"] = ""
os.environ["AWS_SECRET_ACCESS_KEY"] = ""
os.environ["AWS_REGION_NAME"] = ""
# Use 1M context window with Claude Sonnet 4
response = completion(
model="bedrock/anthropic.claude-sonnet-4-20250115-v1:0",
messages=[{"role": "user", "content": "Hello! Testing 1M context window."}],
max_tokens=100,
extra_headers={
"anthropic-beta": "context-1m-2025-08-07" # 👈 Enable 1M context
}
)
```
**Multiple Beta Features**
```python
from litellm import completion
# Combine multiple beta features (comma-separated)
response = completion(
model="bedrock/converse/anthropic.claude-3-5-sonnet-20241022-v2:0",
messages=[{"role": "user", "content": "Testing multiple beta features"}],
max_tokens=100,
extra_headers={
"anthropic-beta": "computer-use-2024-10-22,context-1m-2025-08-07"
}
)
```
**Computer Use Tools with Beta Features**
```python
from litellm import completion
# Computer use tools automatically add computer-use-2024-10-22
# You can add additional beta features
response = completion(
model="bedrock/converse/anthropic.claude-3-5-sonnet-20241022-v2:0",
messages=[{"role": "user", "content": "Take a screenshot"}],
tools=[{
"type": "computer_20241022",
"name": "computer",
"display_width_px": 1920,
"display_height_px": 1080
}],
extra_headers={
"anthropic-beta": "context-1m-2025-08-07" # Additional beta feature
}
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
**Set on YAML Config**
```yaml
model_list:
- model_name: claude-sonnet-4-1m
litellm_params:
model: bedrock/anthropic.claude-sonnet-4-20250115-v1:0
extra_headers:
anthropic-beta: "context-1m-2025-08-07" # 👈 Enable 1M context
- model_name: claude-computer-use
litellm_params:
model: bedrock/converse/anthropic.claude-3-5-sonnet-20241022-v2:0
extra_headers:
anthropic-beta: "computer-use-2024-10-22,context-1m-2025-08-07"
general_settings:
forward_client_headers_to_llm_api: true # 👈 Required for client-side header forwarding
```
**Set on Request**
```python
import openai
client = openai.OpenAI(
api_key="anything",
base_url="http://0.0.0.0:4000"
)
response = client.chat.completions.create(
model="claude-sonnet-4-1m",
messages=[{
"role": "user",
"content": "Testing 1M context window"
}],
extra_headers={
"anthropic-beta": "context-1m-2025-08-07"
}
)
```
:::info
**For client-side header forwarding**: When using the proxy and sending `anthropic-beta` headers from the client (like the OpenAI SDK), you need to enable `forward_client_headers_to_llm_api: true` in your proxy's `general_settings`. This tells the proxy to extract headers from HTTP requests and forward them to the underlying LLM provider.
:::
</TabItem>
</Tabs>
:::info
Beta features may require special access or permissions in your AWS account. Some features are only available in specific AWS regions. Check the [AWS Bedrock documentation](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-anthropic-claude-messages-request-response.html) for availability and access requirements.
:::
## Usage - Structured Output / JSON mode
<Tabs>

144
docs/my-website/docs/providers/cometapi.md Normal file
View file

@ -0,0 +1,144 @@
# CometAPI
LiteLLM supports all AI models from [CometAPI](https://www.cometapi.com/). CometAPI provides access to 500+ AI models through a unified API interface, including cutting-edge models like GPT-5, Claude Opus 4.1, and various other state-of-the-art language models.
## Authentication
To use CometAPI models, you need to obtain an API key from [CometAPI Token Console](https://api.cometapi.com/console/token). CometAPI offers free tokens for new users - you can get your free API key instantly by registering.
## Usage
Set your CometAPI key as an environment variable and use the completion function:
```python
import os
from litellm import completion
# Set API key
os.environ["COMETAPI_KEY"] = "your_comet_api_key_here"
# Define messages
messages = [{"content": "Hello, how are you?", "role": "user"}]
# Method 1: Using environment variable (recommended)
response = completion(
model="cometapi/gpt-5",
messages=messages
)
print(response.choices[0].message.content)
```
### Alternative Usage - Explicit API Key
You can also pass the API key explicitly:
```python
import os
from litellm import completion
# Define messages
messages = [{"content": "Hello, how are you?", "role": "user"}]
# Method 2: Explicitly passing API key
response = completion(
model="cometapi/gpt-4o",
messages=messages,
api_key="your_comet_api_key_here"
)
print(response.choices[0].message.content)
```
## Usage - Streaming
Just set `stream=True` when calling completion:
```python
import os
from litellm import completion
os.environ["COMETAPI_KEY"] = "your_comet_api_key_here"
messages = [{"content": "Hello, how are you?", "role": "user"}]
response = completion(
model="cometapi/gpt-5",
messages=messages,
stream=True
)
for chunk in response:
print(chunk.choices[0].delta.content or "", end="")
```
## Usage - Async Streaming
For async streaming, use `acompletion`:
```python
from litellm import acompletion
import asyncio, os, traceback
async def completion_call():
try:
os.environ["COMETAPI_KEY"] = "your_comet_api_key_here"
print("test acompletion + streaming")
response = await acompletion(
model="cometapi/chatgpt-4o-latest",
messages=[{"content": "Hello, how are you?", "role": "user"}],
stream=True
)
print(f"response: {response}")
async for chunk in response:
print(chunk)
except:
print(f"error occurred: {traceback.format_exc()}")
pass
# Run the async function
await completion_call()
```
## CometAPI Models
CometAPI offers access to 500+ AI models through a unified API. Some popular models include:
| Model Name | Function Call |
|------------|---------------|
| cometapi/gpt-5 | `completion('cometapi/gpt-5', messages)` |
| cometapi/gpt-5-mini | `completion('cometapi/gpt-5-mini', messages)` |
| cometapi/gpt-5-nano | `completion('cometapi/gpt-5-nano', messages)` |
| cometapi/gpt-oss-20b | `completion('cometapi/gpt-oss-20b', messages)` |
| cometapi/gpt-oss-120b | `completion('cometapi/gpt-oss-120b', messages)` |
| cometapi/chatgpt-4o-latest | `completion('cometapi/chatgpt-4o-latest', messages)` |
For a complete list of available models, visit the [CometAPI Models page](https://www.cometapi.com/model/).
## Environment Variables
| Variable | Description | Required |
|----------|-------------|----------|
| `COMETAPI_KEY` | Your CometAPI API key | Yes |
## Error Handling
```python
import os
from litellm import completion
try:
os.environ["COMETAPI_KEY"] = "your_comet_api_key_here"
messages = [{"content": "Hello, how are you?", "role": "user"}]
response = completion(
model="cometapi/gpt-5",
messages=messages
)
print(response.choices[0].message.content)
except Exception as e:
print(f"Error: {e}")
```

140
docs/my-website/docs/providers/deepinfra.md
View file

@ -1,3 +1,6 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# DeepInfra
https://deepinfra.com/
@ -7,6 +10,11 @@ https://deepinfra.com/
:::
## Table of Contents
- [API Key](#api-key)
- [Chat Models](#chat-models)
- [Rerank Endpoint](#rerank-endpoint)
## API Key
```python
@ -53,3 +61,135 @@ for chunk in response:
| codellama/CodeLlama-34b-Instruct-hf | `completion(model="deepinfra/codellama/CodeLlama-34b-Instruct-hf", messages)` |
| mistralai/Mistral-7B-Instruct-v0.1 | `completion(model="deepinfra/mistralai/Mistral-7B-Instruct-v0.1", messages)` |
| jondurbin/airoboros-l2-70b-gpt4-1.4.1 | `completion(model="deepinfra/jondurbin/airoboros-l2-70b-gpt4-1.4.1", messages)` |
## Rerank Endpoint
LiteLLM provides a Cohere API compatible `/rerank` endpoint for DeepInfra rerank models.
### Supported Rerank Models
| Model Name | Description |
|------------|-------------|
| `deepinfra/Qwen/Qwen3-Reranker-0.6B` | Lightweight rerank model (0.6B parameters) |
| `deepinfra/Qwen/Qwen3-Reranker-4B` | Medium rerank model (4B parameters) |
| `deepinfra/Qwen/Qwen3-Reranker-8B` | Large rerank model (8B parameters) |
### Usage - LiteLLM Python SDK
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import rerank
import os
os.environ["DEEPINFRA_API_KEY"] = "your-api-key"
response = rerank(
model="deepinfra/Qwen/Qwen3-Reranker-0.6B",
query="What is the capital of France?",
documents=[
"Paris is the capital of France.",
"London is the capital of the United Kingdom.",
"Berlin is the capital of Germany.",
"Madrid is the capital of Spain.",
"Rome is the capital of Italy."
]
)
print(response)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
1. Add to config.yaml
```yaml
model_list:
- model_name: Qwen/Qwen3-Reranker-0.6B
litellm_params:
model: deepinfra/Qwen/Qwen3-Reranker-0.6B
api_key: os.environ/DEEPINFRA_API_KEY
```
2. Start proxy
```bash
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000/
```
3. Test it!
```bash
curl -L -X POST 'http://0.0.0.0:4000/rerank' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"model": "Qwen/Qwen3-Reranker-0.6B",
"query": "What is the capital of France?",
"documents": [
"Paris is the capital of France.",
"London is the capital of the United Kingdom.",
"Berlin is the capital of Germany.",
"Madrid is the capital of Spain.",
"Rome is the capital of Italy."
]
}'
```
</TabItem>
</Tabs>
### Supported Cohere Rerank API Params
| Param | Type | Description |
| ------------------ | ----------- | ----------------------------------------------- |
| `query` | `str` | The query to rerank the documents against |
| `documents` | `list[str]` | The documents to rerank |
### Provider-specific parameters
Pass any deepinfra specific parameters as a keyword argument to the rerank function, e.g.
```
response = rerank(
model="deepinfra/Qwen/Qwen3-Reranker-0.6B",
query="What is the capital of France?",
documents=[
"Paris is the capital of France.",
"London is the capital of the United Kingdom.",
"Berlin is the capital of Germany.",
"Madrid is the capital of Spain.",
"Rome is the capital of Italy."
],
my_custom_param="my_custom_value", # any other deepinfra specific parameters
)
```
### Response Format
```json
{
"id": "request-id",
"results": [
{
"index": 0,
"relevance_score": 0.9975274205207825
},
{
"index": 1,
"relevance_score": 0.011687257327139378
}
],
"meta": {
"billed_units": {
"total_tokens": 427
},
"tokens": {
"input_tokens": 427,
"output_tokens": 0
}
}
}
```

10
docs/my-website/docs/providers/google_ai_studio/image_gen.md
View file

@ -42,7 +42,7 @@ os.environ["GEMINI_API_KEY"] = "your-api-key-here"
# Generate a single image
response = litellm.image_generation(
model="gemini/imagen-4.0-generate-preview-06-06",
model="gemini/imagen-4.0-generate-001",
prompt="A cute baby sea otter swimming in crystal clear water"
)
@ -64,7 +64,7 @@ async def generate_image():
# Generate image asynchronously
response = await litellm.aimage_generation(
model="gemini/imagen-4.0-generate-preview-06-06",
model="gemini/imagen-4.0-generate-001",
prompt="A beautiful sunset over mountains with vibrant colors",
n=1,
)
@ -89,7 +89,7 @@ os.environ["GEMINI_API_KEY"] = "your-api-key-here"
# Generate image with additional parameters
response = litellm.image_generation(
model="gemini/imagen-4.0-generate-preview-06-06",
model="gemini/imagen-4.0-generate-001",
prompt="A futuristic cityscape at night with neon lights",
n=1,
size="1024x1024",
@ -112,7 +112,7 @@ for image in response.data:
model_list:
- model_name: google-imagen
litellm_params:
model: gemini/imagen-4.0-generate-preview-06-06
model: gemini/imagen-4.0-generate-001
api_key: os.environ/GEMINI_API_KEY
model_info:
mode: image_generation
@ -198,7 +198,7 @@ Google AI Studio Image Generation supports the following OpenAI-compatible param
| Parameter | Type | Description | Default | Example |
|-----------|------|-------------|---------|---------|
| `prompt` | string | Text description of the image to generate | Required | `"A sunset over the ocean"` |
| `model` | string | The model to use for generation | Required | `"gemini/imagen-4.0-generate-preview-06-06"` |
| `model` | string | The model to use for generation | Required | `"gemini/imagen-4.0-generate-001"` |
| `n` | integer | Number of images to generate (1-4) | `1` | `2` |
| `size` | string | Image dimensions | `"1024x1024"` | `"512x512"`, `"1024x1024"` |

17
docs/my-website/docs/providers/litellm_proxy.md
View file

@ -9,7 +9,7 @@ import TabItem from '@theme/TabItem';
| Description | LiteLLM Proxy is an OpenAI-compatible gateway that allows you to interact with multiple LLM providers through a unified API. Simply use the `litellm_proxy/` prefix before the model name to route your requests through the proxy. |
| Provider Route on LiteLLM | `litellm_proxy/` (add this prefix to the model name, to route any requests to litellm_proxy - e.g. `litellm_proxy/your-model-name`) |
| Setup LiteLLM Gateway | [LiteLLM Gateway ↗](../simple_proxy) |
| Supported Endpoints |`/chat/completions`, `/completions`, `/embeddings`, `/audio/speech`, `/audio/transcriptions`, `/images`, `/rerank` |
| Supported Endpoints |`/chat/completions`, `/completions`, `/embeddings`, `/audio/speech`, `/audio/transcriptions`, `/images`, `/images/edits`, `/rerank` |
@ -111,6 +111,21 @@ response = litellm.image_generation(
)
```
## Image Edit
```python
import litellm
with open("your-image.png", "rb") as f:
response = litellm.image_edit(
model="litellm_proxy/gpt-image-1",
prompt="Make this image a watercolor painting",
image=[f],
api_base="your-litellm-proxy-url",
api_key="your-litellm-proxy-api-key",
)
```
## Audio Transcription
```python

4
docs/my-website/docs/providers/vertex_image.md
View file

@ -18,7 +18,7 @@ import litellm
# Generate a single image
response = await litellm.aimage_generation(
prompt="An olympic size swimming pool with crystal clear water and modern architecture",
model="vertex_ai/imagen-4.0-generate-preview-06-06",
model="vertex_ai/imagen-4.0-generate-001",
vertex_ai_project="your-project-id",
vertex_ai_location="us-central1",
)
@ -34,7 +34,7 @@ print(response.data[0].url)
model_list:
- model_name: vertex-imagen
litellm_params:
model: vertex_ai/imagen-4.0-generate-preview-06-06
model: vertex_ai/imagen-4.0-generate-001
vertex_ai_project: "your-project-id"
vertex_ai_location: "us-central1"
vertex_ai_credentials: "path/to/service-account.json" # Optional if using environment auth

87
docs/my-website/docs/providers/vertex_partner.md
View file

@ -14,6 +14,7 @@ import TabItem from '@theme/TabItem';
| Meta/Llama | `vertex_ai/meta/{MODEL}` | [Vertex AI - Meta Models](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/llama) |
| Mistral | `vertex_ai/mistral-*` | [Vertex AI - Mistral Models](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/mistral) |
| AI21 (Jamba) | `vertex_ai/jamba-*` | [Vertex AI - AI21 Models](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/ai21) |
| Qwen | `vertex_ai/qwen/*` | [Vertex AI - Qwen Models](https://cloud.google.com/vertex-ai/generative-ai/docs/maas/qwen) |
| Model Garden | `vertex_ai/openai/{MODEL_ID}` or `vertex_ai/{MODEL_ID}` | [Vertex Model Garden](https://cloud.google.com/model-garden?hl=en) |
## Vertex AI - Anthropic (Claude)
@ -571,6 +572,92 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
</Tabs>
## VertexAI Qwen API
| Property | Details |
|----------|---------|
| Provider Route | `vertex_ai/qwen/{MODEL}` |
| Vertex Documentation | [Vertex AI - Qwen Models](https://cloud.google.com/vertex-ai/generative-ai/docs/maas/qwen) |
**LiteLLM Supports all Vertex AI Qwen Models.** Ensure you use the `vertex_ai/qwen/` prefix for all Vertex AI Qwen models.
| Model Name | Usage |
|------------------|------------------------------|
| vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas | `completion('vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas', messages)` |
| vertex_ai/qwen/qwen3-235b-a22b-instruct-2507-maas | `completion('vertex_ai/qwen/qwen3-235b-a22b-instruct-2507-maas', messages)` |
#### Usage
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
import os
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = ""
model = "qwen/qwen3-coder-480b-a35b-instruct-maas"
vertex_ai_project = "your-vertex-project" # can also set this as os.environ["VERTEXAI_PROJECT"]
vertex_ai_location = "your-vertex-location" # can also set this as os.environ["VERTEXAI_LOCATION"]
response = completion(
model="vertex_ai/" + model,
messages=[{"role": "user", "content": "hi"}],
vertex_ai_project=vertex_ai_project,
vertex_ai_location=vertex_ai_location,
)
print("\nModel Response", response)
```
</TabItem>
<TabItem value="proxy" label="Proxy">
**1. Add to config**
```yaml
model_list:
- model_name: vertex-qwen
litellm_params:
model: vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas
vertex_ai_project: "my-test-project"
vertex_ai_location: "us-east-1"
- model_name: vertex-qwen
litellm_params:
model: vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas
vertex_ai_project: "my-test-project"
vertex_ai_location: "us-west-1"
```
**2. Start proxy**
```bash
litellm --config /path/to/config.yaml
# RUNNING at http://0.0.0.0:4000
```
**3. Test it!**
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"model": "vertex-qwen", # 👈 the 'model_name' in config
"messages": [
{
"role": "user",
"content": "what llm are you"
}
],
}'
```
</TabItem>
</Tabs>
## Model Garden
:::tip

46
docs/my-website/docs/providers/vllm.md
View file

@ -104,6 +104,52 @@ Here's how to call an OpenAI-Compatible Endpoint with the LiteLLM Proxy Server
</Tabs>
## Reasoning Effort
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
response = completion(
model="hosted_vllm/gpt-oss-120b",
messages=[{"role": "user", "content": "whats 2 + 2"}],
reasoning_effort="high",
api_base="https://hosted-vllm-api.co",
)
print(response)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
1. Setup config.yaml
```yaml
model_list:
- model_name: gpt-oss-120b
litellm_params:
model: hosted_vllm/gpt-oss-120b
api_base: https://hosted-vllm-api.co
```
2. Start the proxy
```bash
litellm --config /path/to/config.yaml
```
3. Test it!
```bash
curl http://0.0.0.0:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-d '{"model": "gpt-oss-120b", "messages": [{"role": "user", "content": "whats 2 + 2"}], "reasoning_effort": "high"}'
```
</TabItem>
</Tabs>
## Embeddings

6
docs/my-website/docs/proxy/config_settings.md
View file

@ -335,12 +335,16 @@ router_settings:
| ANTHROPIC_API_KEY | API key for Anthropic service
| ANTHROPIC_API_BASE | Base URL for Anthropic API. Default is https://api.anthropic.com
| AWS_ACCESS_KEY_ID | Access Key ID for AWS services
| AWS_DEFAULT_REGION | Default AWS region for service interactions when AWS_REGION is not set
| AWS_PROFILE_NAME | AWS CLI profile name to be used
| AWS_REGION | AWS region for service interactions (takes precedence over AWS_DEFAULT_REGION)
| AWS_REGION_NAME | Default AWS region for service interactions
| AWS_ROLE_ARN | ARN of the AWS IAM role to assume for authentication
| AWS_ROLE_NAME | Role name for AWS IAM usage
| AWS_SECRET_ACCESS_KEY | Secret Access Key for AWS services
| AWS_SESSION_NAME | Name for AWS session
| AWS_WEB_IDENTITY_TOKEN | Web identity token for AWS
| AWS_WEB_IDENTITY_TOKEN_FILE | Path to file containing web identity token for AWS
| AZURE_API_VERSION | Version of the Azure API being used
| AZURE_AUTHORITY_HOST | Azure authority host URL
| AZURE_CERTIFICATE_PASSWORD | Password for Azure OpenAI certificate
@ -349,6 +353,7 @@ router_settings:
| AZURE_CODE_INTERPRETER_COST_PER_SESSION | Cost per session for Azure Code Interpreter service
| AZURE_COMPUTER_USE_INPUT_COST_PER_1K_TOKENS | Input cost per 1K tokens for Azure Computer Use service
| AZURE_COMPUTER_USE_OUTPUT_COST_PER_1K_TOKENS | Output cost per 1K tokens for Azure Computer Use service
| AZURE_DEFAULT_RESPONSES_API_VERSION | Version of the Azure Default Responses API being used. Default is "preview"
| AZURE_TENANT_ID | Tenant ID for Azure Active Directory
| AZURE_USERNAME | Username for Azure services, use in conjunction with AZURE_PASSWORD for azure ad token with basic username/password workflow
| AZURE_PASSWORD | Password for Azure services, use in conjunction with AZURE_USERNAME for azure ad token with basic username/password workflow
@ -565,6 +570,7 @@ router_settings:
| LITELLM_LICENSE | License key for LiteLLM usage
| LITELLM_LOCAL_MODEL_COST_MAP | Local configuration for model cost mapping in LiteLLM
| LITELLM_LOG | Enable detailed logging for LiteLLM
| LITELLM_LOG_FILE | File path to write LiteLLM logs to. When set, logs will be written to both console and the specified file
| LITELLM_MASTER_KEY | Master key for proxy authentication
| LITELLM_MODE | Operating mode for LiteLLM (e.g., production, development)
| LITELLM_RATE_LIMIT_WINDOW_SIZE | Rate limit window size for LiteLLM. Default is 60

16
docs/my-website/docs/proxy/deploy.md
View file

@ -127,6 +127,8 @@ CMD ["--port", "4000", "--config", "config.yaml", "--detailed_debug"]
Follow these instructions to build a docker container from the litellm pip package. If your company has a strict requirement around security / building images you can follow these steps.
**Note:** You'll need to copy the `schema.prisma` file from the [litellm repository](https://github.com/BerriAI/litellm/blob/main/schema.prisma) to your build directory alongside the Dockerfile and requirements.txt.
Dockerfile
```shell
@ -149,6 +151,12 @@ COPY requirements.txt .
RUN --mount=type=cache,target=${HOME}/.cache/pip \
${HOME}/venv/bin/pip install -r requirements.txt
# Copy Prisma schema file
COPY schema.prisma .
# Generate prisma client
RUN prisma generate
EXPOSE 4000/tcp
ENTRYPOINT ["litellm"]
@ -1002,5 +1010,13 @@ User-agent: *
Disallow: /
```
## Deployment FAQ
**Q: Is Postgres the only supported database, or do you support other ones (like Mongo)?**
A: We explored MySQL but that was hard to maintain and led to bugs for customers. Currently, PostgreSQL is our primary supported database for production deployments.
**Q: If there is Postgres downtime, how does LiteLLM react? Does it fail-open or is there API downtime?**
A: You can gracefully handle DB unavailability if it's on your VPC. See our production guide for more details: [Gracefully Handle DB Unavailability](https://docs.litellm.ai/docs/proxy/prod#6-if-running-litellm-on-vpc-gracefully-handle-db-unavailability)

2
docs/my-website/docs/proxy/docker_quick_start.md
View file

@ -2,7 +2,7 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Getting Started - E2E Tutorial
# E2E Tutorial
End-to-End tutorial for LiteLLM Proxy to:
- Add an Azure OpenAI model

5
docs/my-website/docs/proxy/guardrails/custom_guardrail.md
View file

@ -23,15 +23,14 @@ A CustomGuardrail has 4 methods to enforce guardrails
Create a new file called `custom_guardrail.py` and add this code to it
```python
from typing import Any, Dict, List, Literal, Optional, Union
from typing import Any, AsyncGenerator, Literal, Optional, Union
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.caching.caching import DualCache
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.guardrails.guardrail_helpers import should_proceed_based_on_metadata
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.utils import ModelResponseStream
class myCustomGuardrail(CustomGuardrail):

299
docs/my-website/docs/proxy/guardrails/noma_security.md Normal file
View file

@ -0,0 +1,299 @@
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Noma Security
Use [Noma Security](https://noma.security/) to protect your LLM applications with comprehensive AI content moderation and safety guardrails.
## Quick Start
### 1. Define Guardrails on your LiteLLM config.yaml
Define your guardrails under the `guardrails` section:
```yaml showLineNumbers title="litellm config.yaml"
model_list:
- model_name: gpt-4o-mini
litellm_params:
model: openai/gpt-4o-mini
api_key: os.environ/OPENAI_API_KEY
guardrails:
- guardrail_name: "noma-guard"
litellm_params:
guardrail: noma
mode: "during_call"
api_key: os.environ/NOMA_API_KEY
api_base: os.environ/NOMA_API_BASE
- guardrail_name: "noma-pre-guard"
litellm_params:
guardrail: noma
mode: "pre_call"
api_key: os.environ/NOMA_API_KEY
api_base: os.environ/NOMA_API_BASE
```
#### Supported values for `mode`
- `pre_call` Run **before** LLM call, on **input**
- `post_call` Run **after** LLM call, on **input & output**
- `during_call` Run **during** LLM call, on **input**. Same as `pre_call` but runs in parallel with the LLM call. Response not returned until guardrail check completes
### 2. Start LiteLLM Gateway
```shell
litellm --config config.yaml --detailed_debug
```
### 3. Test request
<Tabs>
<TabItem label="Unsuccessful call" value="not-allowed">
Expect this to fail since the request contains harmful content:
```shell showLineNumbers title="Curl Request"
curl -i http://0.0.0.0:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-d '{
"model": "gpt-4o-mini",
"messages": [
{"role": "user", "content": "Tell me how to hack into someone's email account"}
]
}'
```
Expected response on failure:
```json
{
"error": {
"message": "{\n \"error\": \"Request blocked by Noma guardrail\",\n \"details\": {\n \"prompt\": {\n \"harmfulContent\": {\n \"result\": true,\n \"confidence\": 0.95\n }\n }\n }\n }",
"type": "None",
"param": "None",
"code": "400"
}
}
```
</TabItem>
<TabItem label="Successful Call" value="allowed">
```shell showLineNumbers title="Curl Request"
curl -i http://0.0.0.0:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-d '{
"model": "gpt-4o-mini",
"messages": [
{"role": "user", "content": "What is the capital of France?"}
]
}'
```
Expected response:
```json
{
"id": "chatcmpl-123",
"object": "chat.completion",
"created": 1677652288,
"model": "gpt-4o-mini",
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": "The capital of France is Paris."
},
"finish_reason": "stop"
}
],
"usage": {
"prompt_tokens": 9,
"completion_tokens": 12,
"total_tokens": 21
}
}
```
</TabItem>
</Tabs>
## Supported Params
```yaml
guardrails:
- guardrail_name: "noma-guard"
litellm_params:
guardrail: noma
mode: "pre_call"
api_key: os.environ/NOMA_API_KEY
api_base: os.environ/NOMA_API_BASE
### OPTIONAL ###
# application_id: "my-app"
# monitor_mode: false
# block_failures: true
```
### Required Parameters
- **`api_key`**: Your Noma Security API key (set as `os.environ/NOMA_API_KEY` in YAML config)
### Optional Parameters
- **`api_base`**: Noma API base URL (defaults to `https://api.noma.security/`)
- **`application_id`**: Your application identifier (defaults to `"litellm"`)
- **`monitor_mode`**: If `true`, logs violations without blocking (defaults to `false`)
- **`block_failures`**: If `true`, blocks requests when guardrail API failures occur (defaults to `true`)
## Environment Variables
You can set these environment variables instead of hardcoding values in your config:
```shell
export NOMA_API_KEY="your-api-key-here"
export NOMA_API_BASE="https://api.noma.security/" # Optional
export NOMA_APPLICATION_ID="my-app" # Optional
export NOMA_MONITOR_MODE="false" # Optional
export NOMA_BLOCK_FAILURES="true" # Optional
```
## Advanced Configuration
### Monitor Mode
Use monitor mode to test your guardrails without blocking requests:
```yaml
guardrails:
- guardrail_name: "noma-monitor"
litellm_params:
guardrail: noma
mode: "pre_call"
api_key: os.environ/NOMA_API_KEY
monitor_mode: true # Log violations but don't block
```
### Handling API Failures
Control behavior when the Noma API is unavailable:
```yaml
guardrails:
- guardrail_name: "noma-failopen"
litellm_params:
guardrail: noma
mode: "pre_call"
api_key: os.environ/NOMA_API_KEY
block_failures: false # Allow requests to proceed if guardrail API fails
```
### Multiple Guardrails
Apply different configurations for input and output:
```yaml
guardrails:
- guardrail_name: "noma-strict-input"
litellm_params:
guardrail: noma
mode: "pre_call"
api_key: os.environ/NOMA_API_KEY
block_failures: true
- guardrail_name: "noma-monitor-output"
litellm_params:
guardrail: noma
mode: "post_call"
api_key: os.environ/NOMA_API_KEY
monitor_mode: true
```
## ✨ Pass Additional Parameters
Use `extra_body` to pass additional parameters to the Noma Security API call, such as dynamically setting the application ID for specific requests.
<Tabs>
<TabItem value="openai" label="OpenAI Python">
```python
import openai
client = openai.OpenAI(
api_key="your-api-key",
base_url="http://0.0.0.0:4000"
)
response = client.chat.completions.create(
model="gpt-4o-mini",
messages=[{"role": "user", "content": "Hello, how are you?"}],
extra_body={
"guardrails": {
"noma-guard": {
"extra_body": {
"application_id": "my-specific-app-id"
}
}
}
}
)
```
</TabItem>
<TabItem value="curl" label="Curl">
```shell
curl 'http://0.0.0.0:4000/v1/chat/completions' \
-H 'Content-Type: application/json' \
-d '{
"model": "gpt-4o-mini",
"messages": [
{
"role": "user",
"content": "Hello, how are you?"
}
],
"guardrails": {
"noma-guard": {
"extra_body": {
"application_id": "my-specific-app-id"
}
}
}
}'
```
</TabItem>
</Tabs>
This allows you to override the default `application_id` parameter for specific requests, which is useful for tracking usage across different applications or components.
## Response Details
When content is blocked, Noma provides detailed information about the violations as JSON inside the `message` field, with the following structure:
```json
{
"error": "Request blocked by Noma guardrail",
"details": {
"prompt": {
"harmfulContent": {
"result": true,
"confidence": 0.95
},
"sensitiveData": {
"email": {
"result": true,
"entities": ["user@example.com"]
}
},
"bannedTopics": {
"violence": {
"result": true,
"confidence": 0.88
}
}
}
}
}
```

5
docs/my-website/docs/proxy/quick_start.md
View file

@ -2,8 +2,9 @@ import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Quick Start
Quick start CLI, Config, Docker
# CLI - Quick Start
Setup LiteLLM Proxy quickly via CLI.
LiteLLM Server (LLM Gateway) manages:

31
docs/my-website/docs/proxy/self_serve.md
View file

@ -309,6 +309,37 @@ curl -X POST '<PROXY_BASE_URL>/team/new' \
</TabItem>
</Tabs>
### Team Member Rate Limits
Set a default tpm/rpm limit for an individual team member.
You can do this when creating a new team, or by updating an existing team.
<Tabs>
<TabItem value="ui" label="UI">
<Image img={require('../../img/create_team_member_rate_limits.png')} style={{ width: '600px', height: 'auto' }} />
</TabItem>
<TabItem value="api" label="API">
```bash
curl -X POST '<PROXY_BASE_URL>/team/new' \
-H 'Authorization: Bearer <PROXY_MASTER_KEY>' \
-H 'Content-Type: application/json' \
-D '{
"team_alias": "team_1",
"team_member_rpm_limit": 100,
"team_member_tpm_limit": 1000
}'
```
</TabItem>
</Tabs>
### Set default params for new teams
When you connect litellm to your SSO provider, litellm can auto-create teams. Use this to set the default `models`, `max_budget`, `budget_duration` for these auto-created teams.

6
docs/my-website/docs/proxy/team_budgets.md
View file

@ -4,6 +4,12 @@ import TabItem from '@theme/TabItem';
# Setting Team Budgets
# Pre-Requisites
- You must set up a Postgres database (e.g. Supabase, Neon, etc.)
- To enable team member rate limits, set the environment variable `EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING=true` **before starting the proxy server**. Without this, team member rate limits will not be enforced.
Track spend, set budgets for your Internal Team
## Setting Monthly Team Budgets

8
docs/my-website/docs/proxy/users.md
View file

@ -58,6 +58,9 @@ You can:
**Step-by step tutorial on setting, resetting budgets on Teams here (API or using Admin UI)**
> **Prerequisite:**
> To enable team member rate limits, you must set the environment variable `EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING=true` before starting the proxy server. Without this, team member rate limits will not be enforced.
👉 [https://docs.litellm.ai/docs/proxy/team_budgets](https://docs.litellm.ai/docs/proxy/team_budgets)
:::
@ -793,6 +796,11 @@ Expected Response:
Enable multi-instance rate limiting with the env var `EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING="True"`
**Important Notes:**
- Setting `EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING="True"` is required for team member rate limits to function, not just for multi-instance scenarios.
- **Rate limits do not apply to proxy admin users.**
- When testing rate limits, use internal user roles (non-admin) to ensure limits are enforced as expected.
Changes:
- This moves to using async_increment instead of async_set_cache when updating current requests/tokens.
- The in-memory cache is synced with redis every 0.01s, to avoid calling redis for every request.

3
docs/my-website/docs/rerank.md
View file

@ -118,4 +118,5 @@ curl http://0.0.0.0:4000/rerank \
| AWS Bedrock| [Usage](../docs/providers/bedrock#rerank-api) |
| HuggingFace| [Usage](../docs/providers/huggingface_rerank) |
| Infinity| [Usage](../docs/providers/infinity) |
| vLLM| [Usage](../docs/providers/vllm#rerank-endpoint) |
| vLLM| [Usage](../docs/providers/vllm#rerank-endpoint) |
| DeepInfra| [Usage](../docs/providers/deepinfra#rerank-endpoint) |

12
docs/my-website/docs/response_api.md
View file

@ -803,10 +803,18 @@ LiteLLM Proxy supports session management for non-OpenAI models. This allows you
1. Enable storing request / response content in the database
Set `store_prompts_in_spend_logs: true` in your proxy config.yaml. When this is enabled, LiteLLM will store the request and response content in the database.
Set `store_prompts_in_cold_storage: true` in your proxy config.yaml. When this is enabled, LiteLLM will store the request and response content in the s3 bucket you specify.
```yaml showLineNumbers title="config.yaml with Session Continuity"
litellm_settings:
callbacks: ["s3_v2"]
cold_storage_custom_logger: s3_v2
s3_callback_params: # learn more https://docs.litellm.ai/docs/proxy/logging#s3-buckets
s3_bucket_name: litellm-logs # AWS Bucket Name for S3
s3_region_name: us-west-2
```yaml
general_settings:
store_prompts_in_cold_storage: true
store_prompts_in_spend_logs: true
```

1353
docs/my-website/docs/simple_proxy_old_doc.md
View file

File diff suppressed because it is too large Load diff

95
docs/my-website/docs/tutorials/claude_responses_api.md
View file

@ -4,11 +4,11 @@ import TabItem from '@theme/TabItem';
# Claude Code
This tutorial shows how to call the Responses API models like `codex-mini` and `o3-pro` from the Claude Code endpoint on LiteLLM.
This tutorial shows how to call Claude models through LiteLLM proxy from Claude Code.
:::info
This tutorial is based on [Anthropic's official LiteLLM configuration documentation](https://docs.anthropic.com/en/docs/claude-code/llm-gateway#litellm-configuration). This integration allows you to use any LiteLLM supported model through Claude Code.
This tutorial is based on [Anthropic's official LiteLLM configuration documentation](https://docs.anthropic.com/en/docs/claude-code/llm-gateway#litellm-configuration). This integration allows you to use any LiteLLM supported model through Claude Code with centralized authentication, usage tracking, and cost controls.
:::
@ -31,19 +31,18 @@ Create a secure configuration using environment variables:
```yaml
model_list:
# Responses API models
- model_name: codex-mini
# Claude models
- model_name: claude-3-5-sonnet-20241022
litellm_params:
model: openai/codex-mini
api_key: os.environ/OPENAI_API_KEY
api_base: https://api.openai.com/v1
model: anthropic/claude-3-5-sonnet-20241022
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: o3-pro
- model_name: claude-3-5-haiku-20241022
litellm_params:
model: openai/o3-pro
api_key: os.environ/OPENAI_API_KEY
api_base: https://api.openai.com/v1
model: anthropic/claude-3-5-haiku-20241022
api_key: os.environ/ANTHROPIC_API_KEY
litellm_settings:
master_key: os.environ/LITELLM_MASTER_KEY
```
@ -51,7 +50,7 @@ litellm_settings:
Set your environment variables:
```bash
export OPENAI_API_KEY="your-openai-api-key"
export ANTHROPIC_API_KEY="your-anthropic-api-key"
export LITELLM_MASTER_KEY="sk-1234567890" # Generate a secure key
```
@ -72,31 +71,43 @@ curl -X POST http://0.0.0.0:4000/v1/messages \
-H "Authorization: Bearer $LITELLM_MASTER_KEY" \
-H "Content-Type: application/json" \
-d '{
"model": "codex-mini",
"model": "claude-3-5-sonnet-20241022",
"max_tokens": 1000,
"messages": [{"role": "user", "content": "What is the capital of France?"}]
}'
```
### 4. Configure Claude Code
Setup Claude Code to use your LiteLLM proxy:
#### Method 1: Unified Endpoint (Recommended)
Configure Claude Code to use LiteLLM's unified endpoint:
```bash
export ANTHROPIC_BASE_URL="http://0.0.0.0:4000"
export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY"
```
### 5. Use Claude Code
#### Method 2: Provider-specific Pass-through Endpoint
Start Claude Code with any configured model:
Alternatively, use the Anthropic pass-through endpoint:
```bash
# Use Responses API models
claude --model codex-mini
claude --model o3-pro
export ANTHROPIC_BASE_URL="http://0.0.0.0:4000/anthropic"
export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY"
```
# Or use the latest model alias
claude --model codex-mini-latest
### 5. Use Claude Code
Start Claude Code and it will automatically use your configured models:
```bash
# Claude Code will use the models configured in your LiteLLM proxy
claude
# Or specify a model if you have multiple configured
claude --model claude-3-5-sonnet-20241022
claude --model claude-3-5-haiku-20241022
```
Example conversation:
@ -112,7 +123,8 @@ Common issues and solutions:
**Authentication errors:**
- Verify your environment variables are set: `echo $LITELLM_MASTER_KEY`
- Check that your OpenAI API key is valid and has sufficient credits
- Check that your API keys are valid and have sufficient credits
- Ensure the `ANTHROPIC_AUTH_TOKEN` matches your LiteLLM master key
**Model not found:**
- Ensure the model name in Claude Code matches exactly with your `config.yaml`
@ -123,33 +135,47 @@ Common issues and solutions:
Expand your configuration to support multiple providers and models:
<Tabs>
<TabItem value="responses-plus" label="Responses + Standard Models">
<TabItem value="multi-provider" label="Multi-Provider Setup">
```yaml
model_list:
# Responses API models
# OpenAI models
- model_name: codex-mini
litellm_params:
litellm_params:
model: openai/codex-mini
api_key: os.environ/OPENAI_API_KEY
api_base: https://api.openai.com/v1
- model_name: o3-pro
litellm_params:
model: openai/o3-pro
api_key: os.environ/OPENAI_API_KEY
api_base: https://api.openai.com/v1
# Standard models
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY
api_base: https://api.openai.com/v1
- model_name: claude-3-5-sonnet
# Anthropic models
- model_name: claude-3-5-sonnet-20241022
litellm_params:
model: anthropic/claude-3-5-sonnet-20241022
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: claude-3-5-haiku-20241022
litellm_params:
model: anthropic/claude-3-5-haiku-20241022
api_key: os.environ/ANTHROPIC_API_KEY
# AWS Bedrock
- model_name: claude-bedrock
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1
litellm_settings:
master_key: os.environ/LITELLM_MASTER_KEY
@ -158,13 +184,14 @@ litellm_settings:
Switch between models seamlessly:
```bash
# Use Responses API models for advanced reasoning
claude --model o3-pro
claude --model codex-mini
# Use Claude for complex reasoning
claude --model claude-3-5-sonnet-20241022
# Use standard models for general tasks
claude --model gpt-4o
claude --model claude-3-5-sonnet
# Use Haiku for fast responses
claude --model claude-3-5-haiku-20241022
# Use Bedrock deployment
claude --model claude-bedrock
```
</TabItem>

BIN
docs/my-website/img/create_team_member_rate_limits.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 100 KiB

BIN
docs/my-website/img/release_notes/faster_caching_calls.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 270 KiB

BIN
docs/my-website/img/release_notes/responses_api_session_mgt_images.jpg Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 472 KiB

BIN
docs/my-website/img/release_notes/team_member_rate_limits.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 372 KiB

6
docs/my-website/package-lock.json generated
View file

@ -12933,9 +12933,9 @@
}
},
"node_modules/mermaid": {
"version": "11.9.0",
"resolved": "https://registry.npmjs.org/mermaid/-/mermaid-11.9.0.tgz",
"integrity": "sha512-YdPXn9slEwO0omQfQIsW6vS84weVQftIyyTGAZCwM//MGhPzL1+l6vO6bkf0wnP4tHigH1alZ5Ooy3HXI2gOag==",
"version": "11.10.0",
"resolved": "https://registry.npmjs.org/mermaid/-/mermaid-11.10.0.tgz",
"integrity": "sha512-oQsFzPBy9xlpnGxUqLbVY8pvknLlsNIJ0NWwi8SUJjhbP1IT0E0o1lfhU4iYV3ubpy+xkzkaOyDUQMn06vQElQ==",
"dependencies": {
"@braintree/sanitize-url": "^7.0.4",
"@iconify/utils": "^2.1.33",

3
docs/my-website/package.json
View file

@ -49,6 +49,7 @@
},
"overrides": {
"webpack-dev-server": ">=5.2.1",
"form-data": ">=4.0.4"
"form-data": ">=4.0.4",
"mermaid": ">=11.10.0"
}
}

6
docs/my-website/release_notes/v1.74.15-stable/index.md
View file

@ -86,9 +86,9 @@ This is great to central AI Platform teams looking to track how they are helping
| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Cost per Image |
| ----------- | -------------------------------------- | -------------- | ------------------- | -------------------- | -------------- |
| OpenRouter | `openrouter/x-ai/grok-4` | 256k | $3 | $15 | N/A |
| Google AI Studio | `gemini/imagen-4.0-generate-preview-06-06` | N/A | N/A | N/A | $0.04 |
| Google AI Studio | `gemini/imagen-4.0-ultra-generate-preview-06-06` | N/A | N/A | N/A | $0.06 |
| Google AI Studio | `gemini/imagen-4.0-fast-generate-preview-06-06` | N/A | N/A | N/A | $0.02 |
| Google AI Studio | `gemini/imagen-4.0-generate-001` | N/A | N/A | N/A | $0.04 |
| Google AI Studio | `gemini/imagen-4.0-ultra-generate-001` | N/A | N/A | N/A | $0.06 |
| Google AI Studio | `gemini/imagen-4.0-fast-generate-001` | N/A | N/A | N/A | $0.02 |
| Google AI Studio | `gemini/imagen-3.0-generate-002` | N/A | N/A | N/A | $0.04 |
| Google AI Studio | `gemini/imagen-3.0-generate-001` | N/A | N/A | N/A | $0.04 |
| Google AI Studio | `gemini/imagen-3.0-fast-generate-001` | N/A | N/A | N/A | $0.02 |

47
docs/my-website/release_notes/v1.75.5-stable/index.md
View file

@ -1,5 +1,5 @@
---
title: "[PRE-RELEASE]v1.75.5-stable"
title: "v1.75.5-stable - Redis latency improvements"
slug: "v1-75-5"
date: 2025-08-10T10:00:00
authors:
@ -28,14 +28,14 @@ import TabItem from '@theme/TabItem';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:v1.75.5.rc.1
ghcr.io/berriai/litellm:v1.75.5-stable
```
</TabItem>
<TabItem value="pip" label="Pip">
``` showLineNumbers title="pip install litellm"
pip install litellm==1.75.5.post1
pip install litellm==1.75.5.post2
```
</TabItem>
@ -43,8 +43,49 @@ pip install litellm==1.75.5.post1
---
## Key Highlights
- **Redis - Latency Improvements** - Reduces P99 latency by 50% with Redis enabled.
- **Responses API Session Management** - Support for managing responses API sessions with images.
- **Oracle Cloud Infrastructure** - New LLM provider for calling models on Oracle Cloud Infrastructure.
- **Digital Ocean's Gradient AI** - New LLM provider for calling models on Digital Ocean's Gradient AI platform.
### Risk of Upgrade
If you build the proxy from the pip package, you should hold off on upgrading. This version makes `prisma migrate deploy` our default for managing the DB. This is safer, as it doesn't reset the DB, but it requires a manual `prisma generate` step.
Users of our Docker image, are **not** affected by this change.
---
## Redis Latency Improvements
<Image
img={require('../../img/release_notes/faster_caching_calls.png')}
style={{width: '100%', display: 'block', margin: '2rem auto'}}
/>
<br/>
This release adds in-memory caching for Redis requests, enabling faster response times in high-traffic. Now, LiteLLM instances will check their in-memory cache for a cache hit, before checking Redis. This reduces caching-related latency from 100ms for LLM API calls to sub-1ms, on cache hits.
---
## Responses API Session Management w/ Images
<Image
img={require('../../img/release_notes/responses_api_session_mgt_images.jpg')}
style={{width: '100%', display: 'block', margin: '2rem auto'}}
/>
<br/>
LiteLLM now supports session management for Responses API requests with images. This is great for use-cases like chatbots, that are using the Responses API to track the state of a conversation. LiteLLM session management works across **ALL** LLM API's (including Anthropic, Bedrock, OpenAI, etc). LiteLLM session management works by storing the request and response content in an s3 bucket, you can specify.
---
## New Models / Updated Models
#### New Model Support

247
docs/my-website/release_notes/v1.75.8/index.md Normal file
View file

@ -0,0 +1,247 @@
---
title: "v1.75.8-stable - Team Member Rate Limits"
slug: "v1-75-8"
date: 2025-08-16T10:00:00
authors:
- name: Krrish Dholakia
title: CEO, LiteLLM
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaffer
title: CTO, LiteLLM
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
hide_table_of_contents: false
---
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
## Deploy this version
<Tabs>
<TabItem value="docker" label="Docker">
``` showLineNumbers title="docker run litellm"
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:v1.75.8-stable
```
</TabItem>
<TabItem value="pip" label="Pip">
``` showLineNumbers title="pip install litellm"
pip install litellm==1.75.8
```
</TabItem>
</Tabs>
---
## Key Highlights
- **Team Member Rate Limits** - Individual rate limiting for team members with JWT authentication support.
- **Performance Improvements** - New experimental HTTP handler flag for 100+ RPS improvement on OpenAI calls.
- **GPT-5 Model Family Support** - Full support for OpenAI's GPT-5 models with `reasoning_effort` parameter and Azure OpenAI integration.
- **Azure AI Flux Image Generation** - Support for Azure AI's Flux image generation models.
---
## Team Member Rate Limits
<Image
img={require('../../img/release_notes/team_member_rate_limits.png')}
style={{width: '100%', display: 'block', margin: '2rem auto'}}
/>
<p style={{textAlign: 'left', color: '#666'}}>
LiteLLM MCP Architecture: Use MCP tools with all LiteLLM supported models
</p>
This release adds support for setting rate limits on individual members (including machine users) within a team. Teams can now give each agent its own rate limits—so that heavy-traffic agents don’t impact other agents or human users.
Agents can authenticate with LiteLLM using JWT and the same team role as human users, while still enforcing per-agent rate limits.
## New Models / Updated Models
#### New Model Support
| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features |
| ----------- | -------------------------------------- | -------------- | ------------------- | -------------------- | -------- |
| Azure AI | `azure_ai/FLUX-1.1-pro` | - | - | $40/image | Image generation |
| Azure AI | `azure_ai/FLUX.1-Kontext-pro` | - | - | $40/image | Image generation |
| Vertex AI | `vertex_ai/deepseek-ai/deepseek-r1-0528-maas` | 65k | $1.35 | $5.4 | Chat completions + reasoning |
| OpenRouter | `openrouter/deepseek/deepseek-chat-v3-0324` | 65k | $0.14 | $0.28 | Chat completions |
#### Features
- **[OpenAI](../../docs/providers/openai)**
- Added `reasoning_effort` parameter support for GPT-5 model family - [PR #13475](https://github.com/BerriAI/litellm/pull/13475), [Get Started](../../docs/providers/openai#openai-chat-completion-models)
- Support for `reasoning` parameter in Responses API - [PR #13475](https://github.com/BerriAI/litellm/pull/13475), [Get Started](../../docs/response_api)
- **[Azure OpenAI](../../docs/providers/azure/azure)**
- GPT-5 support with max_tokens and `reasoning` parameter - [PR #13510](https://github.com/BerriAI/litellm/pull/13510), [Get Started](../../docs/providers/azure/azure#gpt-5-models)
- **[AWS Bedrock](../../docs/providers/bedrock)**
- Streaming support for bedrock gpt-oss model family - [PR #13346](https://github.com/BerriAI/litellm/pull/13346), [Get Started](../../docs/providers/bedrock#openai-gpt-oss)
- `/messages` endpoint compatibility with `bedrock/converse/<model>` - [PR #13627](https://github.com/BerriAI/litellm/pull/13627)
- Cache point support for assistant and tool messages - [PR #13640](https://github.com/BerriAI/litellm/pull/13640)
- **[Azure AI](../../docs/providers/azure)**
- New Azure AI Flux Image Generation provider - [PR #13592](https://github.com/BerriAI/litellm/pull/13592), [Get Started](../../docs/providers/azure_ai_img)
- Fixed Content-Type header for image generation - [PR #13584](https://github.com/BerriAI/litellm/pull/13584)
- **[CometAPI](../../docs/providers/comet)**
- New provider support with chat completions and streaming - [PR #13458](https://github.com/BerriAI/litellm/pull/13458)
- **[SambaNova](../../docs/providers/sambanova)**
- Added embedding model support - [PR #13308](https://github.com/BerriAI/litellm/pull/13308), [Get Started](../../docs/providers/sambanova#sambanova---embeddings)
- **[Vertex AI](../../docs/providers/vertex)**
- Added `/countTokens` endpoint support for Gemini CLI integration - [PR #13545](https://github.com/BerriAI/litellm/pull/13545)
- Token counter support for VertexAI models - [PR #13558](https://github.com/BerriAI/litellm/pull/13558)
- **[hosted_vllm](../../docs/providers/vllm)**
- Added `reasoning_effort` parameter support - [PR #13620](https://github.com/BerriAI/litellm/pull/13620), [Get Started](../../docs/providers/vllm#reasoning-effort)
#### Bugs
- **[OCI](../../docs/providers/oci)**
- Fixed streaming issues - [PR #13437](https://github.com/BerriAI/litellm/pull/13437)
- **[Ollama](../../docs/providers/ollama)**
- Fixed GPT-OSS streaming with 'thinking' field - [PR #13375](https://github.com/BerriAI/litellm/pull/13375)
- **[VolcEngine](../../docs/providers/volcengine)**
- Fixed thinking disabled parameter handling - [PR #13598](https://github.com/BerriAI/litellm/pull/13598)
- **[Streaming](../../docs/completion/stream)**
- Consistent 'finish_reason' chunk indexing - [PR #13560](https://github.com/BerriAI/litellm/pull/13560)
---
## LLM API Endpoints
#### Features
- **[/messages](../../docs/anthropic/messages)**
- Tool use arguments properly returned for non-anthropic models - [PR #13638](https://github.com/BerriAI/litellm/pull/13638)
#### Bugs
- **[Real-time API](../../docs/realtime)**
- Fixed endpoint for no intent scenarios - [PR #13476](https://github.com/BerriAI/litellm/pull/13476)
- **[Responses API](../../docs/response_api)**
- Fixed `stream=True` + `background=True` with Responses API - [PR #13654](https://github.com/BerriAI/litellm/pull/13654)
---
## [MCP Gateway](../../docs/mcp)
#### Features
- **Access Control & Configuration**
- Enhanced MCPServerManager with access groups and description support - [PR #13549](https://github.com/BerriAI/litellm/pull/13549)
#### Bugs
- **Authentication**
- Fixed MCP gateway key authentication - [PR #13630](https://github.com/BerriAI/litellm/pull/13630)
[Read More](../../docs/mcp)
---
## Management Endpoints / UI
#### Features
- **Team Management**
- Team Member Rate Limits implementation - [PR #13601](https://github.com/BerriAI/litellm/pull/13601)
- JWT authentication support for team member rate limits - [PR #13601](https://github.com/BerriAI/litellm/pull/13601)
- Show team member TPM/RPM limits in UI - [PR #13662](https://github.com/BerriAI/litellm/pull/13662)
- Allow editing team member RPM/TPM limits - [PR #13669](https://github.com/BerriAI/litellm/pull/13669)
- Allow unsetting TPM and RPM in Teams Settings - [PR #13430](https://github.com/BerriAI/litellm/pull/13430)
- Team Member Permissions Page access column changes - [PR #13145](https://github.com/BerriAI/litellm/pull/13145)
- **Key Management**
- Display errors from backend on the UI Keys page - [PR #13435](https://github.com/BerriAI/litellm/pull/13435)
- Added confirmation modal before deleting keys - [PR #13655](https://github.com/BerriAI/litellm/pull/13655)
- Support for `user` parameter in LiteLLM SDK to Proxy communication - [PR #13555](https://github.com/BerriAI/litellm/pull/13555)
- **UI Improvements**
- Fixed internal users table overflow - [PR #12736](https://github.com/BerriAI/litellm/pull/12736)
- Enhanced chart readability with short-form notation for large numbers - [PR #12370](https://github.com/BerriAI/litellm/pull/12370)
- Fixed image overflow in LiteLLM model display - [PR #13639](https://github.com/BerriAI/litellm/pull/13639)
- Removed ambiguous network response errors - [PR #13582](https://github.com/BerriAI/litellm/pull/13582)
- **Credentials**
- Added CredentialDeleteModal component and integration with CredentialsPanel - [PR #13550](https://github.com/BerriAI/litellm/pull/13550)
- **Admin & Permissions**
- Allow routes for admin viewer - [PR #13588](https://github.com/BerriAI/litellm/pull/13588)
#### Bugs
- **SCIM Integration**
- Fixed SCIM Team Memberships metadata handling - [PR #13553](https://github.com/BerriAI/litellm/pull/13553)
- **Authentication**
- Fixed incorrect key info endpoint - [PR #13633](https://github.com/BerriAI/litellm/pull/13633)
---
## Logging / Guardrail Integrations
#### Features
- **[Langfuse OTEL](../../docs/proxy/logging#langfuse)**
- Added key/team logging for Langfuse OTEL Logger - [PR #13512](https://github.com/BerriAI/litellm/pull/13512)
- Fixed LangfuseOtelSpanAttributes constants to match expected values - [PR #13659](https://github.com/BerriAI/litellm/pull/13659)
- **[MLflow](../../docs/proxy/logging#mlflow)**
- Updated MLflow logger usage span attributes - [PR #13561](https://github.com/BerriAI/litellm/pull/13561)
#### Bugs
- **Security**
- Hide sensitive data in `/model/info` - azure entra client_secret - [PR #13577](https://github.com/BerriAI/litellm/pull/13577)
- Fixed trivy/secrets false positives - [PR #13631](https://github.com/BerriAI/litellm/pull/13631)
---
## Performance / Loadbalancing / Reliability improvements
#### Features
- **HTTP Performance**
- New 'EXPERIMENTAL_OPENAI_BASE_LLM_HTTP_HANDLER' flag for +100 RPS improvement on OpenAI calls - [PR #13625](https://github.com/BerriAI/litellm/pull/13625)
- **Database Monitoring**
- Added DB metrics to Prometheus - [PR #13626](https://github.com/BerriAI/litellm/pull/13626)
- **Error Handling**
- Added safe divide by 0 protection to prevent crashes - [PR #13624](https://github.com/BerriAI/litellm/pull/13624)
#### Bugs
- **Dependencies**
- Updated boto3 to 1.36.0 and aioboto3 to 13.4.0 - [PR #13665](https://github.com/BerriAI/litellm/pull/13665)
---
## General Proxy Improvements
#### Features
- **Database**
- Removed redundant `use_prisma_migrate` flag - now default - [PR #13555](https://github.com/BerriAI/litellm/pull/13555)
- **LLM Translation**
- Added model ID check - [PR #13507](https://github.com/BerriAI/litellm/pull/13507)
- Refactored Anthropic configurations and added support for `anthropic_beta` headers - [PR #13590](https://github.com/BerriAI/litellm/pull/13590)
---
## New Contributors
* @TensorNull made their first contribution in [PR #13458](https://github.com/BerriAI/litellm/pull/13458)
* @MajorD00m made their first contribution in [PR #13577](https://github.com/BerriAI/litellm/pull/13577)
* @VerunicaM made their first contribution in [PR #13584](https://github.com/BerriAI/litellm/pull/13584)
* @huangyafei made their first contribution in [PR #13607](https://github.com/BerriAI/litellm/pull/13607)
* @TomeHirata made their first contribution in [PR #13561](https://github.com/BerriAI/litellm/pull/13561)
* @willfinnigan made their first contribution in [PR #13659](https://github.com/BerriAI/litellm/pull/13659)
* @dcbark01 made their first contribution in [PR #13633](https://github.com/BerriAI/litellm/pull/13633)
* @javacruft made their first contribution in [PR #13631](https://github.com/BerriAI/litellm/pull/13631)
---
## **[Full Changelog](https://github.com/BerriAI/litellm/compare/v1.75.5-stable.rc-draft...v1.75.8-nightly)**

12
docs/my-website/sidebars.js
View file

@ -40,6 +40,7 @@ const sidebars = {
"proxy/guardrails/guardrails_ai",
"proxy/guardrails/lakera_ai",
"proxy/guardrails/model_armor",
"proxy/guardrails/noma_security",
"proxy/guardrails/openai_moderation",
"proxy/guardrails/pangea",
"proxy/guardrails/pillar_security",
@ -108,6 +109,7 @@ const sidebars = {
type: "category",
label: "Setup & Deployment",
items: [
"proxy/quick_start",
"proxy/deploy",
"proxy/prod",
"proxy/cli",
@ -372,7 +374,14 @@ const sidebars = {
"providers/azure/azure_embedding",
]
},
"providers/azure_ai",
{
type: "category",
label: "Azure AI",
items: [
"providers/azure_ai",
"providers/azure_ai_img",
]
},
{
type: "category",
label: "Vertex AI",
@ -484,6 +493,7 @@ const sidebars = {
"guides/finetuned_models",
"guides/security_settings",
"completion/audio",
"completion/image_generation_chat",
"completion/web_search",
"completion/document_understanding",
"completion/vision",

16
enterprise/litellm_enterprise/proxy/management_endpoints/internal_user_endpoints.py
View file

@ -2,6 +2,8 @@
Enterprise internal user management endpoints
"""
import os
from fastapi import APIRouter, Depends, HTTPException
from litellm.proxy._types import UserAPIKeyAuth
@ -21,7 +23,7 @@ async def available_enterprise_users(
"""
For keys with `max_users` set, return the list of users that are allowed to use the key.
"""
from litellm.proxy._types import CommonProxyErrors
from litellm.proxy._types import CommonProxyErrors, EnterpriseLicenseData
from litellm.proxy.proxy_server import (
premium_user,
premium_user_data,
@ -34,10 +36,14 @@ async def available_enterprise_users(
detail={"error": CommonProxyErrors.db_not_connected_error.value},
)
if premium_user is None:
raise HTTPException(
status_code=500, detail={"error": CommonProxyErrors.not_premium_user.value}
)
if not premium_user:
# check if SSO is enabled - show 5 user limit
from litellm.proxy.auth.auth_utils import _has_user_setup_sso
if _has_user_setup_sso():
premium_user_data = EnterpriseLicenseData(
max_users=5,
)
# Count number of rows in LiteLLM_UserTable
user_count = await prisma_client.db.litellm_usertable.count()

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.2.17-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.2.17.tar.gz vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.2.18-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.2.18.tar.gz vendored Normal file
View file

Binary file not shown.

2
litellm-proxy-extras/litellm_proxy_extras/utils.py
View file

@ -298,7 +298,7 @@ class ProxyExtrasDBManager:
and "database schema is not empty" in e.stderr
):
logger.info(
"Database schema is not empty, creating baseline migration"
"Database schema is not empty, creating baseline migration. In read-only file system, please set an environment variable `LITELLM_MIGRATION_DIR` to a writable directory to enable migrations. Learn more - https://docs.litellm.ai/docs/proxy/prod#read-only-file-system"
)
ProxyExtrasDBManager._create_baseline_migration(schema_path)
logger.info(

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-proxy-extras"
version = "0.2.16"
version = "0.2.18"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
authors = ["BerriAI"]
readme = "README.md"
@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.2.16"
version = "0.2.18"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-proxy-extras==",

468
litellm/__init__.py
View file

@ -146,6 +146,7 @@ _custom_logger_compatible_callbacks_literal = Literal[
"vector_store_pre_call_hook",
"dotprompt",
]
configured_cold_storage_logger: Optional[_custom_logger_compatible_callbacks_literal] = None
logged_real_time_event_types: Optional[Union[List[str], Literal["*"]]] = None
_known_custom_logger_compatible_callbacks: List = list(
get_args(_custom_logger_compatible_callbacks_literal)
@ -303,7 +304,6 @@ default_in_memory_ttl: Optional[float] = None
default_redis_ttl: Optional[float] = None
default_redis_batch_cache_expiry: Optional[float] = None
model_alias_map: Dict[str, str] = {}
model_group_alias_map: Dict[str, str] = {}
model_group_settings: Optional["ModelGroupSettings"] = None
max_budget: float = 0.0 # set the max budget across all providers
budget_duration: Optional[str] = (
@ -467,78 +467,81 @@ BEDROCK_CONVERSE_MODELS = [
]
####### COMPLETION MODELS ###################
open_ai_chat_completion_models: List = []
open_ai_text_completion_models: List = []
cohere_models: List = []
cohere_chat_models: List = []
mistral_chat_models: List = []
text_completion_codestral_models: List = []
anthropic_models: List = []
openrouter_models: List = []
datarobot_models: List = []
vertex_language_models: List = []
vertex_vision_models: List = []
vertex_chat_models: List = []
vertex_code_chat_models: List = []
vertex_ai_image_models: List = []
vertex_text_models: List = []
vertex_code_text_models: List = []
vertex_embedding_models: List = []
vertex_anthropic_models: List = []
vertex_llama3_models: List = []
vertex_ai_ai21_models: List = []
vertex_mistral_models: List = []
ai21_models: List = []
ai21_chat_models: List = []
nlp_cloud_models: List = []
aleph_alpha_models: List = []
bedrock_models: List = []
bedrock_converse_models: List = BEDROCK_CONVERSE_MODELS
fireworks_ai_models: List = []
fireworks_ai_embedding_models: List = []
deepinfra_models: List = []
perplexity_models: List = []
watsonx_models: List = []
gemini_models: List = []
xai_models: List = []
deepseek_models: List = []
azure_ai_models: List = []
jina_ai_models: List = []
voyage_models: List = []
infinity_models: List = []
databricks_models: List = []
cloudflare_models: List = []
codestral_models: List = []
friendliai_models: List = []
featherless_ai_models: List = []
palm_models: List = []
groq_models: List = []
azure_models: List = []
azure_text_models: List = []
anyscale_models: List = []
cerebras_models: List = []
galadriel_models: List = []
sambanova_models: List = []
sambanova_embedding_models: List = []
novita_models: List = []
assemblyai_models: List = []
snowflake_models: List = []
gradient_ai_models: List = []
llama_models: List = []
nscale_models: List = []
nebius_models: List = []
nebius_embedding_models: List = []
deepgram_models: List = []
elevenlabs_models: List = []
dashscope_models: List = []
moonshot_models: List = []
v0_models: List = []
morph_models: List = []
lambda_ai_models: List = []
hyperbolic_models: List = []
recraft_models: List = []
cometapi_models: List = []
oci_models: List = []
from typing import Set
open_ai_chat_completion_models: Set = set()
open_ai_text_completion_models: Set = set()
cohere_models: Set = set()
cohere_chat_models: Set = set()
mistral_chat_models: Set = set()
text_completion_codestral_models: Set = set()
anthropic_models: Set = set()
openrouter_models: Set = set()
datarobot_models: Set = set()
vertex_language_models: Set = set()
vertex_vision_models: Set = set()
vertex_chat_models: Set = set()
vertex_code_chat_models: Set = set()
vertex_ai_image_models: Set = set()
vertex_text_models: Set = set()
vertex_code_text_models: Set = set()
vertex_embedding_models: Set = set()
vertex_anthropic_models: Set = set()
vertex_llama3_models: Set = set()
vertex_deepseek_models: Set = set()
vertex_ai_ai21_models: Set = set()
vertex_mistral_models: Set = set()
ai21_models: Set = set()
ai21_chat_models: Set = set()
nlp_cloud_models: Set = set()
aleph_alpha_models: Set = set()
bedrock_models: Set = set()
bedrock_converse_models: Set = set(BEDROCK_CONVERSE_MODELS)
fireworks_ai_models: Set = set()
fireworks_ai_embedding_models: Set = set()
deepinfra_models: Set = set()
perplexity_models: Set = set()
watsonx_models: Set = set()
gemini_models: Set = set()
xai_models: Set = set()
deepseek_models: Set = set()
azure_ai_models: Set = set()
jina_ai_models: Set = set()
voyage_models: Set = set()
infinity_models: Set = set()
databricks_models: Set = set()
cloudflare_models: Set = set()
codestral_models: Set = set()
friendliai_models: Set = set()
featherless_ai_models: Set = set()
palm_models: Set = set()
groq_models: Set = set()
azure_models: Set = set()
azure_text_models: Set = set()
anyscale_models: Set = set()
cerebras_models: Set = set()
galadriel_models: Set = set()
sambanova_models: Set = set()
sambanova_embedding_models: Set = set()
novita_models: Set = set()
assemblyai_models: Set = set()
snowflake_models: Set = set()
gradient_ai_models: Set = set()
llama_models: Set = set()
nscale_models: Set = set()
nebius_models: Set = set()
nebius_embedding_models: Set = set()
aiml_models: Set = set()
deepgram_models: Set = set()
elevenlabs_models: Set = set()
dashscope_models: Set = set()
moonshot_models: Set = set()
v0_models: Set = set()
morph_models: Set = set()
lambda_ai_models: Set = set()
hyperbolic_models: Set = set()
recraft_models: Set = set()
cometapi_models: Set = set()
oci_models: Set = set()
def is_bedrock_pricing_only_model(key: str) -> bool:
@ -579,163 +582,168 @@ def add_known_models():
if value.get("litellm_provider") == "openai" and not is_openai_finetune_model(
key
):
open_ai_chat_completion_models.append(key)
open_ai_chat_completion_models.add(key)
elif value.get("litellm_provider") == "text-completion-openai":
open_ai_text_completion_models.append(key)
open_ai_text_completion_models.add(key)
elif value.get("litellm_provider") == "azure_text":
azure_text_models.append(key)
azure_text_models.add(key)
elif value.get("litellm_provider") == "cohere":
cohere_models.append(key)
cohere_models.add(key)
elif value.get("litellm_provider") == "cohere_chat":
cohere_chat_models.append(key)
cohere_chat_models.add(key)
elif value.get("litellm_provider") == "mistral":
mistral_chat_models.append(key)
mistral_chat_models.add(key)
elif value.get("litellm_provider") == "anthropic":
anthropic_models.append(key)
anthropic_models.add(key)
elif value.get("litellm_provider") == "empower":
empower_models.append(key)
empower_models.add(key)
elif value.get("litellm_provider") == "openrouter":
openrouter_models.append(key)
openrouter_models.add(key)
elif value.get("litellm_provider") == "datarobot":
datarobot_models.append(key)
datarobot_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-text-models":
vertex_text_models.append(key)
vertex_text_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-code-text-models":
vertex_code_text_models.append(key)
vertex_code_text_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-language-models":
vertex_language_models.append(key)
vertex_language_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-vision-models":
vertex_vision_models.append(key)
vertex_vision_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-chat-models":
vertex_chat_models.append(key)
vertex_chat_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-code-chat-models":
vertex_code_chat_models.append(key)
vertex_code_chat_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-embedding-models":
vertex_embedding_models.append(key)
vertex_embedding_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-anthropic_models":
key = key.replace("vertex_ai/", "")
vertex_anthropic_models.append(key)
vertex_anthropic_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-llama_models":
key = key.replace("vertex_ai/", "")
vertex_llama3_models.append(key)
vertex_llama3_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-deepseek_models":
key = key.replace("vertex_ai/", "")
vertex_deepseek_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-mistral_models":
key = key.replace("vertex_ai/", "")
vertex_mistral_models.append(key)
vertex_mistral_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-ai21_models":
key = key.replace("vertex_ai/", "")
vertex_ai_ai21_models.append(key)
vertex_ai_ai21_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-image-models":
key = key.replace("vertex_ai/", "")
vertex_ai_image_models.append(key)
vertex_ai_image_models.add(key)
elif value.get("litellm_provider") == "ai21":
if value.get("mode") == "chat":
ai21_chat_models.append(key)
ai21_chat_models.add(key)
else:
ai21_models.append(key)
ai21_models.add(key)
elif value.get("litellm_provider") == "nlp_cloud":
nlp_cloud_models.append(key)
nlp_cloud_models.add(key)
elif value.get("litellm_provider") == "aleph_alpha":
aleph_alpha_models.append(key)
aleph_alpha_models.add(key)
elif value.get(
"litellm_provider"
) == "bedrock" and not is_bedrock_pricing_only_model(key):
bedrock_models.append(key)
bedrock_models.add(key)
elif value.get("litellm_provider") == "bedrock_converse":
bedrock_converse_models.append(key)
bedrock_converse_models.add(key)
elif value.get("litellm_provider") == "deepinfra":
deepinfra_models.append(key)
deepinfra_models.add(key)
elif value.get("litellm_provider") == "perplexity":
perplexity_models.append(key)
perplexity_models.add(key)
elif value.get("litellm_provider") == "watsonx":
watsonx_models.append(key)
watsonx_models.add(key)
elif value.get("litellm_provider") == "gemini":
gemini_models.append(key)
gemini_models.add(key)
elif value.get("litellm_provider") == "fireworks_ai":
# ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params.
if "-to-" not in key and "fireworks-ai-default" not in key:
fireworks_ai_models.append(key)
fireworks_ai_models.add(key)
elif value.get("litellm_provider") == "fireworks_ai-embedding-models":
# ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params.
if "-to-" not in key:
fireworks_ai_embedding_models.append(key)
fireworks_ai_embedding_models.add(key)
elif value.get("litellm_provider") == "text-completion-codestral":
text_completion_codestral_models.append(key)
text_completion_codestral_models.add(key)
elif value.get("litellm_provider") == "xai":
xai_models.append(key)
xai_models.add(key)
elif value.get("litellm_provider") == "deepseek":
deepseek_models.append(key)
deepseek_models.add(key)
elif value.get("litellm_provider") == "meta_llama":
llama_models.append(key)
llama_models.add(key)
elif value.get("litellm_provider") == "nscale":
nscale_models.append(key)
nscale_models.add(key)
elif value.get("litellm_provider") == "azure_ai":
azure_ai_models.append(key)
azure_ai_models.add(key)
elif value.get("litellm_provider") == "voyage":
voyage_models.append(key)
voyage_models.add(key)
elif value.get("litellm_provider") == "infinity":
infinity_models.append(key)
infinity_models.add(key)
elif value.get("litellm_provider") == "databricks":
databricks_models.append(key)
databricks_models.add(key)
elif value.get("litellm_provider") == "cloudflare":
cloudflare_models.append(key)
cloudflare_models.add(key)
elif value.get("litellm_provider") == "codestral":
codestral_models.append(key)
codestral_models.add(key)
elif value.get("litellm_provider") == "friendliai":
friendliai_models.append(key)
friendliai_models.add(key)
elif value.get("litellm_provider") == "palm":
palm_models.append(key)
palm_models.add(key)
elif value.get("litellm_provider") == "groq":
groq_models.append(key)
groq_models.add(key)
elif value.get("litellm_provider") == "azure":
azure_models.append(key)
azure_models.add(key)
elif value.get("litellm_provider") == "anyscale":
anyscale_models.append(key)
anyscale_models.add(key)
elif value.get("litellm_provider") == "cerebras":
cerebras_models.append(key)
cerebras_models.add(key)
elif value.get("litellm_provider") == "galadriel":
galadriel_models.append(key)
galadriel_models.add(key)
elif value.get("litellm_provider") == "sambanova":
sambanova_models.append(key)
sambanova_models.add(key)
elif value.get("litellm_provider") == "sambanova-embedding-models":
sambanova_embedding_models.append(key)
sambanova_embedding_models.add(key)
elif value.get("litellm_provider") == "novita":
novita_models.append(key)
novita_models.add(key)
elif value.get("litellm_provider") == "nebius-chat-models":
nebius_models.append(key)
nebius_models.add(key)
elif value.get("litellm_provider") == "nebius-embedding-models":
nebius_embedding_models.append(key)
nebius_embedding_models.add(key)
elif value.get("litellm_provider") == "aiml":
aiml_models.add(key)
elif value.get("litellm_provider") == "assemblyai":
assemblyai_models.append(key)
assemblyai_models.add(key)
elif value.get("litellm_provider") == "jina_ai":
jina_ai_models.append(key)
jina_ai_models.add(key)
elif value.get("litellm_provider") == "snowflake":
snowflake_models.append(key)
snowflake_models.add(key)
elif value.get("litellm_provider") == "gradient_ai":
gradient_ai_models.append(key)
gradient_ai_models.add(key)
elif value.get("litellm_provider") == "featherless_ai":
featherless_ai_models.append(key)
featherless_ai_models.add(key)
elif value.get("litellm_provider") == "deepgram":
deepgram_models.append(key)
deepgram_models.add(key)
elif value.get("litellm_provider") == "elevenlabs":
elevenlabs_models.append(key)
elevenlabs_models.add(key)
elif value.get("litellm_provider") == "dashscope":
dashscope_models.append(key)
dashscope_models.add(key)
elif value.get("litellm_provider") == "moonshot":
moonshot_models.append(key)
moonshot_models.add(key)
elif value.get("litellm_provider") == "v0":
v0_models.append(key)
v0_models.add(key)
elif value.get("litellm_provider") == "morph":
morph_models.append(key)
morph_models.add(key)
elif value.get("litellm_provider") == "lambda_ai":
lambda_ai_models.append(key)
lambda_ai_models.add(key)
elif value.get("litellm_provider") == "hyperbolic":
hyperbolic_models.append(key)
hyperbolic_models.add(key)
elif value.get("litellm_provider") == "recraft":
recraft_models.append(key)
recraft_models.add(key)
elif value.get("litellm_provider") == "cometapi":
cometapi_models.append(key)
cometapi_models.add(key)
elif value.get("litellm_provider") == "oci":
oci_models.append(key)
oci_models.add(key)
add_known_models()
@ -765,68 +773,68 @@ ollama_models = ["llama2"]
maritalk_models = ["maritalk"]
model_list = (
model_list = list(
open_ai_chat_completion_models
+ open_ai_text_completion_models
+ cohere_models
+ cohere_chat_models
+ anthropic_models
+ replicate_models
+ openrouter_models
+ datarobot_models
+ huggingface_models
+ vertex_chat_models
+ vertex_text_models
+ ai21_models
+ ai21_chat_models
+ together_ai_models
+ baseten_models
+ aleph_alpha_models
+ nlp_cloud_models
+ ollama_models
+ bedrock_models
+ deepinfra_models
+ perplexity_models
+ maritalk_models
+ vertex_language_models
+ watsonx_models
+ gemini_models
+ text_completion_codestral_models
+ xai_models
+ deepseek_models
+ azure_ai_models
+ voyage_models
+ infinity_models
+ databricks_models
+ cloudflare_models
+ codestral_models
+ friendliai_models
+ palm_models
+ groq_models
+ azure_models
+ anyscale_models
+ cerebras_models
+ galadriel_models
+ sambanova_models
+ azure_text_models
+ novita_models
+ assemblyai_models
+ jina_ai_models
+ snowflake_models
+ gradient_ai_models
+ llama_models
+ featherless_ai_models
+ nscale_models
+ deepgram_models
+ elevenlabs_models
+ dashscope_models
+ moonshot_models
+ v0_models
+ morph_models
+ lambda_ai_models
+ recraft_models
+ cometapi_models
+ oci_models
| open_ai_text_completion_models
| cohere_models
| cohere_chat_models
| anthropic_models
| set(replicate_models)
| openrouter_models
| datarobot_models
| set(huggingface_models)
| vertex_chat_models
| vertex_text_models
| ai21_models
| ai21_chat_models
| set(together_ai_models)
| set(baseten_models)
| aleph_alpha_models
| nlp_cloud_models
| set(ollama_models)
| bedrock_models
| deepinfra_models
| perplexity_models
| set(maritalk_models)
| vertex_language_models
| watsonx_models
| gemini_models
| text_completion_codestral_models
| xai_models
| deepseek_models
| azure_ai_models
| voyage_models
| infinity_models
| databricks_models
| cloudflare_models
| codestral_models
| friendliai_models
| palm_models
| groq_models
| azure_models
| anyscale_models
| cerebras_models
| galadriel_models
| sambanova_models
| azure_text_models
| novita_models
| assemblyai_models
| jina_ai_models
| snowflake_models
| gradient_ai_models
| llama_models
| featherless_ai_models
| nscale_models
| deepgram_models
| elevenlabs_models
| dashscope_models
| moonshot_models
| v0_models
| morph_models
| lambda_ai_models
| recraft_models
| cometapi_models
| oci_models
)
model_list_set = set(model_list)
@ -835,9 +843,9 @@ provider_list: List[Union[LlmProviders, str]] = list(LlmProviders)
models_by_provider: dict = {
"openai": open_ai_chat_completion_models + open_ai_text_completion_models,
"openai": open_ai_chat_completion_models | open_ai_text_completion_models,
"text-completion-openai": open_ai_text_completion_models,
"cohere": cohere_models + cohere_chat_models,
"cohere": cohere_models | cohere_chat_models,
"cohere_chat": cohere_chat_models,
"anthropic": anthropic_models,
"replicate": replicate_models,
@ -846,13 +854,9 @@ models_by_provider: dict = {
"baseten": baseten_models,
"openrouter": openrouter_models,
"datarobot": datarobot_models,
"vertex_ai": vertex_chat_models
+ vertex_text_models
+ vertex_anthropic_models
+ vertex_vision_models
+ vertex_language_models,
"vertex_ai": vertex_chat_models | vertex_text_models | vertex_anthropic_models | vertex_vision_models | vertex_language_models | vertex_deepseek_models,
"ai21": ai21_models,
"bedrock": bedrock_models + bedrock_converse_models,
"bedrock": bedrock_models | bedrock_converse_models,
"petals": petals_models,
"ollama": ollama_models,
"ollama_chat": ollama_models,
@ -861,7 +865,7 @@ models_by_provider: dict = {
"maritalk": maritalk_models,
"watsonx": watsonx_models,
"gemini": gemini_models,
"fireworks_ai": fireworks_ai_models + fireworks_ai_embedding_models,
"fireworks_ai": fireworks_ai_models | fireworks_ai_embedding_models,
"aleph_alpha": aleph_alpha_models,
"text-completion-codestral": text_completion_codestral_models,
"xai": xai_models,
@ -877,14 +881,15 @@ models_by_provider: dict = {
"friendliai": friendliai_models,
"palm": palm_models,
"groq": groq_models,
"azure": azure_models + azure_text_models,
"azure": azure_models | azure_text_models,
"azure_text": azure_text_models,
"anyscale": anyscale_models,
"cerebras": cerebras_models,
"galadriel": galadriel_models,
"sambanova": sambanova_models + sambanova_embedding_models,
"sambanova": sambanova_models | sambanova_embedding_models,
"novita": novita_models,
"nebius": nebius_models + nebius_embedding_models,
"nebius": nebius_models | nebius_embedding_models,
"aiml": aiml_models,
"assemblyai": assemblyai_models,
"jina_ai": jina_ai_models,
"snowflake": snowflake_models,
@ -931,12 +936,12 @@ longer_context_model_fallback_dict: dict = {
all_embedding_models = (
open_ai_embedding_models
+ cohere_embedding_models
+ bedrock_embedding_models
+ vertex_embedding_models
+ fireworks_ai_embedding_models
+ nebius_embedding_models
+ sambanova_embedding_models
| set(cohere_embedding_models)
| set(bedrock_embedding_models)
| vertex_embedding_models
| fireworks_ai_embedding_models
| nebius_embedding_models
| sambanova_embedding_models
)
####### IMAGE GENERATION MODELS ###################
@ -1034,6 +1039,7 @@ from .llms.cohere.rerank_v2.transformation import CohereRerankV2Config
from .llms.azure_ai.rerank.transformation import AzureAIRerankConfig
from .llms.infinity.rerank.transformation import InfinityRerankConfig
from .llms.jina_ai.rerank.transformation import JinaAIRerankConfig
from .llms.deepinfra.rerank.transformation import DeepinfraRerankConfig
from .llms.clarifai.chat.transformation import ClarifaiConfig
from .llms.ai21.chat.transformation import AI21ChatConfig, AI21ChatConfig as AI21Config
from .llms.meta_llama.chat.transformation import LlamaAPIConfig
@ -1041,7 +1047,7 @@ from .llms.anthropic.experimental_pass_through.messages.transformation import (
AnthropicMessagesConfig,
)
from .llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import (
AmazonAnthropicClaude3MessagesConfig,
AmazonAnthropicClaudeMessagesConfig,
)
from .llms.together_ai.chat import TogetherAIConfig
from .llms.together_ai.completion.transformation import TogetherAITextCompletionConfig
@ -1101,7 +1107,7 @@ from .llms.bedrock.chat.invoke_transformations.anthropic_claude2_transformation
AmazonAnthropicConfig,
)
from .llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import (
AmazonAnthropicClaude3Config,
AmazonAnthropicClaudeConfig,
)
from .llms.bedrock.chat.invoke_transformations.amazon_cohere_transformation import (
AmazonCohereConfig,
@ -1145,12 +1151,15 @@ from .llms.topaz.image_variations.transformation import TopazImageVariationConfi
from litellm.llms.openai.completion.transformation import OpenAITextCompletionConfig
from .llms.groq.chat.transformation import GroqChatConfig
from .llms.voyage.embedding.transformation import VoyageEmbeddingConfig
from .llms.voyage.embedding.transformation_contextual import VoyageContextualEmbeddingConfig
from .llms.infinity.embedding.transformation import InfinityEmbeddingConfig
from .llms.azure_ai.chat.transformation import AzureAIStudioConfig
from .llms.mistral.chat.transformation import MistralConfig
from .llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from .llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig
from .llms.azure.responses.o_series_transformation import AzureOpenAIOSeriesResponsesAPIConfig
from .llms.azure.responses.o_series_transformation import (
AzureOpenAIOSeriesResponsesAPIConfig,
)
from .llms.openai.chat.o_series_transformation import (
OpenAIOSeriesConfig as OpenAIO1Config, # maintain backwards compatibility
OpenAIOSeriesConfig,
@ -1158,6 +1167,7 @@ from .llms.openai.chat.o_series_transformation import (
from .llms.snowflake.chat.transformation import SnowflakeConfig
from .llms.gradient_ai.chat.transformation import GradientAIConfig
openaiOSeriesConfig = OpenAIOSeriesConfig()
from .llms.openai.chat.gpt_transformation import (
OpenAIGPTConfig,
@ -1188,6 +1198,7 @@ nvidiaNimEmbeddingConfig = NvidiaNimEmbeddingConfig()
from .llms.featherless_ai.chat.transformation import FeatherlessAIConfig
from .llms.cerebras.chat import CerebrasConfig
from .llms.baseten.chat import BasetenConfig
from .llms.sambanova.chat import SambanovaConfig
from .llms.sambanova.embedding.transformation import SambaNovaEmbeddingConfig
from .llms.ai21.chat.transformation import AI21ChatConfig
@ -1203,6 +1214,7 @@ from .llms.friendliai.chat.transformation import FriendliaiChatConfig
from .llms.jina_ai.embedding.transformation import JinaAIEmbeddingConfig
from .llms.xai.chat.transformation import XAIChatConfig
from .llms.xai.common_utils import XAIModelInfo
from .llms.aiml.chat.transformation import AIMLChatConfig
from .llms.volcengine import VolcEngineConfig
from .llms.codestral.completion.transformation import CodestralTextCompletionConfig
from .llms.azure.azure import (

6
litellm/_logging.py
View file

@ -108,6 +108,7 @@ verbose_router_logger.addHandler(handler)
verbose_proxy_logger.addHandler(handler)
verbose_logger.addHandler(handler)
def _suppress_loggers():
"""Suppress noisy loggers at INFO level"""
# Suppress httpx request logging at INFO level
@ -120,6 +121,7 @@ def _suppress_loggers():
apscheduler_scheduler_logger = logging.getLogger("apscheduler.scheduler")
apscheduler_scheduler_logger.setLevel(logging.WARNING)
# Call the suppression function
_suppress_loggers()
@ -187,6 +189,4 @@ def _is_debugging_on() -> bool:
"""
Returns True if debugging is on
"""
if verbose_logger.isEnabledFor(logging.DEBUG) or set_verbose is True:
return True
return False
return verbose_logger.isEnabledFor(logging.DEBUG) or set_verbose is True

4
litellm/caching/caching.py
View file

@ -774,11 +774,9 @@ class Cache:
"""
Internal method to check if the cache type supports async get/set operations
Only S3 Cache Does NOT support async operations
All cache types now support async operations
"""
if self.type and self.type == LiteLLMCacheType.S3:
return False
return True

8
litellm/caching/caching_handler.py
View file

@ -599,7 +599,7 @@ class LLMCachingHandler:
cached_result = await litellm.cache.async_get_cache(
dynamic_cache_object=self.dual_cache, **new_kwargs
)
else: # for s3 caching. [NOT RECOMMENDED IN PROD - this will slow down responses since boto3 is sync]
else: # fallback for caches that don't support async
cached_result = litellm.cache.get_cache(
dynamic_cache_object=self.dual_cache, **new_kwargs
)
@ -806,12 +806,6 @@ class LLMCachingHandler:
result, dynamic_cache_object=self.dual_cache, **new_kwargs
)
)
elif isinstance(litellm.cache.cache, S3Cache):
threading.Thread(
target=litellm.cache.add_cache,
args=(result,),
kwargs=new_kwargs,
).start()
else:
asyncio.create_task(
litellm.cache.async_add_cache(

15
litellm/caching/in_memory_cache.py
View file

@ -112,14 +112,15 @@ class InMemoryCache(BaseCache):
- 3. the size of in-memory cache is bounded
"""
for key in list(self.ttl_dict.keys()):
if self._is_key_expired(key):
self._remove_key(key)
current_time = time.time()
expired_keys = [key for key, ttl in self.ttl_dict.items() if current_time > ttl]
for key in expired_keys:
self._remove_key(key)
# de-reference the removed item
# https://www.geeksforgeeks.org/diagnosing-and-fixing-memory-leaks-in-python/
# One of the most common causes of memory leaks in Python is the retention of objects that are no longer being used.
# This can occur when an object is referenced by another object, but the reference is never removed.
# de-reference the removed item
# https://www.geeksforgeeks.org/diagnosing-and-fixing-memory-leaks-in-python/
# One of the most common causes of memory leaks in Python is the retention of objects that are no longer being used.
# This can occur when an object is referenced by another object, but the reference is never removed.
def allow_ttl_override(self, key: str) -> bool:
"""

56
litellm/caching/s3_cache.py
View file

@ -1,18 +1,19 @@
"""
S3 Cache implementation
WARNING: DO NOT USE THIS IN PRODUCTION - This is not ASYNC
Has 4 methods:
- set_cache
- get_cache
- async_set_cache
- async_get_cache
- async_set_cache (uses run_in_executor)
- async_get_cache (uses run_in_executor)
"""
import ast
import asyncio
import json
from functools import partial
from typing import Optional
from datetime import datetime, timezone, timedelta
from litellm._logging import print_verbose, verbose_logger
@ -55,21 +56,23 @@ class S3Cache(BaseCache):
**kwargs,
)
def _to_s3_key(self, key: str) -> str:
"""Convert cache key to S3 key"""
return self.key_prefix + key.replace(":", "/")
def set_cache(self, key, value, **kwargs):
try:
print_verbose(f"LiteLLM SET Cache - S3. Key={key}. Value={value}")
ttl = kwargs.get("ttl", None)
# Convert value to JSON before storing in S3
serialized_value = json.dumps(value)
key = self.key_prefix + key
key = self._to_s3_key(key)
if ttl is not None:
cache_control = f"immutable, max-age={ttl}, s-maxage={ttl}"
import datetime
# Calculate expiration time
expiration_time = datetime.datetime.now() + ttl
expiration_time = datetime.now(timezone.utc) + timedelta(seconds=ttl)
# Upload the data to S3 with the calculated expiration time
self.s3_client.put_object(
Bucket=self.bucket_name,
@ -94,17 +97,26 @@ class S3Cache(BaseCache):
ContentDisposition=f'inline; filename="{key}.json"',
)
except Exception as e:
# NON blocking - notify users S3 is throwing an exception
print_verbose(f"S3 Caching: set_cache() - Got exception from S3: {e}")
async def async_set_cache(self, key, value, **kwargs):
self.set_cache(key=key, value=value, **kwargs)
"""
Asynchronously set cache using run_in_executor to avoid blocking the event loop.
Compatible with Python 3.8+.
"""
try:
verbose_logger.debug(f"Set ASYNC S3 Cache: Key={key}. Value={value}")
loop = asyncio.get_event_loop()
func = partial(self.set_cache, key, value, **kwargs)
await loop.run_in_executor(None, func)
except Exception as e:
verbose_logger.error(f"S3 Caching: async_set_cache() - Got exception from S3: {e}")
def get_cache(self, key, **kwargs):
import botocore
try:
key = self.key_prefix + key
key = self._to_s3_key(key)
print_verbose(f"Get S3 Cache: key: {key}")
# Download the data from S3
@ -113,6 +125,13 @@ class S3Cache(BaseCache):
)
if cached_response is not None:
if "Expires" in cached_response:
expires_time = cached_response['Expires']
current_time = datetime.now(expires_time.tzinfo)
if current_time > expires_time:
return None
# cached_response is in `b{} convert it to ModelResponse
cached_response = (
cached_response["Body"].read().decode("utf-8")
@ -138,13 +157,26 @@ class S3Cache(BaseCache):
return None
except Exception as e:
# NON blocking - notify users S3 is throwing an exception
verbose_logger.error(
f"S3 Caching: get_cache() - Got exception from S3: {e}"
)
async def async_get_cache(self, key, **kwargs):
return self.get_cache(key=key, **kwargs)
"""
Asynchronously get cache using run_in_executor to avoid blocking the event loop.
Compatible with Python 3.8+.
"""
try:
verbose_logger.debug(f"Get ASYNC S3 Cache: key: {key}")
loop = asyncio.get_event_loop()
func = partial(self.get_cache, key, **kwargs)
result = await loop.run_in_executor(None, func)
return result
except Exception as e:
verbose_logger.error(
f"S3 Caching: async_get_cache() - Got exception from S3: {e}"
)
return None
def flush_cache(self):
pass

112
litellm/constants.py
View file

@ -1,6 +1,9 @@
import os
from typing import List, Literal
AZURE_DEFAULT_RESPONSES_API_VERSION = str(
os.getenv("AZURE_DEFAULT_RESPONSES_API_VERSION", "preview")
)
ROUTER_MAX_FALLBACKS = int(os.getenv("ROUTER_MAX_FALLBACKS", 5))
DEFAULT_BATCH_SIZE = int(os.getenv("DEFAULT_BATCH_SIZE", 512))
DEFAULT_FLUSH_INTERVAL_SECONDS = int(os.getenv("DEFAULT_FLUSH_INTERVAL_SECONDS", 5))
@ -248,6 +251,7 @@ LITELLM_CHAT_PROVIDERS = [
"groq",
"nvidia_nim",
"cerebras",
"baseten",
"ai21_chat",
"volcengine",
"codestral",
@ -424,6 +428,7 @@ openai_compatible_providers: List = [
"groq",
"nvidia_nim",
"cerebras",
"baseten",
"sambanova",
"ai21_chat",
"ai21",
@ -457,6 +462,7 @@ openai_compatible_providers: List = [
"morph",
"lambda_ai",
"hyperbolic",
"aiml",
]
openai_text_completion_compatible_providers: List = (
[ # providers that support `/v1/completions`
@ -480,7 +486,7 @@ _openai_like_providers: List = [
"watsonx",
] # private helper. similar to openai but require some custom auth / endpoint handling, so can't use the openai sdk
# well supported replicate llms
replicate_models: List = [
replicate_models: set = set([
# llama replicate supported LLMs
"replicate/llama-2-70b-chat:2796ee9483c3fd7aa2e171d38f4ca12251a30609463dcfd4cd76703f22e96cdf",
"a16z-infra/llama-2-13b-chat:2a7f981751ec7fdf87b5b91ad4db53683a98082e9ff7bfd12c8cd5ea85980a52",
@ -493,9 +499,9 @@ replicate_models: List = [
# Others
"replicate/dolly-v2-12b:ef0e1aefc61f8e096ebe4db6b2bacc297daf2ef6899f0f7e001ec445893500e5",
"replit/replit-code-v1-3b:b84f4c074b807211cd75e3e8b1589b6399052125b4c27106e43d47189e8415ad",
]
])
clarifai_models: List = [
clarifai_models: set = set([
"clarifai/meta.Llama-3.Llama-3-8B-Instruct",
"clarifai/gcp.generate.gemma-1_1-7b-it",
"clarifai/mistralai.completion.mixtral-8x22B",
@ -559,10 +565,10 @@ clarifai_models: List = [
"clarifai/gcp.generate.gemini-1_5-pro",
"clarifai/gcp.generate.imagen-2",
"clarifai/salesforce.blip.general-english-image-caption-blip-2",
]
])
huggingface_models: List = [
huggingface_models: set = set([
"meta-llama/Llama-2-7b-hf",
"meta-llama/Llama-2-7b-chat-hf",
"meta-llama/Llama-2-13b-hf",
@ -575,13 +581,13 @@ huggingface_models: List = [
"meta-llama/Llama-2-13b-chat",
"meta-llama/Llama-2-70b",
"meta-llama/Llama-2-70b-chat",
] # these have been tested on extensively. But by default all text2text-generation and text-generation models are supported by liteLLM. - https://docs.litellm.ai/docs/providers
empower_models = [
]) # these have been tested on extensively. But by default all text2text-generation and text-generation models are supported by liteLLM. - https://docs.litellm.ai/docs/providers
empower_models = set([
"empower/empower-functions",
"empower/empower-functions-small",
]
])
together_ai_models: List = [
together_ai_models: set = set([
# llama llms - chat
"togethercomputer/llama-2-70b-chat",
# llama llms - language / instruct
@ -609,16 +615,17 @@ together_ai_models: List = [
"Austism/chronos-hermes-13b",
"upstage/SOLAR-0-70b-16bit",
"WizardLM/WizardLM-70B-V1.0",
] # supports all together ai models, just pass in the model id e.g. completion(model="together_computer/replit_code_3b",...)
])
# supports all together ai models, just pass in the model id e.g. completion(model="together_computer/replit_code_3b",...)
baseten_models: List = [
baseten_models: set = set([
"qvv0xeq",
"q841o8w",
"31dxrj3",
] # FALCON 7B # WizardLM # Mosaic ML
]) # FALCON 7B # WizardLM # Mosaic ML
featherless_ai_models: List = [
featherless_ai_models: set = set([
"featherless-ai/Qwerky-72B",
"featherless-ai/Qwerky-QwQ-32B",
"Qwen/Qwen2.5-72B-Instruct",
@ -628,24 +635,61 @@ featherless_ai_models: List = [
"mistralai/Mistral-Small-24B-Instruct-2501",
"mistralai/Mistral-Nemo-Instruct-2407",
"ProdeusUnity/Stellar-Odyssey-12b-v0.0",
]
])
nebius_models: List = [
nebius_models: set = set([
# deepseek models
"deepseek-ai/DeepSeek-R1-0528",
"deepseek-ai/DeepSeek-V3-0324",
"deepseek-ai/DeepSeek-V3",
"deepseek-ai/DeepSeek-R1",
"deepseek-ai/DeepSeek-R1-Distill-Llama-70B",
# google models
"google/gemma-2-2b-it",
"google/gemma-2-9b-it-fast",
# llama models
"meta-llama/Llama-3.3-70B-Instruct",
"meta-llama/Meta-Llama-3.1-70B-Instruct",
"meta-llama/Meta-Llama-3.1-8B-Instruct",
"meta-llama/Meta-Llama-3.1-405B-Instruct",
"NousResearch/Hermes-3-Llama-405B",
# microsoft models
"microsoft/phi-4",
# mistral models
"mistralai/Mistral-Nemo-Instruct-2407",
"mistralai/Devstral-Small-2505",
# moonshot models
"moonshotai/Kimi-K2-Instruct",
# nvidia models
"nvidia/Llama-3_1-Nemotron-Ultra-253B-v1",
"nvidia/Llama-3_3-Nemotron-Super-49B-v1",
# openai models
"openai/gpt-oss-120b",
"openai/gpt-oss-20b",
# qwen models
"Qwen/Qwen3-Coder-480B-A35B-Instruct",
"Qwen/Qwen3-235B-A22B-Instruct-2507",
"Qwen/Qwen3-235B-A22B",
"Qwen/Qwen3-30B-A3B-fast",
"Qwen/Qwen3-30B-A3B",
"Qwen/Qwen3-32B",
"Qwen/Qwen3-14B",
"nvidia/Llama-3_1-Nemotron-Ultra-253B-v1",
"deepseek-ai/DeepSeek-V3-0324",
"deepseek-ai/DeepSeek-V3-0324-fast",
"deepseek-ai/DeepSeek-R1",
"deepseek-ai/DeepSeek-R1-fast",
"meta-llama/Llama-3.3-70B-Instruct-fast",
"Qwen/Qwen2.5-32B-Instruct-fast",
"Qwen/Qwen2.5-Coder-32B-Instruct-fast",
]
"Qwen/Qwen3-4B-fast",
"Qwen/Qwen2.5-Coder-7B",
"Qwen/Qwen2.5-Coder-32B-Instruct",
"Qwen/Qwen2.5-72B-Instruct",
"Qwen/QwQ-32B",
"Qwen/Qwen3-30B-A3B-Thinking-2507",
"Qwen/Qwen3-30B-A3B-Instruct-2507",
# zai models
"zai-org/GLM-4.5",
"zai-org/GLM-4.5-Air",
# other models
"aaditya/Llama3-OpenBioLLM-70B",
"ProdeusUnity/Stellar-Odyssey-12b-v0.0",
"all-hands/openhands-lm-32b-v0.1",
])
dashscope_models: List = [
dashscope_models: set = set([
"qwen-turbo",
"qwen-plus",
"qwen-max",
@ -656,13 +700,13 @@ dashscope_models: List = [
"qwen3-235b-a22b",
"qwen3-32b",
"qwen3-30b-a3b",
]
])
nebius_embedding_models: List = [
nebius_embedding_models: set = set([
"BAAI/bge-en-icl",
"BAAI/bge-multilingual-gemma2",
"intfloat/e5-mistral-7b-instruct",
]
])
BEDROCK_INVOKE_PROVIDERS_LITERAL = Literal[
"cohere",
@ -676,8 +720,8 @@ BEDROCK_INVOKE_PROVIDERS_LITERAL = Literal[
"deepseek_r1",
]
open_ai_embedding_models: List = ["text-embedding-ada-002"]
cohere_embedding_models: List = [
open_ai_embedding_models: set = set(["text-embedding-ada-002"])
cohere_embedding_models: set = set([
"embed-v4.0",
"embed-english-v3.0",
"embed-english-light-v3.0",
@ -685,12 +729,12 @@ cohere_embedding_models: List = [
"embed-english-v2.0",
"embed-english-light-v2.0",
"embed-multilingual-v2.0",
]
bedrock_embedding_models: List = [
])
bedrock_embedding_models: set = set([
"amazon.titan-embed-text-v1",
"cohere.embed-english-v3",
"cohere.embed-multilingual-v3",
]
])
known_tokenizer_config = {
"mistralai/Mistral-7B-Instruct-v0.1": {

59
litellm/cost_calculator.py
View file

@ -32,9 +32,6 @@ from litellm.llms.azure.cost_calculation import (
from litellm.llms.bedrock.cost_calculation import (
cost_per_token as bedrock_cost_per_token,
)
from litellm.llms.bedrock.image.cost_calculator import (
cost_calculator as bedrock_image_cost_calculator,
)
from litellm.llms.databricks.cost_calculator import (
cost_per_token as databricks_cost_per_token,
)
@ -60,9 +57,6 @@ from litellm.llms.vertex_ai.cost_calculator import (
cost_per_token as google_cost_per_token,
)
from litellm.llms.vertex_ai.cost_calculator import cost_router as google_cost_router
from litellm.llms.vertex_ai.image_generation.cost_calculator import (
cost_calculator as vertex_ai_image_cost_calculator,
)
from litellm.responses.utils import ResponseAPILoggingUtils
from litellm.types.llms.openai import (
HttpxBinaryResponseContent,
@ -768,50 +762,15 @@ def completion_cost( # noqa: PLR0915
)
if CostCalculatorUtils._call_type_has_image_response(call_type):
### IMAGE GENERATION COST CALCULATION ###
if custom_llm_provider == "vertex_ai":
if isinstance(completion_response, ImageResponse):
return vertex_ai_image_cost_calculator(
model=model,
image_response=completion_response,
)
elif custom_llm_provider == "bedrock":
if isinstance(completion_response, ImageResponse):
return bedrock_image_cost_calculator(
model=model,
size=size,
image_response=completion_response,
optional_params=optional_params,
)
raise TypeError(
"completion_response must be of type ImageResponse for bedrock image cost calculation"
)
elif custom_llm_provider == litellm.LlmProviders.RECRAFT.value:
from litellm.llms.recraft.cost_calculator import (
cost_calculator as recraft_image_cost_calculator,
)
return recraft_image_cost_calculator(
model=model,
image_response=completion_response,
)
elif custom_llm_provider == litellm.LlmProviders.GEMINI.value:
from litellm.llms.gemini.image_generation.cost_calculator import (
cost_calculator as gemini_image_cost_calculator,
)
return gemini_image_cost_calculator(
model=model,
image_response=completion_response,
)
else:
return default_image_cost_calculator(
model=model,
quality=quality,
custom_llm_provider=custom_llm_provider,
n=n,
size=size,
optional_params=optional_params,
)
return CostCalculatorUtils.route_image_generation_cost_calculator(
model=model,
custom_llm_provider=custom_llm_provider,
completion_response=completion_response,
quality=quality,
n=n,
size=size,
optional_params=optional_params,
)
elif (
call_type == CallTypes.speech.value
or call_type == CallTypes.aspeech.value

95
litellm/images/main.py
View file

@ -1,7 +1,7 @@
import asyncio
import contextvars
from functools import partial
from typing import Any, Coroutine, Dict, Literal, Optional, Union, cast, overload
from typing import Any, Coroutine, Dict, List, Literal, Optional, Union, cast, overload
import httpx
@ -311,7 +311,7 @@ def image_generation( # noqa: PLR0915
) or get_secret_str("AZURE_AD_TOKEN")
default_headers = {
"Content-Type": "application/json;",
"Content-Type": "application/json",
"api-key": api_key,
}
for k, v in default_headers.items():
@ -335,8 +335,64 @@ def image_generation( # noqa: PLR0915
headers=headers,
litellm_params=litellm_params_dict,
)
#########################################################
# Providers using llm_http_handler
#########################################################
elif custom_llm_provider in (
litellm.LlmProviders.RECRAFT,
litellm.LlmProviders.AIML,
litellm.LlmProviders.GEMINI,
):
if image_generation_config is None:
raise ValueError(f"image generation config is not supported for {custom_llm_provider}")
return llm_http_handler.image_generation_handler(
api_key=api_key,
model=model,
prompt=prompt,
image_generation_provider_config=image_generation_config,
image_generation_optional_request_params=optional_params,
custom_llm_provider=custom_llm_provider,
litellm_params=litellm_params_dict,
logging_obj=litellm_logging_obj,
timeout=timeout,
client=client,
)
elif custom_llm_provider == "azure_ai":
from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
api_base = AzureFoundryModelInfo.get_api_base(api_base)
api_key = AzureFoundryModelInfo.get_api_key(api_key)
if extra_headers is not None:
optional_params["extra_headers"] = extra_headers
default_headers = {
"Content-Type": "application/json",
"api-key": api_key,
}
for k, v in default_headers.items():
if k not in headers:
headers[k] = v
model_response = azure_chat_completions.image_generation(
model=model,
prompt=prompt,
timeout=timeout,
api_key=api_key,
api_base=api_base,
azure_ad_token=None,
azure_ad_token_provider=azure_ad_token_provider,
logging_obj=litellm_logging_obj,
optional_params=optional_params,
model_response=model_response,
api_version=api_version,
aimg_generation=aimg_generation,
client=client,
headers=headers,
litellm_params=litellm_params_dict,
)
elif (
custom_llm_provider == "openai"
or custom_llm_provider == LlmProviders.LITELLM_PROXY.value
or custom_llm_provider in litellm.openai_compatible_providers
):
model_response = openai_chat_completions.image_generation(
@ -406,28 +462,6 @@ def image_generation( # noqa: PLR0915
api_base=api_base,
client=client,
)
#########################################################
# Providers using llm_http_handler
#########################################################
elif custom_llm_provider in (
litellm.LlmProviders.RECRAFT,
litellm.LlmProviders.GEMINI,
):
if image_generation_config is None:
raise ValueError(f"image generation config is not supported for {custom_llm_provider}")
return llm_http_handler.image_generation_handler(
model=model,
prompt=prompt,
image_generation_provider_config=image_generation_config,
image_generation_optional_request_params=optional_params,
custom_llm_provider=custom_llm_provider,
litellm_params=litellm_params_dict,
logging_obj=litellm_logging_obj,
timeout=timeout,
client=client,
)
elif (
custom_llm_provider in litellm._custom_providers
): # Assume custom LLM provider
@ -643,7 +677,7 @@ def image_variation(
@client
def image_edit(
image: FileTypes,
image: Union[FileTypes, List[FileTypes]],
prompt: str,
model: Optional[str] = None,
mask: Optional[str] = None,
@ -671,6 +705,9 @@ def image_edit(
litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None)
_is_async = kwargs.pop("async_call", False) is True
#add images / or return a single image
images = image if isinstance(image, list) else [image]
# get llm provider logic
litellm_params = GenericLiteLLMParams(**kwargs)
model, custom_llm_provider, _, _ = get_llm_provider(
@ -719,7 +756,7 @@ def image_edit(
# Call the handler with _is_async flag instead of directly calling the async handler
return base_llm_http_handler.image_edit_handler(
model=model,
image=image,
image=images,
prompt=prompt,
image_edit_provider_config=image_edit_provider_config,
image_edit_optional_request_params=image_edit_request_params,
@ -745,7 +782,7 @@ def image_edit(
@client
async def aimage_edit(
image: FileTypes,
image: Union[FileTypes, List[FileTypes]],
model: str,
prompt: str,
mask: Optional[str] = None,
@ -785,9 +822,11 @@ async def aimage_edit(
model=model, api_base=local_vars.get("base_url", None)
)
images = image if isinstance(image, list) else [image]
func = partial(
image_edit,
image=image,
image=images,
prompt=prompt,
mask=mask,
model=model,

32
litellm/integrations/braintrust_logging.py
View file

@ -19,10 +19,6 @@ from litellm.llms.custom_httpx.http_handler import (
)
from litellm.utils import print_verbose
global_braintrust_http_handler = get_async_httpx_client(
llm_provider=httpxSpecialProvider.LoggingCallback
)
global_braintrust_sync_http_handler = HTTPHandler()
API_BASE = "https://api.braintrustdata.com/v1"
@ -52,6 +48,10 @@ class BraintrustLogger(CustomLogger):
self._project_id_cache: Dict[
str, str
] = {} # Cache mapping project names to IDs
self.global_braintrust_http_handler = get_async_httpx_client(
llm_provider=httpxSpecialProvider.LoggingCallback
)
self.global_braintrust_sync_http_handler = HTTPHandler()
def validate_environment(self, api_key: Optional[str]):
"""
@ -76,7 +76,7 @@ class BraintrustLogger(CustomLogger):
return self._project_id_cache[project_name]
try:
response = global_braintrust_sync_http_handler.post(
response = self.global_braintrust_sync_http_handler.post(
f"{self.api_base}/project",
headers=self.headers,
json={"name": project_name},
@ -96,7 +96,7 @@ class BraintrustLogger(CustomLogger):
return self._project_id_cache[project_name]
try:
response = await global_braintrust_http_handler.post(
response = await self.global_braintrust_http_handler.post(
f"{self.api_base}/project/register",
headers=self.headers,
json={"name": project_name},
@ -146,7 +146,7 @@ class BraintrustLogger(CustomLogger):
return metadata
async def create_default_project_and_experiment(self):
project = await global_braintrust_http_handler.post(
project = await self.global_braintrust_http_handler.post(
f"{self.api_base}/project", headers=self.headers, json={"name": "litellm"}
)
@ -155,7 +155,7 @@ class BraintrustLogger(CustomLogger):
self.default_project_id = project_dict["id"]
def create_sync_default_project_and_experiment(self):
project = global_braintrust_sync_http_handler.post(
project = self.global_braintrust_sync_http_handler.post(
f"{self.api_base}/project", headers=self.headers, json={"name": "litellm"}
)
@ -274,12 +274,15 @@ class BraintrustLogger(CustomLogger):
"end": end_time.timestamp(),
}
# Allow metadata override for span name
span_name = metadata.get("span_name", "Chat Completion")
request_data = {
"id": litellm_call_id,
"input": prompt["messages"],
"metadata": clean_metadata,
"tags": tags,
"span_attributes": {"name": "Chat Completion", "type": "llm"},
"span_attributes": {"name": span_name, "type": "llm"},
}
if choices is not None:
request_data["output"] = [choice.dict() for choice in choices]
@ -291,9 +294,9 @@ class BraintrustLogger(CustomLogger):
try:
print_verbose(
f"global_braintrust_sync_http_handler.post: {global_braintrust_sync_http_handler.post}"
f"self.global_braintrust_sync_http_handler.post: {self.global_braintrust_sync_http_handler.post}"
)
global_braintrust_sync_http_handler.post(
self.global_braintrust_sync_http_handler.post(
url=f"{self.api_base}/project_logs/{project_id}/insert",
json={"events": [request_data]},
headers=self.headers,
@ -426,13 +429,16 @@ class BraintrustLogger(CustomLogger):
- api_call_start_time.timestamp()
)
# Allow metadata override for span name
span_name = metadata.get("span_name", "Chat Completion")
request_data = {
"id": litellm_call_id,
"input": prompt["messages"],
"output": output,
"metadata": clean_metadata,
"tags": tags,
"span_attributes": {"name": "Chat Completion", "type": "llm"},
"span_attributes": {"name": span_name, "type": "llm"},
}
if choices is not None:
request_data["output"] = [choice.dict() for choice in choices]
@ -446,7 +452,7 @@ class BraintrustLogger(CustomLogger):
request_data["metrics"] = metrics
try:
await global_braintrust_http_handler.post(
await self.global_braintrust_http_handler.post(
url=f"{self.api_base}/project_logs/{project_id}/insert",
json={"events": [request_data]},
headers=self.headers,

101
litellm/integrations/datadog/datadog_llm_obs.py
View file

@ -27,7 +27,12 @@ from litellm.llms.custom_httpx.http_handler import (
httpxSpecialProvider,
)
from litellm.types.integrations.datadog_llm_obs import *
from litellm.types.utils import CallTypes, StandardLoggingPayload
from litellm.types.utils import (
CallTypes,
StandardLoggingGuardrailInformation,
StandardLoggingPayload,
StandardLoggingPayloadErrorInformation,
)
class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
@ -102,6 +107,24 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
verbose_logger.exception(
f"DataDogLLMObs: Error logging success event - {str(e)}"
)
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
try:
verbose_logger.debug(
f"DataDogLLMObs: Logging failure event for model {kwargs.get('model', 'unknown')}"
)
payload = self.create_llm_obs_payload(
kwargs, start_time, end_time
)
verbose_logger.debug(f"DataDogLLMObs: Payload: {payload}")
self.log_queue.append(payload)
if len(self.log_queue) >= self.batch_size:
await self.async_send_batch()
except Exception as e:
verbose_logger.exception(
f"DataDogLLMObs: Error logging failure event - {str(e)}"
)
async def async_send_batch(self):
try:
@ -174,11 +197,14 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
call_type=standard_logging_payload.get("call_type")
))
error_info = self._assemble_error_info(standard_logging_payload)
meta = Meta(
kind=self._get_datadog_span_kind(standard_logging_payload.get("call_type")),
input=input_meta,
output=output_meta,
metadata=self._get_dd_llm_obs_payload_metadata(standard_logging_payload),
error=error_info,
)
# Calculate metrics (you may need to adjust these based on available data)
@ -199,11 +225,31 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
start_ns=int(start_time.timestamp() * 1e9),
duration=int((end_time - start_time).total_seconds() * 1e9),
metrics=metrics,
status="error" if error_info else "ok",
tags=[
self._get_datadog_tags(standard_logging_object=standard_logging_payload)
],
)
def _assemble_error_info(self, standard_logging_payload: StandardLoggingPayload) -> Optional[DDLLMObsError]:
"""
Assemble error information for failure cases according to DD LLM Obs API spec
"""
# Handle error information for failure cases according to DD LLM Obs API spec
error_info: Optional[DDLLMObsError] = None
if standard_logging_payload.get("status") == "failure":
# Try to get structured error information first
error_information: Optional[StandardLoggingPayloadErrorInformation] = standard_logging_payload.get("error_information")
if error_information:
error_info = DDLLMObsError(
message=error_information.get("error_message") or standard_logging_payload.get("error_str") or "Unknown error",
type=error_information.get("error_class"),
stack=error_information.get("traceback")
)
return error_info
def _get_time_to_first_token_seconds(self, standard_logging_payload: StandardLoggingPayload) -> float:
"""
Get the time to first token in seconds
@ -232,8 +278,20 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
for now this handles logging /chat/completions responses
"""
if response_obj is None:
return []
if call_type in [CallTypes.completion.value, CallTypes.acompletion.value]:
return [response_obj["choices"][0]["message"]]
try:
# Safely extract message from response_obj, handle failure cases
if isinstance(response_obj, dict) and "choices" in response_obj:
choices = response_obj["choices"]
if choices and len(choices) > 0 and "message" in choices[0]:
return [choices[0]["message"]]
return []
except (KeyError, IndexError, TypeError):
# In case of any error accessing the response structure, return empty list
return []
return []
def _get_datadog_span_kind(self, call_type: Optional[str]) -> Literal["llm", "tool", "task", "embedding", "retrieval"]:
@ -350,11 +408,11 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
def _get_dd_llm_obs_payload_metadata(
self, standard_logging_payload: StandardLoggingPayload
) -> Dict:
) -> Dict[str, Any]:
"""
Fields to track in DD LLM Observability metadata from litellm standard logging payload
"""
_metadata = {
_metadata: Dict[str, Any] = {
"model_name": standard_logging_payload.get("model", "unknown"),
"model_provider": standard_logging_payload.get(
"custom_llm_provider", "unknown"
@ -364,9 +422,44 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
"cache_hit": standard_logging_payload.get("cache_hit", "unknown"),
"cache_key": standard_logging_payload.get("cache_key", "unknown"),
"saved_cache_cost": standard_logging_payload.get("saved_cache_cost", 0),
"guardrail_information": standard_logging_payload.get("guardrail_information", None),
}
#########################################################
# Add latency metrics to metadata
#########################################################
latency_metrics = self._get_latency_metrics(standard_logging_payload)
_metadata.update({"latency_metrics": dict(latency_metrics)})
_standard_logging_metadata: dict = (
dict(standard_logging_payload.get("metadata", {})) or {}
)
_metadata.update(_standard_logging_metadata)
return _metadata
def _get_latency_metrics(self, standard_logging_payload: StandardLoggingPayload) -> DDLLMObsLatencyMetrics:
"""
Get the latency metrics from the standard logging payload
"""
latency_metrics: DDLLMObsLatencyMetrics = DDLLMObsLatencyMetrics()
# Add latency metrics to metadata
# Time to first token (convert from seconds to milliseconds for consistency)
time_to_first_token_seconds = self._get_time_to_first_token_seconds(standard_logging_payload)
if time_to_first_token_seconds > 0:
latency_metrics["time_to_first_token_ms"] = time_to_first_token_seconds * 1000
# LiteLLM overhead time
hidden_params = standard_logging_payload.get("hidden_params", {})
litellm_overhead_ms = hidden_params.get("litellm_overhead_time_ms")
if litellm_overhead_ms is not None:
latency_metrics["litellm_overhead_time_ms"] = litellm_overhead_ms
# Guardrail overhead latency
guardrail_info: Optional[StandardLoggingGuardrailInformation] = standard_logging_payload.get("guardrail_information")
if guardrail_info is not None:
_guardrail_duration_seconds: Optional[float] = guardrail_info.get("duration")
if _guardrail_duration_seconds is not None:
# Convert from seconds to milliseconds for consistency
latency_metrics["guardrail_overhead_time_ms"] = _guardrail_duration_seconds * 1000
return latency_metrics

13
litellm/integrations/langfuse/langfuse_otel.py
View file

@ -141,6 +141,17 @@ class LangfuseOtelLogger(OpenTelemetry):
value = str(value)
safe_set_attribute(span, enum_attr.value, value)
@staticmethod
def _get_langfuse_otel_host() -> Optional[str]:
"""
Returns the Langfuse OTEL host based on environment variables.
Returned in the following order of precedence:
1. LANGFUSE_OTEL_HOST
2. LANGFUSE_HOST
"""
return os.environ.get("LANGFUSE_OTEL_HOST") or os.environ.get("LANGFUSE_HOST")
@staticmethod
def get_langfuse_otel_config() -> LangfuseOtelConfig:
"""
@ -166,7 +177,7 @@ class LangfuseOtelLogger(OpenTelemetry):
)
# Determine endpoint - default to US cloud
langfuse_host = os.environ.get("LANGFUSE_HOST", None)
langfuse_host = LangfuseOtelLogger._get_langfuse_otel_host()
if langfuse_host:
# If LANGFUSE_HOST is provided, construct OTEL endpoint from it

17
litellm/integrations/mlflow.py
View file

@ -60,10 +60,7 @@ class MlflowLogger(CustomLogger):
inputs = self._construct_input(kwargs)
input_messages = inputs.get("messages", [])
output_messages = [
c.message.model_dump(exclude_none=True)
for c in getattr(response_obj, "choices", [])
]
output_messages = [c.message.model_dump(exclude_none=True) for c in getattr(response_obj, "choices", [])]
if messages := [*input_messages, *output_messages]:
set_span_chat_messages(span, messages)
if tools := inputs.get("tools"):
@ -168,6 +165,10 @@ class MlflowLogger(CustomLogger):
for key in ["functions", "tools", "stream", "tool_choice", "user"]:
if value := kwargs.get("optional_params", {}).pop(key, None):
inputs[key] = value
if prediction := kwargs.get("prediction"):
inputs["prediction"] = prediction
return inputs
def _extract_attributes(self, kwargs):
@ -189,9 +190,9 @@ class MlflowLogger(CustomLogger):
{
"api_base": standard_obj.get("api_base"),
"cache_hit": standard_obj.get("cache_hit"),
"usage": {
"completion_tokens": standard_obj.get("completion_tokens"),
"prompt_tokens": standard_obj.get("prompt_tokens"),
"mlflow.chat.tokenUsage": {
"input_tokens": standard_obj.get("prompt_tokens"),
"output_tokens": standard_obj.get("completion_tokens"),
"total_tokens": standard_obj.get("total_tokens"),
},
"raw_llm_response": standard_obj.get("response"),
@ -232,7 +233,6 @@ class MlflowLogger(CustomLogger):
"""
import mlflow
call_type = kwargs.get("call_type", "completion")
span_name = f"litellm-{call_type}"
span_type = self._get_span_type(call_type)
@ -260,6 +260,7 @@ class MlflowLogger(CustomLogger):
tags=self._transform_tag_list_to_dict(attributes.get("request_tags", [])),
start_time_ns=start_time_ns,
)
def _transform_tag_list_to_dict(self, tag_list: list) -> dict:
return {tag: "" for tag in tag_list}

21
litellm/litellm_core_utils/core_helpers.py
View file

@ -37,6 +37,27 @@ def safe_divide_seconds(
return float(seconds / denominator)
def safe_divide(
numerator: Union[int, float],
denominator: Union[int, float],
default: Union[int, float] = 0
) -> Union[int, float]:
"""
Safely divide two numbers, returning a default value if denominator is zero.
Args:
numerator: The number to divide
denominator: The number to divide by
default: Value to return if denominator is zero (defaults to 0)
Returns:
The result of numerator/denominator, or default if denominator is zero
"""
if denominator == 0:
return default
return numerator / denominator
def map_finish_reason(
finish_reason: str,
): # openai supports 5 stop sequences - 'stop', 'length', 'function_call', 'content_filter', 'null'

17
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -196,6 +196,9 @@ def get_llm_provider( # noqa: PLR0915
elif endpoint == "https://api.cerebras.ai/v1":
custom_llm_provider = "cerebras"
dynamic_api_key = get_secret_str("CEREBRAS_API_KEY")
elif endpoint == "https://inference.baseten.co/v1":
custom_llm_provider = "baseten"
dynamic_api_key = get_secret_str("BASETEN_API_KEY")
elif endpoint == "https://api.sambanova.ai/v1":
custom_llm_provider = "sambanova"
dynamic_api_key = get_secret_str("SAMBANOVA_API_KEY")
@ -478,6 +481,13 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915
api_base or get_secret("CEREBRAS_API_BASE") or "https://api.cerebras.ai/v1"
) # type: ignore
dynamic_api_key = api_key or get_secret_str("CEREBRAS_API_KEY")
elif custom_llm_provider == "baseten":
# Use BasetenConfig to determine the appropriate API base URL
if api_base is None:
api_base = litellm.BasetenConfig.get_api_base_for_model(model)
else:
api_base = api_base or get_secret_str("BASETEN_API_BASE") or "https://inference.baseten.co/v1"
dynamic_api_key = api_key or get_secret_str("BASETEN_API_KEY")
elif custom_llm_provider == "sambanova":
api_base = (
api_base
@ -732,6 +742,13 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915
) = litellm.HyperbolicChatConfig()._get_openai_compatible_provider_info(
api_base, api_key
)
elif custom_llm_provider == "aiml":
(
api_base,
dynamic_api_key,
) = litellm.AIMLChatConfig()._get_openai_compatible_provider_info(
api_base, api_key
)
if api_base is not None and not isinstance(api_base, str):
raise Exception("api base needs to be a string. api_base={}".format(api_base))

23
litellm/litellm_core_utils/get_provider_specific_headers.py Normal file
View file

@ -0,0 +1,23 @@
from typing import Dict, Optional
from litellm.types.utils import ProviderSpecificHeader
class ProviderSpecificHeaderUtils:
@staticmethod
def get_provider_specific_headers(
provider_specific_header: Optional[ProviderSpecificHeader],
custom_llm_provider: Optional[str],
) -> Dict:
"""
Get the provider specific headers for the given custom llm provider
Returns:
Optional[Dict]: The provider specific headers for the given custom llm provider
"""
if (
provider_specific_header is not None
and provider_specific_header.get("custom_llm_provider") == custom_llm_provider
):
return provider_specific_header.get("extra_headers", {})
return {}

9
litellm/litellm_core_utils/get_supported_openai_params.py
View file

@ -78,6 +78,8 @@ def get_supported_openai_params( # noqa: PLR0915
return litellm.nvidiaNimEmbeddingConfig.get_supported_openai_params()
elif custom_llm_provider == "cerebras":
return litellm.CerebrasConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "baseten":
return litellm.BasetenConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "xai":
return litellm.XAIChatConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "ai21_chat" or custom_llm_provider == "ai21":
@ -266,10 +268,9 @@ def get_supported_openai_params( # noqa: PLR0915
from litellm.llms.elevenlabs.audio_transcription.transformation import (
ElevenLabsAudioTranscriptionConfig,
)
return (
ElevenLabsAudioTranscriptionConfig().get_supported_openai_params(
model=model
)
return ElevenLabsAudioTranscriptionConfig().get_supported_openai_params(
model=model
)
elif custom_llm_provider in litellm._custom_providers:
if request_type == "chat_completion":

40
litellm/litellm_core_utils/litellm_logging.py
View file

@ -811,7 +811,7 @@ class Logging(LiteLLMLoggingBaseClass):
str(e)
)
)
if self.logger_fn and callable(self.logger_fn):
if getattr(self, "logger_fn", None) and callable(self.logger_fn):
try:
self.logger_fn(
self.model_call_details
@ -999,7 +999,7 @@ class Logging(LiteLLMLoggingBaseClass):
)
)
)
if self.logger_fn and callable(self.logger_fn):
if getattr(self, "logger_fn", None) and callable(self.logger_fn):
try:
self.logger_fn(
self.model_call_details
@ -3919,10 +3919,12 @@ class StandardLoggingPayloadSetup:
# Generate cold storage object key if cold storage is configured
if start_time is not None and response_id is not None:
cold_storage_object_key = StandardLoggingPayloadSetup._generate_cold_storage_object_key(
start_time=start_time,
response_id=response_id,
team_alias=clean_metadata.get("user_api_key_team_alias"),
cold_storage_object_key = (
StandardLoggingPayloadSetup._generate_cold_storage_object_key(
start_time=start_time,
response_id=response_id,
team_alias=clean_metadata.get("user_api_key_team_alias"),
)
)
if cold_storage_object_key:
clean_metadata["cold_storage_object_key"] = cold_storage_object_key
@ -4093,45 +4095,33 @@ class StandardLoggingPayloadSetup:
) -> Optional[str]:
"""
Generate cold storage object key in the same format as S3Logger.
Args:
start_time: The start time of the request
response_id: The response ID
response_id: The response ID
team_alias: Optional team alias for team-based prefixing
Returns:
Optional[str]: The generated object key or None if cold storage not configured
"""
# Generate object key in same format as S3Logger
from litellm.integrations.s3 import get_s3_object_key
from litellm.proxy.spend_tracking.cold_storage_handler import ColdStorageHandler
# Only generate object key if cold storage is configured
try:
configured_cold_storage_logger = (
ColdStorageHandler._get_configured_cold_storage_custom_logger()
)
except Exception as e:
verbose_logger.debug(
f"Cold storage custom logger unavailable: {e}"
)
if litellm.configured_cold_storage_logger is None:
return None
if configured_cold_storage_logger is None:
return None
try:
# Generate file name in same format as litellm.utils.get_logging_id
s3_file_name = f"time-{start_time.strftime('%H-%M-%S-%f')}_{response_id}"
s3_object_key = get_s3_object_key(
s3_path="", # Use empty path as default
team_alias_prefix="", # Don't split by team alias for cold storage
s3_path="", # Use empty path as default
team_alias_prefix="", # Don't split by team alias for cold storage
start_time=start_time,
s3_file_name=s3_file_name,
)
return s3_object_key
except Exception:
# If any error occurs in generating the key, return None

144
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -1,11 +1,17 @@
# What is this?
## Helper utilities for cost_per_token()
from typing import Literal, Optional, Tuple, cast
from typing import Any, Literal, Optional, Tuple, cast
import litellm
from litellm._logging import verbose_logger
from litellm.types.utils import CallTypes, ModelInfo, PassthroughCallTypes, Usage
from litellm.types.utils import (
CallTypes,
ImageResponse,
ModelInfo,
PassthroughCallTypes,
Usage,
)
from litellm.utils import get_model_info
@ -107,15 +113,20 @@ def _generic_cost_per_character(
return prompt_cost, completion_cost
def _get_token_base_cost(model_info: ModelInfo, usage: Usage) -> Tuple[float, float]:
def _get_token_base_cost(model_info: ModelInfo, usage: Usage) -> Tuple[float, float, float, float]:
"""
Return prompt cost for a given model and usage.
Return prompt cost, completion cost, and cache costs for a given model and usage.
If input_tokens > threshold and `input_cost_per_token_above_[x]k_tokens` or `input_cost_per_token_above_[x]_tokens` is set,
then we use the corresponding threshold cost.
then we use the corresponding threshold cost for all token types.
Returns:
Tuple[float, float, float, float] - (prompt_cost, completion_cost, cache_creation_cost, cache_read_cost)
"""
prompt_base_cost = cast(float, _get_cost_per_unit(model_info, "input_cost_per_token"))
completion_base_cost = cast(float, _get_cost_per_unit(model_info, "output_cost_per_token"))
cache_creation_cost = cast(float, _get_cost_per_unit(model_info, "cache_creation_input_token_cost"))
cache_read_cost = cast(float, _get_cost_per_unit(model_info, "cache_read_input_token_cost"))
## CHECK IF ABOVE THRESHOLD
threshold: Optional[float] = None
@ -135,13 +146,28 @@ def _get_token_base_cost(model_info: ModelInfo, usage: Usage) -> Tuple[float, fl
f"output_cost_per_token_above_{threshold_str}_tokens",
completion_base_cost,
))
# Apply tiered pricing to cache costs
cache_creation_tiered_key = f"cache_creation_input_token_cost_above_{threshold_str}_tokens"
cache_read_tiered_key = f"cache_read_input_token_cost_above_{threshold_str}_tokens"
if cache_creation_tiered_key in model_info:
cache_creation_cost = cast(float, _get_cost_per_unit(
model_info, cache_creation_tiered_key, cache_creation_cost
))
if cache_read_tiered_key in model_info:
cache_read_cost = cast(float, _get_cost_per_unit(
model_info, cache_read_tiered_key, cache_read_cost
))
break
except (IndexError, ValueError):
continue
except Exception:
continue
return prompt_base_cost, completion_base_cost
return prompt_base_cost, completion_base_cost, cache_creation_cost, cache_read_cost
def calculate_cost_component(
@ -256,28 +282,22 @@ def generic_cost_per_token(
if text_tokens == 0:
text_tokens = usage.prompt_tokens - cache_hit_tokens - audio_tokens
prompt_base_cost, completion_base_cost = _get_token_base_cost(
prompt_base_cost, completion_base_cost, cache_creation_cost, cache_read_cost = _get_token_base_cost(
model_info=model_info, usage=usage
)
prompt_cost = float(text_tokens) * prompt_base_cost
### CACHE READ COST
prompt_cost += calculate_cost_component(
model_info, "cache_read_input_token_cost", cache_hit_tokens
)
### CACHE READ COST - Now uses tiered pricing
prompt_cost += float(cache_hit_tokens) * cache_read_cost
### AUDIO COST
prompt_cost += calculate_cost_component(
model_info, "input_cost_per_audio_token", audio_tokens
)
### CACHE WRITING COST
prompt_cost += calculate_cost_component(
model_info,
"cache_creation_input_token_cost",
usage._cache_creation_input_tokens,
)
### CACHE WRITING COST - Now uses tiered pricing
prompt_cost += float(usage._cache_creation_input_tokens or 0) * cache_creation_cost
### CHARACTER COST
@ -377,3 +397,93 @@ class CostCalculatorUtils:
]:
return True
return False
@staticmethod
def route_image_generation_cost_calculator(
model: str,
completion_response: Any,
custom_llm_provider: Optional[str] = None,
quality: Optional[str] = None,
n: Optional[int] = None,
size: Optional[str] = None,
optional_params: Optional[dict] = None,
) -> float:
"""
Route the image generation cost calculator based on the custom_llm_provider
"""
from litellm.cost_calculator import default_image_cost_calculator
from litellm.llms.azure_ai.image_generation.cost_calculator import (
cost_calculator as azure_ai_image_cost_calculator,
)
from litellm.llms.bedrock.image.cost_calculator import (
cost_calculator as bedrock_image_cost_calculator,
)
from litellm.llms.gemini.image_generation.cost_calculator import (
cost_calculator as gemini_image_cost_calculator,
)
from litellm.llms.recraft.cost_calculator import (
cost_calculator as recraft_image_cost_calculator,
)
from litellm.llms.vertex_ai.image_generation.cost_calculator import (
cost_calculator as vertex_ai_image_cost_calculator,
)
if custom_llm_provider == litellm.LlmProviders.VERTEX_AI.value:
if isinstance(completion_response, ImageResponse):
return vertex_ai_image_cost_calculator(
model=model,
image_response=completion_response,
)
elif custom_llm_provider == litellm.LlmProviders.BEDROCK.value:
if isinstance(completion_response, ImageResponse):
return bedrock_image_cost_calculator(
model=model,
size=size,
image_response=completion_response,
optional_params=optional_params,
)
raise TypeError(
"completion_response must be of type ImageResponse for bedrock image cost calculation"
)
elif custom_llm_provider == litellm.LlmProviders.RECRAFT.value:
from litellm.llms.recraft.cost_calculator import (
cost_calculator as recraft_image_cost_calculator,
)
return recraft_image_cost_calculator(
model=model,
image_response=completion_response,
)
elif custom_llm_provider == litellm.LlmProviders.AIML.value:
from litellm.llms.aiml.image_generation.cost_calculator import (
cost_calculator as aiml_image_cost_calculator,
)
return aiml_image_cost_calculator(
model=model,
image_response=completion_response,
)
elif custom_llm_provider == litellm.LlmProviders.GEMINI.value:
from litellm.llms.gemini.image_generation.cost_calculator import (
cost_calculator as gemini_image_cost_calculator,
)
return gemini_image_cost_calculator(
model=model,
image_response=completion_response,
)
elif custom_llm_provider == litellm.LlmProviders.AZURE_AI.value:
return azure_ai_image_cost_calculator(
model=model,
image_response=completion_response,
)
else:
return default_image_cost_calculator(
model=model,
quality=quality,
custom_llm_provider=custom_llm_provider,
n=n,
size=size,
optional_params=optional_params,
)
return 0.0

119
litellm/litellm_core_utils/logging_utils.py
View file

@ -1,5 +1,6 @@
import asyncio
import functools
import time
from datetime import datetime
from typing import TYPE_CHECKING, Any, List, Optional, Union
@ -11,15 +12,19 @@ from litellm.types.utils import (
)
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
from litellm import ModelResponse as _ModelResponse
from litellm.litellm_core_utils.litellm_logging import (
Logging as LiteLLMLoggingObject,
)
LiteLLMModelResponse = _ModelResponse
Span = Union[_Span, Any]
else:
LiteLLMModelResponse = Any
LiteLLMLoggingObject = Any
Span = Any
import litellm
@ -28,9 +33,52 @@ import litellm
Helper utils used for logging callbacks
"""
# Global service logger instance to avoid recreating it
_service_logger = None
def _get_service_logger():
"""Get or create the global ServiceLogging instance"""
global _service_logger
if _service_logger is None:
from litellm._service_logger import ServiceLogging
_service_logger = ServiceLogging()
return _service_logger
def _get_parent_otel_span_from_logging_obj(
logging_obj: Optional[LiteLLMLoggingObject] = None,
) -> Optional[Span]:
"""
Extract the parent OTEL span from the logging object using existing helper.
Args:
logging_obj: The LiteLLM logging object containing model call details
Returns:
The parent OTEL span if found, None otherwise
"""
try:
if logging_obj is None or not hasattr(logging_obj, "model_call_details"):
return None
# Reuse existing function by passing model_call_details as kwargs
from litellm.litellm_core_utils.core_helpers import (
_get_parent_otel_span_from_kwargs,
)
return _get_parent_otel_span_from_kwargs(logging_obj.model_call_details)
except Exception as e:
verbose_logger.exception(
f"Error in _get_parent_otel_span_from_logging_obj: {str(e)}"
)
return None
def convert_litellm_response_object_to_str(
response_obj: Union[Any, LiteLLMModelResponse]
response_obj: Union[Any, LiteLLMModelResponse],
) -> Optional[str]:
"""
Get the string of the response object from LiteLLM
@ -125,37 +173,102 @@ def track_llm_api_timing():
"""
Decorator to track LLM API call timing for both sync and async functions.
The logging_obj is expected to be passed as an argument to the decorated function.
Logs timing using ServiceLogging similar to Redis cache.
"""
def decorator(func):
@functools.wraps(func)
async def async_wrapper(*args, **kwargs):
start_time = datetime.now()
start_time_float = time.time()
logging_obj = kwargs.get("logging_obj", None)
# Extract parent OTEL span from logging object
parent_otel_span = _get_parent_otel_span_from_logging_obj(logging_obj)
try:
result = await func(*args, **kwargs)
return result
finally:
end_time = datetime.now()
end_time_float = time.time()
duration = end_time_float - start_time_float
# Set duration in model call details
_set_duration_in_model_call_details(
logging_obj=kwargs.get("logging_obj", None),
logging_obj=logging_obj,
start_time=start_time,
end_time=end_time,
)
# Log timing using ServiceLogging (like Redis cache)
try:
from litellm.types.services import ServiceTypes
service_logger = _get_service_logger()
# Get function name for call_type
call_type = f"{func.__name__} <- track_llm_api_timing"
# Create async task for service logging (similar to Redis cache pattern)
asyncio.create_task(
service_logger.async_service_success_hook(
service=ServiceTypes.LITELLM,
duration=duration,
call_type=call_type,
start_time=start_time_float,
end_time=end_time_float,
parent_otel_span=parent_otel_span,
)
)
except Exception as e:
verbose_logger.debug(f"Error in service logging: {str(e)}")
@functools.wraps(func)
def sync_wrapper(*args, **kwargs):
start_time = datetime.now()
start_time_float = time.time()
logging_obj = kwargs.get("logging_obj", None)
# Extract parent OTEL span from logging object
parent_otel_span = _get_parent_otel_span_from_logging_obj(logging_obj)
try:
result = func(*args, **kwargs)
return result
finally:
end_time = datetime.now()
end_time_float = time.time()
duration = end_time_float - start_time_float
# Set duration in model call details
_set_duration_in_model_call_details(
logging_obj=kwargs.get("logging_obj", None),
logging_obj=logging_obj,
start_time=start_time,
end_time=end_time,
)
# Log timing using ServiceLogging (like Redis cache)
try:
from litellm.types.services import ServiceTypes
service_logger = _get_service_logger()
# Get function name for call_type
call_type = f"{func.__name__} <- track_llm_api_timing"
# Use sync service logging for sync functions
service_logger.service_success_hook(
service=ServiceTypes.LITELLM,
duration=duration,
call_type=call_type,
start_time=start_time_float,
end_time=end_time_float,
parent_otel_span=parent_otel_span,
)
except Exception as e:
verbose_logger.debug(f"Error in service logging: {str(e)}")
# Check if the function is async or sync
if asyncio.iscoroutinefunction(func):
return async_wrapper

131
litellm/litellm_core_utils/logging_worker.py Normal file
View file

@ -0,0 +1,131 @@
import asyncio
import contextlib
from typing import Coroutine, Optional
from litellm._logging import verbose_logger
class LoggingWorker:
"""
A simple, async logging worker that processes log coroutines in the background.
Designed to be best-effort with bounded queues to prevent backpressure.
This leads to a +200 RPS performance improvement when using LiteLLM Python SDK or Proxy Server.
- Use this to queue coroutine tasks that are not critical to the main flow of the application. e.g Success/Error callbacks, logging, etc.
"""
LOGGING_WORKER_MAX_QUEUE_SIZE = 50_000
LOGGING_WORKER_MAX_TIME_PER_COROUTINE = 20.0
MAX_ITERATIONS_TO_CLEAR_QUEUE = 200
MAX_TIME_TO_CLEAR_QUEUE = 5.0
def __init__(
self,
timeout: float = LOGGING_WORKER_MAX_TIME_PER_COROUTINE,
max_queue_size: int = LOGGING_WORKER_MAX_QUEUE_SIZE,
):
self.timeout = timeout
self.max_queue_size = max_queue_size
self._queue: Optional[asyncio.Queue] = None
self._worker_task: Optional[asyncio.Task] = None
def _ensure_queue(self) -> None:
"""Initialize the queue if it doesn't exist."""
if self._queue is None:
self._queue = asyncio.Queue(maxsize=self.max_queue_size)
def start(self) -> None:
"""Start the logging worker. Idempotent - safe to call multiple times."""
self._ensure_queue()
if self._worker_task is None or self._worker_task.done():
self._worker_task = asyncio.create_task(self._worker_loop())
async def _worker_loop(self) -> None:
"""Main worker loop that processes log coroutines sequentially."""
try:
if self._queue is None:
return
while True:
# Process one coroutine at a time to keep event loop load predictable
coroutine = await self._queue.get()
try:
await asyncio.wait_for(coroutine, timeout=self.timeout)
except Exception as e:
verbose_logger.exception(f"LoggingWorker error: {e}")
pass
finally:
self._queue.task_done()
except asyncio.CancelledError as e:
verbose_logger.exception(f"LoggingWorker cancelled: {e}")
pass
def enqueue(self, coroutine: Coroutine) -> None:
"""
Add a coroutine to the logging queue.
Hot path: never blocks, drops logs if queue is full.
"""
if self._queue is None:
return
try:
self._queue.put_nowait(coroutine)
except asyncio.QueueFull as e:
verbose_logger.exception(f"LoggingWorker queue is full: {e}")
# Drop logs on overload to protect request throughput
pass
def ensure_initialized_and_enqueue(self, async_coroutine: Coroutine):
"""
Ensure the logging worker is initialized and enqueue the coroutine.
"""
self.start()
self.enqueue(async_coroutine)
async def stop(self) -> None:
"""Stop the logging worker and clean up resources."""
if self._worker_task:
self._worker_task.cancel()
with contextlib.suppress(Exception):
await self._worker_task
self._worker_task = None
async def flush(self) -> None:
"""Flush the logging queue."""
if self._queue is None:
return
while not self._queue.empty():
await self._queue.join()
async def clear_queue(self):
"""
Clear the queue with a maximum time limit.
"""
if self._queue is None:
return
start_time = asyncio.get_event_loop().time()
for _ in range(self.MAX_ITERATIONS_TO_CLEAR_QUEUE):
# Check if we've exceeded the maximum time
if asyncio.get_event_loop().time() - start_time >= self.MAX_TIME_TO_CLEAR_QUEUE:
verbose_logger.warning(f"clear_queue exceeded max_time of {self.MAX_TIME_TO_CLEAR_QUEUE}s, stopping early")
break
try:
coroutine = self._queue.get_nowait()
# Await the coroutine to properly execute and avoid "never awaited" warnings
try:
await asyncio.wait_for(coroutine, timeout=self.timeout)
except Exception:
# Suppress errors during cleanup
pass
self._queue.task_done() # If you're using join() elsewhere
except asyncio.QueueEmpty:
break
# Global instance for backward compatibility
GLOBAL_LOGGING_WORKER = LoggingWorker()

213
litellm/litellm_core_utils/model_response_utils.py Normal file
View file

@ -0,0 +1,213 @@
"""
Utility functions for ModelResponse and ModelResponseStream objects.
"""
from typing import Any
from litellm.types.utils import Delta, ModelResponseBase, ModelResponseStream
def is_model_response_stream_empty(model_response: ModelResponseStream) -> bool:
"""
Check if a ModelResponseStream is empty based on:
- If finish_reason is set -> it's non empty
- If any field in choices is set (e.g. content, tool calls, etc.) it's non empty
- If usage exists -> it's non empty
This function is robust and ignores fields that are always set (from ModelResponseBase)
and checks for any meaningful content in other fields.
Args:
model_response: The ModelResponseStream to check
Returns:
bool: True if the stream is empty, False if it contains meaningful data
"""
# Fields that are always set in ModelResponseBase and should be ignored
# These are structural fields that don't indicate content
BASE_FIELDS = ModelResponseBase.model_fields.keys()
# Check if usage exists - this indicates meaningful data
if getattr(model_response, "usage", None) is not None:
return False
# Check provider_specific_fields at the top level
if (
hasattr(model_response, "provider_specific_fields")
and model_response.provider_specific_fields is not None
and model_response.provider_specific_fields != {}
):
return False
# Check model_extra for dynamically added fields (this is where Pydantic stores them)
if hasattr(model_response, "model_extra") and model_response.model_extra:
for extra_field_name, extra_field_value in model_response.model_extra.items():
if _has_meaningful_content(extra_field_value):
return False
# Check for any non-base fields that are set
for model_response_field in model_response.model_fields.keys():
# Skip base fields that are always set
if model_response_field in BASE_FIELDS:
continue
# Skip choices - we'll handle them separately with deep inspection
if model_response_field == "choices":
continue
# Check if any other field has meaningful content
model_response_value = getattr(model_response, model_response_field, None)
if _has_meaningful_content(model_response_value):
return False
# Deep check of choices for any meaningful content
if hasattr(model_response, "choices") and model_response.choices:
for choice in model_response.choices:
if _is_choice_non_empty(choice):
return False
# If we get here, the stream is empty
return True
def _has_meaningful_content(value: Any) -> bool:
"""
Check if a value contains meaningful content.
Args:
value: The value to check
Returns:
bool: True if the value has meaningful content, False otherwise
"""
if value is None:
return False
if isinstance(value, str):
return len(value.strip()) > 0
if isinstance(value, (list, dict)):
return len(value) > 0
if isinstance(value, bool):
return True # Any boolean value is meaningful
if isinstance(value, (int, float)):
return True # Any numeric value is meaningful
# For other types (objects), consider them meaningful if they exist
return True
def _is_choice_non_empty(choice: Any) -> bool:
"""
Deep check if a choice contains any meaningful content.
Args:
choice: The choice object to check
Returns:
bool: True if the choice has meaningful content, False otherwise
"""
# Check finish_reason
if hasattr(choice, "finish_reason") and choice.finish_reason is not None:
return True
# Check logprobs
if hasattr(choice, "logprobs") and choice.logprobs is not None:
return True
# Check enhancements (if present)
if hasattr(choice, "enhancements") and choice.enhancements is not None:
return True
# Deep check delta object
if hasattr(choice, "delta") and choice.delta is not None:
if _is_delta_non_empty(choice.delta):
return True
# Check model_extra for dynamically added fields on the choice
if hasattr(choice, "model_extra") and choice.model_extra:
for extra_field_name, extra_field_value in choice.model_extra.items():
# Skip certain structural fields that are just default/None placeholders
if extra_field_name == "index" and extra_field_value == 0:
continue
if (
extra_field_name in {"finish_reason", "logprobs"}
and extra_field_value is None
):
continue
if extra_field_name == "delta":
continue
if _has_meaningful_content(extra_field_value):
return True
# Check for any other non-standard fields on the choice
for attr_name in dir(choice):
# Skip private attributes, methods, and known empty fields
if (
attr_name.startswith("_")
or callable(getattr(choice, attr_name))
or attr_name.startswith("model_")
or attr_name
in {
"finish_reason",
"index",
"delta",
"logprobs",
"enhancements",
}
):
continue
attr_value = getattr(choice, attr_name, None)
if _has_meaningful_content(attr_value):
return True
return False
def _is_delta_non_empty(delta: Delta) -> bool:
"""
Deep check if a delta object contains any meaningful content.
Args:
delta: The delta object to check
Returns:
bool: True if the delta has meaningful content, False otherwise
"""
# Check model_extra for dynamically added fields (this is where Pydantic stores them)
if hasattr(delta, "model_extra") and delta.model_extra:
for extra_field_name, extra_field_value in delta.model_extra.items():
# Even structural fields are meaningful if they have actual content
if _has_meaningful_content(extra_field_value):
return True
# Check all regular attributes of the delta object
for attr_name in dir(delta):
# Skip private attributes, methods, and Pydantic-specific fields
if (
attr_name.startswith("_")
or callable(getattr(delta, attr_name))
or attr_name.startswith("model_")
):
continue
attr_value = getattr(delta, attr_name, None)
if _has_meaningful_content(attr_value):
return True
return False

5
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -18,6 +18,7 @@ from typing import (
cast,
)
from litellm.router_utils.batch_utils import InMemoryFile
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionAssistantMessage,
@ -453,6 +454,10 @@ def extract_file_data(file_data: FileTypes) -> ExtractedFileData:
filename, file_content, content_type = file_data
elif len(file_data) == 4:
filename, file_content, content_type, file_headers = file_data
elif isinstance(file_data, InMemoryFile):
filename = file_data.name
file_content = file_data
content_type = file_data.content_type
else:
file_content = file_data
# Convert content to bytes

102
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -17,6 +17,7 @@ from litellm.llms.custom_httpx.http_handler import HTTPHandler, get_async_httpx_
from litellm.types.files import get_file_extension_from_mime_type
from litellm.types.llms.anthropic import *
from litellm.types.llms.bedrock import MessageBlock as BedrockMessageBlock
from litellm.types.llms.bedrock import CachePointBlock
from litellm.types.llms.custom_http import httpxSpecialProvider
from litellm.types.llms.ollama import OllamaVisionModelObject
from litellm.types.llms.openai import (
@ -2685,6 +2686,11 @@ def _convert_to_bedrock_tool_call_invoke(
)
bedrock_content_block = BedrockContentBlock(toolUse=bedrock_tool)
_parts_list.append(bedrock_content_block)
# Check for cache_control and add a separate cachePoint block
if tool.get("cache_control", None) is not None:
cache_point_block = BedrockContentBlock(cachePoint=CachePointBlock(type="default"))
_parts_list.append(cache_point_block)
return _parts_list
except Exception as e:
raise Exception(
@ -2745,6 +2751,7 @@ def _convert_to_bedrock_tool_call_result(
for content in content_list:
if content["type"] == "text":
content_str += content["text"]
message.get("name", "")
id = str(message.get("tool_call_id", str(uuid.uuid4())))
@ -2753,6 +2760,7 @@ def _convert_to_bedrock_tool_call_result(
content=[tool_result_content_block],
toolUseId=id,
)
content_block = BedrockContentBlock(toolResult=tool_result)
return content_block
@ -3185,9 +3193,30 @@ class BedrockConverseMessagesProcessor:
## MERGE CONSECUTIVE TOOL CALL MESSAGES ##
tool_content: List[BedrockContentBlock] = []
while msg_i < len(messages) and messages[msg_i]["role"] == "tool":
tool_call_result = _convert_to_bedrock_tool_call_result(messages[msg_i])
current_message = messages[msg_i]
tool_call_result = _convert_to_bedrock_tool_call_result(current_message)
tool_content.append(tool_call_result)
# Check if we need to add a separate cachePoint block
has_cache_control = False
# Check for message-level cache_control
if current_message.get("cache_control", None) is not None:
has_cache_control = True
# Check for content-level cache_control in list content
elif isinstance(current_message.get("content"), list):
for content_element in current_message["content"]:
if (isinstance(content_element, dict) and
content_element.get("cache_control", None) is not None):
has_cache_control = True
break
# Add a separate cachePoint block if cache_control is present
if has_cache_control:
cache_point_block = BedrockContentBlock(cachePoint=CachePointBlock(type="default"))
tool_content.append(cache_point_block)
msg_i += 1
if tool_content:
# if last message was a 'user' message, then add a blank assistant message (bedrock requires alternating roles)
@ -3267,13 +3296,29 @@ class BedrockConverseMessagesProcessor:
image_url=image_url
)
assistants_parts.append(assistants_part)
# Add cache point block for assistant content elements
_cache_point_block = (
litellm.AmazonConverseConfig()._get_cache_point_block(
message_block=cast(
OpenAIMessageContentListBlock, element
),
block_type="content_block",
)
)
if _cache_point_block is not None:
assistants_parts.append(_cache_point_block)
assistant_content.extend(assistants_parts)
elif _assistant_content is not None and isinstance(
_assistant_content, str
):
assistant_content.append(
BedrockContentBlock(text=_assistant_content)
elif _assistant_content is not None and isinstance(_assistant_content, str):
assistant_content.append(BedrockContentBlock(text=_assistant_content))
# Add cache point block for assistant string content
_cache_point_block = (
litellm.AmazonConverseConfig()._get_cache_point_block(
assistant_message_block, block_type="content_block"
)
)
if _cache_point_block is not None:
assistant_content.append(_cache_point_block)
_tool_calls = assistant_message_block.get("tool_calls", [])
if _tool_calls:
assistant_content.extend(
@ -3516,8 +3561,30 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915
tool_content: List[BedrockContentBlock] = []
while msg_i < len(messages) and messages[msg_i]["role"] == "tool":
tool_call_result = _convert_to_bedrock_tool_call_result(messages[msg_i])
current_message = messages[msg_i]
# Add the tool result first
tool_content.append(tool_call_result)
# Check if we need to add a separate cachePoint block
has_cache_control = False
# Check for message-level cache_control
if current_message.get("cache_control", None) is not None:
has_cache_control = True
# Check for content-level cache_control in list content
elif isinstance(current_message.get("content"), list):
for content_element in current_message["content"]:
if (isinstance(content_element, dict) and
content_element.get("cache_control", None) is not None):
has_cache_control = True
break
# Add a separate cachePoint block if cache_control is present
if has_cache_control:
cache_point_block = BedrockContentBlock(cachePoint=CachePointBlock(type="default"))
tool_content.append(cache_point_block)
msg_i += 1
if tool_content:
# if last message was a 'user' message, then add a blank assistant message (bedrock requires alternating roles)
@ -3589,9 +3656,28 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915
image_url=image_url
)
assistants_parts.append(assistants_part)
# Add cache point block for assistant content elements
_cache_point_block = (
litellm.AmazonConverseConfig()._get_cache_point_block(
message_block=cast(
OpenAIMessageContentListBlock, element
),
block_type="content_block",
)
)
if _cache_point_block is not None:
assistants_parts.append(_cache_point_block)
assistant_content.extend(assistants_parts)
elif _assistant_content is not None and isinstance(_assistant_content, str):
assistant_content.append(BedrockContentBlock(text=_assistant_content))
# Add cache point block for assistant string content
_cache_point_block = (
litellm.AmazonConverseConfig()._get_cache_point_block(
assistant_message_block, block_type="content_block"
)
)
if _cache_point_block is not None:
assistant_content.append(_cache_point_block)
_tool_calls = assistant_message_block.get("tool_calls", [])
if _tool_calls:
assistant_content.extend(

7
litellm/litellm_core_utils/sensitive_data_masker.py
View file

@ -33,7 +33,12 @@ class SensitiveDataMasker:
value_str = str(value)
masked_length = len(value_str) - (self.visible_prefix + self.visible_suffix)
return f"{value_str[:self.visible_prefix]}{self.mask_char * masked_length}{value_str[-self.visible_suffix:]}"
# Handle the case where visible_suffix is 0 to avoid showing the entire string
if self.visible_suffix == 0:
return f"{value_str[:self.visible_prefix]}{self.mask_char * masked_length}"
else:
return f"{value_str[:self.visible_prefix]}{self.mask_char * masked_length}{value_str[-self.visible_suffix:]}"
def is_sensitive_key(self, key: str) -> bool:
key_lower = str(key).lower()

114
litellm/litellm_core_utils/streaming_handler.py
View file

@ -13,11 +13,16 @@ from pydantic import BaseModel
import litellm
from litellm import verbose_logger
from litellm.litellm_core_utils.model_response_utils import (
is_model_response_stream_empty,
)
from litellm.litellm_core_utils.redact_messages import LiteLLMLoggingObject
from litellm.litellm_core_utils.thread_pool_executor import executor
from litellm.types.llms.openai import ChatCompletionChunk
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import Delta
from litellm.types.utils import (
Delta,
)
from litellm.types.utils import GenericStreamingChunk as GChunk
from litellm.types.utils import (
ModelResponse,
@ -32,6 +37,12 @@ from .exception_mapping_utils import exception_type
from .llm_response_utils.get_api_base import get_api_base
from .rules import Rules
# Constants for special delta attribute names
AUDIO_ATTRIBUTE = "audio"
IMAGE_ATTRIBUTE = "image"
TOOL_CALLS_ATTRIBUTE = "tool_calls"
FUNCTION_CALL_ATTRIBUTE = "function_call"
def is_async_iterable(obj: Any) -> bool:
"""
@ -763,6 +774,66 @@ class CustomStreamWrapper:
model_response.choices[0].delta = Delta(**_initial_delta)
return model_response
def _has_special_delta_content(self, model_response: ModelResponseStream) -> bool:
"""
Check if the delta contains special content types (tool_calls, function_call, audio, or image).
"""
if len(model_response.choices) == 0:
return False
delta = model_response.choices[0].delta
# Check for tool_calls or function_call
if getattr(delta, TOOL_CALLS_ATTRIBUTE, None) is not None or getattr(delta, FUNCTION_CALL_ATTRIBUTE, None) is not None:
return True
# Check for audio
if hasattr(delta, AUDIO_ATTRIBUTE) and getattr(delta, AUDIO_ATTRIBUTE, None) is not None:
return True
# Check for image
if hasattr(delta, IMAGE_ATTRIBUTE) and getattr(delta, IMAGE_ATTRIBUTE, None) is not None:
return True
return False
def _handle_special_delta_content(self, model_response: ModelResponseStream) -> ModelResponseStream:
"""
Handle special delta content types by stripping role and returning the response.
"""
return self.strip_role_from_delta(model_response)
def _has_special_delta_attribute(self, delta, attribute_name: str) -> bool:
"""
Check if delta has a specific attribute and it's not None.
"""
return delta is not None and getattr(delta, attribute_name, None) is not None
def _copy_delta_attribute(self, source_delta, target_delta, attribute_name: str) -> None:
"""
Copy a specific attribute from source delta to target delta.
"""
setattr(target_delta, attribute_name, getattr(source_delta, attribute_name))
def _has_any_special_delta_attributes(self, delta) -> bool:
"""
Check if delta has any special attributes (audio, image).
"""
special_attributes = [AUDIO_ATTRIBUTE, IMAGE_ATTRIBUTE]
for attribute in special_attributes:
if self._has_special_delta_attribute(delta, attribute):
return True
return False
def _handle_special_delta_attributes(self, delta, model_response: "ModelResponseStream") -> None:
"""
Handle special delta attributes (audio, image) by copying them to model_response.
"""
special_attributes = [AUDIO_ATTRIBUTE, IMAGE_ATTRIBUTE]
for attribute in special_attributes:
if self._has_special_delta_attribute(delta, attribute):
self._copy_delta_attribute(delta, model_response.choices[0].delta, attribute)
def return_processed_chunk_logic( # noqa
self,
completion_obj: Dict[str, Any],
@ -885,20 +956,8 @@ class CustomStreamWrapper:
self.sent_last_chunk = True
return model_response
elif (
model_response.choices[0].delta.tool_calls is not None
or model_response.choices[0].delta.function_call is not None
):
model_response = self.strip_role_from_delta(model_response)
return model_response
elif (
len(model_response.choices) > 0
and hasattr(model_response.choices[0].delta, "audio")
and model_response.choices[0].delta.audio is not None
):
model_response = self.strip_role_from_delta(model_response)
return model_response
elif self._has_special_delta_content(model_response):
return self._handle_special_delta_content(model_response)
else:
if hasattr(model_response, "usage"):
self.chunks.append(model_response)
@ -1371,10 +1430,8 @@ class CustomStreamWrapper:
)
)
model_response.choices[0].delta = Delta()
elif (
delta is not None and getattr(delta, "audio", None) is not None
):
model_response.choices[0].delta.audio = delta.audio
elif self._has_any_special_delta_attributes(delta):
self._handle_special_delta_attributes(delta, model_response)
else:
try:
delta = (
@ -1574,6 +1631,13 @@ class CustomStreamWrapper:
response = self.model_response_creator(
chunk=obj_dict, hidden_params=response._hidden_params
)
## check if empty
is_empty = is_model_response_stream_empty(
model_response=cast(ModelResponseStream, response)
)
if is_empty:
continue
# add usage as hidden param
if self.sent_last_chunk is True and self.stream_options is None:
usage = calculate_total_usage(chunks=self.chunks)
@ -1730,7 +1794,18 @@ class CustomStreamWrapper:
# Create a new object without the removed attribute
processed_chunk = self.model_response_creator(chunk=obj_dict)
is_empty = is_model_response_stream_empty(
model_response=cast(ModelResponseStream, processed_chunk)
)
if is_empty:
continue
print_verbose(f"final returned processed chunk: {processed_chunk}")
# add usage as hidden param
if self.sent_last_chunk is True and self.stream_options is None:
usage = calculate_total_usage(chunks=self.chunks)
processed_chunk._hidden_params["usage"] = usage
return processed_chunk
raise StopAsyncIteration
else: # temporary patch for non-aiohttp async calls
@ -1774,6 +1849,7 @@ class CustomStreamWrapper:
messages=self.messages,
logging_obj=self.logging_obj,
)
response = self.model_response_creator()
if complete_streaming_response is not None:
setattr(

2
litellm/litellm_core_utils/token_counter.py
View file

@ -529,7 +529,7 @@ def _get_count_function(
encoding = tiktoken.get_encoding("cl100k_base")
def count_tokens(text: str) -> int:
return len(encoding.encode(text))
return len(encoding.encode(text, disallowed_special=()))
else:
raise ValueError("Unsupported tokenizer type")

5
litellm/llms/aiml/__init__.py Normal file
View file

@ -0,0 +1,5 @@
from .image_generation import get_aiml_image_generation_config
__all__ = [
"get_aiml_image_generation_config",
]

23
litellm/llms/aiml/chat/transformation.py Normal file
View file

@ -0,0 +1,23 @@
from typing import Optional, Tuple
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
from litellm.secret_managers.main import get_secret_str
class AIMLChatConfig(OpenAIGPTConfig):
@property
def custom_llm_provider(self) -> Optional[str]:
return "aiml"
def _get_openai_compatible_provider_info(
self, api_base: Optional[str], api_key: Optional[str]
) -> Tuple[Optional[str], Optional[str]]:
# AIML is openai compatible, we just need to set the api_base
api_base = (
api_base
or get_secret_str("AIML_API_BASE")
or "https://api.aimlapi.com/v1" # Default AIML API base URL
) # type: ignore
dynamic_api_key = api_key or get_secret_str("AIML_API_KEY")
return api_base, dynamic_api_key
pass

13
litellm/llms/aiml/image_generation/__init__.py Normal file
View file

@ -0,0 +1,13 @@
from litellm.llms.base_llm.image_generation.transformation import (
BaseImageGenerationConfig,
)
from .transformation import AimlImageGenerationConfig
__all__ = [
"AimlImageGenerationConfig",
]
def get_aiml_image_generation_config(model: str) -> BaseImageGenerationConfig:
return AimlImageGenerationConfig()

25
litellm/llms/aiml/image_generation/cost_calculator.py Normal file
View file

@ -0,0 +1,25 @@
from typing import Any
import litellm
from litellm.types.utils import ImageResponse
def cost_calculator(
model: str,
image_response: Any,
) -> float:
"""
AI/ML flux image generation cost calculator
"""
_model_info = litellm.get_model_info(
model=model,
custom_llm_provider=litellm.LlmProviders.AIML.value,
)
output_cost_per_image: float = _model_info.get("output_cost_per_image") or 0.0
num_images: int = 0
if isinstance(image_response, ImageResponse):
if image_response.data:
num_images = len(image_response.data)
return output_cost_per_image * num_images
else:
raise ValueError(f"image_response must be of type ImageResponse got type={type(image_response)}")

204
litellm/llms/aiml/image_generation/transformation.py Normal file
View file

@ -0,0 +1,204 @@
from typing import TYPE_CHECKING, Any, List, Optional
import httpx
from litellm.llms.base_llm.image_generation.transformation import (
BaseImageGenerationConfig,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.aiml import AimlImageGenerationRequestParams
from litellm.types.llms.openai import (
AllMessageValues,
OpenAIImageGenerationOptionalParams,
)
from litellm.types.utils import ImageObject, ImageResponse
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
LiteLLMLoggingObj = Any
class AimlImageGenerationConfig(BaseImageGenerationConfig):
DEFAULT_BASE_URL: str = "https://api.aimlapi.com"
IMAGE_GENERATION_ENDPOINT: str = "v1/images/generations"
def get_supported_openai_params(
self, model: str
) -> List[OpenAIImageGenerationOptionalParams]:
"""
https://api.aimlapi.com/v1/images/generations
"""
return [
"n",
"response_format",
"size"
]
def map_openai_params(
self,
non_default_params: dict,
optional_params: dict,
model: str,
drop_params: bool,
) -> dict:
supported_params = self.get_supported_openai_params(model)
for k in non_default_params.keys():
if k not in optional_params.keys():
if k in supported_params:
# Map OpenAI params to AI/ML params
if k == "n":
optional_params["num_images"] = non_default_params[k]
elif k == "response_format":
optional_params["output_format"] = non_default_params[k]
elif k == "size":
# Map OpenAI size format to AI/ML image_size
size_value = non_default_params[k]
if isinstance(size_value, str):
# Handle standard OpenAI sizes like "1024x1024"
if "x" in size_value:
width, height = map(int, size_value.split("x"))
optional_params["image_size"] = {"width": width, "height": height}
else:
# Pass through predefined sizes
optional_params["image_size"] = size_value
else:
optional_params["image_size"] = size_value
else:
optional_params[k] = non_default_params[k]
elif drop_params:
pass
else:
raise ValueError(
f"Parameter {k} is not supported for model {model}. Supported parameters are {supported_params}. Set drop_params=True to drop unsupported parameters."
)
return optional_params
def get_complete_url(
self,
api_base: Optional[str],
api_key: Optional[str],
model: str,
optional_params: dict,
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
"""
Get the complete url for the request
"""
complete_url: str = (
api_base
or get_secret_str("AIML_API_BASE")
or self.DEFAULT_BASE_URL
)
complete_url = complete_url.rstrip("/")
complete_url = f"{complete_url}/{self.IMAGE_GENERATION_ENDPOINT}"
return complete_url
def validate_environment(
self,
headers: dict,
model: str,
messages: List[AllMessageValues],
optional_params: dict,
litellm_params: dict,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
) -> dict:
final_api_key: Optional[str] = (
api_key or
get_secret_str("AIML_API_KEY") or
get_secret_str("AIMLAPI_KEY") # Alternative name
)
if not final_api_key:
raise ValueError("AIML_API_KEY or AIMLAPI_KEY is not set")
headers["Authorization"] = f"Bearer {final_api_key}"
headers["Content-Type"] = "application/json"
return headers
def transform_image_generation_request(
self,
model: str,
prompt: str,
optional_params: dict,
litellm_params: dict,
headers: dict,
) -> dict:
"""
Transform the image generation request to the AI/ML flux image generation request body
https://api.aimlapi.com/v1/images/generations
"""
aiml_image_generation_request_body: AimlImageGenerationRequestParams = AimlImageGenerationRequestParams(
prompt=prompt,
model=model,
**optional_params,
)
return dict(aiml_image_generation_request_body)
def transform_image_generation_response(
self,
model: str,
raw_response: httpx.Response,
model_response: ImageResponse,
logging_obj: LiteLLMLoggingObj,
request_data: dict,
optional_params: dict,
litellm_params: dict,
encoding: Any,
api_key: Optional[str] = None,
json_mode: Optional[bool] = None,
) -> ImageResponse:
"""
Transform the image generation response to the litellm image response
https://api.aimlapi.com/v1/images/generations
"""
try:
response_data = raw_response.json()
except Exception as e:
raise self.get_error_class(
error_message=f"Error transforming image generation response: {e}",
status_code=raw_response.status_code,
headers=raw_response.headers,
)
if not model_response.data:
model_response.data = []
# AI/ML API can return images in two different formats:
# 1. output.choices array with image_base64
# 2. images array with url (and optional width, height, content_type)
if "output" in response_data and "choices" in response_data["output"]:
for choice in response_data["output"]["choices"]:
if "image_base64" in choice:
model_response.data.append(ImageObject(
b64_json=choice["image_base64"],
url=None, # AI/ML API returns base64, not URLs
))
elif "url" in choice:
model_response.data.append(ImageObject(
b64_json=None,
url=choice["url"],
))
elif "images" in response_data:
# Handle alternative format: {"images": [{"url": "...", "width": 1024, "height": 768, "content_type": "image/jpeg"}]}
for image in response_data["images"]:
if "url" in image:
model_response.data.append(ImageObject(
b64_json=None,
url=image["url"],
))
elif "image_base64" in image:
model_response.data.append(ImageObject(
b64_json=image["image_base64"],
url=None,
))
return model_response

3
litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py
View file

@ -489,7 +489,7 @@ class LiteLLMAnthropicMessagesAdapter:
text: str = ""
partial_json: Optional[str] = None
for choice in choices:
if choice.delta.content is not None:
if choice.delta.content is not None and len(choice.delta.content) > 0:
text += choice.delta.content
elif choice.delta.tool_calls is not None:
partial_json = ""
@ -499,7 +499,6 @@ class LiteLLMAnthropicMessagesAdapter:
and tool.function.arguments is not None
):
partial_json += tool.function.arguments
if partial_json is not None:
return "input_json_delta", ContentJsonBlockDelta(
type="input_json_delta", partial_json=partial_json

3
litellm/llms/azure/chat/gpt_5_transformation.py
View file

@ -10,6 +10,7 @@ from .gpt_transformation import AzureOpenAIConfig
class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
"""Azure specific handling for gpt-5 models."""
GPT5_SERIES_ROUTE = "gpt5_series/"
@classmethod
@ -23,7 +24,7 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
def get_supported_openai_params(self, model: str) -> List[str]:
return OpenAIGPT5Config.get_supported_openai_params(self, model=model)
def map_openai_params(
self,
non_default_params: dict,

Some files were not shown because too many files have changed in this diff Show more