scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-08 03:08:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 115

Merge remote-tracking branch 'origin/main' into fix_vertex_expired_tokens

Browse source
This commit is contained in:
Oz Ben-Ami 2025-08-24 11:04:19 -04:00
commit 3f57d67c85
541 changed files with 46518 additions and 10634 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

112
.circleci/config.yml
View file

@ -83,8 +83,8 @@ jobs:
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install pyarrow
pip install "boto3==1.34.34"
pip install "aioboto3==12.3.0"
pip install "boto3==1.36.0"
pip install "aioboto3==13.4.0"
pip install langchain
pip install lunary==0.2.5
pip install "azure-identity==1.16.1"
@ -95,7 +95,7 @@ jobs:
pip install opentelemetry-api==1.25.0
pip install opentelemetry-sdk==1.25.0
pip install opentelemetry-exporter-otlp==1.25.0
pip install openai==1.81.0
pip install openai==1.100.1
pip install prisma==0.11.0
pip install "detect_secrets==1.5.0"
pip install "httpx==0.24.1"
@ -208,8 +208,8 @@ jobs:
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install pyarrow
pip install "boto3==1.34.34"
pip install "aioboto3==12.3.0"
pip install "boto3==1.36.0"
pip install "aioboto3==13.4.0"
pip install langchain
pip install lunary==0.2.5
pip install "azure-identity==1.16.1"
@ -220,7 +220,7 @@ jobs:
pip install opentelemetry-api==1.25.0
pip install opentelemetry-sdk==1.25.0
pip install opentelemetry-exporter-otlp==1.25.0
pip install openai==1.81.0
pip install openai==1.100.1
pip install prisma==0.11.0
pip install "detect_secrets==1.5.0"
pip install "httpx==0.24.1"
@ -315,8 +315,8 @@ jobs:
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install pyarrow
pip install "boto3==1.34.34"
pip install "aioboto3==12.3.0"
pip install "boto3==1.36.0"
pip install "aioboto3==13.4.0"
pip install langchain
pip install lunary==0.2.5
pip install "azure-identity==1.16.1"
@ -327,7 +327,7 @@ jobs:
pip install opentelemetry-api==1.25.0
pip install opentelemetry-sdk==1.25.0
pip install opentelemetry-exporter-otlp==1.25.0
pip install openai==1.81.0
pip install openai==1.100.1
pip install prisma==0.11.0
pip install "detect_secrets==1.5.0"
pip install "httpx==0.24.1"
@ -439,6 +439,7 @@ jobs:
paths:
- auth_ui_unit_tests_coverage.xml
- auth_ui_unit_tests_coverage
litellm_router_testing: # Runs all tests with the "router" keyword
docker:
- image: cimg/python:3.11
@ -469,7 +470,55 @@ jobs:
command: |
pwd
ls
python -m pytest tests/local_testing tests/router_unit_tests --cov=litellm --cov-report=xml -vv -k "router" -x -v --junitxml=test-results/junit.xml --durations=5
python -m pytest tests/local_testing --cov=litellm --cov-report=xml -vv -k "router" -x -v --junitxml=test-results/junit.xml --durations=5
no_output_timeout: 120m
- run:
name: Rename the coverage files
command: |
mv coverage.xml litellm_router_coverage.xml
mv .coverage litellm_router_coverage
# Store test results
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- litellm_router_coverage.xml
- litellm_router_coverage
litellm_router_unit_testing: # Runs all tests with the "router" keyword
docker:
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
steps:
- checkout
- setup_google_dns
- run:
name: Install Dependencies
command: |
python -m pip install --upgrade pip
python -m pip install -r requirements.txt
pip install "pytest==7.3.1"
pip install "respx==0.22.0"
pip install "pytest-cov==5.0.0"
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
pip install semantic_router --no-deps
pip install aurelio_sdk --no-deps
pip install "pytest-xdist==3.6.1"
# Run pytest and generate JUnit XML report
- setup_litellm_enterprise_pip
- run:
name: Run tests
command: |
pwd
ls
python -m pytest -vv tests/router_unit_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
no_output_timeout: 120m
- run:
name: Rename the coverage files
@ -590,8 +639,8 @@ jobs:
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install pyarrow
pip install "boto3==1.34.34"
pip install "aioboto3==12.3.0"
pip install "boto3==1.36.0"
pip install "aioboto3==13.4.0"
pip install langchain
pip install lunary==0.2.5
pip install "azure-identity==1.16.1"
@ -602,7 +651,7 @@ jobs:
pip install opentelemetry-api==1.25.0
pip install opentelemetry-sdk==1.25.0
pip install opentelemetry-exporter-otlp==1.25.0
pip install openai==1.81.0
pip install openai==1.100.1
pip install prisma==0.11.0
pip install "detect_secrets==1.5.0"
pip install "httpx==0.24.1"
@ -816,7 +865,7 @@ jobs:
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.22.0"
pip install "pydantic==2.10.2"
pip install "boto3==1.34.34"
pip install "boto3==1.36.0"
# Run pytest and generate JUnit XML report
- run:
name: Run tests
@ -957,6 +1006,7 @@ jobs:
pip install "responses==0.25.7"
pip install "pytest-xdist==3.6.1"
pip install "semantic_router==0.1.10"
pip install "fastapi-offline==1.7.3"
- setup_litellm_enterprise_pip
# Run pytest and generate JUnit XML report
- run:
@ -1237,7 +1287,7 @@ jobs:
pip install aiohttp
pip install openai
pip install click
pip install "boto3==1.34.34"
pip install "boto3==1.36.0"
pip install jinja2
pip install "tokenizers==0.20.0"
pip install "uvloop==0.21.0"
@ -1506,8 +1556,8 @@ jobs:
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install pyarrow
pip install "boto3==1.34.34"
pip install "aioboto3==12.3.0"
pip install "boto3==1.36.0"
pip install "aioboto3==13.4.0"
pip install langchain
pip install "langfuse>=2.0.0"
pip install "logfire==0.29.0"
@ -1521,7 +1571,7 @@ jobs:
pip install "aiodynamo==23.10.1"
pip install "asyncio==3.4.3"
pip install "PyGithub==1.59.1"
pip install "openai==1.81.0"
pip install "openai==1.100.1"
- run:
name: Install dockerize
command: |
@ -1662,8 +1712,8 @@ jobs:
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install pyarrow
pip install "boto3==1.34.34"
pip install "aioboto3==12.3.0"
pip install "boto3==1.36.0"
pip install "aioboto3==13.4.0"
pip install langchain
pip install "langchain_mcp_adapters==0.0.5"
pip install "langfuse>=2.0.0"
@ -1678,7 +1728,7 @@ jobs:
pip install "aiodynamo==23.10.1"
pip install "asyncio==3.4.3"
pip install "PyGithub==1.59.1"
pip install "openai==1.81.0"
pip install "openai==1.100.1"
# Run pytest and generate JUnit XML report
- run:
name: Install dockerize
@ -1803,8 +1853,8 @@ jobs:
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install pyarrow
pip install "boto3==1.34.34"
pip install "aioboto3==12.3.0"
pip install "boto3==1.36.0"
pip install "aioboto3==13.4.0"
pip install langchain
pip install "langfuse>=2.0.0"
pip install "logfire==0.29.0"
@ -1818,7 +1868,7 @@ jobs:
pip install "aiodynamo==23.10.1"
pip install "asyncio==3.4.3"
pip install "PyGithub==1.59.1"
pip install "openai==1.81.0"
pip install "openai==1.100.1"
- run:
name: Install dockerize
command: |
@ -2398,14 +2448,14 @@ jobs:
pip install "pytest-asyncio==0.21.1"
pip install "google-cloud-aiplatform==1.43.0"
pip install aiohttp
pip install "openai==1.81.0"
pip install "openai==1.100.1"
pip install "assemblyai==0.37.0"
python -m pip install --upgrade pip
pip install "pydantic==2.10.2"
pip install "pytest==7.3.1"
pip install "pytest-mock==3.12.0"
pip install "pytest-asyncio==0.21.1"
pip install "boto3==1.34.34"
pip install "boto3==1.36.0"
pip install mypy
pip install pyarrow
pip install numpydoc
@ -2789,7 +2839,7 @@ jobs:
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
pip install aiohttp
pip install "openai==1.81.0"
pip install "openai==1.100.1"
python -m pip install --upgrade pip
pip install "pydantic==2.10.2"
pip install "pytest==7.3.1"
@ -2981,6 +3031,12 @@ workflows:
only:
- main
- /litellm_.*/
- litellm_router_unit_testing:
filters:
branches:
only:
- main
- /litellm_.*/
- check_code_and_doc_quality:
filters:
branches:
@ -3127,6 +3183,7 @@ workflows:
- image_gen_testing
- logging_testing
- litellm_router_testing
- litellm_router_unit_testing
- caching_unit_tests
- litellm_proxy_unit_testing
- litellm_security_tests
@ -3186,6 +3243,7 @@ workflows:
- image_gen_testing
- logging_testing
- litellm_router_testing
- litellm_router_unit_testing
- caching_unit_tests
- langfuse_logging_unit_tests
- litellm_assistants_api_testing

2
.circleci/requirements.txt
View file

@ -1,5 +1,5 @@
# used by CI/CD testing
openai==1.81.0
openai==1.100.1
python-dotenv
tiktoken
importlib_metadata

133
.github/scripts/scan_keywords.py vendored Normal file
View file

@ -0,0 +1,133 @@
#!/usr/bin/env python3
import json
import os
import sys
import urllib.request
import urllib.error
def read_event_payload() -> dict:
event_path = os.environ.get("GITHUB_EVENT_PATH")
if not event_path or not os.path.exists(event_path):
return {}
with open(event_path, "r", encoding="utf-8") as f:
return json.load(f)
def get_issue_text(event: dict) -> tuple[str, str, int, str, str]:
issue = event.get("issue") or {}
title = (issue.get("title") or "").strip()
body = (issue.get("body") or "").strip()
number = issue.get("number") or 0
html_url = issue.get("html_url") or ""
author = ((issue.get("user") or {}).get("login") or "").strip()
return title, body, number, html_url, author
def detect_keywords(text: str, keywords: list[str]) -> list[str]:
lowered = text.lower()
matches = []
for keyword in keywords:
k = keyword.strip().lower()
if not k:
continue
if k in lowered:
matches.append(keyword.strip())
# Deduplicate while preserving order
seen = set()
unique_matches = []
for m in matches:
if m not in seen:
unique_matches.append(m)
seen.add(m)
return unique_matches
def send_webhook(webhook_url: str, payload: dict) -> None:
if not webhook_url:
return
data = json.dumps(payload).encode("utf-8")
req = urllib.request.Request(
webhook_url,
data=data,
headers={"Content-Type": "application/json"},
method="POST",
)
try:
with urllib.request.urlopen(req, timeout=10) as resp:
resp.read()
except urllib.error.HTTPError as e:
print(f"Webhook HTTP error: {e.code} {e.reason}", file=sys.stderr)
except urllib.error.URLError as e:
print(f"Webhook URL error: {e.reason}", file=sys.stderr)
except Exception as e:
print(f"Webhook unexpected error: {e}", file=sys.stderr)
def _excerpt(text: str, max_len: int = 400) -> str:
if not text:
return ""
# Keep original formatting
if len(text) <= max_len:
return text
return text[: max_len - 1] + "…"
def main() -> int:
event = read_event_payload()
if not event:
print("::warning::No event payload found; exiting without labeling.")
return 0
# Read issue details
title, body, number, html_url, author = get_issue_text(event)
combined_text = f"{title}\n\n{body}".strip()
# Keywords from env or defaults
keywords_env = os.environ.get("KEYWORDS", "")
default_keywords = ["azure", "openai", "bedrock", "vertexai", "vertex ai", "anthropic"]
keywords = [k.strip() for k in keywords_env.split(",")] if keywords_env else default_keywords
matches = detect_keywords(combined_text, keywords)
found = bool(matches)
# Emit outputs
github_output = os.environ.get("GITHUB_OUTPUT")
if github_output:
with open(github_output, "a", encoding="utf-8") as fh:
fh.write(f"found={'true' if found else 'false'}\n")
fh.write(f"matches={','.join(matches)}\n")
# Optional webhook notification
webhook_url = os.environ.get("PROVIDER_ISSUE_WEBHOOK_URL", "").strip()
if found and webhook_url:
repo_full = (event.get("repository") or {}).get("full_name", "")
title_part = f"*{title}*" if title else "New issue"
author_part = f" by @{author}" if author else ""
body_preview = _excerpt(body)
preview_block = f"\n{body_preview}" if body_preview else ""
payload = {
"text": (
f"New issue 🚨\n"
f"{title_part}\n\n{preview_block}\n"
f"<{html_url}|View issue>\n"
f"Author: {author}"
)
}
send_webhook(webhook_url, payload)
# Print a short log line for Actions UI
if found:
print(f"Detected provider keywords: {', '.join(matches)}")
else:
print("No provider keywords detected.")
return 0
if __name__ == "__main__":
raise SystemExit(main())

64
.github/workflows/issue-keyword-labeler.yml vendored Normal file
View file

@ -0,0 +1,64 @@
name: Issue Keyword Labeler
on:
issues:
types:
- opened
jobs:
scan-and-label:
runs-on: ubuntu-latest
permissions:
issues: write
contents: read
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Scan for provider keywords
id: scan
env:
PROVIDER_ISSUE_WEBHOOK_URL: ${{ secrets.PROVIDER_ISSUE_WEBHOOK_URL }}
KEYWORDS: azure,openai,bedrock,vertexai,vertex ai,anthropic
run: python3 .github/scripts/scan_keywords.py
- name: Ensure label exists
if: steps.scan.outputs.found == 'true'
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const labelName = 'llm translation';
try {
await github.rest.issues.getLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName
});
} catch (error) {
if (error.status === 404) {
await github.rest.issues.createLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName,
color: 'c1ff72',
description: 'Issues related to LLM provider translation/mapping'
});
} else {
throw error;
}
}
- name: Add label to the issue
if: steps.scan.outputs.found == 'true'
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
labels: ['llm translation']
});

9
.github/workflows/test-linting.yml vendored
View file

@ -22,11 +22,8 @@ jobs:
- name: Install dependencies
run: |
pip install openai==1.81.0
poetry install --with dev
pip install openai==1.81.0
poetry run pip install openai==1.100.1
- name: Run Black formatting
run: |
@ -40,6 +37,10 @@ jobs:
poetry run ruff check .
cd ..
- name: Print OpenAI version
run: |
poetry run python -c "import openai; print(f'OpenAI version: {openai.__version__}')"
- name: Run MyPy type checking
run: |
cd litellm

3
.github/workflows/test-litellm.yml vendored
View file

@ -7,7 +7,7 @@ on:
jobs:
test:
runs-on: ubuntu-latest
timeout-minutes: 20
timeout-minutes: 25
steps:
- uses: actions/checkout@v4
@ -31,6 +31,7 @@ jobs:
poetry run pip install "pytest-retry==1.6.3"
poetry run pip install pytest-xdist
poetry run pip install "google-genai==1.22.0"
poetry run pip install "fastapi-offline==1.7.3"
- name: Setup litellm-enterprise as local package
run: |
cd enterprise

4
.gitignore vendored
View file

@ -86,6 +86,7 @@ litellm/proxy/db/migrations/0_init/migration.sql
litellm/proxy/db/migrations/*
litellm/proxy/migrations/*config.yaml
litellm/proxy/migrations/*
litellm/proxy/to_delete_loadtest_work/*
config.yaml
tests/litellm/litellm_core_utils/llm_cost_calc/log.txt
tests/test_custom_dir/*
@ -93,4 +94,5 @@ test.py
litellm_config.yaml
.cursor
.vscode/launch.json
.vscode/launch.json
litellm/proxy/to_delete_loadtest_work/*

4
Dockerfile
View file

@ -65,8 +65,8 @@ COPY --from=builder /wheels/ /wheels/
# Install the built wheel using pip; again using a wildcard if it's the only file
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
# Install semantic_router without dependencies
RUN pip install semantic_router --no-deps
# Install semantic_router and aurelio-sdk using script
RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
# Generate prisma client
RUN prisma generate

6
Makefile
View file

@ -34,13 +34,13 @@ install-proxy-dev:
# CI-compatible installations (matches GitHub workflows exactly)
install-dev-ci:
pip install openai==1.81.0
pip install openai==1.99.5
poetry install --with dev
pip install openai==1.81.0
pip install openai==1.99.5
install-proxy-dev-ci:
poetry install --with dev,proxy-dev --extras proxy
pip install openai==1.81.0
pip install openai==1.99.5
install-test-deps: install-proxy-dev
poetry run pip install "pytest-retry==1.6.3"

23
README.md
View file

@ -47,7 +47,7 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature
# Usage ([**Docs**](https://docs.litellm.ai/docs/))
> [!IMPORTANT]
> LiteLLM v1.0.0 now requires `openai>=1.0.0`. Migration guide [here](https://docs.litellm.ai/docs/migration)
> LiteLLM v1.0.0 now requires `openai>=1.0.0`. Migration guide [here](https://docs.litellm.ai/docs/migration)
> LiteLLM v1.40.14+ now requires `pydantic>=2.0.0`. No changes required.
<a target="_blank" href="https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/liteLLM_Getting_Started.ipynb">
@ -132,7 +132,7 @@ print(response)
## Streaming ([Docs](https://docs.litellm.ai/docs/completion/stream))
liteLLM supports streaming the model response back, pass `stream=True` to get a streaming iterator in response.
liteLLM supports streaming the model response back, pass `stream=True` to get a streaming iterator in response.
Streaming is supported for all models (Bedrock, Huggingface, TogetherAI, Azure, OpenAI, etc.)
```python
@ -234,7 +234,7 @@ $ litellm --model huggingface/bigcode/starcoder
> [!IMPORTANT]
> 💡 [Use LiteLLM Proxy with Langchain (Python, JS), OpenAI SDK (Python, JS) Anthropic SDK, Mistral SDK, LlamaIndex, Instructor, Curl](https://docs.litellm.ai/docs/proxy/user_keys)
> 💡 [Use LiteLLM Proxy with Langchain (Python, JS), OpenAI SDK (Python, JS) Anthropic SDK, Mistral SDK, LlamaIndex, Instructor, Curl](https://docs.litellm.ai/docs/proxy/user_keys)
```python
import openai # openai v1.0.0+
@ -266,7 +266,7 @@ echo 'LITELLM_MASTER_KEY="sk-1234"' > .env
# Add the litellm salt key - you cannot change this after adding a model
# It is used to encrypt / decrypt your LLM API Key credentials
# We recommend - https://1password.com/password-generator/
# We recommend - https://1password.com/password-generator/
# password generator to get a random hash for litellm salt key
echo 'LITELLM_SALT_KEY="sk-1234"' >> .env
@ -340,6 +340,7 @@ curl 'http://0.0.0.0:4000/key/generate' \
| [xinference [Xorbits Inference]](https://docs.litellm.ai/docs/providers/xinference) | | | | | ✅ | |
| [FriendliAI](https://docs.litellm.ai/docs/providers/friendliai) | ✅ | ✅ | ✅ | ✅ | | |
| [Galadriel](https://docs.litellm.ai/docs/providers/galadriel) | ✅ | ✅ | ✅ | ✅ | | |
| [GradientAI](https://docs.litellm.ai/docs/providers/gradient_ai) | ✅ | ✅ | | | | |
| [Novita AI](https://novita.ai/models/llm?utm_source=github_litellm&utm_medium=github_readme&utm_campaign=github_link) | ✅ | ✅ | ✅ | ✅ | | |
| [Featherless AI](https://docs.litellm.ai/docs/providers/featherless_ai) | ✅ | ✅ | ✅ | ✅ | | |
| [Nebius AI Studio](https://docs.litellm.ai/docs/providers/nebius) | ✅ | ✅ | ✅ | ✅ | ✅ | |
@ -348,7 +349,7 @@ curl 'http://0.0.0.0:4000/key/generate' \
## Contributing
Interested in contributing? Contributions to LiteLLM Python SDK, Proxy Server, and LLM integrations are both accepted and highly encouraged!
Interested in contributing? Contributions to LiteLLM Python SDK, Proxy Server, and LLM integrations are both accepted and highly encouraged!
**Quick start:** `git clone` → `make install-dev` → `make format` → `make lint` → `make test-unit`
@ -359,7 +360,7 @@ For companies that need better security, user management and professional suppor
[Talk to founders](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
This covers:
This covers:
- ✅ **Features under the [LiteLLM Commercial License](https://docs.litellm.ai/docs/proxy/enterprise):**
- ✅ **Feature Prioritization**
- ✅ **Custom Integrations**
@ -373,6 +374,8 @@ We welcome contributions to LiteLLM! Whether you're fixing bugs, adding features
## Quick Start for Contributors
This requires poetry to be installed.
```bash
git clone https://github.com/BerriAI/litellm.git
cd litellm
@ -380,6 +383,7 @@ make install-dev # Install development dependencies
make format # Format your code
make lint # Run all linting checks
make test-unit # Run unit tests
make format-check # Check formatting only
```
For detailed contributing guidelines, see [CONTRIBUTING.md](CONTRIBUTING.md).
@ -395,11 +399,6 @@ Our automated checks include:
- **Circular import detection**
- **Import safety checks**
Run all checks locally:
```bash
make lint # Run all linting (matches CI)
make format-check # Check formatting only
```
All these checks must pass before your PR can be merged.
@ -441,7 +440,7 @@ All these checks must pass before your PR can be merged.
1. (In root) create virtual environment `python -m venv .venv`
2. Activate virtual environment `source .venv/bin/activate`
3. Install dependencies `pip install -e ".[all]"`
4. Start proxy backend `uvicorn litellm.proxy.proxy_server:app --host localhost --port 4000 --reload`
4. Start proxy backend `python3 /path/to/litellm/proxy_cli.py`
### Frontend
1. Navigate to `ui/litellm-dashboard`

213
cookbook/liteLLM_Baseten.ipynb vendored
View file

@ -6,19 +6,21 @@
"id": "gZx-wHJapG5w"
},
"source": [
"# Use liteLLM to call Falcon, Wizard, MPT 7B using OpenAI chatGPT Input/output\n",
"# LiteLLM with Baseten Model APIs\n",
"\n",
"* Falcon 7B: https://app.baseten.co/explore/falcon_7b\n",
"* Wizard LM: https://app.baseten.co/explore/wizardlm\n",
"* MPT 7B Base: https://app.baseten.co/explore/mpt_7b_instruct\n",
"This notebook demonstrates how to use LiteLLM with Baseten's Model APIs instead of dedicated deployments.\n",
"\n",
"\n",
"## Call all baseten llm models using OpenAI chatGPT Input/Output using liteLLM\n",
"Example call\n",
"## Example Usage\n",
"```python\n",
"model = \"q841o8w\" # baseten model version ID\n",
"response = completion(model=model, messages=messages, custom_llm_provider=\"baseten\")\n",
"```"
"response = completion(\n",
" model=\"baseten/openai/gpt-oss-120b\",\n",
" messages=[{\"role\": \"user\", \"content\": \"Hello!\"}],\n",
" max_tokens=1000,\n",
" temperature=0.7\n",
")\n",
"```\n",
"\n",
"## Setup"
]
},
{
@ -29,20 +31,25 @@
},
"outputs": [],
"source": [
"!pip install litellm==0.1.399\n",
"!pip install baseten urllib3"
"%pip install litellm"
]
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": null,
"metadata": {
"id": "VEukLhDzo4vw"
},
"outputs": [],
"source": [
"import os\n",
"from litellm import completion"
"from litellm import completion\n",
"\n",
"# Set your Baseten API key\n",
"os.environ['BASETEN_API_KEY'] = \"\" #@param {type:\"string\"}\n",
"\n",
"# Test message\n",
"messages = [{\"role\": \"user\", \"content\": \"What is AGI?\"}]"
]
},
{
@ -51,19 +58,31 @@
"id": "4STYM2OHFNlc"
},
"source": [
"## Setup"
"## Example 1: Basic Completion\n",
"\n",
"Simple completion with the GPT-OSS 120B model"
]
},
{
"cell_type": "code",
"execution_count": 21,
"execution_count": null,
"metadata": {
"id": "DorpLxw1FHbC"
},
"outputs": [],
"source": [
"os.environ['BASETEN_API_KEY'] = \"\" #@param\n",
"messages = [{ \"content\": \"what does Baseten do? \",\"role\": \"user\"}]"
"print(\"=== Basic Completion ===\")\n",
"response = completion(\n",
" model=\"baseten/openai/gpt-oss-120b\",\n",
" messages=messages,\n",
" max_tokens=1000,\n",
" temperature=0.7,\n",
" top_p=0.9,\n",
" presence_penalty=0.1,\n",
" frequency_penalty=0.1,\n",
")\n",
"print(f\"Response: {response.choices[0].message.content}\")\n",
"print(f\"Usage: {response.usage}\")"
]
},
{
@ -72,13 +91,14 @@
"id": "syF3dTdKFSQQ"
},
"source": [
"## Calling Falcon 7B: https://app.baseten.co/explore/falcon_7b\n",
"### Pass Your Baseten model `Version ID` as `model`"
"## Example 2: Streaming Completion\n",
"\n",
"Streaming completion with usage statistics"
]
},
{
"cell_type": "code",
"execution_count": 18,
"execution_count": null,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
@ -86,137 +106,26 @@
"id": "rPgSoMlsojz0",
"outputId": "81d6dc7b-1681-4ae4-e4c8-5684eb1bd050"
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[32mINFO\u001b[0m API key set.\n",
"INFO:baseten:API key set.\n"
]
},
{
"data": {
"text/plain": [
"{'choices': [{'finish_reason': 'stop',\n",
" 'index': 0,\n",
" 'message': {'role': 'assistant',\n",
" 'content': \"what does Baseten do? \\nI'm sorry, I cannot provide a specific answer as\"}}],\n",
" 'created': 1692135883.699066,\n",
" 'model': 'qvv0xeq'}"
]
},
"execution_count": 18,
"metadata": {},
"output_type": "execute_result"
}
],
"outputs": [],
"source": [
"model = \"qvv0xeq\"\n",
"response = completion(model=model, messages=messages, custom_llm_provider=\"baseten\")\n",
"response"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "7n21UroEGCGa"
},
"source": [
"## Calling Wizard LM https://app.baseten.co/explore/wizardlm\n",
"### Pass Your Baseten model `Version ID` as `model`"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "uLVWFH899lAF",
"outputId": "61c2bc74-673b-413e-bb40-179cf408523d"
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[32mINFO\u001b[0m API key set.\n",
"INFO:baseten:API key set.\n"
]
},
{
"data": {
"text/plain": [
"{'choices': [{'finish_reason': 'stop',\n",
" 'index': 0,\n",
" 'message': {'role': 'assistant',\n",
" 'content': 'As an AI language model, I do not have personal beliefs or practices, but based on the information available online, Baseten is a popular name for a traditional Ethiopian dish made with injera, a spongy flatbread, and wat, a spicy stew made with meat or vegetables. It is typically served for breakfast or dinner and is a staple in Ethiopian cuisine. The name Baseten is also used to refer to a traditional Ethiopian coffee ceremony, where coffee is brewed and served in a special ceremony with music and food.'}}],\n",
" 'created': 1692135900.2806294,\n",
" 'model': 'q841o8w'}"
]
},
"execution_count": 19,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model = \"q841o8w\"\n",
"response = completion(model=model, messages=messages, custom_llm_provider=\"baseten\")\n",
"response"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "6-TFwmPAGPXq"
},
"source": [
"## Calling mosaicml/mpt-7b https://app.baseten.co/explore/mpt_7b_instruct\n",
"### Pass Your Baseten model `Version ID` as `model`"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "gbeYZOrUE_Bp",
"outputId": "838d86ea-2143-4cb3-bc80-2acc2346c37a"
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[32mINFO\u001b[0m API key set.\n",
"INFO:baseten:API key set.\n"
]
},
{
"data": {
"text/plain": [
"{'choices': [{'finish_reason': 'stop',\n",
" 'index': 0,\n",
" 'message': {'role': 'assistant',\n",
" 'content': \"\\n===================\\n\\nIt's a tool to build a local version of a game on your own machine to host\\non your website.\\n\\nIt's used to make game demos and show them on Twitter, Tumblr, and Facebook.\\n\\n\\n\\n## What's built\\n\\n- A directory of all your game directories, named with a version name and build number, with images linked to.\\n- Includes HTML to include in another site.\\n- Includes images for your icons and\"}}],\n",
" 'created': 1692135914.7472186,\n",
" 'model': '31dxrj3'}"
]
},
"execution_count": 20,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model = \"31dxrj3\"\n",
"response = completion(model=model, messages=messages, custom_llm_provider=\"baseten\")\n",
"response"
"print(\"=== Streaming Completion ===\")\n",
"response = completion(\n",
" model=\"baseten/openai/gpt-oss-120b\",\n",
" messages=[{\"role\": \"user\", \"content\": \"Write a short poem about AI\"}],\n",
" stream=True,\n",
" max_tokens=500,\n",
" temperature=0.8,\n",
" stream_options={\n",
" \"include_usage\": True,\n",
" \"continuous_usage_stats\": True\n",
" },\n",
")\n",
"\n",
"print(\"Streaming response:\")\n",
"for chunk in response:\n",
" if chunk.choices and chunk.choices[0].delta.content:\n",
" print(chunk.choices[0].delta.content, end=\"\", flush=True)\n",
"print(\"\\n\")"
]
}
],
@ -234,4 +143,4 @@
},
"nbformat": 4,
"nbformat_minor": 0
}
}

2
deploy/charts/litellm-helm/Chart.yaml
View file

@ -18,7 +18,7 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.4.4
version: 0.4.5
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to

4
deploy/charts/litellm-helm/README.md
View file

@ -24,7 +24,7 @@ If `db.useStackgresOperator` is used (not yet implemented):
| `replicaCount` | The number of LiteLLM Proxy pods to be deployed | `1` |
| `masterkeySecretName` | The name of the Kubernetes Secret that contains the Master API Key for LiteLLM. If not specified, use the generated secret name. | N/A |
| `masterkeySecretKey` | The key within the Kubernetes Secret that contains the Master API Key for LiteLLM. If not specified, use `masterkey` as the key. | N/A |
| `masterkey` | The Master API Key for LiteLLM. If not specified, a random key is generated. | N/A |
| `masterkey` | The Master API Key for LiteLLM. If not specified, a random key in the `sk-...` format is generated. | N/A |
| `environmentSecrets` | An optional array of Secret object names. The keys and values in these secrets will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` |
| `environmentConfigMaps` | An optional array of ConfigMap object names. The keys and values in these configmaps will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` |
| `image.repository` | LiteLLM Proxy image repository | `ghcr.io/berriai/litellm` |
@ -135,7 +135,7 @@ service, the **Proxy Endpoint** should be set to `http://<RELEASE>-litellm:4000`
The **Proxy Key** is the value specified for `masterkey` or, if a `masterkey`
was not provided to the helm command line, the `masterkey` is a randomly
generated string stored in the `<RELEASE>-litellm-masterkey` Kubernetes Secret.
generated string in the `sk-...` format stored in the `<RELEASE>-litellm-masterkey` Kubernetes Secret.
```bash
kubectl -n litellm get secret <RELEASE>-litellm-masterkey -o jsonpath="{.data.masterkey}"

7
deploy/charts/litellm-helm/templates/deployment.yaml
View file

@ -71,7 +71,14 @@ spec:
name: {{ .Values.db.secret.name }}
key: {{ .Values.db.secret.passwordKey }}
- name: DATABASE_HOST
{{- if .Values.db.secret.endpointKey }}
valueFrom:
secretKeyRef:
name: {{ .Values.db.secret.name }}
key: {{ .Values.db.secret.endpointKey }}
{{- else }}
value: {{ .Values.db.endpoint }}
{{- end }}
- name: DATABASE_NAME
value: {{ .Values.db.database }}
- name: DATABASE_URL

11
deploy/charts/litellm-helm/templates/migrations-job.yaml
View file

@ -49,7 +49,14 @@ spec:
name: {{ .Values.db.secret.name }}
key: {{ .Values.db.secret.passwordKey }}
- name: DATABASE_HOST
{{- if .Values.db.secret.endpointKey }}
valueFrom:
secretKeyRef:
name: {{ .Values.db.secret.name }}
key: {{ .Values.db.secret.endpointKey }}
{{- else }}
value: {{ .Values.db.endpoint }}
{{- end }}
- name: DATABASE_NAME
value: {{ .Values.db.database }}
- name: DATABASE_URL
@ -73,6 +80,10 @@ spec:
volumeMounts:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.migrationJob.resources }}
resources:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.migrationJob.extraContainers }}
{{- toYaml . | nindent 8 }}
{{- end }}

2
deploy/charts/litellm-helm/templates/secret-masterkey.yaml
View file

@ -1,5 +1,5 @@
{{- if not .Values.masterkeySecretName }}
{{ $masterkey := (.Values.masterkey | default (randAlphaNum 17)) }}
{{ $masterkey := (.Values.masterkey | default (printf "sk-%s" (randAlphaNum 18))) }}
apiVersion: v1
kind: Secret
metadata:

8
deploy/charts/litellm-helm/tests/masterkey-secret_tests.yaml
View file

@ -2,13 +2,19 @@ suite: test masterkey secret
templates:
- secret-masterkey.yaml
tests:
- it: should create a secret if masterkeySecretName is not set
- it: should create a secret if masterkeySecretName is not set. should start with sk-xxxx (base64 encoded as c2st*)
template: secret-masterkey.yaml
set:
masterkeySecretName: ""
asserts:
- isKind:
of: Secret
- matchRegex:
path: data.masterkey
pattern: ^c2st
# Note: The masterkey is generated as "sk-<18-random-chars>" in plain text,
# but stored as base64 encoded in Kubernetes secret (requirement).
# "sk-" base64 encodes to "c2st", so we check for "^c2st" pattern.
- it: should not create a secret if masterkeySecretName is set
template: secret-masterkey.yaml
set:

6
deploy/charts/litellm-helm/values.yaml
View file

@ -161,6 +161,8 @@ db:
name: postgres
usernameKey: username
passwordKey: password
# Optional: when set, DATABASE_HOST will be sourced from this secret key instead of db.endpoint
endpointKey: ""
# Use the Stackgres Helm chart to deploy an instance of a Stackgres cluster.
# The Stackgres Operator must already be installed within the target
@ -206,6 +208,10 @@ migrationJob:
disableSchemaUpdate: false # Skip schema migrations for specific environments. When True, the job will exit with code 0.
annotations: {}
ttlSecondsAfterFinished: 120
resources: {}
# requests:
# cpu: 100m
# memory: 100Mi
extraContainers: []
# Hook configuration

4
docker/Dockerfile.database
View file

@ -57,8 +57,8 @@ COPY --from=builder /wheels/ /wheels/
# Install the built wheel using pip; again using a wildcard if it's the only file
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
# Install semantic_router without dependencies
RUN pip install semantic_router --no-deps
# Install semantic_router and aurelio-sdk using script
RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
# ensure pyjwt is used, not jwt
RUN pip uninstall jwt -y

8
docker/Dockerfile.non_root
View file

@ -47,8 +47,8 @@ RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ \
&& rm -f *.whl \
&& rm -rf /wheels
# Install semantic_router without dependencies
RUN pip install semantic_router --no-deps
# Install semantic_router and aurelio-sdk using script
RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
# Ensure correct JWT library is used (pyjwt not jwt)
RUN pip uninstall jwt -y && \
@ -70,7 +70,9 @@ RUN mkdir -p /nonexistent /.npm && \
chown -R nobody:nogroup /app && \
chown -R nobody:nogroup /nonexistent /.npm && \
PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \
chown -R nobody:nogroup $PRISMA_PATH
chown -R nobody:nogroup $PRISMA_PATH && \
LITELLM_PKG_MIGRATIONS_PATH="$(python -c 'import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))' 2>/dev/null || echo '')/migrations" && \
[ -n "$LITELLM_PKG_MIGRATIONS_PATH" ] && chown -R nobody:nogroup $LITELLM_PKG_MIGRATIONS_PATH
# --- OpenShift Compatibility: Apply Red Hat recommended pattern ---
# Get paths for directories that need write access at runtime

1
docker/build_from_pip/requirements.txt
View file

@ -2,4 +2,5 @@ litellm[proxy]==1.67.4.dev1 # Specify the litellm version you want to use
prometheus_client
langfuse
prisma
openai==1.99.9
ddtrace==2.19.0 # for advanced DD tracing / profiling

3
docker/install_auto_router.sh Executable file
View file

@ -0,0 +1,3 @@
#!/bin/bash
pip install semantic_router==0.1.11 --no-deps
pip install aurelio-sdk==0.0.19

66
docs/my-website/docs/completion/document_understanding.md
View file

@ -10,6 +10,7 @@ Works for:
- Bedrock Models
- Anthropic API Models
- OpenAI API Models
- Mistral (Only using file ID of already uploaded file, similar to OpenAI file_id input)
## Quick Start
@ -279,6 +280,71 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
</Tabs>
## Mistral Example
Here is a sample payload for using the Mistral model for document understanding:
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm.utils import completion
# pdf file_id received from files endpoint
file_id = "fa778e5e-46ec-4562-8418-36623fe25a71"
# model
model = "mistral/mistral-large-latest"
file_content = [
{"type": "text", "text": "What's this file about?"},
{
"type": "file",
"file": {
"file_id": file_id,
}
},
]
response = completion(
model=model,
messages=[{"role": "user", "content": file_content}],
)
assert response is not None
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```bash
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-d '{
"model": "mistral/mistral-large-latest",
"messages": [
{
"role": "user",
"content": [
{
"type": "text",
"text": "What is the content of the file?"
},
{
"type": "file",
"file": {
"file_id": "fa778e5e-46ec-4562-8418-36623fe25a71"
}
}
]
}
]
}
```
</TabItem>
</Tabs>
## Checking if a model supports pdf input
<Tabs>

85
docs/my-website/docs/mcp.md
View file

@ -1134,6 +1134,91 @@ When MCP tools are called, your custom hook will:
2. Modify the response if needed
3. Track costs in LiteLLM's logging system
## MCP Guardrails
LiteLLM supports applying guardrails to MCP tool calls to ensure security and compliance. You can configure guardrails to run before or during MCP calls to validate inputs and block or mask sensitive information.
### Supported MCP Guardrail Modes
MCP guardrails support the following modes:
- `pre_mcp_call`: Run **before** MCP call, on **input**. Use this mode when you want to apply validation/masking/blocking for MCP requests
- `during_mcp_call`: Run **during** MCP call execution. Use this mode for real-time monitoring and intervention
### Configuration Examples
Configure guardrails to run before MCP tool calls to validate and sanitize inputs:
```yaml title="config.yaml" showLineNumbers
guardrails:
- guardrail_name: "mcp-input-validation"
litellm_params:
guardrail: presidio # or other supported guardrails
mode: "pre_mcp_call" # or during_mcp_call
pii_entities_config:
CREDIT_CARD: "BLOCK" # Will block requests containing credit card numbers
EMAIL_ADDRESS: "MASK" # Will mask email addresses
PHONE_NUMBER: "MASK" # Will mask phone numbers
default_on: true
```
### Usage Examples
#### Testing Pre-MCP Call Guardrails
Test your MCP guardrails with a request that includes sensitive information:
```bash title="Test MCP Guardrail" showLineNumbers
curl http://localhost:4000/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gpt-3.5-turbo",
"messages": [
{"role": "user", "content": "My credit card is 4111-1111-1111-1111 and my email is john@example.com"}
],
"guardrails": ["mcp-input-validation"]
}'
```
The request will be processed as follows:
1. Credit card number will be blocked (request rejected)
2. Email address will be masked (e.g., replaced with `<EMAIL_ADDRESS>`)
#### Using with MCP Tools
When using MCP tools, guardrails will be applied to the tool inputs:
```python title="Python Example with MCP Guardrails" showLineNumbers
import openai
client = openai.OpenAI(
api_key="your-api-key",
base_url="http://localhost:4000"
)
# This request will trigger MCP guardrails
response = client.chat.completions.create(
model="gpt-3.5-turbo",
messages=[
{"role": "user", "content": "Send an email to 555-123-4567 with my SSN 123-45-6789"}
],
tools=[{"type": "mcp", "server_label": "litellm", "server_url": "litellm_proxy"}],
guardrails=["mcp-input-validation"]
)
```
### Supported Guardrail Providers
MCP guardrails work with all LiteLLM-supported guardrail providers:
- **Presidio**: PII detection and masking
- **Bedrock**: AWS Bedrock guardrails
- **Lakera**: Content moderation
- **Aporia**: Custom guardrails
- **Custom**: Your own guardrail implementations
## MCP Permission Management
LiteLLM supports managing permissions for MCP Servers by Keys, Teams, Organizations (entities) on LiteLLM. When a MCP client attempts to list tools, LiteLLM will only return the tools the entity has permissions to access.

30
docs/my-website/docs/providers/aiml.md
View file

@ -1,5 +1,23 @@
# AI/ML API
https://aimlapi.com/
## Overview
| Property | Details |
|-------|-------|
| Description | AI/ML API provides access to state-of-the-art AI models including flux-pro/v1.1 for high-quality image generation. |
| Provider Route on LiteLLM | `aiml/` |
| Link to Provider Doc | [AI/ML API ↗](https://docs.aimlapi.com/) |
| Supported Operations | [`/chat/completions`], [`/images/generations`](#image-generation) |
LiteLLM supports AI/ML API Image Generation calls.
## API Base, Key
```python
# env variable
os.environ['AIML_API_KEY'] = "your-api-key"
os.environ['AIML_API_BASE'] = "https://api.aimlapi.com" # [optional]
```
Getting started with the AI/ML API is simple. Follow these steps to set up your integration:
### 1. Get Your API Key
@ -24,7 +42,7 @@ You can choose from LLama, Qwen, Flux, and 200+ other open and closed-source mod
import litellm
response = litellm.completion(
model="openai/meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo", # The model name must include prefix "openai" + the model name from ai/ml api
model="aiml/meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo", # The model name must include prefix "openai" + the model name from ai/ml api
api_key="", # your aiml api-key
api_base="https://api.aimlapi.com/v2",
messages=[
@ -42,7 +60,7 @@ response = litellm.completion(
import litellm
response = litellm.completion(
model="openai/Qwen/Qwen2-72B-Instruct", # The model name must include prefix "openai" + the model name from ai/ml api
model="aiml/Qwen/Qwen2-72B-Instruct", # The model name must include prefix "openai" + the model name from ai/ml api
api_key="", # your aiml api-key
api_base="https://api.aimlapi.com/v2",
messages=[
@ -67,7 +85,7 @@ import litellm
async def main():
response = await litellm.acompletion(
model="openai/anthropic/claude-3-5-haiku", # The model name must include prefix "openai" + the model name from ai/ml api
model="aiml/anthropic/claude-3-5-haiku", # The model name must include prefix "openai" + the model name from ai/ml api
api_key="", # your aiml api-key
api_base="https://api.aimlapi.com/v2",
messages=[
@ -97,7 +115,7 @@ async def main():
try:
print("test acompletion + streaming")
response = await litellm.acompletion(
model="openai/nvidia/Llama-3.1-Nemotron-70B-Instruct-HF", # The model name must include prefix "openai" + the model name from ai/ml api
model="aiml/nvidia/Llama-3.1-Nemotron-70B-Instruct-HF", # The model name must include prefix "openai" + the model name from ai/ml api
api_key="", # your aiml api-key
api_base="https://api.aimlapi.com/v2",
messages=[{"content": "Hey, how's it going?", "role": "user"}],
@ -125,7 +143,7 @@ import litellm
async def main():
response = await litellm.aembedding(
model="openai/text-embedding-3-small", # The model name must include prefix "openai" + the model name from ai/ml api
model="aiml/text-embedding-3-small", # The model name must include prefix "openai" + the model name from ai/ml api
api_key="", # your aiml api-key
api_base="https://api.aimlapi.com/v1", # 👈 the URL has changed from v2 to v1
input="Your text string",
@ -147,7 +165,7 @@ import litellm
async def main():
response = await litellm.aimage_generation(
model="openai/dall-e-3", # The model name must include prefix "openai" + the model name from ai/ml api
model="aiml/dall-e-3", # The model name must include prefix "openai" + the model name from ai/ml api
api_key="", # your aiml api-key
api_base="https://api.aimlapi.com/v1", # 👈 the URL has changed from v2 to v1
prompt="A cute baby sea otter",

81
docs/my-website/docs/providers/azure/azure.md
View file

@ -9,8 +9,8 @@ import TabItem from '@theme/TabItem';
| Property | Details |
|-------|-------|
| Description | Azure OpenAI Service provides REST API access to OpenAI's powerful language models including o1, o1-mini, GPT-4o, GPT-4o mini, GPT-4 Turbo with Vision, GPT-4, GPT-3.5-Turbo, and Embeddings model series |
| Provider Route on LiteLLM | `azure/`, [`azure/o_series/`](#azure-o-series-models) |
| Description | Azure OpenAI Service provides REST API access to OpenAI's powerful language models including o1, o1-mini, GPT-5, GPT-4o, GPT-4o mini, GPT-4 Turbo with Vision, GPT-4, GPT-3.5-Turbo, and Embeddings model series |
| Provider Route on LiteLLM | `azure/`, [`azure/o_series/`](#o-series-models), [`azure/gpt5_series/`](#gpt-5-models) |
| Supported Operations | [`/chat/completions`](#azure-openai-chat-completion-models), [`/responses`](./azure_responses), [`/completions`](#azure-instruct-models), [`/embeddings`](./azure_embedding), [`/audio/speech`](#azure-text-to-speech-tts), [`/audio/transcriptions`](../audio_transcription), `/fine_tuning`, [`/batches`](#azure-batches-api), `/files`, [`/images`](../image_generation#azure-openai-image-generation-models) |
| Link to Provider Doc | [Azure OpenAI ↗](https://learn.microsoft.com/en-us/azure/ai-services/openai/overview)
@ -207,6 +207,7 @@ model_list:
|------------------|----------------------------------------|
| o1-mini | `response = completion(model="azure/<your deployment name>", messages=messages)` |
| o1-preview | `response = completion(model="azure/<your deployment name>", messages=messages)` |
| gpt-5 | `response = completion(model="azure/<your deployment name>", messages=messages)` |
| gpt-4o-mini | `completion('azure/<your deployment name>', messages)` |
| gpt-4o | `completion('azure/<your deployment name>', messages)` |
| gpt-4 | `completion('azure/<your deployment name>', messages)` |
@ -368,6 +369,82 @@ model_list:
</Tabs>
## GPT-5 Models
| Property | Details |
|-------|-------|
| Description | Azure OpenAI GPT-5 models |
| Provider Route on LiteLLM | `azure/gpt5_series/<custom-name>` or `azure/gpt-5-deployment-name` |
LiteLLM supports using Azure GPT-5 models in one of the two ways:
1. Explicit Routing: `model = azure/gpt5_series/<deployment-name>`. In this scenario the model onboarded to litellm follows the format `model=azure/gpt5_series/<deployment-name>`.
2. Inferred Routing (If the azure deployment name contains `gpt-5` in the name): `model = azure/gpt-5-mini`. In this scenario the model onboarded to litellm follows the format `model=azure/gpt-5-mini`.
#### Explicit Routing
Use `azure/gpt5_series/<deployment-name>` for explicit GPT-5 model routing.
<Tabs>
<TabItem value="sdk" label="SDK">
```python
import litellm
response = litellm.completion(
model="azure/gpt5_series/my-gpt-5-deployment",
messages=[{"role": "user", "content": "Hello, world!"}]
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```yaml
model_list:
- model_name: gpt-5
litellm_params:
model: azure/gpt5_series/my-gpt-5-deployment
api_base: os.environ/AZURE_API_BASE
api_key: os.environ/AZURE_API_KEY
```
</TabItem>
</Tabs>
#### Inferred Routing (gpt-5 in the deployment name)
If your Azure deployment name contains `gpt-5`, LiteLLM automatically recognizes it as a GPT-5 model.
<Tabs>
<TabItem value="sdk" label="SDK">
```python
import litellm
# Deployment name contains 'gpt-5' - automatically inferred
response = litellm.completion(
model="azure/my-gpt-5-deployment",
messages=[{"role": "user", "content": "Hello, world!"}]
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```yaml
model_list:
- model_name: gpt-5-mini
litellm_params:
model: azure/my-gpt-5-deployment # deployment name contains 'gpt-5'
api_base: os.environ/AZURE_API_BASE
api_key: os.environ/AZURE_API_KEY
```
</TabItem>
</Tabs>
## Azure Audio Model
<Tabs>

266
docs/my-website/docs/providers/azure_ai_img.md Normal file
View file

@ -0,0 +1,266 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Azure AI Image Generation
Azure AI provides powerful image generation capabilities using FLUX models from Black Forest Labs to create high-quality images from text descriptions.
## Overview
| Property | Details |
|----------|---------|
| Description | Azure AI Image Generation uses FLUX models to generate high-quality images from text descriptions. |
| Provider Route on LiteLLM | `azure_ai/` |
| Provider Doc | [Azure AI FLUX Models ↗](https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/black-forest-labs-flux-1-kontext-pro-and-flux1-1-pro-now-available-in-azure-ai-f/4434659) |
| Supported Operations | [`/images/generations`](#image-generation) |
## Setup
### API Key & Base URL
```python showLineNumbers
# Set your Azure AI API credentials
import os
os.environ["AZURE_AI_API_KEY"] = "your-api-key-here"
os.environ["AZURE_AI_API_BASE"] = "your-azure-ai-endpoint" # e.g., https://your-endpoint.eastus2.inference.ai.azure.com/
```
Get your API key and endpoint from [Azure AI Studio](https://ai.azure.com/).
## Supported Models
| Model Name | Description | Cost per Image |
|------------|-------------|----------------|
| `azure_ai/FLUX-1.1-pro` | Latest FLUX 1.1 Pro model for high-quality image generation | $0.04 |
| `azure_ai/FLUX.1-Kontext-pro` | FLUX 1 Kontext Pro model with enhanced context understanding | $0.04 |
## Image Generation
### Usage - LiteLLM Python SDK
<Tabs>
<TabItem value="basic" label="Basic Usage">
```python showLineNumbers title="Basic Image Generation"
import litellm
import os
# Set your API credentials
os.environ["AZURE_AI_API_KEY"] = "your-api-key-here"
os.environ["AZURE_AI_API_BASE"] = "your-azure-ai-endpoint"
# Generate a single image
response = litellm.image_generation(
model="azure_ai/FLUX.1-Kontext-pro",
prompt="A cute baby sea otter swimming in crystal clear water",
api_base=os.environ["AZURE_AI_API_BASE"],
api_key=os.environ["AZURE_AI_API_KEY"]
)
print(response.data[0].url)
```
</TabItem>
<TabItem value="flux11" label="FLUX 1.1 Pro">
```python showLineNumbers title="FLUX 1.1 Pro Image Generation"
import litellm
import os
# Set your API credentials
os.environ["AZURE_AI_API_KEY"] = "your-api-key-here"
os.environ["AZURE_AI_API_BASE"] = "your-azure-ai-endpoint"
# Generate image with FLUX 1.1 Pro
response = litellm.image_generation(
model="azure_ai/FLUX-1.1-pro",
prompt="A futuristic cityscape at night with neon lights and flying cars",
api_base=os.environ["AZURE_AI_API_BASE"],
api_key=os.environ["AZURE_AI_API_KEY"]
)
print(response.data[0].url)
```
</TabItem>
<TabItem value="async" label="Async Usage">
```python showLineNumbers title="Async Image Generation"
import litellm
import asyncio
import os
async def generate_image():
# Set your API credentials
os.environ["AZURE_AI_API_KEY"] = "your-api-key-here"
os.environ["AZURE_AI_API_BASE"] = "your-azure-ai-endpoint"
# Generate image asynchronously
response = await litellm.aimage_generation(
model="azure_ai/FLUX.1-Kontext-pro",
prompt="A beautiful sunset over mountains with vibrant colors",
api_base=os.environ["AZURE_AI_API_BASE"],
api_key=os.environ["AZURE_AI_API_KEY"],
n=1,
)
print(response.data[0].url)
return response
# Run the async function
asyncio.run(generate_image())
```
</TabItem>
<TabItem value="advanced" label="Advanced Parameters">
```python showLineNumbers title="Advanced Image Generation with Parameters"
import litellm
import os
# Set your API credentials
os.environ["AZURE_AI_API_KEY"] = "your-api-key-here"
os.environ["AZURE_AI_API_BASE"] = "your-azure-ai-endpoint"
# Generate image with additional parameters
response = litellm.image_generation(
model="azure_ai/FLUX-1.1-pro",
prompt="A majestic dragon soaring over a medieval castle at dawn",
api_base=os.environ["AZURE_AI_API_BASE"],
api_key=os.environ["AZURE_AI_API_KEY"],
n=1,
size="1024x1024",
quality="standard"
)
for image in response.data:
print(f"Generated image URL: {image.url}")
```
</TabItem>
</Tabs>
### Usage - LiteLLM Proxy Server
#### 1. Configure your config.yaml
```yaml showLineNumbers title="Azure AI Image Generation Configuration"
model_list:
- model_name: azure-flux-kontext
litellm_params:
model: azure_ai/FLUX.1-Kontext-pro
api_key: os.environ/AZURE_AI_API_KEY
api_base: os.environ/AZURE_AI_API_BASE
model_info:
mode: image_generation
- model_name: azure-flux-11-pro
litellm_params:
model: azure_ai/FLUX-1.1-pro
api_key: os.environ/AZURE_AI_API_KEY
api_base: os.environ/AZURE_AI_API_BASE
model_info:
mode: image_generation
general_settings:
master_key: sk-1234
```
#### 2. Start LiteLLM Proxy Server
```bash showLineNumbers title="Start LiteLLM Proxy Server"
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
#### 3. Make requests with OpenAI Python SDK
<Tabs>
<TabItem value="openai-sdk" label="OpenAI SDK">
```python showLineNumbers title="Azure AI Image Generation via Proxy - OpenAI SDK"
from openai import OpenAI
# Initialize client with your proxy URL
client = OpenAI(
base_url="http://localhost:4000", # Your proxy URL
api_key="sk-1234" # Your proxy API key
)
# Generate image with FLUX Kontext Pro
response = client.images.generate(
model="azure-flux-kontext",
prompt="A serene Japanese garden with cherry blossoms and a peaceful pond",
n=1,
size="1024x1024"
)
print(response.data[0].url)
```
</TabItem>
<TabItem value="litellm-sdk" label="LiteLLM SDK">
```python showLineNumbers title="Azure AI Image Generation via Proxy - LiteLLM SDK"
import litellm
# Configure LiteLLM to use your proxy
response = litellm.image_generation(
model="litellm_proxy/azure-flux-11-pro",
prompt="A cyberpunk warrior in a neon-lit alleyway",
api_base="http://localhost:4000",
api_key="sk-1234"
)
print(response.data[0].url)
```
</TabItem>
<TabItem value="curl" label="cURL">
```bash showLineNumbers title="Azure AI Image Generation via Proxy - cURL"
curl --location 'http://localhost:4000/v1/images/generations' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer sk-1234' \
--data '{
"model": "azure-flux-kontext",
"prompt": "A cozy coffee shop interior with warm lighting and rustic wooden furniture",
"n": 1,
"size": "1024x1024"
}'
```
</TabItem>
</Tabs>
## Supported Parameters
Azure AI Image Generation supports the following OpenAI-compatible parameters:
| Parameter | Type | Description | Default | Example |
|-----------|------|-------------|---------|---------|
| `prompt` | string | Text description of the image to generate | Required | `"A sunset over the ocean"` |
| `model` | string | The FLUX model to use for generation | Required | `"azure_ai/FLUX.1-Kontext-pro"` |
| `n` | integer | Number of images to generate (1-4) | `1` | `2` |
| `size` | string | Image dimensions | `"1024x1024"` | `"512x512"`, `"1024x1024"` |
| `api_base` | string | Your Azure AI endpoint URL | Required | `"https://your-endpoint.eastus2.inference.ai.azure.com/"` |
| `api_key` | string | Your Azure AI API key | Required | Environment variable or direct value |
## Getting Started
1. Create an account at [Azure AI Studio](https://ai.azure.com/)
2. Deploy a FLUX model in your Azure AI Studio workspace
3. Get your API key and endpoint from the deployment details
4. Set your `AZURE_AI_API_KEY` and `AZURE_AI_API_BASE` environment variables
5. Start generating images using LiteLLM
## Additional Resources
- [Azure AI Studio Documentation](https://docs.microsoft.com/en-us/azure/ai-services/)
- [FLUX Models Announcement](https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/black-forest-labs-flux-1-kontext-pro-and-flux1-1-pro-now-available-in-azure-ai-f/4434659)

113
docs/my-website/docs/providers/baseten.md
View file

@ -1,23 +1,106 @@
# Baseten
LiteLLM supports any Text-Gen-Interface models on Baseten.
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
[Here's a tutorial on deploying a huggingface TGI model (Llama2, CodeLlama, WizardCoder, Falcon, etc.) on Baseten](https://truss.baseten.co/examples/performance/tgi-server)
# Baseten
LiteLLM supports both Baseten Model APIs and dedicated deployments with automatic routing.
## API Types
### Model API (Default)
- **URL**: `https://inference.baseten.co/v1`
- **Format**: `baseten/<model-name>` (e.g., `baseten/openai/gpt-oss-120b`)
- **Best for**: Quick access to popular models
### Dedicated Deployments
- **URL**: `https://model-{id}.api.baseten.co/environments/production/sync/v1`
- **Format**: `baseten/{8-digit-alphanumeric-code}` (e.g., `baseten/abcd1234`)
- **Best for**: Custom models, latency SLAs
:::tip
**Automatic Routing**: LiteLLM detects the type based on model format:
- 8-digit alphanumeric codes → Dedicated deployment
- All other formats → Model API
:::
## Quick Start
### API KEYS
```python
import os
os.environ["BASETEN_API_KEY"] = ""
import os
from litellm import completion
os.environ['BASETEN_API_KEY'] = "your-api-key"
# Model API (default)
response = completion(
model="baseten/openai/gpt-oss-120b",
messages=[{"role": "user", "content": "Hello!"}]
)
# Dedicated deployment (8-digit ID)
response = completion(
model="baseten/abcd1234",
messages=[{"role": "user", "content": "Hello!"}]
)
```
### Baseten Models
Baseten provides infrastructure to deploy and serve ML models https://www.baseten.co/. Use liteLLM to easily call models deployed on Baseten.
## Examples
Example Baseten Usage - Note: liteLLM supports all models deployed on Baseten
### Basic Usage
```python
# Model API
response = completion(
model="baseten/openai/gpt-oss-120b",
messages=[{"role": "user", "content": "Explain quantum computing"}],
max_tokens=500,
temperature=0.7
)
Usage: Pass `model=baseten/<Model ID>`
# Dedicated deployment
response = completion(
model="baseten/abcd1234",
messages=[{"role": "user", "content": "Explain quantum computing"}],
max_tokens=500,
temperature=0.7
)
```
| Model Name | Function Call | Required OS Variables |
|------------------|--------------------------------------------|------------------------------------|
| Falcon 7B | `completion(model='baseten/qvv0xeq', messages=messages)` | `os.environ['BASETEN_API_KEY']` |
| Wizard LM | `completion(model='baseten/q841o8w', messages=messages)` | `os.environ['BASETEN_API_KEY']` |
| MPT 7B Base | `completion(model='baseten/31dxrj3', messages=messages)` | `os.environ['BASETEN_API_KEY']` |
### Streaming (Model API only)
```python
response = completion(
model="baseten/openai/gpt-oss-120b",
messages=[{"role": "user", "content": "Write a poem"}],
stream=True,
stream_options={"include_usage": True}
)
for chunk in response:
if chunk.choices and chunk.choices[0].delta.content:
print(chunk.choices[0].delta.content, end="")
```
## Usage with LiteLLM Proxy
1. **Config**:
```yaml
model_list:
- model_name: baseten-model
litellm_params:
model: baseten/openai/gpt-oss-120b
api_key: your-baseten-api-key
```
2. **Request**:
```python
import openai
client = openai.OpenAI(
api_key="sk-1234",
base_url="http://0.0.0.0:4000"
)
response = client.chat.completions.create(
model="baseten-model",
messages=[{"role": "user", "content": "Hello!"}]
)
```

231
docs/my-website/docs/providers/bedrock.md
View file

@ -584,6 +584,150 @@ Same as [Anthropic API response](../providers/anthropic#usage---thinking--reason
Same as [Anthropic API response](../providers/anthropic#usage---thinking--reasoning_content).
## Usage - Anthropic Beta Features
LiteLLM supports Anthropic's beta features on AWS Bedrock through the `anthropic-beta` header. This enables access to experimental features like:
- **1M Context Window** - Up to 1 million tokens of context (Claude Sonnet 4)
- **Computer Use Tools** - AI that can interact with computer interfaces
- **Token-Efficient Tools** - More efficient tool usage patterns
- **Extended Output** - Up to 128K output tokens
- **Enhanced Thinking** - Advanced reasoning capabilities
### Supported Beta Features
| Beta Feature | Header Value | Compatible Models | Description |
|--------------|-------------|------------------|-------------|
| 1M Context Window | `context-1m-2025-08-07` | Claude Sonnet 4 | Enable 1 million token context window |
| Computer Use (Latest) | `computer-use-2025-01-24` | Claude 3.7 Sonnet | Latest computer use tools |
| Computer Use (Legacy) | `computer-use-2024-10-22` | Claude 3.5 Sonnet v2 | Computer use tools for Claude 3.5 |
| Token-Efficient Tools | `token-efficient-tools-2025-02-19` | Claude 3.7 Sonnet | More efficient tool usage |
| Interleaved Thinking | `interleaved-thinking-2025-05-14` | Claude 4 models | Enhanced thinking capabilities |
| Extended Output | `output-128k-2025-02-19` | Claude 3.7 Sonnet | Up to 128K output tokens |
| Developer Thinking | `dev-full-thinking-2025-05-14` | Claude 4 models | Raw thinking mode for developers |
<Tabs>
<TabItem value="sdk" label="SDK">
**Single Beta Feature**
```python
from litellm import completion
import os
# set env
os.environ["AWS_ACCESS_KEY_ID"] = ""
os.environ["AWS_SECRET_ACCESS_KEY"] = ""
os.environ["AWS_REGION_NAME"] = ""
# Use 1M context window with Claude Sonnet 4
response = completion(
model="bedrock/anthropic.claude-sonnet-4-20250115-v1:0",
messages=[{"role": "user", "content": "Hello! Testing 1M context window."}],
max_tokens=100,
extra_headers={
"anthropic-beta": "context-1m-2025-08-07" # 👈 Enable 1M context
}
)
```
**Multiple Beta Features**
```python
from litellm import completion
# Combine multiple beta features (comma-separated)
response = completion(
model="bedrock/converse/anthropic.claude-3-5-sonnet-20241022-v2:0",
messages=[{"role": "user", "content": "Testing multiple beta features"}],
max_tokens=100,
extra_headers={
"anthropic-beta": "computer-use-2024-10-22,context-1m-2025-08-07"
}
)
```
**Computer Use Tools with Beta Features**
```python
from litellm import completion
# Computer use tools automatically add computer-use-2024-10-22
# You can add additional beta features
response = completion(
model="bedrock/converse/anthropic.claude-3-5-sonnet-20241022-v2:0",
messages=[{"role": "user", "content": "Take a screenshot"}],
tools=[{
"type": "computer_20241022",
"name": "computer",
"display_width_px": 1920,
"display_height_px": 1080
}],
extra_headers={
"anthropic-beta": "context-1m-2025-08-07" # Additional beta feature
}
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
**Set on YAML Config**
```yaml
model_list:
- model_name: claude-sonnet-4-1m
litellm_params:
model: bedrock/anthropic.claude-sonnet-4-20250115-v1:0
extra_headers:
anthropic-beta: "context-1m-2025-08-07" # 👈 Enable 1M context
- model_name: claude-computer-use
litellm_params:
model: bedrock/converse/anthropic.claude-3-5-sonnet-20241022-v2:0
extra_headers:
anthropic-beta: "computer-use-2024-10-22,context-1m-2025-08-07"
general_settings:
forward_client_headers_to_llm_api: true # 👈 Required for client-side header forwarding
```
**Set on Request**
```python
import openai
client = openai.OpenAI(
api_key="anything",
base_url="http://0.0.0.0:4000"
)
response = client.chat.completions.create(
model="claude-sonnet-4-1m",
messages=[{
"role": "user",
"content": "Testing 1M context window"
}],
extra_headers={
"anthropic-beta": "context-1m-2025-08-07"
}
)
```
:::info
**For client-side header forwarding**: When using the proxy and sending `anthropic-beta` headers from the client (like the OpenAI SDK), you need to enable `forward_client_headers_to_llm_api: true` in your proxy's `general_settings`. This tells the proxy to extract headers from HTTP requests and forward them to the underlying LLM provider.
:::
</TabItem>
</Tabs>
:::info
Beta features may require special access or permissions in your AWS account. Some features are only available in specific AWS regions. Check the [AWS Bedrock documentation](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-anthropic-claude-messages-request-response.html) for availability and access requirements.
:::
## Usage - Structured Output / JSON mode
<Tabs>
@ -1488,6 +1632,91 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
### OpenAI GPT OSS
| Property | Details |
|----------|---------|
| Provider Route | `bedrock/converse/openai.gpt-oss-20b-1:0`, `bedrock/converse/openai.gpt-oss-120b-1:0` |
| Provider Documentation | [Amazon Bedrock ↗](https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-bedrock.html) |
<Tabs>
<TabItem value="sdk" label="SDK">
```python title="GPT OSS SDK Usage" showLineNumbers
from litellm import completion
import os
# Set AWS credentials
os.environ["AWS_ACCESS_KEY_ID"] = "your-aws-access-key"
os.environ["AWS_SECRET_ACCESS_KEY"] = "your-aws-secret-key"
os.environ["AWS_REGION_NAME"] = "us-east-1"
# GPT OSS 20B model
response = completion(
model="bedrock/converse/openai.gpt-oss-20b-1:0",
messages=[{"role": "user", "content": "Hello, how are you?"}],
)
print(response.choices[0].message.content)
# GPT OSS 120B model
response = completion(
model="bedrock/converse/openai.gpt-oss-120b-1:0",
messages=[{"role": "user", "content": "Explain machine learning in simple terms"}],
)
print(response.choices[0].message.content)
```
</TabItem>
<TabItem value="proxy" label="Proxy">
**1. Add to config**
```yaml title="config.yaml" showLineNumbers
model_list:
- model_name: gpt-oss-20b
litellm_params:
model: bedrock/converse/openai.gpt-oss-20b-1:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: os.environ/AWS_REGION_NAME
- model_name: gpt-oss-120b
litellm_params:
model: bedrock/converse/openai.gpt-oss-120b-1:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: os.environ/AWS_REGION_NAME
```
**2. Start proxy**
```bash title="Start LiteLLM Proxy" showLineNumbers
litellm --config /path/to/config.yaml
# RUNNING at http://0.0.0.0:4000
```
**3. Test it!**
```bash title="Test GPT OSS via Proxy" showLineNumbers
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"model": "gpt-oss-20b",
"messages": [
{
"role": "user",
"content": "What are the key benefits of open source AI?"
}
]
}'
```
</TabItem>
</Tabs>
## Provisioned throughput models
To use provisioned throughput Bedrock models pass
- `model=bedrock/<base-model>`, example `model=bedrock/anthropic.claude-v2`. Set `model` to any of the [Supported AWS models](#supported-aws-bedrock-models)
@ -1522,6 +1751,8 @@ Here's an example of using a bedrock model with LiteLLM. For a complete list, re
| Model Name | Command |
|----------------------------|------------------------------------------------------------------|
| GPT-OSS 20B | `completion(model='bedrock/converse/openai.gpt-oss-20b-1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` |
| GPT-OSS 120B | `completion(model='bedrock/converse/openai.gpt-oss-120b-1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` |
| Deepseek R1 | `completion(model='bedrock/us.deepseek.r1-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` |
| Anthropic Claude-V3.5 Sonnet | `completion(model='bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` |
| Anthropic Claude-V3 sonnet | `completion(model='bedrock/anthropic.claude-3-sonnet-20240229-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` |

140
docs/my-website/docs/providers/deepinfra.md
View file

@ -1,3 +1,6 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# DeepInfra
https://deepinfra.com/
@ -7,6 +10,11 @@ https://deepinfra.com/
:::
## Table of Contents
- [API Key](#api-key)
- [Chat Models](#chat-models)
- [Rerank Endpoint](#rerank-endpoint)
## API Key
```python
@ -53,3 +61,135 @@ for chunk in response:
| codellama/CodeLlama-34b-Instruct-hf | `completion(model="deepinfra/codellama/CodeLlama-34b-Instruct-hf", messages)` |
| mistralai/Mistral-7B-Instruct-v0.1 | `completion(model="deepinfra/mistralai/Mistral-7B-Instruct-v0.1", messages)` |
| jondurbin/airoboros-l2-70b-gpt4-1.4.1 | `completion(model="deepinfra/jondurbin/airoboros-l2-70b-gpt4-1.4.1", messages)` |
## Rerank Endpoint
LiteLLM provides a Cohere API compatible `/rerank` endpoint for DeepInfra rerank models.
### Supported Rerank Models
| Model Name | Description |
|------------|-------------|
| `deepinfra/Qwen/Qwen3-Reranker-0.6B` | Lightweight rerank model (0.6B parameters) |
| `deepinfra/Qwen/Qwen3-Reranker-4B` | Medium rerank model (4B parameters) |
| `deepinfra/Qwen/Qwen3-Reranker-8B` | Large rerank model (8B parameters) |
### Usage - LiteLLM Python SDK
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import rerank
import os
os.environ["DEEPINFRA_API_KEY"] = "your-api-key"
response = rerank(
model="deepinfra/Qwen/Qwen3-Reranker-0.6B",
query="What is the capital of France?",
documents=[
"Paris is the capital of France.",
"London is the capital of the United Kingdom.",
"Berlin is the capital of Germany.",
"Madrid is the capital of Spain.",
"Rome is the capital of Italy."
]
)
print(response)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
1. Add to config.yaml
```yaml
model_list:
- model_name: Qwen/Qwen3-Reranker-0.6B
litellm_params:
model: deepinfra/Qwen/Qwen3-Reranker-0.6B
api_key: os.environ/DEEPINFRA_API_KEY
```
2. Start proxy
```bash
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000/
```
3. Test it!
```bash
curl -L -X POST 'http://0.0.0.0:4000/rerank' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"model": "Qwen/Qwen3-Reranker-0.6B",
"query": "What is the capital of France?",
"documents": [
"Paris is the capital of France.",
"London is the capital of the United Kingdom.",
"Berlin is the capital of Germany.",
"Madrid is the capital of Spain.",
"Rome is the capital of Italy."
]
}'
```
</TabItem>
</Tabs>
### Supported Cohere Rerank API Params
| Param | Type | Description |
| ------------------ | ----------- | ----------------------------------------------- |
| `query` | `str` | The query to rerank the documents against |
| `documents` | `list[str]` | The documents to rerank |
### Provider-specific parameters
Pass any deepinfra specific parameters as a keyword argument to the rerank function, e.g.
```
response = rerank(
model="deepinfra/Qwen/Qwen3-Reranker-0.6B",
query="What is the capital of France?",
documents=[
"Paris is the capital of France.",
"London is the capital of the United Kingdom.",
"Berlin is the capital of Germany.",
"Madrid is the capital of Spain.",
"Rome is the capital of Italy."
],
my_custom_param="my_custom_value", # any other deepinfra specific parameters
)
```
### Response Format
```json
{
"id": "request-id",
"results": [
{
"index": 0,
"relevance_score": 0.9975274205207825
},
{
"index": 1,
"relevance_score": 0.011687257327139378
}
],
"meta": {
"billed_units": {
"total_tokens": 427
},
"tokens": {
"input_tokens": 427,
"output_tokens": 0
}
}
}
```

79
docs/my-website/docs/providers/gradient_ai.md Normal file
View file

@ -0,0 +1,79 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# GradientAI
https://digitalocean.com/products/gradientai
LiteLLM provides native support for GradientAI models.
To use a GradientAI model, specify it as `gradient_ai/<model-name>` in your LiteLLM requests.
## API Key & Endpoint
Set your credentials and endpoint as environment variables:
```python
import os
os.environ['GRADIENT_AI_API_KEY'] = "your-api-key"
os.environ['GRADIENT_AI_AGENT_ENDPOINT'] = "https://api.gradient_ai.com/api/v1/chat" # default endpoint
```
## Sample Usage
```python
from litellm import completion
import os
os.environ['GRADIENT_AI_API_KEY'] = "your-api-key"
response = completion(
model="gradient_ai/model-name",
messages=[
{"role": "user", "content": "Hello, how are you?"}
],
)
print(response.choices[0].message.content)
```
## Streaming Example
```python
from litellm import completion
import os
os.environ['GRADIENT_AI_API_KEY'] = "your-api-key"
response = completion(
model="gradient_ai/model-name",
messages=[
{"role": "user", "content": "Write a story about a robot learning to love"}
],
stream=True,
)
for chunk in response:
print(chunk.choices[0].delta.content or "", end="")
```
## Supported Parameters
| Parameter | Type | Description |
|-----------------------------------|--------------|--------------------------------------------------------------------|
| `temperature` | float | Controls randomness (0.0-2.0) |
| `top_p` | float | Nucleus sampling parameter (0.0-1.0) |
| `max_tokens` | int | Maximum tokens to generate |
| `max_completion_tokens` | int | Alternative to max_tokens |
| `stream` | bool | Whether to stream the response |
| `k` | int | Top results to return from knowledge bases |
| `retrieval_method` | string | Retrieval strategy (rewrite/step_back/sub_queries/none) |
| `frequency_penalty` | float | Penalizes repeated tokens (-2.0 to 2.0) |
| `presence_penalty` | float | Penalizes tokens based on presence (-2.0 to 2.0) |
| `stop` | string/list | Sequences to stop generation |
| `kb_filters` | List[Dict] | Filters for knowledge base retrieval |
| `instruction_override` | string | Override agent's default instruction |
| `include_retrieval_info` | bool | Include document retrieval metadata |
| `include_guardrails_info` | bool | Include guardrail trigger metadata |
| `provide_citations` | bool | Include citations in response |
---
For more details, see [DigitalOcean GradientAI documentation](https://digitalocean.com/products/gradientai).

52
docs/my-website/docs/providers/litellm_proxy.md
View file

@ -9,7 +9,7 @@ import TabItem from '@theme/TabItem';
| Description | LiteLLM Proxy is an OpenAI-compatible gateway that allows you to interact with multiple LLM providers through a unified API. Simply use the `litellm_proxy/` prefix before the model name to route your requests through the proxy. |
| Provider Route on LiteLLM | `litellm_proxy/` (add this prefix to the model name, to route any requests to litellm_proxy - e.g. `litellm_proxy/your-model-name`) |
| Setup LiteLLM Gateway | [LiteLLM Gateway ↗](../simple_proxy) |
| Supported Endpoints |`/chat/completions`, `/completions`, `/embeddings`, `/audio/speech`, `/audio/transcriptions`, `/images`, `/rerank` |
| Supported Endpoints |`/chat/completions`, `/completions`, `/embeddings`, `/audio/speech`, `/audio/transcriptions`, `/images`, `/images/edits`, `/rerank` |
@ -111,6 +111,21 @@ response = litellm.image_generation(
)
```
## Image Edit
```python
import litellm
with open("your-image.png", "rb") as f:
response = litellm.image_edit(
model="litellm_proxy/gpt-image-1",
prompt="Make this image a watercolor painting",
image=[f],
api_base="your-litellm-proxy-url",
api_key="your-litellm-proxy-api-key",
)
```
## Audio Transcription
```python
@ -211,3 +226,38 @@ response = litellm.completion(
use_litellm_proxy=True
)
```
## Sending `tags` to LiteLLM Proxy
Tags allow you to categorize and track your API requests for monitoring, debugging, and analytics purposes. You can send tags as a list of strings to the LiteLLM Proxy using the `extra_body` parameter.
### Usage
Send tags by including them in the `extra_body` parameter of your completion request:
```python showLineNumbers title="Usage"
import litellm
response = litellm.completion(
model="gpt-4",
messages=[{"role": "user", "content": "What is the capital of France?"}],
api_base="http://localhost:4000",
api_key="sk-1234",
extra_body={"tags": ["user:ishaan", "department:engineering", "priority:high"]}
)
```
### Async Usage
```python showLineNumbers title="Async Usage"
import litellm
response = await litellm.acompletion(
model="gpt-4",
messages=[{"role": "user", "content": "What is the capital of France?"}],
api_base="http://localhost:4000",
api_key="sk-1234",
extra_body={"tags": ["user:ishaan", "department:engineering"]}
)
```

352
docs/my-website/docs/providers/openai/responses_api.md
View file

@ -492,3 +492,355 @@ print(response_with_mcp_call)
</Tabs>
## Verbosity Parameter
The `verbosity` parameter is supported for the `responses` API.
<Tabs>
<TabItem value="sdk" label="LiteLLM Python SDK">
```python showLineNumbers title="Verbosity Parameter"
from litellm import responses
question = "Write a poem about a boy and his first pet dog."
for verbosity in ["low", "medium", "high"]:
response = responses(
model="gpt-5-mini",
input=question,
text={"verbosity": verbosity}
)
print(response)
```
</TabItem>
<TabItem value="proxy" label="LiteLLM Proxy">
```python
from openai import OpenAI
import pandas as pd
from IPython.display import display
client = OpenAI(
base_url="http://localhost:4000", # Your proxy URL
api_key="your-api-key" # Your proxy API key
)
question = "Write a poem about a boy and his first pet dog."
data = []
for verbosity in ["low", "medium", "high"]:
response = client.responses.create(
model="gpt-5-mini",
input=question,
text={"verbosity": verbosity}
)
# Extract text
output_text = ""
for item in response.output:
if hasattr(item, "content"):
for content in item.content:
if hasattr(content, "text"):
output_text += content.text
usage = response.usage
data.append({
"Verbosity": verbosity,
"Sample Output": output_text,
"Output Tokens": usage.output_tokens
})
# Create DataFrame
df = pd.DataFrame(data)
# Display nicely with centered headers
pd.set_option('display.max_colwidth', None)
styled_df = df.style.set_table_styles(
[
{'selector': 'th', 'props': [('text-align', 'center')]}, # Center column headers
{'selector': 'td', 'props': [('text-align', 'left')]} # Left-align table cells
]
)
display(styled_df)
```
</TabItem>
</Tabs>
## Free-form Function Calling
<Tabs>
<TabItem value="sdk" label="LiteLLM Python SDK">
```python showLineNumbers title="Free-form Function Calling"
import litellm
response = litellm.responses(
response = client.responses.create(
model="gpt-5-mini",
input="Please use the code_exec tool to calculate the area of a circle with radius equal to the number of 'r's in strawberry",
text={"format": {"type": "text"}},
tools=[
{
"type": "custom",
"name": "code_exec",
"description": "Executes arbitrary python code",
}
]
)
print(response.output)
```
</TabItem>
<TabItem value="proxy" label="LiteLLM Proxy">
```python showLineNumbers title="Free-form Function Calling"
from openai import OpenAI
client = OpenAI(
base_url="http://localhost:4000", # Your proxy URL
api_key="your-api-key" # Your proxy API key
)
response = client.responses.create(
model="gpt-5-mini",
input="Please use the code_exec tool to calculate the area of a circle with radius equal to the number of 'r's in strawberry",
text={"format": {"type": "text"}},
tools=[
{
"type": "custom",
"name": "code_exec",
"description": "Executes arbitrary python code",
}
]
)
print(response.output)
```
</TabItem>
</Tabs>
## Context-Free Grammar
<Tabs>
<TabItem value="sdk" label="LiteLLM Python SDK">
```python showLineNumbers title="Context-Free Grammar"
import litellm
import textwrap
# ----------------- grammars for MS SQL dialect -----------------
mssql_grammar = textwrap.dedent(r"""
// ---------- Punctuation & operators ----------
SP: " "
COMMA: ","
GT: ">"
EQ: "="
SEMI: ";"
// ---------- Start ----------
start: "SELECT" SP "TOP" SP NUMBER SP select_list SP "FROM" SP table SP "WHERE" SP amount_filter SP "AND" SP date_filter SP "ORDER" SP "BY" SP sort_cols SEMI
// ---------- Projections ----------
select_list: column (COMMA SP column)*
column: IDENTIFIER
// ---------- Tables ----------
table: IDENTIFIER
// ---------- Filters ----------
amount_filter: "total_amount" SP GT SP NUMBER
date_filter: "order_date" SP GT SP DATE
// ---------- Sorting ----------
sort_cols: "order_date" SP "DESC"
// ---------- Terminals ----------
IDENTIFIER: /[A-Za-z_][A-Za-z0-9_]*/
NUMBER: /[0-9]+/
DATE: /'[0-9]{4}-[0-9]{2}-[0-9]{2}'/
""")
sql_prompt_mssql = (
"Call the mssql_grammar to generate a query for Microsoft SQL Server that retrieve the "
"five most recent orders per customer, showing customer_id, order_id, order_date, and total_amount, "
"where total_amount > 500 and order_date is after '2025-01-01'. "
)
response = litellm.responses(
model="gpt-5",
input=sql_prompt_mssql,
text={"format": {"type": "text"}},
tools=[
{
"type": "custom",
"name": "mssql_grammar",
"description": "Executes read-only Microsoft SQL Server queries limited to SELECT statements with TOP and basic WHERE/ORDER BY. YOU MUST REASON HEAVILY ABOUT THE QUERY AND MAKE SURE IT OBEYS THE GRAMMAR.",
"format": {
"type": "grammar",
"syntax": "lark",
"definition": mssql_grammar
}
},
],
parallel_tool_calls=False
)
print("--- MS SQL Query ---")
print(response_mssql.output[1].input)
```
</TabItem>
<TabItem value="proxy" label="LiteLLM Proxy">
```python showLineNumbers title="Context-Free Grammar"
from openai import OpenAI
client = OpenAI(
base_url="http://localhost:4000", # Your proxy URL
api_key="your-api-key" # Your proxy API key
)
import textwrap
# ----------------- grammars for MS SQL dialect -----------------
mssql_grammar = textwrap.dedent(r"""
// ---------- Punctuation & operators ----------
SP: " "
COMMA: ","
GT: ">"
EQ: "="
SEMI: ";"
// ---------- Start ----------
start: "SELECT" SP "TOP" SP NUMBER SP select_list SP "FROM" SP table SP "WHERE" SP amount_filter SP "AND" SP date_filter SP "ORDER" SP "BY" SP sort_cols SEMI
// ---------- Projections ----------
select_list: column (COMMA SP column)*
column: IDENTIFIER
// ---------- Tables ----------
table: IDENTIFIER
// ---------- Filters ----------
amount_filter: "total_amount" SP GT SP NUMBER
date_filter: "order_date" SP GT SP DATE
// ---------- Sorting ----------
sort_cols: "order_date" SP "DESC"
// ---------- Terminals ----------
IDENTIFIER: /[A-Za-z_][A-Za-z0-9_]*/
NUMBER: /[0-9]+/
DATE: /'[0-9]{4}-[0-9]{2}-[0-9]{2}'/
""")
sql_prompt_mssql = (
"Call the mssql_grammar to generate a query for Microsoft SQL Server that retrieve the "
"five most recent orders per customer, showing customer_id, order_id, order_date, and total_amount, "
"where total_amount > 500 and order_date is after '2025-01-01'. "
)
response = client.responses.create(
model="gpt-5",
input=sql_prompt_mssql,
text={"format": {"type": "text"}},
tools=[
{
"type": "custom",
"name": "mssql_grammar",
"description": "Executes read-only Microsoft SQL Server queries limited to SELECT statements with TOP and basic WHERE/ORDER BY. YOU MUST REASON HEAVILY ABOUT THE QUERY AND MAKE SURE IT OBEYS THE GRAMMAR.",
"format": {
"type": "grammar",
"syntax": "lark",
"definition": mssql_grammar
}
},
],
parallel_tool_calls=False
)
print("--- MS SQL Query ---")
print(response_mssql.output[1].input)
```
</TabItem>
</Tabs>
## Minimal Reasoning
<Tabs>
<TabItem value="sdk" label="LiteLLM Python SDK">
```python showLineNumbers title="Minimal Reasoning"
import litellm
response = litellm.responses(
model="gpt-5",
input= [{ 'role': 'developer', 'content': prompt },
{ 'role': 'user', 'content': 'The food that the restaurant was great! I recommend it to everyone.' }],
reasoning = {
"effort": "minimal"
},
)
print(response)
```
</TabItem>
<TabItem value="proxy" label="LiteLLM Proxy">
```python showLineNumbers title="Minimal Reasoning"
from openai import OpenAI
client = OpenAI(
base_url="http://localhost:4000", # Your proxy URL
api_key="your-api-key" # Your proxy API key
)
prompt = "Classify sentiment of the review as positive|neutral|negative. Return one word only."
response = client.responses.create(
model="gpt-5",
input= [{ 'role': 'developer', 'content': prompt },
{ 'role': 'user', 'content': 'The food that the restaurant was great! I recommend it to everyone.' }],
reasoning = {
"effort": "minimal"
},
)
# Extract model's text output
output_text = ""
for item in response.output:
if hasattr(item, "content"):
for content in item.content:
if hasattr(content, "text"):
output_text += content.text
# Token usage details
usage = response.usage
print("--------------------------------")
print("Output:")
print(output_text)
```
</TabItem>
</Tabs>

13
docs/my-website/docs/providers/sambanova.md
View file

@ -307,3 +307,16 @@ response = litellm.completion(
print(response.choices[0].message.content))
```
## SambaNova - Embeddings
```python
import litellm
response = litellm.embedding(
model="sambanova/E5-Mistral-7B-Instruct",
input=["sample text to embed", "another sample text to embed"]
)
print(response.data)
```

87
docs/my-website/docs/providers/vertex_partner.md
View file

@ -14,6 +14,7 @@ import TabItem from '@theme/TabItem';
| Meta/Llama | `vertex_ai/meta/{MODEL}` | [Vertex AI - Meta Models](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/llama) |
| Mistral | `vertex_ai/mistral-*` | [Vertex AI - Mistral Models](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/mistral) |
| AI21 (Jamba) | `vertex_ai/jamba-*` | [Vertex AI - AI21 Models](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/ai21) |
| Qwen | `vertex_ai/qwen/*` | [Vertex AI - Qwen Models](https://cloud.google.com/vertex-ai/generative-ai/docs/maas/qwen) |
| Model Garden | `vertex_ai/openai/{MODEL_ID}` or `vertex_ai/{MODEL_ID}` | [Vertex Model Garden](https://cloud.google.com/model-garden?hl=en) |
## Vertex AI - Anthropic (Claude)
@ -571,6 +572,92 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
</Tabs>
## VertexAI Qwen API
| Property | Details |
|----------|---------|
| Provider Route | `vertex_ai/qwen/{MODEL}` |
| Vertex Documentation | [Vertex AI - Qwen Models](https://cloud.google.com/vertex-ai/generative-ai/docs/maas/qwen) |
**LiteLLM Supports all Vertex AI Qwen Models.** Ensure you use the `vertex_ai/qwen/` prefix for all Vertex AI Qwen models.
| Model Name | Usage |
|------------------|------------------------------|
| vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas | `completion('vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas', messages)` |
| vertex_ai/qwen/qwen3-235b-a22b-instruct-2507-maas | `completion('vertex_ai/qwen/qwen3-235b-a22b-instruct-2507-maas', messages)` |
#### Usage
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
import os
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = ""
model = "qwen/qwen3-coder-480b-a35b-instruct-maas"
vertex_ai_project = "your-vertex-project" # can also set this as os.environ["VERTEXAI_PROJECT"]
vertex_ai_location = "your-vertex-location" # can also set this as os.environ["VERTEXAI_LOCATION"]
response = completion(
model="vertex_ai/" + model,
messages=[{"role": "user", "content": "hi"}],
vertex_ai_project=vertex_ai_project,
vertex_ai_location=vertex_ai_location,
)
print("\nModel Response", response)
```
</TabItem>
<TabItem value="proxy" label="Proxy">
**1. Add to config**
```yaml
model_list:
- model_name: vertex-qwen
litellm_params:
model: vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas
vertex_ai_project: "my-test-project"
vertex_ai_location: "us-east-1"
- model_name: vertex-qwen
litellm_params:
model: vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas
vertex_ai_project: "my-test-project"
vertex_ai_location: "us-west-1"
```
**2. Start proxy**
```bash
litellm --config /path/to/config.yaml
# RUNNING at http://0.0.0.0:4000
```
**3. Test it!**
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"model": "vertex-qwen", # 👈 the 'model_name' in config
"messages": [
{
"role": "user",
"content": "what llm are you"
}
],
}'
```
</TabItem>
</Tabs>
## Model Garden
:::tip

46
docs/my-website/docs/providers/vllm.md
View file

@ -104,6 +104,52 @@ Here's how to call an OpenAI-Compatible Endpoint with the LiteLLM Proxy Server
</Tabs>
## Reasoning Effort
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
response = completion(
model="hosted_vllm/gpt-oss-120b",
messages=[{"role": "user", "content": "whats 2 + 2"}],
reasoning_effort="high",
api_base="https://hosted-vllm-api.co",
)
print(response)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
1. Setup config.yaml
```yaml
model_list:
- model_name: gpt-oss-120b
litellm_params:
model: hosted_vllm/gpt-oss-120b
api_base: https://hosted-vllm-api.co
```
2. Start the proxy
```bash
litellm --config /path/to/config.yaml
```
3. Test it!
```bash
curl http://0.0.0.0:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-d '{"model": "gpt-oss-120b", "messages": [{"role": "user", "content": "whats 2 + 2"}], "reasoning_effort": "high"}'
```
</TabItem>
</Tabs>
## Embeddings

5
docs/my-website/docs/proxy/config_settings.md
View file

@ -335,12 +335,16 @@ router_settings:
| ANTHROPIC_API_KEY | API key for Anthropic service
| ANTHROPIC_API_BASE | Base URL for Anthropic API. Default is https://api.anthropic.com
| AWS_ACCESS_KEY_ID | Access Key ID for AWS services
| AWS_DEFAULT_REGION | Default AWS region for service interactions when AWS_REGION is not set
| AWS_PROFILE_NAME | AWS CLI profile name to be used
| AWS_REGION | AWS region for service interactions (takes precedence over AWS_DEFAULT_REGION)
| AWS_REGION_NAME | Default AWS region for service interactions
| AWS_ROLE_ARN | ARN of the AWS IAM role to assume for authentication
| AWS_ROLE_NAME | Role name for AWS IAM usage
| AWS_SECRET_ACCESS_KEY | Secret Access Key for AWS services
| AWS_SESSION_NAME | Name for AWS session
| AWS_WEB_IDENTITY_TOKEN | Web identity token for AWS
| AWS_WEB_IDENTITY_TOKEN_FILE | Path to file containing web identity token for AWS
| AZURE_API_VERSION | Version of the Azure API being used
| AZURE_AUTHORITY_HOST | Azure authority host URL
| AZURE_CERTIFICATE_PASSWORD | Password for Azure OpenAI certificate
@ -349,6 +353,7 @@ router_settings:
| AZURE_CODE_INTERPRETER_COST_PER_SESSION | Cost per session for Azure Code Interpreter service
| AZURE_COMPUTER_USE_INPUT_COST_PER_1K_TOKENS | Input cost per 1K tokens for Azure Computer Use service
| AZURE_COMPUTER_USE_OUTPUT_COST_PER_1K_TOKENS | Output cost per 1K tokens for Azure Computer Use service
| AZURE_DEFAULT_RESPONSES_API_VERSION | Version of the Azure Default Responses API being used. Default is "preview"
| AZURE_TENANT_ID | Tenant ID for Azure Active Directory
| AZURE_USERNAME | Username for Azure services, use in conjunction with AZURE_PASSWORD for azure ad token with basic username/password workflow
| AZURE_PASSWORD | Password for Azure services, use in conjunction with AZURE_USERNAME for azure ad token with basic username/password workflow

16
docs/my-website/docs/proxy/deploy.md
View file

@ -127,6 +127,8 @@ CMD ["--port", "4000", "--config", "config.yaml", "--detailed_debug"]
Follow these instructions to build a docker container from the litellm pip package. If your company has a strict requirement around security / building images you can follow these steps.
**Note:** You'll need to copy the `schema.prisma` file from the [litellm repository](https://github.com/BerriAI/litellm/blob/main/schema.prisma) to your build directory alongside the Dockerfile and requirements.txt.
Dockerfile
```shell
@ -149,6 +151,12 @@ COPY requirements.txt .
RUN --mount=type=cache,target=${HOME}/.cache/pip \
${HOME}/venv/bin/pip install -r requirements.txt
# Copy Prisma schema file
COPY schema.prisma .
# Generate prisma client
RUN prisma generate
EXPOSE 4000/tcp
ENTRYPOINT ["litellm"]
@ -1002,5 +1010,13 @@ User-agent: *
Disallow: /
```
## Deployment FAQ
**Q: Is Postgres the only supported database, or do you support other ones (like Mongo)?**
A: We explored MySQL but that was hard to maintain and led to bugs for customers. Currently, PostgreSQL is our primary supported database for production deployments.
**Q: If there is Postgres downtime, how does LiteLLM react? Does it fail-open or is there API downtime?**
A: You can gracefully handle DB unavailability if it's on your VPC. See our production guide for more details: [Gracefully Handle DB Unavailability](https://docs.litellm.ai/docs/proxy/prod#6-if-running-litellm-on-vpc-gracefully-handle-db-unavailability)

2
docs/my-website/docs/proxy/docker_quick_start.md
View file

@ -2,7 +2,7 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Getting Started - E2E Tutorial
# E2E Tutorial
End-to-End tutorial for LiteLLM Proxy to:
- Add an Azure OpenAI model

162
docs/my-website/docs/proxy/native_litellm_prompt.md Normal file
View file

@ -0,0 +1,162 @@
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# LiteLLM Prompt Management (GitOps)
Store prompts as `.prompt` files in your repository and use them directly with LiteLLM. No external services required.
## Quick Start
<Tabs>
<TabItem value="sdk" label="SDK">
**1. Create a .prompt file**
Create `prompts/hello.prompt`:
```yaml
---
model: gpt-4
temperature: 0.7
---
System: You are a helpful assistant.
User: {{user_message}}
```
**2. Use with LiteLLM**
```python
import litellm
# Set the global prompt directory
litellm.global_prompt_directory = "prompts/"
response = litellm.completion(
model="dotprompt/gpt-4",
prompt_id="hello",
prompt_variables={"user_message": "What is the capital of France?"}
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
**1. Create a .prompt file**
Create `prompts/hello.prompt`:
```yaml
---
model: gpt-4
temperature: 0.7
---
System: You are a helpful assistant.
User: {{user_message}}
```
**2. Setup config.yaml**
```yaml
model_list:
- model_name: my-dotprompt-model
litellm_params:
model: dotprompt/gpt-4
prompt_id: "hello"
api_key: os.environ/OPENAI_API_KEY
litellm_settings:
global_prompt_directory: "./prompts"
```
**3. Start the proxy**
```bash
litellm --config config.yaml --detailed_debug
```
**4. Test it!**
```bash
curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-d '{
"model": "my-dotprompt-model",
"messages": [{"role": "user", "content": "IGNORED"}],
"prompt_variables": {
"user_message": "What is the capital of France?"
}
}'
```
</TabItem>
</Tabs>
### .prompt File Format
`.prompt` files use YAML frontmatter for metadata and support Jinja2 templating:
```yaml
---
model: gpt-4 # Model to use
temperature: 0.7 # Optional parameters
max_tokens: 1000
input:
schema:
user_message: string # Input validation (optional)
---
System: You are a helpful {{role}} assistant.
User: {{user_message}}
```
### Advanced Features
**Multi-role conversations:**
```yaml
---
model: gpt-4
temperature: 0.3
---
System: You are a helpful coding assistant.
User: {{user_question}}
```
**Dynamic model selection:**
```yaml
---
model: "{{preferred_model}}" # Model can be a variable
temperature: 0.7
---
System: You are a helpful assistant specialized in {{domain}}.
User: {{user_message}}
```
### API Reference
For dotprompt integration, use these parameters:
```
model: dotprompt/<base_model> # required (e.g., dotprompt/gpt-4)
prompt_id: str # required - the .prompt filename without extension
prompt_variables: Optional[dict] # optional - variables for template rendering
```
**Example API call:**
```python
response = litellm.completion(
model="dotprompt/gpt-4",
prompt_id="hello",
prompt_variables={"user_message": "Hello world"},
messages=[{"role": "user", "content": "This will be ignored"}]
)
```

2
docs/my-website/docs/proxy/prod.md
View file

@ -199,7 +199,7 @@ USE_PRISMA_MIGRATE="True"
<TabItem value="cli" label="CLI">
```bash
litellm --use_prisma_migrate
litellm
```
</TabItem>

1
docs/my-website/docs/proxy/prompt_management.md
View file

@ -8,6 +8,7 @@ Run experiments or change the specific model (e.g. from gpt-4o to gpt4o-mini fin
| Supported Integrations | Link |
|------------------------|------|
| Native LiteLLM GitOps (.prompt files) | [Get Started](native_litellm_prompt) |
| Langfuse | [Get Started](https://langfuse.com/docs/prompts/get-started) |
| Humanloop | [Get Started](../observability/humanloop) |

5
docs/my-website/docs/proxy/quick_start.md
View file

@ -2,8 +2,9 @@ import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Quick Start
Quick start CLI, Config, Docker
# CLI - Quick Start
Setup LiteLLM Proxy quickly via CLI.
LiteLLM Server (LLM Gateway) manages:

6
docs/my-website/docs/proxy/team_budgets.md
View file

@ -4,6 +4,12 @@ import TabItem from '@theme/TabItem';
# Setting Team Budgets
# Pre-Requisites
- You must set up a Postgres database (e.g. Supabase, Neon, etc.)
- To enable team member rate limits, set the environment variable `EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING=true` **before starting the proxy server**. Without this, team member rate limits will not be enforced.
Track spend, set budgets for your Internal Team
## Setting Monthly Team Budgets

8
docs/my-website/docs/proxy/users.md
View file

@ -58,6 +58,9 @@ You can:
**Step-by step tutorial on setting, resetting budgets on Teams here (API or using Admin UI)**
> **Prerequisite:**
> To enable team member rate limits, you must set the environment variable `EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING=true` before starting the proxy server. Without this, team member rate limits will not be enforced.
👉 [https://docs.litellm.ai/docs/proxy/team_budgets](https://docs.litellm.ai/docs/proxy/team_budgets)
:::
@ -793,6 +796,11 @@ Expected Response:
Enable multi-instance rate limiting with the env var `EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING="True"`
**Important Notes:**
- Setting `EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING="True"` is required for team member rate limits to function, not just for multi-instance scenarios.
- **Rate limits do not apply to proxy admin users.**
- When testing rate limits, use internal user roles (non-admin) to ensure limits are enforced as expected.
Changes:
- This moves to using async_increment instead of async_set_cache when updating current requests/tokens.
- The in-memory cache is synced with redis every 0.01s, to avoid calling redis for every request.

3
docs/my-website/docs/rerank.md
View file

@ -118,4 +118,5 @@ curl http://0.0.0.0:4000/rerank \
| AWS Bedrock| [Usage](../docs/providers/bedrock#rerank-api) |
| HuggingFace| [Usage](../docs/providers/huggingface_rerank) |
| Infinity| [Usage](../docs/providers/infinity) |
| vLLM| [Usage](../docs/providers/vllm#rerank-endpoint) |
| vLLM| [Usage](../docs/providers/vllm#rerank-endpoint) |
| DeepInfra| [Usage](../docs/providers/deepinfra#rerank-endpoint) |

12
docs/my-website/docs/response_api.md
View file

@ -803,10 +803,18 @@ LiteLLM Proxy supports session management for non-OpenAI models. This allows you
1. Enable storing request / response content in the database
Set `store_prompts_in_spend_logs: true` in your proxy config.yaml. When this is enabled, LiteLLM will store the request and response content in the database.
Set `store_prompts_in_cold_storage: true` in your proxy config.yaml. When this is enabled, LiteLLM will store the request and response content in the s3 bucket you specify.
```yaml showLineNumbers title="config.yaml with Session Continuity"
litellm_settings:
callbacks: ["s3_v2"]
cold_storage_custom_logger: s3_v2
s3_callback_params: # learn more https://docs.litellm.ai/docs/proxy/logging#s3-buckets
s3_bucket_name: litellm-logs # AWS Bucket Name for S3
s3_region_name: us-west-2
```yaml
general_settings:
store_prompts_in_cold_storage: true
store_prompts_in_spend_logs: true
```

96
docs/my-website/docs/routing.md
View file

@ -1000,6 +1000,102 @@ router_settings:
</TabItem>
</Tabs>
### How Cooldowns Work
Cooldowns apply to individual deployments, not entire model groups. The router isolates failures to specific deployments while keeping healthy alternatives available.
#### What is a deployment?
A deployment is a single entry in your `config.yaml` model list. Each deployment represents a unique configuration with its own `litellm_params`.
LiteLLM generates a unique `model_id` for each deployment by creating a deterministic hash of all the `litellm_params`. This allows the router to track and manage each deployment independently.
**Example: Multiple deployments for the same model**
```yaml showLineNumbers title="Load Balancing config.yaml"
model_list:
- model_name: sonnet-4 # Deployment 1
litellm_params:
model: anthropic/claude-sonnet-4-20250514
api_key: <our-real-key>
- model_name: byok-sonnet-4 # Deployment 2
litellm_params:
model: anthropic/claude-sonnet-4-20250514
api_key: <customer-managed-key>
api_base: https://proxy.litellm.ai/api.anthropic.com
- model_name: sonnet-4 # Deployment 3
litellm_params:
model: vertex_ai/claude-sonnet-4-20250514
vertex_project: my-project
```
Each deployment gets a unique `model_id` (e.g., `1234567890`, `9129922`, `4982929292`) that the router uses for tracking health and cooldown status.
#### When are deployments cooled down?
The router automatically cools down deployments based on the following conditions:
| Condition | Trigger | Cooldown Duration |
|-----------|---------|-------------------|
| **Rate Limiting (429)** | Immediate on 429 response | 5 seconds (default) |
| **High Failure Rate** | >50% failures in current minute | 5 seconds (default) |
| **Non-Retryable Errors** | 401 (Auth), 404 (Not Found), 408 (Timeout) | 5 seconds (default) |
During cooldown, the specific deployment is temporarily removed from the available pool, while other healthy deployments continue serving requests.
#### Cooldown Recovery
Deployments automatically recover from cooldown after the cooldown period expires. The router will:
1. **Monitor cooldown timers** for each deployment
2. **Automatically re-enable** deployments when cooldown expires
3. **Gradually reintroduce** cooled-down deployments to the rotation
4. **Reset failure counters** once the deployment is healthy again
#### Real-World Example
Consider this high-availability setup with multiple providers:
```yaml showLineNumbers title="Load Balancing config.yaml"
model_list:
- model_name: sonnet-4 # Primary: Anthropic Direct
litellm_params:
model: anthropic/claude-sonnet-4-20250514
api_key: <anthropic-key>
- model_name: byok-sonnet-4 # BYOK: Customer-managed keys
litellm_params:
model: anthropic/claude-sonnet-4-20250514
api_key: <customer-managed-key>
api_base: https://proxy.litellm.ai/api.anthropic.com
- model_name: sonnet-4 # Fallback: Vertex AI
litellm_params:
model: vertex_ai/claude-sonnet-4-20250514
vertex_project: my-project
```
**Failure Scenario:**
```mermaid
flowchart TD
A["Request for 'sonnet-4'"] --> B["Router finds available deployments"]
B --> C["Available:<br/>• Anthropic Direct<br/>• Vertex AI"]
C --> D["Selects Anthropic Direct"]
D --> E{"Request fails with 429?"}
E -->|No| F["Success ✅"]
E -->|Yes| G["Cooldown Anthropic Direct<br/>for 5 seconds"]
G --> H["Next request for 'sonnet-4'"]
H --> I["Route to Vertex AI<br/>(only available deployment for model_name='sonnet-4')"]
I --> J["Success ✅"]
style G fill:#ffcccc
style I fill:#ccffcc
```
### Retries
For both async + sync functions, we support retrying failed requests.

1353
docs/my-website/docs/simple_proxy_old_doc.md
View file

File diff suppressed because it is too large Load diff

95
docs/my-website/docs/tutorials/claude_responses_api.md
View file

@ -4,11 +4,11 @@ import TabItem from '@theme/TabItem';
# Claude Code
This tutorial shows how to call the Responses API models like `codex-mini` and `o3-pro` from the Claude Code endpoint on LiteLLM.
This tutorial shows how to call Claude models through LiteLLM proxy from Claude Code.
:::info
This tutorial is based on [Anthropic's official LiteLLM configuration documentation](https://docs.anthropic.com/en/docs/claude-code/llm-gateway#litellm-configuration). This integration allows you to use any LiteLLM supported model through Claude Code.
This tutorial is based on [Anthropic's official LiteLLM configuration documentation](https://docs.anthropic.com/en/docs/claude-code/llm-gateway#litellm-configuration). This integration allows you to use any LiteLLM supported model through Claude Code with centralized authentication, usage tracking, and cost controls.
:::
@ -31,19 +31,18 @@ Create a secure configuration using environment variables:
```yaml
model_list:
# Responses API models
- model_name: codex-mini
# Claude models
- model_name: claude-3-5-sonnet-20241022
litellm_params:
model: openai/codex-mini
api_key: os.environ/OPENAI_API_KEY
api_base: https://api.openai.com/v1
model: anthropic/claude-3-5-sonnet-20241022
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: o3-pro
- model_name: claude-3-5-haiku-20241022
litellm_params:
model: openai/o3-pro
api_key: os.environ/OPENAI_API_KEY
api_base: https://api.openai.com/v1
model: anthropic/claude-3-5-haiku-20241022
api_key: os.environ/ANTHROPIC_API_KEY
litellm_settings:
master_key: os.environ/LITELLM_MASTER_KEY
```
@ -51,7 +50,7 @@ litellm_settings:
Set your environment variables:
```bash
export OPENAI_API_KEY="your-openai-api-key"
export ANTHROPIC_API_KEY="your-anthropic-api-key"
export LITELLM_MASTER_KEY="sk-1234567890" # Generate a secure key
```
@ -72,31 +71,43 @@ curl -X POST http://0.0.0.0:4000/v1/messages \
-H "Authorization: Bearer $LITELLM_MASTER_KEY" \
-H "Content-Type: application/json" \
-d '{
"model": "codex-mini",
"model": "claude-3-5-sonnet-20241022",
"max_tokens": 1000,
"messages": [{"role": "user", "content": "What is the capital of France?"}]
}'
```
### 4. Configure Claude Code
Setup Claude Code to use your LiteLLM proxy:
#### Method 1: Unified Endpoint (Recommended)
Configure Claude Code to use LiteLLM's unified endpoint:
```bash
export ANTHROPIC_BASE_URL="http://0.0.0.0:4000"
export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY"
```
### 5. Use Claude Code
#### Method 2: Provider-specific Pass-through Endpoint
Start Claude Code with any configured model:
Alternatively, use the Anthropic pass-through endpoint:
```bash
# Use Responses API models
claude --model codex-mini
claude --model o3-pro
export ANTHROPIC_BASE_URL="http://0.0.0.0:4000/anthropic"
export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY"
```
# Or use the latest model alias
claude --model codex-mini-latest
### 5. Use Claude Code
Start Claude Code and it will automatically use your configured models:
```bash
# Claude Code will use the models configured in your LiteLLM proxy
claude
# Or specify a model if you have multiple configured
claude --model claude-3-5-sonnet-20241022
claude --model claude-3-5-haiku-20241022
```
Example conversation:
@ -112,7 +123,8 @@ Common issues and solutions:
**Authentication errors:**
- Verify your environment variables are set: `echo $LITELLM_MASTER_KEY`
- Check that your OpenAI API key is valid and has sufficient credits
- Check that your API keys are valid and have sufficient credits
- Ensure the `ANTHROPIC_AUTH_TOKEN` matches your LiteLLM master key
**Model not found:**
- Ensure the model name in Claude Code matches exactly with your `config.yaml`
@ -123,33 +135,47 @@ Common issues and solutions:
Expand your configuration to support multiple providers and models:
<Tabs>
<TabItem value="responses-plus" label="Responses + Standard Models">
<TabItem value="multi-provider" label="Multi-Provider Setup">
```yaml
model_list:
# Responses API models
# OpenAI models
- model_name: codex-mini
litellm_params:
litellm_params:
model: openai/codex-mini
api_key: os.environ/OPENAI_API_KEY
api_base: https://api.openai.com/v1
- model_name: o3-pro
litellm_params:
model: openai/o3-pro
api_key: os.environ/OPENAI_API_KEY
api_base: https://api.openai.com/v1
# Standard models
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY
api_base: https://api.openai.com/v1
- model_name: claude-3-5-sonnet
# Anthropic models
- model_name: claude-3-5-sonnet-20241022
litellm_params:
model: anthropic/claude-3-5-sonnet-20241022
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: claude-3-5-haiku-20241022
litellm_params:
model: anthropic/claude-3-5-haiku-20241022
api_key: os.environ/ANTHROPIC_API_KEY
# AWS Bedrock
- model_name: claude-bedrock
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1
litellm_settings:
master_key: os.environ/LITELLM_MASTER_KEY
@ -158,13 +184,14 @@ litellm_settings:
Switch between models seamlessly:
```bash
# Use Responses API models for advanced reasoning
claude --model o3-pro
claude --model codex-mini
# Use Claude for complex reasoning
claude --model claude-3-5-sonnet-20241022
# Use standard models for general tasks
claude --model gpt-4o
claude --model claude-3-5-sonnet
# Use Haiku for fast responses
claude --model claude-3-5-haiku-20241022
# Use Bedrock deployment
claude --model claude-bedrock
```
</TabItem>

5
docs/my-website/docusaurus.config.js
View file

@ -136,6 +136,11 @@ const config = {
],
],
themes: ['@docusaurus/theme-mermaid'],
markdown: {
mermaid: true,
},
scripts: [
{
async: true,

BIN
docs/my-website/img/release_notes/faster_caching_calls.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 270 KiB

BIN
docs/my-website/img/release_notes/responses_api_session_mgt_images.jpg Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 472 KiB

1196
docs/my-website/package-lock.json generated
View file

File diff suppressed because it is too large Load diff

4
docs/my-website/package.json
View file

@ -18,6 +18,7 @@
"@docusaurus/plugin-google-gtag": "3.8.1",
"@docusaurus/plugin-ideal-image": "3.8.1",
"@docusaurus/preset-classic": "3.8.1",
"@docusaurus/theme-mermaid": "^3.8.1",
"@inkeep/cxkit-docusaurus": "^0.5.89",
"@mdx-js/react": "^3.0.0",
"clsx": "^1.2.1",
@ -48,6 +49,7 @@
},
"overrides": {
"webpack-dev-server": ">=5.2.1",
"form-data": ">=4.0.4"
"form-data": ">=4.0.4",
"mermaid": ">=11.10.0"
}
}

6
docs/my-website/release_notes/v1.74.15-stable/index.md
View file

@ -1,5 +1,5 @@
---
title: "[Pre-Release] v1.74.15-stable"
title: "v1.74.15-stable"
slug: "v1-74-15"
date: 2025-08-02T10:00:00
authors:
@ -28,14 +28,14 @@ import TabItem from '@theme/TabItem';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:1.74.15.rc.1
ghcr.io/berriai/litellm:v1.74.15-stable
```
</TabItem>
<TabItem value="pip" label="Pip">
``` showLineNumbers title="pip install litellm"
pip install litellm==1.74.15.post1
pip install litellm==1.74.15.post2
```
</TabItem>

299
docs/my-website/release_notes/v1.75.5-stable/index.md Normal file
View file

@ -0,0 +1,299 @@
---
title: "v1.75.5-stable - Redis latency improvements"
slug: "v1-75-5"
date: 2025-08-10T10:00:00
authors:
- name: Krrish Dholakia
title: CEO, LiteLLM
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaffer
title: CTO, LiteLLM
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
hide_table_of_contents: false
---
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
## Deploy this version
<Tabs>
<TabItem value="docker" label="Docker">
``` showLineNumbers title="docker run litellm"
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:v1.75.5-stable
```
</TabItem>
<TabItem value="pip" label="Pip">
``` showLineNumbers title="pip install litellm"
pip install litellm==1.75.5.post2
```
</TabItem>
</Tabs>
---
## Key Highlights
- **Redis - Latency Improvements** - Reduces P99 latency by 50% with Redis enabled.
- **Responses API Session Management** - Support for managing responses API sessions with images.
- **Oracle Cloud Infrastructure** - New LLM provider for calling models on Oracle Cloud Infrastructure.
- **Digital Ocean's Gradient AI** - New LLM provider for calling models on Digital Ocean's Gradient AI platform.
### Risk of Upgrade
If you build the proxy from the pip package, you should hold off on upgrading. This version makes `prisma migrate deploy` our default for managing the DB. This is safer, as it doesn't reset the DB, but it requires a manual `prisma generate` step.
Users of our Docker image, are **not** affected by this change.
---
## Redis Latency Improvements
<Image
img={require('../../img/release_notes/faster_caching_calls.png')}
style={{width: '100%', display: 'block', margin: '2rem auto'}}
/>
<br/>
This release adds in-memory caching for Redis requests, enabling faster response times in high-traffic. Now, LiteLLM instances will check their in-memory cache for a cache hit, before checking Redis. This reduces caching-related latency from 100ms for LLM API calls to sub-1ms, on cache hits.
---
## Responses API Session Management w/ Images
<Image
img={require('../../img/release_notes/responses_api_session_mgt_images.jpg')}
style={{width: '100%', display: 'block', margin: '2rem auto'}}
/>
<br/>
LiteLLM now supports session management for Responses API requests with images. This is great for use-cases like chatbots, that are using the Responses API to track the state of a conversation. LiteLLM session management works across **ALL** LLM API's (including Anthropic, Bedrock, OpenAI, etc). LiteLLM session management works by storing the request and response content in an s3 bucket, you can specify.
---
## New Models / Updated Models
#### New Model Support
| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) |
| ----------- | -------------------------------------- | -------------- | ------------------- | -------------------- |
| Bedrock | `bedrock/us.anthropic.claude-opus-4-1-20250805-v1:0` | 200k | $15 | $75 |
| Bedrock | `bedrock/openai.gpt-oss-20b-1:0` | 200k | 0.07 | 0.3 |
| Bedrock | `bedrock/openai.gpt-oss-120b-1:0` | 200k | 0.15 | 0.6 |
| Fireworks AI | `fireworks_ai/accounts/fireworks/models/glm-4p5` | 128k | 0.55 | 2.19 |
| Fireworks AI | `fireworks_ai/accounts/fireworks/models/glm-4p5-air` | 128k | 0.22 | 0.88 |
| Fireworks AI | `fireworks_ai/accounts/fireworks/models/gpt-oss-120b` | 131072 | 0.15 | 0.6 |
| Fireworks AI | `fireworks_ai/accounts/fireworks/models/gpt-oss-20b` | 131072 | 0.05 | 0.2 |
| Groq | `groq/openai/gpt-oss-20b` | 131072 | 0.1 | 0.5 |
| Groq | `groq/openai/gpt-oss-120b` | 131072 | 0.15 | 0.75 |
| OpenAI | `openai/gpt-5` | 400k | 1.25 | 10 |
| OpenAI | `openai/gpt-5-2025-08-07` | 400k | 1.25 | 10 |
| OpenAI | `openai/gpt-5-mini` | 400k | 0.25 | 2 |
| OpenAI | `openai/gpt-5-mini-2025-08-07` | 400k | 0.25 | 2 |
| OpenAI | `openai/gpt-5-nano` | 400k | 0.05 | 0.4 |
| OpenAI | `openai/gpt-5-nano-2025-08-07` | 400k | 0.05 | 0.4 |
| OpenAI | `openai/gpt-5-chat` | 400k | 1.25 | 10 |
| OpenAI | `openai/gpt-5-chat-latest` | 400k | 1.25 | 10 |
| Azure | `azure/gpt-5` | 400k | 1.25 | 10 |
| Azure | `azure/gpt-5-2025-08-07` | 400k | 1.25 | 10 |
| Azure | `azure/gpt-5-mini` | 400k | 0.25 | 2 |
| Azure | `azure/gpt-5-mini-2025-08-07` | 400k | 0.25 | 2 |
| Azure | `azure/gpt-5-nano-2025-08-07` | 400k | 0.05 | 0.4 |
| Azure | `azure/gpt-5-nano` | 400k | 0.05 | 0.4 |
| Azure | `azure/gpt-5-chat` | 400k | 1.25 | 10 |
| Azure | `azure/gpt-5-chat-latest` | 400k | 1.25 | 10 |
#### Features
- **[OCI](../../docs/providers/oci)**
- New LLM provider - [PR #13206](https://github.com/BerriAI/litellm/pull/13206)
- **[JinaAI](../../docs/providers/jina_ai)**
- support multimodal embedding models - [PR #13181](https://github.com/BerriAI/litellm/pull/13181)
- **GPT-5 ([OpenAI](../../docs/providers/openai)/[Azure](../../docs/providers/azure))**
- Support drop_params for temperature - [PR #13390](https://github.com/BerriAI/litellm/pull/13390)
- Map max_tokens to max_completion_tokens - [PR #13390](https://github.com/BerriAI/litellm/pull/13390)
- **[Anthropic](../../docs/providers/anthropic)**
- Add claude-opus-4-1 on model cost map - [PR #13384](https://github.com/BerriAI/litellm/pull/13384)
- **[OpenRouter](../../docs/providers/openrouter)**
- Add gpt-oss to model cost map - [PR #13442](https://github.com/BerriAI/litellm/pull/13442)
- **[Cerebras](../../docs/providers/cerebras)**
- Add gpt-oss to model cost map - [PR #13442](https://github.com/BerriAI/litellm/pull/13442)
- **[Azure](../../docs/providers/azure)**
- Support drop params for ‘temperature’ on o-series models - [PR #13353](https://github.com/BerriAI/litellm/pull/13353)
- **[GradientAI](../../docs/providers/gradient_ai)**
- New LLM Provider - [PR #12169](https://github.com/BerriAI/litellm/pull/12169)
#### Bugs
- **[OpenAI](../../docs/providers/openai)**
- Add ‘service_tier’ and ‘safety_identifier’ as supported responses api params - [PR #13258](https://github.com/BerriAI/litellm/pull/13258)
- Correct pricing for web search on 4o-mini - [PR #13269](https://github.com/BerriAI/litellm/pull/13269)
- **[Mistral](../../docs/providers/mistral)**
- Handle $id and $schema fields when calling mistral - [PR #13389](https://github.com/BerriAI/litellm/pull/13389)
---
## LLM API Endpoints
#### Features
- `/responses`
- Responses API Session Handling w/ support for images - [PR #13347](https://github.com/BerriAI/litellm/pull/13347)
- failed if input containing ResponseReasoningItem - [PR #13465](https://github.com/BerriAI/litellm/pull/13465)
- Support custom tools - [PR #13418](https://github.com/BerriAI/litellm/pull/13418)
#### Bugs
- `/chat/completions`
- Fix completion_token_details usage object missing ‘text’ tokens - [PR #13234](https://github.com/BerriAI/litellm/pull/13234)
- (SDK) handle tool being a pydantic object - [PR #13274](https://github.com/BerriAI/litellm/pull/13274)
- include cost in streaming usage object - [PR #13418](https://github.com/BerriAI/litellm/pull/13418)
- Exclude none fields on /chat/completion - allows usage with n8n - [PR #13320](https://github.com/BerriAI/litellm/pull/13320)
- `/responses`
- Transform function call in response for non-openai models (gemini/anthropic) - [PR #13260](https://github.com/BerriAI/litellm/pull/13260)
- Fix unsupported operand error with model groups - [PR #13293](https://github.com/BerriAI/litellm/pull/13293)
- Responses api session management for streaming responses - [PR #13396](https://github.com/BerriAI/litellm/pull/13396)
- `/v1/messages`
- Added litellm claude code count tokens - [PR #13261](https://github.com/BerriAI/litellm/pull/13261)
- `/vector_stores`
- Fix create/search vector store errors - [PR #13285](https://github.com/BerriAI/litellm/pull/13285)
---
## [MCP Gateway](../../docs/mcp)
#### Features
- Add route check for internal users - [PR #13350](https://github.com/BerriAI/litellm/pull/13350)
- MCP Guardrails - docs - [PR #13392](https://github.com/BerriAI/litellm/pull/13392)
#### Bugs
- Fix auth on UI for bearer token servers - [PR #13312](https://github.com/BerriAI/litellm/pull/13312)
- allow access group on mcp tool retrieval - [PR #13425](https://github.com/BerriAI/litellm/pull/13425)
---
## Management Endpoints / UI
#### Features
- **Teams**
- Add team deletion check for teams with keys - [PR #12953](https://github.com/BerriAI/litellm/pull/12953)
- **Models**
- Add ability to set model alias per key/team - [PR #13276](https://github.com/BerriAI/litellm/pull/13276)
- New button to reload model pricing from model cost map - [PR #13464](https://github.com/BerriAI/litellm/pull/13464), [PR #13470](https://github.com/BerriAI/litellm/pull/13470)
- **Keys**
- Make ‘team’ field required when creating service account keys - [PR #13302](https://github.com/BerriAI/litellm/pull/13302)
- Gray out key-based logging settings for non-enterprise users - prevents confusion on if ‘logging’ all up is supported - [PR #13431](https://github.com/BerriAI/litellm/pull/13431)
- **Navbar**
- Add logo customization for LiteLLM admin UI - [PR #12958](https://github.com/BerriAI/litellm/pull/12958)
- **Logs**
- Add token breakdowns on logs + session page - [PR #13357](https://github.com/BerriAI/litellm/pull/13357)
- **Usage**
- Ensure Usage Page loads after the DB has large entries - [PR #13400](https://github.com/BerriAI/litellm/pull/13400)
- **Test Key Page**
- allow uploading images for /chat/completions and /responses - [PR #13445](https://github.com/BerriAI/litellm/pull/13445)
- **MCP**
- Add auth tokens to local storage auth - [PR #13473](https://github.com/BerriAI/litellm/pull/13473)
#### Bugs
- **Custom Root Path**
- Fix login route when SSO is enabled - [PR #13267](https://github.com/BerriAI/litellm/pull/13267)
- **Customers/End-users**
- Allow calling /v1/models when end user over budget - allows model listing to work on OpenWebUI when customer over budget - [PR #13320](https://github.com/BerriAI/litellm/pull/13320)
- **Teams**
- Remove user - team membership, when user removed from team - [PR #13433](https://github.com/BerriAI/litellm/pull/13433)
- **Errors**
- Bubble up network errors to user for Logging and Alerts page - [PR #13427](https://github.com/BerriAI/litellm/pull/13427)
- **Model Hub**
- Show pricing for azure models, when base model is set - [PR #13418](https://github.com/BerriAI/litellm/pull/13418)
---
## Logging / Guardrail Integrations
#### Features
- **Bedrock Guardrails**
- Redacted sensitive information in bedrock guardrails error message - [PR #13356](https://github.com/BerriAI/litellm/pull/13356)
- **Standard Logging Payload**
- Fix ‘can’t register atextexit’ bug - [PR #13436](https://github.com/BerriAI/litellm/pull/13436)
#### Bugs
- **Braintrust**
- Allow setting of braintrust callback base url - [PR #13368](https://github.com/BerriAI/litellm/pull/13368)
- **OTEL**
- Track pre_call hook latency - [PR #13362](https://github.com/BerriAI/litellm/pull/13362)
---
## Performance / Loadbalancing / Reliability improvements
#### Features
- **Team-BYOK models**
- Add wildcard model support - [PR #13278](https://github.com/BerriAI/litellm/pull/13278)
- **Caching**
- GCP IAM auth support for caching - [PR #13275](https://github.com/BerriAI/litellm/pull/13275)
- **Latency**
- reduce p99 latency w/ redis enabled by 50% - only updates model usage if tpm/rpm limits set - [PR #13362](https://github.com/BerriAI/litellm/pull/13362)
---
## General Proxy Improvements
#### Features
- **Models**
- Support /v1/models/\{model_id\} retrieval - [PR #13268](https://github.com/BerriAI/litellm/pull/13268)
- **Multi-instance**
- Ensure disable_llm_api_endpoints works - [PR #13278](https://github.com/BerriAI/litellm/pull/13278)
- **Logs**
- Add apscheduler log suppress - [PR #13299](https://github.com/BerriAI/litellm/pull/13299)
- **Helm**
- Add labels to migrations job template - [PR #13343](https://github.com/BerriAI/litellm/pull/13343) s/o [@unique-jakub](https://github.com/unique-jakub)
#### Bugs
- **Non-root image**
- Fix non-root image for migration - [PR #13379](https://github.com/BerriAI/litellm/pull/13379)
- **Get Routes**
- Load get routes when using fastapi-offline - [PR #13466](https://github.com/BerriAI/litellm/pull/13466)
- **Health checks**
- Generate unique trace IDs for Langfuse health checks - [PR #13468](https://github.com/BerriAI/litellm/pull/13468)
- **Swagger**
- Allow using Swagger for /chat/completions - [PR #13469](https://github.com/BerriAI/litellm/pull/13469)
- **Auth**
- Fix JWTs access not working with model access groups - [PR #13474](https://github.com/BerriAI/litellm/pull/13474)
---
## New Contributors
* @bbartels made their first contribution in https://github.com/BerriAI/litellm/pull/13244
* @breno-aumo made their first contribution in https://github.com/BerriAI/litellm/pull/13206
* @pascalwhoop made their first contribution in https://github.com/BerriAI/litellm/pull/13122
* @ZPerling made their first contribution in https://github.com/BerriAI/litellm/pull/13045
* @zjx20 made their first contribution in https://github.com/BerriAI/litellm/pull/13181
* @edwarddamato made their first contribution in https://github.com/BerriAI/litellm/pull/13368
* @msannan2 made their first contribution in https://github.com/BerriAI/litellm/pull/12169
## **[Full Changelog](https://github.com/BerriAI/litellm/compare/v1.74.15-stable...v1.75.5-stable.rc-draft)**

231
docs/my-website/release_notes/v1.75.8/index.md Normal file
View file

@ -0,0 +1,231 @@
---
title: "[PRE-RELEASE]v1.75.8"
slug: "v1-75-8"
date: 2025-08-16T10:00:00
authors:
- name: Krrish Dholakia
title: CEO, LiteLLM
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaffer
title: CTO, LiteLLM
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
hide_table_of_contents: false
---
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
## Deploy this version
<Tabs>
<TabItem value="docker" label="Docker">
``` showLineNumbers title="docker run litellm"
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:v1.75.8
```
</TabItem>
<TabItem value="pip" label="Pip">
``` showLineNumbers title="pip install litellm"
pip install litellm==1.75.8
```
</TabItem>
</Tabs>
---
## Key Highlights
- **Team Member Rate Limits** - Individual rate limiting for team members with JWT authentication support.
- **Performance Improvements** - New experimental HTTP handler flag for 100+ RPS improvement on OpenAI calls.
- **GPT-5 Model Family Support** - Full support for OpenAI's GPT-5 models with `reasoning_effort` parameter and Azure OpenAI integration.
- **Azure AI Flux Image Generation** - Support for Azure AI's Flux image generation models.
---
## New Models / Updated Models
#### New Model Support
| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features |
| ----------- | -------------------------------------- | -------------- | ------------------- | -------------------- | -------- |
| Azure AI | `azure_ai/FLUX-1.1-pro` | - | - | $40/image | Image generation |
| Azure AI | `azure_ai/FLUX.1-Kontext-pro` | - | - | $40/image | Image generation |
| Vertex AI | `vertex_ai/deepseek-ai/deepseek-r1-0528-maas` | 65k | $1.35 | $5.4 | Chat completions + reasoning |
| OpenRouter | `openrouter/deepseek/deepseek-chat-v3-0324` | 65k | $0.14 | $0.28 | Chat completions |
#### Features
- **[OpenAI](../../docs/providers/openai)**
- Added `reasoning_effort` parameter support for GPT-5 model family - [PR #13475](https://github.com/BerriAI/litellm/pull/13475), [Get Started](../../docs/providers/openai#openai-chat-completion-models)
- Support for `reasoning` parameter in Responses API - [PR #13475](https://github.com/BerriAI/litellm/pull/13475), [Get Started](../../docs/response_api)
- **[Azure OpenAI](../../docs/providers/azure/azure)**
- GPT-5 support with max_tokens and `reasoning` parameter - [PR #13510](https://github.com/BerriAI/litellm/pull/13510), [Get Started](../../docs/providers/azure/azure#gpt-5-models)
- **[AWS Bedrock](../../docs/providers/bedrock)**
- Streaming support for bedrock gpt-oss model family - [PR #13346](https://github.com/BerriAI/litellm/pull/13346), [Get Started](../../docs/providers/bedrock#openai-gpt-oss)
- `/messages` endpoint compatibility with `bedrock/converse/<model>` - [PR #13627](https://github.com/BerriAI/litellm/pull/13627)
- Cache point support for assistant and tool messages - [PR #13640](https://github.com/BerriAI/litellm/pull/13640)
- **[Azure AI](../../docs/providers/azure)**
- New Azure AI Flux Image Generation provider - [PR #13592](https://github.com/BerriAI/litellm/pull/13592), [Get Started](../../docs/providers/azure_ai_img)
- Fixed Content-Type header for image generation - [PR #13584](https://github.com/BerriAI/litellm/pull/13584)
- **[CometAPI](../../docs/providers/comet)**
- New provider support with chat completions and streaming - [PR #13458](https://github.com/BerriAI/litellm/pull/13458)
- **[SambaNova](../../docs/providers/sambanova)**
- Added embedding model support - [PR #13308](https://github.com/BerriAI/litellm/pull/13308), [Get Started](../../docs/providers/sambanova#sambanova---embeddings)
- **[Vertex AI](../../docs/providers/vertex)**
- Added `/countTokens` endpoint support for Gemini CLI integration - [PR #13545](https://github.com/BerriAI/litellm/pull/13545)
- Token counter support for VertexAI models - [PR #13558](https://github.com/BerriAI/litellm/pull/13558)
- **[hosted_vllm](../../docs/providers/vllm)**
- Added `reasoning_effort` parameter support - [PR #13620](https://github.com/BerriAI/litellm/pull/13620), [Get Started](../../docs/providers/vllm#reasoning-effort)
#### Bugs
- **[OCI](../../docs/providers/oci)**
- Fixed streaming issues - [PR #13437](https://github.com/BerriAI/litellm/pull/13437)
- **[Ollama](../../docs/providers/ollama)**
- Fixed GPT-OSS streaming with 'thinking' field - [PR #13375](https://github.com/BerriAI/litellm/pull/13375)
- **[VolcEngine](../../docs/providers/volcengine)**
- Fixed thinking disabled parameter handling - [PR #13598](https://github.com/BerriAI/litellm/pull/13598)
- **[Streaming](../../docs/completion/stream)**
- Consistent 'finish_reason' chunk indexing - [PR #13560](https://github.com/BerriAI/litellm/pull/13560)
---
## LLM API Endpoints
#### Features
- **[/messages](../../docs/anthropic/messages)**
- Tool use arguments properly returned for non-anthropic models - [PR #13638](https://github.com/BerriAI/litellm/pull/13638)
#### Bugs
- **[Real-time API](../../docs/realtime)**
- Fixed endpoint for no intent scenarios - [PR #13476](https://github.com/BerriAI/litellm/pull/13476)
- **[Responses API](../../docs/response_api)**
- Fixed `stream=True` + `background=True` with Responses API - [PR #13654](https://github.com/BerriAI/litellm/pull/13654)
---
## [MCP Gateway](../../docs/mcp)
#### Features
- **Access Control & Configuration**
- Enhanced MCPServerManager with access groups and description support - [PR #13549](https://github.com/BerriAI/litellm/pull/13549)
#### Bugs
- **Authentication**
- Fixed MCP gateway key authentication - [PR #13630](https://github.com/BerriAI/litellm/pull/13630)
[Read More](../../docs/mcp)
---
## Management Endpoints / UI
#### Features
- **Team Management**
- Team Member Rate Limits implementation - [PR #13601](https://github.com/BerriAI/litellm/pull/13601)
- JWT authentication support for team member rate limits - [PR #13601](https://github.com/BerriAI/litellm/pull/13601)
- Show team member TPM/RPM limits in UI - [PR #13662](https://github.com/BerriAI/litellm/pull/13662)
- Allow editing team member RPM/TPM limits - [PR #13669](https://github.com/BerriAI/litellm/pull/13669)
- Allow unsetting TPM and RPM in Teams Settings - [PR #13430](https://github.com/BerriAI/litellm/pull/13430)
- Team Member Permissions Page access column changes - [PR #13145](https://github.com/BerriAI/litellm/pull/13145)
- **Key Management**
- Display errors from backend on the UI Keys page - [PR #13435](https://github.com/BerriAI/litellm/pull/13435)
- Added confirmation modal before deleting keys - [PR #13655](https://github.com/BerriAI/litellm/pull/13655)
- Support for `user` parameter in LiteLLM SDK to Proxy communication - [PR #13555](https://github.com/BerriAI/litellm/pull/13555)
- **UI Improvements**
- Fixed internal users table overflow - [PR #12736](https://github.com/BerriAI/litellm/pull/12736)
- Enhanced chart readability with short-form notation for large numbers - [PR #12370](https://github.com/BerriAI/litellm/pull/12370)
- Fixed image overflow in LiteLLM model display - [PR #13639](https://github.com/BerriAI/litellm/pull/13639)
- Removed ambiguous network response errors - [PR #13582](https://github.com/BerriAI/litellm/pull/13582)
- **Credentials**
- Added CredentialDeleteModal component and integration with CredentialsPanel - [PR #13550](https://github.com/BerriAI/litellm/pull/13550)
- **Admin & Permissions**
- Allow routes for admin viewer - [PR #13588](https://github.com/BerriAI/litellm/pull/13588)
#### Bugs
- **SCIM Integration**
- Fixed SCIM Team Memberships metadata handling - [PR #13553](https://github.com/BerriAI/litellm/pull/13553)
- **Authentication**
- Fixed incorrect key info endpoint - [PR #13633](https://github.com/BerriAI/litellm/pull/13633)
---
## Logging / Guardrail Integrations
#### Features
- **[Langfuse OTEL](../../docs/proxy/logging#langfuse)**
- Added key/team logging for Langfuse OTEL Logger - [PR #13512](https://github.com/BerriAI/litellm/pull/13512)
- Fixed LangfuseOtelSpanAttributes constants to match expected values - [PR #13659](https://github.com/BerriAI/litellm/pull/13659)
- **[MLflow](../../docs/proxy/logging#mlflow)**
- Updated MLflow logger usage span attributes - [PR #13561](https://github.com/BerriAI/litellm/pull/13561)
#### Bugs
- **Security**
- Hide sensitive data in `/model/info` - azure entra client_secret - [PR #13577](https://github.com/BerriAI/litellm/pull/13577)
- Fixed trivy/secrets false positives - [PR #13631](https://github.com/BerriAI/litellm/pull/13631)
---
## Performance / Loadbalancing / Reliability improvements
#### Features
- **HTTP Performance**
- New 'EXPERIMENTAL_OPENAI_BASE_LLM_HTTP_HANDLER' flag for +100 RPS improvement on OpenAI calls - [PR #13625](https://github.com/BerriAI/litellm/pull/13625)
- **Database Monitoring**
- Added DB metrics to Prometheus - [PR #13626](https://github.com/BerriAI/litellm/pull/13626)
- **Error Handling**
- Added safe divide by 0 protection to prevent crashes - [PR #13624](https://github.com/BerriAI/litellm/pull/13624)
#### Bugs
- **Dependencies**
- Updated boto3 to 1.36.0 and aioboto3 to 13.4.0 - [PR #13665](https://github.com/BerriAI/litellm/pull/13665)
---
## General Proxy Improvements
#### Features
- **Database**
- Removed redundant `use_prisma_migrate` flag - now default - [PR #13555](https://github.com/BerriAI/litellm/pull/13555)
- **LLM Translation**
- Added model ID check - [PR #13507](https://github.com/BerriAI/litellm/pull/13507)
- Refactored Anthropic configurations and added support for `anthropic_beta` headers - [PR #13590](https://github.com/BerriAI/litellm/pull/13590)
---
## New Contributors
* @TensorNull made their first contribution in [PR #13458](https://github.com/BerriAI/litellm/pull/13458)
* @MajorD00m made their first contribution in [PR #13577](https://github.com/BerriAI/litellm/pull/13577)
* @VerunicaM made their first contribution in [PR #13584](https://github.com/BerriAI/litellm/pull/13584)
* @huangyafei made their first contribution in [PR #13607](https://github.com/BerriAI/litellm/pull/13607)
* @TomeHirata made their first contribution in [PR #13561](https://github.com/BerriAI/litellm/pull/13561)
* @willfinnigan made their first contribution in [PR #13659](https://github.com/BerriAI/litellm/pull/13659)
* @dcbark01 made their first contribution in [PR #13633](https://github.com/BerriAI/litellm/pull/13633)
* @javacruft made their first contribution in [PR #13631](https://github.com/BerriAI/litellm/pull/13631)
---
## **[Full Changelog](https://github.com/BerriAI/litellm/compare/v1.75.5-stable.rc-draft...v1.75.8-nightly)**

22
docs/my-website/sidebars.js
View file

@ -65,6 +65,7 @@ const sidebars = {
label: "[Beta] Prompt Management",
items: [
"proxy/prompt_management",
"proxy/native_litellm_prompt",
"proxy/custom_prompt_management"
].sort()
},
@ -81,12 +82,12 @@ const sidebars = {
"tutorials/cost_tracking_coding",
]
},
],
// But you can create a sidebar manually
tutorialSidebar: [
{ type: "doc", id: "index" }, // NEW
{
type: "category",
label: "LiteLLM Proxy Server",
@ -107,6 +108,7 @@ const sidebars = {
type: "category",
label: "Setup & Deployment",
items: [
"proxy/quick_start",
"proxy/deploy",
"proxy/prod",
"proxy/cli",
@ -213,7 +215,7 @@ const sidebars = {
"proxy/dynamic_logging"
],
},
{
type: "category",
label: "Secret Managers",
@ -371,7 +373,14 @@ const sidebars = {
"providers/azure/azure_embedding",
]
},
"providers/azure_ai",
{
type: "category",
label: "Azure AI",
items: [
"providers/azure_ai",
"providers/azure_ai_img",
]
},
{
type: "category",
label: "Vertex AI",
@ -466,6 +475,7 @@ const sidebars = {
"providers/custom_llm_server",
"providers/petals",
"providers/snowflake",
"providers/gradient_ai",
"providers/featherless_ai",
"providers/nebius",
"providers/dashscope",
@ -504,7 +514,7 @@ const sidebars = {
]
},
{
type: "category",
label: "Routing, Loadbalancing & Fallbacks",
@ -535,7 +545,7 @@ const sidebars = {
},
],
},
{
type: "category",
label: "Load Testing",

16
enterprise/litellm_enterprise/proxy/management_endpoints/internal_user_endpoints.py
View file

@ -2,6 +2,8 @@
Enterprise internal user management endpoints
"""
import os
from fastapi import APIRouter, Depends, HTTPException
from litellm.proxy._types import UserAPIKeyAuth
@ -21,7 +23,7 @@ async def available_enterprise_users(
"""
For keys with `max_users` set, return the list of users that are allowed to use the key.
"""
from litellm.proxy._types import CommonProxyErrors
from litellm.proxy._types import CommonProxyErrors, EnterpriseLicenseData
from litellm.proxy.proxy_server import (
premium_user,
premium_user_data,
@ -34,10 +36,14 @@ async def available_enterprise_users(
detail={"error": CommonProxyErrors.db_not_connected_error.value},
)
if premium_user is None:
raise HTTPException(
status_code=500, detail={"error": CommonProxyErrors.not_premium_user.value}
)
if not premium_user:
# check if SSO is enabled - show 5 user limit
from litellm.proxy.auth.auth_utils import _has_user_setup_sso
if _has_user_setup_sso():
premium_user_data = EnterpriseLicenseData(
max_users=5,
)
# Count number of rows in LiteLLM_UserTable
user_count = await prisma_client.db.litellm_usertable.count()

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.2.17-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.2.17.tar.gz vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.2.18-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.2.18.tar.gz vendored Normal file
View file

Binary file not shown.

3
litellm-proxy-extras/litellm_proxy_extras/utils.py
View file

@ -243,7 +243,6 @@ class ProxyExtrasDBManager:
bool: True if setup was successful, False otherwise
"""
schema_path = ProxyExtrasDBManager._get_prisma_dir() + "/schema.prisma"
use_migrate = str_to_bool(os.getenv("USE_PRISMA_MIGRATE")) or use_migrate
for attempt in range(4):
original_dir = os.getcwd()
migrations_dir = ProxyExtrasDBManager._get_prisma_dir()
@ -299,7 +298,7 @@ class ProxyExtrasDBManager:
and "database schema is not empty" in e.stderr
):
logger.info(
"Database schema is not empty, creating baseline migration"
"Database schema is not empty, creating baseline migration. In read-only file system, please set an environment variable `LITELLM_MIGRATION_DIR` to a writable directory to enable migrations. Learn more - https://docs.litellm.ai/docs/proxy/prod#read-only-file-system"
)
ProxyExtrasDBManager._create_baseline_migration(schema_path)
logger.info(

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-proxy-extras"
version = "0.2.16"
version = "0.2.18"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
authors = ["BerriAI"]
readme = "README.md"
@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.2.16"
version = "0.2.18"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-proxy-extras==",

469
litellm/__init__.py
View file

@ -146,6 +146,7 @@ _custom_logger_compatible_callbacks_literal = Literal[
"vector_store_pre_call_hook",
"dotprompt",
]
configured_cold_storage_logger: Optional[_custom_logger_compatible_callbacks_literal] = None
logged_real_time_event_types: Optional[Union[List[str], Literal["*"]]] = None
_known_custom_logger_compatible_callbacks: List = list(
get_args(_custom_logger_compatible_callbacks_literal)
@ -231,7 +232,9 @@ aleph_alpha_key: Optional[str] = None
nlp_cloud_key: Optional[str] = None
novita_api_key: Optional[str] = None
snowflake_key: Optional[str] = None
gradient_ai_api_key: Optional[str] = None
nebius_key: Optional[str] = None
cometapi_key: Optional[str] = None
common_cloud_provider_auth_params: dict = {
"params": ["project", "region_name", "token"],
"providers": ["vertex_ai", "bedrock", "watsonx", "azure", "vertex_ai_beta"],
@ -301,7 +304,6 @@ default_in_memory_ttl: Optional[float] = None
default_redis_ttl: Optional[float] = None
default_redis_batch_cache_expiry: Optional[float] = None
model_alias_map: Dict[str, str] = {}
model_group_alias_map: Dict[str, str] = {}
model_group_settings: Optional["ModelGroupSettings"] = None
max_budget: float = 0.0 # set the max budget across all providers
budget_duration: Optional[str] = (
@ -465,75 +467,81 @@ BEDROCK_CONVERSE_MODELS = [
]
####### COMPLETION MODELS ###################
open_ai_chat_completion_models: List = []
open_ai_text_completion_models: List = []
cohere_models: List = []
cohere_chat_models: List = []
mistral_chat_models: List = []
text_completion_codestral_models: List = []
anthropic_models: List = []
openrouter_models: List = []
datarobot_models: List = []
vertex_language_models: List = []
vertex_vision_models: List = []
vertex_chat_models: List = []
vertex_code_chat_models: List = []
vertex_ai_image_models: List = []
vertex_text_models: List = []
vertex_code_text_models: List = []
vertex_embedding_models: List = []
vertex_anthropic_models: List = []
vertex_llama3_models: List = []
vertex_ai_ai21_models: List = []
vertex_mistral_models: List = []
ai21_models: List = []
ai21_chat_models: List = []
nlp_cloud_models: List = []
aleph_alpha_models: List = []
bedrock_models: List = []
bedrock_converse_models: List = BEDROCK_CONVERSE_MODELS
fireworks_ai_models: List = []
fireworks_ai_embedding_models: List = []
deepinfra_models: List = []
perplexity_models: List = []
watsonx_models: List = []
gemini_models: List = []
xai_models: List = []
deepseek_models: List = []
azure_ai_models: List = []
jina_ai_models: List = []
voyage_models: List = []
infinity_models: List = []
databricks_models: List = []
cloudflare_models: List = []
codestral_models: List = []
friendliai_models: List = []
featherless_ai_models: List = []
palm_models: List = []
groq_models: List = []
azure_models: List = []
azure_text_models: List = []
anyscale_models: List = []
cerebras_models: List = []
galadriel_models: List = []
sambanova_models: List = []
novita_models: List = []
assemblyai_models: List = []
snowflake_models: List = []
llama_models: List = []
nscale_models: List = []
nebius_models: List = []
nebius_embedding_models: List = []
deepgram_models: List = []
elevenlabs_models: List = []
dashscope_models: List = []
moonshot_models: List = []
v0_models: List = []
morph_models: List = []
lambda_ai_models: List = []
hyperbolic_models: List = []
recraft_models: List = []
oci_models: List = []
from typing import Set
open_ai_chat_completion_models: Set = set()
open_ai_text_completion_models: Set = set()
cohere_models: Set = set()
cohere_chat_models: Set = set()
mistral_chat_models: Set = set()
text_completion_codestral_models: Set = set()
anthropic_models: Set = set()
openrouter_models: Set = set()
datarobot_models: Set = set()
vertex_language_models: Set = set()
vertex_vision_models: Set = set()
vertex_chat_models: Set = set()
vertex_code_chat_models: Set = set()
vertex_ai_image_models: Set = set()
vertex_text_models: Set = set()
vertex_code_text_models: Set = set()
vertex_embedding_models: Set = set()
vertex_anthropic_models: Set = set()
vertex_llama3_models: Set = set()
vertex_deepseek_models: Set = set()
vertex_ai_ai21_models: Set = set()
vertex_mistral_models: Set = set()
ai21_models: Set = set()
ai21_chat_models: Set = set()
nlp_cloud_models: Set = set()
aleph_alpha_models: Set = set()
bedrock_models: Set = set()
bedrock_converse_models: Set = set(BEDROCK_CONVERSE_MODELS)
fireworks_ai_models: Set = set()
fireworks_ai_embedding_models: Set = set()
deepinfra_models: Set = set()
perplexity_models: Set = set()
watsonx_models: Set = set()
gemini_models: Set = set()
xai_models: Set = set()
deepseek_models: Set = set()
azure_ai_models: Set = set()
jina_ai_models: Set = set()
voyage_models: Set = set()
infinity_models: Set = set()
databricks_models: Set = set()
cloudflare_models: Set = set()
codestral_models: Set = set()
friendliai_models: Set = set()
featherless_ai_models: Set = set()
palm_models: Set = set()
groq_models: Set = set()
azure_models: Set = set()
azure_text_models: Set = set()
anyscale_models: Set = set()
cerebras_models: Set = set()
galadriel_models: Set = set()
sambanova_models: Set = set()
sambanova_embedding_models: Set = set()
novita_models: Set = set()
assemblyai_models: Set = set()
snowflake_models: Set = set()
gradient_ai_models: Set = set()
llama_models: Set = set()
nscale_models: Set = set()
nebius_models: Set = set()
nebius_embedding_models: Set = set()
aiml_models: Set = set()
deepgram_models: Set = set()
elevenlabs_models: Set = set()
dashscope_models: Set = set()
moonshot_models: Set = set()
v0_models: Set = set()
morph_models: Set = set()
lambda_ai_models: Set = set()
hyperbolic_models: Set = set()
recraft_models: Set = set()
cometapi_models: Set = set()
oci_models: Set = set()
def is_bedrock_pricing_only_model(key: str) -> bool:
@ -574,157 +582,168 @@ def add_known_models():
if value.get("litellm_provider") == "openai" and not is_openai_finetune_model(
key
):
open_ai_chat_completion_models.append(key)
open_ai_chat_completion_models.add(key)
elif value.get("litellm_provider") == "text-completion-openai":
open_ai_text_completion_models.append(key)
open_ai_text_completion_models.add(key)
elif value.get("litellm_provider") == "azure_text":
azure_text_models.append(key)
azure_text_models.add(key)
elif value.get("litellm_provider") == "cohere":
cohere_models.append(key)
cohere_models.add(key)
elif value.get("litellm_provider") == "cohere_chat":
cohere_chat_models.append(key)
cohere_chat_models.add(key)
elif value.get("litellm_provider") == "mistral":
mistral_chat_models.append(key)
mistral_chat_models.add(key)
elif value.get("litellm_provider") == "anthropic":
anthropic_models.append(key)
anthropic_models.add(key)
elif value.get("litellm_provider") == "empower":
empower_models.append(key)
empower_models.add(key)
elif value.get("litellm_provider") == "openrouter":
openrouter_models.append(key)
openrouter_models.add(key)
elif value.get("litellm_provider") == "datarobot":
datarobot_models.append(key)
datarobot_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-text-models":
vertex_text_models.append(key)
vertex_text_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-code-text-models":
vertex_code_text_models.append(key)
vertex_code_text_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-language-models":
vertex_language_models.append(key)
vertex_language_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-vision-models":
vertex_vision_models.append(key)
vertex_vision_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-chat-models":
vertex_chat_models.append(key)
vertex_chat_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-code-chat-models":
vertex_code_chat_models.append(key)
vertex_code_chat_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-embedding-models":
vertex_embedding_models.append(key)
vertex_embedding_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-anthropic_models":
key = key.replace("vertex_ai/", "")
vertex_anthropic_models.append(key)
vertex_anthropic_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-llama_models":
key = key.replace("vertex_ai/", "")
vertex_llama3_models.append(key)
vertex_llama3_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-deepseek_models":
key = key.replace("vertex_ai/", "")
vertex_deepseek_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-mistral_models":
key = key.replace("vertex_ai/", "")
vertex_mistral_models.append(key)
vertex_mistral_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-ai21_models":
key = key.replace("vertex_ai/", "")
vertex_ai_ai21_models.append(key)
vertex_ai_ai21_models.add(key)
elif value.get("litellm_provider") == "vertex_ai-image-models":
key = key.replace("vertex_ai/", "")
vertex_ai_image_models.append(key)
vertex_ai_image_models.add(key)
elif value.get("litellm_provider") == "ai21":
if value.get("mode") == "chat":
ai21_chat_models.append(key)
ai21_chat_models.add(key)
else:
ai21_models.append(key)
ai21_models.add(key)
elif value.get("litellm_provider") == "nlp_cloud":
nlp_cloud_models.append(key)
nlp_cloud_models.add(key)
elif value.get("litellm_provider") == "aleph_alpha":
aleph_alpha_models.append(key)
aleph_alpha_models.add(key)
elif value.get(
"litellm_provider"
) == "bedrock" and not is_bedrock_pricing_only_model(key):
bedrock_models.append(key)
bedrock_models.add(key)
elif value.get("litellm_provider") == "bedrock_converse":
bedrock_converse_models.append(key)
bedrock_converse_models.add(key)
elif value.get("litellm_provider") == "deepinfra":
deepinfra_models.append(key)
deepinfra_models.add(key)
elif value.get("litellm_provider") == "perplexity":
perplexity_models.append(key)
perplexity_models.add(key)
elif value.get("litellm_provider") == "watsonx":
watsonx_models.append(key)
watsonx_models.add(key)
elif value.get("litellm_provider") == "gemini":
gemini_models.append(key)
gemini_models.add(key)
elif value.get("litellm_provider") == "fireworks_ai":
# ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params.
if "-to-" not in key and "fireworks-ai-default" not in key:
fireworks_ai_models.append(key)
fireworks_ai_models.add(key)
elif value.get("litellm_provider") == "fireworks_ai-embedding-models":
# ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params.
if "-to-" not in key:
fireworks_ai_embedding_models.append(key)
fireworks_ai_embedding_models.add(key)
elif value.get("litellm_provider") == "text-completion-codestral":
text_completion_codestral_models.append(key)
text_completion_codestral_models.add(key)
elif value.get("litellm_provider") == "xai":
xai_models.append(key)
xai_models.add(key)
elif value.get("litellm_provider") == "deepseek":
deepseek_models.append(key)
deepseek_models.add(key)
elif value.get("litellm_provider") == "meta_llama":
llama_models.append(key)
llama_models.add(key)
elif value.get("litellm_provider") == "nscale":
nscale_models.append(key)
nscale_models.add(key)
elif value.get("litellm_provider") == "azure_ai":
azure_ai_models.append(key)
azure_ai_models.add(key)
elif value.get("litellm_provider") == "voyage":
voyage_models.append(key)
voyage_models.add(key)
elif value.get("litellm_provider") == "infinity":
infinity_models.append(key)
infinity_models.add(key)
elif value.get("litellm_provider") == "databricks":
databricks_models.append(key)
databricks_models.add(key)
elif value.get("litellm_provider") == "cloudflare":
cloudflare_models.append(key)
cloudflare_models.add(key)
elif value.get("litellm_provider") == "codestral":
codestral_models.append(key)
codestral_models.add(key)
elif value.get("litellm_provider") == "friendliai":
friendliai_models.append(key)
friendliai_models.add(key)
elif value.get("litellm_provider") == "palm":
palm_models.append(key)
palm_models.add(key)
elif value.get("litellm_provider") == "groq":
groq_models.append(key)
groq_models.add(key)
elif value.get("litellm_provider") == "azure":
azure_models.append(key)
azure_models.add(key)
elif value.get("litellm_provider") == "anyscale":
anyscale_models.append(key)
anyscale_models.add(key)
elif value.get("litellm_provider") == "cerebras":
cerebras_models.append(key)
cerebras_models.add(key)
elif value.get("litellm_provider") == "galadriel":
galadriel_models.append(key)
galadriel_models.add(key)
elif value.get("litellm_provider") == "sambanova":
sambanova_models.append(key)
sambanova_models.add(key)
elif value.get("litellm_provider") == "sambanova-embedding-models":
sambanova_embedding_models.add(key)
elif value.get("litellm_provider") == "novita":
novita_models.append(key)
novita_models.add(key)
elif value.get("litellm_provider") == "nebius-chat-models":
nebius_models.append(key)
nebius_models.add(key)
elif value.get("litellm_provider") == "nebius-embedding-models":
nebius_embedding_models.append(key)
nebius_embedding_models.add(key)
elif value.get("litellm_provider") == "aiml":
aiml_models.add(key)
elif value.get("litellm_provider") == "assemblyai":
assemblyai_models.append(key)
assemblyai_models.add(key)
elif value.get("litellm_provider") == "jina_ai":
jina_ai_models.append(key)
jina_ai_models.add(key)
elif value.get("litellm_provider") == "snowflake":
snowflake_models.append(key)
snowflake_models.add(key)
elif value.get("litellm_provider") == "gradient_ai":
gradient_ai_models.add(key)
elif value.get("litellm_provider") == "featherless_ai":
featherless_ai_models.append(key)
featherless_ai_models.add(key)
elif value.get("litellm_provider") == "deepgram":
deepgram_models.append(key)
deepgram_models.add(key)
elif value.get("litellm_provider") == "elevenlabs":
elevenlabs_models.append(key)
elevenlabs_models.add(key)
elif value.get("litellm_provider") == "dashscope":
dashscope_models.append(key)
dashscope_models.add(key)
elif value.get("litellm_provider") == "moonshot":
moonshot_models.append(key)
moonshot_models.add(key)
elif value.get("litellm_provider") == "v0":
v0_models.append(key)
v0_models.add(key)
elif value.get("litellm_provider") == "morph":
morph_models.append(key)
morph_models.add(key)
elif value.get("litellm_provider") == "lambda_ai":
lambda_ai_models.append(key)
lambda_ai_models.add(key)
elif value.get("litellm_provider") == "hyperbolic":
hyperbolic_models.append(key)
hyperbolic_models.add(key)
elif value.get("litellm_provider") == "recraft":
recraft_models.append(key)
recraft_models.add(key)
elif value.get("litellm_provider") == "cometapi":
cometapi_models.add(key)
elif value.get("litellm_provider") == "oci":
oci_models.append(key)
oci_models.add(key)
add_known_models()
@ -754,66 +773,68 @@ ollama_models = ["llama2"]
maritalk_models = ["maritalk"]
model_list = (
model_list = list(
open_ai_chat_completion_models
+ open_ai_text_completion_models
+ cohere_models
+ cohere_chat_models
+ anthropic_models
+ replicate_models
+ openrouter_models
+ datarobot_models
+ huggingface_models
+ vertex_chat_models
+ vertex_text_models
+ ai21_models
+ ai21_chat_models
+ together_ai_models
+ baseten_models
+ aleph_alpha_models
+ nlp_cloud_models
+ ollama_models
+ bedrock_models
+ deepinfra_models
+ perplexity_models
+ maritalk_models
+ vertex_language_models
+ watsonx_models
+ gemini_models
+ text_completion_codestral_models
+ xai_models
+ deepseek_models
+ azure_ai_models
+ voyage_models
+ infinity_models
+ databricks_models
+ cloudflare_models
+ codestral_models
+ friendliai_models
+ palm_models
+ groq_models
+ azure_models
+ anyscale_models
+ cerebras_models
+ galadriel_models
+ sambanova_models
+ azure_text_models
+ novita_models
+ assemblyai_models
+ jina_ai_models
+ snowflake_models
+ llama_models
+ featherless_ai_models
+ nscale_models
+ deepgram_models
+ elevenlabs_models
+ dashscope_models
+ moonshot_models
+ v0_models
+ morph_models
+ lambda_ai_models
+ recraft_models
+ oci_models
| open_ai_text_completion_models
| cohere_models
| cohere_chat_models
| anthropic_models
| set(replicate_models)
| openrouter_models
| datarobot_models
| set(huggingface_models)
| vertex_chat_models
| vertex_text_models
| ai21_models
| ai21_chat_models
| set(together_ai_models)
| set(baseten_models)
| aleph_alpha_models
| nlp_cloud_models
| set(ollama_models)
| bedrock_models
| deepinfra_models
| perplexity_models
| set(maritalk_models)
| vertex_language_models
| watsonx_models
| gemini_models
| text_completion_codestral_models
| xai_models
| deepseek_models
| azure_ai_models
| voyage_models
| infinity_models
| databricks_models
| cloudflare_models
| codestral_models
| friendliai_models
| palm_models
| groq_models
| azure_models
| anyscale_models
| cerebras_models
| galadriel_models
| sambanova_models
| azure_text_models
| novita_models
| assemblyai_models
| jina_ai_models
| snowflake_models
| gradient_ai_models
| llama_models
| featherless_ai_models
| nscale_models
| deepgram_models
| elevenlabs_models
| dashscope_models
| moonshot_models
| v0_models
| morph_models
| lambda_ai_models
| recraft_models
| cometapi_models
| oci_models
)
model_list_set = set(model_list)
@ -822,9 +843,9 @@ provider_list: List[Union[LlmProviders, str]] = list(LlmProviders)
models_by_provider: dict = {
"openai": open_ai_chat_completion_models + open_ai_text_completion_models,
"openai": open_ai_chat_completion_models | open_ai_text_completion_models,
"text-completion-openai": open_ai_text_completion_models,
"cohere": cohere_models + cohere_chat_models,
"cohere": cohere_models | cohere_chat_models,
"cohere_chat": cohere_chat_models,
"anthropic": anthropic_models,
"replicate": replicate_models,
@ -833,13 +854,9 @@ models_by_provider: dict = {
"baseten": baseten_models,
"openrouter": openrouter_models,
"datarobot": datarobot_models,
"vertex_ai": vertex_chat_models
+ vertex_text_models
+ vertex_anthropic_models
+ vertex_vision_models
+ vertex_language_models,
"vertex_ai": vertex_chat_models | vertex_text_models | vertex_anthropic_models | vertex_vision_models | vertex_language_models | vertex_deepseek_models,
"ai21": ai21_models,
"bedrock": bedrock_models + bedrock_converse_models,
"bedrock": bedrock_models | bedrock_converse_models,
"petals": petals_models,
"ollama": ollama_models,
"ollama_chat": ollama_models,
@ -848,7 +865,7 @@ models_by_provider: dict = {
"maritalk": maritalk_models,
"watsonx": watsonx_models,
"gemini": gemini_models,
"fireworks_ai": fireworks_ai_models + fireworks_ai_embedding_models,
"fireworks_ai": fireworks_ai_models | fireworks_ai_embedding_models,
"aleph_alpha": aleph_alpha_models,
"text-completion-codestral": text_completion_codestral_models,
"xai": xai_models,
@ -864,17 +881,19 @@ models_by_provider: dict = {
"friendliai": friendliai_models,
"palm": palm_models,
"groq": groq_models,
"azure": azure_models + azure_text_models,
"azure": azure_models | azure_text_models,
"azure_text": azure_text_models,
"anyscale": anyscale_models,
"cerebras": cerebras_models,
"galadriel": galadriel_models,
"sambanova": sambanova_models,
"sambanova": sambanova_models | sambanova_embedding_models,
"novita": novita_models,
"nebius": nebius_models + nebius_embedding_models,
"nebius": nebius_models | nebius_embedding_models,
"aiml": aiml_models,
"assemblyai": assemblyai_models,
"jina_ai": jina_ai_models,
"snowflake": snowflake_models,
"gradient_ai": gradient_ai_models,
"meta_llama": llama_models,
"nscale": nscale_models,
"featherless_ai": featherless_ai_models,
@ -887,6 +906,7 @@ models_by_provider: dict = {
"lambda_ai": lambda_ai_models,
"hyperbolic": hyperbolic_models,
"recraft": recraft_models,
"cometapi": cometapi_models,
"oci": oci_models,
}
@ -916,11 +936,12 @@ longer_context_model_fallback_dict: dict = {
all_embedding_models = (
open_ai_embedding_models
+ cohere_embedding_models
+ bedrock_embedding_models
+ vertex_embedding_models
+ fireworks_ai_embedding_models
+ nebius_embedding_models
| set(cohere_embedding_models)
| set(bedrock_embedding_models)
| vertex_embedding_models
| fireworks_ai_embedding_models
| nebius_embedding_models
| sambanova_embedding_models
)
####### IMAGE GENERATION MODELS ###################
@ -1018,6 +1039,7 @@ from .llms.cohere.rerank_v2.transformation import CohereRerankV2Config
from .llms.azure_ai.rerank.transformation import AzureAIRerankConfig
from .llms.infinity.rerank.transformation import InfinityRerankConfig
from .llms.jina_ai.rerank.transformation import JinaAIRerankConfig
from .llms.deepinfra.rerank.transformation import DeepinfraRerankConfig
from .llms.clarifai.chat.transformation import ClarifaiConfig
from .llms.ai21.chat.transformation import AI21ChatConfig, AI21ChatConfig as AI21Config
from .llms.meta_llama.chat.transformation import LlamaAPIConfig
@ -1025,7 +1047,7 @@ from .llms.anthropic.experimental_pass_through.messages.transformation import (
AnthropicMessagesConfig,
)
from .llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import (
AmazonAnthropicClaude3MessagesConfig,
AmazonAnthropicClaudeMessagesConfig,
)
from .llms.together_ai.chat import TogetherAIConfig
from .llms.together_ai.completion.transformation import TogetherAITextCompletionConfig
@ -1085,7 +1107,7 @@ from .llms.bedrock.chat.invoke_transformations.anthropic_claude2_transformation
AmazonAnthropicConfig,
)
from .llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import (
AmazonAnthropicClaude3Config,
AmazonAnthropicClaudeConfig,
)
from .llms.bedrock.chat.invoke_transformations.amazon_cohere_transformation import (
AmazonCohereConfig,
@ -1129,17 +1151,22 @@ from .llms.topaz.image_variations.transformation import TopazImageVariationConfi
from litellm.llms.openai.completion.transformation import OpenAITextCompletionConfig
from .llms.groq.chat.transformation import GroqChatConfig
from .llms.voyage.embedding.transformation import VoyageEmbeddingConfig
from .llms.voyage.embedding.transformation_contextual import VoyageContextualEmbeddingConfig
from .llms.infinity.embedding.transformation import InfinityEmbeddingConfig
from .llms.azure_ai.chat.transformation import AzureAIStudioConfig
from .llms.mistral.chat.transformation import MistralConfig
from .llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from .llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig
from .llms.azure.responses.o_series_transformation import (
AzureOpenAIOSeriesResponsesAPIConfig,
)
from .llms.openai.chat.o_series_transformation import (
OpenAIOSeriesConfig as OpenAIO1Config, # maintain backwards compatibility
OpenAIOSeriesConfig,
)
from .llms.snowflake.chat.transformation import SnowflakeConfig
from .llms.gradient_ai.chat.transformation import GradientAIConfig
openaiOSeriesConfig = OpenAIOSeriesConfig()
from .llms.openai.chat.gpt_transformation import (
@ -1171,7 +1198,9 @@ nvidiaNimEmbeddingConfig = NvidiaNimEmbeddingConfig()
from .llms.featherless_ai.chat.transformation import FeatherlessAIConfig
from .llms.cerebras.chat import CerebrasConfig
from .llms.baseten.chat import BasetenConfig
from .llms.sambanova.chat import SambanovaConfig
from .llms.sambanova.embedding.transformation import SambaNovaEmbeddingConfig
from .llms.ai21.chat.transformation import AI21ChatConfig
from .llms.fireworks_ai.chat.transformation import FireworksAIConfig
from .llms.fireworks_ai.completion.transformation import FireworksAITextCompletionConfig
@ -1185,14 +1214,16 @@ from .llms.friendliai.chat.transformation import FriendliaiChatConfig
from .llms.jina_ai.embedding.transformation import JinaAIEmbeddingConfig
from .llms.xai.chat.transformation import XAIChatConfig
from .llms.xai.common_utils import XAIModelInfo
from .llms.aiml.chat.transformation import AIMLChatConfig
from .llms.volcengine import VolcEngineConfig
from .llms.codestral.completion.transformation import CodestralTextCompletionConfig
from .llms.azure.azure import (
AzureOpenAIError,
AzureOpenAIAssistantsAPIConfig,
)
from .llms.cometapi.chat.transformation import CometAPIConfig
from .llms.azure.chat.gpt_transformation import AzureOpenAIConfig
from .llms.azure.chat.gpt_5_transformation import AzureOpenAIGPT5Config
from .llms.azure.completion.transformation import AzureOpenAITextConfig
from .llms.hosted_vllm.chat.transformation import HostedVLLMChatConfig
from .llms.llamafile.chat.transformation import LlamafileChatConfig

78
litellm/caching/caching.py
View file

@ -28,13 +28,13 @@ from .azure_blob_cache import AzureBlobCache
from .base_cache import BaseCache
from .disk_cache import DiskCache
from .dual_cache import DualCache # noqa
from .gcs_cache import GCSCache
from .in_memory_cache import InMemoryCache
from .qdrant_semantic_cache import QdrantSemanticCache
from .redis_cache import RedisCache
from .redis_cluster_cache import RedisClusterCache
from .redis_semantic_cache import RedisSemanticCache
from .s3_cache import S3Cache
from .gcs_cache import GCSCache
def print_verbose(print_statement):
@ -177,7 +177,7 @@ class Cache:
cluster_kwargs["gcp_service_account"] = gcp_service_account
if gcp_ssl_ca_certs is not None:
cluster_kwargs["gcp_ssl_ca_certs"] = gcp_ssl_ca_certs
self.cache: BaseCache = RedisClusterCache(**cluster_kwargs)
else:
self.cache = RedisCache(
@ -481,7 +481,7 @@ class Cache:
return cached_response
return cached_result
def get_cache(self, **kwargs):
def get_cache(self, dynamic_cache_object: Optional[BaseCache] = None, **kwargs):
"""
Retrieves the cached result for the given arguments.
@ -507,8 +507,12 @@ class Cache:
or cache_control_args.get("s-max-age")
or float("inf")
)
cached_result = self.cache.get_cache(cache_key, messages=messages)
cached_result = self.cache.get_cache(cache_key, messages=messages)
if dynamic_cache_object is not None:
cached_result = dynamic_cache_object.get_cache(
cache_key, messages=messages
)
else:
cached_result = self.cache.get_cache(cache_key, messages=messages)
return self._get_cache_logic(
cached_result=cached_result, max_age=max_age
)
@ -516,7 +520,9 @@ class Cache:
print_verbose(f"An exception occurred: {traceback.format_exc()}")
return None
async def async_get_cache(self, **kwargs):
async def async_get_cache(
self, dynamic_cache_object: Optional[BaseCache] = None, **kwargs
):
"""
Async get cache implementation.
@ -537,7 +543,14 @@ class Cache:
max_age = cache_control_args.get(
"s-max-age", cache_control_args.get("s-maxage", float("inf"))
)
cached_result = await self.cache.async_get_cache(cache_key, **kwargs)
if dynamic_cache_object is not None:
cached_result = await dynamic_cache_object.async_get_cache(
cache_key, **kwargs
)
else:
cached_result = await self.cache.async_get_cache(
cache_key, **kwargs
)
return self._get_cache_logic(
cached_result=cached_result, max_age=max_age
)
@ -596,7 +609,9 @@ class Cache:
except Exception as e:
verbose_logger.exception(f"LiteLLM Cache: Excepton add_cache: {str(e)}")
async def async_add_cache(self, result, **kwargs):
async def async_add_cache(
self, result, dynamic_cache_object: Optional[BaseCache] = None, **kwargs
):
"""
Async implementation of add_cache
"""
@ -610,12 +625,18 @@ class Cache:
cache_key, cached_data, kwargs = self._add_cache_logic(
result=result, **kwargs
)
await self.cache.async_set_cache(cache_key, cached_data, **kwargs)
if dynamic_cache_object is not None:
await dynamic_cache_object.async_set_cache(
cache_key, cached_data, **kwargs
)
else:
await self.cache.async_set_cache(cache_key, cached_data, **kwargs)
except Exception as e:
verbose_logger.exception(f"LiteLLM Cache: Excepton add_cache: {str(e)}")
def _convert_to_cached_embedding(self, embedding_response: Any, model: Optional[str]) -> CachedEmbedding:
def _convert_to_cached_embedding(
self, embedding_response: Any, model: Optional[str]
) -> CachedEmbedding:
"""
Convert any embedding response into the standardized CachedEmbedding TypedDict format.
"""
@ -627,7 +648,7 @@ class Cache:
"object": embedding_response.get("object"),
"model": model,
}
elif hasattr(embedding_response, 'model_dump'):
elif hasattr(embedding_response, "model_dump"):
data = embedding_response.model_dump()
return {
"embedding": data.get("embedding"),
@ -646,7 +667,6 @@ class Cache:
except KeyError as e:
raise ValueError(f"Missing expected key in embedding response: {e}")
def add_embedding_response_to_cache(
self,
result: EmbeddingResponse,
@ -657,18 +677,22 @@ class Cache:
preset_cache_key = self.get_cache_key(**{**kwargs, "input": input})
kwargs["cache_key"] = preset_cache_key
embedding_response = result.data[idx_in_result_data]
# Always convert to properly typed CachedEmbedding
model_name = result.model
embedding_dict: CachedEmbedding = self._convert_to_cached_embedding(embedding_response, model_name)
embedding_dict: CachedEmbedding = self._convert_to_cached_embedding(
embedding_response, model_name
)
cache_key, cached_data, kwargs = self._add_cache_logic(
result=embedding_dict,
**kwargs,
)
return cache_key, cached_data, kwargs
async def async_add_cache_pipeline(self, result, **kwargs):
async def async_add_cache_pipeline(
self, result, dynamic_cache_object: Optional[BaseCache] = None, **kwargs
):
"""
Async implementation of add_cache for Embedding calls
@ -697,14 +721,14 @@ class Cache:
)
cache_list.append((cache_key, cached_data))
await self.cache.async_set_cache_pipeline(cache_list=cache_list, **kwargs)
# if async_set_cache_pipeline:
# await async_set_cache_pipeline(cache_list=cache_list, **kwargs)
# else:
# tasks = []
# for val in cache_list:
# tasks.append(self.cache.async_set_cache(val[0], val[1], **kwargs))
# await asyncio.gather(*tasks)
if dynamic_cache_object is not None:
await dynamic_cache_object.async_set_cache_pipeline(
cache_list=cache_list, **kwargs
)
else:
await self.cache.async_set_cache_pipeline(
cache_list=cache_list, **kwargs
)
except Exception as e:
verbose_logger.exception(f"LiteLLM Cache: Excepton add_cache: {str(e)}")
@ -750,11 +774,9 @@ class Cache:
"""
Internal method to check if the cache type supports async get/set operations
Only S3 Cache Does NOT support async operations
All cache types now support async operations
"""
if self.type and self.type == LiteLLMCacheType.S3:
return False
return True

78
litellm/caching/caching_handler.py
View file

@ -1,5 +1,5 @@
"""
This contains LLMCachingHandler
This contains LLMCachingHandler
This exposes two methods:
- async_get_cache
@ -35,11 +35,12 @@ from pydantic import BaseModel
import litellm
from litellm._logging import print_verbose, verbose_logger
from litellm.caching import InMemoryCache
from litellm.caching.caching import S3Cache
from litellm.types.caching import CachedEmbedding
from litellm.litellm_core_utils.logging_utils import (
_assemble_complete_response_from_streaming_chunks,
)
from litellm.types.caching import CachedEmbedding
from litellm.types.rerank import RerankResponse
from litellm.types.utils import (
CallTypes,
@ -68,7 +69,12 @@ class CachingHandlerResponse(BaseModel):
cached_result: Optional[Any] = None
final_embedding_cached_response: Optional[EmbeddingResponse] = None
embedding_all_elements_cache_hit: bool = False # this is set to True when all elements in the list have a cache hit in the embedding cache, if true return the final_embedding_cached_response no need to make an API call
embedding_all_elements_cache_hit: bool = (
False # this is set to True when all elements in the list have a cache hit in the embedding cache, if true return the final_embedding_cached_response no need to make an API call
)
in_memory_cache_obj = InMemoryCache()
class LLMCachingHandler:
@ -78,11 +84,20 @@ class LLMCachingHandler:
request_kwargs: Dict[str, Any],
start_time: datetime.datetime,
):
from litellm.caching import DualCache, RedisCache
self.async_streaming_chunks: List[ModelResponse] = []
self.sync_streaming_chunks: List[ModelResponse] = []
self.request_kwargs = request_kwargs
self.original_function = original_function
self.start_time = start_time
if litellm.cache is not None and isinstance(litellm.cache.cache, RedisCache):
self.dual_cache: Optional[DualCache] = DualCache(
redis_cache=litellm.cache.cache,
in_memory_cache=in_memory_cache_obj,
)
else:
self.dual_cache = None
pass
async def _async_get_cache(
@ -115,10 +130,16 @@ class LLMCachingHandler:
Raises:
None
"""
from litellm.litellm_core_utils.core_helpers import (
_get_parent_otel_span_from_kwargs,
)
from litellm.utils import CustomStreamWrapper
kwargs = kwargs.copy()
args = args or ()
parent_otel_span = _get_parent_otel_span_from_kwargs(kwargs)
kwargs["parent_otel_span"] = parent_otel_span
final_embedding_cached_response: Optional[EmbeddingResponse] = None
embedding_all_elements_cache_hit: bool = False
cached_result: Optional[Any] = None
@ -306,13 +327,15 @@ class LLMCachingHandler:
else:
raise ValueError("input must be a string or a list")
def _extract_model_from_cached_results(self, non_null_list: List[Tuple[int, CachedEmbedding]]) -> Optional[str]:
def _extract_model_from_cached_results(
self, non_null_list: List[Tuple[int, CachedEmbedding]]
) -> Optional[str]:
"""
Helper method to extract the model name from cached results.
Args:
non_null_list: List of (idx, cr) tuples where cr is the cached result dict
Returns:
Optional[str]: The model name if found, None otherwise
"""
@ -558,7 +581,12 @@ class LLMCachingHandler:
preset_cache_key = litellm.cache.get_cache_key(
**{**new_kwargs, "input": i}
)
tasks.append(litellm.cache.async_get_cache(cache_key=preset_cache_key))
tasks.append(
litellm.cache.async_get_cache(
cache_key=preset_cache_key,
dynamic_cache_object=self.dual_cache,
)
)
cached_result = await asyncio.gather(*tasks)
## check if cached result is None ##
if cached_result is not None and isinstance(cached_result, list):
@ -567,9 +595,14 @@ class LLMCachingHandler:
cached_result = None
else:
if litellm.cache._supports_async() is True:
cached_result = await litellm.cache.async_get_cache(**new_kwargs)
else: # for s3 caching. [NOT RECOMMENDED IN PROD - this will slow down responses since boto3 is sync]
cached_result = litellm.cache.get_cache(**new_kwargs)
## check if dual cache is supported ##
cached_result = await litellm.cache.async_get_cache(
dynamic_cache_object=self.dual_cache, **new_kwargs
)
else: # fallback for caches that don't support async
cached_result = litellm.cache.get_cache(
dynamic_cache_object=self.dual_cache, **new_kwargs
)
return cached_result
def _convert_cached_result_to_model_response(
@ -735,6 +768,9 @@ class LLMCachingHandler:
Raises:
None
"""
from litellm.litellm_core_utils.core_helpers import (
_get_parent_otel_span_from_kwargs,
)
if litellm.cache is None:
return
@ -746,6 +782,8 @@ class LLMCachingHandler:
args,
)
)
parent_otel_span = _get_parent_otel_span_from_kwargs(new_kwargs)
new_kwargs["parent_otel_span"] = parent_otel_span
# [OPTIONAL] ADD TO CACHE
if self._should_store_result_in_cache(
original_function=original_function, kwargs=new_kwargs
@ -764,18 +802,16 @@ class LLMCachingHandler:
) # s3 doesn't support bulk writing. Exclude.
):
asyncio.create_task(
litellm.cache.async_add_cache_pipeline(result, **new_kwargs)
litellm.cache.async_add_cache_pipeline(
result, dynamic_cache_object=self.dual_cache, **new_kwargs
)
)
elif isinstance(litellm.cache.cache, S3Cache):
threading.Thread(
target=litellm.cache.add_cache,
args=(result,),
kwargs=new_kwargs,
).start()
else:
asyncio.create_task(
litellm.cache.async_add_cache(
result.model_dump_json(), **new_kwargs
result.model_dump_json(),
dynamic_cache_object=self.dual_cache,
**new_kwargs,
)
)
else:
@ -933,9 +969,9 @@ class LLMCachingHandler:
}
if litellm.cache is not None:
litellm_params[
"preset_cache_key"
] = litellm.cache._get_preset_cache_key_from_kwargs(**kwargs)
litellm_params["preset_cache_key"] = (
litellm.cache._get_preset_cache_key_from_kwargs(**kwargs)
)
else:
litellm_params["preset_cache_key"] = None

103
litellm/caching/redis_cache.py
View file

@ -43,6 +43,45 @@ else:
Span = Any
def _get_call_stack_info(num_frames: int = 2) -> str:
"""
Get the function names from the previous 1-2 functions in the call stack.
Args:
num_frames: Number of previous frames to include (default: 2)
Returns:
A string with format "current_function <- caller_function [<- grandparent_function]"
"""
try:
current_frame = inspect.currentframe()
if current_frame is None:
return "unknown"
# Skip this function and the immediate caller (which sets call_type)
f_back = current_frame.f_back
if f_back is None:
return "unknown"
frame = f_back.f_back
if frame is None:
return "unknown"
function_names = []
for _ in range(num_frames):
if frame is None:
break
func_name = frame.f_code.co_name
function_names.append(func_name)
frame = frame.f_back
if not function_names:
return "unknown"
return " <- ".join(function_names)
except Exception:
return "unknown"
class RedisCache(BaseCache):
# if users don't provider one, use the default litellm cache
@ -181,7 +220,7 @@ class RedisCache(BaseCache):
self.service_logger_obj.service_success_hook(
service=ServiceTypes.REDIS,
duration=_duration,
call_type="set_cache",
call_type=f"set_cache <- {_get_call_stack_info()}",
start_time=start_time,
end_time=end_time,
)
@ -205,7 +244,7 @@ class RedisCache(BaseCache):
self.service_logger_obj.service_success_hook(
service=ServiceTypes.REDIS,
duration=_duration,
call_type="increment_cache",
call_type=f"increment_cache <- {_get_call_stack_info()}",
start_time=start_time,
end_time=end_time,
)
@ -219,7 +258,7 @@ class RedisCache(BaseCache):
self.service_logger_obj.service_success_hook(
service=ServiceTypes.REDIS,
duration=_duration,
call_type="increment_cache_ttl",
call_type=f"increment_cache_ttl <- {_get_call_stack_info()}",
start_time=start_time,
end_time=end_time,
)
@ -232,7 +271,7 @@ class RedisCache(BaseCache):
self.service_logger_obj.service_success_hook(
service=ServiceTypes.REDIS,
duration=_duration,
call_type="increment_cache_expire",
call_type=f"increment_cache_expire <- {_get_call_stack_info()}",
start_time=start_time,
end_time=end_time,
)
@ -271,7 +310,7 @@ class RedisCache(BaseCache):
self.service_logger_obj.async_service_success_hook(
service=ServiceTypes.REDIS,
duration=_duration,
call_type="async_scan_iter",
call_type=f"async_scan_iter <- {_get_call_stack_info()}",
start_time=start_time,
end_time=end_time,
)
@ -287,7 +326,7 @@ class RedisCache(BaseCache):
service=ServiceTypes.REDIS,
duration=_duration,
error=e,
call_type="async_scan_iter",
call_type=f"async_scan_iter <- {_get_call_stack_info()}",
start_time=start_time,
end_time=end_time,
)
@ -341,7 +380,7 @@ class RedisCache(BaseCache):
start_time=start_time,
end_time=end_time,
parent_otel_span=_get_parent_otel_span_from_kwargs(kwargs),
call_type="async_set_cache",
call_type=f"async_set_cache <- {_get_call_stack_info()}",
)
)
verbose_logger.error(
@ -374,7 +413,7 @@ class RedisCache(BaseCache):
self.service_logger_obj.async_service_success_hook(
service=ServiceTypes.REDIS,
duration=_duration,
call_type="async_set_cache",
call_type=f"async_set_cache <- {_get_call_stack_info()}",
start_time=start_time,
end_time=end_time,
parent_otel_span=_get_parent_otel_span_from_kwargs(kwargs),
@ -390,7 +429,7 @@ class RedisCache(BaseCache):
service=ServiceTypes.REDIS,
duration=_duration,
error=e,
call_type="async_set_cache",
call_type=f"async_set_cache <- {_get_call_stack_info()}",
start_time=start_time,
end_time=end_time,
parent_otel_span=_get_parent_otel_span_from_kwargs(kwargs),
@ -463,7 +502,7 @@ class RedisCache(BaseCache):
self.service_logger_obj.async_service_success_hook(
service=ServiceTypes.REDIS,
duration=_duration,
call_type="async_set_cache_pipeline",
call_type=f"async_set_cache_pipeline <- {_get_call_stack_info()}",
start_time=start_time,
end_time=end_time,
parent_otel_span=_get_parent_otel_span_from_kwargs(kwargs),
@ -479,7 +518,7 @@ class RedisCache(BaseCache):
service=ServiceTypes.REDIS,
duration=_duration,
error=e,
call_type="async_set_cache_pipeline",
call_type=f"async_set_cache_pipeline <- {_get_call_stack_info()}",
start_time=start_time,
end_time=end_time,
parent_otel_span=_get_parent_otel_span_from_kwargs(kwargs),
@ -528,7 +567,7 @@ class RedisCache(BaseCache):
start_time=start_time,
end_time=end_time,
parent_otel_span=_get_parent_otel_span_from_kwargs(kwargs),
call_type="async_set_cache_sadd",
call_type=f"async_set_cache_sadd <- {_get_call_stack_info()}",
)
)
# NON blocking - notify users Redis is throwing an exception
@ -554,7 +593,7 @@ class RedisCache(BaseCache):
self.service_logger_obj.async_service_success_hook(
service=ServiceTypes.REDIS,
duration=_duration,
call_type="async_set_cache_sadd",
call_type=f"async_set_cache_sadd <- {_get_call_stack_info()}",
start_time=start_time,
end_time=end_time,
parent_otel_span=_get_parent_otel_span_from_kwargs(kwargs),
@ -568,7 +607,7 @@ class RedisCache(BaseCache):
service=ServiceTypes.REDIS,
duration=_duration,
error=e,
call_type="async_set_cache_sadd",
call_type=f"async_set_cache_sadd <- {_get_call_stack_info()}",
start_time=start_time,
end_time=end_time,
parent_otel_span=_get_parent_otel_span_from_kwargs(kwargs),
@ -620,7 +659,7 @@ class RedisCache(BaseCache):
self.service_logger_obj.async_service_success_hook(
service=ServiceTypes.REDIS,
duration=_duration,
call_type="async_increment",
call_type=f"async_increment <- {_get_call_stack_info()}",
start_time=start_time,
end_time=end_time,
parent_otel_span=parent_otel_span,
@ -636,7 +675,7 @@ class RedisCache(BaseCache):
service=ServiceTypes.REDIS,
duration=_duration,
error=e,
call_type="async_increment",
call_type=f"async_increment <- {_get_call_stack_info()}",
start_time=start_time,
end_time=end_time,
parent_otel_span=parent_otel_span,
@ -683,7 +722,7 @@ class RedisCache(BaseCache):
self.service_logger_obj.service_success_hook(
service=ServiceTypes.REDIS,
duration=_duration,
call_type="get_cache",
call_type=f"get_cache <- {_get_call_stack_info()}",
start_time=start_time,
end_time=end_time,
parent_otel_span=parent_otel_span,
@ -745,7 +784,7 @@ class RedisCache(BaseCache):
self.service_logger_obj.service_success_hook(
service=ServiceTypes.REDIS,
duration=_duration,
call_type="batch_get_cache",
call_type=f"batch_get_cache <- {_get_call_stack_info()}",
start_time=start_time,
end_time=end_time,
parent_otel_span=parent_otel_span,
@ -790,7 +829,7 @@ class RedisCache(BaseCache):
self.service_logger_obj.async_service_success_hook(
service=ServiceTypes.REDIS,
duration=_duration,
call_type="async_get_cache",
call_type=f"async_get_cache <- {_get_call_stack_info()}",
start_time=start_time,
end_time=end_time,
parent_otel_span=parent_otel_span,
@ -806,7 +845,7 @@ class RedisCache(BaseCache):
service=ServiceTypes.REDIS,
duration=_duration,
error=e,
call_type="async_get_cache",
call_type=f"async_get_cache <- {_get_call_stack_info()}",
start_time=start_time,
end_time=end_time,
parent_otel_span=parent_otel_span,
@ -851,7 +890,7 @@ class RedisCache(BaseCache):
self.service_logger_obj.async_service_success_hook(
service=ServiceTypes.REDIS,
duration=_duration,
call_type="async_batch_get_cache",
call_type=f"async_batch_get_cache <- {_get_call_stack_info()}",
start_time=start_time,
end_time=end_time,
parent_otel_span=parent_otel_span,
@ -879,7 +918,7 @@ class RedisCache(BaseCache):
service=ServiceTypes.REDIS,
duration=_duration,
error=e,
call_type="async_batch_get_cache",
call_type=f"async_batch_get_cache <- {_get_call_stack_info()}",
start_time=start_time,
end_time=end_time,
parent_otel_span=parent_otel_span,
@ -903,7 +942,7 @@ class RedisCache(BaseCache):
self.service_logger_obj.service_success_hook(
service=ServiceTypes.REDIS,
duration=_duration,
call_type="sync_ping",
call_type=f"sync_ping <- {_get_call_stack_info()}",
start_time=start_time,
end_time=end_time,
)
@ -917,7 +956,7 @@ class RedisCache(BaseCache):
service=ServiceTypes.REDIS,
duration=_duration,
error=e,
call_type="sync_ping",
call_type=f"sync_ping <- {_get_call_stack_info()}",
)
verbose_logger.error(
f"LiteLLM Redis Cache PING: - Got exception from REDIS : {str(e)}"
@ -938,7 +977,7 @@ class RedisCache(BaseCache):
self.service_logger_obj.async_service_success_hook(
service=ServiceTypes.REDIS,
duration=_duration,
call_type="async_ping",
call_type=f"async_ping <- {_get_call_stack_info()}",
)
)
return response
@ -952,7 +991,7 @@ class RedisCache(BaseCache):
service=ServiceTypes.REDIS,
duration=_duration,
error=e,
call_type="async_ping",
call_type=f"async_ping <- {_get_call_stack_info()}",
)
)
verbose_logger.error(
@ -1051,7 +1090,7 @@ class RedisCache(BaseCache):
self.service_logger_obj.async_service_success_hook(
service=ServiceTypes.REDIS,
duration=_duration,
call_type="async_increment_pipeline",
call_type=f"async_increment_pipeline <- {_get_call_stack_info()}",
start_time=start_time,
end_time=end_time,
parent_otel_span=_get_parent_otel_span_from_kwargs(kwargs),
@ -1067,7 +1106,7 @@ class RedisCache(BaseCache):
service=ServiceTypes.REDIS,
duration=_duration,
error=e,
call_type="async_increment_pipeline",
call_type=f"async_increment_pipeline <- {_get_call_stack_info()}",
start_time=start_time,
end_time=end_time,
parent_otel_span=_get_parent_otel_span_from_kwargs(kwargs),
@ -1131,7 +1170,7 @@ class RedisCache(BaseCache):
self.service_logger_obj.async_service_success_hook(
service=ServiceTypes.REDIS,
duration=_duration,
call_type="async_rpush",
call_type=f"async_rpush <- {_get_call_stack_info()}",
)
)
return response
@ -1145,7 +1184,7 @@ class RedisCache(BaseCache):
service=ServiceTypes.REDIS,
duration=_duration,
error=e,
call_type="async_rpush",
call_type=f"async_rpush <- {_get_call_stack_info()}",
)
)
verbose_logger.error(
@ -1202,7 +1241,7 @@ class RedisCache(BaseCache):
self.service_logger_obj.async_service_success_hook(
service=ServiceTypes.REDIS,
duration=_duration,
call_type="async_lpop",
call_type=f"async_lpop <- {_get_call_stack_info()}",
)
)
@ -1230,7 +1269,7 @@ class RedisCache(BaseCache):
service=ServiceTypes.REDIS,
duration=_duration,
error=e,
call_type="async_lpop",
call_type=f"async_lpop <- {_get_call_stack_info()}",
)
)
verbose_logger.error(

46
litellm/caching/s3_cache.py
View file

@ -1,17 +1,17 @@
"""
S3 Cache implementation
WARNING: DO NOT USE THIS IN PRODUCTION - This is not ASYNC
Has 4 methods:
- set_cache
- get_cache
- async_set_cache
- async_get_cache
- async_set_cache (uses run_in_executor)
- async_get_cache (uses run_in_executor)
"""
import ast
import asyncio
import json
from functools import partial
from typing import Optional
from litellm._logging import print_verbose, verbose_logger
@ -55,20 +55,24 @@ class S3Cache(BaseCache):
**kwargs,
)
def _to_s3_key(self, key: str) -> str:
"""Convert cache key to S3 key"""
return self.key_prefix + key.replace(":", "/")
def set_cache(self, key, value, **kwargs):
try:
print_verbose(f"LiteLLM SET Cache - S3. Key={key}. Value={value}")
ttl = kwargs.get("ttl", None)
# Convert value to JSON before storing in S3
serialized_value = json.dumps(value)
key = self.key_prefix + key
key = self._to_s3_key(key)
if ttl is not None:
cache_control = f"immutable, max-age={ttl}, s-maxage={ttl}"
import datetime
# Calculate expiration time
expiration_time = datetime.datetime.now() + ttl
expiration_time = datetime.datetime.now() + datetime.timedelta(seconds=ttl)
# Upload the data to S3 with the calculated expiration time
self.s3_client.put_object(
@ -94,17 +98,26 @@ class S3Cache(BaseCache):
ContentDisposition=f'inline; filename="{key}.json"',
)
except Exception as e:
# NON blocking - notify users S3 is throwing an exception
print_verbose(f"S3 Caching: set_cache() - Got exception from S3: {e}")
async def async_set_cache(self, key, value, **kwargs):
self.set_cache(key=key, value=value, **kwargs)
"""
Asynchronously set cache using run_in_executor to avoid blocking the event loop.
Compatible with Python 3.8+.
"""
try:
verbose_logger.debug(f"Set ASYNC S3 Cache: Key={key}. Value={value}")
loop = asyncio.get_event_loop()
func = partial(self.set_cache, key, value, **kwargs)
await loop.run_in_executor(None, func)
except Exception as e:
verbose_logger.error(f"S3 Caching: async_set_cache() - Got exception from S3: {e}")
def get_cache(self, key, **kwargs):
import botocore
try:
key = self.key_prefix + key
key = self._to_s3_key(key)
print_verbose(f"Get S3 Cache: key: {key}")
# Download the data from S3
@ -138,13 +151,26 @@ class S3Cache(BaseCache):
return None
except Exception as e:
# NON blocking - notify users S3 is throwing an exception
verbose_logger.error(
f"S3 Caching: get_cache() - Got exception from S3: {e}"
)
async def async_get_cache(self, key, **kwargs):
return self.get_cache(key=key, **kwargs)
"""
Asynchronously get cache using run_in_executor to avoid blocking the event loop.
Compatible with Python 3.8+.
"""
try:
verbose_logger.debug(f"Get ASYNC S3 Cache: key: {key}")
loop = asyncio.get_event_loop()
func = partial(self.get_cache, key, **kwargs)
result = await loop.run_in_executor(None, func)
return result
except Exception as e:
verbose_logger.error(
f"S3 Caching: async_get_cache() - Got exception from S3: {e}"
)
return None
def flush_cache(self):
pass

2
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -460,6 +460,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
return Reasoning(effort="medium", summary="auto")
elif reasoning_effort == "low":
return Reasoning(effort="low", summary="auto")
elif reasoning_effort == "minimal":
return Reasoning(effort="minimal", summary="auto")
return None
def _map_responses_status_to_finish_reason(self, status: Optional[str]) -> str:

60
litellm/constants.py
View file

@ -1,6 +1,9 @@
import os
from typing import List, Literal
AZURE_DEFAULT_RESPONSES_API_VERSION = str(
os.getenv("AZURE_DEFAULT_RESPONSES_API_VERSION", "preview")
)
ROUTER_MAX_FALLBACKS = int(os.getenv("ROUTER_MAX_FALLBACKS", 5))
DEFAULT_BATCH_SIZE = int(os.getenv("DEFAULT_BATCH_SIZE", 512))
DEFAULT_FLUSH_INTERVAL_SECONDS = int(os.getenv("DEFAULT_FLUSH_INTERVAL_SECONDS", 5))
@ -224,6 +227,7 @@ LITELLM_CHAT_PROVIDERS = [
"together_ai",
"datarobot",
"openrouter",
"cometapi",
"vertex_ai",
"vertex_ai_beta",
"gemini",
@ -247,6 +251,7 @@ LITELLM_CHAT_PROVIDERS = [
"groq",
"nvidia_nim",
"cerebras",
"baseten",
"ai21_chat",
"volcengine",
"codestral",
@ -270,6 +275,7 @@ LITELLM_CHAT_PROVIDERS = [
"llamafile",
"lm_studio",
"galadriel",
"gradient_ai",
"github_copilot", # GitHub Copilot Chat API
"novita",
"meta_llama",
@ -422,6 +428,7 @@ openai_compatible_providers: List = [
"groq",
"nvidia_nim",
"cerebras",
"baseten",
"sambanova",
"ai21_chat",
"ai21",
@ -455,6 +462,7 @@ openai_compatible_providers: List = [
"morph",
"lambda_ai",
"hyperbolic",
"aiml",
]
openai_text_completion_compatible_providers: List = (
[ # providers that support `/v1/completions`
@ -478,7 +486,7 @@ _openai_like_providers: List = [
"watsonx",
] # private helper. similar to openai but require some custom auth / endpoint handling, so can't use the openai sdk
# well supported replicate llms
replicate_models: List = [
replicate_models: set = set([
# llama replicate supported LLMs
"replicate/llama-2-70b-chat:2796ee9483c3fd7aa2e171d38f4ca12251a30609463dcfd4cd76703f22e96cdf",
"a16z-infra/llama-2-13b-chat:2a7f981751ec7fdf87b5b91ad4db53683a98082e9ff7bfd12c8cd5ea85980a52",
@ -491,9 +499,9 @@ replicate_models: List = [
# Others
"replicate/dolly-v2-12b:ef0e1aefc61f8e096ebe4db6b2bacc297daf2ef6899f0f7e001ec445893500e5",
"replit/replit-code-v1-3b:b84f4c074b807211cd75e3e8b1589b6399052125b4c27106e43d47189e8415ad",
]
])
clarifai_models: List = [
clarifai_models: set = set([
"clarifai/meta.Llama-3.Llama-3-8B-Instruct",
"clarifai/gcp.generate.gemma-1_1-7b-it",
"clarifai/mistralai.completion.mixtral-8x22B",
@ -557,10 +565,10 @@ clarifai_models: List = [
"clarifai/gcp.generate.gemini-1_5-pro",
"clarifai/gcp.generate.imagen-2",
"clarifai/salesforce.blip.general-english-image-caption-blip-2",
]
])
huggingface_models: List = [
huggingface_models: set = set([
"meta-llama/Llama-2-7b-hf",
"meta-llama/Llama-2-7b-chat-hf",
"meta-llama/Llama-2-13b-hf",
@ -573,13 +581,13 @@ huggingface_models: List = [
"meta-llama/Llama-2-13b-chat",
"meta-llama/Llama-2-70b",
"meta-llama/Llama-2-70b-chat",
] # these have been tested on extensively. But by default all text2text-generation and text-generation models are supported by liteLLM. - https://docs.litellm.ai/docs/providers
empower_models = [
]) # these have been tested on extensively. But by default all text2text-generation and text-generation models are supported by liteLLM. - https://docs.litellm.ai/docs/providers
empower_models = set([
"empower/empower-functions",
"empower/empower-functions-small",
]
])
together_ai_models: List = [
together_ai_models: set = set([
# llama llms - chat
"togethercomputer/llama-2-70b-chat",
# llama llms - language / instruct
@ -607,16 +615,17 @@ together_ai_models: List = [
"Austism/chronos-hermes-13b",
"upstage/SOLAR-0-70b-16bit",
"WizardLM/WizardLM-70B-V1.0",
] # supports all together ai models, just pass in the model id e.g. completion(model="together_computer/replit_code_3b",...)
])
# supports all together ai models, just pass in the model id e.g. completion(model="together_computer/replit_code_3b",...)
baseten_models: List = [
baseten_models: set = set([
"qvv0xeq",
"q841o8w",
"31dxrj3",
] # FALCON 7B # WizardLM # Mosaic ML
]) # FALCON 7B # WizardLM # Mosaic ML
featherless_ai_models: List = [
featherless_ai_models: set = set([
"featherless-ai/Qwerky-72B",
"featherless-ai/Qwerky-QwQ-32B",
"Qwen/Qwen2.5-72B-Instruct",
@ -626,9 +635,9 @@ featherless_ai_models: List = [
"mistralai/Mistral-Small-24B-Instruct-2501",
"mistralai/Mistral-Nemo-Instruct-2407",
"ProdeusUnity/Stellar-Odyssey-12b-v0.0",
]
])
nebius_models: List = [
nebius_models: set = set([
"Qwen/Qwen3-235B-A22B",
"Qwen/Qwen3-30B-A3B-fast",
"Qwen/Qwen3-32B",
@ -641,9 +650,9 @@ nebius_models: List = [
"meta-llama/Llama-3.3-70B-Instruct-fast",
"Qwen/Qwen2.5-32B-Instruct-fast",
"Qwen/Qwen2.5-Coder-32B-Instruct-fast",
]
])
dashscope_models: List = [
dashscope_models: set = set([
"qwen-turbo",
"qwen-plus",
"qwen-max",
@ -654,13 +663,13 @@ dashscope_models: List = [
"qwen3-235b-a22b",
"qwen3-32b",
"qwen3-30b-a3b",
]
])
nebius_embedding_models: List = [
nebius_embedding_models: set = set([
"BAAI/bge-en-icl",
"BAAI/bge-multilingual-gemma2",
"intfloat/e5-mistral-7b-instruct",
]
])
BEDROCK_INVOKE_PROVIDERS_LITERAL = Literal[
"cohere",
@ -674,8 +683,8 @@ BEDROCK_INVOKE_PROVIDERS_LITERAL = Literal[
"deepseek_r1",
]
open_ai_embedding_models: List = ["text-embedding-ada-002"]
cohere_embedding_models: List = [
open_ai_embedding_models: set = set(["text-embedding-ada-002"])
cohere_embedding_models: set = set([
"embed-v4.0",
"embed-english-v3.0",
"embed-english-light-v3.0",
@ -683,12 +692,12 @@ cohere_embedding_models: List = [
"embed-english-v2.0",
"embed-english-light-v2.0",
"embed-multilingual-v2.0",
]
bedrock_embedding_models: List = [
])
bedrock_embedding_models: set = set([
"amazon.titan-embed-text-v1",
"cohere.embed-english-v3",
"cohere.embed-multilingual-v3",
]
])
known_tokenizer_config = {
"mistralai/Mistral-7B-Instruct-v0.1": {
@ -877,6 +886,7 @@ SENTRY_DENYLIST = [
"CLOUDFLARE_API_KEY",
"BASETEN_KEY",
"OPENROUTER_KEY",
"COMETAPI_KEY",
"DATAROBOT_API_TOKEN",
"FIREWORKS_API_KEY",
"FIREWORKS_AI_API_KEY",

59
litellm/cost_calculator.py
View file

@ -32,9 +32,6 @@ from litellm.llms.azure.cost_calculation import (
from litellm.llms.bedrock.cost_calculation import (
cost_per_token as bedrock_cost_per_token,
)
from litellm.llms.bedrock.image.cost_calculator import (
cost_calculator as bedrock_image_cost_calculator,
)
from litellm.llms.databricks.cost_calculator import (
cost_per_token as databricks_cost_per_token,
)
@ -60,9 +57,6 @@ from litellm.llms.vertex_ai.cost_calculator import (
cost_per_token as google_cost_per_token,
)
from litellm.llms.vertex_ai.cost_calculator import cost_router as google_cost_router
from litellm.llms.vertex_ai.image_generation.cost_calculator import (
cost_calculator as vertex_ai_image_cost_calculator,
)
from litellm.responses.utils import ResponseAPILoggingUtils
from litellm.types.llms.openai import (
HttpxBinaryResponseContent,
@ -768,50 +762,15 @@ def completion_cost( # noqa: PLR0915
)
if CostCalculatorUtils._call_type_has_image_response(call_type):
### IMAGE GENERATION COST CALCULATION ###
if custom_llm_provider == "vertex_ai":
if isinstance(completion_response, ImageResponse):
return vertex_ai_image_cost_calculator(
model=model,
image_response=completion_response,
)
elif custom_llm_provider == "bedrock":
if isinstance(completion_response, ImageResponse):
return bedrock_image_cost_calculator(
model=model,
size=size,
image_response=completion_response,
optional_params=optional_params,
)
raise TypeError(
"completion_response must be of type ImageResponse for bedrock image cost calculation"
)
elif custom_llm_provider == litellm.LlmProviders.RECRAFT.value:
from litellm.llms.recraft.cost_calculator import (
cost_calculator as recraft_image_cost_calculator,
)
return recraft_image_cost_calculator(
model=model,
image_response=completion_response,
)
elif custom_llm_provider == litellm.LlmProviders.GEMINI.value:
from litellm.llms.gemini.image_generation.cost_calculator import (
cost_calculator as gemini_image_cost_calculator,
)
return gemini_image_cost_calculator(
model=model,
image_response=completion_response,
)
else:
return default_image_cost_calculator(
model=model,
quality=quality,
custom_llm_provider=custom_llm_provider,
n=n,
size=size,
optional_params=optional_params,
)
return CostCalculatorUtils.route_image_generation_cost_calculator(
model=model,
custom_llm_provider=custom_llm_provider,
completion_response=completion_response,
quality=quality,
n=n,
size=size,
optional_params=optional_params,
)
elif (
call_type == CallTypes.speech.value
or call_type == CallTypes.aspeech.value

79
litellm/images/main.py
View file

@ -311,7 +311,7 @@ def image_generation( # noqa: PLR0915
) or get_secret_str("AZURE_AD_TOKEN")
default_headers = {
"Content-Type": "application/json;",
"Content-Type": "application/json",
"api-key": api_key,
}
for k, v in default_headers.items():
@ -335,8 +335,63 @@ def image_generation( # noqa: PLR0915
headers=headers,
litellm_params=litellm_params_dict,
)
#########################################################
# Providers using llm_http_handler
#########################################################
elif custom_llm_provider in (
litellm.LlmProviders.RECRAFT,
litellm.LlmProviders.AIML,
litellm.LlmProviders.GEMINI,
):
if image_generation_config is None:
raise ValueError(f"image generation config is not supported for {custom_llm_provider}")
return llm_http_handler.image_generation_handler(
model=model,
prompt=prompt,
image_generation_provider_config=image_generation_config,
image_generation_optional_request_params=optional_params,
custom_llm_provider=custom_llm_provider,
litellm_params=litellm_params_dict,
logging_obj=litellm_logging_obj,
timeout=timeout,
client=client,
)
elif custom_llm_provider == "azure_ai":
from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
api_base = AzureFoundryModelInfo.get_api_base(api_base)
api_key = AzureFoundryModelInfo.get_api_key(api_key)
if extra_headers is not None:
optional_params["extra_headers"] = extra_headers
default_headers = {
"Content-Type": "application/json",
"api-key": api_key,
}
for k, v in default_headers.items():
if k not in headers:
headers[k] = v
model_response = azure_chat_completions.image_generation(
model=model,
prompt=prompt,
timeout=timeout,
api_key=api_key,
api_base=api_base,
azure_ad_token=None,
azure_ad_token_provider=azure_ad_token_provider,
logging_obj=litellm_logging_obj,
optional_params=optional_params,
model_response=model_response,
api_version=api_version,
aimg_generation=aimg_generation,
client=client,
headers=headers,
litellm_params=litellm_params_dict,
)
elif (
custom_llm_provider == "openai"
or custom_llm_provider == LlmProviders.LITELLM_PROXY.value
or custom_llm_provider in litellm.openai_compatible_providers
):
model_response = openai_chat_completions.image_generation(
@ -406,28 +461,6 @@ def image_generation( # noqa: PLR0915
api_base=api_base,
client=client,
)
#########################################################
# Providers using llm_http_handler
#########################################################
elif custom_llm_provider in (
litellm.LlmProviders.RECRAFT,
litellm.LlmProviders.GEMINI,
):
if image_generation_config is None:
raise ValueError(f"image generation config is not supported for {custom_llm_provider}")
return llm_http_handler.image_generation_handler(
model=model,
prompt=prompt,
image_generation_provider_config=image_generation_config,
image_generation_optional_request_params=optional_params,
custom_llm_provider=custom_llm_provider,
litellm_params=litellm_params_dict,
logging_obj=litellm_logging_obj,
timeout=timeout,
client=client,
)
elif (
custom_llm_provider in litellm._custom_providers
): # Assume custom LLM provider

22
litellm/integrations/braintrust_logging.py
View file

@ -19,10 +19,6 @@ from litellm.llms.custom_httpx.http_handler import (
)
from litellm.utils import print_verbose
global_braintrust_http_handler = get_async_httpx_client(
llm_provider=httpxSpecialProvider.LoggingCallback
)
global_braintrust_sync_http_handler = HTTPHandler()
API_BASE = "https://api.braintrustdata.com/v1"
@ -52,6 +48,10 @@ class BraintrustLogger(CustomLogger):
self._project_id_cache: Dict[
str, str
] = {} # Cache mapping project names to IDs
self.global_braintrust_http_handler = get_async_httpx_client(
llm_provider=httpxSpecialProvider.LoggingCallback
)
self.global_braintrust_sync_http_handler = HTTPHandler()
def validate_environment(self, api_key: Optional[str]):
"""
@ -76,7 +76,7 @@ class BraintrustLogger(CustomLogger):
return self._project_id_cache[project_name]
try:
response = global_braintrust_sync_http_handler.post(
response = self.global_braintrust_sync_http_handler.post(
f"{self.api_base}/project",
headers=self.headers,
json={"name": project_name},
@ -96,7 +96,7 @@ class BraintrustLogger(CustomLogger):
return self._project_id_cache[project_name]
try:
response = await global_braintrust_http_handler.post(
response = await self.global_braintrust_http_handler.post(
f"{self.api_base}/project/register",
headers=self.headers,
json={"name": project_name},
@ -146,7 +146,7 @@ class BraintrustLogger(CustomLogger):
return metadata
async def create_default_project_and_experiment(self):
project = await global_braintrust_http_handler.post(
project = await self.global_braintrust_http_handler.post(
f"{self.api_base}/project", headers=self.headers, json={"name": "litellm"}
)
@ -155,7 +155,7 @@ class BraintrustLogger(CustomLogger):
self.default_project_id = project_dict["id"]
def create_sync_default_project_and_experiment(self):
project = global_braintrust_sync_http_handler.post(
project = self.global_braintrust_sync_http_handler.post(
f"{self.api_base}/project", headers=self.headers, json={"name": "litellm"}
)
@ -291,9 +291,9 @@ class BraintrustLogger(CustomLogger):
try:
print_verbose(
f"global_braintrust_sync_http_handler.post: {global_braintrust_sync_http_handler.post}"
f"self.global_braintrust_sync_http_handler.post: {self.global_braintrust_sync_http_handler.post}"
)
global_braintrust_sync_http_handler.post(
self.global_braintrust_sync_http_handler.post(
url=f"{self.api_base}/project_logs/{project_id}/insert",
json={"events": [request_data]},
headers=self.headers,
@ -446,7 +446,7 @@ class BraintrustLogger(CustomLogger):
request_data["metrics"] = metrics
try:
await global_braintrust_http_handler.post(
await self.global_braintrust_http_handler.post(
url=f"{self.api_base}/project_logs/{project_id}/insert",
json={"events": [request_data]},
headers=self.headers,

101
litellm/integrations/datadog/datadog_llm_obs.py
View file

@ -27,7 +27,12 @@ from litellm.llms.custom_httpx.http_handler import (
httpxSpecialProvider,
)
from litellm.types.integrations.datadog_llm_obs import *
from litellm.types.utils import CallTypes, StandardLoggingPayload
from litellm.types.utils import (
CallTypes,
StandardLoggingGuardrailInformation,
StandardLoggingPayload,
StandardLoggingPayloadErrorInformation,
)
class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
@ -102,6 +107,24 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
verbose_logger.exception(
f"DataDogLLMObs: Error logging success event - {str(e)}"
)
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
try:
verbose_logger.debug(
f"DataDogLLMObs: Logging failure event for model {kwargs.get('model', 'unknown')}"
)
payload = self.create_llm_obs_payload(
kwargs, start_time, end_time
)
verbose_logger.debug(f"DataDogLLMObs: Payload: {payload}")
self.log_queue.append(payload)
if len(self.log_queue) >= self.batch_size:
await self.async_send_batch()
except Exception as e:
verbose_logger.exception(
f"DataDogLLMObs: Error logging failure event - {str(e)}"
)
async def async_send_batch(self):
try:
@ -174,11 +197,14 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
call_type=standard_logging_payload.get("call_type")
))
error_info = self._assemble_error_info(standard_logging_payload)
meta = Meta(
kind=self._get_datadog_span_kind(standard_logging_payload.get("call_type")),
input=input_meta,
output=output_meta,
metadata=self._get_dd_llm_obs_payload_metadata(standard_logging_payload),
error=error_info,
)
# Calculate metrics (you may need to adjust these based on available data)
@ -199,11 +225,31 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
start_ns=int(start_time.timestamp() * 1e9),
duration=int((end_time - start_time).total_seconds() * 1e9),
metrics=metrics,
status="error" if error_info else "ok",
tags=[
self._get_datadog_tags(standard_logging_object=standard_logging_payload)
],
)
def _assemble_error_info(self, standard_logging_payload: StandardLoggingPayload) -> Optional[DDLLMObsError]:
"""
Assemble error information for failure cases according to DD LLM Obs API spec
"""
# Handle error information for failure cases according to DD LLM Obs API spec
error_info: Optional[DDLLMObsError] = None
if standard_logging_payload.get("status") == "failure":
# Try to get structured error information first
error_information: Optional[StandardLoggingPayloadErrorInformation] = standard_logging_payload.get("error_information")
if error_information:
error_info = DDLLMObsError(
message=error_information.get("error_message") or standard_logging_payload.get("error_str") or "Unknown error",
type=error_information.get("error_class"),
stack=error_information.get("traceback")
)
return error_info
def _get_time_to_first_token_seconds(self, standard_logging_payload: StandardLoggingPayload) -> float:
"""
Get the time to first token in seconds
@ -232,8 +278,20 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
for now this handles logging /chat/completions responses
"""
if response_obj is None:
return []
if call_type in [CallTypes.completion.value, CallTypes.acompletion.value]:
return [response_obj["choices"][0]["message"]]
try:
# Safely extract message from response_obj, handle failure cases
if isinstance(response_obj, dict) and "choices" in response_obj:
choices = response_obj["choices"]
if choices and len(choices) > 0 and "message" in choices[0]:
return [choices[0]["message"]]
return []
except (KeyError, IndexError, TypeError):
# In case of any error accessing the response structure, return empty list
return []
return []
def _get_datadog_span_kind(self, call_type: Optional[str]) -> Literal["llm", "tool", "task", "embedding", "retrieval"]:
@ -350,11 +408,11 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
def _get_dd_llm_obs_payload_metadata(
self, standard_logging_payload: StandardLoggingPayload
) -> Dict:
) -> Dict[str, Any]:
"""
Fields to track in DD LLM Observability metadata from litellm standard logging payload
"""
_metadata = {
_metadata: Dict[str, Any] = {
"model_name": standard_logging_payload.get("model", "unknown"),
"model_provider": standard_logging_payload.get(
"custom_llm_provider", "unknown"
@ -364,9 +422,44 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
"cache_hit": standard_logging_payload.get("cache_hit", "unknown"),
"cache_key": standard_logging_payload.get("cache_key", "unknown"),
"saved_cache_cost": standard_logging_payload.get("saved_cache_cost", 0),
"guardrail_information": standard_logging_payload.get("guardrail_information", None),
}
#########################################################
# Add latency metrics to metadata
#########################################################
latency_metrics = self._get_latency_metrics(standard_logging_payload)
_metadata.update({"latency_metrics": dict(latency_metrics)})
_standard_logging_metadata: dict = (
dict(standard_logging_payload.get("metadata", {})) or {}
)
_metadata.update(_standard_logging_metadata)
return _metadata
def _get_latency_metrics(self, standard_logging_payload: StandardLoggingPayload) -> DDLLMObsLatencyMetrics:
"""
Get the latency metrics from the standard logging payload
"""
latency_metrics: DDLLMObsLatencyMetrics = DDLLMObsLatencyMetrics()
# Add latency metrics to metadata
# Time to first token (convert from seconds to milliseconds for consistency)
time_to_first_token_seconds = self._get_time_to_first_token_seconds(standard_logging_payload)
if time_to_first_token_seconds > 0:
latency_metrics["time_to_first_token_ms"] = time_to_first_token_seconds * 1000
# LiteLLM overhead time
hidden_params = standard_logging_payload.get("hidden_params", {})
litellm_overhead_ms = hidden_params.get("litellm_overhead_time_ms")
if litellm_overhead_ms is not None:
latency_metrics["litellm_overhead_time_ms"] = litellm_overhead_ms
# Guardrail overhead latency
guardrail_info: Optional[StandardLoggingGuardrailInformation] = standard_logging_payload.get("guardrail_information")
if guardrail_info is not None:
_guardrail_duration_seconds: Optional[float] = guardrail_info.get("duration")
if _guardrail_duration_seconds is not None:
# Convert from seconds to milliseconds for consistency
latency_metrics["guardrail_overhead_time_ms"] = _guardrail_duration_seconds * 1000
return latency_metrics

57
litellm/integrations/langfuse/langfuse_otel.py
View file

@ -1,15 +1,16 @@
import base64
import os
import json # <--- NEW
from typing import TYPE_CHECKING, Any, Union
from urllib.parse import quote
import os
from typing import TYPE_CHECKING, Any, Optional, Union
from litellm._logging import verbose_logger
from litellm.integrations.arize import _utils
from litellm.integrations.opentelemetry import OpenTelemetry
from litellm.types.integrations.langfuse_otel import (
LangfuseOtelConfig,
LangfuseSpanAttributes,
)
from litellm.types.utils import StandardCallbackDynamicParams
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
@ -33,7 +34,11 @@ LANGFUSE_CLOUD_US_ENDPOINT = "https://us.cloud.langfuse.com/api/public/otel"
class LangfuseOtelLogger:
class LangfuseOtelLogger(OpenTelemetry):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
@staticmethod
def set_langfuse_otel_attributes(span: Span, kwargs, response_obj):
"""
@ -174,11 +179,11 @@ class LangfuseOtelLogger:
endpoint = LANGFUSE_CLOUD_US_ENDPOINT
verbose_logger.debug(f"Using Langfuse US cloud endpoint: {endpoint}")
# Create Basic Auth header
auth_string = f"{public_key}:{secret_key}"
auth_header = base64.b64encode(auth_string.encode()).decode()
# URL encode the entire header value as required by OpenTelemetry specification
otlp_auth_headers = f"Authorization={quote(f'Basic {auth_header}')}"
auth_header = LangfuseOtelLogger._get_langfuse_authorization_header(
public_key=public_key,
secret_key=secret_key
)
otlp_auth_headers = f"Authorization={auth_header}"
# Set standard OTEL environment variables
os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = endpoint
@ -187,3 +192,37 @@ class LangfuseOtelLogger:
return LangfuseOtelConfig(
otlp_auth_headers=otlp_auth_headers, protocol="otlp_http"
)
@staticmethod
def _get_langfuse_authorization_header(public_key: str, secret_key: str) -> str:
"""
Get the authorization header for Langfuse OpenTelemetry.
"""
auth_string = f"{public_key}:{secret_key}"
auth_header = base64.b64encode(auth_string.encode()).decode()
return f'Basic {auth_header}'
def construct_dynamic_otel_headers(
self,
standard_callback_dynamic_params: StandardCallbackDynamicParams
) -> Optional[dict]:
"""
Construct dynamic Langfuse headers from standard callback dynamic params
This is used for team/key based logging.
Returns:
dict: A dictionary of dynamic Langfuse headers
"""
dynamic_headers = {}
dynamic_langfuse_public_key = standard_callback_dynamic_params.get("langfuse_public_key")
dynamic_langfuse_secret_key = standard_callback_dynamic_params.get("langfuse_secret_key")
if dynamic_langfuse_public_key and dynamic_langfuse_secret_key:
auth_header = LangfuseOtelLogger._get_langfuse_authorization_header(
public_key=dynamic_langfuse_public_key,
secret_key=dynamic_langfuse_secret_key
)
dynamic_headers["Authorization"] = auth_header
return dynamic_headers

17
litellm/integrations/mlflow.py
View file

@ -60,10 +60,7 @@ class MlflowLogger(CustomLogger):
inputs = self._construct_input(kwargs)
input_messages = inputs.get("messages", [])
output_messages = [
c.message.model_dump(exclude_none=True)
for c in getattr(response_obj, "choices", [])
]
output_messages = [c.message.model_dump(exclude_none=True) for c in getattr(response_obj, "choices", [])]
if messages := [*input_messages, *output_messages]:
set_span_chat_messages(span, messages)
if tools := inputs.get("tools"):
@ -168,6 +165,10 @@ class MlflowLogger(CustomLogger):
for key in ["functions", "tools", "stream", "tool_choice", "user"]:
if value := kwargs.get("optional_params", {}).pop(key, None):
inputs[key] = value
if prediction := kwargs.get("prediction"):
inputs["prediction"] = prediction
return inputs
def _extract_attributes(self, kwargs):
@ -189,9 +190,9 @@ class MlflowLogger(CustomLogger):
{
"api_base": standard_obj.get("api_base"),
"cache_hit": standard_obj.get("cache_hit"),
"usage": {
"completion_tokens": standard_obj.get("completion_tokens"),
"prompt_tokens": standard_obj.get("prompt_tokens"),
"mlflow.chat.tokenUsage": {
"input_tokens": standard_obj.get("prompt_tokens"),
"output_tokens": standard_obj.get("completion_tokens"),
"total_tokens": standard_obj.get("total_tokens"),
},
"raw_llm_response": standard_obj.get("response"),
@ -232,7 +233,6 @@ class MlflowLogger(CustomLogger):
"""
import mlflow
call_type = kwargs.get("call_type", "completion")
span_name = f"litellm-{call_type}"
span_type = self._get_span_type(call_type)
@ -260,6 +260,7 @@ class MlflowLogger(CustomLogger):
tags=self._transform_tag_list_to_dict(attributes.get("request_tags", [])),
start_time_ns=start_time_ns,
)
def _transform_tag_list_to_dict(self, tag_list: list) -> dict:
return {tag: "" for tag in tag_list}

21
litellm/litellm_core_utils/core_helpers.py
View file

@ -37,6 +37,27 @@ def safe_divide_seconds(
return float(seconds / denominator)
def safe_divide(
numerator: Union[int, float],
denominator: Union[int, float],
default: Union[int, float] = 0
) -> Union[int, float]:
"""
Safely divide two numbers, returning a default value if denominator is zero.
Args:
numerator: The number to divide
denominator: The number to divide by
default: Value to return if denominator is zero (defaults to 0)
Returns:
The result of numerator/denominator, or default if denominator is zero
"""
if denominator == 0:
return default
return numerator / denominator
def map_finish_reason(
finish_reason: str,
): # openai supports 5 stop sequences - 'stop', 'length', 'function_call', 'content_filter', 'null'

29
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -196,6 +196,9 @@ def get_llm_provider( # noqa: PLR0915
elif endpoint == "https://api.cerebras.ai/v1":
custom_llm_provider = "cerebras"
dynamic_api_key = get_secret_str("CEREBRAS_API_KEY")
elif endpoint == "https://inference.baseten.co/v1":
custom_llm_provider = "baseten"
dynamic_api_key = get_secret_str("BASETEN_API_KEY")
elif endpoint == "https://api.sambanova.ai/v1":
custom_llm_provider = "sambanova"
dynamic_api_key = get_secret_str("SAMBANOVA_API_KEY")
@ -351,11 +354,16 @@ def get_llm_provider( # noqa: PLR0915
custom_llm_provider = "openai"
elif model in litellm.empower_models:
custom_llm_provider = "empower"
elif model in litellm.gradient_ai_models:
custom_llm_provider = "gradient_ai"
elif model == "*":
custom_llm_provider = "openai"
# bytez models
elif model.startswith("bytez/"):
custom_llm_provider = "bytez"
# cometapi models
elif model.startswith("cometapi/"):
custom_llm_provider = "cometapi"
elif model.startswith("oci/"):
custom_llm_provider = "oci"
if not custom_llm_provider:
@ -473,6 +481,13 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915
api_base or get_secret("CEREBRAS_API_BASE") or "https://api.cerebras.ai/v1"
) # type: ignore
dynamic_api_key = api_key or get_secret_str("CEREBRAS_API_KEY")
elif custom_llm_provider == "baseten":
# Use BasetenConfig to determine the appropriate API base URL
if api_base is None:
api_base = litellm.BasetenConfig.get_api_base_for_model(model)
else:
api_base = api_base or get_secret_str("BASETEN_API_BASE") or "https://inference.baseten.co/v1"
dynamic_api_key = api_key or get_secret_str("BASETEN_API_KEY")
elif custom_llm_provider == "sambanova":
api_base = (
api_base
@ -664,6 +679,13 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915
or f"https://{get_secret('SNOWFLAKE_ACCOUNT_ID')}.snowflakecomputing.com/api/v2/cortex/inference:complete"
) # type: ignore
dynamic_api_key = api_key or get_secret_str("SNOWFLAKE_JWT")
elif custom_llm_provider == "gradient_ai":
(
api_base,
dynamic_api_key,
) = litellm.GradientAIConfig()._get_openai_compatible_provider_info(
api_base, api_key
)
elif custom_llm_provider == "featherless_ai":
(
api_base,
@ -720,6 +742,13 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915
) = litellm.HyperbolicChatConfig()._get_openai_compatible_provider_info(
api_base, api_key
)
elif custom_llm_provider == "aiml":
(
api_base,
dynamic_api_key,
) = litellm.AIMLChatConfig()._get_openai_compatible_provider_info(
api_base, api_key
)
if api_base is not None and not isinstance(api_base, str):
raise Exception("api base needs to be a string. api_base={}".format(api_base))

11
litellm/litellm_core_utils/get_supported_openai_params.py
View file

@ -78,6 +78,8 @@ def get_supported_openai_params( # noqa: PLR0915
return litellm.nvidiaNimEmbeddingConfig.get_supported_openai_params()
elif custom_llm_provider == "cerebras":
return litellm.CerebrasConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "baseten":
return litellm.BasetenConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "xai":
return litellm.XAIChatConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "ai21_chat" or custom_llm_provider == "ai21":
@ -121,6 +123,10 @@ def get_supported_openai_params( # noqa: PLR0915
return litellm.AzureOpenAIO1Config().get_supported_openai_params(
model=model
)
elif litellm.AzureOpenAIGPT5Config.is_model_gpt_5_model(model=model):
return litellm.AzureOpenAIGPT5Config().get_supported_openai_params(
model=model
)
else:
return litellm.AzureOpenAIConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "openrouter":
@ -136,7 +142,10 @@ def get_supported_openai_params( # noqa: PLR0915
model=model
)
elif custom_llm_provider == "sambanova":
return litellm.SambanovaConfig().get_supported_openai_params(model=model)
if request_type == "embeddings":
litellm.SambaNovaEmbeddingConfig().get_supported_openai_params(model=model)
else:
return litellm.SambanovaConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "nebius":
if request_type == "chat_completion":
return litellm.NebiusConfig().get_supported_openai_params(model=model)

12
litellm/litellm_core_utils/health_check_helpers.py
View file

@ -1,12 +1,13 @@
"""
Helper functions for health check calls.
"""
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging
class HealthCheckHelpers:
@staticmethod
@ -38,10 +39,9 @@ class HealthCheckHelpers:
model_params["model"] = cheapest_models[0]
model_params["litellm_logging_obj"] = litellm_logging_obj
model_params["fallbacks"] = fallback_models
model_params["max_tokens"] = 1
model_params["max_tokens"] = 10 # gpt-5-nano throws errors for max_tokens=1
await acompletion(**model_params)
return {}
@staticmethod
def _update_model_params_with_health_check_tracking_information(
@ -57,6 +57,7 @@ class HealthCheckHelpers:
"""
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
_metadata_variable_name = "litellm_metadata"
litellm_metadata = HealthCheckHelpers._get_metadata_for_health_check_call()
model_params[_metadata_variable_name] = litellm_metadata
@ -66,13 +67,14 @@ class HealthCheckHelpers:
_metadata_variable_name=_metadata_variable_name,
)
return model_params
@staticmethod
def _get_metadata_for_health_check_call():
"""
Returns the metadata for the health check call.
"""
from litellm.constants import LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME
return {
"tags": [LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME],
}
}

40
litellm/litellm_core_utils/litellm_logging.py
View file

@ -131,7 +131,6 @@ from ..integrations.humanloop import HumanloopLogger
from ..integrations.lago import LagoLogger
from ..integrations.langfuse.langfuse import LangFuseLogger
from ..integrations.langfuse.langfuse_handler import LangFuseHandler
from ..integrations.langfuse.langfuse_otel import LangfuseOtelLogger
from ..integrations.langfuse.langfuse_prompt_management import LangfusePromptManagement
from ..integrations.langsmith import LangsmithLogger
from ..integrations.literal_ai import LiteralAILogger
@ -812,7 +811,7 @@ class Logging(LiteLLMLoggingBaseClass):
str(e)
)
)
if self.logger_fn and callable(self.logger_fn):
if getattr(self, "logger_fn", None) and callable(self.logger_fn):
try:
self.logger_fn(
self.model_call_details
@ -1000,7 +999,7 @@ class Logging(LiteLLMLoggingBaseClass):
)
)
)
if self.logger_fn and callable(self.logger_fn):
if getattr(self, "logger_fn", None) and callable(self.logger_fn):
try:
self.logger_fn(
self.model_call_details
@ -3457,6 +3456,7 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
_in_memory_loggers.append(langfuse_logger)
return langfuse_logger # type: ignore
elif logging_integration == "langfuse_otel":
from litellm.integrations.langfuse.langfuse_otel import LangfuseOtelLogger
from litellm.integrations.opentelemetry import (
OpenTelemetry,
OpenTelemetryConfig,
@ -3467,15 +3467,16 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
# The endpoint and headers are now set as environment variables by get_langfuse_otel_config()
otel_config = OpenTelemetryConfig(
exporter=langfuse_otel_config.protocol,
headers=langfuse_otel_config.otlp_auth_headers,
)
for callback in _in_memory_loggers:
if (
isinstance(callback, OpenTelemetry)
isinstance(callback, LangfuseOtelLogger)
and callback.callback_name == "langfuse_otel"
):
return callback # type: ignore
_otel_logger = OpenTelemetry(
_otel_logger = LangfuseOtelLogger(
config=otel_config, callback_name="langfuse_otel"
)
_in_memory_loggers.append(_otel_logger)
@ -3918,10 +3919,12 @@ class StandardLoggingPayloadSetup:
# Generate cold storage object key if cold storage is configured
if start_time is not None and response_id is not None:
cold_storage_object_key = StandardLoggingPayloadSetup._generate_cold_storage_object_key(
start_time=start_time,
response_id=response_id,
team_alias=clean_metadata.get("user_api_key_team_alias"),
cold_storage_object_key = (
StandardLoggingPayloadSetup._generate_cold_storage_object_key(
start_time=start_time,
response_id=response_id,
team_alias=clean_metadata.get("user_api_key_team_alias"),
)
)
if cold_storage_object_key:
clean_metadata["cold_storage_object_key"] = cold_storage_object_key
@ -4092,36 +4095,33 @@ class StandardLoggingPayloadSetup:
) -> Optional[str]:
"""
Generate cold storage object key in the same format as S3Logger.
Args:
start_time: The start time of the request
response_id: The response ID
response_id: The response ID
team_alias: Optional team alias for team-based prefixing
Returns:
Optional[str]: The generated object key or None if cold storage not configured
"""
# Generate object key in same format as S3Logger
from litellm.integrations.s3 import get_s3_object_key
from litellm.proxy.spend_tracking.cold_storage_handler import ColdStorageHandler
# Only generate object key if cold storage is configured
configured_cold_storage_logger = ColdStorageHandler._get_configured_cold_storage_custom_logger()
if configured_cold_storage_logger is None:
if litellm.configured_cold_storage_logger is None:
return None
try:
# Generate file name in same format as litellm.utils.get_logging_id
s3_file_name = f"time-{start_time.strftime('%H-%M-%S-%f')}_{response_id}"
s3_object_key = get_s3_object_key(
s3_path="", # Use empty path as default
team_alias_prefix="", # Don't split by team alias for cold storage
s3_path="", # Use empty path as default
team_alias_prefix="", # Don't split by team alias for cold storage
start_time=start_time,
s3_file_name=s3_file_name,
)
return s3_object_key
except Exception:
# If any error occurs in generating the key, return None

144
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -1,11 +1,17 @@
# What is this?
## Helper utilities for cost_per_token()
from typing import Literal, Optional, Tuple, cast
from typing import Any, Literal, Optional, Tuple, cast
import litellm
from litellm._logging import verbose_logger
from litellm.types.utils import CallTypes, ModelInfo, PassthroughCallTypes, Usage
from litellm.types.utils import (
CallTypes,
ImageResponse,
ModelInfo,
PassthroughCallTypes,
Usage,
)
from litellm.utils import get_model_info
@ -107,15 +113,20 @@ def _generic_cost_per_character(
return prompt_cost, completion_cost
def _get_token_base_cost(model_info: ModelInfo, usage: Usage) -> Tuple[float, float]:
def _get_token_base_cost(model_info: ModelInfo, usage: Usage) -> Tuple[float, float, float, float]:
"""
Return prompt cost for a given model and usage.
Return prompt cost, completion cost, and cache costs for a given model and usage.
If input_tokens > threshold and `input_cost_per_token_above_[x]k_tokens` or `input_cost_per_token_above_[x]_tokens` is set,
then we use the corresponding threshold cost.
then we use the corresponding threshold cost for all token types.
Returns:
Tuple[float, float, float, float] - (prompt_cost, completion_cost, cache_creation_cost, cache_read_cost)
"""
prompt_base_cost = cast(float, _get_cost_per_unit(model_info, "input_cost_per_token"))
completion_base_cost = cast(float, _get_cost_per_unit(model_info, "output_cost_per_token"))
cache_creation_cost = cast(float, _get_cost_per_unit(model_info, "cache_creation_input_token_cost"))
cache_read_cost = cast(float, _get_cost_per_unit(model_info, "cache_read_input_token_cost"))
## CHECK IF ABOVE THRESHOLD
threshold: Optional[float] = None
@ -135,13 +146,28 @@ def _get_token_base_cost(model_info: ModelInfo, usage: Usage) -> Tuple[float, fl
f"output_cost_per_token_above_{threshold_str}_tokens",
completion_base_cost,
))
# Apply tiered pricing to cache costs
cache_creation_tiered_key = f"cache_creation_input_token_cost_above_{threshold_str}_tokens"
cache_read_tiered_key = f"cache_read_input_token_cost_above_{threshold_str}_tokens"
if cache_creation_tiered_key in model_info:
cache_creation_cost = cast(float, _get_cost_per_unit(
model_info, cache_creation_tiered_key, cache_creation_cost
))
if cache_read_tiered_key in model_info:
cache_read_cost = cast(float, _get_cost_per_unit(
model_info, cache_read_tiered_key, cache_read_cost
))
break
except (IndexError, ValueError):
continue
except Exception:
continue
return prompt_base_cost, completion_base_cost
return prompt_base_cost, completion_base_cost, cache_creation_cost, cache_read_cost
def calculate_cost_component(
@ -256,28 +282,22 @@ def generic_cost_per_token(
if text_tokens == 0:
text_tokens = usage.prompt_tokens - cache_hit_tokens - audio_tokens
prompt_base_cost, completion_base_cost = _get_token_base_cost(
prompt_base_cost, completion_base_cost, cache_creation_cost, cache_read_cost = _get_token_base_cost(
model_info=model_info, usage=usage
)
prompt_cost = float(text_tokens) * prompt_base_cost
### CACHE READ COST
prompt_cost += calculate_cost_component(
model_info, "cache_read_input_token_cost", cache_hit_tokens
)
### CACHE READ COST - Now uses tiered pricing
prompt_cost += float(cache_hit_tokens) * cache_read_cost
### AUDIO COST
prompt_cost += calculate_cost_component(
model_info, "input_cost_per_audio_token", audio_tokens
)
### CACHE WRITING COST
prompt_cost += calculate_cost_component(
model_info,
"cache_creation_input_token_cost",
usage._cache_creation_input_tokens,
)
### CACHE WRITING COST - Now uses tiered pricing
prompt_cost += float(usage._cache_creation_input_tokens or 0) * cache_creation_cost
### CHARACTER COST
@ -377,3 +397,93 @@ class CostCalculatorUtils:
]:
return True
return False
@staticmethod
def route_image_generation_cost_calculator(
model: str,
completion_response: Any,
custom_llm_provider: Optional[str] = None,
quality: Optional[str] = None,
n: Optional[int] = None,
size: Optional[str] = None,
optional_params: Optional[dict] = None,
) -> float:
"""
Route the image generation cost calculator based on the custom_llm_provider
"""
from litellm.cost_calculator import default_image_cost_calculator
from litellm.llms.azure_ai.image_generation.cost_calculator import (
cost_calculator as azure_ai_image_cost_calculator,
)
from litellm.llms.bedrock.image.cost_calculator import (
cost_calculator as bedrock_image_cost_calculator,
)
from litellm.llms.gemini.image_generation.cost_calculator import (
cost_calculator as gemini_image_cost_calculator,
)
from litellm.llms.recraft.cost_calculator import (
cost_calculator as recraft_image_cost_calculator,
)
from litellm.llms.vertex_ai.image_generation.cost_calculator import (
cost_calculator as vertex_ai_image_cost_calculator,
)
if custom_llm_provider == litellm.LlmProviders.VERTEX_AI.value:
if isinstance(completion_response, ImageResponse):
return vertex_ai_image_cost_calculator(
model=model,
image_response=completion_response,
)
elif custom_llm_provider == litellm.LlmProviders.BEDROCK.value:
if isinstance(completion_response, ImageResponse):
return bedrock_image_cost_calculator(
model=model,
size=size,
image_response=completion_response,
optional_params=optional_params,
)
raise TypeError(
"completion_response must be of type ImageResponse for bedrock image cost calculation"
)
elif custom_llm_provider == litellm.LlmProviders.RECRAFT.value:
from litellm.llms.recraft.cost_calculator import (
cost_calculator as recraft_image_cost_calculator,
)
return recraft_image_cost_calculator(
model=model,
image_response=completion_response,
)
elif custom_llm_provider == litellm.LlmProviders.AIML.value:
from litellm.llms.aiml.image_generation.cost_calculator import (
cost_calculator as aiml_image_cost_calculator,
)
return aiml_image_cost_calculator(
model=model,
image_response=completion_response,
)
elif custom_llm_provider == litellm.LlmProviders.GEMINI.value:
from litellm.llms.gemini.image_generation.cost_calculator import (
cost_calculator as gemini_image_cost_calculator,
)
return gemini_image_cost_calculator(
model=model,
image_response=completion_response,
)
elif custom_llm_provider == litellm.LlmProviders.AZURE_AI.value:
return azure_ai_image_cost_calculator(
model=model,
image_response=completion_response,
)
else:
return default_image_cost_calculator(
model=model,
quality=quality,
custom_llm_provider=custom_llm_provider,
n=n,
size=size,
optional_params=optional_params,
)
return 0.0

119
litellm/litellm_core_utils/logging_utils.py
View file

@ -1,5 +1,6 @@
import asyncio
import functools
import time
from datetime import datetime
from typing import TYPE_CHECKING, Any, List, Optional, Union
@ -11,15 +12,19 @@ from litellm.types.utils import (
)
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
from litellm import ModelResponse as _ModelResponse
from litellm.litellm_core_utils.litellm_logging import (
Logging as LiteLLMLoggingObject,
)
LiteLLMModelResponse = _ModelResponse
Span = Union[_Span, Any]
else:
LiteLLMModelResponse = Any
LiteLLMLoggingObject = Any
Span = Any
import litellm
@ -28,9 +33,52 @@ import litellm
Helper utils used for logging callbacks
"""
# Global service logger instance to avoid recreating it
_service_logger = None
def _get_service_logger():
"""Get or create the global ServiceLogging instance"""
global _service_logger
if _service_logger is None:
from litellm._service_logger import ServiceLogging
_service_logger = ServiceLogging()
return _service_logger
def _get_parent_otel_span_from_logging_obj(
logging_obj: Optional[LiteLLMLoggingObject] = None,
) -> Optional[Span]:
"""
Extract the parent OTEL span from the logging object using existing helper.
Args:
logging_obj: The LiteLLM logging object containing model call details
Returns:
The parent OTEL span if found, None otherwise
"""
try:
if logging_obj is None or not hasattr(logging_obj, "model_call_details"):
return None
# Reuse existing function by passing model_call_details as kwargs
from litellm.litellm_core_utils.core_helpers import (
_get_parent_otel_span_from_kwargs,
)
return _get_parent_otel_span_from_kwargs(logging_obj.model_call_details)
except Exception as e:
verbose_logger.exception(
f"Error in _get_parent_otel_span_from_logging_obj: {str(e)}"
)
return None
def convert_litellm_response_object_to_str(
response_obj: Union[Any, LiteLLMModelResponse]
response_obj: Union[Any, LiteLLMModelResponse],
) -> Optional[str]:
"""
Get the string of the response object from LiteLLM
@ -125,37 +173,102 @@ def track_llm_api_timing():
"""
Decorator to track LLM API call timing for both sync and async functions.
The logging_obj is expected to be passed as an argument to the decorated function.
Logs timing using ServiceLogging similar to Redis cache.
"""
def decorator(func):
@functools.wraps(func)
async def async_wrapper(*args, **kwargs):
start_time = datetime.now()
start_time_float = time.time()
logging_obj = kwargs.get("logging_obj", None)
# Extract parent OTEL span from logging object
parent_otel_span = _get_parent_otel_span_from_logging_obj(logging_obj)
try:
result = await func(*args, **kwargs)
return result
finally:
end_time = datetime.now()
end_time_float = time.time()
duration = end_time_float - start_time_float
# Set duration in model call details
_set_duration_in_model_call_details(
logging_obj=kwargs.get("logging_obj", None),
logging_obj=logging_obj,
start_time=start_time,
end_time=end_time,
)
# Log timing using ServiceLogging (like Redis cache)
try:
from litellm.types.services import ServiceTypes
service_logger = _get_service_logger()
# Get function name for call_type
call_type = f"{func.__name__} <- track_llm_api_timing"
# Create async task for service logging (similar to Redis cache pattern)
asyncio.create_task(
service_logger.async_service_success_hook(
service=ServiceTypes.LITELLM,
duration=duration,
call_type=call_type,
start_time=start_time_float,
end_time=end_time_float,
parent_otel_span=parent_otel_span,
)
)
except Exception as e:
verbose_logger.debug(f"Error in service logging: {str(e)}")
@functools.wraps(func)
def sync_wrapper(*args, **kwargs):
start_time = datetime.now()
start_time_float = time.time()
logging_obj = kwargs.get("logging_obj", None)
# Extract parent OTEL span from logging object
parent_otel_span = _get_parent_otel_span_from_logging_obj(logging_obj)
try:
result = func(*args, **kwargs)
return result
finally:
end_time = datetime.now()
end_time_float = time.time()
duration = end_time_float - start_time_float
# Set duration in model call details
_set_duration_in_model_call_details(
logging_obj=kwargs.get("logging_obj", None),
logging_obj=logging_obj,
start_time=start_time,
end_time=end_time,
)
# Log timing using ServiceLogging (like Redis cache)
try:
from litellm.types.services import ServiceTypes
service_logger = _get_service_logger()
# Get function name for call_type
call_type = f"{func.__name__} <- track_llm_api_timing"
# Use sync service logging for sync functions
service_logger.service_success_hook(
service=ServiceTypes.LITELLM,
duration=duration,
call_type=call_type,
start_time=start_time_float,
end_time=end_time_float,
parent_otel_span=parent_otel_span,
)
except Exception as e:
verbose_logger.debug(f"Error in service logging: {str(e)}")
# Check if the function is async or sync
if asyncio.iscoroutinefunction(func):
return async_wrapper

131
litellm/litellm_core_utils/logging_worker.py Normal file
View file

@ -0,0 +1,131 @@
import asyncio
import contextlib
from typing import Coroutine, Optional
from litellm._logging import verbose_logger
class LoggingWorker:
"""
A simple, async logging worker that processes log coroutines in the background.
Designed to be best-effort with bounded queues to prevent backpressure.
This leads to a +200 RPS performance improvement when using LiteLLM Python SDK or Proxy Server.
- Use this to queue coroutine tasks that are not critical to the main flow of the application. e.g Success/Error callbacks, logging, etc.
"""
LOGGING_WORKER_MAX_QUEUE_SIZE = 50_000
LOGGING_WORKER_MAX_TIME_PER_COROUTINE = 20.0
MAX_ITERATIONS_TO_CLEAR_QUEUE = 200
MAX_TIME_TO_CLEAR_QUEUE = 5.0
def __init__(
self,
timeout: float = LOGGING_WORKER_MAX_TIME_PER_COROUTINE,
max_queue_size: int = LOGGING_WORKER_MAX_QUEUE_SIZE,
):
self.timeout = timeout
self.max_queue_size = max_queue_size
self._queue: Optional[asyncio.Queue] = None
self._worker_task: Optional[asyncio.Task] = None
def _ensure_queue(self) -> None:
"""Initialize the queue if it doesn't exist."""
if self._queue is None:
self._queue = asyncio.Queue(maxsize=self.max_queue_size)
def start(self) -> None:
"""Start the logging worker. Idempotent - safe to call multiple times."""
self._ensure_queue()
if self._worker_task is None or self._worker_task.done():
self._worker_task = asyncio.create_task(self._worker_loop())
async def _worker_loop(self) -> None:
"""Main worker loop that processes log coroutines sequentially."""
try:
if self._queue is None:
return
while True:
# Process one coroutine at a time to keep event loop load predictable
coroutine = await self._queue.get()
try:
await asyncio.wait_for(coroutine, timeout=self.timeout)
except Exception as e:
verbose_logger.exception(f"LoggingWorker error: {e}")
pass
finally:
self._queue.task_done()
except asyncio.CancelledError as e:
verbose_logger.exception(f"LoggingWorker cancelled: {e}")
pass
def enqueue(self, coroutine: Coroutine) -> None:
"""
Add a coroutine to the logging queue.
Hot path: never blocks, drops logs if queue is full.
"""
if self._queue is None:
return
try:
self._queue.put_nowait(coroutine)
except asyncio.QueueFull as e:
verbose_logger.exception(f"LoggingWorker queue is full: {e}")
# Drop logs on overload to protect request throughput
pass
def ensure_initialized_and_enqueue(self, async_coroutine: Coroutine):
"""
Ensure the logging worker is initialized and enqueue the coroutine.
"""
self.start()
self.enqueue(async_coroutine)
async def stop(self) -> None:
"""Stop the logging worker and clean up resources."""
if self._worker_task:
self._worker_task.cancel()
with contextlib.suppress(Exception):
await self._worker_task
self._worker_task = None
async def flush(self) -> None:
"""Flush the logging queue."""
if self._queue is None:
return
while not self._queue.empty():
await self._queue.join()
async def clear_queue(self):
"""
Clear the queue with a maximum time limit.
"""
if self._queue is None:
return
start_time = asyncio.get_event_loop().time()
for _ in range(self.MAX_ITERATIONS_TO_CLEAR_QUEUE):
# Check if we've exceeded the maximum time
if asyncio.get_event_loop().time() - start_time >= self.MAX_TIME_TO_CLEAR_QUEUE:
verbose_logger.warning(f"clear_queue exceeded max_time of {self.MAX_TIME_TO_CLEAR_QUEUE}s, stopping early")
break
try:
coroutine = self._queue.get_nowait()
# Await the coroutine to properly execute and avoid "never awaited" warnings
try:
await asyncio.wait_for(coroutine, timeout=self.timeout)
except Exception:
# Suppress errors during cleanup
pass
self._queue.task_done() # If you're using join() elsewhere
except asyncio.QueueEmpty:
break
# Global instance for backward compatibility
GLOBAL_LOGGING_WORKER = LoggingWorker()

213
litellm/litellm_core_utils/model_response_utils.py Normal file
View file

@ -0,0 +1,213 @@
"""
Utility functions for ModelResponse and ModelResponseStream objects.
"""
from typing import Any
from litellm.types.utils import Delta, ModelResponseBase, ModelResponseStream
def is_model_response_stream_empty(model_response: ModelResponseStream) -> bool:
"""
Check if a ModelResponseStream is empty based on:
- If finish_reason is set -> it's non empty
- If any field in choices is set (e.g. content, tool calls, etc.) it's non empty
- If usage exists -> it's non empty
This function is robust and ignores fields that are always set (from ModelResponseBase)
and checks for any meaningful content in other fields.
Args:
model_response: The ModelResponseStream to check
Returns:
bool: True if the stream is empty, False if it contains meaningful data
"""
# Fields that are always set in ModelResponseBase and should be ignored
# These are structural fields that don't indicate content
BASE_FIELDS = ModelResponseBase.model_fields.keys()
# Check if usage exists - this indicates meaningful data
if getattr(model_response, "usage", None) is not None:
return False
# Check provider_specific_fields at the top level
if (
hasattr(model_response, "provider_specific_fields")
and model_response.provider_specific_fields is not None
and model_response.provider_specific_fields != {}
):
return False
# Check model_extra for dynamically added fields (this is where Pydantic stores them)
if hasattr(model_response, "model_extra") and model_response.model_extra:
for extra_field_name, extra_field_value in model_response.model_extra.items():
if _has_meaningful_content(extra_field_value):
return False
# Check for any non-base fields that are set
for model_response_field in model_response.model_fields.keys():
# Skip base fields that are always set
if model_response_field in BASE_FIELDS:
continue
# Skip choices - we'll handle them separately with deep inspection
if model_response_field == "choices":
continue
# Check if any other field has meaningful content
model_response_value = getattr(model_response, model_response_field, None)
if _has_meaningful_content(model_response_value):
return False
# Deep check of choices for any meaningful content
if hasattr(model_response, "choices") and model_response.choices:
for choice in model_response.choices:
if _is_choice_non_empty(choice):
return False
# If we get here, the stream is empty
return True
def _has_meaningful_content(value: Any) -> bool:
"""
Check if a value contains meaningful content.
Args:
value: The value to check
Returns:
bool: True if the value has meaningful content, False otherwise
"""
if value is None:
return False
if isinstance(value, str):
return len(value.strip()) > 0
if isinstance(value, (list, dict)):
return len(value) > 0
if isinstance(value, bool):
return True # Any boolean value is meaningful
if isinstance(value, (int, float)):
return True # Any numeric value is meaningful
# For other types (objects), consider them meaningful if they exist
return True
def _is_choice_non_empty(choice: Any) -> bool:
"""
Deep check if a choice contains any meaningful content.
Args:
choice: The choice object to check
Returns:
bool: True if the choice has meaningful content, False otherwise
"""
# Check finish_reason
if hasattr(choice, "finish_reason") and choice.finish_reason is not None:
return True
# Check logprobs
if hasattr(choice, "logprobs") and choice.logprobs is not None:
return True
# Check enhancements (if present)
if hasattr(choice, "enhancements") and choice.enhancements is not None:
return True
# Deep check delta object
if hasattr(choice, "delta") and choice.delta is not None:
if _is_delta_non_empty(choice.delta):
return True
# Check model_extra for dynamically added fields on the choice
if hasattr(choice, "model_extra") and choice.model_extra:
for extra_field_name, extra_field_value in choice.model_extra.items():
# Skip certain structural fields that are just default/None placeholders
if extra_field_name == "index" and extra_field_value == 0:
continue
if (
extra_field_name in {"finish_reason", "logprobs"}
and extra_field_value is None
):
continue
if extra_field_name == "delta":
continue
if _has_meaningful_content(extra_field_value):
return True
# Check for any other non-standard fields on the choice
for attr_name in dir(choice):
# Skip private attributes, methods, and known empty fields
if (
attr_name.startswith("_")
or callable(getattr(choice, attr_name))
or attr_name.startswith("model_")
or attr_name
in {
"finish_reason",
"index",
"delta",
"logprobs",
"enhancements",
}
):
continue
attr_value = getattr(choice, attr_name, None)
if _has_meaningful_content(attr_value):
return True
return False
def _is_delta_non_empty(delta: Delta) -> bool:
"""
Deep check if a delta object contains any meaningful content.
Args:
delta: The delta object to check
Returns:
bool: True if the delta has meaningful content, False otherwise
"""
# Check model_extra for dynamically added fields (this is where Pydantic stores them)
if hasattr(delta, "model_extra") and delta.model_extra:
for extra_field_name, extra_field_value in delta.model_extra.items():
# Even structural fields are meaningful if they have actual content
if _has_meaningful_content(extra_field_value):
return True
# Check all regular attributes of the delta object
for attr_name in dir(delta):
# Skip private attributes, methods, and Pydantic-specific fields
if (
attr_name.startswith("_")
or callable(getattr(delta, attr_name))
or attr_name.startswith("model_")
):
continue
attr_value = getattr(delta, attr_name, None)
if _has_meaningful_content(attr_value):
return True
return False

5
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -18,6 +18,7 @@ from typing import (
cast,
)
from litellm.router_utils.batch_utils import InMemoryFile
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionAssistantMessage,
@ -453,6 +454,10 @@ def extract_file_data(file_data: FileTypes) -> ExtractedFileData:
filename, file_content, content_type = file_data
elif len(file_data) == 4:
filename, file_content, content_type, file_headers = file_data
elif isinstance(file_data, InMemoryFile):
filename = file_data.name
file_content = file_data
content_type = file_data.content_type
else:
file_content = file_data
# Convert content to bytes

102
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -17,6 +17,7 @@ from litellm.llms.custom_httpx.http_handler import HTTPHandler, get_async_httpx_
from litellm.types.files import get_file_extension_from_mime_type
from litellm.types.llms.anthropic import *
from litellm.types.llms.bedrock import MessageBlock as BedrockMessageBlock
from litellm.types.llms.bedrock import CachePointBlock
from litellm.types.llms.custom_http import httpxSpecialProvider
from litellm.types.llms.ollama import OllamaVisionModelObject
from litellm.types.llms.openai import (
@ -2685,6 +2686,11 @@ def _convert_to_bedrock_tool_call_invoke(
)
bedrock_content_block = BedrockContentBlock(toolUse=bedrock_tool)
_parts_list.append(bedrock_content_block)
# Check for cache_control and add a separate cachePoint block
if tool.get("cache_control", None) is not None:
cache_point_block = BedrockContentBlock(cachePoint=CachePointBlock(type="default"))
_parts_list.append(cache_point_block)
return _parts_list
except Exception as e:
raise Exception(
@ -2745,6 +2751,7 @@ def _convert_to_bedrock_tool_call_result(
for content in content_list:
if content["type"] == "text":
content_str += content["text"]
message.get("name", "")
id = str(message.get("tool_call_id", str(uuid.uuid4())))
@ -2753,6 +2760,7 @@ def _convert_to_bedrock_tool_call_result(
content=[tool_result_content_block],
toolUseId=id,
)
content_block = BedrockContentBlock(toolResult=tool_result)
return content_block
@ -3185,9 +3193,30 @@ class BedrockConverseMessagesProcessor:
## MERGE CONSECUTIVE TOOL CALL MESSAGES ##
tool_content: List[BedrockContentBlock] = []
while msg_i < len(messages) and messages[msg_i]["role"] == "tool":
tool_call_result = _convert_to_bedrock_tool_call_result(messages[msg_i])
current_message = messages[msg_i]
tool_call_result = _convert_to_bedrock_tool_call_result(current_message)
tool_content.append(tool_call_result)
# Check if we need to add a separate cachePoint block
has_cache_control = False
# Check for message-level cache_control
if current_message.get("cache_control", None) is not None:
has_cache_control = True
# Check for content-level cache_control in list content
elif isinstance(current_message.get("content"), list):
for content_element in current_message["content"]:
if (isinstance(content_element, dict) and
content_element.get("cache_control", None) is not None):
has_cache_control = True
break
# Add a separate cachePoint block if cache_control is present
if has_cache_control:
cache_point_block = BedrockContentBlock(cachePoint=CachePointBlock(type="default"))
tool_content.append(cache_point_block)
msg_i += 1
if tool_content:
# if last message was a 'user' message, then add a blank assistant message (bedrock requires alternating roles)
@ -3267,13 +3296,29 @@ class BedrockConverseMessagesProcessor:
image_url=image_url
)
assistants_parts.append(assistants_part)
# Add cache point block for assistant content elements
_cache_point_block = (
litellm.AmazonConverseConfig()._get_cache_point_block(
message_block=cast(
OpenAIMessageContentListBlock, element
),
block_type="content_block",
)
)
if _cache_point_block is not None:
assistants_parts.append(_cache_point_block)
assistant_content.extend(assistants_parts)
elif _assistant_content is not None and isinstance(
_assistant_content, str
):
assistant_content.append(
BedrockContentBlock(text=_assistant_content)
elif _assistant_content is not None and isinstance(_assistant_content, str):
assistant_content.append(BedrockContentBlock(text=_assistant_content))
# Add cache point block for assistant string content
_cache_point_block = (
litellm.AmazonConverseConfig()._get_cache_point_block(
assistant_message_block, block_type="content_block"
)
)
if _cache_point_block is not None:
assistant_content.append(_cache_point_block)
_tool_calls = assistant_message_block.get("tool_calls", [])
if _tool_calls:
assistant_content.extend(
@ -3516,8 +3561,30 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915
tool_content: List[BedrockContentBlock] = []
while msg_i < len(messages) and messages[msg_i]["role"] == "tool":
tool_call_result = _convert_to_bedrock_tool_call_result(messages[msg_i])
current_message = messages[msg_i]
# Add the tool result first
tool_content.append(tool_call_result)
# Check if we need to add a separate cachePoint block
has_cache_control = False
# Check for message-level cache_control
if current_message.get("cache_control", None) is not None:
has_cache_control = True
# Check for content-level cache_control in list content
elif isinstance(current_message.get("content"), list):
for content_element in current_message["content"]:
if (isinstance(content_element, dict) and
content_element.get("cache_control", None) is not None):
has_cache_control = True
break
# Add a separate cachePoint block if cache_control is present
if has_cache_control:
cache_point_block = BedrockContentBlock(cachePoint=CachePointBlock(type="default"))
tool_content.append(cache_point_block)
msg_i += 1
if tool_content:
# if last message was a 'user' message, then add a blank assistant message (bedrock requires alternating roles)
@ -3589,9 +3656,28 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915
image_url=image_url
)
assistants_parts.append(assistants_part)
# Add cache point block for assistant content elements
_cache_point_block = (
litellm.AmazonConverseConfig()._get_cache_point_block(
message_block=cast(
OpenAIMessageContentListBlock, element
),
block_type="content_block",
)
)
if _cache_point_block is not None:
assistants_parts.append(_cache_point_block)
assistant_content.extend(assistants_parts)
elif _assistant_content is not None and isinstance(_assistant_content, str):
assistant_content.append(BedrockContentBlock(text=_assistant_content))
# Add cache point block for assistant string content
_cache_point_block = (
litellm.AmazonConverseConfig()._get_cache_point_block(
assistant_message_block, block_type="content_block"
)
)
if _cache_point_block is not None:
assistant_content.append(_cache_point_block)
_tool_calls = assistant_message_block.get("tool_calls", [])
if _tool_calls:
assistant_content.extend(

7
litellm/litellm_core_utils/sensitive_data_masker.py
View file

@ -33,7 +33,12 @@ class SensitiveDataMasker:
value_str = str(value)
masked_length = len(value_str) - (self.visible_prefix + self.visible_suffix)
return f"{value_str[:self.visible_prefix]}{self.mask_char * masked_length}{value_str[-self.visible_suffix:]}"
# Handle the case where visible_suffix is 0 to avoid showing the entire string
if self.visible_suffix == 0:
return f"{value_str[:self.visible_prefix]}{self.mask_char * masked_length}"
else:
return f"{value_str[:self.visible_prefix]}{self.mask_char * masked_length}{value_str[-self.visible_suffix:]}"
def is_sensitive_key(self, key: str) -> bool:
key_lower = str(key).lower()

22
litellm/litellm_core_utils/streaming_handler.py
View file

@ -13,6 +13,9 @@ from pydantic import BaseModel
import litellm
from litellm import verbose_logger
from litellm.litellm_core_utils.model_response_utils import (
is_model_response_stream_empty,
)
from litellm.litellm_core_utils.redact_messages import LiteLLMLoggingObject
from litellm.litellm_core_utils.thread_pool_executor import executor
from litellm.types.llms.openai import ChatCompletionChunk
@ -1574,6 +1577,13 @@ class CustomStreamWrapper:
response = self.model_response_creator(
chunk=obj_dict, hidden_params=response._hidden_params
)
## check if empty
is_empty = is_model_response_stream_empty(
model_response=cast(ModelResponseStream, response)
)
if is_empty:
continue
# add usage as hidden param
if self.sent_last_chunk is True and self.stream_options is None:
usage = calculate_total_usage(chunks=self.chunks)
@ -1730,7 +1740,18 @@ class CustomStreamWrapper:
# Create a new object without the removed attribute
processed_chunk = self.model_response_creator(chunk=obj_dict)
is_empty = is_model_response_stream_empty(
model_response=cast(ModelResponseStream, processed_chunk)
)
if is_empty:
continue
print_verbose(f"final returned processed chunk: {processed_chunk}")
# add usage as hidden param
if self.sent_last_chunk is True and self.stream_options is None:
usage = calculate_total_usage(chunks=self.chunks)
processed_chunk._hidden_params["usage"] = usage
return processed_chunk
raise StopAsyncIteration
else: # temporary patch for non-aiohttp async calls
@ -1774,6 +1795,7 @@ class CustomStreamWrapper:
messages=self.messages,
logging_obj=self.logging_obj,
)
response = self.model_response_creator()
if complete_streaming_response is not None:
setattr(

5
litellm/llms/aiml/__init__.py Normal file
View file

@ -0,0 +1,5 @@
from .image_generation import get_aiml_image_generation_config
__all__ = [
"get_aiml_image_generation_config",
]

23
litellm/llms/aiml/chat/transformation.py Normal file
View file

@ -0,0 +1,23 @@
from typing import Optional, Tuple
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
from litellm.secret_managers.main import get_secret_str
class AIMLChatConfig(OpenAIGPTConfig):
@property
def custom_llm_provider(self) -> Optional[str]:
return "aiml"
def _get_openai_compatible_provider_info(
self, api_base: Optional[str], api_key: Optional[str]
) -> Tuple[Optional[str], Optional[str]]:
# AIML is openai compatible, we just need to set the api_base
api_base = (
api_base
or get_secret_str("AIML_API_BASE")
or "https://api.aimlapi.com/v1" # Default AIML API base URL
) # type: ignore
dynamic_api_key = api_key or get_secret_str("AIML_API_KEY")
return api_base, dynamic_api_key
pass

13
litellm/llms/aiml/image_generation/__init__.py Normal file
View file

@ -0,0 +1,13 @@
from litellm.llms.base_llm.image_generation.transformation import (
BaseImageGenerationConfig,
)
from .transformation import AimlImageGenerationConfig
__all__ = [
"AimlImageGenerationConfig",
]
def get_aiml_image_generation_config(model: str) -> BaseImageGenerationConfig:
return AimlImageGenerationConfig()

25
litellm/llms/aiml/image_generation/cost_calculator.py Normal file
View file

@ -0,0 +1,25 @@
from typing import Any
import litellm
from litellm.types.utils import ImageResponse
def cost_calculator(
model: str,
image_response: Any,
) -> float:
"""
AI/ML flux image generation cost calculator
"""
_model_info = litellm.get_model_info(
model=model,
custom_llm_provider=litellm.LlmProviders.AIML.value,
)
output_cost_per_image: float = _model_info.get("output_cost_per_image") or 0.0
num_images: int = 0
if isinstance(image_response, ImageResponse):
if image_response.data:
num_images = len(image_response.data)
return output_cost_per_image * num_images
else:
raise ValueError(f"image_response must be of type ImageResponse got type={type(image_response)}")

204
litellm/llms/aiml/image_generation/transformation.py Normal file
View file

@ -0,0 +1,204 @@
from typing import TYPE_CHECKING, Any, List, Optional
import httpx
from litellm.llms.base_llm.image_generation.transformation import (
BaseImageGenerationConfig,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.aiml import AimlImageGenerationRequestParams
from litellm.types.llms.openai import (
AllMessageValues,
OpenAIImageGenerationOptionalParams,
)
from litellm.types.utils import ImageObject, ImageResponse
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
LiteLLMLoggingObj = Any
class AimlImageGenerationConfig(BaseImageGenerationConfig):
DEFAULT_BASE_URL: str = "https://api.aimlapi.com"
IMAGE_GENERATION_ENDPOINT: str = "v1/images/generations"
def get_supported_openai_params(
self, model: str
) -> List[OpenAIImageGenerationOptionalParams]:
"""
https://api.aimlapi.com/v1/images/generations
"""
return [
"n",
"response_format",
"size"
]
def map_openai_params(
self,
non_default_params: dict,
optional_params: dict,
model: str,
drop_params: bool,
) -> dict:
supported_params = self.get_supported_openai_params(model)
for k in non_default_params.keys():
if k not in optional_params.keys():
if k in supported_params:
# Map OpenAI params to AI/ML params
if k == "n":
optional_params["num_images"] = non_default_params[k]
elif k == "response_format":
optional_params["output_format"] = non_default_params[k]
elif k == "size":
# Map OpenAI size format to AI/ML image_size
size_value = non_default_params[k]
if isinstance(size_value, str):
# Handle standard OpenAI sizes like "1024x1024"
if "x" in size_value:
width, height = map(int, size_value.split("x"))
optional_params["image_size"] = {"width": width, "height": height}
else:
# Pass through predefined sizes
optional_params["image_size"] = size_value
else:
optional_params["image_size"] = size_value
else:
optional_params[k] = non_default_params[k]
elif drop_params:
pass
else:
raise ValueError(
f"Parameter {k} is not supported for model {model}. Supported parameters are {supported_params}. Set drop_params=True to drop unsupported parameters."
)
return optional_params
def get_complete_url(
self,
api_base: Optional[str],
api_key: Optional[str],
model: str,
optional_params: dict,
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
"""
Get the complete url for the request
"""
complete_url: str = (
api_base
or get_secret_str("AIML_API_BASE")
or self.DEFAULT_BASE_URL
)
complete_url = complete_url.rstrip("/")
complete_url = f"{complete_url}/{self.IMAGE_GENERATION_ENDPOINT}"
return complete_url
def validate_environment(
self,
headers: dict,
model: str,
messages: List[AllMessageValues],
optional_params: dict,
litellm_params: dict,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
) -> dict:
final_api_key: Optional[str] = (
api_key or
get_secret_str("AIML_API_KEY") or
get_secret_str("AIMLAPI_KEY") # Alternative name
)
if not final_api_key:
raise ValueError("AIML_API_KEY or AIMLAPI_KEY is not set")
headers["Authorization"] = f"Bearer {final_api_key}"
headers["Content-Type"] = "application/json"
return headers
def transform_image_generation_request(
self,
model: str,
prompt: str,
optional_params: dict,
litellm_params: dict,
headers: dict,
) -> dict:
"""
Transform the image generation request to the AI/ML flux image generation request body
https://api.aimlapi.com/v1/images/generations
"""
aiml_image_generation_request_body: AimlImageGenerationRequestParams = AimlImageGenerationRequestParams(
prompt=prompt,
model=model,
**optional_params,
)
return dict(aiml_image_generation_request_body)
def transform_image_generation_response(
self,
model: str,
raw_response: httpx.Response,
model_response: ImageResponse,
logging_obj: LiteLLMLoggingObj,
request_data: dict,
optional_params: dict,
litellm_params: dict,
encoding: Any,
api_key: Optional[str] = None,
json_mode: Optional[bool] = None,
) -> ImageResponse:
"""
Transform the image generation response to the litellm image response
https://api.aimlapi.com/v1/images/generations
"""
try:
response_data = raw_response.json()
except Exception as e:
raise self.get_error_class(
error_message=f"Error transforming image generation response: {e}",
status_code=raw_response.status_code,
headers=raw_response.headers,
)
if not model_response.data:
model_response.data = []
# AI/ML API can return images in two different formats:
# 1. output.choices array with image_base64
# 2. images array with url (and optional width, height, content_type)
if "output" in response_data and "choices" in response_data["output"]:
for choice in response_data["output"]["choices"]:
if "image_base64" in choice:
model_response.data.append(ImageObject(
b64_json=choice["image_base64"],
url=None, # AI/ML API returns base64, not URLs
))
elif "url" in choice:
model_response.data.append(ImageObject(
b64_json=None,
url=choice["url"],
))
elif "images" in response_data:
# Handle alternative format: {"images": [{"url": "...", "width": 1024, "height": 768, "content_type": "image/jpeg"}]}
for image in response_data["images"]:
if "url" in image:
model_response.data.append(ImageObject(
b64_json=None,
url=image["url"],
))
elif "image_base64" in image:
model_response.data.append(ImageObject(
b64_json=image["image_base64"],
url=None,
))
return model_response

42
litellm/llms/anthropic/common_utils.py
View file

@ -10,10 +10,11 @@ import litellm
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_file_ids_from_messages,
)
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.llms.anthropic import AllAnthropicToolsValues, AnthropicMcpServerTool
from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import TokenCountResponse
class AnthropicError(BaseLLMException):
@ -229,7 +230,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
litellm_model_names.append(litellm_model_name)
return litellm_model_names
def get_token_counter(self) -> Optional["AnthropicTokenCounter"]:
def get_token_counter(self) -> Optional[BaseTokenCounter]:
"""
Factory method to create an Anthropic token counter.
@ -239,32 +240,24 @@ class AnthropicModelInfo(BaseLLMModelInfo):
return AnthropicTokenCounter()
class AnthropicTokenCounter:
class AnthropicTokenCounter(BaseTokenCounter):
"""Token counter implementation for Anthropic provider."""
def supports_provider(
def should_use_token_counting_api(
self,
deployment: Optional[Dict[str, Any]] = None,
from_endpoint: bool = False
custom_llm_provider: Optional[str] = None,
) -> bool:
if not from_endpoint:
return False
if deployment is None:
return False
full_model = deployment.get("litellm_params", {}).get("model", "")
is_anthropic_provider = full_model.startswith("anthropic/") or "anthropic" in full_model.lower()
return is_anthropic_provider
from litellm.types.utils import LlmProviders
return custom_llm_provider == LlmProviders.ANTHROPIC.value
async def count_tokens(
self,
model_to_use: str,
messages: Optional[List[Dict[str, Any]]],
contents: Optional[List[Dict[str, Any]]],
deployment: Optional[Dict[str, Any]] = None,
request_model: str = "",
) -> Optional[Dict[str, Any]]:
) -> Optional[TokenCountResponse]:
from litellm.proxy.utils import count_tokens_with_anthropic_api
result = await count_tokens_with_anthropic_api(
@ -274,12 +267,13 @@ class AnthropicTokenCounter:
)
if result is not None:
return {
"total_tokens": result["total_tokens"],
"request_model": request_model,
"model_used": model_to_use,
"tokenizer_type": result["tokenizer_used"],
}
return TokenCountResponse(
total_tokens=result.get("total_tokens", 0),
request_model=request_model,
model_used=model_to_use,
tokenizer_type=result.get("tokenizer_used", ""),
original_response=result,
)
return None

3
litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py
View file

@ -489,7 +489,7 @@ class LiteLLMAnthropicMessagesAdapter:
text: str = ""
partial_json: Optional[str] = None
for choice in choices:
if choice.delta.content is not None:
if choice.delta.content is not None and len(choice.delta.content) > 0:
text += choice.delta.content
elif choice.delta.tool_calls is not None:
partial_json = ""
@ -499,7 +499,6 @@ class LiteLLMAnthropicMessagesAdapter:
and tool.function.arguments is not None
):
partial_json += tool.function.arguments
if partial_json is not None:
return "input_json_delta", ContentJsonBlockDelta(
type="input_json_delta", partial_json=partial_json

8
litellm/llms/azure/azure.py
View file

@ -230,6 +230,14 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
)
data = {"model": None, "messages": messages, **optional_params}
elif litellm.AzureOpenAIGPT5Config.is_model_gpt_5_model(model=model):
data = litellm.AzureOpenAIGPT5Config().transform_request(
model=model,
messages=messages,
optional_params=optional_params,
litellm_params=litellm_params,
headers=headers or {},
)
else:
data = litellm.AzureOpenAIConfig().transform_request(
model=model,

Some files were not shown because too many files have changed in this diff Show more