scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-07 02:59:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

Merge remote-tracking branch 'origin' into litellm_deleted_keys_team

Browse source
This commit is contained in:
yuneng-jiang 2025-12-19 11:36:27 -08:00
commit 2a4f883e78
163 changed files with 10546 additions and 433 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
.circleci/config.yml
View file

@ -657,7 +657,7 @@ jobs:
docker run -d \
--name postgres-db \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_PASSWORD=test-postgres \
-e POSTGRES_DB=circle_test \
-p 5432:5432 \
postgres:14
@ -2108,7 +2108,7 @@ jobs:
docker run -d \
--name postgres-db \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_PASSWORD=test-postgres \
-e POSTGRES_DB=circle_test \
-p 5432:5432 \
postgres:14
@ -2250,7 +2250,7 @@ jobs:
docker run -d \
--name postgres-db \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_PASSWORD=test-postgres \
-e POSTGRES_DB=circle_test \
-p 5432:5432 \
postgres:14
@ -2390,7 +2390,7 @@ jobs:
docker run -d \
--name postgres-db \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_PASSWORD=test-postgres \
-e POSTGRES_DB=circle_test \
-p 5432:5432 \
postgres:14
@ -2551,7 +2551,7 @@ jobs:
docker run -d \
--name postgres-db \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_PASSWORD=test-postgres \
-e POSTGRES_DB=circle_test \
-p 5432:5432 \
postgres:14
@ -2664,7 +2664,7 @@ jobs:
docker run -d \
--name postgres-db \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_PASSWORD=test-postgres \
-e POSTGRES_DB=circle_test \
-p 5432:5432 \
postgres:14
@ -2800,7 +2800,7 @@ jobs:
docker run -d \
--name postgres-db \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_PASSWORD=test-postgres \
-e POSTGRES_DB=circle_test \
-p 5432:5432 \
postgres:14
@ -3032,7 +3032,7 @@ jobs:
docker run -d \
--name postgres-db \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_PASSWORD=test-postgres \
-e POSTGRES_DB=circle_test \
-p 5432:5432 \
postgres:14
@ -3549,7 +3549,7 @@ jobs:
docker run -d \
--name postgres-db \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_PASSWORD=test-postgres \
-e POSTGRES_DB=circle_test \
-p 5432:5432 \
postgres:14

84
.gitguardian.yaml Normal file
View file

@ -0,0 +1,84 @@
version: 2
secret:
# Exclude files and paths by globbing
ignored_paths:
- "**/*.whl"
- "**/*.pyc"
- "**/__pycache__/**"
- "**/node_modules/**"
- "**/dist/**"
- "**/build/**"
- "**/.git/**"
- "**/venv/**"
- "**/.venv/**"
# Large data/metadata files that don't need scanning
- "**/model_prices_and_context_window*.json"
- "**/*_metadata/*.txt"
- "**/tokenizers/*.json"
- "**/tokenizers/*"
- "miniconda.sh"
# Build outputs and static assets
- "litellm/proxy/_experimental/out/**"
- "ui/litellm-dashboard/public/**"
- "**/swagger/*.js"
- "**/*.woff"
- "**/*.woff2"
- "**/*.avif"
- "**/*.webp"
# Test data files
- "**/tests/**/data_map.txt"
- "tests/**/*.txt"
# Documentation and other non-code files
- "docs/**"
- "**/*.md"
- "**/*.lock"
- "poetry.lock"
- "package-lock.json"
# Ignore security incidents with the SHA256 of the occurrence (false positives)
ignored_matches:
# === Current detected false positives (SHA-based) ===
# gcs_pub_sub_body - folder name, not a password
- name: GCS pub/sub test folder name
match: 75f377c456eede69e5f6e47399ccee6016a2a93cc5dd11db09cc5b1359ae569a
# os.environ/APORIA_API_KEY_1 - environment variable reference
- name: Environment variable reference APORIA_API_KEY_1
match: e2ddeb8b88eca97a402559a2be2117764e11c074d86159ef9ad2375dea188094
# os.environ/APORIA_API_KEY_2 - environment variable reference
- name: Environment variable reference APORIA_API_KEY_2
match: 09aa39a29e050b86603aa55138af1ff08fb86a4582aa965c1bd0672e1575e052
# oidc/circleci_v2/ - test authentication path, not a secret
- name: OIDC CircleCI test path
match: feb3475e1f89a65b7b7815ac4ec597e18a9ec1847742ad445c36ca617b536e15
# text-davinci-003 - OpenAI model identifier, not a secret
- name: OpenAI model identifier text-davinci-003
match: c489000cf6c7600cee0eefb80ad0965f82921cfb47ece880930eb7e7635cf1f1
# === Preventive patterns for test keys (pattern-based) ===
# Test API keys (124 instances across 45 files)
- name: Test API keys with sk-test prefix
match: sk-test-
# Mock API keys
- name: Mock API keys with sk-mock prefix
match: sk-mock-
# Fake API keys
- name: Fake API keys with sk-fake prefix
match: sk-fake-
# Generic test API key patterns
- name: Test API key patterns
match: test-api-key

2
.github/workflows/locustfile.py vendored
View file

@ -8,7 +8,7 @@ class MyUser(HttpUser):
def chat_completion(self):
headers = {
"Content-Type": "application/json",
"Authorization": "Bearer sk-8N1tLOOyH8TIxwOLahhIVg",
"Authorization": "Bearer sk-test-load-test-key-123",
# Include any additional headers you may need for authentication, etc.
}

4
.github/workflows/publish-migrations.yml vendored
View file

@ -20,7 +20,7 @@ jobs:
env:
POSTGRES_DB: temp_db
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_PASSWORD: test-postgres
ports:
- 5432:5432
options: >-
@ -35,7 +35,7 @@ jobs:
env:
POSTGRES_DB: shadow_db
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_PASSWORD: test-postgres
ports:
- 5433:5432
options: >-

1
README.md
View file

@ -266,6 +266,7 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature
| [AI21 (`ai21`)](https://docs.litellm.ai/docs/providers/ai21) | ✅ | ✅ | ✅ | | | | | | | |
| [AI21 Chat (`ai21_chat`)](https://docs.litellm.ai/docs/providers/ai21) | ✅ | ✅ | ✅ | | | | | | | |
| [Aleph Alpha](https://docs.litellm.ai/docs/providers/aleph_alpha) | ✅ | ✅ | ✅ | | | | | | | |
| [Amazon Nova](https://docs.litellm.ai/docs/providers/amazon_nova) | ✅ | ✅ | ✅ | | | | | | | |
| [Anthropic (`anthropic`)](https://docs.litellm.ai/docs/providers/anthropic) | ✅ | ✅ | ✅ | | | | | | ✅ | |
| [Anthropic Text (`anthropic_text`)](https://docs.litellm.ai/docs/providers/anthropic) | ✅ | ✅ | ✅ | | | | | | ✅ | |
| [Anyscale](https://docs.litellm.ai/docs/providers/anyscale) | ✅ | ✅ | ✅ | | | | | | | |

40
ci_cd/TEST_KEY_PATTERNS.md Normal file
View file

@ -0,0 +1,40 @@
# Test Key Patterns Standard
Standard patterns for test/mock keys and credentials in the LiteLLM codebase to avoid triggering secret detection.
## How GitGuardian Works
GitGuardian uses **machine learning and entropy analysis**, not just pattern matching:
- **Low entropy** values (like `sk-1234`, `postgres`) are automatically ignored
- **High entropy** values (realistic-looking secrets) trigger detection
- **Context-aware** detection understands code syntax like `os.environ["KEY"]`
## Recommended Test Key Patterns
### Option 1: Low Entropy Values (Simplest)
These won't trigger GitGuardian's ML detector:
```python
api_key = "sk-1234"
api_key = "sk-12345"
database_password = "postgres"
token = "test123"
```
### Option 2: High Entropy with Test Prefixes
If you need realistic-looking test keys with high entropy, use these prefixes:
```python
api_key = "sk-test-abc123def456ghi789..." # OpenAI-style test key
api_key = "sk-mock-1234567890abcdef1234..." # Mock key
api_key = "sk-fake-xyz789uvw456rst123..." # Fake key
token = "test-api-key-with-high-entropy"
```
## Configured Ignore Patterns
These patterns are in `.gitguardian.yaml` for high-entropy test keys:
- `sk-test-*` - OpenAI-style test keys
- `sk-mock-*` - Mock API keys
- `sk-fake-*` - Fake API keys
- `test-api-key` - Generic test tokens

2
cookbook/LiteLLM_PromptLayer.ipynb vendored
View file

@ -39,7 +39,7 @@
"import os\n",
"os.environ['OPENAI_API_KEY'] = \"\"\n",
"os.environ['REPLICATE_API_TOKEN'] = \"\"\n",
"os.environ['PROMPTLAYER_API_KEY'] = \"pl_4ea2bb00a4dca1b8a70cebf2e9e11564\"\n",
"os.environ['PROMPTLAYER_API_KEY'] = \"test-promptlayer-key-123\"\n",
"\n",
"# Set Promptlayer as a success callback\n",
"litellm.success_callback =['promptlayer']\n",

278
cookbook/Migrating_to_LiteLLM_Proxy_from_OpenAI_Azure_OpenAI.ipynb vendored
View file

@ -1,21 +1,10 @@
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
},
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "kccfk0mHZ4Ad"
},
"source": [
"# Migrating to LiteLLM Proxy from OpenAI/Azure OpenAI\n",
"\n",
@ -32,29 +21,26 @@
"To pass provider-specific args, [go here](https://docs.litellm.ai/docs/completion/provider_specific_params#proxy-usage)\n",
"\n",
"To drop unsupported params (E.g. frequency_penalty for bedrock with librechat), [go here](https://docs.litellm.ai/docs/completion/drop_params#openai-proxy-usage)\n"
],
"metadata": {
"id": "kccfk0mHZ4Ad"
}
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "nmSClzCPaGH6"
},
"source": [
"## /chat/completion\n",
"\n"
],
"metadata": {
"id": "nmSClzCPaGH6"
}
]
},
{
"cell_type": "markdown",
"source": [
"### OpenAI Python SDK"
],
"metadata": {
"id": "_vqcjwOVaKpO"
}
},
"source": [
"### OpenAI Python SDK"
]
},
{
"cell_type": "code",
@ -94,15 +80,20 @@
},
{
"cell_type": "markdown",
"source": [
"## Function Calling"
],
"metadata": {
"id": "AqkyKk9Scxgj"
}
},
"source": [
"## Function Calling"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "wDg10VqLczE1"
},
"outputs": [],
"source": [
"from openai import OpenAI\n",
"client = OpenAI(\n",
@ -139,24 +130,24 @@
")\n",
"\n",
"print(completion)\n"
],
"metadata": {
"id": "wDg10VqLczE1"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
"source": [
"### Azure OpenAI Python SDK"
],
"metadata": {
"id": "YYoxLloSaNWW"
}
},
"source": [
"### Azure OpenAI Python SDK"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "yA1XcgowaSRy"
},
"outputs": [],
"source": [
"import openai\n",
"client = openai.AzureOpenAI(\n",
@ -184,24 +175,24 @@
")\n",
"\n",
"print(response)"
],
"metadata": {
"id": "yA1XcgowaSRy"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
"source": [
"### Langchain Python"
],
"metadata": {
"id": "yl9qhDvnaTpL"
}
},
"source": [
"### Langchain Python"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "5MUZgSquaW5t"
},
"outputs": [],
"source": [
"from langchain.chat_models import ChatOpenAI\n",
"from langchain.prompts.chat import (\n",
@ -239,24 +230,22 @@
"response = chat(messages)\n",
"\n",
"print(response)"
],
"metadata": {
"id": "5MUZgSquaW5t"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
"source": [
"### Curl"
],
"metadata": {
"id": "B9eMgnULbRaz"
}
},
"source": [
"### Curl"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "VWCCk5PFcmhS"
},
"source": [
"\n",
"\n",
@ -280,22 +269,24 @@
"}'\n",
"```\n",
"\n"
],
"metadata": {
"id": "VWCCk5PFcmhS"
}
]
},
{
"cell_type": "markdown",
"source": [
"### LlamaIndex"
],
"metadata": {
"id": "drBAm2e1b6xe"
}
},
"source": [
"### LlamaIndex"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "d0bZcv8fb9mL"
},
"outputs": [],
"source": [
"import os, dotenv\n",
"\n",
@ -326,24 +317,24 @@
"query_engine = index.as_query_engine()\n",
"response = query_engine.query(\"What did the author do growing up?\")\n",
"print(response)\n"
],
"metadata": {
"id": "d0bZcv8fb9mL"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
"source": [
"### Langchain JS"
],
"metadata": {
"id": "xypvNdHnb-Yy"
}
},
"source": [
"### Langchain JS"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "R55mK2vCcBN2"
},
"outputs": [],
"source": [
"import { ChatOpenAI } from \"@langchain/openai\";\n",
"\n",
@ -359,24 +350,24 @@
"const message = await model.invoke(\"Hi there!\");\n",
"\n",
"console.log(message);\n"
],
"metadata": {
"id": "R55mK2vCcBN2"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
"source": [
"### OpenAI JS"
],
"metadata": {
"id": "nC4bLifCcCiW"
}
},
"source": [
"### OpenAI JS"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "MICH8kIMcFpg"
},
"outputs": [],
"source": [
"const { OpenAI } = require('openai');\n",
"\n",
@ -398,24 +389,24 @@
"}\n",
"\n",
"main();\n"
],
"metadata": {
"id": "MICH8kIMcFpg"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
"source": [
"### Anthropic SDK"
],
"metadata": {
"id": "D1Q07pEAcGTb"
}
},
"source": [
"### Anthropic SDK"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "qBjFcAvgcI3t"
},
"outputs": [],
"source": [
"import os\n",
"\n",
@ -423,7 +414,7 @@
"\n",
"client = Anthropic(\n",
" base_url=\"http://localhost:4000\", # proxy endpoint\n",
" api_key=\"sk-s4xN1IiLTCytwtZFJaYQrA\", # litellm proxy virtual key\n",
" api_key=\"sk-test-proxy-key-123\", # litellm proxy virtual key (example)\n",
")\n",
"\n",
"message = client.messages.create(\n",
@ -437,33 +428,33 @@
" model=\"claude-3-opus-20240229\",\n",
")\n",
"print(message.content)"
],
"metadata": {
"id": "qBjFcAvgcI3t"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
"source": [
"## /embeddings"
],
"metadata": {
"id": "dFAR4AJGcONI"
}
},
"source": [
"## /embeddings"
]
},
{
"cell_type": "markdown",
"source": [
"### OpenAI Python SDK"
],
"metadata": {
"id": "lgNoM281cRzR"
}
},
"source": [
"### OpenAI Python SDK"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "NY3DJhPfcQhA"
},
"outputs": [],
"source": [
"import openai\n",
"from openai import OpenAI\n",
@ -478,24 +469,24 @@
")\n",
"\n",
"print(response)\n"
],
"metadata": {
"id": "NY3DJhPfcQhA"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
"source": [
"### Langchain Embeddings"
],
"metadata": {
"id": "hmbg-DW6cUZs"
}
},
"source": [
"### Langchain Embeddings"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "lX2S8Nl1cWVP"
},
"outputs": [],
"source": [
"from langchain.embeddings import OpenAIEmbeddings\n",
"\n",
@ -526,24 +517,22 @@
"\n",
"print(f\"TITAN EMBEDDINGS\")\n",
"print(query_result[:5])"
],
"metadata": {
"id": "lX2S8Nl1cWVP"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
"source": [
"### Curl Request"
],
"metadata": {
"id": "oqGbWBCQcYfd"
}
},
"source": [
"### Curl Request"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "7rkIMV9LcdwQ"
},
"source": [
"\n",
"\n",
@ -556,10 +545,21 @@
" }'\n",
"```\n",
"\n"
],
"metadata": {
"id": "7rkIMV9LcdwQ"
}
]
}
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 0
}

9
docs/my-website/blog/gemini_3_flash/index.md
View file

@ -27,6 +27,10 @@ import TabItem from '@theme/TabItem';
LiteLLM now supports `gemini-3-flash-preview` and all the new API changes along with it.
:::note
If you only want cost tracking, you need no change in your current Litellm version. But if you want the support for new features introduced along with it like thinking levels, you will need to use v1.80.8-stable.1 or above.
:::
## Deploy this version
<Tabs>
@ -232,6 +236,11 @@ response = completion(
print(response)
```
:::note
If using this model via vertex_ai, keep the location as global as this is the only supported location as of now.
:::
## `reasoning_effort` Mapping for Gemini 3+
| reasoning_effort | thinking_level |

2
docs/my-website/docs/image_edits.md
View file

@ -16,7 +16,7 @@ LiteLLM provides image editing functionality that maps to OpenAI's `/images/edit
| Supported operations | Create image edits | Single and multiple images supported |
| Supported LiteLLM SDK Versions | 1.63.8+ | Gemini support requires 1.79.3+ |
| Supported LiteLLM Proxy Versions | 1.71.1+ | Gemini support requires 1.79.3+ |
| Supported LLM providers | **OpenAI**, **Gemini (Google AI Studio)**, **Vertex AI** | Gemini supports the new `gemini-2.5-flash-image` family. Vertex AI supports both Gemini and Imagen models. |
| Supported LLM providers | **OpenAI**, **Gemini (Google AI Studio)**, **Vertex AI**, **Stability AI**, **AWS Bedrock (Stability)** | Gemini supports the new `gemini-2.5-flash-image` family. Vertex AI supports both Gemini and Imagen models. Stability AI and Bedrock Stability support various image editing operations. |
#### ⚡️See all supported models and providers at [models.litellm.ai](https://models.litellm.ai/)

2
docs/my-website/docs/oidc.md
View file

@ -106,7 +106,7 @@ model_list:
aws_region_name: us-west-2
aws_session_name: "my-test-session"
aws_role_name: "arn:aws:iam::335785316107:role/litellm-github-unit-tests-circleci"
aws_web_identity_token: "oidc/circleci_v2/"
aws_web_identity_token: "oidc/example-provider/"
```
#### Amazon IAM Role Configuration for CircleCI v2 -> Bedrock

53
docs/my-website/docs/providers/openai/responses_api.md
View file

@ -623,6 +623,58 @@ display(styled_df)
</TabItem>
</Tabs>
## Function Calling
```python showLineNumbers title="Function Calling with Parallel Tool Calls"
import litellm
import json
tools = [
{
"type": "function",
"name": "get_weather",
"description": "Get current weather for a location",
"parameters": {
"type": "object",
"properties": {
"location": {"type": "string"}
},
"required": ["location"]
}
}
]
# Step 1: Request with tools (parallel_tool_calls=True allows multiple calls)
response = litellm.responses(
model="openai/gpt-4o",
input=[{"role": "user", "content": "What's the weather in Paris and Tokyo?"}],
tools=tools,
parallel_tool_calls=True, # Defaults = True
)
# Step 2: Execute tool calls and collect results
tool_results = []
for output in response.output:
if output.type == "function_call":
result = {"temperature": 15, "condition": "sunny"} # Your function logic here
tool_results.append({
"type": "function_call_output",
"call_id": output.call_id,
"output": json.dumps(result)
})
# Step 3: Send results back
final_response = litellm.responses(
model="openai/gpt-4o",
input=tool_results,
tools=tools,
)
print(final_response.output)
```
Set `parallel_tool_calls=False` to ensure zero or one tool is called per turn. [More details](https://platform.openai.com/docs/guides/function-calling#parallel-function-calling).
## Free-form Function Calling
<Tabs>
@ -633,7 +685,6 @@ display(styled_df)
import litellm
response = litellm.responses(
response = client.responses.create(
model="gpt-5-mini",
input="Please use the code_exec tool to calculate the area of a circle with radius equal to the number of 'r's in strawberry",
text={"format": {"type": "text"}},

284
docs/my-website/docs/providers/stability.md
View file

@ -8,7 +8,7 @@ https://stability.ai/
| Description | Stability AI creates open AI models for image, video, audio, and 3D generation. Known for Stable Diffusion. |
| Provider Route on LiteLLM | `stability/` |
| Link to Provider Doc | [Stability AI API ↗](https://platform.stability.ai/docs/api-reference) |
| Supported Operations | [`/images/generations`](#image-generation) |
| Supported Operations | [`/images/generations`](#image-generation), [`/images/edits`](#image-editing) |
LiteLLM supports Stability AI Image Generation calls via the Stability AI REST API (not via Bedrock).
@ -169,13 +169,285 @@ Stability AI returns images in base64 format. The response is OpenAI-compatible:
}
```
## Comparing with Bedrock
## Image Editing
Stability AI supports various image editing operations including inpainting, upscaling, outpainting, background removal, and more.
### Usage - LiteLLM Python SDK
#### Inpainting (Edit with Mask)
```python showLineNumbers
from litellm import image_edit
import os
os.environ['STABILITY_API_KEY'] = "your-api-key"
# Inpainting - edit specific areas using a mask
response = image_edit(
model="stability/stable-image-inpaint-v1:0",
image=open("original_image.png", "rb"),
mask=open("mask_image.png", "rb"),
prompt="Add a beautiful sunset in the masked area",
size="1024x1024",
)
print(response)
```
#### Image Upscaling
```python showLineNumbers
from litellm import image_edit
import os
os.environ['STABILITY_API_KEY'] = "your-api-key"
# Conservative upscaling - preserves details
response = image_edit(
model="stability/stable-conservative-upscale-v1:0",
image=open("low_res_image.png", "rb"),
prompt="Upscale this image while preserving details",
)
# Creative upscaling - adds creative details
response = image_edit(
model="stability/stable-creative-upscale-v1:0",
image=open("low_res_image.png", "rb"),
prompt="Upscale and enhance with creative details",
creativity=0.3, # 0-0.35, higher = more creative
)
# Fast upscaling - quick upscaling
response = image_edit(
model="stability/stable-fast-upscale-v1:0",
image=open("low_res_image.png", "rb"),
prompt="Quickly upscale this image",
)
print(response)
```
#### Image Outpainting
```python showLineNumbers
from litellm import image_edit
import os
os.environ['STABILITY_API_KEY'] = "your-api-key"
# Extend image beyond its borders
response = image_edit(
model="stability/stable-outpaint-v1:0",
image=open("original_image.png", "rb"),
prompt="Extend this landscape with mountains",
left=100, # Pixels to extend on the left
right=100, # Pixels to extend on the right
up=50, # Pixels to extend on top
down=50, # Pixels to extend on bottom
)
print(response)
```
#### Background Removal
```python showLineNumbers
from litellm import image_edit
import os
os.environ['STABILITY_API_KEY'] = "your-api-key"
# Remove background from image
response = image_edit(
model="stability/stable-image-remove-background-v1:0",
image=open("portrait.png", "rb"),
prompt="Remove the background",
)
print(response)
```
#### Search and Replace
```python showLineNumbers
from litellm import image_edit
import os
os.environ['STABILITY_API_KEY'] = "your-api-key"
# Search and replace objects in image
response = image_edit(
model="stability/stable-image-search-replace-v1:0",
image=open("scene.png", "rb"),
prompt="A red sports car",
search_prompt="blue sedan", # What to replace
)
# Search and recolor
response = image_edit(
model="stability/stable-image-search-recolor-v1:0",
image=open("scene.png", "rb"),
prompt="Make it golden yellow",
select_prompt="the car", # What to recolor
)
print(response)
```
#### Image Control (Sketch/Structure)
```python showLineNumbers
from litellm import image_edit
import os
os.environ['STABILITY_API_KEY'] = "your-api-key"
# Control with sketch
response = image_edit(
model="stability/stable-image-control-sketch-v1:0",
image=open("sketch.png", "rb"),
prompt="Turn this sketch into a realistic photo",
control_strength=0.7, # 0-1, higher = more control
)
# Control with structure
response = image_edit(
model="stability/stable-image-control-structure-v1:0",
image=open("structure_reference.png", "rb"),
prompt="Generate image following this structure",
control_strength=0.7,
)
print(response)
```
#### Erase Objects
```python showLineNumbers
from litellm import image_edit
import os
os.environ['STABILITY_API_KEY'] = "your-api-key"
# Erase objects from image
response = image_edit(
model="stability/stable-image-erase-object-v1:0",
image=open("scene.png", "rb"),
mask=open("object_mask.png", "rb"), # Mask the object to erase
prompt="Remove the object",
)
print(response)
```
### Supported Image Edit Models
| Model Name | Function Call | Description |
|------------|---------------|-------------|
| stable-image-inpaint-v1:0 | `image_edit(model="stability/stable-image-inpaint-v1:0", ...)` | Inpainting with mask |
| stable-conservative-upscale-v1:0 | `image_edit(model="stability/stable-conservative-upscale-v1:0", ...)` | Conservative upscaling |
| stable-creative-upscale-v1:0 | `image_edit(model="stability/stable-creative-upscale-v1:0", ...)` | Creative upscaling |
| stable-fast-upscale-v1:0 | `image_edit(model="stability/stable-fast-upscale-v1:0", ...)` | Fast upscaling |
| stable-outpaint-v1:0 | `image_edit(model="stability/stable-outpaint-v1:0", ...)` | Extend image borders |
| stable-image-remove-background-v1:0 | `image_edit(model="stability/stable-image-remove-background-v1:0", ...)` | Remove background |
| stable-image-search-replace-v1:0 | `image_edit(model="stability/stable-image-search-replace-v1:0", ...)` | Search and replace objects |
| stable-image-search-recolor-v1:0 | `image_edit(model="stability/stable-image-search-recolor-v1:0", ...)` | Search and recolor |
| stable-image-control-sketch-v1:0 | `image_edit(model="stability/stable-image-control-sketch-v1:0", ...)` | Control with sketch |
| stable-image-control-structure-v1:0 | `image_edit(model="stability/stable-image-control-structure-v1:0", ...)` | Control with structure |
| stable-image-erase-object-v1:0 | `image_edit(model="stability/stable-image-erase-object-v1:0", ...)` | Erase objects |
| stable-image-style-guide-v1:0 | `image_edit(model="stability/stable-image-style-guide-v1:0", ...)` | Apply style guide |
| stable-style-transfer-v1:0 | `image_edit(model="stability/stable-style-transfer-v1:0", ...)` | Transfer style |
### Usage - LiteLLM Proxy Server
#### 1. Setup config.yaml
```yaml showLineNumbers
model_list:
- model_name: stability-inpaint
litellm_params:
model: stability/stable-image-inpaint-v1:0
api_key: os.environ/STABILITY_API_KEY
model_info:
mode: image_edit
- model_name: stability-upscale
litellm_params:
model: stability/stable-conservative-upscale-v1:0
api_key: os.environ/STABILITY_API_KEY
model_info:
mode: image_edit
general_settings:
master_key: sk-1234
```
#### 2. Start the proxy
```bash showLineNumbers
litellm --config config.yaml
# RUNNING on http://0.0.0.0:4000
```
#### 3. Test it
```bash showLineNumbers
curl -X POST "http://0.0.0.0:4000/v1/images/edits" \
-H "Authorization: Bearer sk-1234" \
-F "model=stability-inpaint" \
-F "image=@original_image.png" \
-F "mask=@mask_image.png" \
-F "prompt=Add a beautiful garden in the masked area"
```
## AWS Bedrock (Stability)
LiteLLM also supports Stability AI models via AWS Bedrock. This is useful if you're already using AWS infrastructure.
### Usage - Bedrock Stability
```python showLineNumbers
from litellm import image_edit
import os
# Set AWS credentials
os.environ["AWS_ACCESS_KEY_ID"] = "your-access-key"
os.environ["AWS_SECRET_ACCESS_KEY"] = "your-secret-key"
os.environ["AWS_REGION_NAME"] = "us-east-1"
# Bedrock Stability inpainting
response = image_edit(
model="bedrock/us.stability.stable-image-inpaint-v1:0",
image=open("original_image.png", "rb"),
mask=open("mask_image.png", "rb"),
prompt="Add flowers in the masked area",
size="1024x1024",
)
print(response)
```
### Supported Bedrock Stability Models
All Stability AI image edit models are available via Bedrock with the `bedrock/` prefix:
| Direct API Model | Bedrock Model | Description |
|------------------|---------------|-------------|
| stability/stable-image-inpaint-v1:0 | bedrock/us.stability.stable-image-inpaint-v1:0 | Inpainting |
| stability/stable-conservative-upscale-v1:0 | bedrock/stability.stable-conservative-upscale-v1:0 | Conservative upscaling |
| stability/stable-creative-upscale-v1:0 | bedrock/stability.stable-creative-upscale-v1:0 | Creative upscaling |
| stability/stable-fast-upscale-v1:0 | bedrock/stability.stable-fast-upscale-v1:0 | Fast upscaling |
| stability/stable-outpaint-v1:0 | bedrock/stability.stable-outpaint-v1:0 | Outpainting |
| stability/stable-image-remove-background-v1:0 | bedrock/stability.stable-image-remove-background-v1:0 | Remove background |
| stability/stable-image-search-replace-v1:0 | bedrock/stability.stable-image-search-replace-v1:0 | Search and replace |
| stability/stable-image-search-recolor-v1:0 | bedrock/stability.stable-image-search-recolor-v1:0 | Search and recolor |
| stability/stable-image-control-sketch-v1:0 | bedrock/stability.stable-image-control-sketch-v1:0 | Control with sketch |
| stability/stable-image-control-structure-v1:0 | bedrock/stability.stable-image-control-structure-v1:0 | Control with structure |
| stability/stable-image-erase-object-v1:0 | bedrock/stability.stable-image-erase-object-v1:0 | Erase objects |
**Note:** Bedrock model IDs may use `us.stability.*` or `stability.*` prefix depending on the region and model.
## Comparing Routes
LiteLLM supports Stability AI models via two routes:
| Route | Provider | Use Case |
|-------|----------|----------|
| `stability/` | Stability AI Direct API | Direct access, all latest models |
| `bedrock/stability.*` | AWS Bedrock | AWS integration, enterprise features |
| Route | Provider | Use Case | Image Generation | Image Editing |
|-------|----------|----------|------------------|---------------|
| `stability/` | Stability AI Direct API | Direct access, all latest models | ✅ | ✅ |
| `bedrock/stability.*` | AWS Bedrock | AWS integration, enterprise features | ✅ | ✅ |
Use `stability/` for direct API access. Use `bedrock/stability.*` if you're already using AWS Bedrock.

7
docs/my-website/docs/providers/vertex_ocr.md
View file

@ -140,7 +140,7 @@ with open("document.pdf", "rb") as f:
pdf_base64 = base64.b64encode(f.read()).decode()
response = litellm.ocr(
model="vertex_ai/mistral-ocr-2505",
model="vertex_ai/mistral-ocr-2505", # This doesn't work for deepseek
document={
"type": "document_url",
"document_url": f"data:application/pdf;base64,{pdf_base64}"
@ -219,7 +219,7 @@ print(f"Cost: ${response._hidden_params.get('response_cost', 0)}")
## Important Notes
:::info URL Conversion
Vertex AI OCR endpoints don't have internet access. LiteLLM automatically converts public URLs to base64 data URIs before sending requests to Vertex AI.
Vertex AI Mistral OCR endpoints don't have internet access. LiteLLM automatically converts public URLs to base64 data URIs before sending requests to Vertex AI.
:::
:::tip Regional Availability
@ -227,11 +227,14 @@ Mistral OCR is available in multiple regions. Specify `vertex_location` to use a
- `us-central1` (default)
- `europe-west1`
- `asia-southeast1`
Deepseek OCR is only available in global region.
:::
## Supported Models
- `mistral-ocr-2505` - Latest Mistral OCR model on Vertex AI
- `deepseek-ocr-maas` - Lates Deepseek OCR model on Vertex AI
Use the Vertex AI provider prefix: `vertex_ai/<model-name>`

22
docs/my-website/docs/proxy/alerting.md
View file

@ -215,16 +215,16 @@ general_settings:
alerting: ["slack"]
alerting_threshold: 0.0001 # (Seconds) set an artificially low threshold for testing alerting
alert_to_webhook_url: {
"llm_exceptions": "https://hooks.slack.com/services/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",
"llm_too_slow": "https://hooks.slack.com/services/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",
"llm_requests_hanging": "https://hooks.slack.com/services/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",
"budget_alerts": "https://hooks.slack.com/services/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",
"db_exceptions": "https://hooks.slack.com/services/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",
"daily_reports": "https://hooks.slack.com/services/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",
"spend_reports": "https://hooks.slack.com/services/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",
"cooldown_deployment": "https://hooks.slack.com/services/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",
"new_model_added": "https://hooks.slack.com/services/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",
"outage_alerts": "https://hooks.slack.com/services/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",
"llm_exceptions": "example-slack-webhook-url",
"llm_too_slow": "example-slack-webhook-url",
"llm_requests_hanging": "example-slack-webhook-url",
"budget_alerts": "example-slack-webhook-url",
"db_exceptions": "example-slack-webhook-url",
"daily_reports": "example-slack-webhook-url",
"spend_reports": "example-slack-webhook-url",
"cooldown_deployment": "example-slack-webhook-url",
"new_model_added": "example-slack-webhook-url",
"outage_alerts": "example-slack-webhook-url",
}
litellm_settings:
@ -399,7 +399,7 @@ curl -X GET --location 'http://0.0.0.0:4000/health/services?service=webhook' \
{
"spend": 1, # the spend for the 'event_group'
"max_budget": 0, # the 'max_budget' set for the 'event_group'
"token": "88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b",
"token": "example-api-key-123",
"user_id": "default_user_id",
"team_id": null,
"user_email": null,

6
docs/my-website/docs/proxy/cost_tracking.md
View file

@ -722,7 +722,7 @@ curl -X GET 'http://localhost:4000/global/spend/report?start_date=2024-04-01&end
```shell
[
{
"api_key": "88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b",
"api_key": "example-api-key-123",
"total_cost": 0.3201286305151999,
"total_input_tokens": 36.0,
"total_output_tokens": 1593.0,
@ -766,7 +766,7 @@ curl -X GET 'http://localhost:4000/global/spend/report?start_date=2024-04-01&end
```shell
[
{
"api_key": "88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b",
"api_key": "example-api-key-123",
"total_cost": 0.00013132,
"total_input_tokens": 105.0,
"total_output_tokens": 872.0,
@ -1151,7 +1151,7 @@ curl -X GET "http://0.0.0.0:4000/spend/logs?request_id=<your-call-id" \ # e.g.:
"request_id": "chatcmpl-9ZKMURhVYSi9D6r6PJ9vLcayIK0Vm",
"call_type": "acompletion",
"metadata": {
"user_api_key": "88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b",
"user_api_key": "example-api-key-123",
"user_api_key_alias": null,
"spend_logs_metadata": { # 👈 LOGGED CUSTOM METADATA
"hello": "world"

2
docs/my-website/docs/proxy/customers.md
View file

@ -103,7 +103,7 @@ Expected Response
{
"spend": 0.0011120000000000001, # 👈 SPEND
"max_budget": null,
"token": "88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b",
"token": "example-api-key-123",
"customer_id": "krrish12", # 👈 CUSTOMER ID
"user_id": null,
"team_id": null,

2
docs/my-website/docs/proxy/guardrails/litellm_content_filter.md
View file

@ -257,7 +257,7 @@ Contact me at [EMAIL_REDACTED]
| `amex` | American Express cards | `3782-822463-10005` |
| `aws_access_key` | AWS access keys | `AKIAIOSFODNN7EXAMPLE` |
| `aws_secret_key` | AWS secret keys | `wJalrXUtnFEMI/K7MDENG/bPxRfi...` |
| `github_token` | GitHub tokens | `ghp_16C7e42F292c6912E7710c838347Ae178B4a` |
| `github_token` | GitHub tokens | `example-github-token-123` |
### Using Prebuilt Patterns

4
docs/my-website/docs/proxy/guardrails/pillar_security.md
View file

@ -790,7 +790,7 @@ curl -X POST "http://localhost:4000/v1/chat/completions" \
"messages": [
{
"role": "user",
"content": "Generate python code that accesses my Github repo using this PAT: ghp_A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8"
"content": "Generate python code that accesses my Github repo using this PAT: example-github-token-123"
}
],
"max_tokens": 50
@ -815,7 +815,7 @@ curl -X POST "http://localhost:4000/v1/chat/completions" \
"type": "github_token",
"start_idx": 66,
"end_idx": 106,
"evidence": "ghp_A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8",
"evidence": "example-github-token-123",
}
]
}

2
docs/my-website/docs/proxy/multiple_admins.md
View file

@ -89,7 +89,7 @@ curl -X POST 'http://0.0.0.0:4000/team/update' \
"id": "bd136c28-edd0-4cb6-b963-f35464cf6f5a",
"updated_at": "2024-06-08 23:41:14.793",
"changed_by": "krrish@berri.ai", # 👈 CHANGED BY
"changed_by_api_key": "88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b",
"changed_by_api_key": "example-api-key-123",
"action": "updated",
"table_name": "LiteLLM_TeamTable",
"object_id": "8bf18b11-7f52-4717-8e1f-7c65f9d01e52",

2
docs/my-website/docs/proxy/prod.md
View file

@ -33,7 +33,7 @@ litellm_settings:
Set slack webhook url in your env
```shell
export SLACK_WEBHOOK_URL="https://hooks.slack.com/services/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"
export SLACK_WEBHOOK_URL="example-slack-webhook-url"
```
Turn off FASTAPI's default info logs

2
docs/my-website/docs/proxy/quick_start.md
View file

@ -400,7 +400,7 @@ from anthropic import Anthropic
client = Anthropic(
base_url="http://localhost:4000", # proxy endpoint
api_key="sk-s4xN1IiLTCytwtZFJaYQrA", # litellm proxy virtual key
api_key="sk-test-proxy-key-123", # litellm proxy virtual key (example)
)
message = client.messages.create(

2
docs/my-website/docs/proxy/user_keys.md
View file

@ -285,7 +285,7 @@ from anthropic import Anthropic
client = Anthropic(
base_url="http://localhost:4000", # proxy endpoint
api_key="sk-s4xN1IiLTCytwtZFJaYQrA", # litellm proxy virtual key
api_key="sk-test-proxy-key-123", # litellm proxy virtual key (example)
)
message = client.messages.create(

2
docs/my-website/docs/response_api.md
View file

@ -4,7 +4,7 @@ import TabItem from '@theme/TabItem';
# /responses
LiteLLM provides a BETA endpoint in the spec of [OpenAI's `/responses` API](https://platform.openai.com/docs/api-reference/responses)
LiteLLM provides an endpoint in the spec of [OpenAI's `/responses` API](https://platform.openai.com/docs/api-reference/responses)
Requests to /chat/completions may be bridged here automatically when the provider lacks support for that endpoint. The model’s default `mode` determines how bridging works.(see `model_prices_and_context_window`)

9
docs/my-website/docs/secret_managers/hashicorp_vault.md
View file

@ -198,10 +198,13 @@ LiteLLM stores secret under the `prefix_for_stored_virtual_keys` path (default:
<Image img={require('../../img/hcorp_virtual_key.png')} />
### Team-specific overrides (proxy)
### Team-specific overrides
When running the LiteLLM proxy you can override the Vault location per team. Set
`Secret Manager Settings` on the team with the following structure:
When running the LiteLLM proxy you can override the Vault location per team. Use the [Team-Level Secret Manager Settings](./overview.md#team-level-secret-manager-settings) flow in the dashboard and configure the panel shown below:
<Image img={require('../../img/secret_manager_hashicorp_vault_settings.png')} />
Use the following structure for the JSON payload:
```json
{

25
docs/my-website/docs/secret_managers/overview.md
View file

@ -49,9 +49,28 @@ general_settings:
## Team-Level Secret Manager Settings
From the **Teams** page in the LiteLLM dashboard you can configure a secret manager per team. Open the team (or the “Create New Team” modal), find the **Secret Manager Settings** panel, and enter the provider-specific JSON configuration (e.g. `{"namespace": "admin", "mount": "secret", "path_prefix": "litellm"}`). This configuration is applied whenever LiteLLM writes secrets (e.g., storing virtual keys) on behalf of that team.
Team-level secret manager settings let every team bring their own key-management configuration. These settings are used when creating virtual keys tied to the team.
<Image img={require('../../img/secret_manager_settings.png')} />
Follow these steps to configure it:
1. **Create a team**
Open the Teams page and click `Create Team` to launch the modal.
Refer to each provider’s documentation (AWS, Azure, Google, Hashicorp, etc.) for the supported keys/values you can place inside `secret_manager_settings`.
<Image img={require('../../img/secret_manager_settings_create_team.png')} />
2. **Expand Additional Settings**
Use the `Additional Settings` toggle to reveal the advanced configuration panel.
<Image img={require('../../img/secret_manager_settings_additional_settings.png')} />
3. **Configure the Secret Manager**
In the `Secret Manager Settings` panel, paste the provider-specific JSON. Refer to each provider page (AWS, Azure, Google, Hashicorp, etc.) for the supported keys/values. JSON is required today, but we plan to add a more UI-friendly editor.
<Image img={require('../../img/secret_manager_settings.png')} />
4. **Create the team**
Review the inputs and click `Create Team` to save.
<Image img={require('../../img/secret_manager_settings_create_button.png')} />
Once saved, LiteLLM will use this configuration.

2
docs/my-website/docusaurus.config.js
View file

@ -8,7 +8,7 @@ const darkCodeTheme = require('prism-react-renderer/themes/dracula');
const inkeepConfig = {
baseSettings: {
apiKey: "0cb9c9916ec71bfe0e53c9d7f83ff046daee3fa9ef318f6a",
apiKey: "test-inkeep-api-key-123",
organizationDisplayName: 'liteLLM',
primaryBrandColor: '#4965f5',
theme: {

BIN
docs/my-website/img/secret_manager_hashicorp_vault_settings.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 107 KiB

BIN
docs/my-website/img/secret_manager_settings.png
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 117 KiB

After

Width:  |  Height:  |  Size: 680 KiB

BIN
docs/my-website/img/secret_manager_settings_additional_settings.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 683 KiB

BIN
docs/my-website/img/secret_manager_settings_create_button.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 691 KiB

BIN
docs/my-website/img/secret_manager_settings_create_team.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 524 KiB

1
docs/my-website/sidebars.js
View file

@ -671,6 +671,7 @@ const sidebars = {
"providers/ai21",
"providers/aiml",
"providers/aleph_alpha",
"providers/amazon_nova",
"providers/anyscale",
"providers/baseten",
"providers/bytez",

110
enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py Normal file
View file

@ -0,0 +1,110 @@
"""
Polls LiteLLM_ManagedObjectTable to check if the response is complete.
Cost tracking is handled automatically by litellm.aget_responses().
"""
from typing import TYPE_CHECKING
import litellm
from litellm._logging import verbose_proxy_logger
if TYPE_CHECKING:
from litellm.proxy.utils import PrismaClient, ProxyLogging
from litellm.router import Router
class CheckResponsesCost:
def __init__(
self,
proxy_logging_obj: "ProxyLogging",
prisma_client: "PrismaClient",
llm_router: "Router",
):
from litellm.proxy.utils import PrismaClient, ProxyLogging
from litellm.router import Router
self.proxy_logging_obj: ProxyLogging = proxy_logging_obj
self.prisma_client: PrismaClient = prisma_client
self.llm_router: Router = llm_router
async def check_responses_cost(self):
"""
Check if background responses are complete and track their cost.
- Get all status="queued" or "in_progress" and file_purpose="response" jobs
- Query the provider to check if response is complete
- Cost is automatically tracked by litellm.aget_responses()
- Mark completed/failed/cancelled responses as complete in the database
"""
jobs = await self.prisma_client.db.litellm_managedobjecttable.find_many(
where={
"status": {"in": ["queued", "in_progress"]},
"file_purpose": "response",
}
)
verbose_proxy_logger.debug(f"Found {len(jobs)} response jobs to check")
completed_jobs = []
for job in jobs:
unified_object_id = job.unified_object_id
try:
from litellm.proxy.hooks.responses_id_security import (
ResponsesIDSecurity,
)
# Get the stored response object to extract model information
stored_response = job.file_object
model_name = stored_response.get("model", None)
# Decrypt the response ID
responses_id_security, _, _ = ResponsesIDSecurity()._decrypt_response_id(unified_object_id)
# Prepare metadata with model information for cost tracking
litellm_metadata = {
"user_api_key_user_id": job.created_by or "default-user-id",
}
# Add model information if available
if model_name:
litellm_metadata["model"] = model_name
litellm_metadata["model_group"] = model_name # Use same value for model_group
response = await litellm.aget_responses(
response_id=responses_id_security,
litellm_metadata=litellm_metadata,
)
verbose_proxy_logger.debug(
f"Response {unified_object_id} status: {response.status}, model: {model_name}"
)
except Exception as e:
verbose_proxy_logger.info(
f"Skipping job {unified_object_id} due to error: {e}"
)
continue
# Check if response is in a terminal state
if response.status == "completed":
verbose_proxy_logger.info(
f"Response {unified_object_id} is complete. Cost automatically tracked by aget_responses."
)
completed_jobs.append(job)
elif response.status in ["failed", "cancelled"]:
verbose_proxy_logger.info(
f"Response {unified_object_id} has status {response.status}, marking as complete"
)
completed_jobs.append(job)
# Mark completed jobs in the database
if len(completed_jobs) > 0:
await self.prisma_client.db.litellm_managedobjecttable.update_many(
where={"id": {"in": [job.id for job in completed_jobs]}},
data={"status": "completed"},
)
verbose_proxy_logger.info(
f"Marked {len(completed_jobs)} response jobs as completed"
)

15
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -23,7 +23,9 @@ from litellm.proxy._types import (
from litellm.proxy.openai_files_endpoints.common_utils import (
_is_base64_encoded_unified_file_id,
get_batch_id_from_unified_batch_id,
get_content_type_from_file_object,
get_model_id_from_unified_batch_id,
normalize_mime_type_for_provider,
)
from litellm.types.llms.openai import (
AllMessageValues,
@ -33,6 +35,7 @@ from litellm.types.llms.openai import (
FileObject,
OpenAIFileObject,
OpenAIFilesPurpose,
ResponsesAPIResponse,
)
from litellm.types.utils import (
CallTypesLiteral,
@ -41,10 +44,6 @@ from litellm.types.utils import (
LLMResponseTypes,
SpecialEnums,
)
from litellm.proxy.openai_files_endpoints.common_utils import (
get_content_type_from_file_object,
normalize_mime_type_for_provider,
)
if TYPE_CHECKING:
from litellm.types.llms.openai import HttpxBinaryResponseContent
@ -133,10 +132,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
async def store_unified_object_id(
self,
unified_object_id: str,
file_object: Union[LiteLLMBatch, LiteLLMFineTuningJob],
file_object: Union[LiteLLMBatch, LiteLLMFineTuningJob, "ResponsesAPIResponse"],
litellm_parent_otel_span: Optional[Span],
model_object_id: str,
file_purpose: Literal["batch", "fine-tune"],
file_purpose: Literal["batch", "fine-tune", "response"],
user_api_key_dict: UserAPIKeyAuth,
) -> None:
verbose_logger.info(
@ -946,7 +945,9 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
# File is stored in a storage backend, download and convert to base64
try:
from litellm.llms.base_llm.files.storage_backend_factory import get_storage_backend
from litellm.llms.base_llm.files.storage_backend_factory import (
get_storage_backend,
)
storage_backend_name = db_file.storage_backend
storage_url = db_file.storage_url

18
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -824,4 +824,22 @@ model LiteLLM_UISettings {
ui_settings Json
created_at DateTime @default(now())
updated_at DateTime @updatedAt
}
// Skills table for storing LiteLLM-managed skills
model LiteLLM_SkillsTable {
skill_id String @id @default(uuid())
display_title String?
description String?
instructions String? // The skill instructions/prompt (from SKILL.md)
source String @default("custom") // "custom" or "anthropic"
latest_version String?
file_content Bytes? // Binary content of the skill files (zip)
file_name String? // Original filename
file_type String? // MIME type (e.g., "application/zip")
metadata Json? @default("{}")
created_at DateTime @default(now())
created_by String?
updated_at DateTime @default(now()) @updatedAt
updated_by String?
}

6
litellm/__init__.py
View file

@ -1203,9 +1203,9 @@ from .llms.bedrock.chat.invoke_transformations.amazon_openai_transformation impo
AmazonBedrockOpenAIConfig,
)
from .llms.bedrock.image.amazon_stability1_transformation import AmazonStabilityConfig
from .llms.bedrock.image.amazon_stability3_transformation import AmazonStability3Config
from .llms.bedrock.image.amazon_nova_canvas_transformation import AmazonNovaCanvasConfig
from .llms.bedrock.image_generation.amazon_stability1_transformation import AmazonStabilityConfig
from .llms.bedrock.image_generation.amazon_stability3_transformation import AmazonStability3Config
from .llms.bedrock.image_generation.amazon_nova_canvas_transformation import AmazonNovaCanvasConfig
from .llms.bedrock.embed.amazon_titan_g1_transformation import AmazonTitanG1Config
from .llms.bedrock.embed.amazon_titan_multimodal_transformation import (
AmazonTitanMultimodalEmbeddingG1Config,

5
litellm/anthropic_interface/messages/__init__.py
View file

@ -37,6 +37,7 @@ async def acreate(
tools: Optional[List[Dict]] = None,
top_k: Optional[int] = None,
top_p: Optional[float] = None,
container: Optional[Dict] = None,
**kwargs
) -> Union[AnthropicMessagesResponse, AsyncIterator]:
"""
@ -56,6 +57,7 @@ async def acreate(
tools (List[Dict], optional): List of tool definitions
top_k (int, optional): Top K sampling parameter
top_p (float, optional): Nucleus sampling parameter
container (Dict, optional): Container config with skills for code execution
**kwargs: Additional arguments
Returns:
@ -75,6 +77,7 @@ async def acreate(
tools=tools,
top_k=top_k,
top_p=top_p,
container=container,
**kwargs,
)
@ -93,6 +96,7 @@ def create(
tools: Optional[List[Dict]] = None,
top_k: Optional[int] = None,
top_p: Optional[float] = None,
container: Optional[Dict] = None,
**kwargs
) -> Union[
AnthropicMessagesResponse,
@ -135,5 +139,6 @@ def create(
tools=tools,
top_k=top_k,
top_p=top_p,
container=container,
**kwargs,
)

63
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -167,24 +167,28 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
)
elif role == "tool":
# Convert tool message to function call output format
# Transform content to responses format (handles str, list, and other types)
# _convert_content_to_responses_format always returns List[Dict[str, Any]]
# The Responses API expects 'output' to be a string, not a list
if content is None:
transformed_output: list[dict[str, Any]] = []
elif isinstance(content, (str, list)):
transformed_output = self._convert_content_to_responses_format(
content, "tool"
)
output_str = ""
elif isinstance(content, str):
output_str = content
elif isinstance(content, list):
# If content is a list, extract text parts and join them
text_parts = []
for item in content:
if isinstance(item, str):
text_parts.append(item)
elif isinstance(item, dict) and item.get("type") == "text":
text_parts.append(item.get("text", ""))
output_str = " ".join(text_parts) if text_parts else str(content)
else:
# Fallback: convert unexpected types to string first
transformed_output = self._convert_content_to_responses_format(
str(content), "tool"
)
# Fallback: convert unexpected types to string
output_str = str(content)
input_items.append(
{
"type": "function_call_output",
"call_id": tool_call_id,
"output": transformed_output,
"output": output_str,
}
)
elif role == "assistant" and tool_calls and isinstance(tool_calls, list):
@ -345,6 +349,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
index = 0
reasoning_content: Optional[str] = None
# Collect all tool calls to put them in a single choice
# (Chat Completions API expects all tool calls in one message)
accumulated_tool_calls: List[Dict[str, Any]] = []
tool_call_index = 0
for item in output_items:
if isinstance(item, ResponseReasoningItem):
for summary_item in item.summary:
@ -378,20 +387,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
tool_call_dict = LiteLLMCompletionResponsesConfig.convert_response_function_tool_call_to_chat_completion_tool_call(
tool_call_item=item,
index=index,
index=tool_call_index,
)
msg = Message(
content=None,
tool_calls=[tool_call_dict],
reasoning_content=reasoning_content,
)
choices.append(
Choices(message=msg, finish_reason="tool_calls", index=index)
)
reasoning_content = None # flush reasoning content
index += 1
accumulated_tool_calls.append(tool_call_dict)
tool_call_index += 1
elif isinstance(item, dict) and handle_raw_dict_callback is not None:
# Handle raw dict responses (e.g., from GPT-5 Codex)
@ -401,6 +400,18 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
else:
pass # don't fail request if item in list is not supported
# If we accumulated tool calls, create a single choice with all of them
if accumulated_tool_calls:
msg = Message(
content=None,
tool_calls=accumulated_tool_calls,
reasoning_content=reasoning_content,
)
choices.append(
Choices(message=msg, finish_reason="tool_calls", index=index)
)
reasoning_content = None
return choices
def transform_response( # noqa: PLR0915
@ -492,7 +503,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
def _convert_content_str_to_input_text(
self, content: str, role: str
) -> Dict[str, Any]:
if role == "user" or role == "system":
if role == "user" or role == "system" or role == "tool":
return {"type": "input_text", "text": content}
else:
return {"type": "output_text", "text": content}

1
litellm/constants.py
View file

@ -892,6 +892,7 @@ BEDROCK_INVOKE_PROVIDERS_LITERAL = Literal[
"qwen2",
"twelvelabs",
"openai",
"stability",
]
BEDROCK_EMBEDDING_PROVIDERS_LITERAL = Literal[

63
litellm/images/main.py
View file

@ -33,6 +33,7 @@ from litellm.main import (
base_llm_aiohttp_handler,
base_llm_http_handler,
bedrock_image_generation,
bedrock_image_edit,
openai_chat_completions,
openai_image_variations,
)
@ -670,7 +671,7 @@ def image_variation(
@client
def image_edit(
def image_edit( # noqa: PLR0915
image: Union[FileTypes, List[FileTypes]],
prompt: str,
model: Optional[str] = None,
@ -695,6 +696,29 @@ def image_edit(
"""
local_vars = locals()
try:
openai_params = [
"user",
"request_timeout",
"api_base",
"api_version",
"api_key",
"deployment_id",
"organization",
"base_url",
"default_headers",
"timeout",
"max_retries",
"n",
"quality",
"size",
"style",
"async_call",
]
litellm_params_list = all_litellm_params
default_params = openai_params + litellm_params_list
non_default_params = {
k: v for k, v in kwargs.items() if k not in default_params
} # model-specific params - pass them straight to the model/provider
litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj") # type: ignore
litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None)
_is_async = kwargs.pop("async_call", False) is True
@ -788,7 +812,6 @@ def image_edit(
image_edit_optional_params: ImageEditOptionalRequestParams = (
_get_ImageEditRequestUtils().get_requested_image_edit_optional_param(local_vars)
)
# Get optional parameters for the responses API
image_edit_request_params: Dict = (
_get_ImageEditRequestUtils().get_optional_params_image_edit(
@ -812,6 +835,42 @@ def image_edit(
custom_llm_provider=custom_llm_provider,
)
# Route bedrock to its specific handler (AWS signing required)
if custom_llm_provider == "bedrock":
if model is None:
raise Exception("Model needs to be set for bedrock")
image_edit_request_params.update(non_default_params)
return bedrock_image_edit.image_edit( # type: ignore
model=model,
image=images,
prompt=prompt,
timeout=timeout,
logging_obj=litellm_logging_obj,
optional_params=image_edit_request_params,
model_response=ImageResponse(),
aimage_edit=_is_async,
client=kwargs.get("client"),
api_base=kwargs.get("api_base"),
extra_headers=extra_headers,
api_key=kwargs.get("api_key"),
)
elif custom_llm_provider == "stability":
image_edit_request_params.update(non_default_params)
return base_llm_http_handler.image_edit_handler(
model=model,
image=images,
prompt=prompt,
image_edit_provider_config=image_edit_provider_config,
image_edit_optional_request_params=image_edit_request_params,
custom_llm_provider=custom_llm_provider,
litellm_params=litellm_params,
logging_obj=litellm_logging_obj,
extra_headers=extra_headers,
extra_body=extra_body,
timeout=timeout or DEFAULT_REQUEST_TIMEOUT,
_is_async=_is_async,
client=kwargs.get("client"),
)
# Call the handler with _is_async flag instead of directly calling the async handler
return base_llm_http_handler.image_edit_handler(
model=model,

1
litellm/images/utils.py
View file

@ -82,7 +82,6 @@ class ImageEditRequestUtils:
filtered_params = {
k: v for k, v in params.items() if k in valid_keys and v is not None
}
return cast(ImageEditOptionalRequestParams, filtered_params)
@staticmethod

24
litellm/integrations/custom_guardrail.py
View file

@ -240,6 +240,28 @@ class CustomGuardrail(CustomLogger):
return metadata["disable_global_guardrail"]
return False
def _is_valid_response_type(self, result: Any) -> bool:
"""
Check if result is a valid LLMResponseTypes instance.
Safely handles TypedDict types which don't support isinstance checks.
For non-LiteLLM responses (like passthrough httpx.Response), returns True
to allow them through.
"""
if result is None:
return False
try:
# Try isinstance check on valid types that support it
response_types = get_args(LLMResponseTypes)
return isinstance(result, response_types)
except TypeError as e:
# TypedDict types don't support isinstance checks
# In this case, we can't validate the type, so we allow it through
if "TypedDict" in str(e):
return True
raise
def get_guardrail_from_metadata(
self, data: dict
) -> Union[List[str], List[Dict[str, DynamicGuardrailParams]]]:
@ -342,7 +364,7 @@ class CustomGuardrail(CustomLogger):
response=response,
)
if result is None or not isinstance(result, get_args(LLMResponseTypes)):
if not self._is_valid_response_type(result):
return response
return result

5
litellm/integrations/langfuse/langfuse_prompt_management.py
View file

@ -294,6 +294,11 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge
self.async_log_success_event, kwargs, response_obj, start_time, end_time
)
def log_failure_event(self, kwargs, response_obj, start_time, end_time):
return run_async_function(
self.async_log_failure_event, kwargs, response_obj, start_time, end_time
)
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
standard_callback_dynamic_params = kwargs.get(
"standard_callback_dynamic_params"

8
litellm/litellm_core_utils/api_route_to_call_types.py
View file

@ -5,10 +5,12 @@ This dictionary maps each API endpoint to the CallTypes that can be used for tha
Each route can have both async (prefixed with 'a') and sync call types.
"""
from typing import List, Optional
from litellm.types.utils import API_ROUTE_TO_CALL_TYPES, CallTypes
def get_call_types_for_route(route: str) -> list:
def get_call_types_for_route(route: str) -> Optional[List[CallTypes]]:
"""
Get the list of CallTypes for a given API route.
@ -16,9 +18,9 @@ def get_call_types_for_route(route: str) -> list:
route: API route path (e.g., "/chat/completions")
Returns:
List of CallTypes for that route, or empty list if route not found
List of CallTypes for that route, or None if route not found
"""
return API_ROUTE_TO_CALL_TYPES.get(route, [])
return API_ROUTE_TO_CALL_TYPES.get(route, None)
def get_routes_for_call_type(call_type: CallTypes) -> list:

2
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -674,7 +674,7 @@ class CostCalculatorUtils:
from litellm.llms.azure_ai.image_generation.cost_calculator import (
cost_calculator as azure_ai_image_cost_calculator,
)
from litellm.llms.bedrock.image.cost_calculator import (
from litellm.llms.bedrock.image_generation.cost_calculator import (
cost_calculator as bedrock_image_cost_calculator,
)
from litellm.llms.gemini.image_generation.cost_calculator import (

41
litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py
View file

@ -169,7 +169,7 @@ class LiteLLMAnthropicMessagesAdapter:
"""
Which anthropic params, we need to translate to the openai format.
"""
return ["messages", "metadata", "system", "tool_choice", "tools"]
return ["messages", "metadata", "system", "tool_choice", "tools", "thinking"]
def translate_anthropic_messages_to_openai( # noqa: PLR0915
self,
@ -420,6 +420,35 @@ class LiteLLMAnthropicMessagesAdapter:
return new_messages
def translate_anthropic_thinking_to_openai(
self, thinking: Dict[str, Any]
) -> Optional[str]:
"""
Translate Anthropic's thinking parameter to OpenAI's reasoning_effort.
Anthropic thinking format: {'type': 'enabled'|'disabled', 'budget_tokens': int}
OpenAI reasoning_effort: 'none' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh' | 'default'
"""
if not isinstance(thinking, dict):
return None
thinking_type = thinking.get("type", "disabled")
if thinking_type == "disabled":
return None
elif thinking_type == "enabled":
budget_tokens = thinking.get("budget_tokens", 0)
if budget_tokens >= 10000:
return "high"
elif budget_tokens >= 5000:
return "medium"
elif budget_tokens >= 2000:
return "low"
else:
return "minimal"
return None
def translate_anthropic_tool_choice_to_openai(
self, tool_choice: AnthropicMessagesToolChoice
) -> ChatCompletionToolChoiceValues:
@ -529,6 +558,16 @@ class LiteLLMAnthropicMessagesAdapter:
tools=cast(List[AllAnthropicToolsValues], tools)
)
## CONVERT THINKING
if "thinking" in anthropic_message_request:
thinking = anthropic_message_request["thinking"]
if thinking:
reasoning_effort = self.translate_anthropic_thinking_to_openai(
thinking=cast(Dict[str, Any], thinking)
)
if reasoning_effort:
new_kwargs["reasoning_effort"] = reasoning_effort
translatable_params = self.translatable_anthropic_params()
for k, v in anthropic_message_request.items():
if k not in translatable_params: # pass remaining params as is

4
litellm/llms/anthropic/experimental_pass_through/messages/handler.py
View file

@ -119,6 +119,7 @@ def anthropic_messages_handler(
tools: Optional[List[Dict]] = None,
top_k: Optional[int] = None,
top_p: Optional[float] = None,
container: Optional[Dict] = None,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
client: Optional[AsyncHTTPHandler] = None,
@ -131,6 +132,9 @@ def anthropic_messages_handler(
]:
"""
Makes Anthropic `/v1/messages` API calls In the Anthropic API Spec
Args:
container: Container config with skills for code execution
"""
from litellm.types.utils import LlmProviders

4
litellm/llms/bedrock/base_aws_llm.py
View file

@ -365,6 +365,10 @@ class BaseAWSLLM:
model_id = BaseAWSLLM._get_model_id_from_model_with_spec(
model_id, spec="qwen3"
)
elif provider == "stability" and "stability/" in model_id:
model_id = BaseAWSLLM._get_model_id_from_model_with_spec(
model_id, spec="stability"
)
return model_id
@staticmethod

10
litellm/llms/bedrock/image_edit/__init__.py Normal file
View file

@ -0,0 +1,10 @@
"""
Bedrock Image Edit Module
Handles image edit operations for Bedrock stability models.
"""
from .handler import BedrockImageEdit
__all__ = ["BedrockImageEdit"]

310
litellm/llms/bedrock/image_edit/handler.py Normal file
View file

@ -0,0 +1,310 @@
"""
Bedrock Image Edit Handler
Handles image edit requests for Bedrock stability models.
"""
from __future__ import annotations
import json
from typing import TYPE_CHECKING, Any, Optional, Union
import httpx
from pydantic import BaseModel
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging
from litellm.llms.bedrock.image_edit.stability_transformation import (
BedrockStabilityImageEditConfig,
)
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
_get_httpx_client,
get_async_httpx_client,
)
from litellm.types.utils import ImageResponse
from ..base_aws_llm import BaseAWSLLM
from ..common_utils import BedrockError
if TYPE_CHECKING:
from botocore.awsrequest import AWSPreparedRequest
else:
AWSPreparedRequest = Any
class BedrockImageEditPreparedRequest(BaseModel):
"""
Internal/Helper class for preparing the request for bedrock image edit
"""
endpoint_url: str
prepped: AWSPreparedRequest
body: bytes
data: dict
class BedrockImageEdit(BaseAWSLLM):
"""
Bedrock Image Edit handler
"""
@classmethod
def get_config_class(cls, model: str | None):
if BedrockStabilityImageEditConfig._is_stability_edit_model(model):
return BedrockStabilityImageEditConfig
else:
raise ValueError(f"Unsupported model for bedrock image edit: {model}")
def image_edit(
self,
model: str,
image: list,
prompt: str,
model_response: ImageResponse,
optional_params: dict,
logging_obj: LitellmLogging,
timeout: Optional[Union[float, httpx.Timeout]],
aimage_edit: bool = False,
api_base: Optional[str] = None,
extra_headers: Optional[dict] = None,
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
api_key: Optional[str] = None,
):
prepared_request = self._prepare_request(
model=model,
image=image,
prompt=prompt,
optional_params=optional_params,
api_base=api_base,
extra_headers=extra_headers,
logging_obj=logging_obj,
api_key=api_key,
)
if aimage_edit is True:
return self.async_image_edit(
prepared_request=prepared_request,
timeout=timeout,
model=model,
logging_obj=logging_obj,
prompt=prompt,
model_response=model_response,
client=(
client
if client is not None and isinstance(client, AsyncHTTPHandler)
else None
),
)
if client is None or not isinstance(client, HTTPHandler):
client = _get_httpx_client()
try:
response = client.post(url=prepared_request.endpoint_url, headers=prepared_request.prepped.headers, data=prepared_request.body) # type: ignore
response.raise_for_status()
except httpx.HTTPStatusError as err:
error_code = err.response.status_code
raise BedrockError(status_code=error_code, message=err.response.text)
except httpx.TimeoutException:
raise BedrockError(status_code=408, message="Timeout error occurred.")
### FORMAT RESPONSE TO OPENAI FORMAT ###
model_response = self._transform_response_dict_to_openai_response(
model_response=model_response,
model=model,
logging_obj=logging_obj,
prompt=prompt,
response=response,
data=prepared_request.data,
)
return model_response
async def async_image_edit(
self,
prepared_request: BedrockImageEditPreparedRequest,
timeout: Optional[Union[float, httpx.Timeout]],
model: str,
logging_obj: LitellmLogging,
prompt: str,
model_response: ImageResponse,
client: Optional[AsyncHTTPHandler] = None,
) -> ImageResponse:
"""
Asynchronous handler for bedrock image edit
"""
async_client = client or get_async_httpx_client(
llm_provider=litellm.LlmProviders.BEDROCK,
params={"timeout": timeout},
)
try:
response = await async_client.post(url=prepared_request.endpoint_url, headers=prepared_request.prepped.headers, data=prepared_request.body) # type: ignore
response.raise_for_status()
except httpx.HTTPStatusError as err:
error_code = err.response.status_code
raise BedrockError(status_code=error_code, message=err.response.text)
except httpx.TimeoutException:
raise BedrockError(status_code=408, message="Timeout error occurred.")
### FORMAT RESPONSE TO OPENAI FORMAT ###
model_response = self._transform_response_dict_to_openai_response(
model=model,
logging_obj=logging_obj,
prompt=prompt,
response=response,
data=prepared_request.data,
model_response=model_response,
)
return model_response
def _prepare_request(
self,
model: str,
image: list,
prompt: str,
optional_params: dict,
api_base: Optional[str],
extra_headers: Optional[dict],
logging_obj: LitellmLogging,
api_key: Optional[str],
) -> BedrockImageEditPreparedRequest:
"""
Prepare the request body, headers, and endpoint URL for the Bedrock Image Edit API
Args:
model (str): The model to use for the image edit
image (list): The images to edit
prompt (str): The prompt for the edit
optional_params (dict): The optional parameters for the image edit
api_base (Optional[str]): The base URL for the Bedrock API
extra_headers (Optional[dict]): The extra headers to include in the request
logging_obj (LitellmLogging): The logging object to use for logging
api_key (Optional[str]): The API key to use
Returns:
BedrockImageEditPreparedRequest: The prepared request object
"""
boto3_credentials_info = self._get_boto_credentials_from_optional_params(
optional_params, model
)
# Use the existing ARN-aware provider detection method
bedrock_provider = self.get_bedrock_invoke_provider(model)
### SET RUNTIME ENDPOINT ###
modelId = self.get_bedrock_model_id(
model=model,
provider=bedrock_provider,
optional_params=optional_params,
)
_, proxy_endpoint_url = self.get_runtime_endpoint(
api_base=api_base,
aws_bedrock_runtime_endpoint=boto3_credentials_info.aws_bedrock_runtime_endpoint,
aws_region_name=boto3_credentials_info.aws_region_name,
)
proxy_endpoint_url = f"{proxy_endpoint_url}/model/{modelId}/invoke"
data = self._get_request_body(
model=model,
image=image,
prompt=prompt,
optional_params=optional_params,
)
# Make POST Request
body = json.dumps(data).encode("utf-8")
headers = {"Content-Type": "application/json"}
if extra_headers is not None:
headers = {"Content-Type": "application/json", **extra_headers}
prepped = self.get_request_headers(
credentials=boto3_credentials_info.credentials,
aws_region_name=boto3_credentials_info.aws_region_name,
extra_headers=extra_headers,
endpoint_url=proxy_endpoint_url,
data=body,
headers=headers,
api_key=api_key,
)
## LOGGING
logging_obj.pre_call(
input=prompt,
api_key="",
additional_args={
"complete_input_dict": data,
"api_base": proxy_endpoint_url,
"headers": prepped.headers,
},
)
return BedrockImageEditPreparedRequest(
endpoint_url=proxy_endpoint_url,
prepped=prepped,
body=body,
data=data,
)
def _get_request_body(
self,
model: str,
image: list,
prompt: str,
optional_params: dict,
) -> dict:
"""
Get the request body for the Bedrock Image Edit API
Checks the model/provider and transforms the request body accordingly
Returns:
dict: The request body to use for the Bedrock Image Edit API
"""
config_class = self.get_config_class(model=model)
config_instance = config_class()
request_body = config_instance.transform_image_edit_request(
model=model,
prompt=prompt,
image=image[0] if image else None,
image_edit_optional_request_params=optional_params,
litellm_params={},
headers={},
)
return dict(request_body)
def _transform_response_dict_to_openai_response(
self,
model_response: ImageResponse,
model: str,
logging_obj: LitellmLogging,
prompt: str,
response: httpx.Response,
data: dict,
) -> ImageResponse:
"""
Transforms the Image Edit response from Bedrock to OpenAI format
"""
## LOGGING
if logging_obj is not None:
logging_obj.post_call(
input=prompt,
api_key="",
original_response=response.text,
additional_args={"complete_input_dict": data},
)
verbose_logger.debug("raw model_response: %s", response.text)
response_dict = response.json()
if response_dict is None:
raise ValueError("Error in response object format, got None")
config_class = self.get_config_class(model=model)
config_instance = config_class()
model_response = config_instance.transform_image_edit_response(
model=model,
raw_response=response,
logging_obj=logging_obj,
)
return model_response

377
litellm/llms/bedrock/image_edit/stability_transformation.py Normal file
View file

@ -0,0 +1,377 @@
"""
Bedrock Stability AI Image Edit Transformation
Handles transformation between OpenAI-compatible format and Bedrock Stability AI Image Edit API format.
Supported models:
- stability.stable-conservative-upscale-v1:0
- stability.stable-creative-upscale-v1:0
- stability.stable-fast-upscale-v1:0
- stability.stable-outpaint-v1:0
- stability.stable-image-control-sketch-v1:0
- stability.stable-image-control-structure-v1:0
- stability.stable-image-erase-object-v1:0
- stability.stable-image-inpaint-v1:0
- stability.stable-image-remove-background-v1:0
- stability.stable-image-search-recolor-v1:0
- stability.stable-image-search-replace-v1:0
- stability.stable-image-style-guide-v1:0
- stability.stable-style-transfer-v1:0
API Reference: https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters.html
"""
import json
import base64
from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple
import httpx
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
from litellm.types.images.main import ImageEditOptionalRequestParams
from litellm.types.router import GenericLiteLLMParams
from litellm.types.llms.stability import (
OPENAI_SIZE_TO_STABILITY_ASPECT_RATIO,
)
from litellm.types.utils import FileTypes, ImageObject, ImageResponse
from litellm.utils import get_model_info
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
LiteLLMLoggingObj = Any
class BedrockStabilityImageEditConfig(BaseImageEditConfig):
"""
Configuration for Bedrock Stability AI image edit.
Supports all Stability image edit operations through Bedrock.
"""
@classmethod
def _is_stability_edit_model(cls, model: Optional[str] = None) -> bool:
"""
Returns True if the model is a Bedrock Stability edit model.
Bedrock Stability edit models follow this pattern:
stability.stable-conservative-upscale-v1:0
stability.stable-creative-upscale-v1:0
stability.stable-fast-upscale-v1:0
stability.stable-outpaint-v1:0
stability.stable-image-inpaint-v1:0
stability.stable-image-erase-object-v1:0
etc.
"""
if model:
model_lower = model.lower()
if "stability." in model_lower and any([
"upscale" in model_lower,
"outpaint" in model_lower,
"inpaint" in model_lower,
"erase" in model_lower,
"remove-background" in model_lower,
"search-recolor" in model_lower,
"search-replace" in model_lower,
"control-sketch" in model_lower,
"control-structure" in model_lower,
"style-guide" in model_lower,
"style-transfer" in model_lower,
]):
return True
return False
def get_supported_openai_params(
self, model: str
) -> list:
"""
Return list of OpenAI params supported by Bedrock Stability.
"""
return [
"n", # Number of images (Stability always returns 1, we can loop)
"size", # Maps to aspect_ratio
"response_format", # b64_json or url (Stability only returns b64)
"mask",
]
def map_openai_params(
self,
image_edit_optional_params: ImageEditOptionalRequestParams,
model: str,
drop_params: bool,
) -> Dict:
"""
Map OpenAI parameters to Bedrock Stability parameters.
OpenAI -> Stability mappings:
- size -> aspect_ratio
- n -> (handled separately, Stability returns 1 image per request)
"""
supported_params = self.get_supported_openai_params(model)
# Define mapping from OpenAI params to Stability params
param_mapping = {
"size": "aspect_ratio",
# "n" and "response_format" are handled separately
}
# Create a copy to not mutate original - convert TypedDict to regular dict
mapped_params: Dict[str, Any] = dict(image_edit_optional_params)
for k, v in image_edit_optional_params.items():
if k in param_mapping:
# Map param if mapping exists and value is valid
if k == "size" and v in OPENAI_SIZE_TO_STABILITY_ASPECT_RATIO:
mapped_params[param_mapping[k]] = OPENAI_SIZE_TO_STABILITY_ASPECT_RATIO[v] # type: ignore
# Don't copy "size" itself to final dict
elif k == "n":
# Store for logic but do not add to outgoing params
mapped_params["_n"] = v
elif k == "response_format":
# Only b64 supported at Stability; store for postprocessing
mapped_params["_response_format"] = v
elif k not in supported_params:
if not drop_params:
raise ValueError(
f"Parameter {k} is not supported for model {model}. "
f"Supported parameters are {supported_params}. "
f"Set drop_params=True to drop unsupported parameters."
)
# Otherwise, param will simply be dropped
else:
# param is supported and not mapped, keep as-is
continue
# Remove OpenAI params that have been mapped unless they're in stability
for mapped in ["size", "n", "response_format"]:
if mapped in mapped_params:
del mapped_params[mapped]
return mapped_params
def transform_image_edit_request(
self,
model: str,
prompt: str,
image: FileTypes,
image_edit_optional_request_params: Dict,
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> Tuple[Dict, Any]:
"""
Transform OpenAI-style request to Bedrock Stability request format.
Returns the request body dict that will be JSON-encoded by the handler.
"""
# Build Bedrock Stability request
data: Dict[str, Any] = {
"prompt": prompt,
"output_format": "png", # Default to PNG
}
# Convert image to base64
image_b64: str
if hasattr(image, 'read') and callable(getattr(image, 'read', None)):
# File-like object (e.g., BufferedReader from open())
image_bytes = image.read() # type: ignore
image_b64 = base64.b64encode(image_bytes).decode('utf-8') # type: ignore
elif isinstance(image, bytes):
# Raw bytes
image_b64 = base64.b64encode(image).decode('utf-8')
elif isinstance(image, str):
# Already a base64 string
image_b64 = image
else:
# Try to handle as bytes
image_b64 = base64.b64encode(bytes(image)).decode('utf-8') # type: ignore
data["image"] = image_b64
# Add optional params (already mapped in map_openai_params)
for key, value in image_edit_optional_request_params.items(): # type: ignore
# Skip internal params (prefixed with _)
if key.startswith("_") or value is None:
continue
# File-like optional params (mask, init_image, style_image, etc.)
if key in ["mask", "init_image", "style_image"]:
# Handle case where value might be in a list
file_value = value
if isinstance(value, list) and len(value) > 0:
file_value = value[0]
if hasattr(file_value, 'read') and callable(getattr(file_value, 'read', None)):
file_bytes = file_value.read() # type: ignore
elif isinstance(file_value, bytes):
file_bytes = file_value
elif isinstance(file_value, str):
# Already a base64 string
data[key] = file_value
continue
else:
file_bytes = file_value # type: ignore
if isinstance(file_bytes, bytes):
file_b64 = base64.b64encode(file_bytes).decode('utf-8')
else:
file_b64 = str(file_bytes)
data[key] = file_b64
continue
# Supported text fields
if key in [
"negative_prompt",
"aspect_ratio",
"seed",
"output_format",
"model",
"mode",
"strength",
"style_preset",
"creativity",
"control_strength",
"grow_mask",
"left",
"right",
"up",
"down",
"select_prompt",
"search_prompt",
"fidelity",
"composition_fidelity",
"style_strength",
"change_strength",
]:
data[key] = value # type: ignore
return data, {}
def transform_image_edit_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
api_key: Optional[str] = None,
json_mode: Optional[bool] = None,
) -> ImageResponse:
"""
Transform Bedrock Stability response to OpenAI-compatible ImageResponse.
Bedrock returns: {"images": ["base64..."], "finish_reasons": [null], "seeds": [123]}
OpenAI expects: {"data": [{"b64_json": "base64..."}], "created": timestamp}
"""
try:
response_data = raw_response.json()
with open("response_data.json", "w") as f:
json.dump(response_data, f)
except Exception as e:
raise self.get_error_class(
error_message=f"Error parsing Bedrock Stability response: {e}",
status_code=raw_response.status_code,
headers=raw_response.headers,
)
# Check for errors in response
if "errors" in response_data:
raise self.get_error_class(
error_message=f"Bedrock Stability error: {response_data['errors']}",
status_code=raw_response.status_code,
headers=raw_response.headers,
)
# Check finish_reasons
finish_reasons = response_data.get("finish_reasons", [])
if finish_reasons and finish_reasons[0]:
raise self.get_error_class(
error_message=f"Bedrock Stability error: {finish_reasons[0]}",
status_code=400,
headers=raw_response.headers,
)
model_response = ImageResponse()
if not model_response.data:
model_response.data = []
# Extract images from response
images = response_data.get("images", [])
if images:
for image_b64 in images:
if image_b64:
model_response.data.append(
ImageObject(
b64_json=image_b64,
url=None,
revised_prompt=None,
)
)
if not hasattr(model_response, "_hidden_params"):
model_response._hidden_params = {}
if "additional_headers" not in model_response._hidden_params:
model_response._hidden_params["additional_headers"] = {}
# Set cost based on model
model_info = get_model_info(model, custom_llm_provider="bedrock")
cost_per_image = model_info.get("output_cost_per_image", 0)
if cost_per_image is not None:
model_response._hidden_params["additional_headers"]["llm_provider-x-litellm-response-cost"] = float(cost_per_image)
return model_response
def use_multipart_form_data(self) -> bool:
"""
Bedrock Stability uses JSON format, not multipart/form-data.
"""
return False
def get_complete_url(
self,
model: str,
api_base: Optional[str],
litellm_params: dict,
) -> str:
"""
Get the complete URL for the Bedrock Image Edit API.
For Bedrock, this is handled by the handler which constructs the endpoint URL
based on the model ID and AWS region. This method is required by the base class
but the actual URL construction happens in BedrockImageEdit.image_edit().
Returns a placeholder - the real endpoint is constructed in the handler.
"""
# Bedrock URLs are constructed in the handler using boto3
# This is a placeholder for the abstract method requirement
return "bedrock://image-edit"
def validate_environment(
self,
headers: dict,
model: str,
api_key: Optional[str] = None,
) -> dict:
"""
Validate environment for Bedrock Stability image edit.
For Bedrock, AWS credentials are managed by the BaseAWSLLM class.
This method validates that headers are properly set up.
Args:
headers: The request headers to validate/update
model: The model name being used
api_key: Optional API key (not used for Bedrock, which uses AWS credentials)
Returns:
Updated headers dict
"""
if headers is None:
headers = {}
# Bedrock uses AWS credentials, not API keys
# Headers are set up by the handler's get_request_headers() method
# This just ensures basic headers are present
if "Content-Type" not in headers:
headers["Content-Type"] = "application/json"
return headers

0
litellm/llms/bedrock/image/amazon_nova_canvas_transformation.py → litellm/llms/bedrock/image_generation/amazon_nova_canvas_transformation.py
View file

0
litellm/llms/bedrock/image/amazon_stability1_transformation.py → litellm/llms/bedrock/image_generation/amazon_stability1_transformation.py
View file

0
litellm/llms/bedrock/image/amazon_stability3_transformation.py → litellm/llms/bedrock/image_generation/amazon_stability3_transformation.py
View file

0
litellm/llms/bedrock/image/amazon_titan_transformation.py → litellm/llms/bedrock/image_generation/amazon_titan_transformation.py
View file

2
litellm/llms/bedrock/image/cost_calculator.py → litellm/llms/bedrock/image_generation/cost_calculator.py
View file

@ -1,6 +1,6 @@
from typing import Optional
from litellm.llms.bedrock.image.image_handler import BedrockImageGeneration
from litellm.llms.bedrock.image_generation.image_handler import BedrockImageGeneration
from litellm.types.utils import ImageResponse

6
litellm/llms/bedrock/image/image_handler.py → litellm/llms/bedrock/image_generation/image_handler.py
View file

@ -9,13 +9,13 @@ from pydantic import BaseModel
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging
from litellm.llms.bedrock.image.amazon_nova_canvas_transformation import (
from litellm.llms.bedrock.image_generation.amazon_nova_canvas_transformation import (
AmazonNovaCanvasConfig,
)
from litellm.llms.bedrock.image.amazon_stability3_transformation import (
from litellm.llms.bedrock.image_generation.amazon_stability3_transformation import (
AmazonStability3Config,
)
from litellm.llms.bedrock.image.amazon_titan_transformation import (
from litellm.llms.bedrock.image_generation.amazon_titan_transformation import (
AmazonTitanImageGenerationConfig,
)
from litellm.llms.custom_httpx.http_handler import (

2
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -3761,7 +3761,7 @@ class BaseLLMHTTPHandler:
input=prompt,
api_key="",
additional_args={
"complete_input_dict": data,
"complete_input_dict": files,
"api_base": api_base,
"headers": headers,
},

381
litellm/llms/litellm_proxy/skills/README.md Normal file
View file

@ -0,0 +1,381 @@
# LiteLLM Skills - Database-Backed Skills Storage
This module provides database-backed skills storage as an alternative to Anthropic's cloud-based Skills API. It enables using skills with **any LLM provider** (Bedrock, OpenAI, Azure, etc.) by storing skills locally and converting them to tools + system prompt injection.
## Architecture
```mermaid
flowchart TB
subgraph "Skill Creation"
A[User creates skill with ZIP file] --> B{custom_llm_provider?}
B -->|anthropic| C[Forward to Anthropic API]
B -->|litellm_proxy| D[Store in LiteLLM Database]
D --> E[Extract & store:<br/>- display_title<br/>- description<br/>- instructions<br/>- file_content ZIP]
end
subgraph "Skill Usage in Messages API"
F[Request with container.skills] --> G[SkillsInjectionHook]
G --> H{skill_id prefix?}
H -->|"litellm:skill_abc"| I[Fetch from LiteLLM DB]
H -->|"skill_xyz" no prefix| J[Pass to Anthropic as native skill]
I --> K{Model provider?}
K -->|Anthropic API| L[Convert to tools]
K -->|Bedrock/OpenAI/etc| M[Convert to tools +<br/>Inject SKILL.md into system prompt]
J --> N[Keep in container.skills]
end
subgraph "Skill Resolution for Non-Anthropic"
M --> O[Extract SKILL.md from ZIP]
O --> P[Add to system prompt:<br/># Available Skills<br/>## Skill: My Skill<br/>SKILL.md content...]
P --> Q[Create OpenAI-style tool:<br/>type: function<br/>name: skill_id<br/>description: instructions]
Q --> R[Send to LLM Provider]
end
```
## Automatic Code Execution
For skills that include executable code (Python files), LiteLLM automatically handles:
1. **Pre-call hook** (`async_pre_call_hook`): Adds `litellm_code_execution` tool, injects SKILL.md content
2. **Post-call hook** (`async_post_call_success_deployment_hook`): Detects tool calls, executes code in Docker sandbox, continues loop
3. **Returns files**: Generated files (GIFs, images, etc.) returned directly on response
```mermaid
sequenceDiagram
participant User
participant LiteLLM as LiteLLM SDK
participant PreHook as async_pre_call_hook
participant LLM as LLM Provider
participant PostHook as async_post_call_success_deployment_hook
participant Sandbox as Docker Sandbox
User->>LiteLLM: litellm.acompletion(model, messages, container={skills: [...]})
Note over LiteLLM,PreHook: PRE-CALL HOOK
LiteLLM->>PreHook: Intercept request
PreHook->>PreHook: Fetch skill from DB (litellm:skill_id)
PreHook->>PreHook: Extract SKILL.md from ZIP
PreHook->>PreHook: Inject SKILL.md into system prompt
PreHook->>PreHook: Add litellm_code_execution tool
PreHook->>PreHook: Store skill files in metadata
PreHook-->>LiteLLM: Modified request
LiteLLM->>LLM: Forward to provider (OpenAI/Bedrock/etc)
LLM-->>LiteLLM: Response with tool_calls
Note over LiteLLM,PostHook: POST-CALL HOOK (Agentic Loop)
LiteLLM->>PostHook: Check response
loop Until no more tool calls
PostHook->>PostHook: Check for litellm_code_execution tool call
alt Has code execution tool call
PostHook->>Sandbox: Execute Python code
Sandbox->>Sandbox: Copy skill files to /sandbox
Sandbox->>Sandbox: Install requirements.txt
Sandbox->>Sandbox: Run code
Sandbox-->>PostHook: Result + generated files
PostHook->>PostHook: Add tool result to messages
PostHook->>LLM: Make another LLM call
LLM-->>PostHook: New response
else No code execution
PostHook->>PostHook: Break loop
end
end
PostHook->>PostHook: Attach files to response._litellm_generated_files
PostHook-->>LiteLLM: Modified response with files
LiteLLM-->>User: Final response with generated files
```
```python
import litellm
from litellm.proxy.hooks.litellm_skills import SkillsInjectionHook
# Register the hook (done once at startup)
hook = SkillsInjectionHook()
litellm.callbacks.append(hook)
# ONE request - LiteLLM handles everything automatically
# The container parameter triggers the SkillsInjectionHook
response = await litellm.acompletion(
model="gpt-4o-mini",
messages=[{"role": "user", "content": "Create a bouncing ball GIF"}],
container={
"skills": [{"type": "custom", "skill_id": "litellm:skill_abc123"}]
},
)
# Files are attached directly to response
generated_files = response._litellm_generated_files
for f in generated_files:
print(f"Generated: {f['name']} ({f['size']} bytes)")
# f['content_base64'] contains the file data
```
This mimics Anthropic's behavior - no manual agentic loop needed!
### How it works
The `SkillsInjectionHook` uses two hooks:
1. **`async_pre_call_hook`** (proxy only): Transforms the request before LLM call
- Fetches skills from DB
- Injects SKILL.md into system prompt
- Adds `litellm_code_execution` tool
- Sets `_litellm_code_execution_enabled=True` in metadata
2. **`async_post_call_success_deployment_hook`** (SDK + proxy): Called after LLM response
- Checks if response has `litellm_code_execution` tool call
- Executes code in Docker sandbox
- Adds result to messages, makes another LLM call
- Repeats until model gives final response
- Attaches generated files to `response._litellm_generated_files`
## File Structure
```
litellm/llms/litellm_proxy/skills/
├── __init__.py # Exports all skill components
├── handler.py # LiteLLMSkillsHandler - database CRUD operations (Prisma)
├── transformation.py # LiteLLMSkillsTransformationHandler - SDK transformation layer
├── prompt_injection.py # SkillPromptInjectionHandler - SKILL.md extraction and injection
├── sandbox_executor.py # SkillsSandboxExecutor - Docker sandbox code execution
├── code_execution.py # CodeExecutionHandler - automatic agentic loop
└── README.md # This file
litellm/proxy/hooks/litellm_skills/
├── __init__.py # Re-exports from SDK + SkillsInjectionHook
└── main.py # SkillsInjectionHook - CustomLogger hook for proxy
```
## Components
### 1. `handler.py` - LiteLLMSkillsHandler
Database operations for skills CRUD:
```python
from litellm.llms.litellm_proxy.skills import LiteLLMSkillsHandler
# Create skill
skill = await LiteLLMSkillsHandler.create_skill(
data=NewSkillRequest(
display_title="My Skill",
description="A helpful skill",
instructions="Use this skill when...",
file_content=zip_bytes, # ZIP file content
file_name="my-skill.zip",
file_type="application/zip",
),
user_id="user_123"
)
# List skills
skills = await LiteLLMSkillsHandler.list_skills(limit=10, offset=0)
# Get skill
skill = await LiteLLMSkillsHandler.get_skill(skill_id="skill_abc123")
# Delete skill
await LiteLLMSkillsHandler.delete_skill(skill_id="skill_abc123")
```
### 2. `transformation.py` - LiteLLMSkillsTransformationHandler
SDK-level transformation layer that wraps handler operations:
```python
from litellm.llms.litellm_proxy.skills import LiteLLMSkillsTransformationHandler
handler = LiteLLMSkillsTransformationHandler()
# Async create
skill = await handler.create_skill_handler(
display_title="My Skill",
files=[zip_file],
_is_async=True
)
```
## Skill ZIP Format
Skills must be packaged as ZIP files with a `SKILL.md` file:
```
my-skill.zip
└── my-skill/
└── SKILL.md
```
### SKILL.md Format
```markdown
---
name: my-skill
description: A brief description of what this skill does
---
# My Skill
Detailed instructions for the LLM on how to use this skill.
## Usage
When the user asks about X, use this skill to...
## Examples
- Example 1: ...
- Example 2: ...
```
## SDK Usage
### Create Skill in LiteLLM Database
```python
import litellm
# Create skill stored in LiteLLM DB
skill = litellm.create_skill(
display_title="Data Analysis Skill",
files=[open("data-analysis.zip", "rb")],
custom_llm_provider="litellm_proxy", # Store in LiteLLM DB
)
print(f"Created skill: {skill.id}") # skill_abc123
```
### Use Skill with Any Provider
```python
import litellm
# Use LiteLLM-stored skill with Bedrock
response = litellm.completion(
model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
messages=[{"role": "user", "content": "Analyze this data..."}],
container={
"skills": [
{"type": "custom", "skill_id": "litellm:skill_abc123"} # litellm: prefix
]
}
)
```
## How Skill Resolution Works
### Step 1: Request with Skills
```python
{
"model": "bedrock/claude-3-sonnet",
"messages": [{"role": "user", "content": "Help me analyze data"}],
"container": {
"skills": [
{"type": "custom", "skill_id": "litellm:skill_abc123"}
]
}
}
```
### Step 2: SkillsInjectionHook Processing
The hook (`litellm/proxy/hooks/litellm_skills/main.py`) intercepts the request:
1. **Detects `litellm:` prefix** → Fetches skill from database
2. **Checks model provider** → Bedrock is not Anthropic
3. **Extracts SKILL.md** from stored ZIP file
4. **Converts skill to tool** + **Injects content into system prompt**
### Step 3: Transformed Request
```python
{
"model": "bedrock/claude-3-sonnet",
"messages": [
{
"role": "system",
"content": """
---
# Available Skills
## Skill: Data Analysis Skill
# Data Analysis Skill
This skill helps with data analysis tasks...
## Usage
When the user asks about data analysis...
"""
},
{"role": "user", "content": "Help me analyze data"}
],
"tools": [
{
"type": "function",
"function": {
"name": "skill_abc123",
"description": "This skill helps with data analysis tasks...",
"parameters": {"type": "object", "properties": {}, "required": []}
}
}
]
# container is removed for non-Anthropic providers
}
```
## Database Schema
Skills are stored in `LiteLLM_SkillsTable`:
```prisma
model LiteLLM_SkillsTable {
skill_id String @id @default(uuid())
display_title String?
description String?
instructions String?
source String @default("custom")
latest_version String?
metadata Json? @default("{}")
file_content Bytes? // ZIP file binary content
file_name String? // Original filename
file_type String? // MIME type
created_at DateTime @default(now())
created_by String?
updated_at DateTime @default(now()) @updatedAt
updated_by String?
}
```
## Routing Summary
| Scenario | custom_llm_provider | skill_id Format | Behavior |
|----------|---------------------|-----------------|----------|
| Create skill on Anthropic | `anthropic` | N/A | Forward to Anthropic API |
| Create skill in LiteLLM DB | `litellm_proxy` | N/A | Store in database |
| Use Anthropic native skill | N/A | `skill_xyz` | Pass to Anthropic container.skills |
| Use LiteLLM skill on Anthropic | N/A | `litellm:skill_abc` | Convert to tools |
| Use LiteLLM skill on Bedrock/OpenAI | N/A | `litellm:skill_abc` | Convert to tools + inject SKILL.md |
## Testing
Run the tests:
```bash
pytest tests/proxy_unit_tests/test_skills_db.py -v
```
Tests cover:
- Creating skills with file content
- Listing and retrieving skills
- Deleting skills
- Hook resolution with ZIP file extraction
- System prompt injection for non-Anthropic models

54
litellm/llms/litellm_proxy/skills/__init__.py Normal file
View file

@ -0,0 +1,54 @@
"""
LiteLLM Proxy Skills - Database-backed skills storage and execution
This module provides:
- Database-backed skills storage (alternative to Anthropic's cloud-based skills API)
- Skill content extraction and prompt injection
- Sandboxed code execution for skills
- Automatic code execution handler
Main components:
- handler.py: LiteLLMSkillsHandler - database CRUD operations
- transformation.py: LiteLLMSkillsTransformationHandler - SDK transformation layer
- prompt_injection.py: SkillPromptInjectionHandler - SKILL.md extraction and injection
- sandbox_executor.py: SkillsSandboxExecutor - Docker sandbox execution
- code_execution.py: CodeExecutionHandler - automatic agentic loop
"""
from litellm.llms.litellm_proxy.skills.code_execution import (
LITELLM_CODE_EXECUTION_TOOL,
CodeExecutionHandler,
LiteLLMInternalTools,
add_code_execution_tool,
code_execution_handler,
get_litellm_code_execution_tool,
has_code_execution_tool,
)
from litellm.llms.litellm_proxy.skills.constants import (
DEFAULT_MAX_ITERATIONS,
DEFAULT_SANDBOX_TIMEOUT,
)
from litellm.llms.litellm_proxy.skills.handler import LiteLLMSkillsHandler
from litellm.llms.litellm_proxy.skills.prompt_injection import (
SkillPromptInjectionHandler,
)
from litellm.llms.litellm_proxy.skills.sandbox_executor import SkillsSandboxExecutor
from litellm.llms.litellm_proxy.skills.transformation import (
LiteLLMSkillsTransformationHandler,
)
__all__ = [
"LiteLLMSkillsHandler",
"LiteLLMSkillsTransformationHandler",
"SkillPromptInjectionHandler",
"SkillsSandboxExecutor",
"CodeExecutionHandler",
"LiteLLMInternalTools",
"LITELLM_CODE_EXECUTION_TOOL",
"get_litellm_code_execution_tool",
"code_execution_handler",
"has_code_execution_tool",
"add_code_execution_tool",
"DEFAULT_MAX_ITERATIONS",
"DEFAULT_SANDBOX_TIMEOUT",
]

311
litellm/llms/litellm_proxy/skills/code_execution.py Normal file
View file

@ -0,0 +1,311 @@
"""
Automatic Code Execution Handler for LiteLLM Skills
When `litellm_code_execution` tool is present, this handler automatically:
1. Makes the LLM call
2. Executes any code the model generates
3. Continues the conversation with results
4. Returns final response with generated files inline (base64)
This mimics Anthropic's behavior where code execution happens automatically.
Generated files are returned directly in the response - no separate storage needed.
"""
import base64
import json
from enum import Enum
from typing import Any, Dict, List, Optional
from litellm._logging import verbose_logger
class LiteLLMInternalTools(str, Enum):
"""
Enum for internal LiteLLM tools that are injected into requests.
These tools are handled automatically by LiteLLM hooks and are not
passed to the underlying LLM provider directly.
"""
CODE_EXECUTION = "litellm_code_execution"
def get_litellm_code_execution_tool() -> Dict[str, Any]:
"""
Returns the litellm_code_execution tool definition in OpenAI format.
This tool enables automatic code execution in a sandboxed environment
when skills include executable Python code.
"""
return {
"type": "function",
"function": {
"name": LiteLLMInternalTools.CODE_EXECUTION.value,
"description": "Execute Python code in a sandboxed environment. Use this to run code that generates files, processes data, or performs computations. Generated files will be returned directly.",
"parameters": {
"type": "object",
"properties": {
"code": {
"type": "string",
"description": "Python code to execute"
}
},
"required": ["code"]
}
}
}
def get_litellm_code_execution_tool_anthropic() -> Dict[str, Any]:
"""
Returns the litellm_code_execution tool definition in Anthropic/messages API format.
This tool enables automatic code execution in a sandboxed environment
when skills include executable Python code.
"""
return {
"name": LiteLLMInternalTools.CODE_EXECUTION.value,
"description": "Execute Python code in a sandboxed environment. Use this to run code that generates files, processes data, or performs computations. Generated files will be returned directly.",
"input_schema": {
"type": "object",
"properties": {
"code": {
"type": "string",
"description": "Python code to execute"
}
},
"required": ["code"]
}
}
# Singleton tool definition for backwards compatibility
LITELLM_CODE_EXECUTION_TOOL = get_litellm_code_execution_tool()
class CodeExecutionHandler:
"""
Handles automatic code execution for LiteLLM skills.
When enabled, this handler intercepts LLM responses with code execution
tool calls, executes them in a sandbox, and continues the conversation
automatically until completion.
"""
def __init__(
self,
max_iterations: Optional[int] = None,
sandbox_timeout: Optional[int] = None,
):
from litellm.llms.litellm_proxy.skills.constants import (
DEFAULT_MAX_ITERATIONS,
DEFAULT_SANDBOX_TIMEOUT,
)
self.max_iterations = max_iterations or DEFAULT_MAX_ITERATIONS
self.sandbox_timeout = sandbox_timeout or DEFAULT_SANDBOX_TIMEOUT
async def execute_with_code_execution(
self,
model: str,
messages: List[Dict],
tools: List[Dict],
skill_files: Dict[str, bytes],
skill_id: Optional[str] = None,
**kwargs,
) -> Dict[str, Any]:
"""
Execute an LLM call with automatic code execution handling.
This method:
1. Makes the initial LLM call
2. If model calls litellm_code_execution, executes the code
3. Continues conversation with results
4. Repeats until model stops calling tools
5. Returns final response with generated files inline
Args:
model: Model to use
messages: Initial messages
tools: Tools including litellm_code_execution
skill_files: Dict of skill files for execution
skill_id: Optional skill ID for tracking
**kwargs: Additional args for litellm.acompletion
Returns:
Dict with:
- response: Final LLM response
- files: List of generated files with content (base64)
- execution_results: List of code execution results
"""
import litellm
from litellm.llms.litellm_proxy.skills.sandbox_executor import (
SkillsSandboxExecutor,
)
current_messages = list(messages)
generated_files: List[Dict[str, Any]] = [] # Files returned directly
execution_results: List[Dict] = []
executor = SkillsSandboxExecutor(timeout=self.sandbox_timeout)
response: Any = None # Initialize to avoid possibly unbound error
for iteration in range(self.max_iterations):
verbose_logger.debug(
f"CodeExecutionHandler: Iteration {iteration + 1}/{self.max_iterations}"
)
# Make LLM call
response = await litellm.acompletion(
model=model,
messages=current_messages,
tools=tools,
**kwargs,
)
assistant_message = response.choices[0].message # type: ignore
stop_reason = response.choices[0].finish_reason # type: ignore
# Build assistant message for conversation history
assistant_msg_dict: Dict[str, Any] = {
"role": "assistant",
"content": assistant_message.content,
}
if assistant_message.tool_calls:
assistant_msg_dict["tool_calls"] = [
{
"id": tc.id,
"type": "function",
"function": {
"name": tc.function.name,
"arguments": tc.function.arguments
}
}
for tc in assistant_message.tool_calls
]
current_messages.append(assistant_msg_dict)
# Check if we're done (no tool calls or not tool_calls finish reason)
if stop_reason != "tool_calls" or not assistant_message.tool_calls:
verbose_logger.debug(
f"CodeExecutionHandler: Completed after {iteration + 1} iterations"
)
return {
"response": response,
"files": generated_files, # Files returned directly with base64 content
"execution_results": execution_results,
"messages": current_messages,
}
# Handle tool calls
for tool_call in assistant_message.tool_calls:
tool_name = tool_call.function.name
if tool_name == LiteLLMInternalTools.CODE_EXECUTION.value:
# Execute code in sandbox
try:
args = json.loads(tool_call.function.arguments)
code = args.get("code", "")
verbose_logger.debug(
f"CodeExecutionHandler: Executing code ({len(code)} chars)"
)
exec_result = executor.execute(
code=code,
skill_files=skill_files,
)
verbose_logger.debug(
f"CodeExecutionHandler: Execution result: {exec_result}"
)
execution_results.append({
"iteration": iteration,
"success": exec_result["success"],
"output": exec_result["output"],
"error": exec_result["error"],
"files": [f["name"] for f in exec_result["files"]],
})
# Build tool result content
tool_result = exec_result["output"] or ""
# Collect generated files (returned directly, no storage)
if exec_result["files"]:
tool_result += "\n\nGenerated files:"
for f in exec_result["files"]:
file_content = base64.b64decode(f["content_base64"])
# Add to generated files list (returned in response)
generated_files.append({
"name": f["name"],
"mime_type": f["mime_type"],
"content_base64": f["content_base64"],
"size": len(file_content),
})
tool_result += f"\n- {f['name']} ({len(file_content)} bytes)"
verbose_logger.debug(
f"CodeExecutionHandler: Generated file {f['name']} ({len(file_content)} bytes)"
)
if exec_result["error"]:
tool_result += f"\n\nError:\n{exec_result['error']}"
except Exception as e:
tool_result = f"Code execution failed: {str(e)}"
execution_results.append({
"iteration": iteration,
"success": False,
"error": str(e),
})
# Add tool result to messages
current_messages.append({
"role": "tool",
"tool_call_id": tool_call.id,
"content": tool_result,
})
else:
# Non-code-execution tool - pass through
# In a full implementation, this would call other tool handlers
current_messages.append({
"role": "tool",
"tool_call_id": tool_call.id,
"content": f"Tool '{tool_name}' not handled by code execution handler",
})
# Max iterations reached
verbose_logger.warning(
f"CodeExecutionHandler: Max iterations ({self.max_iterations}) reached"
)
return {
"response": response,
"files": generated_files,
"execution_results": execution_results,
"messages": current_messages,
"max_iterations_reached": True,
}
def has_code_execution_tool(tools: Optional[List[Dict]]) -> bool:
"""Check if litellm_code_execution tool is in the tools list."""
if not tools:
return False
for tool in tools:
func = tool.get("function", {})
if func.get("name") == LiteLLMInternalTools.CODE_EXECUTION.value:
return True
return False
def add_code_execution_tool(tools: Optional[List[Dict]]) -> List[Dict]:
"""Add litellm_code_execution tool if not already present."""
tools = tools or []
if not has_code_execution_tool(tools):
tools.append(LITELLM_CODE_EXECUTION_TOOL)
return tools
# Global handler instance
code_execution_handler = CodeExecutionHandler()

13
litellm/llms/litellm_proxy/skills/constants.py Normal file
View file

@ -0,0 +1,13 @@
"""
Constants for LiteLLM Skills
Centralized constants for skills processing, code execution, and sandbox configuration.
"""
# Code execution loop settings
DEFAULT_MAX_ITERATIONS: int = 10
"""Maximum number of iterations for the automatic code execution loop."""
DEFAULT_SANDBOX_TIMEOUT: int = 120
"""Default timeout in seconds for sandbox code execution."""

219
litellm/llms/litellm_proxy/skills/handler.py Normal file
View file

@ -0,0 +1,219 @@
"""
Handler for LiteLLM database-backed skills operations.
This module contains the actual database operations for skills CRUD.
Used by the transformation layer and skills injection hook.
"""
import uuid
from typing import Any, Dict, List, Optional
from litellm._logging import verbose_logger
from litellm.proxy._types import LiteLLM_SkillsTable, NewSkillRequest
def _prisma_skill_to_litellm(prisma_skill) -> LiteLLM_SkillsTable:
"""
Convert a Prisma skill record to LiteLLM_SkillsTable.
Handles Base64 decoding of file_content field.
"""
import base64
data = prisma_skill.model_dump()
# Decode Base64 file_content back to bytes
# model_dump() converts Base64 field to base64-encoded string
if data.get("file_content") is not None:
if isinstance(data["file_content"], str):
data["file_content"] = base64.b64decode(data["file_content"])
elif isinstance(data["file_content"], bytes):
# Already bytes, no conversion needed
pass
return LiteLLM_SkillsTable(**data)
class LiteLLMSkillsHandler:
"""
Handler for LiteLLM database-backed skills operations.
This class provides static methods for CRUD operations on skills
stored in the LiteLLM proxy database (LiteLLM_SkillsTable).
"""
@staticmethod
async def _get_prisma_client():
"""Get the prisma client from proxy server."""
from litellm.proxy.proxy_server import prisma_client
if prisma_client is None:
raise ValueError(
"Prisma client is not initialized. "
"Database connection required for LiteLLM skills."
)
return prisma_client
@staticmethod
async def create_skill(
data: NewSkillRequest,
user_id: Optional[str] = None,
) -> LiteLLM_SkillsTable:
"""
Create a new skill in the LiteLLM database.
Args:
data: NewSkillRequest with skill details
user_id: Optional user ID for tracking
Returns:
LiteLLM_SkillsTable record
"""
prisma_client = await LiteLLMSkillsHandler._get_prisma_client()
skill_id = f"litellm_skill_{uuid.uuid4()}"
skill_data: Dict[str, Any] = {
"skill_id": skill_id,
"display_title": data.display_title,
"description": data.description,
"instructions": data.instructions,
"source": "custom",
"created_by": user_id,
"updated_by": user_id,
}
# Handle metadata
if data.metadata is not None:
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
skill_data["metadata"] = safe_dumps(data.metadata)
# Handle file content - wrap bytes in Base64 for Prisma
if data.file_content is not None:
from prisma.fields import Base64
skill_data["file_content"] = Base64.encode(data.file_content)
if data.file_name is not None:
skill_data["file_name"] = data.file_name
if data.file_type is not None:
skill_data["file_type"] = data.file_type
verbose_logger.debug(
f"LiteLLMSkillsHandler: Creating skill {skill_id} with title={data.display_title}"
)
new_skill = await prisma_client.db.litellm_skillstable.create(data=skill_data)
return _prisma_skill_to_litellm(new_skill)
@staticmethod
async def list_skills(
limit: int = 20,
offset: int = 0,
) -> List[LiteLLM_SkillsTable]:
"""
List skills from the LiteLLM database.
Args:
limit: Maximum number of skills to return
offset: Number of skills to skip
Returns:
List of LiteLLM_SkillsTable records
"""
prisma_client = await LiteLLMSkillsHandler._get_prisma_client()
verbose_logger.debug(
f"LiteLLMSkillsHandler: Listing skills with limit={limit}, offset={offset}"
)
skills = await prisma_client.db.litellm_skillstable.find_many(
take=limit,
skip=offset,
order={"created_at": "desc"},
)
return [_prisma_skill_to_litellm(s) for s in skills]
@staticmethod
async def get_skill(skill_id: str) -> LiteLLM_SkillsTable:
"""
Get a skill by ID from the LiteLLM database.
Args:
skill_id: The skill ID to retrieve
Returns:
LiteLLM_SkillsTable record
Raises:
ValueError: If skill not found
"""
prisma_client = await LiteLLMSkillsHandler._get_prisma_client()
verbose_logger.debug(f"LiteLLMSkillsHandler: Getting skill {skill_id}")
skill = await prisma_client.db.litellm_skillstable.find_unique(
where={"skill_id": skill_id}
)
if skill is None:
raise ValueError(f"Skill not found: {skill_id}")
return _prisma_skill_to_litellm(skill)
@staticmethod
async def delete_skill(skill_id: str) -> Dict[str, str]:
"""
Delete a skill by ID from the LiteLLM database.
Args:
skill_id: The skill ID to delete
Returns:
Dict with id and type of deleted skill
Raises:
ValueError: If skill not found
"""
prisma_client = await LiteLLMSkillsHandler._get_prisma_client()
verbose_logger.debug(f"LiteLLMSkillsHandler: Deleting skill {skill_id}")
# Check if skill exists
skill = await prisma_client.db.litellm_skillstable.find_unique(
where={"skill_id": skill_id}
)
if skill is None:
raise ValueError(f"Skill not found: {skill_id}")
# Delete the skill
await prisma_client.db.litellm_skillstable.delete(where={"skill_id": skill_id})
return {"id": skill_id, "type": "skill_deleted"}
@staticmethod
async def fetch_skill_from_db(skill_id: str) -> Optional[LiteLLM_SkillsTable]:
"""
Fetch a skill from the database (used by skills injection hook).
This is a convenience method that returns None instead of raising
an exception if the skill is not found.
Args:
skill_id: The skill ID to fetch
Returns:
LiteLLM_SkillsTable or None if not found
"""
try:
return await LiteLLMSkillsHandler.get_skill(skill_id)
except ValueError:
return None
except Exception as e:
verbose_logger.warning(
f"LiteLLMSkillsHandler: Error fetching skill {skill_id}: {e}"
)
return None

305
litellm/llms/litellm_proxy/skills/prompt_injection.py Normal file
View file

@ -0,0 +1,305 @@
"""
Prompt Injection Handler for LiteLLM Skills
Handles extraction of skill content (SKILL.md) from stored ZIP files
and injection into the system prompt for non-Anthropic models.
"""
import zipfile
from io import BytesIO
from typing import Any, Dict, List, Optional
from litellm._logging import verbose_logger
from litellm.proxy._types import LiteLLM_SkillsTable
class SkillPromptInjectionHandler:
"""
Handles skill content extraction and system prompt injection.
Responsibilities:
- Extract SKILL.md content from skill ZIP files
- Extract ALL files from ZIP for code execution
- Inject skill content into system message
- Create execute_code tool definition
"""
def extract_skill_content(self, skill: LiteLLM_SkillsTable) -> Optional[str]:
"""
Extract skill content from the stored zip file.
Looks for SKILL.md or README.md in the zip and returns its content.
This content describes the skill's capabilities and instructions.
Args:
skill: The skill from LiteLLM database
Returns:
The skill content as a string, or None if not available
"""
if not skill.file_content:
return skill.instructions
try:
zip_buffer = BytesIO(skill.file_content)
with zipfile.ZipFile(zip_buffer, "r") as zf:
# Look for SKILL.md first
for name in zf.namelist():
if name.endswith("SKILL.md"):
content = zf.read(name).decode("utf-8")
if content:
return f"## Skill: {skill.display_title or skill.skill_id}\n\n{content}"
# Fall back to README.md
for name in zf.namelist():
if name.endswith("README.md"):
content = zf.read(name).decode("utf-8")
if content:
return f"## Skill: {skill.display_title or skill.skill_id}\n\n{content}"
# Fall back to any .md file
for name in zf.namelist():
if name.endswith(".md"):
content = zf.read(name).decode("utf-8")
if content:
return f"## Skill: {skill.display_title or skill.skill_id}\n\n{content}"
except Exception as e:
verbose_logger.warning(
f"SkillPromptInjectionHandler: Error extracting content from skill {skill.skill_id}: {e}"
)
return skill.instructions
def extract_all_files(self, skill: LiteLLM_SkillsTable) -> Dict[str, bytes]:
"""
Extract ALL files from skill ZIP for code execution.
Returns a dict mapping file paths to their binary content.
The paths have the skill folder prefix removed (e.g., "slack-gif-creator/core/..." -> "core/...").
Args:
skill: The skill from LiteLLM database
Returns:
Dict mapping file paths to binary content
"""
files: Dict[str, bytes] = {}
if not skill.file_content:
return files
try:
zip_buffer = BytesIO(skill.file_content)
with zipfile.ZipFile(zip_buffer, "r") as zf:
for name in zf.namelist():
# Skip directories
if name.endswith("/"):
continue
# Remove skill folder prefix (first path component)
parts = name.split("/")
if len(parts) > 1:
clean_path = "/".join(parts[1:])
else:
clean_path = name
if clean_path:
files[clean_path] = zf.read(name)
except Exception as e:
verbose_logger.warning(
f"SkillPromptInjectionHandler: Error extracting files from skill {skill.skill_id}: {e}"
)
return files
def inject_skill_content_to_messages(
self, data: dict, skill_contents: List[str], use_anthropic_format: bool = False
) -> dict:
"""
Inject skill content into the system prompt.
For Anthropic messages API (use_anthropic_format=True):
- Injects into top-level 'system' parameter (not in messages array)
For OpenAI-style APIs (use_anthropic_format=False):
- Injects into messages array with role="system"
Args:
data: The request data dict
skill_contents: List of skill content strings to inject
use_anthropic_format: If True, use top-level 'system' param for Anthropic
Returns:
Modified data dict with skill content in system prompt
"""
if not skill_contents:
return data
# Build the skill injection text
skill_section = "\n\n---\n\n# Available Skills\n\n" + "\n\n---\n\n".join(skill_contents)
if use_anthropic_format:
# Anthropic messages API: use top-level 'system' parameter
current_system = data.get("system", "")
if current_system:
data["system"] = current_system + skill_section
else:
data["system"] = skill_section.strip()
return data
# OpenAI-style: inject into messages array
messages = data.get("messages", [])
if not messages:
return data
# Find or create system message
system_msg_idx = None
for i, msg in enumerate(messages):
if isinstance(msg, dict) and msg.get("role") == "system":
system_msg_idx = i
break
if system_msg_idx is not None:
# Append to existing system message
current_content = messages[system_msg_idx].get("content", "")
messages[system_msg_idx]["content"] = current_content + skill_section
else:
# Create new system message at the beginning
messages.insert(0, {"role": "system", "content": skill_section.strip()})
data["messages"] = messages
return data
def create_execute_code_tool(self, skill_modules: List[str]) -> Dict[str, Any]:
"""
Create the execute_code tool definition.
This tool allows the model to execute Python code with access
to the skill's modules (e.g., 'from core.gif_builder import GIFBuilder').
Args:
skill_modules: List of available module paths (e.g., ["core/gif_builder.py"])
Returns:
OpenAI-style tool definition
"""
# Format module list for description
module_examples = []
for mod in skill_modules[:5]: # Limit to 5 examples
if mod.endswith(".py"):
# Convert path to import: "core/gif_builder.py" -> "from core.gif_builder import ..."
import_path = mod.replace("/", ".").replace(".py", "")
module_examples.append(f"from {import_path} import ...")
module_hint = ""
if module_examples:
module_hint = f" Available modules: {', '.join(module_examples)}"
return {
"type": "function",
"function": {
"name": "execute_code",
"description": f"Execute Python code in a sandboxed environment. Generated files will be returned.{module_hint}",
"parameters": {
"type": "object",
"properties": {
"code": {
"type": "string",
"description": "Python code to execute. You can import skill modules and use standard libraries."
}
},
"required": ["code"]
}
}
}
def convert_skill_to_tool(self, skill: LiteLLM_SkillsTable) -> Dict[str, Any]:
"""
Convert a LiteLLM skill to an OpenAI-style tool.
The skill's instructions are used as the function description,
allowing the model to understand when and how to use the skill.
Args:
skill: The skill from LiteLLM database
Returns:
OpenAI-style tool definition
"""
# Create a function name from skill_id (sanitize for function naming)
func_name = skill.skill_id.replace("-", "_").replace(" ", "_")
# Use instructions as description, fall back to description or title
description = (
skill.instructions
or skill.description
or skill.display_title
or f"Skill: {skill.skill_id}"
)
# Truncate description if too long (OpenAI has limits)
max_desc_length = 1024
if len(description) > max_desc_length:
description = description[: max_desc_length - 3] + "..."
tool: Dict[str, Any] = {
"type": "function",
"function": {
"name": func_name,
"description": description,
"parameters": {
"type": "object",
"properties": {},
"required": [],
},
},
}
# If skill has metadata with parameter definitions, use them
if skill.metadata and isinstance(skill.metadata, dict):
params = skill.metadata.get("parameters")
if params and isinstance(params, dict):
tool["function"]["parameters"] = params
return tool
def convert_skill_to_anthropic_tool(self, skill: LiteLLM_SkillsTable) -> Dict[str, Any]:
"""
Convert a LiteLLM skill to an Anthropic-style tool (messages API format).
Args:
skill: The skill from LiteLLM database
Returns:
Anthropic-style tool definition with name, description, input_schema
"""
func_name = skill.skill_id.replace("-", "_").replace(" ", "_")
description = (
skill.instructions
or skill.description
or skill.display_title
or f"Skill: {skill.skill_id}"
)
max_desc_length = 1024
if len(description) > max_desc_length:
description = description[: max_desc_length - 3] + "..."
input_schema: Dict[str, Any] = {
"type": "object",
"properties": {},
"required": [],
}
if skill.metadata and isinstance(skill.metadata, dict):
params = skill.metadata.get("parameters")
if params and isinstance(params, dict):
input_schema = params
return {
"name": func_name,
"description": description,
"input_schema": input_schema,
}

286
litellm/llms/litellm_proxy/skills/sandbox_executor.py Normal file
View file

@ -0,0 +1,286 @@
"""
Sandbox Executor for LiteLLM Skills
Executes skill code in a sandboxed environment using llm-sandbox.
Supports Docker, Podman, and Kubernetes backends.
"""
import base64
import os
from typing import Any, Dict, List, Optional
from litellm._logging import verbose_logger
class SkillsSandboxExecutor:
"""
Executes skill code in llm-sandbox Docker container.
Responsibilities:
- Create sandbox session with skill files
- Install requirements
- Execute model-generated code
- Collect generated files (GIFs, images, etc.)
"""
def __init__(
self,
timeout: int = 60,
backend: str = "docker",
image: Optional[str] = None,
):
"""
Initialize the sandbox executor.
Args:
timeout: Maximum execution time in seconds
backend: Sandbox backend ("docker", "podman", "kubernetes")
image: Custom Docker image (default: uses llm-sandbox default)
"""
self.timeout = timeout
self.backend = backend
self.image = image
self._session = None
def execute(
self,
code: str,
skill_files: Dict[str, bytes],
requirements: Optional[str] = None,
) -> Dict[str, Any]:
"""
Execute code with skill files in sandbox.
Args:
code: Python code to execute
skill_files: Dict mapping file paths to binary content
requirements: Optional requirements.txt content
Returns:
{
"success": bool,
"output": str,
"error": str (if failed),
"files": [{"name": str, "content_base64": str, "mime_type": str}]
}
"""
try:
from llm_sandbox import SandboxSession
except ImportError:
verbose_logger.error(
"SkillsSandboxExecutor: llm-sandbox not installed. "
"Install with: pip install llm-sandbox"
)
return {
"success": False,
"output": "",
"error": "llm-sandbox not installed. Install with: pip install llm-sandbox",
"files": [],
}
try:
# Create sandbox session
session_kwargs: Dict[str, Any] = {
"lang": "python",
"verbose": False,
}
if self.image:
session_kwargs["image"] = self.image
with SandboxSession(**session_kwargs) as session:
# 1. Copy skill files into sandbox using copy_to_runtime
import tempfile
# Create a temp directory to stage files
with tempfile.TemporaryDirectory() as tmpdir:
for path, content in skill_files.items():
# Create the file in temp directory
local_path = os.path.join(tmpdir, path)
os.makedirs(os.path.dirname(local_path), exist_ok=True)
with open(local_path, "wb") as f:
f.write(content)
# Copy to sandbox
sandbox_path = f"/sandbox/{path}"
session.copy_to_runtime(local_path, sandbox_path)
verbose_logger.debug(
f"SkillsSandboxExecutor: Copied {len(skill_files)} files to sandbox"
)
# 2. Install requirements if present
req_packages = None
if requirements:
req_packages = requirements.strip().replace("\n", " ")
elif "requirements.txt" in skill_files:
req_content = skill_files["requirements.txt"].decode("utf-8")
req_packages = req_content.strip().replace("\n", " ")
if req_packages:
# Run pip install as code
pip_code = f"""
import subprocess
subprocess.run(['pip', 'install'] + '{req_packages}'.split(), check=True)
"""
result = session.run(pip_code)
verbose_logger.debug(
"SkillsSandboxExecutor: Installed requirements"
)
# 3. Execute the code
# Wrap code to run from /sandbox directory
wrapped_code = f"""
import os
os.chdir('/sandbox')
import sys
sys.path.insert(0, '/sandbox')
{code}
"""
result = session.run(wrapped_code)
success = result.exit_code == 0
output = result.stdout or ""
error = result.stderr or ""
if success:
verbose_logger.debug(
"SkillsSandboxExecutor: Code execution succeeded"
)
else:
verbose_logger.debug(
f"SkillsSandboxExecutor: Code execution failed with exit code {result.exit_code}"
)
verbose_logger.debug(
f"SkillsSandboxExecutor: stderr: {error[:500] if error else 'No stderr'}"
)
verbose_logger.debug(
f"SkillsSandboxExecutor: stdout: {output[:500] if output else 'No stdout'}"
)
# 4. Collect generated files
generated_files = self._collect_generated_files(session, skill_files)
return {
"success": success,
"output": output,
"error": error,
"files": generated_files,
}
except Exception as e:
verbose_logger.error(
f"SkillsSandboxExecutor: Execution failed: {e}"
)
return {
"success": False,
"output": "",
"error": str(e),
"files": [],
}
def _collect_generated_files(
self,
session: Any,
original_files: Dict[str, bytes],
) -> List[Dict[str, Any]]:
"""
Collect files generated during execution.
Looks for new files in /sandbox that weren't in the original skill files.
Focuses on common output types: GIF, PNG, JPG, PDF, CSV, etc.
Args:
session: The sandbox session
original_files: Original skill files (to exclude)
Returns:
List of generated files with base64 content
"""
generated_files: List[Dict[str, Any]] = []
try:
import tempfile
# List files in /sandbox using Python code
list_code = """
import os
import json
files = []
for root, dirs, filenames in os.walk('/sandbox'):
for f in filenames:
if f.endswith(('.gif', '.png', '.jpg', '.jpeg', '.pdf', '.csv', '.json')):
files.append(os.path.join(root, f))
print(json.dumps(files))
"""
result = session.run(list_code)
if result.exit_code == 0 and result.stdout:
import json
try:
filepaths = json.loads(result.stdout.strip())
except json.JSONDecodeError:
filepaths = []
for filepath in filepaths:
if not filepath:
continue
# Get relative path
rel_path = filepath.replace("/sandbox/", "")
# Skip if it was an original file
if rel_path in original_files:
continue
# Copy file from sandbox using copy_from_runtime
with tempfile.NamedTemporaryFile(delete=False) as tmp:
tmp_path = tmp.name
try:
session.copy_from_runtime(filepath, tmp_path)
with open(tmp_path, "rb") as f:
content = f.read()
content_b64 = base64.b64encode(content).decode("utf-8")
generated_files.append({
"name": os.path.basename(filepath),
"path": rel_path,
"content_base64": content_b64,
"mime_type": self._get_mime_type(filepath),
})
verbose_logger.debug(
f"SkillsSandboxExecutor: Collected generated file: {rel_path}"
)
except Exception as e:
verbose_logger.warning(
f"SkillsSandboxExecutor: Error copying file {filepath}: {e}"
)
finally:
if os.path.exists(tmp_path):
os.unlink(tmp_path)
except Exception as e:
verbose_logger.warning(
f"SkillsSandboxExecutor: Error collecting generated files: {e}"
)
return generated_files
def _get_mime_type(self, filename: str) -> str:
"""Get MIME type for a file based on extension."""
ext = filename.lower().split(".")[-1]
return {
"gif": "image/gif",
"png": "image/png",
"jpg": "image/jpeg",
"jpeg": "image/jpeg",
"pdf": "application/pdf",
"csv": "text/csv",
"json": "application/json",
"txt": "text/plain",
}.get(ext, "application/octet-stream")

336
litellm/llms/litellm_proxy/skills/transformation.py Normal file
View file

@ -0,0 +1,336 @@
"""
Transformation handler for LiteLLM database-backed skills.
This module provides the SDK-level transformation layer that converts
API requests to database operations via LiteLLMSkillsHandler.
Pattern follows litellm/llms/litellm_proxy/responses/transformation.py
"""
from typing import TYPE_CHECKING, Any, Coroutine, Dict, List, Optional, Union
from litellm.types.llms.anthropic_skills import (
DeleteSkillResponse,
ListSkillsResponse,
Skill,
)
from litellm.types.utils import LlmProviders
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
class LiteLLMSkillsTransformationHandler:
"""
Transformation handler for skills API requests to LiteLLM database operations.
This is used when custom_llm_provider="litellm_proxy" to store/retrieve skills
from the LiteLLM proxy database instead of calling an external API.
"""
@property
def custom_llm_provider(self) -> str:
"""Return the provider name for logging."""
return LlmProviders.LITELLM_PROXY.value
def create_skill_handler(
self,
display_title: Optional[str] = None,
description: Optional[str] = None,
instructions: Optional[str] = None,
files: Optional[List[Any]] = None,
file_content: Optional[bytes] = None,
file_name: Optional[str] = None,
file_type: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None,
user_id: Optional[str] = None,
_is_async: bool = False,
logging_obj: Optional["LiteLLMLoggingObj"] = None,
litellm_call_id: Optional[str] = None,
**kwargs,
) -> Union[Skill, Coroutine[Any, Any, Skill]]:
"""
Create a skill in LiteLLM database.
Args:
display_title: Display title for the skill
description: Description of the skill
instructions: Instructions/prompt for the skill
files: Files to upload - list of tuples (filename, content, content_type)
file_content: Binary content of skill files (alternative to files)
file_name: Original filename (alternative to files)
file_type: MIME type (alternative to files)
metadata: Additional metadata
user_id: User ID for tracking
_is_async: Whether to return a coroutine
Returns:
Skill object or coroutine that returns Skill
"""
# Pre-call logging
if logging_obj:
logging_obj.update_environment_variables(
model=None,
optional_params={"display_title": display_title},
litellm_params={"litellm_call_id": litellm_call_id},
custom_llm_provider=self.custom_llm_provider,
)
# Extract file content from files parameter if provided
# files is a list of tuples: [(filename, content, content_type), ...]
if files and not file_content:
if isinstance(files, list) and len(files) > 0:
first_file = files[0]
if isinstance(first_file, tuple) and len(first_file) >= 2:
file_name = first_file[0]
file_content = first_file[1]
file_type = first_file[2] if len(first_file) > 2 else "application/zip"
if _is_async:
return self._async_create_skill(
display_title=display_title,
description=description,
instructions=instructions,
file_content=file_content,
file_name=file_name,
file_type=file_type,
metadata=metadata,
user_id=user_id,
)
import asyncio
return asyncio.get_event_loop().run_until_complete(
self._async_create_skill(
display_title=display_title,
description=description,
instructions=instructions,
file_content=file_content,
file_name=file_name,
file_type=file_type,
metadata=metadata,
user_id=user_id,
)
)
async def _async_create_skill(
self,
display_title: Optional[str] = None,
description: Optional[str] = None,
instructions: Optional[str] = None,
file_content: Optional[bytes] = None,
file_name: Optional[str] = None,
file_type: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None,
user_id: Optional[str] = None,
) -> Skill:
"""Async implementation of create_skill."""
# Lazy import to avoid SDK dependency on proxy
from litellm.llms.litellm_proxy.skills.handler import LiteLLMSkillsHandler
from litellm.proxy._types import NewSkillRequest
skill_request = NewSkillRequest(
display_title=display_title,
description=description,
instructions=instructions,
file_content=file_content,
file_name=file_name,
file_type=file_type,
metadata=metadata,
)
db_skill = await LiteLLMSkillsHandler.create_skill(
data=skill_request,
user_id=user_id,
)
return self._db_skill_to_response(db_skill)
def list_skills_handler(
self,
limit: int = 20,
offset: int = 0,
_is_async: bool = False,
logging_obj: Optional["LiteLLMLoggingObj"] = None,
litellm_call_id: Optional[str] = None,
**kwargs,
) -> Union[ListSkillsResponse, Coroutine[Any, Any, ListSkillsResponse]]:
"""
List skills from LiteLLM database.
Args:
limit: Maximum number of skills to return
offset: Number of skills to skip
_is_async: Whether to return a coroutine
logging_obj: LiteLLM logging object
litellm_call_id: Call ID for logging
Returns:
ListSkillsResponse or coroutine that returns ListSkillsResponse
"""
# Pre-call logging
if logging_obj:
logging_obj.update_environment_variables(
model=None,
optional_params={"limit": limit, "offset": offset},
litellm_params={"litellm_call_id": litellm_call_id},
custom_llm_provider=self.custom_llm_provider,
)
if _is_async:
return self._async_list_skills(limit=limit, offset=offset)
import asyncio
return asyncio.get_event_loop().run_until_complete(
self._async_list_skills(limit=limit, offset=offset)
)
async def _async_list_skills(
self,
limit: int = 20,
offset: int = 0,
) -> ListSkillsResponse:
"""Async implementation of list_skills."""
# Lazy import to avoid SDK dependency on proxy
from litellm.llms.litellm_proxy.skills.handler import LiteLLMSkillsHandler
db_skills = await LiteLLMSkillsHandler.list_skills(
limit=limit,
offset=offset,
)
skills = [self._db_skill_to_response(s) for s in db_skills]
return ListSkillsResponse(
data=skills,
has_more=len(skills) >= limit,
next_page=None,
)
def get_skill_handler(
self,
skill_id: str,
_is_async: bool = False,
logging_obj: Optional["LiteLLMLoggingObj"] = None,
litellm_call_id: Optional[str] = None,
**kwargs,
) -> Union[Skill, Coroutine[Any, Any, Skill]]:
"""
Get a skill from LiteLLM database.
Args:
skill_id: The skill ID to retrieve
_is_async: Whether to return a coroutine
logging_obj: LiteLLM logging object
litellm_call_id: Call ID for logging
Returns:
Skill or coroutine that returns Skill
"""
# Pre-call logging
if logging_obj:
logging_obj.update_environment_variables(
model=None,
optional_params={"skill_id": skill_id},
litellm_params={"litellm_call_id": litellm_call_id},
custom_llm_provider=self.custom_llm_provider,
)
if _is_async:
return self._async_get_skill(skill_id=skill_id)
import asyncio
return asyncio.get_event_loop().run_until_complete(
self._async_get_skill(skill_id=skill_id)
)
async def _async_get_skill(self, skill_id: str) -> Skill:
"""Async implementation of get_skill."""
# Lazy import to avoid SDK dependency on proxy
from litellm.llms.litellm_proxy.skills.handler import LiteLLMSkillsHandler
db_skill = await LiteLLMSkillsHandler.get_skill(skill_id=skill_id)
return self._db_skill_to_response(db_skill)
def delete_skill_handler(
self,
skill_id: str,
_is_async: bool = False,
logging_obj: Optional["LiteLLMLoggingObj"] = None,
litellm_call_id: Optional[str] = None,
**kwargs,
) -> Union[DeleteSkillResponse, Coroutine[Any, Any, DeleteSkillResponse]]:
"""
Delete a skill from LiteLLM database.
Args:
skill_id: The skill ID to delete
_is_async: Whether to return a coroutine
logging_obj: LiteLLM logging object
litellm_call_id: Call ID for logging
Returns:
DeleteSkillResponse or coroutine that returns DeleteSkillResponse
"""
# Pre-call logging
if logging_obj:
logging_obj.update_environment_variables(
model=None,
optional_params={"skill_id": skill_id},
litellm_params={"litellm_call_id": litellm_call_id},
custom_llm_provider=self.custom_llm_provider,
)
if _is_async:
return self._async_delete_skill(skill_id=skill_id)
import asyncio
return asyncio.get_event_loop().run_until_complete(
self._async_delete_skill(skill_id=skill_id)
)
async def _async_delete_skill(self, skill_id: str) -> DeleteSkillResponse:
"""Async implementation of delete_skill."""
# Lazy import to avoid SDK dependency on proxy
from litellm.llms.litellm_proxy.skills.handler import LiteLLMSkillsHandler
result = await LiteLLMSkillsHandler.delete_skill(skill_id=skill_id)
return DeleteSkillResponse(
id=result["id"],
type=result.get("type", "skill_deleted"),
)
def _db_skill_to_response(self, db_skill: Any) -> Skill:
"""
Convert a database skill record to Anthropic-compatible Skill response.
Args:
db_skill: LiteLLM_SkillsTable record
Returns:
Skill object
"""
created_at = ""
updated_at = ""
if hasattr(db_skill, "created_at") and db_skill.created_at:
created_at = (
db_skill.created_at.isoformat()
if hasattr(db_skill.created_at, "isoformat")
else str(db_skill.created_at)
)
if hasattr(db_skill, "updated_at") and db_skill.updated_at:
updated_at = (
db_skill.updated_at.isoformat()
if hasattr(db_skill.updated_at, "isoformat")
else str(db_skill.updated_at)
)
return Skill(
id=db_skill.skill_id,
created_at=created_at,
updated_at=updated_at,
display_title=db_skill.display_title,
latest_version=db_skill.latest_version,
source=db_skill.source or "custom",
type="skill",
)

3
litellm/llms/openai/responses/transformation.py
View file

@ -96,8 +96,8 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
validated_input.append(item.model_dump(exclude_none=True))
elif isinstance(item, dict):
# Handle reasoning items specifically to filter out status=None
verbose_logger.debug(f"Handling reasoning item: {item}")
if item.get("type") == "reasoning":
verbose_logger.debug(f"Handling reasoning item: {item}")
# Type assertion since we know it's a dict at this point
dict_item = cast(Dict[str, Any], item)
filtered_item = self._handle_reasoning_item(dict_item)
@ -411,7 +411,6 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
)
raw_response_headers = dict(raw_response.headers)
processed_headers = process_response_headers(raw_response_headers)
response = ResponsesAPIResponse(**raw_response_json)
response._hidden_params["additional_headers"] = processed_headers
response._hidden_params["headers"] = raw_response_headers

37
litellm/llms/stability/image_edit/__init__.py Normal file
View file

@ -0,0 +1,37 @@
"""
Stability AI Image Edit Module
Factory function for getting the appropriate config class.
"""
from litellm.llms.base_llm.image_edit.transformation import (
BaseImageEditConfig,
)
from .transformations import StabilityImageEditConfig
__all__ = [
"StabilityImageEditConfig",
"get_stability_image_edit_config",
]
def get_stability_image_edit_config(model: str) -> BaseImageEditConfig:
"""
Get the appropriate Stability AI config for the given model.
Currently all models use the same config class, but this factory
allows for model-specific configs in the future.
Args:
model: The model name (e.g., "stability/inpaint", "stability/outpaint")
Returns:
BaseImageEditConfig instance for Stability AI
"""
# For now, all models use the same config
# In the future, we could have model-specific configs:
# - StabilityInpaintConfig for Inpaint models
# - StabilityOutpaintConfig for Outpaint models
# - etc.
return StabilityImageEditConfig()

314
litellm/llms/stability/image_edit/transformations.py Normal file
View file

@ -0,0 +1,314 @@
"""
Stability AI Image Edit Config
Handles transformation between OpenAI-compatible format and Stability AI API format.
API Reference: https://platform.stability.ai/docs/api-reference
"""
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
import httpx
from httpx._types import RequestFiles
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.images.main import ImageEditOptionalRequestParams
from litellm.types.router import GenericLiteLLMParams
from litellm.types.llms.stability import (
OPENAI_SIZE_TO_STABILITY_ASPECT_RATIO,
STABILITY_EDIT_ENDPOINTS,
)
from litellm.types.utils import FileTypes, ImageObject, ImageResponse
from litellm.utils import get_model_info
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
LiteLLMLoggingObj = Any
class StabilityImageEditConfig(BaseImageEditConfig):
"""
Configuration for Stability AI image edit.
Supports:
- Stable Diffusion 3 (SD3, SD3.5) Image Edit
"""
DEFAULT_BASE_URL: str = "https://api.stability.ai"
def get_supported_openai_params(
self, model: str
) -> List[str]:
"""
Return list of OpenAI params supported by Stability AI.
https://platform.stability.ai/docs/api-reference
"""
return [
"n", # Number of images (Stability always returns 1, we can loop)
"size", # Maps to aspect_ratio
"response_format", # b64_json or url (Stability only returns b64)
"mask"
]
def map_openai_params(
self,
image_edit_optional_params: ImageEditOptionalRequestParams,
model: str,
drop_params: bool,
) -> Dict:
"""
Map OpenAI parameters to Stability AI parameters.
OpenAI -> Stability mappings:
- size -> aspect_ratio
- n -> (handled separately, Stability returns 1 image per request)
"""
supported_params = self.get_supported_openai_params(model)
# Define mapping from OpenAI params to Stability params
param_mapping = {
"size": "aspect_ratio",
# "n" and "response_format" are handled separately
}
# Create a copy to not mutate original - convert TypedDict to regular dict
mapped_params: Dict[str, Any] = dict(image_edit_optional_params)
for k, v in image_edit_optional_params.items():
if k in param_mapping:
# Map param if mapping exists and value is valid
if k == "size" and v in OPENAI_SIZE_TO_STABILITY_ASPECT_RATIO:
mapped_params[param_mapping[k]] = OPENAI_SIZE_TO_STABILITY_ASPECT_RATIO[v] # type: ignore
# Don't copy "size" itself to final dict
elif k == "n":
# Store for logic but do not add to outgoing params
mapped_params["_n"] = v
elif k == "response_format":
# Only b64 supported at Stability; store for postprocessing
mapped_params["_response_format"] = v
elif k not in supported_params:
if not drop_params:
raise ValueError(
f"Parameter {k} is not supported for model {model}. "
f"Supported parameters are {supported_params}. "
f"Set drop_params=True to drop unsupported parameters."
)
# Otherwise, param will simply be dropped
else:
# param is supported and not mapped, keep as-is
continue
# Remove OpenAI params that have been mapped unless they're in stability
for mapped in ["size", "n", "response_format"]:
if mapped in mapped_params:
del mapped_params[mapped]
return mapped_params
def _get_model_endpoint(self, model: str) -> str:
"""
Get the API endpoint for a given model.
"""
# Remove "stability/" prefix if present
model_name = model.lower()
if model_name.startswith("stability/"):
model_name = model_name[10:] # Remove "stability/" prefix
# Check if model is in our mapping
for key, endpoint in STABILITY_EDIT_ENDPOINTS.items():
if key in model_name:
return endpoint
# Default to SD3 endpoint
return "/v2beta/stable-image/edit/inpaint"
def get_complete_url(
self,
model: str,
api_base: Optional[str],
litellm_params: dict,
) -> str:
"""
Get the complete URL for the Stability AI API request.
"""
base_url: str = (
api_base
or get_secret_str("STABILITY_API_BASE")
or litellm_params.get("api_base", None)
or self.DEFAULT_BASE_URL
)
base_url = base_url.rstrip("/")
endpoint = self._get_model_endpoint(model)
return f"{base_url}{endpoint}"
def validate_environment(
self,
headers: dict,
model: str,
api_key: Optional[str] = None,
) -> dict:
"""
Validate environment and set up headers for Stability AI.
"""
final_api_key: Optional[str] = api_key or get_secret_str("STABILITY_API_KEY")
if not final_api_key:
raise ValueError(
"STABILITY_API_KEY is not set. "
"Please set it via environment variable or pass api_key parameter."
)
headers["Authorization"] = f"Bearer {final_api_key}"
headers["Accept"] = "application/json"
return headers
def transform_image_edit_request(
self,
model: str,
prompt: str,
image: FileTypes,
image_edit_optional_request_params: Dict,
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> Tuple[Dict, RequestFiles]:
"""
Transform OpenAI-style request to Stability AI request format.
Note: Stability AI uses multipart/form-data, but the HTTP handler
will handle the conversion from dict to form data.
"""
# Build Stability request
# Populate multipart form-data as separate text fields (data) and files.
# Stability expects prompt/output_format/etc. as normal form fields, not file parts.
data: Dict[str, Any] = {
"prompt": prompt,
"output_format": "png", # Default to PNG
}
# Handle image parameter - could be a single file or list
image_file = image[0] if isinstance(image, list) else image # type: ignore
files: Dict[str, Any] = {"image": image_file}
# Add optional params (already mapped in map_openai_params)
for key, value in image_edit_optional_request_params.items(): # type: ignore
# Skip internal params (prefixed with _)
if key.startswith("_") or value is None:
continue
# File-like optional param
if key == "mask":
# Handle case where mask might be in a list
mask_value = value
if isinstance(value, list) and len(value) > 0:
mask_value = value[0]
files["mask"] = mask_value # type: ignore
continue
# File-like optional params (init_image, style_image, etc.)
if key in ["init_image", "style_image"]:
# Handle case where value might be in a list
file_value = value
if isinstance(value, list) and len(value) > 0:
file_value = value[0]
files[key] = file_value # type: ignore
continue
# Supported text fields
if key in [
"negative_prompt",
"aspect_ratio",
"seed",
"mode",
"strength",
"style_preset",
"left",
"bottom",
"right",
"top",
"creativity",
"search_prompt",
"grow_mask",
"select_prompt",
"control_strength",
"composition_fidelity",
"change_strength"
]:
data[key] = value # type: ignore
return data, files
def transform_image_edit_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
api_key: Optional[str] = None,
json_mode: Optional[bool] = None,
) -> ImageResponse:
"""
Transform Stability AI response to OpenAI-compatible ImageResponse.
Stability returns: {"image": "base64...", "finish_reason": "SUCCESS", "seed": 123}
OpenAI expects: {"data": [{"b64_json": "base64..."}], "created": timestamp}
"""
try:
response_data = raw_response.json()
except Exception as e:
raise self.get_error_class(
error_message=f"Error parsing Stability AI response: {e}",
status_code=raw_response.status_code,
headers=raw_response.headers,
)
# Check for errors in response
if "errors" in response_data:
raise self.get_error_class(
error_message=f"Stability AI error: {response_data['errors']}",
status_code=raw_response.status_code,
headers=raw_response.headers,
)
# Check finish_reason
finish_reason = response_data.get("finish_reason", "")
if finish_reason == "CONTENT_FILTERED":
raise self.get_error_class(
error_message="Content was filtered by Stability AI safety systems",
status_code=400,
headers=raw_response.headers,
)
model_response = ImageResponse()
if not model_response.data:
model_response.data = []
# Extract image from response
image_b64 = response_data.get("image")
if image_b64:
model_response.data.append(
ImageObject(
b64_json=image_b64,
url=None,
revised_prompt=None,
)
)
if not hasattr(model_response, "_hidden_params"):
model_response._hidden_params = {}
if "additional_headers" not in model_response._hidden_params:
model_response._hidden_params["additional_headers"] = {}
# Override: fetch model-cost from model_cost map based on the provided model name
model_info = get_model_info(model, custom_llm_provider="stability")
cost_per_image = model_info.get("output_cost_per_image", 0)
if cost_per_image is not None:
model_response._hidden_params["additional_headers"]["llm_provider-x-litellm-response-cost"] = float(cost_per_image)
return model_response
def use_multipart_form_data(self) -> bool:
"""
Stability AI requires multipart/form-data for image generation.
"""
return True

20
litellm/llms/vertex_ai/common_utils.py
View file

@ -640,14 +640,28 @@ def add_object_type(schema):
if properties is not None:
if "required" in schema and schema["required"] is None:
schema.pop("required", None)
schema["type"] = "object"
for name, value in properties.items():
add_object_type(value)
# Gemini doesn't accept empty properties for object types
# If properties is empty, remove it and the type field
if not properties:
schema.pop("properties", None)
schema.pop("type", None)
schema.pop("required", None)
else:
schema["type"] = "object"
for name, value in properties.items():
add_object_type(value)
items = schema.get("items", None)
if items is not None:
add_object_type(items)
for key in ["anyOf", "oneOf", "allOf"]:
values = schema.get(key, None)
if values is not None and isinstance(values, list):
for value in values:
if isinstance(value, dict):
add_object_type(value)
def strip_field(schema, field_name: str):
schema.pop(field_name, None)

41
litellm/llms/vertex_ai/ocr/common_utils.py Normal file
View file

@ -0,0 +1,41 @@
"""
Common utilities for Vertex AI OCR providers.
This module provides routing logic to determine which OCR configuration to use
based on the model name.
"""
from typing import TYPE_CHECKING, Optional
if TYPE_CHECKING:
from litellm.llms.base_llm.ocr.transformation import BaseOCRConfig
def get_vertex_ai_ocr_config(model: str) -> Optional["BaseOCRConfig"]:
"""
Determine which Vertex AI OCR configuration to use based on the model name.
Vertex AI supports multiple OCR services:
- Vertex AI OCR: vertex_ai/<model>
Args:
model: The model name (e.g., "vertex_ai/ocr/<model>")
Returns:
OCR configuration instance for the specified model
Examples:
>>> get_vertex_ai_ocr_config("vertex_ai/deepseek-ai/deepseek-ocr-maas")
<VertexAIDeepSeekOCRConfig object>
>>> get_vertex_ai_ocr_config("vertex_ai/ocr/mistral-ocr-maas")
<VertexAIOCRConfig object>
"""
from litellm.llms.vertex_ai.ocr.deepseek_transformation import (
VertexAIDeepSeekOCRConfig,
)
from litellm.llms.vertex_ai.ocr.transformation import VertexAIOCRConfig
if "deepseek" in model:
return VertexAIDeepSeekOCRConfig()
return VertexAIOCRConfig()

394
litellm/llms/vertex_ai/ocr/deepseek_transformation.py Normal file
View file

@ -0,0 +1,394 @@
"""
Vertex AI DeepSeek OCR transformation implementation.
"""
import json
from typing import TYPE_CHECKING, Any, Dict, Optional
import httpx
from litellm._logging import verbose_logger
from litellm.llms.base_llm.ocr.transformation import (
BaseOCRConfig,
DocumentType,
OCRPage,
OCRRequestData,
OCRResponse,
OCRUsageInfo,
)
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
else:
LiteLLMLoggingObj = Any
class VertexAIDeepSeekOCRConfig(BaseOCRConfig):
"""
Vertex AI DeepSeek OCR transformation configuration.
Vertex AI DeepSeek OCR uses the chat completion API format through the openapi endpoint.
This transformation converts OCR requests to chat completion format and vice versa.
"""
def __init__(self) -> None:
super().__init__()
self.vertex_base = VertexBase()
def validate_environment(
self,
headers: Dict,
model: str,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
litellm_params: Optional[dict] = None,
**kwargs,
) -> Dict:
"""
Validate environment and return headers for Vertex AI OCR.
Vertex AI uses Bearer token authentication with access token from credentials.
"""
# Extract Vertex AI parameters using safe helpers from VertexBase
# Use safe_get_* methods that don't mutate litellm_params dict
litellm_params = litellm_params or {}
vertex_project = VertexBase.safe_get_vertex_ai_project(litellm_params=litellm_params)
vertex_credentials = VertexBase.safe_get_vertex_ai_credentials(litellm_params=litellm_params)
# Get access token from Vertex credentials
access_token, project_id = self.vertex_base.get_access_token(
credentials=vertex_credentials,
project_id=vertex_project,
)
headers = {
"Authorization": f"Bearer {access_token}",
"Content-Type": "application/json",
**headers,
}
return headers
def get_complete_url(
self,
api_base: Optional[str],
model: str,
optional_params: dict,
litellm_params: Optional[dict] = None,
**kwargs,
) -> str:
"""
Get complete URL for Vertex AI DeepSeek OCR endpoint.
Vertex AI endpoint format:
https://{location}-aiplatform.googleapis.com/v1/projects/{project}/locations/{location}/endpoints/openapi/chat/completions
Args:
api_base: Vertex AI API base URL (optional)
model: Model name (e.g., "deepseek-ai/deepseek-ocr-maas")
optional_params: Optional parameters
litellm_params: LiteLLM parameters containing vertex_project, vertex_location
Returns: Complete URL for Vertex AI OCR endpoint
"""
# Extract Vertex AI parameters using safe helpers from VertexBase
# Use safe_get_* methods that don't mutate litellm_params dict
litellm_params = litellm_params or {}
vertex_project = VertexBase.safe_get_vertex_ai_project(litellm_params=litellm_params)
vertex_location = VertexBase.safe_get_vertex_ai_location(litellm_params=litellm_params)
if vertex_project is None:
raise ValueError(
"Missing vertex_project - Set VERTEXAI_PROJECT environment variable or pass vertex_project parameter"
)
if vertex_location is None:
vertex_location = "us-central1"
# Get API base URL
if api_base is None:
api_base = "https://aiplatform.googleapis.com"
# Ensure no trailing slash
api_base = api_base.rstrip("/")
# Vertex AI DeepSeek OCR endpoint format
# Format: https://{region}-aiplatform.googleapis.com/v1/projects/{project}/locations/{region}/endpoints/openapi/chat/completions
return f"{api_base}/v1/projects/{vertex_project}/locations/{vertex_location}/endpoints/openapi/chat/completions"
def transform_ocr_request(
self,
model: str,
document: DocumentType,
optional_params: dict,
headers: dict,
**kwargs,
) -> OCRRequestData:
"""
Transform OCR request to chat completion format for Vertex AI DeepSeek OCR.
Converts OCR document format to chat completion messages format:
- Input: {"type": "image_url", "image_url": "gs://..."}
- Output: {"model": "deepseek-ai/deepseek-ocr-maas", "messages": [{"role": "user", "content": [{"type": "image_url", "image_url": "gs://..."}]}]}
Args:
model: Model name (e.g., "deepseek-ai/deepseek-ocr-maas")
document: Document dict from user (Mistral OCR format)
optional_params: Already mapped optional parameters
headers: Request headers
**kwargs: Additional arguments
Returns:
OCRRequestData with JSON data in chat completion format
"""
verbose_logger.debug("Vertex AI DeepSeek OCR transform_ocr_request (sync) called")
if not isinstance(document, dict):
raise ValueError(f"Expected document dict, got {type(document)}")
# Extract document type and URL
doc_type = document.get("type")
image_url = None
document_url = None
if doc_type == "image_url":
image_url = document.get("image_url", "")
elif doc_type == "document_url":
document_url = document.get("document_url", "")
else:
raise ValueError(f"Unsupported document type: {doc_type}. Expected 'image_url' or 'document_url'")
# Build chat completion message content
content_item = {}
if image_url:
content_item = {
"type": "image_url",
"image_url": image_url
}
elif document_url:
# For document URLs, we use image_url type as well (Vertex AI supports both)
content_item = {
"type": "image_url",
"image_url": document_url
}
# Build chat completion request
data = {
"model": "deepseek-ai/" + model,
"messages": [
{
"role": "user",
"content": [content_item]
}
]
}
# Add optional parameters (stream, temperature, etc.)
# Filter out OCR-specific params that don't apply to chat completion
chat_completion_params = {}
for key, value in optional_params.items():
# Include common chat completion params
if key in ["stream", "temperature", "max_tokens", "top_p", "n", "stop"]:
chat_completion_params[key] = value
data.update(chat_completion_params)
verbose_logger.debug("Vertex AI DeepSeek OCR: Transformed request to chat completion format")
return OCRRequestData(data=data, files=None)
async def async_transform_ocr_request(
self,
model: str,
document: DocumentType,
optional_params: dict,
headers: dict,
**kwargs,
) -> OCRRequestData:
"""
Transform OCR request to chat completion format for Vertex AI DeepSeek OCR (async).
Same as sync version - no async-specific logic needed.
Args:
model: Model name
document: Document dict from user
optional_params: Already mapped optional parameters
headers: Request headers
**kwargs: Additional arguments
Returns:
OCRRequestData with JSON data in chat completion format
"""
return self.transform_ocr_request(
model=model,
document=document,
optional_params=optional_params,
headers=headers,
**kwargs,
)
def transform_ocr_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
**kwargs,
) -> OCRResponse:
"""
Transform chat completion response to OCR format.
Vertex AI DeepSeek OCR returns chat completion format:
{
"id": "...",
"object": "chat.completion",
"choices": [{
"message": {
"role": "assistant",
"content": "<OCR result as JSON string or markdown>"
}
}],
"usage": {...}
}
We need to extract the content and convert it to OCRResponse format.
Args:
model: Model name
raw_response: Raw HTTP response from Vertex AI
logging_obj: Logging object
**kwargs: Additional arguments
Returns:
OCRResponse in standard format
"""
verbose_logger.debug("Vertex AI DeepSeek OCR transform_ocr_response called")
verbose_logger.debug(f"Raw response: {raw_response.text}")
try:
response_json = raw_response.json()
# Extract content from chat completion response
choices = response_json.get("choices", [])
if not choices:
raise ValueError("No choices in chat completion response")
message = choices[0].get("message", {})
content = message.get("content", "")
if not content:
raise ValueError("No content in chat completion response")
# Try to parse content as JSON (OCR result might be JSON string)
ocr_data = None
try:
# If content is a JSON string, parse it
if isinstance(content, str) and content.strip().startswith("{"):
ocr_data = json.loads(content)
elif isinstance(content, dict):
ocr_data = content
else:
# If content is markdown text, create a single page with the markdown
ocr_data = {
"pages": [
{
"index": 0,
"markdown": content
}
],
"model": model,
"usage_info": response_json.get("usage", {})
}
except json.JSONDecodeError:
# If JSON parsing fails, treat content as markdown
ocr_data = {
"pages": [
{
"index": 0,
"markdown": content
}
],
"model": model,
"usage_info": response_json.get("usage", {})
}
# Ensure we have the expected structure
if "pages" not in ocr_data:
# If OCR data doesn't have pages, wrap the content in a page
ocr_data = {
"pages": [
{
"index": 0,
"markdown": content if isinstance(content, str) else json.dumps(content)
}
],
"model": ocr_data.get("model", model),
"usage_info": ocr_data.get("usage_info", response_json.get("usage", {}))
}
# Convert usage info if present
usage_info = None
if "usage_info" in ocr_data:
usage_dict = ocr_data["usage_info"]
if isinstance(usage_dict, dict):
usage_info = OCRUsageInfo(**usage_dict)
# Build OCRResponse
pages = []
for page_data in ocr_data.get("pages", []):
# Ensure page has required fields
if isinstance(page_data, dict):
page = OCRPage(
index=page_data.get("index", 0),
markdown=page_data.get("markdown", ""),
images=page_data.get("images"),
dimensions=page_data.get("dimensions")
)
pages.append(page)
if not pages:
# Create a default page if none exist
pages = [OCRPage(index=0, markdown=content if isinstance(content, str) else "")]
return OCRResponse(
pages=pages,
model=ocr_data.get("model", model),
document_annotation=ocr_data.get("document_annotation"),
usage_info=usage_info,
object="ocr",
)
except Exception as e:
verbose_logger.error(f"Error parsing Vertex AI DeepSeek OCR response: {e}")
raise e
async def async_transform_ocr_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
**kwargs,
) -> OCRResponse:
"""
Async transform chat completion response to OCR format.
Same as sync version - no async-specific logic needed.
Args:
model: Model name
raw_response: Raw HTTP response
logging_obj: Logging object
**kwargs: Additional arguments
Returns:
OCRResponse in standard format
"""
return self.transform_ocr_response(
model=model,
raw_response=raw_response,
logging_obj=logging_obj,
**kwargs,
)

4
litellm/main.py
View file

@ -165,7 +165,8 @@ from .llms.azure_ai.anthropic.handler import AzureAnthropicChatCompletion
from .llms.azure_ai.embed import AzureAIEmbedding
from .llms.bedrock.chat import BedrockConverseLLM, BedrockLLM
from .llms.bedrock.embed.embedding import BedrockEmbedding
from .llms.bedrock.image.image_handler import BedrockImageGeneration
from .llms.bedrock.image_generation.image_handler import BedrockImageGeneration
from .llms.bedrock.image_edit.handler import BedrockImageEdit
from .llms.bytez.chat.transformation import BytezChatConfig
from .llms.clarifai.chat.transformation import ClarifaiConfig
from .llms.codestral.completion.handler import CodestralTextCompletion
@ -271,6 +272,7 @@ codestral_text_completions = CodestralTextCompletion()
bedrock_converse_chat_completion = BedrockConverseLLM()
bedrock_embedding = BedrockEmbedding()
bedrock_image_generation = BedrockImageGeneration()
bedrock_image_edit = BedrockImageEdit()
vertex_chat_completion = VertexLLM()
vertex_embedding = VertexEmbedding()
vertex_multimodal_embedding = VertexMultimodalEmbedding()

170
litellm/model_prices_and_context_window_backup.json
View file

@ -24483,6 +24483,90 @@
"output_cost_per_image": 0.08,
"supported_endpoints": ["/v1/images/generations"]
},
"stability/inpaint": {
"litellm_provider": "stability",
"mode": "image_edit",
"output_cost_per_image": 0.005,
"supported_endpoints": ["/v1/images/edits"]
},
"stability/outpaint": {
"litellm_provider": "stability",
"mode": "image_edit",
"output_cost_per_image": 0.004,
"supported_endpoints": ["/v1/images/edits"]
},
"stability/erase": {
"litellm_provider": "stability",
"mode": "image_edit",
"output_cost_per_image": 0.005,
"supported_endpoints": ["/v1/images/edits"]
},
"stability/search-and-replace": {
"litellm_provider": "stability",
"mode": "image_edit",
"output_cost_per_image": 0.005,
"supported_endpoints": ["/v1/images/edits"]
},
"stability/search-and-recolor": {
"litellm_provider": "stability",
"mode": "image_edit",
"output_cost_per_image": 0.005,
"supported_endpoints": ["/v1/images/edits"]
},
"stability/remove-background": {
"litellm_provider": "stability",
"mode": "image_edit",
"output_cost_per_image": 0.005,
"supported_endpoints": ["/v1/images/edits"]
},
"stability/replace-background-and-relight": {
"litellm_provider": "stability",
"mode": "image_edit",
"output_cost_per_image": 0.008,
"supported_endpoints": ["/v1/images/edits"]
},
"stability/sketch": {
"litellm_provider": "stability",
"mode": "image_edit",
"output_cost_per_image": 0.005,
"supported_endpoints": ["/v1/images/edits"]
},
"stability/structure": {
"litellm_provider": "stability",
"mode": "image_edit",
"output_cost_per_image": 0.005,
"supported_endpoints": ["/v1/images/edits"]
},
"stability/style": {
"litellm_provider": "stability",
"mode": "image_edit",
"output_cost_per_image": 0.005,
"supported_endpoints": ["/v1/images/edits"]
},
"stability/style-transfer": {
"litellm_provider": "stability",
"mode": "image_edit",
"output_cost_per_image": 0.008,
"supported_endpoints": ["/v1/images/edits"]
},
"stability/fast": {
"litellm_provider": "stability",
"mode": "image_edit",
"output_cost_per_image": 0.002,
"supported_endpoints": ["/v1/images/edits"]
},
"stability/conservative": {
"litellm_provider": "stability",
"mode": "image_edit",
"output_cost_per_image": 0.04,
"supported_endpoints": ["/v1/images/edits"]
},
"stability/creative": {
"litellm_provider": "stability",
"mode": "image_edit",
"output_cost_per_image": 0.06,
"supported_endpoints": ["/v1/images/edits"]
},
"stability/stable-image-core": {
"litellm_provider": "stability",
"mode": "image_generation",
@ -24531,6 +24615,84 @@
"mode": "image_generation",
"output_cost_per_image": 0.14
},
"stability.stable-conservative-upscale-v1:0": {
"litellm_provider": "bedrock",
"max_input_tokens": 77,
"mode": "image_edit",
"output_cost_per_image": 0.40
},
"stability.stable-creative-upscale-v1:0": {
"litellm_provider": "bedrock",
"max_input_tokens": 77,
"mode": "image_edit",
"output_cost_per_image": 0.60
},
"stability.stable-fast-upscale-v1:0": {
"litellm_provider": "bedrock",
"max_input_tokens": 77,
"mode": "image_edit",
"output_cost_per_image": 0.03
},
"stability.stable-outpaint-v1:0": {
"litellm_provider": "bedrock",
"max_input_tokens": 77,
"mode": "image_edit",
"output_cost_per_image": 0.06
},
"stability.stable-image-control-sketch-v1:0": {
"litellm_provider": "bedrock",
"max_input_tokens": 77,
"mode": "image_edit",
"output_cost_per_image": 0.07
},
"stability.stable-image-control-structure-v1:0": {
"litellm_provider": "bedrock",
"max_input_tokens": 77,
"mode": "image_edit",
"output_cost_per_image": 0.07
},
"stability.stable-image-erase-object-v1:0": {
"litellm_provider": "bedrock",
"max_input_tokens": 77,
"mode": "image_edit",
"output_cost_per_image": 0.07
},
"stability.stable-image-inpaint-v1:0": {
"litellm_provider": "bedrock",
"max_input_tokens": 77,
"mode": "image_edit",
"output_cost_per_image": 0.07
},
"stability.stable-image-remove-background-v1:0": {
"litellm_provider": "bedrock",
"max_input_tokens": 77,
"mode": "image_edit",
"output_cost_per_image": 0.07
},
"stability.stable-image-search-recolor-v1:0": {
"litellm_provider": "bedrock",
"max_input_tokens": 77,
"mode": "image_edit",
"output_cost_per_image": 0.07
},
"stability.stable-image-search-replace-v1:0": {
"litellm_provider": "bedrock",
"max_input_tokens": 77,
"mode": "image_edit",
"output_cost_per_image": 0.07
},
"stability.stable-image-style-guide-v1:0": {
"litellm_provider": "bedrock",
"max_input_tokens": 77,
"mode": "image_edit",
"output_cost_per_image": 0.07
},
"stability.stable-style-transfer-v1:0": {
"litellm_provider": "bedrock",
"max_input_tokens": 77,
"mode": "image_edit",
"output_cost_per_image": 0.08
},
"standard/1024-x-1024/dall-e-3": {
"input_cost_per_pixel": 3.81469e-08,
"litellm_provider": "openai",
@ -27777,6 +27939,14 @@
],
"source": "https://cloud.google.com/generative-ai-app-builder/pricing"
},
"vertex_ai/deepseek-ai/deepseek-ocr-maas": {
"litellm_provider": "vertex_ai",
"mode": "ocr",
"input_cost_per_token": 3e-07,
"output_cost_per_token": 1.2e-06,
"ocr_cost_per_page": 3e-04,
"source": "https://cloud.google.com/vertex-ai/pricing"
},
"vertex_ai/openai/gpt-oss-120b-maas": {
"input_cost_per_token": 1.5e-07,
"litellm_provider": "vertex_ai-openai_models",

64
litellm/proxy/_types.py
View file

@ -16,7 +16,11 @@ from typing_extensions import Required, TypedDict
from litellm._uuid import uuid
from litellm.types.integrations.slack_alerting import AlertType
from litellm.types.llms.openai import AllMessageValues, OpenAIFileObject
from litellm.types.llms.openai import (
AllMessageValues,
OpenAIFileObject,
ResponsesAPIResponse,
)
from litellm.types.mcp import (
MCPAuth,
MCPAuthType,
@ -1140,6 +1144,60 @@ class MakeMCPServersPublicRequest(LiteLLMPydanticObjectBase):
mcp_server_ids: List[str]
######## Skills API Types ########
class NewSkillRequest(LiteLLMPydanticObjectBase):
"""Request to create a new skill in LiteLLM database"""
display_title: Optional[str] = None
description: Optional[str] = None
instructions: Optional[str] = None
file_content: Optional[bytes] = None # Binary content of skill files (zip)
file_name: Optional[str] = None # Original filename
file_type: Optional[str] = None # MIME type (e.g., "application/zip")
metadata: Optional[Dict[str, Any]] = None
class UpdateSkillRequest(LiteLLMPydanticObjectBase):
"""Request to update an existing skill"""
skill_id: str
display_title: Optional[str] = None
description: Optional[str] = None
instructions: Optional[str] = None
file_content: Optional[bytes] = None # Binary content of skill files (zip)
file_name: Optional[str] = None # Original filename
file_type: Optional[str] = None # MIME type
metadata: Optional[Dict[str, Any]] = None
class LiteLLM_SkillsTable(LiteLLMPydanticObjectBase):
"""Represents a LiteLLM_SkillsTable record"""
skill_id: str
display_title: Optional[str] = None
description: Optional[str] = None
instructions: Optional[str] = None
source: str = "custom"
latest_version: Optional[str] = None
file_content: Optional[bytes] = None # Binary content of skill files (zip)
file_name: Optional[str] = None # Original filename
file_type: Optional[str] = None # MIME type
metadata: Optional[Dict[str, Any]] = None
created_at: Optional[datetime] = None
created_by: Optional[str] = None
updated_at: Optional[datetime] = None
updated_by: Optional[str] = None
class ListSkillsRequest(LiteLLMPydanticObjectBase):
"""Request to list skills from LiteLLM database"""
limit: Optional[int] = 20
offset: Optional[int] = 0
class NewUserRequestTeam(LiteLLMPydanticObjectBase):
team_id: str
max_budget_in_team: Optional[float] = None
@ -3740,8 +3798,8 @@ class LiteLLM_ManagedFileTable(LiteLLMPydanticObjectBase):
class LiteLLM_ManagedObjectTable(LiteLLMPydanticObjectBase):
unified_object_id: str
model_object_id: str
file_purpose: Literal["batch", "fine-tune"]
file_object: Union[LiteLLMBatch, LiteLLMFineTuningJob]
file_purpose: Literal["batch", "fine-tune", "response"]
file_object: Union[LiteLLMBatch, LiteLLMFineTuningJob, ResponsesAPIResponse]
class EnterpriseLicenseData(TypedDict, total=False):

6
litellm/proxy/common_request_processing.py
View file

@ -885,14 +885,16 @@ class ProxyBaseLLMRequestProcessing:
@staticmethod
def _get_pre_call_type(
route_type: Literal["acompletion", "aembedding", "aresponses"],
) -> Literal["completion", "embeddings", "responses"]:
route_type: Literal["acompletion", "aembedding", "aresponses", "allm_passthrough_route"],
) -> Literal["completion", "embeddings", "responses", "allm_passthrough_route"]:
if route_type == "acompletion":
return "completion"
elif route_type == "aembedding":
return "embeddings"
elif route_type == "aresponses":
return "responses"
elif route_type == "allm_passthrough_route":
return "allm_passthrough_route"
#########################################################
# Proxy Level Streaming Data Generator

12
litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py
View file

@ -128,7 +128,7 @@ class UnifiedLLMGuardrails(CustomLogger):
endpoint_guardrail_translation_mappings = (
load_guardrail_translation_mappings()
)
if CallTypes(call_type) not in endpoint_guardrail_translation_mappings:
if call_type is not None and CallTypes(call_type) not in endpoint_guardrail_translation_mappings:
return data
endpoint_translation = endpoint_guardrail_translation_mappings[
@ -180,10 +180,10 @@ class UnifiedLLMGuardrails(CustomLogger):
call_type: Optional[CallTypesLiteral] = None
if user_api_key_dict.request_route is not None:
call_types = get_call_types_for_route(user_api_key_dict.request_route)
if call_types is not None and len(call_types) > 0:
call_type = call_types[0]
if call_types is not None and len(call_types) > 0: # type: ignore
call_type = call_types[0] # type: ignore
if call_type is None:
call_type = _infer_call_type(call_type=None, completion_response=response)
call_type = _infer_call_type(call_type=None, completion_response=response) # type: ignore
if call_type is None:
return response
@ -308,10 +308,10 @@ class UnifiedLLMGuardrails(CustomLogger):
if call_type is None and user_api_key_dict.request_route is not None:
call_types = get_call_types_for_route(user_api_key_dict.request_route)
if call_types is not None:
call_type = call_types[0]
call_type = call_types[0].value
if call_type is None:
call_type = _infer_call_type(call_type=None, completion_response=item)
call_type = _infer_call_type(call_type=None, completion_response=item) # type: ignore
# If call type not supported, just pass through all chunks
if (

2
litellm/proxy/hooks/__init__.py
View file

@ -3,6 +3,7 @@ from typing import Literal, Union
from . import *
from .cache_control_check import _PROXY_CacheControlCheck
from .litellm_skills import SkillsInjectionHook
from .max_budget_limiter import _PROXY_MaxBudgetLimiter
from .parallel_request_limiter import _PROXY_MaxParallelRequestsHandler
from .parallel_request_limiter_v3 import _PROXY_MaxParallelRequestsHandler_v3
@ -21,6 +22,7 @@ PROXY_HOOKS = {
"parallel_request_limiter": _PROXY_MaxParallelRequestsHandler_v3,
"cache_control_check": _PROXY_CacheControlCheck,
"responses_id_security": ResponsesIDSecurity,
"litellm_skills": SkillsInjectionHook,
}
## FEATURE FLAG HOOKS ##

39
litellm/proxy/hooks/litellm_skills/__init__.py Normal file
View file

@ -0,0 +1,39 @@
"""
LiteLLM Skills Hook - Proxy integration for skills
This module provides the CustomLogger hook for skills processing.
The actual skill logic is in litellm/llms/litellm_proxy/skills/.
Usage:
from litellm.proxy.hooks.litellm_skills import SkillsInjectionHook
# Register hook in proxy
litellm.callbacks.append(SkillsInjectionHook())
"""
# Re-export from the SDK location for convenience
from litellm.llms.litellm_proxy.skills import (
LITELLM_CODE_EXECUTION_TOOL,
CodeExecutionHandler,
LiteLLMInternalTools,
SkillPromptInjectionHandler,
SkillsSandboxExecutor,
code_execution_handler,
get_litellm_code_execution_tool,
)
from litellm.proxy.hooks.litellm_skills.main import (
SkillsInjectionHook,
skills_injection_hook,
)
__all__ = [
"SkillsInjectionHook",
"skills_injection_hook",
"CodeExecutionHandler",
"LiteLLMInternalTools",
"LITELLM_CODE_EXECUTION_TOOL",
"get_litellm_code_execution_tool",
"code_execution_handler",
"SkillPromptInjectionHandler",
"SkillsSandboxExecutor",
]

869
litellm/proxy/hooks/litellm_skills/main.py Normal file
View file

@ -0,0 +1,869 @@
"""
Skills Injection Hook for LiteLLM Proxy
Main hook that orchestrates skill processing:
- Fetches skills from LiteLLM DB
- Injects SKILL.md content into system prompt
- Adds litellm_code_execution tool for automatic code execution
- Handles agentic loop internally when litellm_code_execution is called
For non-Anthropic models (e.g., Bedrock, OpenAI, etc.):
- Skills are converted to OpenAI-style tools
- Skill file content (SKILL.md) is extracted and injected into the system prompt
- litellm_code_execution tool is added - when model calls it, LiteLLM handles
execution automatically and returns final response with file_ids
Usage:
# Simple - LiteLLM handles everything automatically via proxy
# The container parameter triggers the SkillsInjectionHook
response = await litellm.acompletion(
model="gpt-4o-mini",
messages=[{"role": "user", "content": "Create a bouncing ball GIF"}],
container={"skills": [{"skill_id": "litellm:skill_abc123"}]},
)
# Response includes file_ids for generated files
"""
import base64
import json
from typing import Any, Dict, List, Optional, Union
from litellm._logging import verbose_proxy_logger
from litellm.caching.caching import DualCache
from litellm.integrations.custom_logger import CustomLogger
from litellm.llms.litellm_proxy.skills.prompt_injection import (
SkillPromptInjectionHandler,
)
from litellm.proxy._types import LiteLLM_SkillsTable, UserAPIKeyAuth
from litellm.types.utils import CallTypes, CallTypesLiteral
class SkillsInjectionHook(CustomLogger):
"""
Pre/Post-call hook that processes skills from container.skills parameter.
Pre-call (async_pre_call_hook):
- Skills with 'litellm:' prefix are fetched from LiteLLM DB
- For Anthropic models: native skills pass through, LiteLLM skills converted to tools
- For non-Anthropic models: LiteLLM skills are converted to tools + execute_code tool
Post-call (async_post_call_success_deployment_hook):
- If response has litellm_code_execution tool call, automatically execute code
- Continue conversation loop until model gives final response
- Return response with generated files inline
This hook is called automatically by litellm during completion calls.
"""
def __init__(self, **kwargs):
from litellm.llms.litellm_proxy.skills.constants import (
DEFAULT_MAX_ITERATIONS,
DEFAULT_SANDBOX_TIMEOUT,
)
self.optional_params = kwargs
self.prompt_handler = SkillPromptInjectionHandler()
self.max_iterations = kwargs.get("max_iterations", DEFAULT_MAX_ITERATIONS)
self.sandbox_timeout = kwargs.get("sandbox_timeout", DEFAULT_SANDBOX_TIMEOUT)
super().__init__(**kwargs)
async def async_pre_call_hook(
self,
user_api_key_dict: UserAPIKeyAuth,
cache: DualCache,
data: dict,
call_type: CallTypesLiteral,
) -> Optional[Union[Exception, str, dict]]:
"""
Process skills from container.skills before the LLM call.
1. Check if container.skills exists in request
2. Separate skills by prefix (litellm: vs native)
3. Fetch LiteLLM skills from database
4. For Anthropic: keep native skills in container
5. For non-Anthropic: convert LiteLLM skills to tools, inject content, add execute_code
"""
# Only process completion-type calls
if call_type not in ["completion", "acompletion", "anthropic_messages"]:
return data
container = data.get("container")
if not container or not isinstance(container, dict):
return data
skills = container.get("skills")
if not skills or not isinstance(skills, list):
return data
verbose_proxy_logger.debug(f"SkillsInjectionHook: Processing {len(skills)} skills")
litellm_skills: List[LiteLLM_SkillsTable] = []
anthropic_skills: List[Dict[str, Any]] = []
# Separate skills by prefix
for skill in skills:
if not isinstance(skill, dict):
continue
skill_id = skill.get("skill_id", "")
if skill_id.startswith("litellm_"):
# Fetch from LiteLLM DB
db_skill = await self._fetch_skill_from_db(skill_id)
if db_skill:
litellm_skills.append(db_skill)
else:
verbose_proxy_logger.warning(
f"SkillsInjectionHook: Skill '{skill_id}' not found in LiteLLM DB"
)
else:
# Native Anthropic skill - pass through
anthropic_skills.append(skill)
# Check if using messages API spec (anthropic_messages call type)
# Messages API always uses Anthropic-style tool format
use_anthropic_format = call_type == "anthropic_messages"
if len(litellm_skills) > 0:
data = self._process_for_messages_api(
data=data,
litellm_skills=litellm_skills,
use_anthropic_format=use_anthropic_format,
)
return data
def _process_for_messages_api(
self,
data: dict,
litellm_skills: List[LiteLLM_SkillsTable],
use_anthropic_format: bool = True,
) -> dict:
"""
Process skills for messages API (Anthropic format tools).
- Converts skills to Anthropic-style tools (name, description, input_schema)
- Extracts and injects SKILL.md content into system prompt
- Adds litellm_code_execution tool for code execution
- Stores skill files in metadata for sandbox execution
"""
from litellm.llms.litellm_proxy.skills.code_execution import (
get_litellm_code_execution_tool_anthropic,
)
tools = data.get("tools", [])
skill_contents: List[str] = []
all_skill_files: Dict[str, Dict[str, bytes]] = {}
all_module_paths: List[str] = []
for skill in litellm_skills:
# Convert skill to Anthropic-style tool
tools.append(self.prompt_handler.convert_skill_to_anthropic_tool(skill))
# Extract skill content from file if available
content = self.prompt_handler.extract_skill_content(skill)
if content:
skill_contents.append(content)
# Extract all files for code execution
skill_files = self.prompt_handler.extract_all_files(skill)
if skill_files:
all_skill_files[skill.skill_id] = skill_files
for path in skill_files.keys():
if path.endswith(".py"):
all_module_paths.append(path)
if tools:
data["tools"] = tools
# Inject skill content into system prompt
# For Anthropic messages API, use top-level 'system' param instead of messages array
if skill_contents:
data = self.prompt_handler.inject_skill_content_to_messages(
data, skill_contents, use_anthropic_format=use_anthropic_format
)
# Add litellm_code_execution tool if we have skill files
if all_skill_files:
code_exec_tool = get_litellm_code_execution_tool_anthropic()
data["tools"] = data.get("tools", []) + [code_exec_tool]
# Store skill files in litellm_metadata for automatic code execution
data["litellm_metadata"] = data.get("litellm_metadata", {})
data["litellm_metadata"]["_skill_files"] = all_skill_files
data["litellm_metadata"]["_litellm_code_execution_enabled"] = True
# Remove container (not supported by underlying providers)
data.pop("container", None)
verbose_proxy_logger.debug(
f"SkillsInjectionHook: Messages API - converted {len(litellm_skills)} skills to Anthropic tools, "
f"injected {len(skill_contents)} skill contents, "
f"added litellm_code_execution tool with {len(all_module_paths)} modules"
)
return data
def _process_non_anthropic_model(
self,
data: dict,
litellm_skills: List[LiteLLM_SkillsTable],
) -> dict:
"""
Process skills for non-Anthropic models (OpenAI format tools).
- Converts skills to OpenAI-style tools
- Extracts and injects SKILL.md content
- Adds execute_code tool for code execution
- Stores skill files in metadata for sandbox execution
"""
tools = data.get("tools", [])
skill_contents: List[str] = []
all_skill_files: Dict[str, Dict[str, bytes]] = {}
all_module_paths: List[str] = []
for skill in litellm_skills:
# Convert skill to OpenAI-style tool
tools.append(self.prompt_handler.convert_skill_to_tool(skill))
# Extract skill content from file if available
content = self.prompt_handler.extract_skill_content(skill)
if content:
skill_contents.append(content)
# Extract all files for code execution
skill_files = self.prompt_handler.extract_all_files(skill)
if skill_files:
all_skill_files[skill.skill_id] = skill_files
# Collect Python module paths
for path in skill_files.keys():
if path.endswith(".py"):
all_module_paths.append(path)
if tools:
data["tools"] = tools
# Inject skill content into system prompt
if skill_contents:
data = self.prompt_handler.inject_skill_content_to_messages(data, skill_contents)
# Add litellm_code_execution tool if we have skill files
if all_skill_files:
from litellm.llms.litellm_proxy.skills.code_execution import (
get_litellm_code_execution_tool,
)
data["tools"] = data.get("tools", []) + [get_litellm_code_execution_tool()]
# Store skill files in litellm_metadata for automatic code execution
# Using litellm_metadata instead of metadata to avoid conflicts with user metadata
data["litellm_metadata"] = data.get("litellm_metadata", {})
data["litellm_metadata"]["_skill_files"] = all_skill_files
data["litellm_metadata"]["_litellm_code_execution_enabled"] = True
# Remove container for non-Anthropic (they don't support it)
data.pop("container", None)
verbose_proxy_logger.debug(
f"SkillsInjectionHook: Non-Anthropic model - converted {len(litellm_skills)} skills to tools, "
f"injected {len(skill_contents)} skill contents, "
f"added execute_code tool with {len(all_module_paths)} modules"
)
return data
async def _fetch_skill_from_db(self, skill_id: str) -> Optional[LiteLLM_SkillsTable]:
"""
Fetch a skill from the LiteLLM database.
Args:
skill_id: The skill ID (without 'litellm:' prefix)
Returns:
LiteLLM_SkillsTable or None if not found
"""
try:
from litellm.llms.litellm_proxy.skills.handler import LiteLLMSkillsHandler
return await LiteLLMSkillsHandler.fetch_skill_from_db(skill_id)
except Exception as e:
verbose_proxy_logger.warning(
f"SkillsInjectionHook: Error fetching skill {skill_id}: {e}"
)
return None
def _is_anthropic_model(self, model: str) -> bool:
"""
Check if the model is an Anthropic model using get_llm_provider.
Args:
model: The model name/identifier
Returns:
True if Anthropic model, False otherwise
"""
try:
from litellm.litellm_core_utils.get_llm_provider_logic import (
get_llm_provider,
)
_, custom_llm_provider, _, _ = get_llm_provider(model=model)
return custom_llm_provider == "anthropic"
except Exception:
# Fallback to simple check if get_llm_provider fails
return "claude" in model.lower() or model.lower().startswith("anthropic/")
async def async_post_call_success_deployment_hook(
self,
request_data: dict,
response: Any,
call_type: Optional[CallTypes],
) -> Optional[Any]:
"""
Post-call hook to handle automatic code execution.
Handles both OpenAI format (response.choices) and Anthropic/messages API
format (response["content"]).
If the response contains a tool call (litellm_code_execution or skill tool):
1. Execute the code in sandbox
2. Add result to messages
3. Make another LLM call
4. Repeat until model gives final response
5. Return modified response with generated files
"""
from litellm.llms.litellm_proxy.skills.code_execution import (
LiteLLMInternalTools,
)
# Check if code execution is enabled for this request
litellm_metadata = request_data.get("litellm_metadata", {})
metadata = request_data.get("metadata", {})
code_exec_enabled = (
litellm_metadata.get("_litellm_code_execution_enabled") or
metadata.get("_litellm_code_execution_enabled")
)
if not code_exec_enabled:
return None
# Get skill files
skill_files_by_id = (
litellm_metadata.get("_skill_files") or
metadata.get("_skill_files", {})
)
all_skill_files: Dict[str, bytes] = {}
for files_dict in skill_files_by_id.values():
all_skill_files.update(files_dict)
if not all_skill_files:
verbose_proxy_logger.warning(
"SkillsInjectionHook: No skill files found, cannot execute code"
)
return None
# Check for tool calls - handle both Anthropic and OpenAI formats
tool_calls = self._extract_tool_calls(response)
if not tool_calls:
return None
# Check if any tool call needs execution (litellm_code_execution or skill tool)
has_executable_tool = False
for tc in tool_calls:
tool_name = tc.get("name", "")
# Execute if it's litellm_code_execution OR a skill tool (skill_xxx)
if tool_name == LiteLLMInternalTools.CODE_EXECUTION.value or tool_name.startswith("skill_"):
has_executable_tool = True
break
if not has_executable_tool:
return None
verbose_proxy_logger.debug(
"SkillsInjectionHook: Detected tool call, starting execution loop"
)
# Start the agentic loop
return await self._execute_code_loop_messages_api(
data=request_data,
response=response,
skill_files=all_skill_files,
)
def _extract_tool_calls(self, response: Any) -> List[Dict[str, Any]]:
"""Extract tool calls from response, handling both formats."""
tool_calls = []
# Get content - handle both dict and object responses
content = None
if isinstance(response, dict):
content = response.get("content", [])
elif hasattr(response, "content"):
content = response.content
# Anthropic/messages API format: response has "content" list with tool_use blocks
if content:
for block in content:
if isinstance(block, dict) and block.get("type") == "tool_use":
tool_calls.append({
"id": block.get("id"),
"name": block.get("name"),
"input": block.get("input", {}),
})
elif hasattr(block, "type") and getattr(block, "type", None) == "tool_use":
tool_calls.append({
"id": getattr(block, "id", None),
"name": getattr(block, "name", None),
"input": getattr(block, "input", {}),
})
# OpenAI format: response has choices[0].message.tool_calls
if not tool_calls and hasattr(response, "choices") and response.choices: # type: ignore[union-attr]
msg = response.choices[0].message # type: ignore[union-attr]
if hasattr(msg, "tool_calls") and msg.tool_calls:
for tc in msg.tool_calls:
tool_calls.append({
"id": tc.id,
"name": tc.function.name,
"input": json.loads(tc.function.arguments) if tc.function.arguments else {},
})
return tool_calls
async def _execute_code_loop_messages_api(
self,
data: dict,
response: Any,
skill_files: Dict[str, bytes],
) -> Any:
"""
Execute the code execution loop for messages API (Anthropic format).
Returns the final response with generated files inline.
"""
import litellm
from litellm.llms.litellm_proxy.skills.code_execution import (
LiteLLMInternalTools,
)
from litellm.llms.litellm_proxy.skills.sandbox_executor import (
SkillsSandboxExecutor,
)
# Ensure response is not None
if response is None:
verbose_proxy_logger.error(
"SkillsInjectionHook: Response is None, cannot execute code loop"
)
return None
model = data.get("model", "")
messages = list(data.get("messages", []))
tools = data.get("tools", [])
max_tokens = data.get("max_tokens", 4096)
executor = SkillsSandboxExecutor(timeout=self.sandbox_timeout)
generated_files: List[Dict[str, Any]] = []
current_response = response
for iteration in range(self.max_iterations):
# Extract tool calls from current response
tool_calls = self._extract_tool_calls(current_response)
stop_reason = current_response.get("stop_reason") if isinstance(current_response, dict) else getattr(current_response, "stop_reason", None)
# Get content for assistant message - convert to plain dicts
raw_content = current_response.get("content", []) if isinstance(current_response, dict) else getattr(current_response, "content", [])
content_blocks = []
for block in raw_content or []:
if isinstance(block, dict):
content_blocks.append(block)
elif hasattr(block, "model_dump"):
content_blocks.append(block.model_dump())
elif hasattr(block, "__dict__"):
content_blocks.append(dict(block.__dict__))
else:
content_blocks.append({"type": "text", "text": str(block)})
# Build assistant message for conversation history (Anthropic format)
assistant_msg = {"role": "assistant", "content": content_blocks}
messages.append(assistant_msg)
# Check if we're done (no tool calls)
if stop_reason != "tool_use" or not tool_calls:
verbose_proxy_logger.debug(
f"SkillsInjectionHook: Loop completed after {iteration + 1} iterations, "
f"{len(generated_files)} files generated"
)
return self._attach_files_to_response(current_response, generated_files)
# Process tool calls
tool_results = []
for tc in tool_calls:
tool_name = tc.get("name", "")
tool_id = tc.get("id", "")
tool_input = tc.get("input", {})
# Execute if it's litellm_code_execution OR a skill tool
if tool_name == LiteLLMInternalTools.CODE_EXECUTION.value:
code = tool_input.get("code", "")
result = await self._execute_code(code, skill_files, executor, generated_files)
elif tool_name.startswith("skill_"):
# Skill tool - execute the skill's code
result = await self._execute_skill_tool(tool_name, tool_input, skill_files, executor, generated_files)
else:
result = f"Tool '{tool_name}' not handled"
tool_results.append({
"type": "tool_result",
"tool_use_id": tool_id,
"content": result,
})
# Add tool results to messages (Anthropic format)
messages.append({"role": "user", "content": tool_results})
# Make next LLM call
verbose_proxy_logger.debug(
f"SkillsInjectionHook: Making LLM call iteration {iteration + 2}"
)
try:
current_response = await litellm.anthropic.acreate(
model=model,
messages=messages,
tools=tools,
max_tokens=max_tokens,
)
if current_response is None:
verbose_proxy_logger.error(
"SkillsInjectionHook: LLM call returned None"
)
return self._attach_files_to_response(response, generated_files)
except Exception as e:
verbose_proxy_logger.error(
f"SkillsInjectionHook: LLM call failed: {e}"
)
return self._attach_files_to_response(response, generated_files)
verbose_proxy_logger.warning(
f"SkillsInjectionHook: Max iterations ({self.max_iterations}) reached"
)
return self._attach_files_to_response(current_response, generated_files)
async def _execute_code(
self,
code: str,
skill_files: Dict[str, bytes],
executor: Any,
generated_files: List[Dict[str, Any]],
) -> str:
"""Execute code in sandbox and return result string."""
try:
verbose_proxy_logger.debug(f"SkillsInjectionHook: Executing code ({len(code)} chars)")
exec_result = executor.execute(code=code, skill_files=skill_files)
result = exec_result.get("output", "") or ""
# Collect generated files
if exec_result.get("files"):
for f in exec_result["files"]:
generated_files.append({
"name": f["name"],
"mime_type": f["mime_type"],
"content_base64": f["content_base64"],
"size": len(base64.b64decode(f["content_base64"])),
})
result += f"\n\nGenerated file: {f['name']}"
if exec_result.get("error"):
result += f"\n\nError: {exec_result['error']}"
return result or "Code executed successfully"
except Exception as e:
return f"Code execution failed: {str(e)}"
async def _execute_skill_tool(
self,
tool_name: str,
tool_input: Dict[str, Any],
skill_files: Dict[str, bytes],
executor: Any,
generated_files: List[Dict[str, Any]],
) -> str:
"""Execute a skill tool by generating and running code based on skill content."""
# Generate code based on available skill modules
# Look for Python modules in the skill
python_modules = [p for p in skill_files.keys() if p.endswith(".py") and not p.endswith("__init__.py")]
# Try to find the main builder/creator module
main_module = None
for mod in python_modules:
if "builder" in mod.lower() or "creator" in mod.lower() or "generator" in mod.lower():
main_module = mod
break
if not main_module and python_modules:
# Use first non-init module
main_module = python_modules[0]
if main_module:
# Convert path to import: "core/gif_builder.py" -> "core.gif_builder"
import_path = main_module.replace("/", ".").replace(".py", "")
# Generate code that imports and uses the module
code = f"""
# Auto-generated code to execute skill
import sys
sys.path.insert(0, '/sandbox')
from {import_path} import *
# Try to find and use a Builder/Creator class
import inspect
module = __import__('{import_path}', fromlist=[''])
for name, obj in inspect.getmembers(module):
if inspect.isclass(obj) and name != 'object':
try:
instance = obj()
# Try common methods
if hasattr(instance, 'create'):
result = instance.create()
elif hasattr(instance, 'build'):
result = instance.build()
elif hasattr(instance, 'generate'):
result = instance.generate()
elif hasattr(instance, 'save'):
instance.save('output.gif')
print(f'Used {{name}} class')
break
except Exception as e:
print(f'Error with {{name}}: {{e}}')
continue
# List generated files
import os
for f in os.listdir('.'):
if f.endswith(('.gif', '.png', '.jpg')):
print(f'Generated: {{f}}')
"""
else:
# Fallback generic code
code = """
print('No executable skill module found')
"""
return await self._execute_code(code, skill_files, executor, generated_files)
async def _execute_code_loop(
self,
data: dict,
response: Any,
skill_files: Dict[str, bytes],
) -> Any:
"""
Execute the code execution loop until model gives final response.
Returns the final response with generated files inline.
"""
import litellm
from litellm.llms.litellm_proxy.skills.code_execution import (
LiteLLMInternalTools,
)
from litellm.llms.litellm_proxy.skills.sandbox_executor import (
SkillsSandboxExecutor,
)
model = data.get("model", "")
messages = list(data.get("messages", []))
tools = data.get("tools", [])
# Keys to exclude when passing through to acompletion
# These are either handled explicitly or are internal LiteLLM fields
_EXCLUDED_ACOMPLETION_KEYS = frozenset({
"messages",
"model",
"tools",
"metadata",
"litellm_metadata",
"container",
})
kwargs = {
k: v for k, v in data.items()
if k not in _EXCLUDED_ACOMPLETION_KEYS
}
executor = SkillsSandboxExecutor(timeout=self.sandbox_timeout)
generated_files: List[Dict[str, Any]] = []
current_response: Any = response
for iteration in range(self.max_iterations):
# OpenAI format response has choices[0].message
assistant_message = current_response.choices[0].message # type: ignore[union-attr]
stop_reason = current_response.choices[0].finish_reason # type: ignore[union-attr]
# Build assistant message for conversation history
assistant_msg_dict: Dict[str, Any] = {
"role": "assistant",
"content": assistant_message.content,
}
if assistant_message.tool_calls:
assistant_msg_dict["tool_calls"] = [
{
"id": tc.id,
"type": "function",
"function": {
"name": tc.function.name,
"arguments": tc.function.arguments
}
}
for tc in assistant_message.tool_calls
]
messages.append(assistant_msg_dict)
# Check if we're done (no tool calls)
if stop_reason != "tool_calls" or not assistant_message.tool_calls:
verbose_proxy_logger.debug(
f"SkillsInjectionHook: Code execution loop completed after "
f"{iteration + 1} iterations, {len(generated_files)} files generated"
)
# Attach generated files to response
return self._attach_files_to_response(current_response, generated_files)
# Process tool calls
for tool_call in assistant_message.tool_calls:
tool_name = tool_call.function.name
if tool_name == LiteLLMInternalTools.CODE_EXECUTION.value:
tool_result = await self._execute_code_tool(
tool_call=tool_call,
skill_files=skill_files,
executor=executor,
generated_files=generated_files,
)
else:
# Non-code-execution tool - cannot handle
tool_result = f"Tool '{tool_name}' not handled automatically"
messages.append({
"role": "tool",
"tool_call_id": tool_call.id,
"content": tool_result,
})
# Make next LLM call using the messages API
verbose_proxy_logger.debug(
f"SkillsInjectionHook: Making LLM call iteration {iteration + 2}"
)
current_response = await litellm.anthropic.acreate(
model=model,
messages=messages,
tools=tools,
max_tokens=kwargs.get("max_tokens", 4096),
)
# Max iterations reached
verbose_proxy_logger.warning(
f"SkillsInjectionHook: Max iterations ({self.max_iterations}) reached"
)
return self._attach_files_to_response(current_response, generated_files)
async def _execute_code_tool(
self,
tool_call: Any,
skill_files: Dict[str, bytes],
executor: Any,
generated_files: List[Dict[str, Any]],
) -> str:
"""Execute a litellm_code_execution tool call and return result string."""
try:
args = json.loads(tool_call.function.arguments)
code = args.get("code", "")
verbose_proxy_logger.debug(
f"SkillsInjectionHook: Executing code ({len(code)} chars)"
)
exec_result = executor.execute(
code=code,
skill_files=skill_files,
)
# Build tool result content
tool_result = exec_result.get("output", "") or ""
# Collect generated files
if exec_result.get("files"):
tool_result += "\n\nGenerated files:"
for f in exec_result["files"]:
file_content = base64.b64decode(f["content_base64"])
generated_files.append({
"name": f["name"],
"mime_type": f["mime_type"],
"content_base64": f["content_base64"],
"size": len(file_content),
})
tool_result += f"\n- {f['name']} ({len(file_content)} bytes)"
verbose_proxy_logger.debug(
f"SkillsInjectionHook: Generated file {f['name']} "
f"({len(file_content)} bytes)"
)
if exec_result.get("error"):
tool_result += f"\n\nError:\n{exec_result['error']}"
return tool_result
except Exception as e:
verbose_proxy_logger.error(
f"SkillsInjectionHook: Code execution failed: {e}"
)
return f"Code execution failed: {str(e)}"
def _attach_files_to_response(
self,
response: Any,
generated_files: List[Dict[str, Any]],
) -> Any:
"""
Attach generated files to the response object.
Files are added to response._litellm_generated_files for easy access.
For dict responses, files are added as a key.
"""
if not generated_files:
return response
# Handle dict response (Anthropic/messages API format)
if isinstance(response, dict):
response["_litellm_generated_files"] = generated_files
verbose_proxy_logger.debug(
f"SkillsInjectionHook: Attached {len(generated_files)} files to dict response"
)
return response
# Handle object response (OpenAI format)
try:
response._litellm_generated_files = generated_files
except AttributeError:
pass
# Also add to model_extra if available (for serialization)
if hasattr(response, "model_extra"):
if response.model_extra is None:
response.model_extra = {}
response.model_extra["_litellm_generated_files"] = generated_files
verbose_proxy_logger.debug(
f"SkillsInjectionHook: Attached {len(generated_files)} files to response"
)
return response
# Global instance for registration
skills_injection_hook = SkillsInjectionHook()
import litellm
litellm.logging_callback_manager.add_litellm_callback(skills_injection_hook)

87
litellm/proxy/management_endpoints/key_management_endpoints.py
View file

@ -1912,14 +1912,14 @@ async def info_key_fn(
Example Curl:
```
curl -X GET "http://0.0.0.0:4000/key/info?key=sk-02Wr4IAlN3NvPXvL5JVvDA" \
curl -X GET "http://0.0.0.0:4000/key/info?key=sk-test-example-key-123" \
-H "Authorization: Bearer sk-1234"
```
Example Curl - if no key is passed, it will use the Key Passed in Authorization Header
```
curl -X GET "http://0.0.0.0:4000/key/info" \
-H "Authorization: Bearer sk-02Wr4IAlN3NvPXvL5JVvDA"
-H "Authorization: Bearer sk-test-example-key-123"
```
"""
from litellm.proxy.proxy_server import prisma_client
@ -2312,31 +2312,70 @@ async def _team_key_deletion_check(
return False
async def can_delete_verification_token(
async def can_modify_verification_token(
key_info: LiteLLM_VerificationToken,
user_api_key_cache: DualCache,
user_api_key_dict: UserAPIKeyAuth,
prisma_client: PrismaClient,
) -> bool:
"""
- check if user is proxy admin
- check if user is team admin and key is a team key
- check if key is personal key
Check if user has permission to modify (delete/regenerate) a verification token.
Rules:
- Proxy admin can modify any key
- For team keys: only team admin or key owner can modify
- For personal keys: only key owner can modify
Args:
key_info: The verification token to check
user_api_key_cache: Cache for user API keys
user_api_key_dict: The user making the request
prisma_client: Prisma client for database access
Returns:
True if user can modify the key, False otherwise
"""
is_team_key = _is_team_key(data=key_info)
# 1. Proxy admin can modify any key
if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value:
return True
elif is_team_key and key_info.team_id is not None:
return await _team_key_deletion_check(
user_api_key_dict=user_api_key_dict,
key_info=key_info,
# 2. For team keys: only team admin or key owner can modify
if is_team_key and key_info.team_id is not None:
# Get team object to check if user is team admin
team_table = await get_team_object(
team_id=key_info.team_id,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
check_db_only=True,
)
elif key_info.user_id is not None and key_info.user_id == user_api_key_dict.user_id:
return True
else:
if team_table is None:
return False
# Check if user is team admin
if _is_user_team_admin(
user_api_key_dict=user_api_key_dict,
team_obj=team_table,
):
return True
# Check if the key belongs to the user (they own it)
if key_info.user_id is not None and key_info.user_id == user_api_key_dict.user_id:
return True
# Not team admin and doesn't own the key
return False
# 3. For personal keys: only key owner can modify
if key_info.user_id is not None and key_info.user_id == user_api_key_dict.user_id:
return True
# Default: deny
return False
async def delete_verification_tokens(
@ -2385,7 +2424,7 @@ async def delete_verification_tokens(
else:
authorized_keys: List[LiteLLM_VerificationToken] = []
for key in _keys_being_deleted:
if await can_delete_verification_token(
if await can_modify_verification_token(
key_info=key,
user_api_key_cache=user_api_key_cache,
user_api_key_dict=user_api_key_dict,
@ -2818,6 +2857,18 @@ async def regenerate_key_fn(
user_api_key_cache=user_api_key_cache,
)
# check if user has ownership permission to regenerate key
if not await can_modify_verification_token(
key_info=_key_in_db,
user_api_key_cache=user_api_key_cache,
user_api_key_dict=user_api_key_dict,
prisma_client=prisma_client,
):
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail={"error": "You are not authorized to regenerate this key"},
)
verbose_proxy_logger.debug("key_in_db: %s", _key_in_db)
new_token = get_new_token(data=data)
@ -2856,14 +2907,8 @@ async def regenerate_key_fn(
### 3. remove existing key entry from cache
######################################################################
if key:
await _delete_cache_key_object(
hashed_token=hash_token(key),
user_api_key_cache=user_api_key_cache,
proxy_logging_obj=proxy_logging_obj,
)
if hashed_api_key:
if hashed_api_key or key:
await _delete_cache_key_object(
hashed_token=hash_token(key),
user_api_key_cache=user_api_key_cache,

14
litellm/proxy/openai_files_endpoints/common_utils.py
View file

@ -2,13 +2,13 @@ import base64
import mimetypes
import re
from dataclasses import dataclass, field
from typing import List, Literal, Optional, Union
from typing import TYPE_CHECKING, List, Literal, Optional, Union
from fastapi import Request
from litellm.proxy.common_utils.http_parsing_utils import _read_request_body
from litellm.types.utils import SpecialEnums
if TYPE_CHECKING:
from fastapi import Request
def _is_base64_encoded_unified_file_id(b64_uid: str) -> Union[str, Literal[False]]:
# Ensure b64_uid is a string and not a mock object
@ -554,7 +554,7 @@ class FileCreationParams:
async def extract_file_creation_params(
request: Request,
request: "Request",
request_body: Optional[dict] = None,
target_model_names_form: Optional[str] = None,
target_storage_form: Optional[str] = None,
@ -571,6 +571,8 @@ async def extract_file_creation_params(
Returns:
FileCreationParams: Structured parameters extracted from the request
"""
from litellm.proxy.common_utils.http_parsing_utils import _read_request_body
if request_body is None:
request_body = await _read_request_body(request=request) or {}
@ -621,7 +623,7 @@ def _extract_target_model_names_simple(target_model_names_form: Optional[str] =
return []
def _extract_model_param(request: Request, request_body: dict) -> Optional[str]:
def _extract_model_param(request: "Request", request_body: dict) -> Optional[str]:
"""
Extract model parameter from request.

9
litellm/proxy/proxy_config.yaml
View file

@ -1,10 +1,5 @@
model_list:
- model_name: gemini/*
- model_name: anthropic/*
litellm_params:
model: gemini/*
model: anthropic/*
litellm_settings:
callbacks: ["dynamic_rate_limiter_v3"]
priority_reservation:
"prod": 0.9 # 90% reserved for production
"dev": 0.1 # 10% reserved for development

33
litellm/proxy/proxy_server.py
View file

@ -4434,7 +4434,7 @@ class ProxyStartupEvent:
)
@classmethod
async def initialize_scheduled_background_jobs(
async def initialize_scheduled_background_jobs( # noqa: PLR0915
cls,
general_settings: dict,
prisma_client: PrismaClient,
@ -4629,6 +4629,37 @@ class ProxyStartupEvent:
)
pass
### CHECK RESPONSES COST ###
if llm_router is not None:
try:
from litellm_enterprise.proxy.common_utils.check_responses_cost import (
CheckResponsesCost,
)
check_responses_cost_job = CheckResponsesCost(
proxy_logging_obj=proxy_logging_obj,
prisma_client=prisma_client,
llm_router=llm_router,
)
scheduler.add_job(
check_responses_cost_job.check_responses_cost,
"interval",
seconds=proxy_batch_polling_interval
+ random.randint(0, 30), # Add small random offset
# REMOVED jitter parameter - major cause of memory leak
id="check_responses_cost_job",
replace_existing=True,
misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME,
)
verbose_proxy_logger.info("Responses cost check job scheduled successfully")
except Exception as e:
verbose_proxy_logger.error(f"Failed to setup responses cost checking: {e}")
verbose_proxy_logger.debug(
"Checking responses cost for LiteLLM Managed Files is an Enterprise Feature. Skipping..."
)
pass
# MEMORY LEAK FIX: Start scheduler with paused=False to avoid backlog processing
# Do NOT reset job times to "now" as this can trigger the memory leak
# The misfire_grace_time and coalesce settings will handle any missed runs properly

46
litellm/proxy/response_api_endpoints/endpoints.py
View file

@ -1,6 +1,6 @@
import asyncio
import time
from typing import Any, AsyncIterator, cast
from typing import Any, AsyncIterator, Optional, cast
from uuid import uuid4
from fastapi import APIRouter, Depends, HTTPException, Request, Response
@ -155,7 +155,7 @@ async def responses_api(
# Normal response flow
processor = ProxyBaseLLMRequestProcessing(data=data)
try:
return await processor.base_process_llm_request(
response = await processor.base_process_llm_request(
request=request,
fastapi_response=fastapi_response,
user_api_key_dict=user_api_key_dict,
@ -173,6 +173,48 @@ async def responses_api(
user_api_base=user_api_base,
version=version,
)
# Store in managed objects table if background mode is enabled
if data.get("background") and isinstance(response, ResponsesAPIResponse):
if response.status in ["queued", "in_progress"]:
from litellm_enterprise.proxy.hooks.managed_files import ( # type: ignore
_PROXY_LiteLLMManagedFiles,
)
managed_files_obj = cast(
Optional[_PROXY_LiteLLMManagedFiles],
proxy_logging_obj.get_proxy_hook("managed_files"),
)
if managed_files_obj and llm_router:
try:
# Get the actual deployment model_id from hidden params
hidden_params = getattr(response, "_hidden_params", {}) or {}
model_id = hidden_params.get("model_id", None)
if not model_id:
verbose_proxy_logger.warning(
f"No model_id found in response hidden params for response {response.id}, skipping managed object storage"
)
raise Exception("No model_id found in response hidden params")
# Store in managed objects table
await managed_files_obj.store_unified_object_id(
unified_object_id=response.id,
file_object=response,
litellm_parent_otel_span=None,
model_object_id=response.id,
file_purpose="response",
user_api_key_dict=user_api_key_dict,
)
verbose_proxy_logger.info(
f"Stored background response {response.id} in managed objects table with unified_id={response.id}"
)
except Exception as e:
verbose_proxy_logger.error(
f"Failed to store background response in managed objects table: {str(e)}"
)
return response
except ModifyResponseException as e:
# Guardrail passthrough: return violation message in Responses API format (200)
_data = e.request_data

18
litellm/proxy/schema.prisma
View file

@ -824,4 +824,22 @@ model LiteLLM_UISettings {
ui_settings Json
created_at DateTime @default(now())
updated_at DateTime @updatedAt
}
// Skills table for storing LiteLLM-managed skills
model LiteLLM_SkillsTable {
skill_id String @id @default(uuid())
display_title String?
description String?
instructions String? // The skill instructions/prompt (from SKILL.md)
source String @default("custom") // "custom" or "anthropic"
latest_version String?
file_content Bytes? // Binary content of the skill files (zip)
file_name String? // Original filename
file_type String? // MIME type (e.g., "application/zip")
metadata Json? @default("{}")
created_at DateTime @default(now())
created_by String?
updated_at DateTime @default(now()) @updatedAt
updated_by String?
}

2
litellm/proxy/spend_tracking/spend_management_endpoints.py
View file

@ -1938,7 +1938,7 @@ async def view_spend_logs( # noqa: PLR0915
Example Request for specific api_key
```
curl -X GET "http://0.0.0.0:8000/spend/logs?api_key=sk-Fn8Ej39NkBQmUagFEoUWPQ" \
curl -X GET "http://0.0.0.0:8000/spend/logs?api_key=sk-test-example-key-123" \
-H "Authorization: Bearer sk-1234"
```

4
litellm/rag/__init__.py
View file

@ -5,9 +5,9 @@ Provides an all-in-one API for document ingestion:
Upload -> (OCR) -> Chunk -> Embed -> Vector Store
"""
from litellm.rag.main import aingest, ingest
from litellm.rag.main import aingest, aquery, ingest, query
__all__ = ["ingest", "aingest"]
__all__ = ["ingest", "aingest", "query", "aquery"]
# Expose at litellm.rag level for convenience

180
litellm/rag/main.py
View file

@ -7,12 +7,22 @@ Upload -> (OCR) -> Chunk -> Embed -> Vector Store
from __future__ import annotations
__all__ = ["ingest", "aingest"]
__all__ = ["ingest", "aingest", "query", "aquery"]
import asyncio
import contextvars
from functools import partial
from typing import TYPE_CHECKING, Any, Coroutine, Dict, Optional, Tuple, Type, Union
from typing import (
TYPE_CHECKING,
Any,
Coroutine,
Dict,
List,
Optional,
Tuple,
Type,
Union,
)
import httpx
@ -21,7 +31,14 @@ from litellm.rag.ingestion.base_ingestion import BaseRAGIngestion
from litellm.rag.ingestion.bedrock_ingestion import BedrockRAGIngestion
from litellm.rag.ingestion.gemini_ingestion import GeminiRAGIngestion
from litellm.rag.ingestion.openai_ingestion import OpenAIRAGIngestion
from litellm.types.rag import RAGIngestOptions, RAGIngestResponse
from litellm.rag.rag_query import RAGQuery
from litellm.types.rag import (
RAGIngestOptions,
RAGIngestResponse,
RAGQueryRequest,
RAGQueryResponse,
)
from litellm.types.utils import ModelResponse
from litellm.utils import client
if TYPE_CHECKING:
@ -172,6 +189,163 @@ async def aingest(
)
async def _execute_query_pipeline(
model: str,
messages: List[Any],
retrieval_config: Dict[str, Any],
rerank: Optional[Dict[str, Any]] = None,
stream: bool = False,
**kwargs,
) -> ModelResponse:
"""
Execute the RAG query pipeline.
"""
# 1. Extract query from last user message
query_text = RAGQuery.extract_query_from_messages(messages)
if not query_text:
raise ValueError("No query found in messages for RAG query")
# 2. Search vector store
search_response = await litellm.vector_stores.asearch(
vector_store_id=retrieval_config["vector_store_id"],
query=query_text,
max_num_results=retrieval_config.get("top_k", 10),
custom_llm_provider=retrieval_config.get("custom_llm_provider", "openai"),
**kwargs,
)
rerank_response = None
context_chunks = search_response.get("data", [])
# 3. Optional rerank
if rerank and rerank.get("enabled"):
documents = RAGQuery.extract_documents_from_search(search_response)
if documents:
rerank_response = await litellm.arerank(
model=rerank["model"],
query=query_text,
documents=documents,
top_n=rerank.get("top_n", 5),
)
context_chunks = RAGQuery.get_top_chunks_from_rerank(
search_response, rerank_response
)
# 4. Build context message and call completion
context_message = RAGQuery.build_context_message(context_chunks)
modified_messages = messages[:-1] + [context_message] + [messages[-1]]
response = await litellm.acompletion(
model=model,
messages=modified_messages,
stream=stream,
**kwargs,
)
# 5. Attach search results to response
if not stream and isinstance(response, ModelResponse):
response = RAGQuery.add_search_results_to_response(
response=response,
search_results=search_response,
rerank_results=rerank_response,
)
return response # type: ignore[return-value]
@client
async def aquery(
model: str,
messages: List[Any],
retrieval_config: Dict[str, Any],
rerank: Optional[Dict[str, Any]] = None,
stream: bool = False,
**kwargs,
) -> ModelResponse:
"""
Async: Query a RAG pipeline.
"""
local_vars = locals()
try:
loop = asyncio.get_event_loop()
kwargs["aquery"] = True
func = partial(
query,
model=model,
messages=messages,
retrieval_config=retrieval_config,
rerank=rerank,
stream=stream,
**kwargs,
)
ctx = contextvars.copy_context()
func_with_context = partial(ctx.run, func)
init_response = await loop.run_in_executor(None, func_with_context)
if asyncio.iscoroutine(init_response):
response = await init_response
else:
response = init_response
return response
except Exception as e:
raise litellm.exception_type(
model=model,
custom_llm_provider=retrieval_config.get("custom_llm_provider"),
original_exception=e,
completion_kwargs=local_vars,
extra_kwargs=kwargs,
)
@client
def query(
model: str,
messages: List[Any],
retrieval_config: Dict[str, Any],
rerank: Optional[Dict[str, Any]] = None,
stream: bool = False,
**kwargs,
) -> Union[ModelResponse, Coroutine[Any, Any, ModelResponse]]:
"""
Query a RAG pipeline.
"""
local_vars = locals()
try:
_is_async = kwargs.pop("aquery", False) is True
if _is_async:
return _execute_query_pipeline(
model=model,
messages=messages,
retrieval_config=retrieval_config,
rerank=rerank,
stream=stream,
**kwargs,
)
else:
return asyncio.get_event_loop().run_until_complete(
_execute_query_pipeline(
model=model,
messages=messages,
retrieval_config=retrieval_config,
rerank=rerank,
stream=stream,
**kwargs,
)
)
except Exception as e:
raise litellm.exception_type(
model=model,
custom_llm_provider=retrieval_config.get("custom_llm_provider"),
original_exception=e,
completion_kwargs=local_vars,
extra_kwargs=kwargs,
)
@client
def ingest(
ingest_options: Dict[str, Any],

120
litellm/rag/rag_query.py Normal file
View file

@ -0,0 +1,120 @@
from typing import Any, Dict, List, Optional, Union, cast
import litellm
from litellm.types.llms.openai import AllMessageValues, ChatCompletionUserMessage
from litellm.types.utils import ModelResponse
from litellm.types.vector_stores import (
VectorStoreResultContent,
VectorStoreSearchResponse,
VectorStoreSearchResult,
)
class RAGQuery:
CONTENT_PREFIX_STRING = "Context:\n\n"
@staticmethod
def extract_query_from_messages(messages: List[AllMessageValues]) -> Optional[str]:
"""
Extract the query from the last user message.
"""
if not messages or len(messages) == 0:
return None
last_message = messages[-1]
if not isinstance(last_message, dict) or "content" not in last_message:
return None
content = last_message["content"]
if isinstance(content, str):
return content
elif isinstance(content, list) and len(content) > 0:
# Handle list of content items, extract text from first text item
for item in content:
if (
isinstance(item, dict)
and item.get("type") == "text"
and "text" in item
):
return item["text"]
return None
@staticmethod
def build_context_message(context_chunks: List[Any]) -> ChatCompletionUserMessage:
"""
Process search results and build a context message.
"""
context_content = RAGQuery.CONTENT_PREFIX_STRING
for chunk in context_chunks:
if isinstance(chunk, dict):
result_content: Optional[List[VectorStoreResultContent]] = chunk.get(
"content"
)
if result_content:
for content_item in result_content:
content_text: Optional[str] = content_item.get("text")
if content_text:
context_content += content_text + "\n\n"
elif "text" in chunk: # Fallback for simple dict with text
context_content += chunk["text"] + "\n\n"
elif isinstance(chunk, str):
context_content += chunk + "\n\n"
return {
"role": "user",
"content": context_content,
}
@staticmethod
def add_search_results_to_response(
response: ModelResponse,
search_results: VectorStoreSearchResponse,
rerank_results: Optional[Any] = None,
) -> ModelResponse:
"""
Add search results to the response choices.
"""
if hasattr(response, "choices") and response.choices:
for choice in response.choices:
message = getattr(choice, "message", None)
if message is not None:
# Get existing provider_specific_fields or create new dict
provider_fields = (
getattr(message, "provider_specific_fields", None) or {}
)
# Add search results
provider_fields["search_results"] = search_results
if rerank_results:
provider_fields["rerank_results"] = rerank_results
# Set the provider_specific_fields
setattr(message, "provider_specific_fields", provider_fields)
return response
@staticmethod
def extract_documents_from_search(
search_response: Any,
) -> List[Union[str, Dict[str, Any]]]:
"""Extract text documents from vector store search response."""
documents: List[Union[str, Dict[str, Any]]] = []
for result in search_response.get("data", []):
content_list = result.get("content", [])
for content in content_list:
if content.get("type") == "text" and content.get("text"):
documents.append(content["text"])
return documents
@staticmethod
def get_top_chunks_from_rerank(search_response: Any, rerank_response: Any) -> List[Any]:
"""Get the original search results corresponding to the top reranked results."""
top_chunks = []
original_results = search_response.get("data", [])
for result in rerank_response.get("results", []):
index = result.get("index")
if index is not None and index < len(original_results):
top_chunks.append(original_results[index])
return top_chunks

85
litellm/skills/main.py
View file

@ -23,12 +23,27 @@ from litellm.types.llms.anthropic_skills import (
Skill,
)
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager, client
# Initialize HTTP handler
base_llm_http_handler = BaseLLMHTTPHandler()
DEFAULT_ANTHROPIC_API_BASE = "https://api.anthropic.com/v1"
# Initialize LiteLLM skills handler (lazy - only used when custom_llm_provider="litellm")
_litellm_skills_handler = None
def _get_litellm_skills_handler():
"""Lazy initialization of LiteLLM skills handler to avoid import overhead."""
global _litellm_skills_handler
if _litellm_skills_handler is None:
from litellm.llms.litellm_proxy.skills.transformation import (
LiteLLMSkillsTransformationHandler,
)
_litellm_skills_handler = LiteLLMSkillsTransformationHandler()
return _litellm_skills_handler
@client
async def acreate_skill(
@ -133,18 +148,6 @@ def create_skill(
if custom_llm_provider is None:
custom_llm_provider = "anthropic"
# Get provider config
skills_api_provider_config: Optional[BaseSkillsAPIConfig] = (
ProviderConfigManager.get_provider_skills_api_config(
provider=litellm.LlmProviders(custom_llm_provider),
)
)
if skills_api_provider_config is None:
raise ValueError(
f"CREATE skill is not supported for {custom_llm_provider}"
)
# Build create request
create_request: CreateSkillRequest = {}
if display_title is not None:
@ -156,6 +159,30 @@ def create_skill(
if extra_body:
create_request.update(extra_body) # type: ignore
# Route to LiteLLM DB if custom_llm_provider="litellm_proxy"
if custom_llm_provider == LlmProviders.LITELLM_PROXY.value:
return _get_litellm_skills_handler().create_skill_handler(
display_title=display_title,
files=files,
metadata=extra_body.get("metadata") if extra_body else None,
user_id=kwargs.get("user_id"),
_is_async=_is_async,
logging_obj=litellm_logging_obj,
litellm_call_id=litellm_call_id,
)
# Get provider config for external providers (Anthropic, etc.)
skills_api_provider_config: Optional[BaseSkillsAPIConfig] = (
ProviderConfigManager.get_provider_skills_api_config(
provider=litellm.LlmProviders(custom_llm_provider),
)
)
if skills_api_provider_config is None:
raise ValueError(
f"CREATE skill is not supported for {custom_llm_provider}"
)
# Validate environment and get headers
headers = extra_headers or {}
headers = skills_api_provider_config.validate_environment(
@ -316,7 +343,17 @@ def list_skills(
if custom_llm_provider is None:
custom_llm_provider = "anthropic"
# Get provider config
# Route to LiteLLM DB if custom_llm_provider="litellm_proxy"
if custom_llm_provider == LlmProviders.LITELLM_PROXY.value:
return _get_litellm_skills_handler().list_skills_handler(
limit=limit or 20,
offset=0,
_is_async=_is_async,
logging_obj=litellm_logging_obj,
litellm_call_id=litellm_call_id,
)
# Get provider config for external providers (Anthropic, etc.)
skills_api_provider_config: Optional[BaseSkillsAPIConfig] = (
ProviderConfigManager.get_provider_skills_api_config(
provider=litellm.LlmProviders(custom_llm_provider),
@ -481,7 +518,16 @@ def get_skill(
if custom_llm_provider is None:
custom_llm_provider = "anthropic"
# Get provider config
# Route to LiteLLM DB if custom_llm_provider="litellm_proxy"
if custom_llm_provider == LlmProviders.LITELLM_PROXY.value:
return _get_litellm_skills_handler().get_skill_handler(
skill_id=skill_id,
_is_async=_is_async,
logging_obj=litellm_logging_obj,
litellm_call_id=litellm_call_id,
)
# Get provider config for external providers (Anthropic, etc.)
skills_api_provider_config: Optional[BaseSkillsAPIConfig] = (
ProviderConfigManager.get_provider_skills_api_config(
provider=litellm.LlmProviders(custom_llm_provider),
@ -638,7 +684,16 @@ def delete_skill(
if custom_llm_provider is None:
custom_llm_provider = "anthropic"
# Get provider config
# Route to LiteLLM DB if custom_llm_provider="litellm_proxy"
if custom_llm_provider == LlmProviders.LITELLM_PROXY.value:
return _get_litellm_skills_handler().delete_skill_handler(
skill_id=skill_id,
_is_async=_is_async,
logging_obj=litellm_logging_obj,
litellm_call_id=litellm_call_id,
)
# Get provider config for external providers (Anthropic, etc.)
skills_api_provider_config: Optional[BaseSkillsAPIConfig] = (
ProviderConfigManager.get_provider_skills_api_config(
provider=litellm.LlmProviders(custom_llm_provider),

1
litellm/types/llms/anthropic.py
View file

@ -358,6 +358,7 @@ class AnthropicMessagesRequestOptionalParams(TypedDict, total=False):
top_p: Optional[float]
mcp_servers: Optional[List[AnthropicMcpServerTool]]
context_management: Optional[Dict[str, Any]]
container: Optional[Dict[str, Any]] # Container config with skills for code execution
class AnthropicMessagesRequest(AnthropicMessagesRequestOptionalParams, total=False):

14
litellm/types/llms/openai.py
View file

@ -903,6 +903,7 @@ class ChatCompletionRequest(TypedDict, total=False):
functions: List
user: str
metadata: dict # litellm specific param
reasoning_effort: str # OpenAI o1/o3 reasoning parameter
class ChatCompletionDeltaChunk(TypedDict, total=False):
@ -1028,6 +1029,19 @@ OpenAIImageGenerationOptionalParams = Literal[
"user",
]
OpenAIImageEditOptionalParams = Literal[
"background",
"n",
"mask"
"output_compression",
"output_format",
"quality",
"partial_images",
"response_format",
"size",
"style",
"user",
]
class ComputerToolParam(TypedDict, total=False):
display_height: Required[float]

15
litellm/types/llms/stability.py
View file

@ -29,6 +29,13 @@ class StabilityImageGenerationRequest(TypedDict, total=False):
strength: Optional[float] # How much to transform the image (0-1)
style_preset: Optional[str] # Style preset name
class StabilityImageEditRequest(StabilityImageGenerationRequest):
"""
Request parameters for Stability AI image edit endpoint.
Endpoint: /v2beta/stable-image/edit/inpaint
"""
mask: Optional[str] # Base64-encoded mask (white = edit, black = keep)
class StabilityImageGenerationResponse(TypedDict, total=False):
"""
@ -197,16 +204,12 @@ STABILITY_EDIT_ENDPOINTS = {
"search-and-replace": "/v2beta/stable-image/edit/search-and-replace",
"search-and-recolor": "/v2beta/stable-image/edit/search-and-recolor",
"remove-background": "/v2beta/stable-image/edit/remove-background",
}
STABILITY_UPSCALE_ENDPOINTS = {
"replace-background-and-relight": "/v2beta/stable-image/edit/replace-background-and-relight",
"fast": "/v2beta/stable-image/upscale/fast",
"conservative": "/v2beta/stable-image/upscale/conservative",
"creative": "/v2beta/stable-image/upscale/creative",
}
STABILITY_CONTROL_ENDPOINTS = {
"sketch": "/v2beta/stable-image/control/sketch",
"structure": "/v2beta/stable-image/control/structure",
"style": "/v2beta/stable-image/control/style",
"style-transfer": "/v2beta/stable-image/control/style-transfer",
}

38
litellm/types/rag.py
View file

@ -7,6 +7,8 @@ from typing import Any, Dict, List, Literal, Optional, Union
from pydantic import BaseModel, ConfigDict
from typing_extensions import TypedDict
from litellm.types.utils import ModelResponse
class RAGChunkingStrategy(TypedDict, total=False):
"""
@ -187,3 +189,39 @@ class RAGIngestRequest(BaseModel):
model_config = ConfigDict(extra="allow") # Allow additional fields
class RAGRetrievalConfig(TypedDict, total=False):
"""Configuration for vector store retrieval."""
vector_store_id: str
custom_llm_provider: str
top_k: int # max results from vector store
filters: Optional[Dict[str, Any]] # optional - vector store filters
class RAGRerankConfig(TypedDict, total=False):
"""Configuration for reranking results."""
enabled: bool
model: str
top_n: int # final number of chunks after reranking
return_documents: Optional[bool]
class RAGQueryRequest(BaseModel):
"""Request body for RAG query API."""
model: str
messages: List[Any]
retrieval_config: RAGRetrievalConfig
rerank: Optional[RAGRerankConfig] = None
stream: Optional[bool] = False
model_config = ConfigDict(extra="allow")
class RAGQueryResponse(ModelResponse):
"""Response from RAG query API."""
pass

18
litellm/utils.py
View file

@ -7966,6 +7966,18 @@ class ProviderConfigManager:
)
return get_vertex_ai_image_edit_config(model)
elif LlmProviders.STABILITY == provider:
from litellm.llms.stability.image_edit import (
get_stability_image_edit_config,
)
return get_stability_image_edit_config(model)
elif LlmProviders.BEDROCK == provider:
from litellm.llms.bedrock.image_edit.stability_transformation import (
BedrockStabilityImageEditConfig,
)
return BedrockStabilityImageEditConfig()
return None
@staticmethod
@ -7984,9 +7996,13 @@ class ProviderConfigManager:
return get_azure_ai_ocr_config(model=model)
if provider == litellm.LlmProviders.VERTEX_AI:
from litellm.llms.vertex_ai.ocr.common_utils import get_vertex_ai_ocr_config
return get_vertex_ai_ocr_config(model=model)
PROVIDER_TO_CONFIG_MAP = {
litellm.LlmProviders.MISTRAL: MistralOCRConfig,
litellm.LlmProviders.VERTEX_AI: VertexAIOCRConfig,
}
config_class = PROVIDER_TO_CONFIG_MAP.get(provider, None)
if config_class is None:

170
model_prices_and_context_window.json
View file

@ -24483,6 +24483,90 @@
"output_cost_per_image": 0.08,
"supported_endpoints": ["/v1/images/generations"]
},
"stability/inpaint": {
"litellm_provider": "stability",
"mode": "image_edit",
"output_cost_per_image": 0.005,
"supported_endpoints": ["/v1/images/edits"]
},
"stability/outpaint": {
"litellm_provider": "stability",
"mode": "image_edit",
"output_cost_per_image": 0.004,
"supported_endpoints": ["/v1/images/edits"]
},
"stability/erase": {
"litellm_provider": "stability",
"mode": "image_edit",
"output_cost_per_image": 0.005,
"supported_endpoints": ["/v1/images/edits"]
},
"stability/search-and-replace": {
"litellm_provider": "stability",
"mode": "image_edit",
"output_cost_per_image": 0.005,
"supported_endpoints": ["/v1/images/edits"]
},
"stability/search-and-recolor": {
"litellm_provider": "stability",
"mode": "image_edit",
"output_cost_per_image": 0.005,
"supported_endpoints": ["/v1/images/edits"]
},
"stability/remove-background": {
"litellm_provider": "stability",
"mode": "image_edit",
"output_cost_per_image": 0.005,
"supported_endpoints": ["/v1/images/edits"]
},
"stability/replace-background-and-relight": {
"litellm_provider": "stability",
"mode": "image_edit",
"output_cost_per_image": 0.008,
"supported_endpoints": ["/v1/images/edits"]
},
"stability/sketch": {
"litellm_provider": "stability",
"mode": "image_edit",
"output_cost_per_image": 0.005,
"supported_endpoints": ["/v1/images/edits"]
},
"stability/structure": {
"litellm_provider": "stability",
"mode": "image_edit",
"output_cost_per_image": 0.005,
"supported_endpoints": ["/v1/images/edits"]
},
"stability/style": {
"litellm_provider": "stability",
"mode": "image_edit",
"output_cost_per_image": 0.005,
"supported_endpoints": ["/v1/images/edits"]
},
"stability/style-transfer": {
"litellm_provider": "stability",
"mode": "image_edit",
"output_cost_per_image": 0.008,
"supported_endpoints": ["/v1/images/edits"]
},
"stability/fast": {
"litellm_provider": "stability",
"mode": "image_edit",
"output_cost_per_image": 0.002,
"supported_endpoints": ["/v1/images/edits"]
},
"stability/conservative": {
"litellm_provider": "stability",
"mode": "image_edit",
"output_cost_per_image": 0.04,
"supported_endpoints": ["/v1/images/edits"]
},
"stability/creative": {
"litellm_provider": "stability",
"mode": "image_edit",
"output_cost_per_image": 0.06,
"supported_endpoints": ["/v1/images/edits"]
},
"stability/stable-image-core": {
"litellm_provider": "stability",
"mode": "image_generation",
@ -24510,6 +24594,84 @@
"mode": "image_generation",
"output_cost_per_image": 0.04
},
"stability.stable-conservative-upscale-v1:0": {
"litellm_provider": "bedrock",
"max_input_tokens": 77,
"mode": "image_edit",
"output_cost_per_image": 0.40
},
"stability.stable-creative-upscale-v1:0": {
"litellm_provider": "bedrock",
"max_input_tokens": 77,
"mode": "image_edit",
"output_cost_per_image": 0.60
},
"stability.stable-fast-upscale-v1:0": {
"litellm_provider": "bedrock",
"max_input_tokens": 77,
"mode": "image_edit",
"output_cost_per_image": 0.03
},
"stability.stable-outpaint-v1:0": {
"litellm_provider": "bedrock",
"max_input_tokens": 77,
"mode": "image_edit",
"output_cost_per_image": 0.06
},
"stability.stable-image-control-sketch-v1:0": {
"litellm_provider": "bedrock",
"max_input_tokens": 77,
"mode": "image_edit",
"output_cost_per_image": 0.07
},
"stability.stable-image-control-structure-v1:0": {
"litellm_provider": "bedrock",
"max_input_tokens": 77,
"mode": "image_edit",
"output_cost_per_image": 0.07
},
"stability.stable-image-erase-object-v1:0": {
"litellm_provider": "bedrock",
"max_input_tokens": 77,
"mode": "image_edit",
"output_cost_per_image": 0.07
},
"stability.stable-image-inpaint-v1:0": {
"litellm_provider": "bedrock",
"max_input_tokens": 77,
"mode": "image_edit",
"output_cost_per_image": 0.07
},
"stability.stable-image-remove-background-v1:0": {
"litellm_provider": "bedrock",
"max_input_tokens": 77,
"mode": "image_edit",
"output_cost_per_image": 0.07
},
"stability.stable-image-search-recolor-v1:0": {
"litellm_provider": "bedrock",
"max_input_tokens": 77,
"mode": "image_edit",
"output_cost_per_image": 0.07
},
"stability.stable-image-search-replace-v1:0": {
"litellm_provider": "bedrock",
"max_input_tokens": 77,
"mode": "image_edit",
"output_cost_per_image": 0.07
},
"stability.stable-image-style-guide-v1:0": {
"litellm_provider": "bedrock",
"max_input_tokens": 77,
"mode": "image_edit",
"output_cost_per_image": 0.07
},
"stability.stable-style-transfer-v1:0": {
"litellm_provider": "bedrock",
"max_input_tokens": 77,
"mode": "image_edit",
"output_cost_per_image": 0.08
},
"stability.stable-image-core-v1:1": {
"litellm_provider": "bedrock",
"max_input_tokens": 77,
@ -27777,6 +27939,14 @@
],
"source": "https://cloud.google.com/generative-ai-app-builder/pricing"
},
"vertex_ai/deepseek-ai/deepseek-ocr-maas": {
"litellm_provider": "vertex_ai",
"mode": "ocr",
"input_cost_per_token": 3e-07,
"output_cost_per_token": 1.2e-06,
"ocr_cost_per_page": 3e-04,
"source": "https://cloud.google.com/vertex-ai/pricing"
},
"vertex_ai/openai/gpt-oss-120b-maas": {
"input_cost_per_token": 1.5e-07,
"litellm_provider": "vertex_ai-openai_models",

17
provider_endpoints_support.json
View file

@ -84,6 +84,23 @@
"a2a": true
}
},
"amazon_nova": {
"display_name": "Amazon Nova (`amazon_nova`)",
"url": "https://docs.litellm.ai/docs/providers/amazon_nova",
"endpoints": {
"chat_completions": true,
"messages": true,
"responses": true,
"embeddings": false,
"image_generations": false,
"audio_transcriptions": false,
"audio_speech": false,
"moderations": false,
"batches": false,
"rerank": false,
"a2a": true
}
},
"anthropic": {
"display_name": "Anthropic (`anthropic`)",
"url": "https://docs.litellm.ai/docs/providers/anthropic",

1
requirements.txt
View file

@ -48,6 +48,7 @@ detect-secrets==1.5.0 # Enterprise - secret detection / masking in LLM requests
cryptography==44.0.1
tzdata==2025.1 # IANA time zone database
litellm-proxy-extras==0.4.15 # for proxy extras - e.g. prisma migrations
llm-sandbox==0.3.31 # for skill execution in sandbox
### LITELLM PACKAGE DEPENDENCIES
python-dotenv==1.0.1 # for env
tiktoken==0.8.0 # for calculating usage

Some files were not shown because too many files have changed in this diff Show more