mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-12 23:01:41 +00:00
Merge branch 'main' into fix-lint
This commit is contained in:
Kristin Cowalcijk
GitHub
commit
589cc31f14
717 changed files with 13946 additions and 4535 deletions
13
README.md
13
README.md
|
|
@ -2,20 +2,24 @@
|
|||
🚅 LiteLLM
|
||||
</h1>
|
||||
<p align="center">
|
||||
<p align="center">Call 100+ LLMs in OpenAI format. [Bedrock, Azure, OpenAI, VertexAI, Anthropic, Groq, etc.]
|
||||
<p align="center">LiteLLM AI Gateway
|
||||
</p>
|
||||
<p align="center">Open Source AI Gateway for 100+ LLMs. Self-hosted. Enterprise-ready. Call any LLM in OpenAI format.</p>
|
||||
<p align="center">
|
||||
<a href="https://render.com/deploy?repo=https://github.com/BerriAI/litellm" target="_blank" rel="nofollow"><img src="https://render.com/images/deploy-to-render-button.svg" alt="Deploy to Render"></a>
|
||||
<a href="https://railway.app/template/HLP0Ub?referralCode=jch2ME">
|
||||
<img src="https://railway.app/button.svg" alt="Deploy on Railway">
|
||||
<a href="https://railway.com/deploy/RhvhdC?referralCode=7mRv9K&utm_medium=integration&utm_source=template&utm_campaign=generic">
|
||||
<img src="https://railway.com/button.svg" alt="Deploy on Railway">
|
||||
</a>
|
||||
</p>
|
||||
</p>
|
||||
<h4 align="center"><a href="https://docs.litellm.ai/docs/simple_proxy" target="_blank">LiteLLM Proxy Server (AI Gateway)</a> | <a href="https://docs.litellm.ai/docs/enterprise#hosted-litellm-proxy" target="_blank"> Hosted Proxy</a> | <a href="https://docs.litellm.ai/docs/enterprise"target="_blank">Enterprise Tier</a></h4>
|
||||
<h4 align="center"><a href="https://docs.litellm.ai/docs/simple_proxy" target="_blank">LiteLLM Proxy Server (AI Gateway)</a> | <a href="https://docs.litellm.ai/docs/enterprise#hosted-litellm-proxy" target="_blank"> Hosted Proxy</a> | <a href="https://litellm.ai/enterprise"target="_blank">Enterprise Tier</a> | <a href="https://litellm.ai/" target="_blank">Website</a></h4>
|
||||
<h4 align="center">
|
||||
<a href="https://pypi.org/project/litellm/" target="_blank">
|
||||
<img src="https://img.shields.io/pypi/v/litellm.svg" alt="PyPI Version">
|
||||
</a>
|
||||
<a href="https://github.com/BerriAI/litellm" target="_blank">
|
||||
<img src="https://img.shields.io/github/stars/BerriAI/litellm.svg?style=social" alt="GitHub Stars">
|
||||
</a>
|
||||
<a href="https://www.ycombinator.com/companies/berriai">
|
||||
<img src="https://img.shields.io/badge/Y%20Combinator-W23-orange?style=flat-square" alt="Y Combinator W23">
|
||||
</a>
|
||||
|
|
@ -403,6 +407,7 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature
|
|||
# Enterprise
|
||||
For companies that need better security, user management and professional support
|
||||
|
||||
[Get an Enterprise License](https://litellm.ai/enterprise)
|
||||
[Talk to founders](https://enterprise.litellm.ai/demo)
|
||||
|
||||
This covers:
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ USER root
|
|||
# Install build dependencies in one layer
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
gcc \
|
||||
g++ \
|
||||
python3-dev \
|
||||
libssl-dev \
|
||||
pkg-config \
|
||||
|
|
|
|||
|
|
@ -142,6 +142,9 @@ COPY --from=builder /app/requirements.txt /app/requirements.txt
|
|||
COPY --from=builder /app/docker/entrypoint.sh /app/docker/prod_entrypoint.sh /app/docker/
|
||||
COPY --from=builder /app/docker/supervisord.conf /etc/supervisord.conf
|
||||
COPY --from=builder /app/schema.prisma /app/
|
||||
# Keep enterprise bridge module in runtime so `enterprise.enterprise_hooks`
|
||||
# can load and register managed enterprise hooks (e.g. managed_files).
|
||||
COPY --from=builder /app/enterprise /app/enterprise
|
||||
# Copy prisma_migration.py for Helm migrations job compatibility
|
||||
COPY --from=builder /app/litellm/proxy/prisma_migration.py /app/litellm/proxy/prisma_migration.py
|
||||
COPY --from=builder /wheels/ /wheels/
|
||||
|
|
|
|||
|
|
@ -278,7 +278,8 @@ mcp_servers:
|
|||
url: "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes/<url-encoded-ARN>/invocations"
|
||||
transport: "http"
|
||||
auth_type: "aws_sigv4"
|
||||
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
|
||||
aws_role_name: os.environ/AWS_ROLE_ARN # optional — IAM role to assume
|
||||
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID # optional — falls back to IAM role
|
||||
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
|
||||
aws_region_name: us-east-1
|
||||
aws_service_name: bedrock-agentcore
|
||||
|
|
|
|||
|
|
@ -36,6 +36,8 @@ LiteLLM's `aws_sigv4` auth type handles this automatically: every outgoing MCP r
|
|||
| **AWS Access Key ID** | No | Falls back to boto3 credential chain if blank |
|
||||
| **AWS Secret Access Key** | No | Required if Access Key ID is provided |
|
||||
| **AWS Session Token** | No | Only needed for temporary STS credentials |
|
||||
| **AWS Role ARN** | No | IAM role ARN for STS AssumeRole (e.g., `arn:aws:iam::123456789012:role/MyRole`). If set, LiteLLM assumes this role before signing |
|
||||
| **AWS Session Name** | No | Session name for the AssumeRole call — appears in CloudTrail. Auto-generated if omitted |
|
||||
|
||||
Once created, LiteLLM will sign every outgoing MCP request with SigV4. The server's tools appear automatically in the MCP Tools list.
|
||||
|
||||
|
|
@ -66,8 +68,8 @@ mcp_servers:
|
|||
url: "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes/<url-encoded-ARN>/invocations"
|
||||
transport: "http"
|
||||
auth_type: "aws_sigv4"
|
||||
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
|
||||
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
|
||||
aws_role_name: os.environ/AWS_ROLE_ARN # IAM role to assume (recommended)
|
||||
aws_session_name: "litellm-prod" # optional — for CloudTrail auditing
|
||||
aws_region_name: "us-east-1"
|
||||
aws_service_name: "bedrock-agentcore"
|
||||
```
|
||||
|
|
@ -128,6 +130,8 @@ curl http://localhost:4000/mcp-rest/tools/call \
|
|||
| `aws_region_name` | Yes | AWS region (e.g., `us-east-1`) |
|
||||
| `aws_service_name` | No | AWS service name for signing. Defaults to `bedrock-agentcore` |
|
||||
| `aws_session_token` | No | AWS session token for temporary credentials. Supports `os.environ/VAR_NAME` |
|
||||
| `aws_role_name` | No | IAM role ARN for STS AssumeRole. Supports `os.environ/VAR_NAME`. When set, LiteLLM calls `sts:AssumeRole` to get temporary credentials before signing |
|
||||
| `aws_session_name` | No | Session name for the AssumeRole call (appears in CloudTrail). Auto-generated if omitted. Supports `os.environ/VAR_NAME` |
|
||||
|
||||
## How It Works
|
||||
|
||||
|
|
@ -157,6 +161,42 @@ mcp_servers:
|
|||
aws_service_name: "bedrock-agentcore"
|
||||
```
|
||||
|
||||
## Using IAM Role Assumption (AssumeRole)
|
||||
|
||||
For production environments where your LiteLLM instance authenticates via an IAM role (e.g., EKS pod role, EC2 instance profile), you can configure `aws_role_name` to have LiteLLM call `sts:AssumeRole` before signing MCP requests:
|
||||
|
||||
```yaml title="config.yaml with AssumeRole" showLineNumbers
|
||||
mcp_servers:
|
||||
my_agentcore_mcp:
|
||||
url: "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes/<url-encoded-ARN>/invocations"
|
||||
transport: "http"
|
||||
auth_type: "aws_sigv4"
|
||||
aws_role_name: "arn:aws:iam::123456789012:role/BedrockAgentCoreRole"
|
||||
aws_session_name: "litellm-prod" # optional
|
||||
aws_region_name: "us-east-1"
|
||||
aws_service_name: "bedrock-agentcore"
|
||||
```
|
||||
|
||||
LiteLLM uses the ambient credentials (pod role, instance profile, or env vars) to call `sts:AssumeRole`, then signs MCP requests with the assumed role's temporary credentials.
|
||||
|
||||
You can also combine `aws_role_name` with explicit access keys — the keys are then used as the source identity for the AssumeRole call:
|
||||
|
||||
```yaml title="config.yaml with AssumeRole + explicit source keys" showLineNumbers
|
||||
mcp_servers:
|
||||
my_agentcore_mcp:
|
||||
url: "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes/<url-encoded-ARN>/invocations"
|
||||
transport: "http"
|
||||
auth_type: "aws_sigv4"
|
||||
aws_role_name: os.environ/AWS_ROLE_ARN
|
||||
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
|
||||
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
|
||||
aws_region_name: "us-east-1"
|
||||
```
|
||||
|
||||
:::tip
|
||||
For most Kubernetes deployments, you only need `aws_role_name` and `aws_region_name` — the pod's IAM role provides the source credentials automatically.
|
||||
:::
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### 403 Forbidden from AWS
|
||||
|
|
@ -166,6 +206,15 @@ mcp_servers:
|
|||
- Ensure `aws_service_name` is set to `bedrock-agentcore`
|
||||
- If using STS credentials, confirm `aws_session_token` is set and not expired
|
||||
|
||||
### AssumeRole AccessDenied
|
||||
|
||||
If you get `AccessDenied` when using `aws_role_name`:
|
||||
|
||||
- Verify the role ARN is correct
|
||||
- Check that the trust policy on the target role allows your source identity to assume it
|
||||
- If running on EKS, ensure the pod's service account is annotated with the correct IAM role
|
||||
- Check CloudTrail for the failed `sts:AssumeRole` call to see the exact error
|
||||
|
||||
### Health check errors on startup
|
||||
|
||||
SigV4-authenticated MCP servers skip the standard health check on proxy startup. This is expected — the proxy will still sign requests correctly when tools are invoked.
|
||||
|
|
|
|||
231
docs/my-website/docs/mcp_toolsets.md
Normal file
231
docs/my-website/docs/mcp_toolsets.md
Normal file
|
|
@ -0,0 +1,231 @@
|
|||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
# MCP Toolsets
|
||||
|
||||
A **Toolset** is a named collection of specific tools drawn from one or more MCP servers. Instead of giving an agent access to every tool on every server, you pick exactly which tools it needs — from whichever servers they live on — and bundle them under a single name.
|
||||
|
||||
## How it works
|
||||
|
||||
```
|
||||
┌─────────────────────────────────┐
|
||||
│ MCP Toolset │
|
||||
│ "devtooling-prod" │
|
||||
└────────────┬────────────────────┘
|
||||
│
|
||||
┌──────────────────┴──────────────────┐
|
||||
│ │
|
||||
┌────────▼────────┐ ┌────────▼────────┐
|
||||
│ CircleCI MCP │ │ DeepWiki MCP │
|
||||
│ (10+ tools) │ │ (3 tools) │
|
||||
└────────┬────────┘ └────────┬────────┘
|
||||
│ │
|
||||
┌─────────┴──────────┐ ┌──────────┴──────────┐
|
||||
│ ✓ get_build_logs │ │ ✓ read_wiki_structure│
|
||||
│ ✓ find_flaky_tests │ │ ✓ read_wiki_contents │
|
||||
│ ✓ get_pipeline_ │ │ ✗ ask_question │
|
||||
│ status │ └─────────────────────┘
|
||||
│ ✓ run_pipeline │
|
||||
│ ✗ list_followed_ │
|
||||
│ projects │
|
||||
└────────────────────┘
|
||||
|
||||
Agent sees exactly 6 tools, nothing more.
|
||||
```
|
||||
|
||||
Instead of 13+ tools across two servers, the agent gets 6 — the ones it actually needs.
|
||||
|
||||
**Why this matters:**
|
||||
- Smaller tool lists → fewer tokens, faster responses, less hallucination
|
||||
- Combine tools from GitHub + Linear + CircleCI into one named grant
|
||||
- Assign to keys and teams the same way you assign MCP servers today
|
||||
|
||||
---
|
||||
|
||||
## Create a toolset
|
||||
|
||||
### 1. Go to the MCP page
|
||||
|
||||
Navigate to **MCP** in the left sidebar.
|
||||
|
||||
|
||||
|
||||
### 2. Open the Toolsets tab
|
||||
|
||||
Click the **Toolsets** tab on the MCP page.
|
||||
|
||||
|
||||
|
||||
### 3. Click "New Toolset"
|
||||
|
||||
|
||||
|
||||
### 4. Enter a name
|
||||
|
||||
Type a name for the toolset. Pick something descriptive — this is what agents will reference.
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
### 5. Add the first tool
|
||||
|
||||
Select an MCP server from the dropdown, then choose the tool you want to include from that server.
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
### 6. Add tools from a second server
|
||||
|
||||
Click **Add Tool**, pick a different MCP server, and select another tool. Repeat for as many tools as you need — they can come from any number of servers.
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
### 7. Create the toolset
|
||||
|
||||
Click **Create Toolset** to save.
|
||||
|
||||
|
||||
|
||||
---
|
||||
|
||||
## Use a toolset in the Playground
|
||||
|
||||
Once created, your toolset appears alongside MCP servers in the **MCP Servers** dropdown in the Playground — it's selectable the same way.
|
||||
|
||||
### 1. Go to the Playground
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
### 2. Select your toolset from MCP Servers
|
||||
|
||||
In the left panel under **MCP Servers**, open the dropdown and pick your toolset. The model will only see the tools you included in it.
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
The model now has access to exactly the tools in your toolset and nothing else.
|
||||
|
||||
---
|
||||
|
||||
## Use a toolset via API
|
||||
|
||||
Pass the toolset's route as the `server_url` in your tools list. LiteLLM resolves it server-side — no public URL needed.
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="responses" label="Responses API">
|
||||
|
||||
```python
|
||||
import openai
|
||||
|
||||
client = openai.OpenAI(
|
||||
api_key="your-litellm-key",
|
||||
base_url="http://your-proxy/v1",
|
||||
)
|
||||
|
||||
response = client.responses.create(
|
||||
model="gpt-4o",
|
||||
input="What CI/CD tools do you have?",
|
||||
tools=[
|
||||
{
|
||||
"type": "mcp",
|
||||
"server_label": "devtooling-prod",
|
||||
"server_url": "litellm_proxy/mcp/devtooling-prod",
|
||||
"require_approval": "never",
|
||||
}
|
||||
],
|
||||
)
|
||||
print(response.output_text)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="chat" label="Chat Completions API">
|
||||
|
||||
```python
|
||||
import openai
|
||||
|
||||
client = openai.OpenAI(
|
||||
api_key="your-litellm-key",
|
||||
base_url="http://your-proxy/v1",
|
||||
)
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model="gpt-4o",
|
||||
messages=[{"role": "user", "content": "What CI/CD tools do you have?"}],
|
||||
tools=[
|
||||
{
|
||||
"type": "mcp",
|
||||
"server_label": "devtooling-prod",
|
||||
"server_url": "litellm_proxy/mcp/devtooling-prod",
|
||||
"require_approval": "never",
|
||||
}
|
||||
],
|
||||
)
|
||||
print(response.choices[0].message.content)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="rest" label="REST">
|
||||
|
||||
```bash
|
||||
curl http://your-proxy/v1/responses \
|
||||
-H "Authorization: Bearer your-litellm-key" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "gpt-4o",
|
||||
"input": "What CI/CD tools do you have?",
|
||||
"tools": [
|
||||
{
|
||||
"type": "mcp",
|
||||
"server_label": "devtooling-prod",
|
||||
"server_url": "litellm_proxy/mcp/devtooling-prod",
|
||||
"require_approval": "never"
|
||||
}
|
||||
]
|
||||
}'
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
---
|
||||
|
||||
## Manage toolsets via API
|
||||
|
||||
```bash
|
||||
# List all toolsets
|
||||
curl http://your-proxy/v1/mcp/toolset \
|
||||
-H "Authorization: Bearer your-litellm-key"
|
||||
|
||||
# Create a toolset
|
||||
curl -X POST http://your-proxy/v1/mcp/toolset \
|
||||
-H "Authorization: Bearer your-litellm-key" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"toolset_name": "devtooling-prod",
|
||||
"description": "CircleCI + DeepWiki tools for the dev team",
|
||||
"tools": [
|
||||
{"server_id": "<circleci-server-id>", "tool_name": "get_build_failure_logs"},
|
||||
{"server_id": "<circleci-server-id>", "tool_name": "run_pipeline"},
|
||||
{"server_id": "<deepwiki-server-id>", "tool_name": "read_wiki_structure"}
|
||||
]
|
||||
}'
|
||||
|
||||
# Delete a toolset
|
||||
curl -X DELETE http://your-proxy/v1/mcp/toolset/<toolset_id> \
|
||||
-H "Authorization: Bearer your-litellm-key"
|
||||
```
|
||||
|
|
@ -8,24 +8,54 @@ Check the [OCI Models List](https://docs.oracle.com/en-us/iaas/Content/generativ
|
|||
|
||||
## Supported Models
|
||||
|
||||
### Meta Llama Models
|
||||
### Chat / Text Generation
|
||||
|
||||
#### Meta Llama Models
|
||||
- `meta.llama-4-maverick-17b-128e-instruct-fp8`
|
||||
- `meta.llama-4-scout-17b-16e-instruct`
|
||||
- `meta.llama-3.3-70b-instruct`
|
||||
- `meta.llama-3.3-70b-instruct-fp8-dynamic`
|
||||
- `meta.llama-3.2-90b-vision-instruct`
|
||||
- `meta.llama-3.2-11b-vision-instruct`
|
||||
- `meta.llama-3.1-405b-instruct`
|
||||
- `meta.llama-3.1-70b-instruct`
|
||||
|
||||
### xAI Grok Models
|
||||
#### xAI Grok Models
|
||||
- `xai.grok-4.20`
|
||||
- `xai.grok-4.20-multi-agent`
|
||||
- `xai.grok-4`
|
||||
- `xai.grok-4-fast`
|
||||
- `xai.grok-4.1-fast`
|
||||
- `xai.grok-3`
|
||||
- `xai.grok-3-fast`
|
||||
- `xai.grok-3-mini`
|
||||
- `xai.grok-3-mini-fast`
|
||||
- `xai.grok-code-fast-1`
|
||||
|
||||
### Cohere Models
|
||||
#### Cohere Models
|
||||
- `cohere.command-latest`
|
||||
- `cohere.command-a-03-2025`
|
||||
- `cohere.command-a-reasoning-08-2025`
|
||||
- `cohere.command-a-vision-07-2025`
|
||||
- `cohere.command-a-translate-08-2025`
|
||||
- `cohere.command-plus-latest`
|
||||
- `cohere.command-r-08-2024`
|
||||
- `cohere.command-r-plus-08-2024`
|
||||
|
||||
#### Google Gemini Models (via OCI)
|
||||
- `google.gemini-2.5-pro`
|
||||
- `google.gemini-2.5-flash`
|
||||
- `google.gemini-2.5-flash-lite`
|
||||
|
||||
### Embedding Models
|
||||
- `cohere.embed-english-v3.0` (1024 dimensions)
|
||||
- `cohere.embed-english-light-v3.0` (384 dimensions)
|
||||
- `cohere.embed-multilingual-v3.0` (1024 dimensions)
|
||||
- `cohere.embed-multilingual-light-v3.0` (384 dimensions)
|
||||
- `cohere.embed-english-image-v3.0` (1024 dimensions, multimodal)
|
||||
- `cohere.embed-english-light-image-v3.0` (384 dimensions, multimodal)
|
||||
- `cohere.embed-multilingual-light-image-v3.0` (384 dimensions, multimodal)
|
||||
- `cohere.embed-v4.0` (1536 dimensions, multimodal)
|
||||
|
||||
## Authentication
|
||||
|
||||
|
|
@ -394,4 +424,75 @@ response = completion(
|
|||
| `oci_tenancy` | string | - | (Manual auth) The OCID of your OCI tenancy |
|
||||
| `oci_key` | string | - | (Manual auth) The private key content as a string |
|
||||
| `oci_key_file` | string | - | (Manual auth) Path to the private key file |
|
||||
| `oci_signer` | object | - | (SDK auth) OCI SDK Signer object for authentication |
|
||||
| `oci_signer` | object | - | (SDK auth) OCI SDK Signer object for authentication |
|
||||
|
||||
## Embeddings
|
||||
|
||||
LiteLLM supports OCI Generative AI embedding models. These models use the same authentication methods described above.
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="embed-manual" label="Manual Credentials" default>
|
||||
|
||||
```python
|
||||
from litellm import embedding
|
||||
|
||||
response = embedding(
|
||||
model="oci/cohere.embed-english-v3.0",
|
||||
input=["Hello world", "Goodbye world"],
|
||||
oci_region="us-ashburn-1",
|
||||
oci_user=<your_oci_user>,
|
||||
oci_fingerprint=<your_oci_fingerprint>,
|
||||
oci_tenancy=<your_oci_tenancy>,
|
||||
oci_key=<string_with_content_of_oci_key>,
|
||||
oci_compartment_id=<oci_compartment_id>,
|
||||
)
|
||||
print(response)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="embed-sdk" label="OCI SDK Signer">
|
||||
|
||||
```python
|
||||
from litellm import embedding
|
||||
from oci.signer import Signer
|
||||
|
||||
signer = Signer(
|
||||
tenancy="ocid1.tenancy.oc1..",
|
||||
user="ocid1.user.oc1..",
|
||||
fingerprint="xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx",
|
||||
private_key_file_location="~/.oci/key.pem",
|
||||
)
|
||||
|
||||
response = embedding(
|
||||
model="oci/cohere.embed-english-v3.0",
|
||||
input=["Hello world", "Goodbye world"],
|
||||
oci_signer=signer,
|
||||
oci_region="us-ashburn-1",
|
||||
oci_compartment_id="<oci_compartment_id>",
|
||||
)
|
||||
print(response)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
### Embedding Optional Parameters
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| `input_type` | string | - | The type of input: `search_document`, `search_query`, `classification`, `clustering` |
|
||||
| `truncate` | string | `END` | Truncation strategy when input exceeds max tokens: `END` or `START` |
|
||||
|
||||
### Using Dedicated Embedding Endpoints
|
||||
|
||||
```python
|
||||
response = embedding(
|
||||
model="oci/cohere.embed-english-v3.0",
|
||||
input=["Hello world"],
|
||||
oci_serving_mode="DEDICATED",
|
||||
oci_endpoint_id="ocid1.generativeaiendpoint.oc1...",
|
||||
oci_region="us-ashburn-1",
|
||||
oci_compartment_id="<oci_compartment_id>",
|
||||
# ... auth params
|
||||
)
|
||||
```
|
||||
|
|
@ -201,6 +201,7 @@ router_settings:
|
|||
| enable_json_schema_validation | boolean | If true, enables json schema validation for all requests. |
|
||||
| enable_key_alias_format_validation | boolean | If true, validates `key_alias` format on `/key/generate` and `/key/update`. Must be 2-255 chars, start/end with alphanumeric, only allow `a-zA-Z0-9_-/.@`. Default `false`. |
|
||||
| disable_copilot_system_to_assistant | boolean | **DEPRECATED** - GitHub Copilot API supports system prompts. |
|
||||
| default_team_params | object | Default parameters applied to every new team created via `/team/new` (including SSO auto-created teams). Only fills in fields not explicitly set in the request. Sub-fields: `max_budget` (float), `budget_duration` (string, e.g. `"30d"`), `tpm_limit` (integer), `rpm_limit` (integer), `team_member_permissions` (array of strings, e.g. `["/team/daily/activity", "/key/generate"]`), `models` (array of strings — only applied to SSO auto-created teams). |
|
||||
|
||||
### general_settings - Reference
|
||||
|
||||
|
|
@ -288,6 +289,7 @@ router_settings:
|
|||
| database_connection_pool_timeout | integer | Database connection pool timeout in seconds |
|
||||
| disable_error_logs | boolean | If true, suppresses error tracking and storage in the database |
|
||||
| enable_health_check_routing | boolean | If true, enables health check-driven request routing to avoid unhealthy deployments |
|
||||
| health_check_ignore_transient_errors | boolean | If true, 429 (rate limit) and 408 (timeout) health check failures are ignored and do not affect routing or cooldown |
|
||||
| enable_mcp_registry | boolean | If true, enables access to the centralized MCP server registry |
|
||||
| enforce_rbac | boolean | If true, enables role-based access control (RBAC) for all proxy operations |
|
||||
| forward_llm_provider_auth_headers | boolean | If true, forwards provider-specific auth headers to LLM API calls |
|
||||
|
|
@ -396,6 +398,7 @@ router_settings:
|
|||
| guardrail_list | List[GuardrailTypedDict] | List of guardrail configurations for guardrail load balancing. Enables load balancing across multiple guardrail deployments with the same guardrail_name. [Further Docs](./guardrails/guardrail_load_balancing.md) |
|
||||
| enable_health_check_routing | boolean | If true, enables health check-driven deployment filtering to avoid routing requests to unhealthy deployments |
|
||||
| health_check_staleness_threshold | integer | Maximum age in seconds for cached health check results before marking deployments as stale |
|
||||
| health_check_ignore_transient_errors | boolean | If true, 429 (rate limit) and 408 (timeout) health check failures are ignored and do not affect routing or cooldown |
|
||||
|
||||
|
||||
### environment variables - Reference
|
||||
|
|
@ -820,6 +823,7 @@ router_settings:
|
|||
| LITELLM_KEY_ROTATION_ENABLED | Enable auto-key rotation for LiteLLM (boolean). Default is false.
|
||||
| LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS | Interval in seconds for how often to run job that auto-rotates keys. Default is 86400 (24 hours).
|
||||
| LITELLM_KEY_ROTATION_GRACE_PERIOD | Duration to keep old key valid after rotation (e.g. "24h", "2d"). Default is empty (immediate revoke). Used for scheduled rotations and as fallback when not specified in regenerate request.
|
||||
| LITELLM_KEY_ROTATION_LOCK_TTL_SECONDS | TTL in seconds for the distributed lock used by the key rotation job. Default is 600 (10 minutes).
|
||||
| LITELLM_LICENSE | License key for LiteLLM usage
|
||||
| LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS | Set to `True` to use the local bundled Anthropic beta headers config only, disabling remote fetching. Default is `False`
|
||||
| LITELLM_LOCAL_BLOG_POSTS | When set to `True`, uses the local bundled blog posts only, disabling remote fetching from GitHub. Default is `False`
|
||||
|
|
|
|||
|
|
@ -311,7 +311,7 @@ Response:
|
|||
|
||||
## Policy Flow Builder
|
||||
|
||||
For conditional execution (e.g., run a second guardrail only if the first fails), use the [Policy Flow Builder](./policy_flow_builder) to define pipelines with per-step pass/fail actions.
|
||||
For conditional execution (e.g., run a second guardrail only if the first fails), use the [Policy Flow Builder](./policy_flow_builder) to define pipelines with per-step **pass**, **fail**, and optional **error** actions (`on_pass`, `on_fail`, `on_error`).
|
||||
|
||||
## Config Reference
|
||||
|
||||
|
|
@ -337,7 +337,7 @@ policies:
|
|||
| `guardrails.add` | `list[string]` | Guardrails to enable. |
|
||||
| `guardrails.remove` | `list[string]` | Guardrails to disable (useful with inheritance). |
|
||||
| `condition.model` | `string` or `list[string]` | Optional. Only apply when model matches. Supports regex. |
|
||||
| `pipeline` | `object` | Optional. Ordered guardrail execution with per-step actions. See [Policy Flow Builder](./policy_flow_builder). |
|
||||
| `pipeline` | `object` | Optional. Ordered guardrail execution with per-step actions (`on_pass`, `on_fail`, optional `on_error`). See [Policy Flow Builder](./policy_flow_builder). |
|
||||
|
||||
### `policy_attachments`
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
# Policy Flow Builder
|
||||
|
||||
The Policy Flow Builder lets you design guardrail pipelines with **conditional execution**. Instead of running guardrails independently, you chain them into ordered steps and control what happens when each guardrail passes or fails.
|
||||
The Policy Flow Builder lets you design guardrail pipelines with **conditional execution**. Instead of running guardrails independently, you chain them into ordered steps and control what happens when each guardrail **passes**, **fails a policy check** (content intervention), or hits a **technical error** (e.g. timeout, unreachable provider, missing guardrail).
|
||||
|
||||
Two powerful patterns it enables: **guardrail fallbacks** (try a different guardrail when one fails) and **retrying the same guardrail** (run the same guardrail again if it fails, e.g. to handle transient errors).
|
||||
Two powerful patterns it enables: **guardrail fallbacks** (try a different guardrail when one fails) and **retrying the same guardrail** (run the same guardrail again if it fails, e.g. to handle transient errors). With **`on_error`**, you can treat **technical** failures differently from **policy** failures—for example, fall back to another provider when the primary API errors, while still blocking on flagged content.
|
||||
|
||||
## When to use the Flow Builder
|
||||
|
||||
|
|
@ -19,6 +19,7 @@ Use the Flow Builder when you need:
|
|||
- **Custom responses** — return a specific message when a guardrail fails instead of a generic block
|
||||
- **Data chaining** — pass modified data (e.g., PII-masked content) from one step to the next
|
||||
- **Fine-grained control** — different actions on pass vs. fail per step
|
||||
- **Technical-error routing** — set `on_error` separately from `on_fail` so outages or timeouts can **allow**, **block**, **go to the next step**, or return a **custom response** without conflating them with content violations
|
||||
|
||||
## Concepts
|
||||
|
||||
|
|
@ -29,24 +30,37 @@ A pipeline has:
|
|||
- **Mode**: `pre_call` (before the LLM) or `post_call` (after the LLM)
|
||||
- **Steps**: Ordered list of guardrail steps
|
||||
|
||||
### Outcomes: pass, fail, and error
|
||||
|
||||
Each step run produces one of three outcomes:
|
||||
|
||||
| Outcome | Meaning | Typical cause |
|
||||
|--------|---------|----------------|
|
||||
| **pass** | Guardrail completed without blocking | Content allowed, or data was modified and returned |
|
||||
| **fail** | Policy intervention | Guardrail raised an intervention (e.g. flagged content, blocked request) |
|
||||
| **error** | Technical failure | Timeouts, network errors, guardrail not registered, or other non-intervention exceptions |
|
||||
|
||||
`on_pass` and `on_fail` apply to **pass** and **fail** respectively. **`on_error`** applies only to **error**. If `on_error` is omitted, the pipeline uses **`on_fail`** for error outcomes (backward compatible).
|
||||
|
||||
### Step actions
|
||||
|
||||
Each step defines what happens when the guardrail **passes** and when it **fails**:
|
||||
For each step you choose an action for **pass**, **fail**, and optionally **error**. Allowed values are: `next`, `allow`, `block`, `modify_response`.
|
||||
|
||||
| Action | Description |
|
||||
|--------|-------------|
|
||||
| **Next Step** | Continue to the next guardrail in the pipeline |
|
||||
| **Allow** | Stop the pipeline and allow the request to proceed |
|
||||
| **Block** | Stop the pipeline and block the request |
|
||||
| **Custom Response** | Return a custom message instead of the default block |
|
||||
| **Next Step** (`next`) | Continue to the next guardrail in the pipeline |
|
||||
| **Allow** (`allow`) | Stop the pipeline and allow the request to proceed |
|
||||
| **Block** (`block`) | Stop the pipeline and block the request |
|
||||
| **Custom Response** (`modify_response`) | Return a custom message instead of the default block |
|
||||
|
||||
### Step options
|
||||
|
||||
| Field | Type | Description |
|
||||
|-------|------|--------------|
|
||||
| `guardrail` | `string` | Name of the guardrail to run |
|
||||
| `on_pass` | `string` | Action when guardrail passes: `next`, `allow`, `block`, `modify_response` |
|
||||
| `on_fail` | `string` | Action when guardrail fails: `next`, `allow`, `block`, `modify_response` |
|
||||
| `on_pass` | `string` | Action when outcome is **pass**: `next`, `allow`, `block`, `modify_response` |
|
||||
| `on_fail` | `string` | Action when outcome is **fail** (policy intervention): `next`, `allow`, `block`, `modify_response` |
|
||||
| `on_error` | `string` (optional) | Action when outcome is **error** (technical). If omitted, **error** uses `on_fail`. |
|
||||
| `pass_data` | `boolean` | Forward modified request data (e.g., PII-masked) to the next step |
|
||||
| `modify_response_message` | `string` | Custom message when using `modify_response` action |
|
||||
|
||||
|
|
@ -57,7 +71,7 @@ Each step defines what happens when the guardrail **passes** and when it **fails
|
|||
3. Select **Flow Builder** (instead of the simple form)
|
||||
4. Design your flow:
|
||||
- **Trigger** — Incoming LLM request (runs when the policy matches)
|
||||
- **Steps** — Add guardrails, set ON PASS and ON FAIL actions per step
|
||||
- **Steps** — Add guardrails, set **ON PASS**, **ON FAIL**, and **ON ERROR** actions per step (ON ERROR is optional; when unset, errors follow ON FAIL)
|
||||
- **End** — Request proceeds to the LLM
|
||||
5. Use the **+** between steps to insert new steps
|
||||
6. Use the **Test** panel to run sample messages through the pipeline before saving
|
||||
|
|
@ -151,6 +165,37 @@ policies:
|
|||
|
||||
First attempt passes → allow. First attempt fails → retry the same guardrail; second pass → allow, second fail → block.
|
||||
|
||||
## Technical errors vs policy failures (`on_error`)
|
||||
|
||||
Use **`on_error`** when you want different behavior for **API/infra problems** than for **content policy** violations.
|
||||
|
||||
- **`on_fail`** — Runs when the guardrail **intervenes** (e.g. toxic content, PII detected).
|
||||
- **`on_error`** — Runs when the step ends in **error** (timeout, connection failure, guardrail not loaded, etc.). If you omit `on_error`, **error** outcomes use **`on_fail`**.
|
||||
|
||||
Example: block on bad content, but if the primary scanner is down, fall back to a second guardrail instead of blocking every request:
|
||||
|
||||
```yaml
|
||||
policies:
|
||||
error-fallback-policy:
|
||||
guardrails:
|
||||
add:
|
||||
- primary_scanner
|
||||
- backup_scanner
|
||||
pipeline:
|
||||
mode: pre_call
|
||||
steps:
|
||||
- guardrail: primary_scanner
|
||||
on_pass: allow
|
||||
on_fail: block
|
||||
on_error: next
|
||||
- guardrail: backup_scanner
|
||||
on_pass: allow
|
||||
on_fail: block
|
||||
on_error: allow
|
||||
```
|
||||
|
||||
If `primary_scanner` errors → run `backup_scanner`. If `backup_scanner` errors → allow the request (set `on_error` to `block` if you prefer fail-closed).
|
||||
|
||||
## Example: Custom response on fail
|
||||
|
||||
Return a branded message instead of a generic block:
|
||||
|
|
|
|||
|
|
@ -316,86 +316,9 @@ general_settings:
|
|||
|
||||
## Health Check Driven Routing
|
||||
|
||||
By default, background health checks are observability-only — they populate the `/health` endpoint but don't affect routing. Unhealthy deployments still receive traffic until request failures trigger cooldown.
|
||||
Route traffic away from unhealthy deployments proactively — before user requests hit them. Supports per-error-type failure thresholds, transient error suppression, and automatic safety nets.
|
||||
|
||||
With `enable_health_check_routing: true`, the router **excludes deployments that failed their last background health check** before selecting a candidate. This gives you proactive failover instead of reactive cooldown.
|
||||
|
||||
### How it works
|
||||
|
||||
1. Background health checks run on their configured interval
|
||||
2. After each cycle, every deployment is marked healthy or unhealthy
|
||||
3. On each incoming request, the router filters out unhealthy deployments **before** cooldown filtering and load balancing
|
||||
4. If all deployments are unhealthy, the filter is bypassed (safety net — never causes a total outage)
|
||||
5. If health state is stale (older than `health_check_staleness_threshold`), it is ignored
|
||||
|
||||
### Quick start
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: gpt-4
|
||||
litellm_params:
|
||||
model: openai/gpt-4
|
||||
api_key: os.environ/OPENAI_API_KEY
|
||||
- model_name: gpt-4
|
||||
litellm_params:
|
||||
model: openai/gpt-4
|
||||
api_key: os.environ/OPENAI_API_KEY_SECONDARY
|
||||
|
||||
general_settings:
|
||||
background_health_checks: true
|
||||
health_check_interval: 60
|
||||
enable_health_check_routing: true
|
||||
```
|
||||
|
||||
### Configuration
|
||||
|
||||
| Setting | Where | Default | Description |
|
||||
|---------|-------|---------|-------------|
|
||||
| `enable_health_check_routing` | `general_settings` | `false` | Enable/disable health-check-driven routing |
|
||||
| `health_check_staleness_threshold` | `general_settings` | `health_check_interval * 2` | Seconds before health state is considered stale and ignored |
|
||||
| `background_health_checks` | `general_settings` | `false` | Must be `true` for health check routing to work |
|
||||
| `health_check_interval` | `general_settings` | `300` | Seconds between health check cycles |
|
||||
|
||||
### Interaction with cooldown
|
||||
|
||||
Health check filtering and cooldown are **additive**. A deployment can be excluded by either mechanism:
|
||||
|
||||
- **Health check filter** — proactive, runs on the configured interval, excludes deployments that failed the last check
|
||||
- **Cooldown** — reactive, triggered by request failures, excludes deployments for a short TTL
|
||||
|
||||
This means request failures still provide fast detection between health check intervals.
|
||||
|
||||
### Staleness
|
||||
|
||||
If a health check result is older than `health_check_staleness_threshold`, it is ignored and the deployment is treated as eligible. This prevents stale data from permanently excluding a deployment if the health check loop stops or slows down.
|
||||
|
||||
The default staleness threshold is `health_check_interval * 2`. For a 60s interval, health state expires after 120s.
|
||||
|
||||
### Example: custom staleness
|
||||
|
||||
```yaml
|
||||
general_settings:
|
||||
background_health_checks: true
|
||||
health_check_interval: 30
|
||||
enable_health_check_routing: true
|
||||
health_check_staleness_threshold: 90 # ignore health state older than 90s
|
||||
```
|
||||
|
||||
### Debugging
|
||||
|
||||
Run the proxy with `--detailed_debug` and look for:
|
||||
|
||||
```
|
||||
health_check_routing_state_updated healthy=3 unhealthy=1
|
||||
```
|
||||
|
||||
This is logged after each health check cycle when routing state is written.
|
||||
|
||||
If the safety net triggers (all deployments unhealthy), you'll see:
|
||||
|
||||
```
|
||||
All deployments marked unhealthy by health checks, bypassing health filter
|
||||
```
|
||||
See the full guide: [Health Check Driven Routing](./health_check_routing.md)
|
||||
|
||||
## Health Check Timeout
|
||||
|
||||
|
|
|
|||
340
docs/my-website/docs/proxy/health_check_routing.md
Normal file
340
docs/my-website/docs/proxy/health_check_routing.md
Normal file
|
|
@ -0,0 +1,340 @@
|
|||
# Health Check Driven Routing
|
||||
|
||||
Route traffic away from unhealthy deployments before users hit errors. Background health checks run on a configurable interval, and any deployment that fails gets removed from the routing pool proactively, not after a user request already failed.
|
||||
|
||||
|
||||
## Architecture
|
||||
|
||||
<svg viewBox="0 0 860 600" xmlns="http://www.w3.org/2000/svg" style={{maxWidth: '100%', fontFamily: 'system-ui, sans-serif'}}>
|
||||
{/* Background */}
|
||||
<rect width="860" height="600" fill="#f8fafc" rx="12"/>
|
||||
|
||||
{/* LEFT PANEL: Background health check loop */}
|
||||
<rect x="20" y="20" width="240" height="560" fill="#eff6ff" rx="10" stroke="#bfdbfe" strokeWidth="1.5"/>
|
||||
<text x="140" y="48" textAnchor="middle" fill="#1d4ed8" fontSize="13" fontWeight="600">Background Loop</text>
|
||||
<text x="140" y="64" textAnchor="middle" fill="#3b82f6" fontSize="11">every health_check_interval seconds</text>
|
||||
|
||||
{/* Deployment A */}
|
||||
<rect x="40" y="82" width="200" height="50" fill="white" rx="8" stroke="#93c5fd" strokeWidth="1.5"/>
|
||||
<text x="140" y="102" textAnchor="middle" fill="#1e40af" fontSize="12" fontWeight="500">Deployment A</text>
|
||||
<text x="140" y="120" textAnchor="middle" fill="#64748b" fontSize="11">ahealth_check() → 200 ✓</text>
|
||||
|
||||
{/* Deployment B */}
|
||||
<rect x="40" y="148" width="200" height="50" fill="white" rx="8" stroke="#fca5a5" strokeWidth="1.5"/>
|
||||
<text x="140" y="168" textAnchor="middle" fill="#991b1b" fontSize="12" fontWeight="500">Deployment B</text>
|
||||
<text x="140" y="186" textAnchor="middle" fill="#64748b" fontSize="11">ahealth_check() → 401 ✗</text>
|
||||
|
||||
{/* Deployment C */}
|
||||
<rect x="40" y="214" width="200" height="50" fill="white" rx="8" stroke="#fde68a" strokeWidth="1.5"/>
|
||||
<text x="140" y="234" textAnchor="middle" fill="#92400e" fontSize="12" fontWeight="500">Deployment C</text>
|
||||
<text x="140" y="252" textAnchor="middle" fill="#64748b" fontSize="11">ahealth_check() → 429 ⚡</text>
|
||||
|
||||
{/* ignore_transient box */}
|
||||
<rect x="40" y="282" width="200" height="68" fill="#fefce8" rx="8" stroke="#fde047" strokeWidth="1.5"/>
|
||||
<text x="140" y="302" textAnchor="middle" fill="#713f12" fontSize="11" fontWeight="600">ignore_transient_errors: true</text>
|
||||
<text x="140" y="320" textAnchor="middle" fill="#92400e" fontSize="11">429 / 408 → ignored</text>
|
||||
<text x="140" y="338" textAnchor="middle" fill="#92400e" fontSize="11">not written to cache</text>
|
||||
|
||||
{/* allowed_fails_policy box */}
|
||||
<rect x="40" y="368" width="200" height="84" fill="#f0fdf4" rx="8" stroke="#86efac" strokeWidth="1.5"/>
|
||||
<text x="140" y="388" textAnchor="middle" fill="#166534" fontSize="11" fontWeight="600">allowed_fails_policy</text>
|
||||
<text x="140" y="406" textAnchor="middle" fill="#15803d" fontSize="11">401 → increment counter</text>
|
||||
<text x="140" y="424" textAnchor="middle" fill="#15803d" fontSize="11">counter > threshold</text>
|
||||
<text x="140" y="442" textAnchor="middle" fill="#15803d" fontSize="11">→ cooldown triggered</text>
|
||||
|
||||
{/* CENTER PANEL: Shared State */}
|
||||
<rect x="300" y="20" width="220" height="560" fill="#f5f3ff" rx="10" stroke="#c4b5fd" strokeWidth="1.5"/>
|
||||
<text x="410" y="48" textAnchor="middle" fill="#6d28d9" fontSize="13" fontWeight="600">Shared State</text>
|
||||
|
||||
{/* Health State Cache */}
|
||||
<rect x="320" y="62" width="180" height="116" fill="white" rx="8" stroke="#a78bfa" strokeWidth="1.5"/>
|
||||
<text x="410" y="84" textAnchor="middle" fill="#5b21b6" fontSize="12" fontWeight="600">DeploymentHealthCache</text>
|
||||
<text x="410" y="104" textAnchor="middle" fill="#64748b" fontSize="11">A → healthy ✓</text>
|
||||
<text x="410" y="122" textAnchor="middle" fill="#64748b" fontSize="11">B → unhealthy ✗</text>
|
||||
<text x="410" y="140" textAnchor="middle" fill="#64748b" fontSize="11">C → not written (ignored)</text>
|
||||
<text x="410" y="164" textAnchor="middle" fill="#94a3b8" fontSize="10">TTL: staleness_threshold × 1.5</text>
|
||||
|
||||
{/* Cooldown Cache */}
|
||||
<rect x="320" y="196" width="180" height="104" fill="white" rx="8" stroke="#a78bfa" strokeWidth="1.5"/>
|
||||
<text x="410" y="218" textAnchor="middle" fill="#5b21b6" fontSize="12" fontWeight="600">Cooldown Cache</text>
|
||||
<text x="410" y="238" textAnchor="middle" fill="#64748b" fontSize="11">B → cooling down</text>
|
||||
<text x="410" y="256" textAnchor="middle" fill="#64748b" fontSize="11">(after policy threshold)</text>
|
||||
<text x="410" y="278" textAnchor="middle" fill="#94a3b8" fontSize="10">TTL: cooldown_time</text>
|
||||
|
||||
{/* failed_calls counter */}
|
||||
<rect x="320" y="318" width="180" height="90" fill="white" rx="8" stroke="#a78bfa" strokeWidth="1.5"/>
|
||||
<text x="410" y="340" textAnchor="middle" fill="#5b21b6" fontSize="12" fontWeight="600">failed_calls counter</text>
|
||||
<text x="410" y="360" textAnchor="middle" fill="#64748b" fontSize="11">B: 2 / AuthAllowedFails: 1</text>
|
||||
<text x="410" y="378" textAnchor="middle" fill="#64748b" fontSize="11">→ threshold exceeded</text>
|
||||
<text x="410" y="398" textAnchor="middle" fill="#94a3b8" fontSize="10">TTL: cooldown_time (must > interval)</text>
|
||||
|
||||
{/* RIGHT PANEL: Request path */}
|
||||
<rect x="560" y="20" width="280" height="560" fill="#fff7ed" rx="10" stroke="#fed7aa" strokeWidth="1.5"/>
|
||||
<text x="700" y="48" textAnchor="middle" fill="#c2410c" fontSize="13" fontWeight="600">Request Path</text>
|
||||
|
||||
{/* Incoming request */}
|
||||
<rect x="580" y="62" width="240" height="38" fill="#fff" rx="7" stroke="#fb923c" strokeWidth="1.5"/>
|
||||
<text x="700" y="85" textAnchor="middle" fill="#9a3412" fontSize="12" fontWeight="500">Incoming request</text>
|
||||
|
||||
{/* All deployments */}
|
||||
<rect x="580" y="120" width="240" height="38" fill="#fff" rx="7" stroke="#fb923c" strokeWidth="1.5"/>
|
||||
<text x="700" y="143" textAnchor="middle" fill="#9a3412" fontSize="12">All deployments [A, B, C]</text>
|
||||
|
||||
<line x1="700" y1="100" x2="700" y2="120" stroke="#fb923c" strokeWidth="1.5" markerEnd="url(#arrow-orange)"/>
|
||||
|
||||
{/* Health check filter */}
|
||||
<rect x="580" y="178" width="240" height="62" fill="#fff" rx="7" stroke="#fb923c" strokeWidth="1.5"/>
|
||||
<text x="700" y="200" textAnchor="middle" fill="#9a3412" fontSize="12" fontWeight="600">① Health Check Filter</text>
|
||||
<text x="700" y="218" textAnchor="middle" fill="#64748b" fontSize="11">if policy set → bypass</text>
|
||||
<text x="700" y="234" textAnchor="middle" fill="#64748b" fontSize="11">else → remove unhealthy</text>
|
||||
|
||||
<line x1="700" y1="158" x2="700" y2="178" stroke="#fb923c" strokeWidth="1.5" markerEnd="url(#arrow-orange)"/>
|
||||
|
||||
{/* Cooldown filter */}
|
||||
<rect x="580" y="262" width="240" height="50" fill="#fff" rx="7" stroke="#fb923c" strokeWidth="1.5"/>
|
||||
<text x="700" y="284" textAnchor="middle" fill="#9a3412" fontSize="12" fontWeight="600">② Cooldown Filter</text>
|
||||
<text x="700" y="302" textAnchor="middle" fill="#64748b" fontSize="11">remove deployments in cooldown</text>
|
||||
|
||||
<line x1="700" y1="240" x2="700" y2="262" stroke="#fb923c" strokeWidth="1.5" markerEnd="url(#arrow-orange)"/>
|
||||
|
||||
{/* Safety net */}
|
||||
<rect x="580" y="334" width="240" height="52" fill="#fef9c3" rx="7" stroke="#fbbf24" strokeWidth="1.5"/>
|
||||
<text x="700" y="356" textAnchor="middle" fill="#713f12" fontSize="12" fontWeight="600">Safety Net</text>
|
||||
<text x="700" y="376" textAnchor="middle" fill="#713f12" fontSize="11">if all removed → return all</text>
|
||||
|
||||
<line x1="700" y1="312" x2="700" y2="334" stroke="#fb923c" strokeWidth="1.5" markerEnd="url(#arrow-orange)"/>
|
||||
|
||||
{/* Load balancer */}
|
||||
<rect x="580" y="408" width="240" height="38" fill="#fff" rx="7" stroke="#fb923c" strokeWidth="1.5"/>
|
||||
<text x="700" y="431" textAnchor="middle" fill="#9a3412" fontSize="12" fontWeight="600">③ Load Balancer</text>
|
||||
|
||||
<line x1="700" y1="386" x2="700" y2="408" stroke="#fb923c" strokeWidth="1.5" markerEnd="url(#arrow-orange)"/>
|
||||
|
||||
{/* Selected deployment */}
|
||||
<rect x="580" y="468" width="240" height="38" fill="#dcfce7" rx="7" stroke="#4ade80" strokeWidth="1.5"/>
|
||||
<text x="700" y="491" textAnchor="middle" fill="#14532d" fontSize="12" fontWeight="600">Selected: Deployment A ✓</text>
|
||||
|
||||
<line x1="700" y1="446" x2="700" y2="468" stroke="#4ade80" strokeWidth="1.5" markerEnd="url(#arrow-green)"/>
|
||||
|
||||
{/* ARROWS: left → center */}
|
||||
<line x1="240" y1="107" x2="320" y2="110" stroke="#3b82f6" strokeWidth="1.5" strokeDasharray="4,3" markerEnd="url(#arrow-blue)"/>
|
||||
<line x1="240" y1="173" x2="320" y2="240" stroke="#ef4444" strokeWidth="1.5" strokeDasharray="4,3" markerEnd="url(#arrow-red)"/>
|
||||
<line x1="240" y1="173" x2="320" y2="348" stroke="#ef4444" strokeWidth="1.5" strokeDasharray="4,3" markerEnd="url(#arrow-red)"/>
|
||||
<line x1="240" y1="316" x2="320" y2="130" stroke="#eab308" strokeWidth="1.5" strokeDasharray="4,3" markerEnd="url(#arrow-yellow)"/>
|
||||
|
||||
{/* ARROWS: center → right */}
|
||||
<line x1="500" y1="120" x2="580" y2="190" stroke="#8b5cf6" strokeWidth="1.5" strokeDasharray="4,3" markerEnd="url(#arrow-purple)"/>
|
||||
<line x1="500" y1="248" x2="580" y2="274" stroke="#8b5cf6" strokeWidth="1.5" strokeDasharray="4,3" markerEnd="url(#arrow-purple)"/>
|
||||
|
||||
{/* Arrow markers */}
|
||||
<defs>
|
||||
<marker id="arrow-orange" markerWidth="8" markerHeight="8" refX="6" refY="3" orient="auto">
|
||||
<path d="M0,0 L0,6 L8,3 z" fill="#fb923c"/>
|
||||
</marker>
|
||||
<marker id="arrow-blue" markerWidth="8" markerHeight="8" refX="6" refY="3" orient="auto">
|
||||
<path d="M0,0 L0,6 L8,3 z" fill="#3b82f6"/>
|
||||
</marker>
|
||||
<marker id="arrow-red" markerWidth="8" markerHeight="8" refX="6" refY="3" orient="auto">
|
||||
<path d="M0,0 L0,6 L8,3 z" fill="#ef4444"/>
|
||||
</marker>
|
||||
<marker id="arrow-yellow" markerWidth="8" markerHeight="8" refX="6" refY="3" orient="auto">
|
||||
<path d="M0,0 L0,6 L8,3 z" fill="#eab308"/>
|
||||
</marker>
|
||||
<marker id="arrow-purple" markerWidth="8" markerHeight="8" refX="6" refY="3" orient="auto">
|
||||
<path d="M0,0 L0,6 L8,3 z" fill="#8b5cf6"/>
|
||||
</marker>
|
||||
<marker id="arrow-green" markerWidth="8" markerHeight="8" refX="6" refY="3" orient="auto">
|
||||
<path d="M0,0 L0,6 L8,3 z" fill="#4ade80"/>
|
||||
</marker>
|
||||
</defs>
|
||||
</svg>
|
||||
|
||||
|
||||
## What problem does this solve?
|
||||
|
||||
By default, LiteLLM routes traffic to all deployments and only stops sending to a broken one after it has already failed a user request. The cooldown system is reactive.
|
||||
|
||||
Health check driven routing makes this **proactive**: a background loop pings every deployment on a configurable interval. If a deployment fails its health check, it gets removed from the routing pool immediately, before a user request lands on it.
|
||||
|
||||
When you also set `allowed_fails_policy`, you control exactly how many health check failures of each error type (auth errors, rate limits, timeouts) are needed before a deployment enters cooldown. This avoids false positives from transient noise.
|
||||
|
||||
|
||||
## Setup
|
||||
|
||||
### Step 1: Enable background health checks
|
||||
|
||||
Background health checks are off by default. Turn them on in `general_settings`:
|
||||
|
||||
```yaml
|
||||
general_settings:
|
||||
background_health_checks: true
|
||||
health_check_interval: 60 # seconds between each full check cycle
|
||||
```
|
||||
|
||||
### Step 2: Enable health check routing
|
||||
|
||||
```yaml
|
||||
general_settings:
|
||||
background_health_checks: true
|
||||
health_check_interval: 60
|
||||
enable_health_check_routing: true # ← route away from unhealthy deployments
|
||||
```
|
||||
|
||||
At this point, any deployment that fails its health check is immediately excluded from routing until the next check cycle clears it.
|
||||
|
||||
### Step 3: Add a policy to control how many failures trigger cooldown
|
||||
|
||||
Without a policy, the first health check failure marks a deployment as unhealthy. If you want more tolerance (e.g., only act after 2 consecutive auth failures), use `allowed_fails_policy`:
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: claude-sonnet
|
||||
litellm_params:
|
||||
model: anthropic/claude-sonnet-4-5
|
||||
api_key: os.environ/ANTHROPIC_API_KEY
|
||||
|
||||
- model_name: claude-sonnet
|
||||
litellm_params:
|
||||
model: anthropic/claude-sonnet-4-5
|
||||
api_key: os.environ/ANTHROPIC_API_KEY_SECONDARY
|
||||
|
||||
general_settings:
|
||||
background_health_checks: true
|
||||
health_check_interval: 30
|
||||
enable_health_check_routing: true
|
||||
|
||||
router_settings:
|
||||
cooldown_time: 60 # how long a deployment stays in cooldown
|
||||
allowed_fails_policy:
|
||||
AuthenticationErrorAllowedFails: 1 # cooldown after 2nd auth failure
|
||||
TimeoutErrorAllowedFails: 3 # cooldown after 4th timeout
|
||||
```
|
||||
|
||||
When `allowed_fails_policy` is set, the binary health check filter is bypassed. Only the cooldown system controls routing exclusion, and it only fires after your configured threshold is crossed.
|
||||
|
||||
### Step 4 (optional): Ignore transient errors
|
||||
|
||||
429 (rate limit) and 408 (timeout) from a health check usually mean the deployment is temporarily overloaded, not broken. To prevent these from affecting routing at all:
|
||||
|
||||
```yaml
|
||||
general_settings:
|
||||
background_health_checks: true
|
||||
health_check_interval: 30
|
||||
enable_health_check_routing: true
|
||||
health_check_ignore_transient_errors: true # 429 and 408 never affect routing
|
||||
```
|
||||
|
||||
With this on, only hard failures (401, 404, 5xx) from health checks contribute to cooldown.
|
||||
|
||||
|
||||
## Full example
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: gpt-4o
|
||||
litellm_params:
|
||||
model: openai/gpt-4o
|
||||
api_key: os.environ/OPENAI_API_KEY
|
||||
|
||||
- model_name: gpt-4o
|
||||
litellm_params:
|
||||
model: openai/gpt-4o
|
||||
api_key: os.environ/OPENAI_API_KEY_SECONDARY
|
||||
|
||||
- model_name: gpt-4o
|
||||
litellm_params:
|
||||
model: azure/gpt-4o
|
||||
api_base: os.environ/AZURE_API_BASE
|
||||
api_key: os.environ/AZURE_API_KEY
|
||||
|
||||
general_settings:
|
||||
background_health_checks: true
|
||||
health_check_interval: 30
|
||||
enable_health_check_routing: true
|
||||
health_check_ignore_transient_errors: true
|
||||
|
||||
router_settings:
|
||||
cooldown_time: 60
|
||||
allowed_fails_policy:
|
||||
AuthenticationErrorAllowedFails: 0 # cooldown immediately on auth failure
|
||||
TimeoutErrorAllowedFails: 2 # cooldown after 3 timeouts
|
||||
RateLimitErrorAllowedFails: 5 # cooldown after 6 rate limits (if not ignoring transients)
|
||||
```
|
||||
|
||||
|
||||
## Configuration reference
|
||||
|
||||
| Setting | Where | Default | Description |
|
||||
|---|---|---|---|
|
||||
| `enable_health_check_routing` | `general_settings` | `false` | Route away from deployments that fail health checks |
|
||||
| `background_health_checks` | `general_settings` | `false` | Must be `true` for health check routing to work |
|
||||
| `health_check_interval` | `general_settings` | `300` | Seconds between full health check cycles |
|
||||
| `health_check_staleness_threshold` | `general_settings` | `interval x 2` | Seconds before cached health state is ignored |
|
||||
| `health_check_ignore_transient_errors` | `general_settings` | `false` | Ignore 429 and 408 from health checks; these never affect routing |
|
||||
| `cooldown_time` | `router_settings` | `5` | Seconds a deployment stays in cooldown after threshold is crossed |
|
||||
| `allowed_fails_policy` | `router_settings` | `null` | Per-error-type failure thresholds before cooldown (see below) |
|
||||
|
||||
### `allowed_fails_policy` fields
|
||||
|
||||
| Field | Error type | HTTP status |
|
||||
|---|---|---|
|
||||
| `AuthenticationErrorAllowedFails` | Bad API key | 401 |
|
||||
| `TimeoutErrorAllowedFails` | Request timeout | 408 |
|
||||
| `RateLimitErrorAllowedFails` | Rate limit exceeded | 429 |
|
||||
| `BadRequestErrorAllowedFails` | Malformed request | 400 |
|
||||
| `ContentPolicyViolationErrorAllowedFails` | Content filtered | 400 |
|
||||
|
||||
The value is the number of failures **tolerated** before cooldown. `0` means cooldown on the first failure. `2` means cooldown on the third.
|
||||
|
||||
|
||||
## Things to keep in mind
|
||||
|
||||
- **Counter TTL must be longer than the health check interval.** `allowed_fails_policy` works by incrementing a `failed_calls` counter per deployment. That counter expires after `cooldown_time` seconds. If `cooldown_time` is shorter than `health_check_interval`, the counter resets between every check cycle and failures never accumulate. Set `cooldown_time` greater than `health_check_interval` when using `allowed_fails_policy`.
|
||||
|
||||
```yaml
|
||||
router_settings:
|
||||
cooldown_time: 60 # must be > health_check_interval (30s here)
|
||||
|
||||
general_settings:
|
||||
health_check_interval: 30
|
||||
```
|
||||
|
||||
- **`AllowedFails: N` means cooldown on the (N+1)th failure.** The counter check is `updated_fails > allowed_fails`, so `0` triggers on the 1st failure, `1` on the 2nd, `2` on the 3rd.
|
||||
|
||||
| `AllowedFails` | Cooldown triggers after |
|
||||
|---|---|
|
||||
| `0` | 1st failure |
|
||||
| `1` | 2nd failure |
|
||||
| `2` | 3rd failure |
|
||||
|
||||
- **Without `allowed_fails_policy`, the first failure is enough.** The first failed health check immediately excludes the deployment from routing. Use `allowed_fails_policy` when you want tolerance for flaky checks.
|
||||
|
||||
- **If all deployments are unhealthy, the filter is bypassed.** Traffic keeps flowing rather than returning no deployment at all. Requests will fail, but the router keeps trying.
|
||||
|
||||
- **Health check failures and request failures share the same counters.** When `allowed_fails_policy` is set, both sources increment the same `failed_calls` counter. A deployment at 1 health check failure that then receives 1 failing request will hit the threshold for `AllowedFails: 1` and enter cooldown.
|
||||
|
||||
|
||||
## Debugging
|
||||
|
||||
Run the proxy with `--detailed_debug` and look for these log lines:
|
||||
|
||||
After each health check cycle (written at DEBUG level):
|
||||
```
|
||||
health_check_routing_state_updated healthy=2 unhealthy=1
|
||||
```
|
||||
|
||||
When a health check failure increments the counter and triggers cooldown (DEBUG level):
|
||||
```
|
||||
checks 'should_run_cooldown_logic'
|
||||
Attempting to add <deployment_id> to cooldown list
|
||||
```
|
||||
|
||||
When safety net fires because all deployments are in cooldown:
|
||||
```
|
||||
All deployments in cooldown via health-check routing, bypassing cooldown filter
|
||||
```
|
||||
|
||||
When safety net fires because all deployments are unhealthy (binary filter, no `allowed_fails_policy`):
|
||||
```
|
||||
All deployments marked unhealthy by health checks, bypassing health filter
|
||||
```
|
||||
|
|
@ -358,10 +358,15 @@ When you connect litellm to your SSO provider, litellm can auto-create teams. Us
|
|||
|
||||
```yaml showLineNumbers title="Default Params for new teams"
|
||||
litellm_settings:
|
||||
default_team_params: # Default Params to apply when litellm auto creates a team from SSO IDP provider
|
||||
max_budget: 100 # Optional[float], optional): $100 budget for the team
|
||||
budget_duration: 30d # Optional[str], optional): 30 days budget_duration for the team
|
||||
models: ["gpt-3.5-turbo"] # Optional[List[str]], optional): models to be used by the team
|
||||
default_team_params: # Applied to all /team/new calls (including SSO auto-created teams) when the field is not explicitly set
|
||||
max_budget: 100 # Optional[float]: $100 budget for the team
|
||||
budget_duration: 30d # Optional[str]: 30 days budget_duration for the team
|
||||
models: ["gpt-3.5-turbo"] # Optional[List[str]]: models for the team (only applied to SSO auto-created teams)
|
||||
tpm_limit: 100000 # Optional[int]: tokens per minute limit
|
||||
rpm_limit: 1000 # Optional[int]: requests per minute limit
|
||||
team_member_permissions: # Optional[List[str]]: permissions granted to non-admin team members
|
||||
- "/team/daily/activity" # Allow members to view team usage
|
||||
- "/key/generate" # Allow members to generate API keys
|
||||
```
|
||||
|
||||
|
||||
|
|
@ -390,10 +395,14 @@ litellm_settings:
|
|||
max_budget_in_team: 100 # Optional[float], optional): $100 budget for the team. Defaults to None.
|
||||
user_role: "user" # Optional[str], optional): "user" or "admin". Defaults to "user"
|
||||
|
||||
default_team_params: # Default Params to apply when litellm auto creates a team from SSO IDP provider
|
||||
max_budget: 100 # Optional[float], optional): $100 budget for the team
|
||||
budget_duration: 30d # Optional[str], optional): 30 days budget_duration for the team
|
||||
models: ["gpt-3.5-turbo"] # Optional[List[str]], optional): models to be used by the team
|
||||
default_team_params: # Applied to all /team/new calls (including SSO auto-created teams) when the field is not explicitly set
|
||||
max_budget: 100 # Optional[float]: $100 budget for the team
|
||||
budget_duration: 30d # Optional[str]: 30 days budget_duration for the team
|
||||
models: ["gpt-3.5-turbo"] # Optional[List[str]]: models for the team (only applied to SSO auto-created teams)
|
||||
tpm_limit: 100000 # Optional[int]: tokens per minute limit
|
||||
rpm_limit: 1000 # Optional[int]: requests per minute limit
|
||||
team_member_permissions: # Optional[List[str]]: permissions granted to non-admin team members
|
||||
- "/team/daily/activity"
|
||||
|
||||
|
||||
upperbound_key_generate_params: # Upperbound for /key/generate requests when self-serve flow is on
|
||||
|
|
|
|||
|
|
@ -123,10 +123,12 @@ Navigate to your litellm config file and set the following params
|
|||
|
||||
```yaml showLineNumbers title="litellm config with default_team_params"
|
||||
litellm_settings:
|
||||
default_team_params: # Default Params to apply when litellm auto creates a team from SSO IDP provider
|
||||
max_budget: 100 # Optional[float], optional): $100 budget for the team
|
||||
budget_duration: 30d # Optional[str], optional): 30 days budget_duration for the team
|
||||
models: ["gpt-3.5-turbo"] # Optional[List[str]], optional): models to be used by the team
|
||||
default_team_params: # Applied to all /team/new calls (including SSO auto-created teams) when the field is not explicitly set
|
||||
max_budget: 100 # Optional[float]: $100 budget for the team
|
||||
budget_duration: 30d # Optional[str]: 30 days budget_duration for the team
|
||||
models: ["gpt-3.5-turbo"] # Optional[List[str]]: models for the team (only applied to SSO auto-created teams)
|
||||
team_member_permissions: # Optional[List[str]]: permissions granted to non-admin team members
|
||||
- "/team/daily/activity" # Allow members to view team usage
|
||||
```
|
||||
|
||||
### 3.2 Auto-create a new team on LiteLLM
|
||||
|
|
|
|||
BIN
docs/my-website/img/release_notes/mcp_toolsets.jpeg
Normal file
BIN
docs/my-website/img/release_notes/mcp_toolsets.jpeg
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 111 KiB |
BIN
docs/my-website/img/release_notes/skills_marketplace.png
Normal file
BIN
docs/my-website/img/release_notes/skills_marketplace.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 1 MiB |
236
docs/my-website/release_notes/v1.83.3/index.md
Normal file
236
docs/my-website/release_notes/v1.83.3/index.md
Normal file
|
|
@ -0,0 +1,236 @@
|
|||
---
|
||||
title: "[Preview] v1.83.3.rc.1 - Introducing MCP Skills Marketplace"
|
||||
slug: "v1-83-3-rc-1"
|
||||
date: 2026-04-04T00:00:00
|
||||
authors:
|
||||
- name: Krrish Dholakia
|
||||
title: CEO, LiteLLM
|
||||
url: https://www.linkedin.com/in/krish-d/
|
||||
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
|
||||
- name: Ishaan Jaff
|
||||
title: CTO, LiteLLM
|
||||
url: https://www.linkedin.com/in/reffajnaahsi/
|
||||
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
|
||||
- name: Ryan Crabbe
|
||||
title: Full Stack Engineer, LiteLLM
|
||||
url: https://www.linkedin.com/in/ryan-crabbe-0b9687214
|
||||
image_url: https://media.licdn.com/dms/image/v2/D5603AQHt1t9Z4BJ6Gw/profile-displayphoto-shrink_400_400/profile-displayphoto-shrink_400_400/0/1724453682340?e=1772064000&v=beta&t=VXdmr13rsNB05wyA2F1TENOB5UuDHUZ0FCHTolNyR5M
|
||||
- name: Yuneng Jiang
|
||||
title: Senior Full Stack Engineer, LiteLLM
|
||||
url: https://www.linkedin.com/in/yuneng-david-jiang-455676139/
|
||||
image_url: https://avatars.githubusercontent.com/u/171294688?v=4
|
||||
- name: Shivam Rawat
|
||||
title: Forward Deployed Engineer, LiteLLM
|
||||
url: https://linkedin.com/in/shivam-rawat-482937318
|
||||
image_url: https://github.com/shivamrawat1.png
|
||||
hide_table_of_contents: false
|
||||
---
|
||||
|
||||
## Deploy this version
|
||||
|
||||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="docker" label="Docker">
|
||||
|
||||
```bash
|
||||
docker run \
|
||||
-e STORE_MODEL_IN_DB=True \
|
||||
-p 4000:4000 \
|
||||
docker.litellm.ai/berriai/litellm:main-v1.83.3.rc.1
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="pip" label="Pip">
|
||||
|
||||
```bash
|
||||
pip install litellm==1.83.3rc1
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
## Key Highlights
|
||||
|
||||
- **MCP Toolsets** — [Create curated tool subsets from one or more MCP servers with scoped permissions, and manage them from the UI or API](../../docs/mcp)
|
||||
- **Skills Marketplace** — [Browse, install, and publish Claude Code skills from a self-hosted marketplace — works across Anthropic, Vertex AI, Azure, and Bedrock](../../docs/proxy/skills)
|
||||
- **Guardrail Fallbacks** — [Configure `on_error` behavior so guardrail failures degrade gracefully instead of blocking the request](../../docs/proxy/guardrails)
|
||||
- **Team Bring Your Own Guardrails** — [Teams can now attach and manage their own guardrails directly from team settings in the UI](../../docs/proxy/guardrails)
|
||||
|
||||
---
|
||||
|
||||
|
||||
### Skills Marketplace
|
||||
|
||||
The Skills Marketplace gives teams a self-hosted catalog for discovering, installing, and publishing Claude Code skills. Skills are portable across Anthropic, Vertex AI, Azure, and Bedrock — so a skill published once works everywhere your gateway routes to.
|
||||
|
||||
|
||||
|
||||
[Get Started](../../docs/proxy/skills)
|
||||
|
||||
### Guardrail Fallbacks
|
||||
|
||||
Guardrail pipelines now support an optional `on_error` behavior. When a guardrail check fails or errors out, you can configure the pipeline to fall back gracefully — logging the failure and continuing the request — instead of returning a hard 500 to the caller. This is especially useful for non-critical guardrails where availability matters more than enforcement.
|
||||
|
||||
### Team Bring Your Own Guardrails
|
||||
|
||||
Teams can now attach guardrails directly from the team management UI. Admins configure available guardrails at the project or proxy level, and individual teams select which ones apply to their traffic — no config file changes or proxy restarts needed. This also ships with project-level guardrail support in the project create/edit flows.
|
||||
|
||||
### MCP Toolsets
|
||||
|
||||
MCP Toolsets let AI platform admins create curated subsets of tools from one or more MCP servers and assign them to teams and keys with scoped permissions. Instead of granting access to an entire MCP server, you can now bundle specific tools into a named toolset — controlling exactly which tools each team or API key can invoke. Toolsets are fully managed through the UI (new Toolsets tab) and API, and work seamlessly with the Responses API and Playground.
|
||||
|
||||
|
||||
|
||||
[Get Started](../../docs/mcp)
|
||||
---
|
||||
|
||||
## New Models / Updated Models
|
||||
|
||||
#### New Model Support
|
||||
|
||||
| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features |
|
||||
| -------- | ----- | -------------- | ------------------- | -------------------- | -------- |
|
||||
| Brave Search | `brave/search` | - | - | - | Search tool integration metadata in cost map ([PR #25042](https://github.com/BerriAI/litellm/pull/25042)) |
|
||||
| AWS Bedrock | `nvidia.nemotron-super-3-120b` | 256K | Added | Added | Chat completions, function calling, system messages ([PR #24588](https://github.com/BerriAI/litellm/pull/24588)) |
|
||||
| OCI GenAI | Multiple new chat + embedding entries | Varies | Updated | Updated | Expanded chat + embedding model catalog |
|
||||
|
||||
#### Features
|
||||
|
||||
- **[AWS Bedrock](../../docs/providers/bedrock)**
|
||||
- Add Nova Canvas image edit support - [PR #25110](https://github.com/BerriAI/litellm/pull/25110), [PR #24869](https://github.com/BerriAI/litellm/pull/24869)
|
||||
- Improve cache usage exposure for Claude-compatible streaming paths - [PR #25110](https://github.com/BerriAI/litellm/pull/25110), [PR #24850](https://github.com/BerriAI/litellm/pull/24850)
|
||||
- Bedrock model catalog updates - [PR #24645](https://github.com/BerriAI/litellm/pull/24645)
|
||||
|
||||
- **[OCI GenAI](../../docs/providers/oci)**
|
||||
- Add native embeddings support + expanded model catalog - [PR #25151](https://github.com/BerriAI/litellm/pull/25151), [PR #24887](https://github.com/BerriAI/litellm/pull/24887)
|
||||
|
||||
- **[Google Vertex AI](../../docs/providers/vertex)**
|
||||
- Add unversioned Claude Haiku pricing entry to ensure accurate spend accounting - [PR #25151](https://github.com/BerriAI/litellm/pull/25151)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- **General**
|
||||
- Fix `gpt-5.4` pricing metadata - [PR #24748](https://github.com/BerriAI/litellm/pull/24748)
|
||||
- Fix gov pricing tests and Bedrock model test follow-ups - [PR #25022](https://github.com/BerriAI/litellm/pull/25022), [PR #24947](https://github.com/BerriAI/litellm/pull/24947), [PR #24931](https://github.com/BerriAI/litellm/pull/24931)
|
||||
|
||||
## LLM API Endpoints
|
||||
|
||||
#### Features
|
||||
|
||||
- **[A2A / MCP Gateway API (/a2a, /mcp)](../../docs/mcp)**
|
||||
- Preserve JSON-RPC envelope for AgentCore A2A-native agents - [PR #25092](https://github.com/BerriAI/litellm/pull/25092)
|
||||
- Bedrock Anthropic file/document handling fix from internal staging - [PR #25050](https://github.com/BerriAI/litellm/pull/25050), [PR #25047](https://github.com/BerriAI/litellm/pull/25047)
|
||||
|
||||
#### Bugs
|
||||
|
||||
- **[Search API (/search)](../../docs/search)**
|
||||
- Support self-hosted Firecrawl response format in search transforms - [PR #25110](https://github.com/BerriAI/litellm/pull/25110), [PR #24866](https://github.com/BerriAI/litellm/pull/24866)
|
||||
|
||||
## Management Endpoints / UI
|
||||
|
||||
#### Features
|
||||
|
||||
- **Virtual Keys**
|
||||
- Add substring search for `user_id` and `key_alias` on `/key/list` - [PR #24751](https://github.com/BerriAI/litellm/pull/24751), [PR #24746](https://github.com/BerriAI/litellm/pull/24746)
|
||||
- Wire `team_id` filter to key alias dropdown on Virtual Keys tab - [PR #25119](https://github.com/BerriAI/litellm/pull/25119), [PR #25114](https://github.com/BerriAI/litellm/pull/25114)
|
||||
- Allow hashed `token_id` in `/key/update` endpoint - [PR #24969](https://github.com/BerriAI/litellm/pull/24969)
|
||||
|
||||
- **Teams + Organizations**
|
||||
- Resolve access-group models/MCP servers/agents in team endpoints and UI - [PR #25119](https://github.com/BerriAI/litellm/pull/25119), [PR #25027](https://github.com/BerriAI/litellm/pull/25027)
|
||||
- Allow changing team organization from team settings - [PR #25095](https://github.com/BerriAI/litellm/pull/25095)
|
||||
- Add per-model rate limits to team edit/info views - [PR #25156](https://github.com/BerriAI/litellm/pull/25156), [PR #25144](https://github.com/BerriAI/litellm/pull/25144)
|
||||
|
||||
- **Usage + Analytics**
|
||||
- Add paginated team search on usage page filters - [PR #25107](https://github.com/BerriAI/litellm/pull/25107)
|
||||
- Use entity key for usage export display correctness - [PR #25153](https://github.com/BerriAI/litellm/pull/25153)
|
||||
|
||||
- **Models + Providers**
|
||||
- Include access-group models in UI model listing - [PR #24743](https://github.com/BerriAI/litellm/pull/24743)
|
||||
- Expose Azure Entra ID credential fields in provider forms - [PR #25137](https://github.com/BerriAI/litellm/pull/25137)
|
||||
- Do not inject `vector_store_ids: []` when editing a model - [PR #25133](https://github.com/BerriAI/litellm/pull/25133)
|
||||
|
||||
- **Guardrails UI**
|
||||
- Add project-level guardrails support in project create/edit flows - [PR #25100](https://github.com/BerriAI/litellm/pull/25100)
|
||||
- Allow adding team guardrails from the UI - [PR #25038](https://github.com/BerriAI/litellm/pull/25038)
|
||||
|
||||
- **UI Cleanup**
|
||||
- Migrate Tremor Text/Badge to antd Tag and native spans - [PR #24750](https://github.com/BerriAI/litellm/pull/24750)
|
||||
|
||||
#### Bugs
|
||||
|
||||
- Fix logs page showing unfiltered results when backend filter returns zero rows - [PR #24745](https://github.com/BerriAI/litellm/pull/24745)
|
||||
- Enforce upperbound key params on `/key/update` and bulk update hook paths - [PR #25110](https://github.com/BerriAI/litellm/pull/25110), [PR #25103](https://github.com/BerriAI/litellm/pull/25103)
|
||||
- Fix team model update 500 due to unsupported Prisma JSON path filter - [PR #25152](https://github.com/BerriAI/litellm/pull/25152)
|
||||
|
||||
## AI Integrations
|
||||
|
||||
### Logging
|
||||
|
||||
- **General**
|
||||
- Eliminate race condition in streaming `guardrail_information` logging - [PR #24592](https://github.com/BerriAI/litellm/pull/24592)
|
||||
- Use actual `start_time` in failed request spend logs - [PR #24906](https://github.com/BerriAI/litellm/pull/24906)
|
||||
- Harden credential redaction + stop logging raw sensitive auth values - [PR #25151](https://github.com/BerriAI/litellm/pull/25151)
|
||||
|
||||
### Guardrails
|
||||
|
||||
- Add optional `on_error` for guardrail pipeline failures - [PR #25150](https://github.com/BerriAI/litellm/pull/25150), [PR #24831](https://github.com/BerriAI/litellm/pull/24831)
|
||||
- Return HTTP 400 (vs 500) for Model Armor streaming blocks - [PR #24693](https://github.com/BerriAI/litellm/pull/24693)
|
||||
|
||||
### Prompt Management
|
||||
|
||||
- Add environment + user tracking for prompts (`development/staging/production`) in CRUD + UI flows - [PR #25110](https://github.com/BerriAI/litellm/pull/25110), [PR #24855](https://github.com/BerriAI/litellm/pull/24855)
|
||||
|
||||
### Secret Managers
|
||||
|
||||
- No major new secret manager provider additions in this RC.
|
||||
|
||||
## Spend Tracking, Budgets and Rate Limiting
|
||||
|
||||
- Enforce budget for models not directly present in the cost map - [PR #24949](https://github.com/BerriAI/litellm/pull/24949)
|
||||
- Add per-model rate limits in team settings/info UI - [PR #25144](https://github.com/BerriAI/litellm/pull/25144)
|
||||
- Fix unversioned Vertex Claude Haiku pricing entry to avoid `$0.00` accounting - [PR #25151](https://github.com/BerriAI/litellm/pull/25151)
|
||||
|
||||
## MCP Gateway
|
||||
|
||||
- Introduce **MCP Toolsets** with DB types, CRUD APIs, scoped permissions, and UI management tab - [PR #25155](https://github.com/BerriAI/litellm/pull/25155)
|
||||
- Resolve toolset names and enforce toolset access correctly in Responses API and streamable MCP paths - [PR #25155](https://github.com/BerriAI/litellm/pull/25155)
|
||||
- Switch toolset permission caching to shared cache path and improve cache invalidation behavior - [PR #25155](https://github.com/BerriAI/litellm/pull/25155)
|
||||
- Allow JWT auth for `/v1/mcp/server/*` sub-paths - [PR #25113](https://github.com/BerriAI/litellm/pull/25113), [PR #24698](https://github.com/BerriAI/litellm/pull/24698)
|
||||
- Add STS AssumeRole support for MCP SigV4 auth - [PR #25151](https://github.com/BerriAI/litellm/pull/25151)
|
||||
- Add tag query fix + MCP metadata support cherry-pick - [PR #25145](https://github.com/BerriAI/litellm/pull/25145)
|
||||
|
||||
## Performance / Loadbalancing / Reliability improvements
|
||||
|
||||
- Integrate router health-check failures with cooldown behavior and transient 429/408 handling - [PR #25150](https://github.com/BerriAI/litellm/pull/25150), [PR #24988](https://github.com/BerriAI/litellm/pull/24988)
|
||||
- Add distributed lock for key rotation job execution - [PR #25150](https://github.com/BerriAI/litellm/pull/25150), [PR #23364](https://github.com/BerriAI/litellm/pull/23364), [PR #23834](https://github.com/BerriAI/litellm/pull/23834)
|
||||
- Improve team routing reliability with deterministic grouping, isolation fixes, stale alias controls, and order-based fallback - [PR #25154](https://github.com/BerriAI/litellm/pull/25154), [PR #25148](https://github.com/BerriAI/litellm/pull/25148)
|
||||
- Regenerate GCP IAM token per async Redis cluster connection (fix token TTL failures) - [PR #25155](https://github.com/BerriAI/litellm/pull/25155), [PR #24426](https://github.com/BerriAI/litellm/pull/24426)
|
||||
- Restore MCP server fields dropped by schema sync migration - [PR #24078](https://github.com/BerriAI/litellm/pull/24078)
|
||||
- Proxy server reliability hardening with bounded queue usage - [PR #25155](https://github.com/BerriAI/litellm/pull/25155)
|
||||
|
||||
## Documentation Updates
|
||||
|
||||
- Improve HA control plane diagram clarity + mobile rendering updates - [PR #24747](https://github.com/BerriAI/litellm/pull/24747)
|
||||
- Document `default_team_params` in config reference and examples - [PR #25032](https://github.com/BerriAI/litellm/pull/25032)
|
||||
- Add JWT to Virtual Key mapping guide - [PR #24882](https://github.com/BerriAI/litellm/pull/24882)
|
||||
- Add MCP Toolsets docs and sidebar updates - [PR #25155](https://github.com/BerriAI/litellm/pull/25155)
|
||||
- Security docs updates and April hardening blog - [PR #24867](https://github.com/BerriAI/litellm/pull/24867), [PR #24868](https://github.com/BerriAI/litellm/pull/24868), [PR #24871](https://github.com/BerriAI/litellm/pull/24871), [PR #25102](https://github.com/BerriAI/litellm/pull/25102)
|
||||
- General docs cleanup + townhall announcement updates - [PR #24839](https://github.com/BerriAI/litellm/pull/24839), [PR #25026](https://github.com/BerriAI/litellm/pull/25026), [PR #25021](https://github.com/BerriAI/litellm/pull/25021)
|
||||
|
||||
## Infrastructure / Security Notes
|
||||
|
||||
- Harden npm and Docker supply chain workflows and release pipeline checks - [PR #24838](https://github.com/BerriAI/litellm/pull/24838), [PR #24877](https://github.com/BerriAI/litellm/pull/24877), [PR #24881](https://github.com/BerriAI/litellm/pull/24881), [PR #24905](https://github.com/BerriAI/litellm/pull/24905), [PR #24951](https://github.com/BerriAI/litellm/pull/24951), [PR #25023](https://github.com/BerriAI/litellm/pull/25023), [PR #25034](https://github.com/BerriAI/litellm/pull/25034), [PR #25036](https://github.com/BerriAI/litellm/pull/25036), [PR #25037](https://github.com/BerriAI/litellm/pull/25037), [PR #25136](https://github.com/BerriAI/litellm/pull/25136), [PR #25158](https://github.com/BerriAI/litellm/pull/25158)
|
||||
- Resolve CodeQL/security workflow issues and fix broken action SHA references - [PR #24880](https://github.com/BerriAI/litellm/pull/24880), [PR #24815](https://github.com/BerriAI/litellm/pull/24815)
|
||||
- Re-add Codecov reporting in GHA matrix workflows - [PR #24804](https://github.com/BerriAI/litellm/pull/24804)
|
||||
- Fix(docker): load enterprise hooks in non-root runtime image - [PR #24917](https://github.com/BerriAI/litellm/pull/24917)
|
||||
- Apply Black formatting to 14 files - [PR #24532](https://github.com/BerriAI/litellm/pull/24532)
|
||||
- Fix lint issues - [PR #24932](https://github.com/BerriAI/litellm/pull/24932)
|
||||
|
||||
## New Contributors
|
||||
|
||||
* @vanhtuan0409 made their first contribution in https://github.com/BerriAI/litellm/pull/24078
|
||||
* @clfhhc made their first contribution in https://github.com/BerriAI/litellm/pull/24932
|
||||
|
||||
**Full Changelog**: https://github.com/BerriAI/litellm/compare/v1.83.0-nightly...v1.83.3.rc.1
|
||||
|
|
@ -325,6 +325,7 @@ const sidebars = {
|
|||
"mcp_control",
|
||||
"mcp_cost",
|
||||
"mcp_guardrail",
|
||||
"mcp_toolsets",
|
||||
{
|
||||
type: "link",
|
||||
label: "MCP Troubleshooting Guide",
|
||||
|
|
@ -1051,7 +1052,8 @@ const sidebars = {
|
|||
"proxy/fallback_management",
|
||||
"proxy/tag_routing",
|
||||
"proxy/timeout",
|
||||
"wildcard_routing"
|
||||
"wildcard_routing",
|
||||
"proxy/health_check_routing"
|
||||
],
|
||||
},
|
||||
{
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[tool.poetry]
|
||||
name = "litellm-enterprise"
|
||||
version = "0.1.35"
|
||||
version = "0.1.36"
|
||||
description = "Package for LiteLLM Enterprise features"
|
||||
authors = ["BerriAI"]
|
||||
readme = "README.md"
|
||||
|
|
@ -22,7 +22,7 @@ requires = ["poetry-core"]
|
|||
build-backend = "poetry.core.masonry.api"
|
||||
|
||||
[tool.commitizen]
|
||||
version = "0.1.35"
|
||||
version = "0.1.36"
|
||||
version_files = [
|
||||
"pyproject.toml:version",
|
||||
"../requirements.txt:litellm-enterprise==",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,19 @@
|
|||
-- CreateTable: LiteLLM_MCPToolsetTable
|
||||
CREATE TABLE IF NOT EXISTS "LiteLLM_MCPToolsetTable" (
|
||||
"toolset_id" TEXT NOT NULL,
|
||||
"toolset_name" TEXT NOT NULL,
|
||||
"description" TEXT,
|
||||
"tools" JSONB NOT NULL DEFAULT '[]',
|
||||
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"created_by" TEXT,
|
||||
"updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"updated_by" TEXT,
|
||||
|
||||
CONSTRAINT "LiteLLM_MCPToolsetTable_pkey" PRIMARY KEY ("toolset_id")
|
||||
);
|
||||
|
||||
-- CreateIndex
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_MCPToolsetTable_toolset_name_key" ON "LiteLLM_MCPToolsetTable"("toolset_name");
|
||||
|
||||
-- AlterTable: add mcp_toolsets to ObjectPermissionTable
|
||||
ALTER TABLE "LiteLLM_ObjectPermissionTable" ADD COLUMN IF NOT EXISTS "mcp_toolsets" TEXT[] DEFAULT ARRAY[]::TEXT[];
|
||||
|
|
@ -273,6 +273,7 @@ model LiteLLM_ObjectPermissionTable {
|
|||
agent_access_groups String[] @default([])
|
||||
models String[] @default([])
|
||||
blocked_tools String[] @default([]) // Tool names blocked for any key/team/user with this permission
|
||||
mcp_toolsets String[] @default([]) // Toolset IDs granted to this key/team/user
|
||||
teams LiteLLM_TeamTable[]
|
||||
projects LiteLLM_ProjectTable[]
|
||||
verification_tokens LiteLLM_VerificationToken[]
|
||||
|
|
@ -331,6 +332,18 @@ model LiteLLM_MCPServerTable {
|
|||
@@index([approval_status])
|
||||
}
|
||||
|
||||
// Named collection of {server_id, tool_name} pairs that can be granted to keys/teams
|
||||
model LiteLLM_MCPToolsetTable {
|
||||
toolset_id String @id @default(uuid())
|
||||
toolset_name String @unique
|
||||
description String?
|
||||
tools Json @default("[]") // [{server_id: string, tool_name: string}]
|
||||
created_at DateTime @default(now())
|
||||
created_by String?
|
||||
updated_at DateTime @default(now()) @updatedAt
|
||||
updated_by String?
|
||||
}
|
||||
|
||||
// Per-user BYOK credentials for MCP servers
|
||||
model LiteLLM_MCPUserCredentials {
|
||||
id String @id @default(uuid())
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[tool.poetry]
|
||||
name = "litellm-proxy-extras"
|
||||
version = "0.4.64"
|
||||
version = "0.4.65"
|
||||
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
|
||||
authors = ["BerriAI"]
|
||||
readme = "README.md"
|
||||
|
|
@ -22,7 +22,7 @@ requires = ["poetry-core"]
|
|||
build-backend = "poetry.core.masonry.api"
|
||||
|
||||
[tool.commitizen]
|
||||
version = "0.4.64"
|
||||
version = "0.4.65"
|
||||
version_files = [
|
||||
"pyproject.toml:version",
|
||||
"../requirements.txt:litellm-proxy-extras==",
|
||||
|
|
|
|||
|
|
@ -1850,6 +1850,7 @@ if TYPE_CHECKING:
|
|||
)
|
||||
from .llms.v0.chat.transformation import V0ChatConfig as V0ChatConfig
|
||||
from .llms.oci.chat.transformation import OCIChatConfig as OCIChatConfig
|
||||
from .llms.oci.embed.transformation import OCIEmbeddingConfig as OCIEmbeddingConfig
|
||||
from .llms.morph.chat.transformation import MorphChatConfig as MorphChatConfig
|
||||
from .llms.ragflow.chat.transformation import RAGFlowConfig as RAGFlowConfig
|
||||
from .llms.lambda_ai.chat.transformation import (
|
||||
|
|
|
|||
|
|
@ -26,6 +26,12 @@ _REDACTED = "REDACTED"
|
|||
|
||||
def _build_secret_patterns() -> re.Pattern:
|
||||
patterns: List[str] = [
|
||||
# ── PEM private key / certificate blocks ──
|
||||
r"-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----",
|
||||
# ── GCP OAuth2 access tokens (ya29.*) ──
|
||||
r"\bya29\.[A-Za-z0-9_.~+/-]+",
|
||||
# ── Credential %s formatting (space separator, no key= prefix) ──
|
||||
r"(?:client_secret|azure_password|azure_username)\s+[^\s,'\"})\]{}>]+",
|
||||
# AWS access key IDs
|
||||
r"(?:AKIA|ASIA)[0-9A-Z]{16}",
|
||||
# AWS secrets / session tokens / access key IDs (key=value)
|
||||
|
|
@ -46,7 +52,8 @@ def _build_secret_patterns() -> re.Pattern:
|
|||
# Google API keys
|
||||
r"AIza[0-9A-Za-z\-_]{35}",
|
||||
# Password / secret params (handles key=value and 'key': 'value')
|
||||
r"\w*(?:password|passwd|client_secret|secret_key|_secret)"
|
||||
# Word boundary prevents O(n^2) backtracking on long word-char runs.
|
||||
r"(?:^|(?<=\W))\w*(?:password|passwd|client_secret|secret_key|_secret)"
|
||||
r"['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+",
|
||||
# Database connection string credentials (scheme://user:pass@host)
|
||||
r"(?<=://)[^\s'\"]*:[^\s'\"@]+(?=@)",
|
||||
|
|
@ -56,13 +63,21 @@ def _build_secret_patterns() -> re.Pattern:
|
|||
# Catches secrets inside dicts/config dumps by matching on the KEY name
|
||||
# regardless of what the value looks like.
|
||||
# e.g. 'master_key': 'any-value-here', "database_url": "postgres://..."
|
||||
# private_key with PEM-aware value capture
|
||||
r"""private_key['\"]?\s*[:=]\s*['\"]?(?:-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----|[^\s,'\"})\]{}>]+)""",
|
||||
r"(?:master_key|database_url|db_url|connection_string|"
|
||||
r"private_key|signing_key|encryption_key|"
|
||||
r"signing_key|encryption_key|"
|
||||
r"auth_token|access_token|refresh_token|"
|
||||
r"slack_webhook_url|webhook_url|"
|
||||
r"database_connection_string|"
|
||||
r"huggingface_token|jwt_secret)"
|
||||
r"""['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+""",
|
||||
# ── Raw JWTs (without Bearer prefix) ──
|
||||
r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*",
|
||||
# ── Azure SAS tokens in URLs ──
|
||||
r"[?&]sig=[A-Za-z0-9%+/=]+",
|
||||
# ── Full JSON service-account blobs (single-line and multi-line) ──
|
||||
r'\{[^{}]*"type"\s*:\s*"service_account"[^{}]*(?:\{[^{}]*\}[^{}]*)*\}',
|
||||
]
|
||||
return re.compile("|".join(patterns), re.IGNORECASE)
|
||||
|
||||
|
|
@ -74,6 +89,23 @@ def _redact_string(value: str) -> str:
|
|||
return _SECRET_RE.sub(_REDACTED, value)
|
||||
|
||||
|
||||
def redact_secrets(value: str) -> str:
|
||||
"""Public API: redact known secret/credential patterns from an arbitrary string.
|
||||
|
||||
Use this for code paths that bypass the logging system — e.g. Slack/Teams
|
||||
alerting, HTTP error response bodies, or any other string that may contain
|
||||
secrets and will be sent to an external sink.
|
||||
|
||||
Not to be confused with redact_message_input_output_from_logging() in
|
||||
litellm_core_utils/redact_messages.py, which redacts LLM prompt/response
|
||||
content for privacy — this function redacts credential patterns (API keys,
|
||||
PEM blocks, tokens, etc.) by shape.
|
||||
"""
|
||||
if not _ENABLE_SECRET_REDACTION:
|
||||
return value
|
||||
return _redact_string(value)
|
||||
|
||||
|
||||
class SecretRedactionFilter(logging.Filter):
|
||||
"""Scrubs known secret/credential patterns from log records."""
|
||||
|
||||
|
|
@ -441,7 +473,7 @@ def _enable_debugging():
|
|||
def print_verbose(print_statement):
|
||||
try:
|
||||
if set_verbose:
|
||||
print(print_statement) # noqa
|
||||
print(redact_secrets(str(print_statement))) # noqa
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
|
|
|||
|
|
@ -18,6 +18,10 @@ import redis # type: ignore
|
|||
import redis.asyncio as async_redis # type: ignore
|
||||
|
||||
from litellm import get_secret, get_secret_str
|
||||
from litellm._redis_credential_provider import (
|
||||
GCPIAMCredentialProvider,
|
||||
_generate_gcp_iam_access_token,
|
||||
)
|
||||
from litellm.constants import REDIS_CONNECTION_POOL_TIMEOUT, REDIS_SOCKET_TIMEOUT
|
||||
from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker
|
||||
|
||||
|
|
@ -107,33 +111,6 @@ def _redis_kwargs_from_environment():
|
|||
return return_dict
|
||||
|
||||
|
||||
def _generate_gcp_iam_access_token(service_account: str) -> str:
|
||||
"""
|
||||
Generate GCP IAM access token for Redis authentication.
|
||||
|
||||
Args:
|
||||
service_account: GCP service account in format 'projects/-/serviceAccounts/name@project.iam.gserviceaccount.com'
|
||||
|
||||
Returns:
|
||||
Access token string for GCP IAM authentication
|
||||
"""
|
||||
try:
|
||||
from google.cloud import iam_credentials_v1 # type: ignore
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"google-cloud-iam is required for GCP IAM Redis authentication. "
|
||||
"Install it with: pip install google-cloud-iam"
|
||||
)
|
||||
|
||||
client = iam_credentials_v1.IAMCredentialsClient()
|
||||
request = iam_credentials_v1.GenerateAccessTokenRequest(
|
||||
name=service_account,
|
||||
scope=["https://www.googleapis.com/auth/cloud-platform"],
|
||||
)
|
||||
response = client.generate_access_token(request=request)
|
||||
return str(response.access_token)
|
||||
|
||||
|
||||
def create_gcp_iam_redis_connect_func(
|
||||
service_account: str,
|
||||
ssl_ca_certs: Optional[str] = None,
|
||||
|
|
@ -266,7 +243,7 @@ def _get_redis_client_logic(**env_overrides):
|
|||
service_account=_gcp_service_account, ssl_ca_certs=_gcp_ssl_ca_certs
|
||||
)
|
||||
# Store GCP service account in redis_connect_func for async cluster access
|
||||
redis_kwargs["redis_connect_func"]._gcp_service_account = _gcp_service_account
|
||||
redis_kwargs["redis_connect_func"]._gcp_service_account = _gcp_service_account # type: ignore[attr-defined]
|
||||
|
||||
# Remove GCP-specific kwargs that shouldn't be passed to Redis client
|
||||
redis_kwargs.pop("gcp_service_account", None)
|
||||
|
|
@ -413,41 +390,13 @@ def get_redis_async_client(
|
|||
|
||||
# Handle GCP IAM authentication for async clusters
|
||||
redis_connect_func = cluster_kwargs.pop("redis_connect_func", None)
|
||||
from litellm import get_secret_str
|
||||
|
||||
# Get GCP service account - first try from redis_connect_func, then from environment
|
||||
gcp_service_account = None
|
||||
# Use a CredentialProvider so the IAM token is regenerated on every new
|
||||
# connection — mirrors the sync path where redis_connect_func is invoked
|
||||
# per connection. Without this, the token would expire after ~1 hour.
|
||||
if redis_connect_func and hasattr(redis_connect_func, "_gcp_service_account"):
|
||||
gcp_service_account = redis_connect_func._gcp_service_account
|
||||
else:
|
||||
gcp_service_account = redis_kwargs.get(
|
||||
"gcp_service_account"
|
||||
) or get_secret_str("REDIS_GCP_SERVICE_ACCOUNT")
|
||||
|
||||
verbose_logger.debug(
|
||||
f"DEBUG: Redis cluster kwargs: redis_connect_func={redis_connect_func is not None}, gcp_service_account_provided={gcp_service_account is not None}"
|
||||
)
|
||||
|
||||
# If GCP IAM is configured (indicated by redis_connect_func), generate access token and use as password
|
||||
if redis_connect_func and gcp_service_account:
|
||||
verbose_logger.debug(
|
||||
"DEBUG: Generating IAM token for service account (value not logged for security reasons)"
|
||||
)
|
||||
try:
|
||||
# Generate IAM access token using the helper function
|
||||
access_token = _generate_gcp_iam_access_token(gcp_service_account)
|
||||
cluster_kwargs["password"] = access_token
|
||||
verbose_logger.debug(
|
||||
"DEBUG: Successfully generated GCP IAM access token for async Redis cluster"
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_logger.error(f"Failed to generate GCP IAM access token: {e}")
|
||||
from redis.exceptions import AuthenticationError
|
||||
|
||||
raise AuthenticationError("Failed to generate GCP IAM access token")
|
||||
else:
|
||||
verbose_logger.debug(
|
||||
f"DEBUG: Not using GCP IAM auth - redis_connect_func={redis_connect_func is not None}, gcp_service_account_provided={gcp_service_account is not None}"
|
||||
cluster_kwargs["credential_provider"] = GCPIAMCredentialProvider(
|
||||
redis_connect_func._gcp_service_account
|
||||
)
|
||||
|
||||
new_startup_nodes: List[ClusterNode] = []
|
||||
|
|
|
|||
53
litellm/_redis_credential_provider.py
Normal file
53
litellm/_redis_credential_provider.py
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
import asyncio
|
||||
from typing import Tuple
|
||||
|
||||
from redis.credentials import CredentialProvider # type: ignore[attr-defined]
|
||||
|
||||
|
||||
def _generate_gcp_iam_access_token(service_account: str) -> str:
|
||||
"""
|
||||
Generate GCP IAM access token for Redis authentication.
|
||||
|
||||
Args:
|
||||
service_account: GCP service account in format 'projects/-/serviceAccounts/name@project.iam.gserviceaccount.com'
|
||||
|
||||
Returns:
|
||||
Access token string for GCP IAM authentication
|
||||
"""
|
||||
try:
|
||||
from google.cloud import iam_credentials_v1
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"google-cloud-iam is required for GCP IAM Redis authentication. "
|
||||
"Install it with: pip install google-cloud-iam"
|
||||
)
|
||||
|
||||
client = iam_credentials_v1.IAMCredentialsClient()
|
||||
request = iam_credentials_v1.GenerateAccessTokenRequest(
|
||||
name=service_account,
|
||||
scope=["https://www.googleapis.com/auth/cloud-platform"],
|
||||
)
|
||||
response = client.generate_access_token(request=request)
|
||||
return str(response.access_token)
|
||||
|
||||
|
||||
class GCPIAMCredentialProvider(CredentialProvider):
|
||||
"""
|
||||
redis.credentials.CredentialProvider implementation that generates a fresh GCP IAM
|
||||
token on every new connection. This fixes the 1-hour token expiry issue for async
|
||||
Redis cluster clients, which previously generated the token once at startup and
|
||||
cached it as a static password.
|
||||
"""
|
||||
|
||||
def __init__(self, gcp_service_account: str) -> None:
|
||||
self._gcp_service_account = gcp_service_account
|
||||
|
||||
def get_credentials(self) -> Tuple[str]:
|
||||
token = _generate_gcp_iam_access_token(self._gcp_service_account)
|
||||
return (token,)
|
||||
|
||||
async def get_credentials_async(self) -> Tuple[str]:
|
||||
token = await asyncio.to_thread(
|
||||
_generate_gcp_iam_access_token, self._gcp_service_account
|
||||
)
|
||||
return (token,)
|
||||
|
|
@ -2,7 +2,7 @@
|
|||
Pydantic AI provider configuration.
|
||||
"""
|
||||
|
||||
from typing import Any, AsyncIterator, Dict
|
||||
from typing import Any, AsyncIterator, Dict, Optional
|
||||
|
||||
from litellm.a2a_protocol.providers.base import BaseA2AProviderConfig
|
||||
from litellm.a2a_protocol.providers.pydantic_ai_agents.handler import PydanticAIHandler
|
||||
|
|
@ -20,10 +20,12 @@ class PydanticAIProviderConfig(BaseA2AProviderConfig):
|
|||
self,
|
||||
request_id: str,
|
||||
params: Dict[str, Any],
|
||||
api_base: str,
|
||||
api_base: Optional[str] = None,
|
||||
**kwargs,
|
||||
) -> Dict[str, Any]:
|
||||
"""Handle non-streaming request to Pydantic AI agent."""
|
||||
if not api_base:
|
||||
raise ValueError("api_base is required for Pydantic AI agents")
|
||||
return await PydanticAIHandler.handle_non_streaming(
|
||||
request_id=request_id,
|
||||
params=params,
|
||||
|
|
@ -35,10 +37,12 @@ class PydanticAIProviderConfig(BaseA2AProviderConfig):
|
|||
self,
|
||||
request_id: str,
|
||||
params: Dict[str, Any],
|
||||
api_base: str,
|
||||
api_base: Optional[str] = None,
|
||||
**kwargs,
|
||||
) -> AsyncIterator[Dict[str, Any]]:
|
||||
"""Handle streaming request with fake streaming."""
|
||||
if not api_base:
|
||||
raise ValueError("api_base is required for Pydantic AI agents")
|
||||
async for chunk in PydanticAIHandler.handle_streaming(
|
||||
request_id=request_id,
|
||||
params=params,
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ Pydantic AI agents follow A2A protocol but don't support streaming natively.
|
|||
This handler provides fake streaming by converting non-streaming responses into streaming chunks.
|
||||
"""
|
||||
|
||||
from typing import Any, AsyncIterator, Dict
|
||||
from typing import Any, AsyncIterator, Dict, Optional
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.a2a_protocol.providers.pydantic_ai_agents.transformation import (
|
||||
|
|
@ -26,7 +26,7 @@ class PydanticAIHandler:
|
|||
async def handle_non_streaming(
|
||||
request_id: str,
|
||||
params: Dict[str, Any],
|
||||
api_base: str,
|
||||
api_base: Optional[str] = None,
|
||||
timeout: float = 60.0,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
|
|
@ -41,6 +41,8 @@ class PydanticAIHandler:
|
|||
Returns:
|
||||
A2A SendMessageResponse dict
|
||||
"""
|
||||
if api_base is None:
|
||||
raise ValueError("api_base is required for Pydantic AI agents")
|
||||
verbose_logger.info(f"Pydantic AI: Routing to Pydantic AI agent at {api_base}")
|
||||
|
||||
# Send request directly to Pydantic AI agent
|
||||
|
|
@ -57,7 +59,7 @@ class PydanticAIHandler:
|
|||
async def handle_streaming(
|
||||
request_id: str,
|
||||
params: Dict[str, Any],
|
||||
api_base: str,
|
||||
api_base: Optional[str] = None,
|
||||
timeout: float = 60.0,
|
||||
chunk_size: int = 50,
|
||||
delay_ms: int = 10,
|
||||
|
|
@ -80,6 +82,8 @@ class PydanticAIHandler:
|
|||
Yields:
|
||||
A2A streaming response events
|
||||
"""
|
||||
if api_base is None:
|
||||
raise ValueError("api_base is required for Pydantic AI agents")
|
||||
verbose_logger.info(
|
||||
f"Pydantic AI: Faking streaming for Pydantic AI agent at {api_base}"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1319,6 +1319,9 @@ LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS = int(
|
|||
LITELLM_KEY_ROTATION_GRACE_PERIOD: str = os.getenv(
|
||||
"LITELLM_KEY_ROTATION_GRACE_PERIOD", ""
|
||||
) # Duration to keep old key valid after rotation (e.g. "24h", "2d"); empty = immediate revoke (default)
|
||||
LITELLM_KEY_ROTATION_LOCK_TTL_SECONDS = int(
|
||||
os.getenv("LITELLM_KEY_ROTATION_LOCK_TTL_SECONDS", 600)
|
||||
) # 10 minutes default — caps the deadlock window if a pod crashes mid-rotation
|
||||
UI_SESSION_TOKEN_TEAM_ID = "litellm-dashboard"
|
||||
LITELLM_PROXY_ADMIN_NAME = "default_user_id"
|
||||
|
||||
|
|
@ -1341,12 +1344,14 @@ LITELLM_UI_SESSION_DURATION = os.getenv("LITELLM_UI_SESSION_DURATION", "24h")
|
|||
|
||||
########################### DB CRON JOB NAMES ###########################
|
||||
DB_SPEND_UPDATE_JOB_NAME = "db_spend_update_job"
|
||||
DB_DAILY_TAG_SPEND_UPDATE_JOB_NAME = "db_daily_tag_spend_update_job"
|
||||
PROMETHEUS_EMIT_BUDGET_METRICS_JOB_NAME = "prometheus_emit_budget_metrics"
|
||||
CLOUDZERO_EXPORT_USAGE_DATA_JOB_NAME = "cloudzero_export_usage_data"
|
||||
CLOUDZERO_MAX_FETCHED_DATA_RECORDS = int(
|
||||
os.getenv("CLOUDZERO_MAX_FETCHED_DATA_RECORDS", 50000)
|
||||
)
|
||||
SPEND_LOG_CLEANUP_JOB_NAME = "spend_log_cleanup"
|
||||
KEY_ROTATION_JOB_NAME = "litellm_key_rotation_job"
|
||||
SPEND_LOG_RUN_LOOPS = int(os.getenv("SPEND_LOG_RUN_LOOPS", 500))
|
||||
SPEND_LOG_CLEANUP_BATCH_SIZE = int(os.getenv("SPEND_LOG_CLEANUP_BATCH_SIZE", 1000))
|
||||
SPEND_LOG_QUEUE_SIZE_THRESHOLD = int(os.getenv("SPEND_LOG_QUEUE_SIZE_THRESHOLD", 100))
|
||||
|
|
@ -1393,6 +1398,10 @@ APSCHEDULER_REPLACE_EXISTING = os.getenv(
|
|||
"1",
|
||||
] # always replace existing jobs
|
||||
|
||||
# The number of tag entries are higher than number of user, team entries. This leads to a higher QPS.
|
||||
# This will run tag spcific tasks at a later time to smooth QPS
|
||||
DAILY_TAG_SPEND_BATCH_MULTIPLIER = 2.3
|
||||
|
||||
DEFAULT_HEALTH_CHECK_INTERVAL = int(
|
||||
os.getenv("DEFAULT_HEALTH_CHECK_INTERVAL", 300)
|
||||
) # 5 minutes
|
||||
|
|
|
|||
|
|
@ -82,6 +82,8 @@ class MCPSigV4Auth(httpx.Auth):
|
|||
aws_session_token: Optional[str] = None,
|
||||
aws_region_name: Optional[str] = None,
|
||||
aws_service_name: Optional[str] = None,
|
||||
aws_role_name: Optional[str] = None,
|
||||
aws_session_name: Optional[str] = None,
|
||||
):
|
||||
try:
|
||||
from botocore.credentials import Credentials
|
||||
|
|
@ -97,7 +99,16 @@ class MCPSigV4Auth(httpx.Auth):
|
|||
# Note: os.environ/ prefixed values are already resolved by
|
||||
# ProxyConfig._check_for_os_environ_vars() at config load time.
|
||||
# Values arrive here as plain strings.
|
||||
if aws_access_key_id and aws_secret_access_key:
|
||||
if aws_role_name:
|
||||
self.credentials = self._assume_role(
|
||||
aws_role_name=aws_role_name,
|
||||
aws_session_name=aws_session_name,
|
||||
aws_access_key_id=aws_access_key_id,
|
||||
aws_secret_access_key=aws_secret_access_key,
|
||||
aws_session_token=aws_session_token,
|
||||
aws_region_name=self.region_name,
|
||||
)
|
||||
elif aws_access_key_id and aws_secret_access_key:
|
||||
self.credentials = Credentials(
|
||||
access_key=aws_access_key_id,
|
||||
secret_key=aws_secret_access_key,
|
||||
|
|
@ -116,6 +127,43 @@ class MCPSigV4Auth(httpx.Auth):
|
|||
"(env vars, ~/.aws/credentials, instance profile)."
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _assume_role(
|
||||
aws_role_name: str,
|
||||
aws_session_name: Optional[str],
|
||||
aws_access_key_id: Optional[str],
|
||||
aws_secret_access_key: Optional[str],
|
||||
aws_session_token: Optional[str],
|
||||
aws_region_name: str,
|
||||
):
|
||||
"""Call STS AssumeRole and return temporary credentials."""
|
||||
import boto3
|
||||
from botocore.credentials import Credentials
|
||||
|
||||
session_name = (
|
||||
aws_session_name or f"litellm-mcp-{int(__import__('time').time())}"
|
||||
)
|
||||
|
||||
sts_kwargs: dict = {"region_name": aws_region_name}
|
||||
if aws_access_key_id and aws_secret_access_key:
|
||||
sts_kwargs["aws_access_key_id"] = aws_access_key_id
|
||||
sts_kwargs["aws_secret_access_key"] = aws_secret_access_key
|
||||
if aws_session_token:
|
||||
sts_kwargs["aws_session_token"] = aws_session_token
|
||||
|
||||
sts_client = boto3.client("sts", **sts_kwargs)
|
||||
sts_response = sts_client.assume_role(
|
||||
RoleArn=aws_role_name,
|
||||
RoleSessionName=session_name,
|
||||
)
|
||||
|
||||
sts_creds = sts_response["Credentials"]
|
||||
return Credentials(
|
||||
access_key=sts_creds["AccessKeyId"],
|
||||
secret_key=sts_creds["SecretAccessKey"],
|
||||
token=sts_creds["SessionToken"],
|
||||
)
|
||||
|
||||
def auth_flow(
|
||||
self, request: httpx.Request
|
||||
) -> Generator[httpx.Request, httpx.Response, None]:
|
||||
|
|
|
|||
|
|
@ -275,12 +275,11 @@ class AzureBlobStorageLogger(CustomBatchLogger):
|
|||
"""
|
||||
Gets Azure AD token to use for Azure Storage API requests
|
||||
"""
|
||||
verbose_logger.debug("Getting Azure AD Token from Azure Storage")
|
||||
verbose_logger.debug(
|
||||
"tenant_id %s, client_id %s, client_secret %s",
|
||||
"Getting Azure AD Token from Azure Storage, tenant_id=%s, client_id=%s, client_secret=[set=%s]",
|
||||
tenant_id,
|
||||
client_id,
|
||||
client_secret,
|
||||
client_secret is not None,
|
||||
)
|
||||
if tenant_id is None:
|
||||
raise ValueError(
|
||||
|
|
|
|||
|
|
@ -70,7 +70,9 @@ class GCSBucketBase(CustomBatchLogger):
|
|||
custom_llm_provider="vertex_ai",
|
||||
api_base=None,
|
||||
)
|
||||
verbose_logger.debug("constructed auth_header %s", auth_header)
|
||||
verbose_logger.debug(
|
||||
"constructed auth_header [set=%s]", auth_header is not None
|
||||
)
|
||||
headers = {
|
||||
"Authorization": f"Bearer {auth_header}", # auth_header
|
||||
"Content-Type": "application/json",
|
||||
|
|
@ -106,7 +108,9 @@ class GCSBucketBase(CustomBatchLogger):
|
|||
custom_llm_provider="vertex_ai",
|
||||
api_base=None,
|
||||
)
|
||||
verbose_logger.debug("constructed auth_header %s", auth_header)
|
||||
verbose_logger.debug(
|
||||
"constructed auth_header [set=%s]", auth_header is not None
|
||||
)
|
||||
headers = {
|
||||
"Authorization": f"Bearer {auth_header}", # auth_header
|
||||
"Content-Type": "application/json",
|
||||
|
|
|
|||
|
|
@ -202,8 +202,8 @@ def get_llm_provider( # noqa: PLR0915
|
|||
)
|
||||
if dynamic_api_key is not None and not isinstance(dynamic_api_key, str):
|
||||
raise Exception(
|
||||
"dynamic_api_key needs to be a string. dynamic_api_key={}".format(
|
||||
dynamic_api_key
|
||||
"dynamic_api_key needs to be a string. Got type={}".format(
|
||||
type(dynamic_api_key).__name__
|
||||
)
|
||||
)
|
||||
return model, custom_llm_provider, dynamic_api_key, api_base
|
||||
|
|
|
|||
|
|
@ -101,17 +101,15 @@ def get_azure_ad_token_from_entra_id(
|
|||
_client_secret = client_secret
|
||||
|
||||
verbose_logger.debug(
|
||||
"tenant_id %s, client_id %s, client_secret %s",
|
||||
"tenant_id=%s, client_id=%s, client_secret=[set=%s]",
|
||||
_tenant_id,
|
||||
_client_id,
|
||||
_client_secret,
|
||||
_client_secret is not None,
|
||||
)
|
||||
if _tenant_id is None or _client_id is None or _client_secret is None:
|
||||
raise ValueError("tenant_id, client_id, and client_secret must be provided")
|
||||
credential = ClientSecretCredential(_tenant_id, _client_id, _client_secret)
|
||||
|
||||
verbose_logger.debug("credential %s", credential)
|
||||
|
||||
token_provider = get_bearer_token_provider(credential, scope)
|
||||
|
||||
verbose_logger.debug("token_provider %s", token_provider)
|
||||
|
|
@ -140,10 +138,10 @@ def get_azure_ad_token_from_username_password(
|
|||
from azure.identity import UsernamePasswordCredential, get_bearer_token_provider
|
||||
|
||||
verbose_logger.debug(
|
||||
"client_id %s, azure_username %s, azure_password %s",
|
||||
"client_id=%s, azure_username=[set=%s], azure_password=[set=%s]",
|
||||
client_id,
|
||||
azure_username,
|
||||
azure_password,
|
||||
azure_username is not None,
|
||||
azure_password is not None,
|
||||
)
|
||||
credential = UsernamePasswordCredential(
|
||||
client_id=client_id,
|
||||
|
|
@ -151,8 +149,6 @@ def get_azure_ad_token_from_username_password(
|
|||
password=azure_password,
|
||||
)
|
||||
|
||||
verbose_logger.debug("credential %s", credential)
|
||||
|
||||
token_provider = get_bearer_token_provider(credential, scope)
|
||||
|
||||
verbose_logger.debug("token_provider %s", token_provider)
|
||||
|
|
|
|||
|
|
@ -156,24 +156,24 @@ class BaseAWSLLM:
|
|||
|
||||
verbose_logger.debug(
|
||||
"in get credentials\n"
|
||||
"aws_access_key_id=%s\n"
|
||||
"aws_secret_access_key=%s\n"
|
||||
"aws_session_token=%s\n"
|
||||
"aws_access_key_id=[set=%s]\n"
|
||||
"aws_secret_access_key=[set=%s]\n"
|
||||
"aws_session_token=[set=%s]\n"
|
||||
"aws_region_name=%s\n"
|
||||
"aws_session_name=%s\n"
|
||||
"aws_profile_name=%s\n"
|
||||
"aws_role_name=%s\n"
|
||||
"aws_web_identity_token=%s\n"
|
||||
"aws_web_identity_token=[set=%s]\n"
|
||||
"aws_sts_endpoint=%s\n"
|
||||
"aws_external_id=%s",
|
||||
aws_access_key_id,
|
||||
aws_secret_access_key,
|
||||
aws_session_token,
|
||||
aws_access_key_id is not None,
|
||||
aws_secret_access_key is not None,
|
||||
aws_session_token is not None,
|
||||
aws_region_name,
|
||||
aws_session_name,
|
||||
aws_profile_name,
|
||||
aws_role_name,
|
||||
aws_web_identity_token,
|
||||
aws_web_identity_token is not None,
|
||||
aws_sts_endpoint,
|
||||
aws_external_id,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -174,10 +174,15 @@ def load_private_key_from_file(file_path: str):
|
|||
def get_vendor_from_model(model: str) -> OCIVendors:
|
||||
"""
|
||||
Extracts the vendor from the model name.
|
||||
|
||||
OCI GenAI API uses two apiFormat values:
|
||||
- "COHERE" for Cohere models (command-r, command-a, etc.)
|
||||
- "GENERIC" for all other models (Meta Llama, xAI Grok, Google Gemini, etc.)
|
||||
|
||||
Args:
|
||||
model (str): The model name.
|
||||
model (str): The model name (e.g., "cohere.command-a-03-2025", "meta.llama-3.3-70b-instruct").
|
||||
Returns:
|
||||
str: The vendor name.
|
||||
OCIVendors: The vendor enum value.
|
||||
"""
|
||||
vendor = model.split(".")[0].lower()
|
||||
if vendor == "cohere":
|
||||
|
|
|
|||
0
litellm/llms/oci/embed/__init__.py
Normal file
0
litellm/llms/oci/embed/__init__.py
Normal file
347
litellm/llms/oci/embed/transformation.py
Normal file
347
litellm/llms/oci/embed/transformation.py
Normal file
|
|
@ -0,0 +1,347 @@
|
|||
"""
|
||||
OCI Generative AI Embedding Configuration
|
||||
|
||||
Supports embedding models available on Oracle Cloud Infrastructure Generative AI service.
|
||||
Uses the same authentication mechanisms as OCI chat (manual signing or OCI SDK Signer).
|
||||
|
||||
Supported models:
|
||||
- cohere.embed-english-v3.0
|
||||
- cohere.embed-english-light-v3.0
|
||||
- cohere.embed-multilingual-v3.0
|
||||
- cohere.embed-multilingual-light-v3.0
|
||||
- cohere.embed-english-image-v3.0
|
||||
- cohere.embed-english-light-image-v3.0
|
||||
- cohere.embed-multilingual-light-image-v3.0
|
||||
- cohere.embed-v4.0
|
||||
|
||||
Reference: https://docs.oracle.com/en-us/iaas/api/#/en/generative-ai-inference/latest/EmbedTextResult/EmbedText
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig
|
||||
from litellm.llms.oci.chat.transformation import OCIChatConfig
|
||||
from litellm.llms.oci.common_utils import OCIError
|
||||
from litellm.types.llms.openai import AllEmbeddingInputValues, AllMessageValues
|
||||
from litellm.types.utils import EmbeddingResponse, Usage
|
||||
|
||||
# Input type mapping from OpenAI conventions to OCI/Cohere conventions
|
||||
_INPUT_TYPE_MAP = {
|
||||
"search_document": "SEARCH_DOCUMENT",
|
||||
"search_query": "SEARCH_QUERY",
|
||||
"classification": "CLASSIFICATION",
|
||||
"clustering": "CLUSTERING",
|
||||
}
|
||||
|
||||
|
||||
class OCIEmbeddingConfig(BaseEmbeddingConfig):
|
||||
"""
|
||||
Configuration for OCI Generative AI Embedding API.
|
||||
|
||||
The OCI embedding endpoint uses the Cohere embed models hosted on OCI.
|
||||
Authentication is handled via OCI request signing (manual credentials or OCI SDK Signer).
|
||||
|
||||
Usage:
|
||||
```python
|
||||
import litellm
|
||||
|
||||
response = litellm.embedding(
|
||||
model="oci/cohere.embed-english-v3.0",
|
||||
input=["Hello world", "Goodbye world"],
|
||||
oci_compartment_id="ocid1.compartment.oc1..xxx",
|
||||
oci_region="us-ashburn-1",
|
||||
oci_user="ocid1.user.oc1..xxx",
|
||||
oci_fingerprint="xx:xx:xx:xx",
|
||||
oci_tenancy="ocid1.tenancy.oc1..xxx",
|
||||
oci_key_file="~/.oci/key.pem",
|
||||
)
|
||||
```
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
# We reuse OCIChatConfig for signing logic
|
||||
self._chat_config = OCIChatConfig()
|
||||
|
||||
def get_complete_url(
|
||||
self,
|
||||
api_base: Optional[str],
|
||||
api_key: Optional[str],
|
||||
model: str,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
stream: Optional[bool] = None,
|
||||
) -> str:
|
||||
if api_base:
|
||||
return api_base
|
||||
|
||||
oci_region = optional_params.get("oci_region", "us-ashburn-1")
|
||||
return f"https://inference.generativeai.{oci_region}.oci.oraclecloud.com/20231130/actions/embedText"
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list:
|
||||
return [
|
||||
"dimensions",
|
||||
]
|
||||
|
||||
def map_openai_params(
|
||||
self,
|
||||
non_default_params: dict,
|
||||
optional_params: dict,
|
||||
model: str,
|
||||
drop_params: bool,
|
||||
) -> dict:
|
||||
# Note: OCI Cohere embed does not support custom dimensions natively,
|
||||
# but we pass it through in case future models support it
|
||||
if "dimensions" in non_default_params:
|
||||
optional_params["dimensions"] = non_default_params["dimensions"]
|
||||
return optional_params
|
||||
|
||||
def validate_environment(
|
||||
self,
|
||||
headers: dict,
|
||||
model: str,
|
||||
messages: List[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
api_key: Optional[str] = None,
|
||||
api_base: Optional[str] = None,
|
||||
) -> dict:
|
||||
"""
|
||||
Validate OCI credentials for embedding requests.
|
||||
Supports both OCI SDK Signer and manual credential signing.
|
||||
"""
|
||||
oci_signer = optional_params.get("oci_signer")
|
||||
oci_region = optional_params.get("oci_region", "us-ashburn-1")
|
||||
|
||||
api_base = (
|
||||
api_base
|
||||
or f"https://inference.generativeai.{oci_region}.oci.oraclecloud.com"
|
||||
)
|
||||
|
||||
if oci_signer is None:
|
||||
oci_user = optional_params.get("oci_user")
|
||||
oci_fingerprint = optional_params.get("oci_fingerprint")
|
||||
oci_tenancy = optional_params.get("oci_tenancy")
|
||||
oci_key = optional_params.get("oci_key")
|
||||
oci_key_file = optional_params.get("oci_key_file")
|
||||
oci_compartment_id = optional_params.get("oci_compartment_id")
|
||||
|
||||
if (
|
||||
not oci_user
|
||||
or not oci_fingerprint
|
||||
or not oci_tenancy
|
||||
or not (oci_key or oci_key_file)
|
||||
or not oci_compartment_id
|
||||
):
|
||||
raise Exception(
|
||||
"Missing required parameters: oci_user, oci_fingerprint, oci_tenancy, oci_compartment_id "
|
||||
"and at least one of oci_key or oci_key_file. "
|
||||
"Alternatively, provide an oci_signer object from the OCI SDK."
|
||||
)
|
||||
|
||||
from litellm.llms.custom_httpx.http_handler import version
|
||||
|
||||
headers.update(
|
||||
{
|
||||
"content-type": "application/json",
|
||||
"user-agent": f"litellm/{version}",
|
||||
}
|
||||
)
|
||||
|
||||
return headers
|
||||
|
||||
def sign_request(
|
||||
self,
|
||||
headers: dict,
|
||||
optional_params: dict,
|
||||
request_data: dict,
|
||||
api_base: str,
|
||||
api_key: Optional[str] = None,
|
||||
model: Optional[str] = None,
|
||||
stream: Optional[bool] = None,
|
||||
fake_stream: Optional[bool] = None,
|
||||
):
|
||||
"""Delegate to OCIChatConfig's signing logic."""
|
||||
return self._chat_config.sign_request(
|
||||
headers=headers,
|
||||
optional_params=optional_params,
|
||||
request_data=request_data,
|
||||
api_base=api_base,
|
||||
api_key=api_key,
|
||||
model=model,
|
||||
stream=stream,
|
||||
fake_stream=fake_stream,
|
||||
)
|
||||
|
||||
def transform_embedding_request(
|
||||
self,
|
||||
model: str,
|
||||
input: AllEmbeddingInputValues,
|
||||
optional_params: dict,
|
||||
headers: dict,
|
||||
api_base: Optional[str] = None,
|
||||
) -> dict:
|
||||
"""
|
||||
Transform the embedding request to OCI format.
|
||||
|
||||
OCI embedText API expects:
|
||||
{
|
||||
"compartmentId": "...",
|
||||
"servingMode": {"servingType": "ON_DEMAND", "modelId": "..."},
|
||||
"inputs": ["text1", "text2"],
|
||||
"truncate": "END",
|
||||
"inputType": "SEARCH_DOCUMENT"
|
||||
}
|
||||
"""
|
||||
oci_compartment_id = optional_params.get("oci_compartment_id")
|
||||
if not oci_compartment_id:
|
||||
raise Exception(
|
||||
"kwarg `oci_compartment_id` is required for OCI embedding requests"
|
||||
)
|
||||
|
||||
# Build serving mode
|
||||
oci_serving_mode = optional_params.get("oci_serving_mode", "ON_DEMAND")
|
||||
if oci_serving_mode == "DEDICATED":
|
||||
oci_endpoint_id = optional_params.get("oci_endpoint_id", model)
|
||||
serving_mode = {
|
||||
"servingType": "DEDICATED",
|
||||
"endpointId": oci_endpoint_id,
|
||||
}
|
||||
else:
|
||||
serving_mode = {
|
||||
"servingType": "ON_DEMAND",
|
||||
"modelId": model,
|
||||
}
|
||||
|
||||
# Normalize input to list of strings
|
||||
if isinstance(input, str):
|
||||
inputs = [input]
|
||||
elif isinstance(input, list):
|
||||
inputs = []
|
||||
for item in input:
|
||||
if isinstance(item, str):
|
||||
inputs.append(item)
|
||||
elif isinstance(item, list):
|
||||
raise ValueError(
|
||||
"OCI embedding does not support token-array inputs. "
|
||||
"Please convert token lists to strings before calling embedding()."
|
||||
)
|
||||
else:
|
||||
inputs.append(str(item))
|
||||
else:
|
||||
inputs = [str(input)]
|
||||
|
||||
# Build request data — OCI embedText API expects inputs, truncate,
|
||||
# and inputType at the top level alongside compartmentId and servingMode
|
||||
request_data: Dict[str, Any] = {
|
||||
"compartmentId": oci_compartment_id,
|
||||
"servingMode": serving_mode,
|
||||
"inputs": inputs,
|
||||
"truncate": optional_params.get("truncate", "END"),
|
||||
}
|
||||
|
||||
# Map input_type if provided
|
||||
input_type = optional_params.get("input_type")
|
||||
if input_type:
|
||||
mapped_type = _INPUT_TYPE_MAP.get(input_type.lower(), input_type.upper())
|
||||
request_data["inputType"] = mapped_type
|
||||
|
||||
# Sign the request using the same URL the HTTP handler will POST to
|
||||
signing_url = self.get_complete_url(
|
||||
api_base=api_base,
|
||||
api_key=None,
|
||||
model=model,
|
||||
optional_params=optional_params,
|
||||
litellm_params={},
|
||||
)
|
||||
|
||||
signed_headers, body = self.sign_request(
|
||||
headers=headers,
|
||||
optional_params=optional_params,
|
||||
request_data=request_data,
|
||||
api_base=signing_url,
|
||||
)
|
||||
headers.update(signed_headers)
|
||||
|
||||
return request_data
|
||||
|
||||
def transform_embedding_response(
|
||||
self,
|
||||
model: str,
|
||||
raw_response: httpx.Response,
|
||||
model_response: EmbeddingResponse,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
api_key: Optional[str] = None,
|
||||
request_data: dict = {},
|
||||
optional_params: dict = {},
|
||||
litellm_params: dict = {},
|
||||
) -> EmbeddingResponse:
|
||||
"""
|
||||
Transform OCI embedding response to standard EmbeddingResponse format.
|
||||
|
||||
OCI response format:
|
||||
{
|
||||
"embeddings": [[0.1, 0.2, ...], [0.3, 0.4, ...]],
|
||||
"modelId": "cohere.embed-english-v3.0",
|
||||
"modelVersion": "3.0",
|
||||
"inputTextTokenCounts": [5, 4]
|
||||
}
|
||||
"""
|
||||
if raw_response.status_code != 200:
|
||||
raise OCIError(
|
||||
message=raw_response.text,
|
||||
status_code=raw_response.status_code,
|
||||
)
|
||||
|
||||
try:
|
||||
raw_response_json = raw_response.json()
|
||||
except Exception:
|
||||
raise OCIError(
|
||||
message=raw_response.text,
|
||||
status_code=raw_response.status_code,
|
||||
)
|
||||
|
||||
embeddings = raw_response_json.get("embeddings", [])
|
||||
model_id = raw_response_json.get("modelId", model)
|
||||
|
||||
# Build response data in OpenAI format
|
||||
embedding_data = []
|
||||
for idx, embedding in enumerate(embeddings):
|
||||
embedding_data.append(
|
||||
{
|
||||
"object": "embedding",
|
||||
"index": idx,
|
||||
"embedding": embedding,
|
||||
}
|
||||
)
|
||||
|
||||
model_response.model = model_id
|
||||
model_response.data = embedding_data
|
||||
model_response.object = "list"
|
||||
|
||||
# Calculate token usage
|
||||
input_token_counts = raw_response_json.get("inputTextTokenCounts", [])
|
||||
total_tokens = sum(input_token_counts) if input_token_counts else 0
|
||||
|
||||
usage = Usage(
|
||||
prompt_tokens=total_tokens,
|
||||
total_tokens=total_tokens,
|
||||
)
|
||||
model_response.usage = usage
|
||||
|
||||
return model_response
|
||||
|
||||
def get_error_class(
|
||||
self,
|
||||
error_message: str,
|
||||
status_code: int,
|
||||
headers: Union[dict, httpx.Headers],
|
||||
) -> BaseLLMException:
|
||||
return OCIError(
|
||||
message=error_message,
|
||||
status_code=status_code,
|
||||
headers=headers if isinstance(headers, httpx.Headers) else None,
|
||||
)
|
||||
|
|
@ -81,26 +81,26 @@ class VertexBase:
|
|||
) -> Tuple[Any, str]:
|
||||
if credentials is not None:
|
||||
if isinstance(credentials, str):
|
||||
_is_path = os.path.exists(
|
||||
credentials
|
||||
) # credentials is from server config (litellm_params), not user input
|
||||
verbose_logger.debug(
|
||||
"Vertex: Loading vertex credentials from %s", credentials
|
||||
)
|
||||
verbose_logger.debug(
|
||||
"Vertex: checking if credentials is a valid path, os.path.exists(%s)=%s, current dir %s",
|
||||
credentials,
|
||||
os.path.exists(credentials),
|
||||
"Vertex: Loading vertex credentials, is_file_path=%s, current dir %s",
|
||||
_is_path,
|
||||
os.getcwd(),
|
||||
)
|
||||
|
||||
try:
|
||||
if os.path.exists(credentials):
|
||||
json_obj = json.load(open(credentials))
|
||||
if _is_path:
|
||||
with open(credentials) as f:
|
||||
json_obj = json.load(f)
|
||||
else:
|
||||
json_obj = json.loads(credentials)
|
||||
except Exception:
|
||||
except Exception as e:
|
||||
raise Exception(
|
||||
"Unable to load vertex credentials from environment. Got={}".format(
|
||||
credentials
|
||||
)
|
||||
"Unable to load vertex credentials from environment. "
|
||||
"Ensure the JSON is valid (check for unescaped newlines in private_key). "
|
||||
"Parse error: {}".format(type(e).__name__)
|
||||
)
|
||||
elif isinstance(credentials, dict):
|
||||
json_obj = credentials
|
||||
|
|
@ -668,8 +668,8 @@ class VertexBase:
|
|||
## VALIDATION STEP
|
||||
if _credentials.token is None or not isinstance(_credentials.token, str):
|
||||
raise ValueError(
|
||||
"Could not resolve credentials token. Got None or non-string token - {}".format(
|
||||
_credentials.token
|
||||
"Could not resolve credentials token. Got None or non-string token (type={})".format(
|
||||
type(_credentials.token).__name__
|
||||
)
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -5668,6 +5668,22 @@ def embedding( # noqa: PLR0915
|
|||
aembedding=aembedding,
|
||||
litellm_params={},
|
||||
)
|
||||
elif custom_llm_provider == "oci":
|
||||
response = base_llm_http_handler.embedding(
|
||||
model=model,
|
||||
input=input,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
api_base=api_base,
|
||||
api_key=api_key,
|
||||
logging_obj=logging,
|
||||
timeout=timeout,
|
||||
model_response=EmbeddingResponse(),
|
||||
optional_params=optional_params,
|
||||
client=client,
|
||||
aembedding=aembedding,
|
||||
litellm_params=litellm_params_dict,
|
||||
headers=headers,
|
||||
)
|
||||
elif custom_llm_provider in litellm._custom_providers:
|
||||
custom_handler: Optional[CustomLLM] = None
|
||||
for item in litellm.custom_provider_map:
|
||||
|
|
@ -7234,7 +7250,8 @@ async def ahealth_check(
|
|||
|
||||
if mode is None:
|
||||
return {
|
||||
"error": f"error:{str(e)}. Missing `mode`. Set the `mode` for the model - https://docs.litellm.ai/docs/proxy/health#embedding-models \nstacktrace: {stack_trace}"
|
||||
"error": f"error:{str(e)}. Missing `mode`. Set the `mode` for the model - https://docs.litellm.ai/docs/proxy/health#embedding-models \nstacktrace: {stack_trace}",
|
||||
"exception": e,
|
||||
}
|
||||
|
||||
error_to_return = str(e) + "\nstack trace: " + stack_trace
|
||||
|
|
@ -7246,6 +7263,7 @@ async def ahealth_check(
|
|||
return {
|
||||
"error": error_to_return,
|
||||
"raw_request_typed_dict": raw_request_typed_dict,
|
||||
"exception": e,
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -23803,7 +23803,8 @@
|
|||
"output_cost_per_token": 2e-06,
|
||||
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing",
|
||||
"supports_function_calling": true,
|
||||
"supports_response_schema": false
|
||||
"supports_response_schema": false,
|
||||
"supports_vision": true
|
||||
},
|
||||
"oci/meta.llama-3.3-70b-instruct": {
|
||||
"input_cost_per_token": 7.2e-07,
|
||||
|
|
@ -23937,6 +23938,287 @@
|
|||
"supports_function_calling": true,
|
||||
"supports_response_schema": false
|
||||
},
|
||||
"oci/cohere.command-a-reasoning-08-2025": {
|
||||
"input_cost_per_token": 1.56e-06,
|
||||
"litellm_provider": "oci",
|
||||
"max_input_tokens": 256000,
|
||||
"max_output_tokens": 4000,
|
||||
"max_tokens": 4000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.56e-06,
|
||||
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing",
|
||||
"supports_function_calling": true,
|
||||
"supports_response_schema": false
|
||||
},
|
||||
"oci/cohere.command-a-vision-07-2025": {
|
||||
"input_cost_per_token": 1.56e-06,
|
||||
"litellm_provider": "oci",
|
||||
"max_input_tokens": 128000,
|
||||
"max_output_tokens": 4000,
|
||||
"max_tokens": 4000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.56e-06,
|
||||
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing",
|
||||
"supports_function_calling": true,
|
||||
"supports_response_schema": false,
|
||||
"supports_vision": true
|
||||
},
|
||||
"oci/cohere.command-a-translate-08-2025": {
|
||||
"input_cost_per_token": 9e-08,
|
||||
"litellm_provider": "oci",
|
||||
"max_input_tokens": 256000,
|
||||
"max_output_tokens": 4000,
|
||||
"max_tokens": 4000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 9e-08,
|
||||
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing",
|
||||
"supports_function_calling": false,
|
||||
"supports_response_schema": false
|
||||
},
|
||||
"oci/cohere.command-r-08-2024": {
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
"litellm_provider": "oci",
|
||||
"max_input_tokens": 128000,
|
||||
"max_output_tokens": 4000,
|
||||
"max_tokens": 4000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.5e-07,
|
||||
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing",
|
||||
"supports_function_calling": true,
|
||||
"supports_response_schema": false
|
||||
},
|
||||
"oci/cohere.command-r-plus-08-2024": {
|
||||
"input_cost_per_token": 1.56e-06,
|
||||
"litellm_provider": "oci",
|
||||
"max_input_tokens": 128000,
|
||||
"max_output_tokens": 4000,
|
||||
"max_tokens": 4000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.56e-06,
|
||||
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing",
|
||||
"supports_function_calling": true,
|
||||
"supports_response_schema": false
|
||||
},
|
||||
"oci/meta.llama-3.2-11b-vision-instruct": {
|
||||
"input_cost_per_token": 2e-06,
|
||||
"litellm_provider": "oci",
|
||||
"max_input_tokens": 128000,
|
||||
"max_output_tokens": 4000,
|
||||
"max_tokens": 4000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2e-06,
|
||||
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing",
|
||||
"supports_function_calling": true,
|
||||
"supports_response_schema": false,
|
||||
"supports_vision": true
|
||||
},
|
||||
"oci/meta.llama-3.1-70b-instruct": {
|
||||
"input_cost_per_token": 7.2e-07,
|
||||
"litellm_provider": "oci",
|
||||
"max_input_tokens": 128000,
|
||||
"max_output_tokens": 4000,
|
||||
"max_tokens": 4000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 7.2e-07,
|
||||
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing",
|
||||
"supports_function_calling": true,
|
||||
"supports_response_schema": false
|
||||
},
|
||||
"oci/meta.llama-3.3-70b-instruct-fp8-dynamic": {
|
||||
"input_cost_per_token": 7.2e-07,
|
||||
"litellm_provider": "oci",
|
||||
"max_input_tokens": 128000,
|
||||
"max_output_tokens": 4000,
|
||||
"max_tokens": 4000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 7.2e-07,
|
||||
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing",
|
||||
"supports_function_calling": true,
|
||||
"supports_response_schema": false
|
||||
},
|
||||
"oci/xai.grok-4-fast": {
|
||||
"input_cost_per_token": 5e-06,
|
||||
"litellm_provider": "oci",
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"max_tokens": 131072,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.5e-05,
|
||||
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing",
|
||||
"supports_function_calling": true,
|
||||
"supports_response_schema": false
|
||||
},
|
||||
"oci/xai.grok-4.1-fast": {
|
||||
"input_cost_per_token": 5e-06,
|
||||
"litellm_provider": "oci",
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"max_tokens": 131072,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.5e-05,
|
||||
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing",
|
||||
"supports_function_calling": true,
|
||||
"supports_response_schema": false
|
||||
},
|
||||
"oci/xai.grok-4.20": {
|
||||
"input_cost_per_token": 3e-06,
|
||||
"litellm_provider": "oci",
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"max_tokens": 131072,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.5e-05,
|
||||
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing",
|
||||
"supports_function_calling": true,
|
||||
"supports_response_schema": false
|
||||
},
|
||||
"oci/xai.grok-4.20-multi-agent": {
|
||||
"input_cost_per_token": 3e-06,
|
||||
"litellm_provider": "oci",
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"max_tokens": 131072,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.5e-05,
|
||||
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing",
|
||||
"supports_function_calling": true,
|
||||
"supports_response_schema": false
|
||||
},
|
||||
"oci/xai.grok-code-fast-1": {
|
||||
"input_cost_per_token": 5e-06,
|
||||
"litellm_provider": "oci",
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"max_tokens": 131072,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.5e-05,
|
||||
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing",
|
||||
"supports_function_calling": true,
|
||||
"supports_response_schema": false
|
||||
},
|
||||
"oci/google.gemini-2.5-pro": {
|
||||
"input_cost_per_token": 1.25e-06,
|
||||
"litellm_provider": "oci",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 65536,
|
||||
"max_tokens": 65536,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1e-05,
|
||||
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing",
|
||||
"supports_function_calling": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"oci/google.gemini-2.5-flash": {
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
"litellm_provider": "oci",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 65536,
|
||||
"max_tokens": 65536,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 6e-07,
|
||||
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing",
|
||||
"supports_function_calling": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"oci/google.gemini-2.5-flash-lite": {
|
||||
"input_cost_per_token": 7.5e-08,
|
||||
"litellm_provider": "oci",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 65536,
|
||||
"max_tokens": 65536,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3e-07,
|
||||
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing",
|
||||
"supports_function_calling": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"oci/cohere.embed-english-v3.0": {
|
||||
"input_cost_per_token": 1e-07,
|
||||
"litellm_provider": "oci",
|
||||
"max_input_tokens": 512,
|
||||
"max_tokens": 512,
|
||||
"mode": "embedding",
|
||||
"output_cost_per_token": 0.0,
|
||||
"output_vector_size": 1024,
|
||||
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing"
|
||||
},
|
||||
"oci/cohere.embed-english-light-v3.0": {
|
||||
"input_cost_per_token": 1e-07,
|
||||
"litellm_provider": "oci",
|
||||
"max_input_tokens": 512,
|
||||
"max_tokens": 512,
|
||||
"mode": "embedding",
|
||||
"output_cost_per_token": 0.0,
|
||||
"output_vector_size": 384,
|
||||
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing"
|
||||
},
|
||||
"oci/cohere.embed-multilingual-v3.0": {
|
||||
"input_cost_per_token": 1e-07,
|
||||
"litellm_provider": "oci",
|
||||
"max_input_tokens": 512,
|
||||
"max_tokens": 512,
|
||||
"mode": "embedding",
|
||||
"output_cost_per_token": 0.0,
|
||||
"output_vector_size": 1024,
|
||||
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing"
|
||||
},
|
||||
"oci/cohere.embed-multilingual-light-v3.0": {
|
||||
"input_cost_per_token": 1e-07,
|
||||
"litellm_provider": "oci",
|
||||
"max_input_tokens": 512,
|
||||
"max_tokens": 512,
|
||||
"mode": "embedding",
|
||||
"output_cost_per_token": 0.0,
|
||||
"output_vector_size": 384,
|
||||
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing"
|
||||
},
|
||||
"oci/cohere.embed-english-image-v3.0": {
|
||||
"input_cost_per_token": 1e-07,
|
||||
"litellm_provider": "oci",
|
||||
"max_input_tokens": 512,
|
||||
"max_tokens": 512,
|
||||
"mode": "embedding",
|
||||
"output_cost_per_token": 0.0,
|
||||
"output_vector_size": 1024,
|
||||
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing",
|
||||
"supports_embedding_image_input": true
|
||||
},
|
||||
"oci/cohere.embed-english-light-image-v3.0": {
|
||||
"input_cost_per_token": 1e-07,
|
||||
"litellm_provider": "oci",
|
||||
"max_input_tokens": 512,
|
||||
"max_tokens": 512,
|
||||
"mode": "embedding",
|
||||
"output_cost_per_token": 0.0,
|
||||
"output_vector_size": 384,
|
||||
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing",
|
||||
"supports_embedding_image_input": true
|
||||
},
|
||||
"oci/cohere.embed-multilingual-light-image-v3.0": {
|
||||
"input_cost_per_token": 1e-07,
|
||||
"litellm_provider": "oci",
|
||||
"max_input_tokens": 512,
|
||||
"max_tokens": 512,
|
||||
"mode": "embedding",
|
||||
"output_cost_per_token": 0.0,
|
||||
"output_vector_size": 384,
|
||||
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing",
|
||||
"supports_embedding_image_input": true
|
||||
},
|
||||
"oci/cohere.embed-v4.0": {
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"litellm_provider": "oci",
|
||||
"max_input_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "embedding",
|
||||
"output_cost_per_token": 0.0,
|
||||
"output_vector_size": 1536,
|
||||
"source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing",
|
||||
"supports_embedding_image_input": true
|
||||
},
|
||||
"ollama/codegeex4": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "ollama",
|
||||
|
|
@ -30305,6 +30587,27 @@
|
|||
"supports_pdf_input": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"vertex_ai/claude-haiku-4-5": {
|
||||
"cache_creation_input_token_cost": 1.25e-06,
|
||||
"cache_read_input_token_cost": 1e-07,
|
||||
"input_cost_per_token": 1e-06,
|
||||
"litellm_provider": "vertex_ai-anthropic_models",
|
||||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 8192,
|
||||
"max_tokens": 8192,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 5e-06,
|
||||
"source": "https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude/haiku-4-5",
|
||||
"supports_assistant_prefill": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_native_streaming": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"vertex_ai/claude-haiku-4-5@20251001": {
|
||||
"cache_creation_input_token_cost": 1.25e-06,
|
||||
"cache_read_input_token_cost": 1e-07,
|
||||
|
|
|
|||
16
litellm/proxy/_experimental/mcp_server/mcp_context.py
Normal file
16
litellm/proxy/_experimental/mcp_server/mcp_context.py
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
"""
|
||||
Shared ContextVars for the MCP server layer.
|
||||
|
||||
Lives in its own module to avoid circular imports between
|
||||
mcp_server_manager.py and server.py.
|
||||
"""
|
||||
|
||||
from contextvars import ContextVar
|
||||
from typing import Optional
|
||||
|
||||
# Set server-side in proxy_server.py route handlers when a request arrives via
|
||||
# /toolset/{name}/mcp or the toolset fallback in dynamic_mcp_route.
|
||||
# Never populated from client-supplied headers.
|
||||
_mcp_active_toolset_id: ContextVar[Optional[str]] = ContextVar(
|
||||
"_mcp_active_toolset_id", default=None
|
||||
)
|
||||
|
|
@ -346,6 +346,8 @@ class MCPServerManager:
|
|||
aws_session_token=server_config.get("aws_session_token", None),
|
||||
aws_region_name=server_config.get("aws_region_name", None),
|
||||
aws_service_name=server_config.get("aws_service_name", None),
|
||||
aws_role_name=server_config.get("aws_role_name", None),
|
||||
aws_session_name=server_config.get("aws_session_name", None),
|
||||
)
|
||||
self.config_mcp_servers[server_id] = new_server
|
||||
|
||||
|
|
@ -686,6 +688,8 @@ class MCPServerManager:
|
|||
aws_session_token=aws_creds.get("aws_session_token"),
|
||||
aws_region_name=aws_creds.get("aws_region_name"),
|
||||
aws_service_name=aws_creds.get("aws_service_name"),
|
||||
aws_role_name=aws_creds.get("aws_role_name"),
|
||||
aws_session_name=aws_creds.get("aws_session_name"),
|
||||
)
|
||||
return new_server
|
||||
|
||||
|
|
@ -786,7 +790,18 @@ class MCPServerManager:
|
|||
f"Allowed MCP Servers for user api key auth: {allowed_mcp_servers}"
|
||||
)
|
||||
combined_servers = set(allowed_mcp_servers)
|
||||
combined_servers.update(allow_all_server_ids)
|
||||
# Only skip allow_all_keys servers when the request is inside a toolset
|
||||
# scope. toolset_mcp_route / dynamic_mcp_route set _mcp_active_toolset_id
|
||||
# before calling the handler — that ContextVar is the reliable signal.
|
||||
# Using op.mcp_toolsets==[] would false-positive on DB-default rows where
|
||||
# Postgres initialises the column to ARRAY[]::TEXT[].
|
||||
from litellm.proxy._experimental.mcp_server.mcp_context import ( # noqa: PLC0415
|
||||
_mcp_active_toolset_id,
|
||||
)
|
||||
|
||||
in_toolset_scope = _mcp_active_toolset_id.get() is not None
|
||||
if not in_toolset_scope:
|
||||
combined_servers.update(allow_all_server_ids)
|
||||
|
||||
if len(combined_servers) == 0:
|
||||
verbose_logger.debug(
|
||||
|
|
@ -797,6 +812,132 @@ class MCPServerManager:
|
|||
verbose_logger.warning(f"Failed to get allowed MCP servers: {str(e)}.")
|
||||
return allow_all_server_ids
|
||||
|
||||
async def resolve_toolset_tool_permissions(
|
||||
self,
|
||||
toolset_ids: List[str],
|
||||
) -> Dict[str, List[str]]:
|
||||
"""
|
||||
Resolve a list of toolset IDs into a mcp_tool_permissions dict.
|
||||
|
||||
Returns: {server_id: [tool_name, ...]} — the union of all tools across
|
||||
the given toolsets. Results are cached via ``user_api_key_cache`` (a
|
||||
Redis-backed ``DualCache`` in production) so that cache entries are
|
||||
shared across workers and cold-cache DB hits are minimised.
|
||||
"""
|
||||
from litellm.constants import DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL
|
||||
from litellm.proxy._experimental.mcp_server.toolset_db import list_mcp_toolsets
|
||||
from litellm.proxy.proxy_server import prisma_client, user_api_key_cache
|
||||
|
||||
if not toolset_ids or prisma_client is None:
|
||||
return {}
|
||||
|
||||
cache_key = "toolset_perms:" + ",".join(sorted(toolset_ids))
|
||||
cached = await user_api_key_cache.async_get_cache(key=cache_key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
try:
|
||||
toolsets = await list_mcp_toolsets(prisma_client, toolset_ids=toolset_ids)
|
||||
tool_permissions: Dict[str, List[str]] = {}
|
||||
for toolset in toolsets:
|
||||
for tool in toolset.tools:
|
||||
raw_name = tool["tool_name"]
|
||||
unprefixed, _ = split_server_prefix_from_name(raw_name)
|
||||
tool_permissions.setdefault(tool["server_id"], [])
|
||||
if unprefixed not in tool_permissions[tool["server_id"]]:
|
||||
tool_permissions[tool["server_id"]].append(unprefixed)
|
||||
await user_api_key_cache.async_set_cache(
|
||||
key=cache_key,
|
||||
value=tool_permissions,
|
||||
ttl=DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL,
|
||||
)
|
||||
return tool_permissions
|
||||
except Exception as e:
|
||||
verbose_logger.warning(f"Failed to resolve toolset permissions: {str(e)}")
|
||||
return {}
|
||||
|
||||
def invalidate_toolset_cache(self, toolset_id: Optional[str] = None) -> None:
|
||||
"""Evict cached toolset permission entries.
|
||||
|
||||
Called after create/update/delete of a toolset so stale data is not served.
|
||||
The in-memory layer of ``user_api_key_cache`` is cleared immediately;
|
||||
Redis entries expire naturally after the configured TTL.
|
||||
Pass toolset_id to evict only entries containing that ID, or None to clear all.
|
||||
"""
|
||||
# Clear the in-memory layer of the shared DualCache for affected keys.
|
||||
# We can't enumerate Redis keys by pattern, so Redis entries expire via TTL.
|
||||
try:
|
||||
from litellm.proxy.proxy_server import user_api_key_cache
|
||||
|
||||
in_mem = getattr(user_api_key_cache, "in_memory_cache", None)
|
||||
if in_mem is None:
|
||||
return
|
||||
cache_dict = getattr(in_mem, "cache_dict", {})
|
||||
if toolset_id is None:
|
||||
keys_to_remove = [k for k in cache_dict if k.startswith("toolset_")]
|
||||
else:
|
||||
# Evict permission-cache entries that reference this toolset ID.
|
||||
# Also evict ALL name-cache entries (toolset_name:*): we can't map
|
||||
# toolset_id → toolset_name without a DB call, and the name may have
|
||||
# changed in an update anyway.
|
||||
keys_to_remove = [
|
||||
k
|
||||
for k in cache_dict
|
||||
if (k.startswith("toolset_perms:") and toolset_id in k)
|
||||
or k.startswith("toolset_name:")
|
||||
]
|
||||
for k in keys_to_remove:
|
||||
cache_dict.pop(k, None)
|
||||
except Exception as e:
|
||||
verbose_logger.warning(
|
||||
f"invalidate_toolset_cache: failed to evict in-memory entries: {e}"
|
||||
)
|
||||
|
||||
async def get_toolset_by_name_cached(
|
||||
self,
|
||||
prisma_client: Any,
|
||||
toolset_name: str,
|
||||
) -> Optional[Any]:
|
||||
"""Return a toolset by name, cached in ``user_api_key_cache`` (Redis-backed
|
||||
``DualCache`` in production) to avoid a DB hit on every routed request.
|
||||
|
||||
Serialisation note: the cache value is stored as a plain JSON-safe dict via
|
||||
``model_dump(mode="json")`` so that Redis round-trips correctly in multi-worker
|
||||
deployments. On a cache hit we reconstruct the ``MCPToolset`` Pydantic object
|
||||
so callers can always use attribute access (e.g. ``toolset.toolset_id``).
|
||||
"""
|
||||
from litellm.constants import DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL
|
||||
from litellm.proxy.proxy_server import user_api_key_cache
|
||||
from litellm.types.mcp_server.mcp_toolset import MCPToolset
|
||||
|
||||
cache_key = f"toolset_name:{toolset_name}"
|
||||
cached = await user_api_key_cache.async_get_cache(key=cache_key)
|
||||
if cached is not None:
|
||||
# Sentinel value used to cache "not found" so we don't re-query for
|
||||
# names that don't exist.
|
||||
if cached == "__not_found__":
|
||||
return None
|
||||
# Redis deserialises JSON back as a plain dict — reconstruct the model.
|
||||
if isinstance(cached, dict):
|
||||
return MCPToolset(**cached)
|
||||
return cached
|
||||
|
||||
from litellm.proxy._experimental.mcp_server.toolset_db import (
|
||||
get_mcp_toolset_by_name,
|
||||
)
|
||||
|
||||
toolset = await get_mcp_toolset_by_name(prisma_client, toolset_name)
|
||||
await user_api_key_cache.async_set_cache(
|
||||
key=cache_key,
|
||||
value=(
|
||||
toolset.model_dump(mode="json")
|
||||
if toolset is not None
|
||||
else "__not_found__"
|
||||
),
|
||||
ttl=DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL,
|
||||
)
|
||||
return toolset
|
||||
|
||||
def filter_server_ids_by_ip(
|
||||
self, server_ids: List[str], client_ip: Optional[str]
|
||||
) -> List[str]:
|
||||
|
|
@ -1011,6 +1152,8 @@ class MCPServerManager:
|
|||
aws_session_token=server.aws_session_token,
|
||||
aws_region_name=server.aws_region_name,
|
||||
aws_service_name=server.aws_service_name,
|
||||
aws_role_name=server.aws_role_name,
|
||||
aws_session_name=server.aws_session_name,
|
||||
)
|
||||
|
||||
return MCPClient(
|
||||
|
|
@ -1071,6 +1214,24 @@ class MCPServerManager:
|
|||
tools = global_mcp_tool_registry.convert_tools_to_mcp_sdk_tool_type(
|
||||
_tools
|
||||
)
|
||||
# OpenAPI tools are stored in the registry with their prefix already
|
||||
# applied (e.g. "test_petstore-getinventory"). Do NOT pass them
|
||||
# through _create_prefixed_tools — that would add the prefix a second
|
||||
# time producing "test_petstore-test_petstore-getinventory".
|
||||
if not add_prefix:
|
||||
prefix = get_server_prefix(server)
|
||||
sep = MCP_TOOL_PREFIX_SEPARATOR
|
||||
tools = [
|
||||
(
|
||||
t.model_copy(
|
||||
update={"name": t.name[len(prefix) + len(sep) :]}
|
||||
)
|
||||
if t.name.startswith(f"{prefix}{sep}")
|
||||
else t
|
||||
)
|
||||
for t in tools
|
||||
]
|
||||
return tools
|
||||
else:
|
||||
tools = await self._fetch_tools_with_timeout(client, server.name)
|
||||
|
||||
|
|
@ -1571,6 +1732,8 @@ class MCPServerManager:
|
|||
),
|
||||
"aws_region_name": credentials_dict.get("aws_region_name"),
|
||||
"aws_service_name": credentials_dict.get("aws_service_name"),
|
||||
"aws_role_name": credentials_dict.get("aws_role_name"),
|
||||
"aws_session_name": credentials_dict.get("aws_session_name"),
|
||||
}
|
||||
|
||||
def _extract_scopes(self, scopes_value: Any) -> Optional[List[str]]:
|
||||
|
|
@ -2410,7 +2573,6 @@ class MCPServerManager:
|
|||
|
||||
async def reload_servers_from_database(self):
|
||||
"""Re-synchronize the in-memory MCP server registry with the database."""
|
||||
from litellm.proxy._experimental.mcp_server.db import get_all_mcp_servers
|
||||
from litellm.proxy.management_endpoints.mcp_management_endpoints import (
|
||||
get_prisma_client_or_throw,
|
||||
)
|
||||
|
|
@ -2421,9 +2583,19 @@ class MCPServerManager:
|
|||
prisma_client = get_prisma_client_or_throw(
|
||||
"Database not connected. Connect a database to your proxy"
|
||||
)
|
||||
db_mcp_servers = await get_all_mcp_servers(
|
||||
prisma_client, approval_status="active"
|
||||
# Load only "active", legacy "approved", and NULL (no approval workflow) rows.
|
||||
# Pending/rejected servers are excluded at the DB level so we never load them.
|
||||
from litellm.proxy._experimental.mcp_server.db import LiteLLM_MCPServerTable
|
||||
|
||||
raw_rows = await prisma_client.db.litellm_mcpservertable.find_many(
|
||||
where={
|
||||
"OR": [
|
||||
{"approval_status": None},
|
||||
{"approval_status": {"in": ["active", "approved"]}},
|
||||
]
|
||||
}
|
||||
)
|
||||
db_mcp_servers = [LiteLLM_MCPServerTable(**r.model_dump()) for r in raw_rows]
|
||||
verbose_logger.info(f"Found {len(db_mcp_servers)} MCP servers in database")
|
||||
|
||||
previous_registry = self.registry
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import (
|
|||
from litellm.proxy._experimental.mcp_server.discoverable_endpoints import (
|
||||
get_request_base_url,
|
||||
)
|
||||
from litellm.proxy._experimental.mcp_server.mcp_context import _mcp_active_toolset_id
|
||||
from litellm.proxy._experimental.mcp_server.mcp_debug import MCPDebug
|
||||
from litellm.proxy._experimental.mcp_server.utils import (
|
||||
LITELLM_MCP_SERVER_DESCRIPTION,
|
||||
|
|
@ -171,6 +172,34 @@ if MCP_AVAILABLE:
|
|||
mcp_info: Optional[MCPInfo] = None
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||
|
||||
def _normalize_resource_contents(contents: list) -> List[ReadResourceContents]:
|
||||
"""Normalize ResourceContents to ReadResourceContents, preserving meta (MCP 1.26.0+)."""
|
||||
normalized: List[ReadResourceContents] = []
|
||||
for content in contents:
|
||||
meta = getattr(content, "meta", None)
|
||||
if meta is None and hasattr(content, "model_dump"):
|
||||
d = content.model_dump()
|
||||
meta = d.get("meta")
|
||||
if meta is None:
|
||||
meta = d.get("_meta")
|
||||
if isinstance(content, TextResourceContents):
|
||||
normalized.append(
|
||||
ReadResourceContents(
|
||||
content=content.text,
|
||||
mime_type=content.mimeType,
|
||||
meta=meta,
|
||||
)
|
||||
)
|
||||
elif isinstance(content, BlobResourceContents):
|
||||
normalized.append(
|
||||
ReadResourceContents(
|
||||
content=content.blob,
|
||||
mime_type=content.mimeType,
|
||||
meta=meta,
|
||||
)
|
||||
)
|
||||
return normalized
|
||||
|
||||
########################################################
|
||||
############ Initialize the MCP Server #################
|
||||
########################################################
|
||||
|
|
@ -631,26 +660,7 @@ if MCP_AVAILABLE:
|
|||
raw_headers=raw_headers,
|
||||
)
|
||||
|
||||
normalized_contents: List[ReadResourceContents] = []
|
||||
for content in read_resource_result.contents:
|
||||
if isinstance(content, TextResourceContents):
|
||||
text_content: TextResourceContents = content
|
||||
normalized_contents.append(
|
||||
ReadResourceContents(
|
||||
content=text_content.text,
|
||||
mime_type=text_content.mimeType,
|
||||
)
|
||||
)
|
||||
elif isinstance(content, BlobResourceContents):
|
||||
blob_content: BlobResourceContents = content
|
||||
normalized_contents.append(
|
||||
ReadResourceContents(
|
||||
content=blob_content.blob,
|
||||
mime_type=None,
|
||||
)
|
||||
)
|
||||
|
||||
return normalized_contents
|
||||
return _normalize_resource_contents(read_resource_result.contents)
|
||||
|
||||
########################################################
|
||||
############ End of MCP Server Routes ##################
|
||||
|
|
@ -1456,6 +1466,49 @@ if MCP_AVAILABLE:
|
|||
|
||||
return filtered_tools
|
||||
|
||||
async def _merge_toolset_permissions(
|
||||
user_api_key_auth: Optional[UserAPIKeyAuth],
|
||||
) -> Optional[UserAPIKeyAuth]:
|
||||
"""
|
||||
Resolve mcp_toolsets on the key's object_permission into tool-level permissions
|
||||
and merge them (union) into object_permission.mcp_tool_permissions.
|
||||
|
||||
Returns the (possibly mutated copy of) user_api_key_auth.
|
||||
"""
|
||||
if user_api_key_auth is None:
|
||||
return None
|
||||
op = user_api_key_auth.object_permission
|
||||
if op is None:
|
||||
return user_api_key_auth
|
||||
toolset_ids = getattr(op, "mcp_toolsets", None) or []
|
||||
if not toolset_ids:
|
||||
return user_api_key_auth
|
||||
|
||||
toolset_perms = (
|
||||
await global_mcp_server_manager.resolve_toolset_tool_permissions(
|
||||
toolset_ids=toolset_ids
|
||||
)
|
||||
)
|
||||
if not toolset_perms:
|
||||
return user_api_key_auth
|
||||
|
||||
# Merge toolset_perms into existing mcp_tool_permissions (union)
|
||||
existing = dict(op.mcp_tool_permissions or {})
|
||||
for server_id, tool_names in toolset_perms.items():
|
||||
existing_tools = existing.get(server_id, [])
|
||||
merged = list(set(existing_tools) | set(tool_names))
|
||||
existing[server_id] = merged
|
||||
|
||||
# Build updated object_permission with merged tool permissions and server IDs.
|
||||
# Union the toolset's server IDs into mcp_servers so downstream server-level
|
||||
# filtering doesn't silently drop servers that the toolset references but that
|
||||
# aren't already in the key's explicit mcp_servers list.
|
||||
merged_servers = list(set(op.mcp_servers or []) | set(existing.keys()))
|
||||
updated_op = op.model_copy(
|
||||
update={"mcp_servers": merged_servers, "mcp_tool_permissions": existing}
|
||||
)
|
||||
return user_api_key_auth.model_copy(update={"object_permission": updated_op})
|
||||
|
||||
async def _list_mcp_tools(
|
||||
user_api_key_auth: Optional[UserAPIKeyAuth] = None,
|
||||
mcp_auth_header: Optional[str] = None,
|
||||
|
|
@ -1480,6 +1533,11 @@ if MCP_AVAILABLE:
|
|||
"""
|
||||
if not MCP_AVAILABLE:
|
||||
return []
|
||||
|
||||
# Resolve toolset permissions and merge into the key's object_permission
|
||||
# so that the existing filter_tools_by_key_team_permissions logic picks them up.
|
||||
user_api_key_auth = await _merge_toolset_permissions(user_api_key_auth)
|
||||
|
||||
# Get tools from managed MCP servers with error handling
|
||||
managed_tools = []
|
||||
try:
|
||||
|
|
@ -2359,6 +2417,63 @@ if MCP_AVAILABLE:
|
|||
]
|
||||
return False
|
||||
|
||||
async def _apply_toolset_scope(
|
||||
user_api_key_auth: UserAPIKeyAuth,
|
||||
toolset_id: str,
|
||||
) -> UserAPIKeyAuth:
|
||||
"""
|
||||
Restrict a key's MCP permissions to a single toolset.
|
||||
|
||||
When a request arrives via /toolset/{name}/mcp we override the key's
|
||||
object_permission so that only the toolset's tools are visible.
|
||||
|
||||
Raises HTTPException(403) if the key has an explicit toolset grant list
|
||||
that does not include toolset_id (i.e. mcp_toolsets is set but empty,
|
||||
or set to a list that omits this toolset). Admin keys always pass.
|
||||
"""
|
||||
from litellm.proxy._types import LiteLLM_ObjectPermissionTable
|
||||
from litellm.proxy.management_endpoints.common_utils import _user_has_admin_view
|
||||
|
||||
# Access control: non-admin keys must have this toolset in their grant list.
|
||||
# Use _user_has_admin_view so that PROXY_ADMIN_VIEW_ONLY is also treated as admin.
|
||||
is_admin = _user_has_admin_view(user_api_key_auth)
|
||||
if not is_admin:
|
||||
op = user_api_key_auth.object_permission
|
||||
granted = getattr(op, "mcp_toolsets", None) if op else None
|
||||
# granted=None → key has no explicit toolset grants → deny (same semantics as
|
||||
# fetch_mcp_toolsets which returns [] for non-admin keys with no grants configured).
|
||||
# granted=[] or list without toolset_id → also deny.
|
||||
if granted is None or toolset_id not in granted:
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail=f"API key does not have access to toolset '{toolset_id}'.",
|
||||
)
|
||||
|
||||
tool_permissions = (
|
||||
await global_mcp_server_manager.resolve_toolset_tool_permissions(
|
||||
toolset_ids=[toolset_id]
|
||||
)
|
||||
)
|
||||
server_ids = list(tool_permissions.keys())
|
||||
existing_op = user_api_key_auth.object_permission
|
||||
if existing_op is not None:
|
||||
updated_op = existing_op.model_copy(
|
||||
update={
|
||||
"mcp_servers": server_ids,
|
||||
"mcp_tool_permissions": tool_permissions,
|
||||
"mcp_toolsets": [],
|
||||
# mcp_access_groups is preserved: a key's access-group grants
|
||||
# remain valid even when the request is scoped to a single toolset.
|
||||
}
|
||||
)
|
||||
else:
|
||||
updated_op = LiteLLM_ObjectPermissionTable(
|
||||
object_permission_id="toolset-scope",
|
||||
mcp_servers=server_ids,
|
||||
mcp_tool_permissions=tool_permissions,
|
||||
)
|
||||
return user_api_key_auth.model_copy(update={"object_permission": updated_op})
|
||||
|
||||
async def handle_streamable_http_mcp(
|
||||
scope: Scope, receive: Receive, send: Send
|
||||
) -> None:
|
||||
|
|
@ -2403,6 +2518,21 @@ if MCP_AVAILABLE:
|
|||
headers={"www-authenticate": authorization_uri},
|
||||
)
|
||||
|
||||
# Strip any client-supplied x-mcp-toolset-id to prevent forgery.
|
||||
scope["headers"] = [
|
||||
(k, v)
|
||||
for k, v in scope.get("headers", [])
|
||||
if k.lower() != b"x-mcp-toolset-id"
|
||||
]
|
||||
|
||||
# Apply toolset scope if set server-side via ContextVar (set by
|
||||
# /toolset/{name}/mcp and /{name}/mcp route handlers in proxy_server.py).
|
||||
active_toolset_id = _mcp_active_toolset_id.get()
|
||||
if active_toolset_id and user_api_key_auth is not None:
|
||||
user_api_key_auth = await _apply_toolset_scope(
|
||||
user_api_key_auth, active_toolset_id
|
||||
)
|
||||
|
||||
# Inject masked debug headers when client sends x-litellm-mcp-debug: true
|
||||
_debug_headers = MCPDebug.maybe_build_debug_headers(
|
||||
raw_headers=raw_headers,
|
||||
|
|
|
|||
117
litellm/proxy/_experimental/mcp_server/toolset_db.py
Normal file
117
litellm/proxy/_experimental/mcp_server/toolset_db.py
Normal file
|
|
@ -0,0 +1,117 @@
|
|||
import json
|
||||
from typing import List, Optional
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm._uuid import uuid
|
||||
from litellm.proxy.utils import PrismaClient
|
||||
from litellm.types.mcp_server.mcp_toolset import (
|
||||
MCPToolset,
|
||||
NewMCPToolsetRequest,
|
||||
UpdateMCPToolsetRequest,
|
||||
)
|
||||
|
||||
|
||||
def _toolset_from_row(row) -> MCPToolset:
|
||||
data = row.model_dump()
|
||||
tools = data.get("tools") or []
|
||||
if isinstance(tools, str):
|
||||
tools = json.loads(tools)
|
||||
data["tools"] = tools
|
||||
return MCPToolset(**data)
|
||||
|
||||
|
||||
async def create_mcp_toolset(
|
||||
prisma_client: PrismaClient,
|
||||
data: NewMCPToolsetRequest,
|
||||
touched_by: str,
|
||||
) -> MCPToolset:
|
||||
data_dict = data.model_dump(exclude_none=True)
|
||||
data_dict["toolset_id"] = str(uuid.uuid4())
|
||||
data_dict["tools"] = json.dumps(data_dict.get("tools", []))
|
||||
data_dict["created_by"] = touched_by
|
||||
data_dict["updated_by"] = touched_by
|
||||
row = await prisma_client.db.litellm_mcptoolsettable.create(data=data_dict)
|
||||
return _toolset_from_row(row)
|
||||
|
||||
|
||||
async def get_mcp_toolset(
|
||||
prisma_client: PrismaClient,
|
||||
toolset_id: str,
|
||||
) -> Optional[MCPToolset]:
|
||||
row = await prisma_client.db.litellm_mcptoolsettable.find_unique(
|
||||
where={"toolset_id": toolset_id}
|
||||
)
|
||||
if row is None:
|
||||
return None
|
||||
return _toolset_from_row(row)
|
||||
|
||||
|
||||
async def list_mcp_toolsets(
|
||||
prisma_client: PrismaClient,
|
||||
toolset_ids: Optional[List[str]] = None,
|
||||
) -> List[MCPToolset]:
|
||||
try:
|
||||
where = {}
|
||||
if toolset_ids is not None:
|
||||
where = {"toolset_id": {"in": toolset_ids}}
|
||||
rows = await prisma_client.db.litellm_mcptoolsettable.find_many(where=where)
|
||||
return [_toolset_from_row(r) for r in rows]
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.warning(
|
||||
"litellm.proxy._experimental.mcp_server.toolset_db::list_mcp_toolsets - {}".format(
|
||||
str(e)
|
||||
)
|
||||
)
|
||||
return []
|
||||
|
||||
|
||||
async def get_mcp_toolset_by_name(
|
||||
prisma_client: PrismaClient,
|
||||
toolset_name: str,
|
||||
) -> Optional[MCPToolset]:
|
||||
row = await prisma_client.db.litellm_mcptoolsettable.find_first(
|
||||
where={"toolset_name": toolset_name}
|
||||
)
|
||||
if row is None:
|
||||
return None
|
||||
return _toolset_from_row(row)
|
||||
|
||||
|
||||
async def update_mcp_toolset(
|
||||
prisma_client: PrismaClient,
|
||||
data: UpdateMCPToolsetRequest,
|
||||
touched_by: str,
|
||||
) -> Optional[MCPToolset]:
|
||||
data_dict = data.model_dump(exclude_none=True, exclude={"toolset_id"})
|
||||
if "tools" in data_dict:
|
||||
data_dict["tools"] = json.dumps(data_dict["tools"])
|
||||
data_dict["updated_by"] = touched_by
|
||||
try:
|
||||
row = await prisma_client.db.litellm_mcptoolsettable.update(
|
||||
where={"toolset_id": data.toolset_id},
|
||||
data=data_dict,
|
||||
)
|
||||
except Exception as e:
|
||||
from prisma.errors import RecordNotFoundError
|
||||
|
||||
if isinstance(e, RecordNotFoundError):
|
||||
return None
|
||||
raise
|
||||
return _toolset_from_row(row)
|
||||
|
||||
|
||||
async def delete_mcp_toolset(
|
||||
prisma_client: PrismaClient,
|
||||
toolset_id: str,
|
||||
) -> Optional[MCPToolset]:
|
||||
try:
|
||||
row = await prisma_client.db.litellm_mcptoolsettable.delete(
|
||||
where={"toolset_id": toolset_id}
|
||||
)
|
||||
except Exception as e:
|
||||
from prisma.errors import RecordNotFoundError
|
||||
|
||||
if isinstance(e, RecordNotFoundError):
|
||||
return None
|
||||
raise
|
||||
return _toolset_from_row(row)
|
||||
File diff suppressed because one or more lines are too long
|
|
@ -1,28 +1,28 @@
|
|||
1:"$Sreact.fragment"
|
||||
2:I[347257,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"ClientPageRoot"]
|
||||
3:I[952683,["/litellm-asset-prefix/_next/static/chunks/112ad77f3dd2e3cd.js","/litellm-asset-prefix/_next/static/chunks/65f709264734a9bf.js","/litellm-asset-prefix/_next/static/chunks/e627c7aa5ead52b3.js","/litellm-asset-prefix/_next/static/chunks/142704439974f6b3.js","/litellm-asset-prefix/_next/static/chunks/30539b80ac15aad2.js","/litellm-asset-prefix/_next/static/chunks/99d715502d5069f4.js","/litellm-asset-prefix/_next/static/chunks/53a707a5829899ed.js","/litellm-asset-prefix/_next/static/chunks/1fe0596a309ad6cf.js","/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","/litellm-asset-prefix/_next/static/chunks/ed90bf177ad61e18.js","/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","/litellm-asset-prefix/_next/static/chunks/b4bd164f5553a31d.js","/litellm-asset-prefix/_next/static/chunks/adb8beb738574863.js","/litellm-asset-prefix/_next/static/chunks/b6cdb9a433f054f3.js","/litellm-asset-prefix/_next/static/chunks/af8668386d7005fe.js","/litellm-asset-prefix/_next/static/chunks/9606513e20bc3d4f.js","/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","/litellm-asset-prefix/_next/static/chunks/d512ca3b7169bef6.js","/litellm-asset-prefix/_next/static/chunks/e1e3f652dbc5be03.js","/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","/litellm-asset-prefix/_next/static/chunks/f3e0cbc0e84e0a5d.js","/litellm-asset-prefix/_next/static/chunks/8c13023d89b01566.js","/litellm-asset-prefix/_next/static/chunks/338e84191fe615bf.js","/litellm-asset-prefix/_next/static/chunks/c53c9c7afec96700.js","/litellm-asset-prefix/_next/static/chunks/5f4170980a69ffa3.js","/litellm-asset-prefix/_next/static/chunks/c74f3813068add76.js","/litellm-asset-prefix/_next/static/chunks/99109c78121231a0.js","/litellm-asset-prefix/_next/static/chunks/5929da573d876909.js","/litellm-asset-prefix/_next/static/chunks/58461a445becf104.js","/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","/litellm-asset-prefix/_next/static/chunks/d0d828f9a0668699.js","/litellm-asset-prefix/_next/static/chunks/ce9cf9f407f4b359.js","/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","/litellm-asset-prefix/_next/static/chunks/6b2bc4046c4cbfc8.js","/litellm-asset-prefix/_next/static/chunks/5400ee883dfa8c43.js","/litellm-asset-prefix/_next/static/chunks/bd94e2fe34d8a187.js","/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","/litellm-asset-prefix/_next/static/chunks/7a2dc852f68481ea.js","/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","/litellm-asset-prefix/_next/static/chunks/f9c24d6e7ec43046.js","/litellm-asset-prefix/_next/static/chunks/7c797521435cb59c.js","/litellm-asset-prefix/_next/static/chunks/5855ff7033bd4d2e.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/1da362a651d209bd.js","/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","/litellm-asset-prefix/_next/static/chunks/0b8ec8bf90ea9721.js"],"default"]
|
||||
3:I[952683,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","/litellm-asset-prefix/_next/static/chunks/9a17d35f872a6c38.js","/litellm-asset-prefix/_next/static/chunks/142704439974f6b3.js","/litellm-asset-prefix/_next/static/chunks/30539b80ac15aad2.js","/litellm-asset-prefix/_next/static/chunks/99d715502d5069f4.js","/litellm-asset-prefix/_next/static/chunks/7834a5efb7b5f959.js","/litellm-asset-prefix/_next/static/chunks/1fe0596a309ad6cf.js","/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","/litellm-asset-prefix/_next/static/chunks/a7113797b37526f0.js","/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","/litellm-asset-prefix/_next/static/chunks/db0ac43a898048e2.js","/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","/litellm-asset-prefix/_next/static/chunks/c53c9c7afec96700.js","/litellm-asset-prefix/_next/static/chunks/ee9e514b2c2694f7.js","/litellm-asset-prefix/_next/static/chunks/bb71734679762761.js","/litellm-asset-prefix/_next/static/chunks/1fd9dbe73d002173.js","/litellm-asset-prefix/_next/static/chunks/ed901fab61dc16dc.js","/litellm-asset-prefix/_next/static/chunks/ed90bf177ad61e18.js","/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","/litellm-asset-prefix/_next/static/chunks/ed4f62880278d987.js","/litellm-asset-prefix/_next/static/chunks/60d899dd52430ef8.js","/litellm-asset-prefix/_next/static/chunks/a5774cdb9f28daa1.js","/litellm-asset-prefix/_next/static/chunks/f04f887c803d9e60.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/a7dc5e0c9d37afe3.js","/litellm-asset-prefix/_next/static/chunks/86819b3a4f820602.js","/litellm-asset-prefix/_next/static/chunks/b6cdb9a433f054f3.js","/litellm-asset-prefix/_next/static/chunks/bf01d87225e5be70.js","/litellm-asset-prefix/_next/static/chunks/b3b05b76472ce110.js","/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","/litellm-asset-prefix/_next/static/chunks/47a838c67cdd745e.js","/litellm-asset-prefix/_next/static/chunks/4fc2d71e511309ab.js","/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","/litellm-asset-prefix/_next/static/chunks/360f35fe2e0a4945.js","/litellm-asset-prefix/_next/static/chunks/0d219667baa010f5.js","/litellm-asset-prefix/_next/static/chunks/7a2dc852f68481ea.js","/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","/litellm-asset-prefix/_next/static/chunks/d0d828f9a0668699.js","/litellm-asset-prefix/_next/static/chunks/f9c24d6e7ec43046.js","/litellm-asset-prefix/_next/static/chunks/7c797521435cb59c.js","/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","/litellm-asset-prefix/_next/static/chunks/53ac95bfa383e1b4.js","/litellm-asset-prefix/_next/static/chunks/169b34fe8aeee0c7.js","/litellm-asset-prefix/_next/static/chunks/2e768c2b1dfc8cd5.js"],"default"]
|
||||
18:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"OutletBoundary"]
|
||||
19:"$Sreact.suspense"
|
||||
:HL["/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","style"]
|
||||
0:{"buildId":"Hp-LQxDEAEt-JSJFExm-i","rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/e627c7aa5ead52b3.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/142704439974f6b3.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/30539b80ac15aad2.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/99d715502d5069f4.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/53a707a5829899ed.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/1fe0596a309ad6cf.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/ed90bf177ad61e18.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/b4bd164f5553a31d.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/adb8beb738574863.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/b6cdb9a433f054f3.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/af8668386d7005fe.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/9606513e20bc3d4f.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","async":true}],["$","script","script-18",{"src":"/litellm-asset-prefix/_next/static/chunks/d512ca3b7169bef6.js","async":true}],["$","script","script-19",{"src":"/litellm-asset-prefix/_next/static/chunks/e1e3f652dbc5be03.js","async":true}],["$","script","script-20",{"src":"/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","async":true}],["$","script","script-21",{"src":"/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","async":true}],["$","script","script-22",{"src":"/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","async":true}],["$","script","script-23",{"src":"/litellm-asset-prefix/_next/static/chunks/f3e0cbc0e84e0a5d.js","async":true}],["$","script","script-24",{"src":"/litellm-asset-prefix/_next/static/chunks/8c13023d89b01566.js","async":true}],["$","script","script-25",{"src":"/litellm-asset-prefix/_next/static/chunks/338e84191fe615bf.js","async":true}],["$","script","script-26",{"src":"/litellm-asset-prefix/_next/static/chunks/c53c9c7afec96700.js","async":true}],["$","script","script-27",{"src":"/litellm-asset-prefix/_next/static/chunks/5f4170980a69ffa3.js","async":true}],["$","script","script-28",{"src":"/litellm-asset-prefix/_next/static/chunks/c74f3813068add76.js","async":true}],["$","script","script-29",{"src":"/litellm-asset-prefix/_next/static/chunks/99109c78121231a0.js","async":true}],["$","script","script-30",{"src":"/litellm-asset-prefix/_next/static/chunks/5929da573d876909.js","async":true}],["$","script","script-31",{"src":"/litellm-asset-prefix/_next/static/chunks/58461a445becf104.js","async":true}],["$","script","script-32",{"src":"/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","async":true}],["$","script","script-33",{"src":"/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","async":true}],"$L6","$L7","$L8","$L9","$La","$Lb","$Lc","$Ld","$Le","$Lf","$L10","$L11","$L12","$L13","$L14","$L15","$L16"],"$L17"]}],"loading":null,"isPartial":false}
|
||||
0:{"buildId":"-9iBbUN_ohnDf0d-Ux3Ju","rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/9a17d35f872a6c38.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/142704439974f6b3.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/30539b80ac15aad2.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/99d715502d5069f4.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/7834a5efb7b5f959.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/1fe0596a309ad6cf.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/a7113797b37526f0.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/db0ac43a898048e2.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/c53c9c7afec96700.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/ee9e514b2c2694f7.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/bb71734679762761.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/1fd9dbe73d002173.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/ed901fab61dc16dc.js","async":true}],["$","script","script-18",{"src":"/litellm-asset-prefix/_next/static/chunks/ed90bf177ad61e18.js","async":true}],["$","script","script-19",{"src":"/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","async":true}],["$","script","script-20",{"src":"/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","async":true}],["$","script","script-21",{"src":"/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","async":true}],["$","script","script-22",{"src":"/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","async":true}],["$","script","script-23",{"src":"/litellm-asset-prefix/_next/static/chunks/ed4f62880278d987.js","async":true}],["$","script","script-24",{"src":"/litellm-asset-prefix/_next/static/chunks/60d899dd52430ef8.js","async":true}],["$","script","script-25",{"src":"/litellm-asset-prefix/_next/static/chunks/a5774cdb9f28daa1.js","async":true}],["$","script","script-26",{"src":"/litellm-asset-prefix/_next/static/chunks/f04f887c803d9e60.js","async":true}],["$","script","script-27",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true}],["$","script","script-28",{"src":"/litellm-asset-prefix/_next/static/chunks/a7dc5e0c9d37afe3.js","async":true}],["$","script","script-29",{"src":"/litellm-asset-prefix/_next/static/chunks/86819b3a4f820602.js","async":true}],["$","script","script-30",{"src":"/litellm-asset-prefix/_next/static/chunks/b6cdb9a433f054f3.js","async":true}],["$","script","script-31",{"src":"/litellm-asset-prefix/_next/static/chunks/bf01d87225e5be70.js","async":true}],["$","script","script-32",{"src":"/litellm-asset-prefix/_next/static/chunks/b3b05b76472ce110.js","async":true}],["$","script","script-33",{"src":"/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","async":true}],"$L6","$L7","$L8","$L9","$La","$Lb","$Lc","$Ld","$Le","$Lf","$L10","$L11","$L12","$L13","$L14","$L15","$L16"],"$L17"]}],"loading":null,"isPartial":false}
|
||||
4:{}
|
||||
5:"$0:rsc:props:children:0:props:serverProvidedParams:params"
|
||||
6:["$","script","script-34",{"src":"/litellm-asset-prefix/_next/static/chunks/d0d828f9a0668699.js","async":true}]
|
||||
7:["$","script","script-35",{"src":"/litellm-asset-prefix/_next/static/chunks/ce9cf9f407f4b359.js","async":true}]
|
||||
6:["$","script","script-34",{"src":"/litellm-asset-prefix/_next/static/chunks/47a838c67cdd745e.js","async":true}]
|
||||
7:["$","script","script-35",{"src":"/litellm-asset-prefix/_next/static/chunks/4fc2d71e511309ab.js","async":true}]
|
||||
8:["$","script","script-36",{"src":"/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","async":true}]
|
||||
9:["$","script","script-37",{"src":"/litellm-asset-prefix/_next/static/chunks/6b2bc4046c4cbfc8.js","async":true}]
|
||||
a:["$","script","script-38",{"src":"/litellm-asset-prefix/_next/static/chunks/5400ee883dfa8c43.js","async":true}]
|
||||
b:["$","script","script-39",{"src":"/litellm-asset-prefix/_next/static/chunks/bd94e2fe34d8a187.js","async":true}]
|
||||
c:["$","script","script-40",{"src":"/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","async":true}]
|
||||
d:["$","script","script-41",{"src":"/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","async":true}]
|
||||
9:["$","script","script-37",{"src":"/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","async":true}]
|
||||
a:["$","script","script-38",{"src":"/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","async":true}]
|
||||
b:["$","script","script-39",{"src":"/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","async":true}]
|
||||
c:["$","script","script-40",{"src":"/litellm-asset-prefix/_next/static/chunks/360f35fe2e0a4945.js","async":true}]
|
||||
d:["$","script","script-41",{"src":"/litellm-asset-prefix/_next/static/chunks/0d219667baa010f5.js","async":true}]
|
||||
e:["$","script","script-42",{"src":"/litellm-asset-prefix/_next/static/chunks/7a2dc852f68481ea.js","async":true}]
|
||||
f:["$","script","script-43",{"src":"/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","async":true}]
|
||||
10:["$","script","script-44",{"src":"/litellm-asset-prefix/_next/static/chunks/f9c24d6e7ec43046.js","async":true}]
|
||||
11:["$","script","script-45",{"src":"/litellm-asset-prefix/_next/static/chunks/7c797521435cb59c.js","async":true}]
|
||||
12:["$","script","script-46",{"src":"/litellm-asset-prefix/_next/static/chunks/5855ff7033bd4d2e.js","async":true}]
|
||||
13:["$","script","script-47",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true}]
|
||||
14:["$","script","script-48",{"src":"/litellm-asset-prefix/_next/static/chunks/1da362a651d209bd.js","async":true}]
|
||||
15:["$","script","script-49",{"src":"/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","async":true}]
|
||||
16:["$","script","script-50",{"src":"/litellm-asset-prefix/_next/static/chunks/0b8ec8bf90ea9721.js","async":true}]
|
||||
f:["$","script","script-43",{"src":"/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","async":true}]
|
||||
10:["$","script","script-44",{"src":"/litellm-asset-prefix/_next/static/chunks/d0d828f9a0668699.js","async":true}]
|
||||
11:["$","script","script-45",{"src":"/litellm-asset-prefix/_next/static/chunks/f9c24d6e7ec43046.js","async":true}]
|
||||
12:["$","script","script-46",{"src":"/litellm-asset-prefix/_next/static/chunks/7c797521435cb59c.js","async":true}]
|
||||
13:["$","script","script-47",{"src":"/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","async":true}]
|
||||
14:["$","script","script-48",{"src":"/litellm-asset-prefix/_next/static/chunks/53ac95bfa383e1b4.js","async":true}]
|
||||
15:["$","script","script-49",{"src":"/litellm-asset-prefix/_next/static/chunks/169b34fe8aeee0c7.js","async":true}]
|
||||
16:["$","script","script-50",{"src":"/litellm-asset-prefix/_next/static/chunks/2e768c2b1dfc8cd5.js","async":true}]
|
||||
17:["$","$L18",null,{"children":["$","$19",null,{"name":"Next.MetadataOutlet","children":"$@1a"}]}]
|
||||
1a:null
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -3,4 +3,4 @@
|
|||
3:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"MetadataBoundary"]
|
||||
4:"$Sreact.suspense"
|
||||
5:I[27201,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"IconMark"]
|
||||
0:{"buildId":"Hp-LQxDEAEt-JSJFExm-i","rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"./favicon.ico"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"loading":null,"isPartial":false}
|
||||
0:{"buildId":"-9iBbUN_ohnDf0d-Ux3Ju","rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"./favicon.ico"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"loading":null,"isPartial":false}
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
1:"$Sreact.fragment"
|
||||
2:I[867271,["/litellm-asset-prefix/_next/static/chunks/112ad77f3dd2e3cd.js","/litellm-asset-prefix/_next/static/chunks/65f709264734a9bf.js"],"default"]
|
||||
3:I[71195,["/litellm-asset-prefix/_next/static/chunks/112ad77f3dd2e3cd.js","/litellm-asset-prefix/_next/static/chunks/65f709264734a9bf.js"],"default"]
|
||||
2:I[867271,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js"],"default"]
|
||||
3:I[71195,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js"],"default"]
|
||||
4:I[339756,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
|
||||
5:I[837457,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
|
||||
:HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"]
|
||||
:HL["/litellm-asset-prefix/_next/static/chunks/cab8d46a8c32ec36.css","style"]
|
||||
0:{"buildId":"Hp-LQxDEAEt-JSJFExm-i","rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/cab8d46a8c32ec36.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/112ad77f3dd2e3cd.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/65f709264734a9bf.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]]}],"loading":null,"isPartial":false}
|
||||
:HL["/litellm-asset-prefix/_next/static/chunks/c3f387b3358b56db.css","style"]
|
||||
0:{"buildId":"-9iBbUN_ohnDf0d-Ux3Ju","rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/c3f387b3358b56db.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]]}],"loading":null,"isPartial":false}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
:HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"]
|
||||
:HL["/litellm-asset-prefix/_next/static/chunks/cab8d46a8c32ec36.css","style"]
|
||||
:HL["/litellm-asset-prefix/_next/static/chunks/c3f387b3358b56db.css","style"]
|
||||
:HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.3a6ba036.woff2","font",{"crossOrigin":"","type":"font/woff2"}]
|
||||
:HL["/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","style"]
|
||||
0:{"buildId":"Hp-LQxDEAEt-JSJFExm-i","tree":{"name":"","paramType":null,"paramKey":"","hasRuntimePrefetch":false,"slots":{"children":{"name":"__PAGE__","paramType":null,"paramKey":"__PAGE__","hasRuntimePrefetch":false,"slots":null,"isRootLayout":false}},"isRootLayout":true},"staleTime":300}
|
||||
0:{"buildId":"-9iBbUN_ohnDf0d-Ux3Ju","tree":{"name":"","paramType":null,"paramKey":"","hasRuntimePrefetch":false,"slots":{"children":{"name":"__PAGE__","paramType":null,"paramKey":"__PAGE__","hasRuntimePrefetch":false,"slots":null,"isRootLayout":false}},"isRootLayout":true},"staleTime":300}
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -1 +0,0 @@
|
|||
(globalThis.TURBOPACK||(globalThis.TURBOPACK=[])).push(["object"==typeof document?document.currentScript:void 0,526612,e=>{"use strict";var t=e.i(843476),s=e.i(846835),i=e.i(135214),r=e.i(271645),o=e.i(702597);e.s(["default",0,()=>{let{userId:e,accessToken:u,userRole:a,premiumUser:n}=(0,i.default)(),[c,l]=(0,r.useState)([]),[f,d]=(0,r.useState)([]);return(0,r.useEffect)(()=>{(0,s.fetchOrganizations)(u,l).then(()=>{})},[u]),(0,r.useEffect)(()=>{(0,o.fetchUserModels)(e,a,u,d).then(()=>{})},[e,a,u]),(0,t.jsx)(s.default,{organizations:c,userRole:a,userModels:f,accessToken:u,setOrganizations:l,premiumUser:n})}])},601236,e=>{e.v(e=>Promise.resolve().then(()=>e(764205)))}]);
|
||||
File diff suppressed because one or more lines are too long
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue