mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-10 03:28:53 +00:00
Merge branch 'main' into fix/issue-24442
This commit is contained in:
Roy Vivat
GitHub
commit
759fb983c5
851 changed files with 24927 additions and 7049 deletions
106
docs/my-website/blog/gpt_5_4_mini_nano/index.md
Normal file
106
docs/my-website/blog/gpt_5_4_mini_nano/index.md
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
---
|
||||
slug: gpt_5_4_mini_nano
|
||||
title: "Day 0 Support: GPT-5.4-mini and GPT-5.4-nano"
|
||||
date: 2026-03-17T10:00:00
|
||||
authors:
|
||||
- name: Sameer Kankute
|
||||
title: SWE @ LiteLLM (LLM Translation)
|
||||
url: https://www.linkedin.com/in/sameer-kankute/
|
||||
image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
|
||||
- name: Krrish Dholakia
|
||||
title: "CEO, LiteLLM"
|
||||
url: https://www.linkedin.com/in/krish-d/
|
||||
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
|
||||
- name: Ishaan Jaff
|
||||
title: "CTO, LiteLLM"
|
||||
url: https://www.linkedin.com/in/reffajnaahsi/
|
||||
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
|
||||
description: "GPT-5.4-mini and GPT-5.4-nano model support in LiteLLM"
|
||||
tags: [openai, gpt-5.4-mini, gpt-5.4-nano, completion]
|
||||
hide_table_of_contents: false
|
||||
---
|
||||
|
||||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
LiteLLM now supports GPT-5.4-mini and GPT-5.4-nano — cost-effective models for simple completions and high-throughput workloads.
|
||||
|
||||
:::note
|
||||
If you're on **v1.82.3-stable** or above, you don't need any update to use these models.
|
||||
:::
|
||||
|
||||
## Usage
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="proxy" label="LiteLLM Proxy">
|
||||
|
||||
**1. Setup config.yaml**
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: gpt-5.4-mini
|
||||
litellm_params:
|
||||
model: openai/gpt-5.4-mini
|
||||
api_key: os.environ/OPENAI_API_KEY
|
||||
- model_name: gpt-5.4-nano
|
||||
litellm_params:
|
||||
model: openai/gpt-5.4-nano
|
||||
api_key: os.environ/OPENAI_API_KEY
|
||||
```
|
||||
|
||||
**2. Start the proxy**
|
||||
|
||||
```bash
|
||||
litellm --config /path/to/config.yaml
|
||||
```
|
||||
|
||||
**3. Test it**
|
||||
|
||||
```bash
|
||||
# GPT-5.4-mini
|
||||
curl -X POST "http://localhost:4000/v1/chat/completions" \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer $LITELLM_KEY" \
|
||||
-d '{
|
||||
"model": "gpt-5.4-mini",
|
||||
"messages": [{"role": "user", "content": "What is the capital of France?"}]
|
||||
}'
|
||||
|
||||
# GPT-5.4-nano
|
||||
curl -X POST "http://localhost:4000/v1/chat/completions" \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer $LITELLM_KEY" \
|
||||
-d '{
|
||||
"model": "gpt-5.4-nano",
|
||||
"messages": [{"role": "user", "content": "What is 2 + 2?"}]
|
||||
}'
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="sdk" label="LiteLLM SDK">
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
|
||||
# GPT-5.4-mini
|
||||
response = completion(
|
||||
model="openai/gpt-5.4-mini",
|
||||
messages=[{"role": "user", "content": "What is the capital of France?"}],
|
||||
)
|
||||
print(response.choices[0].message.content)
|
||||
|
||||
# GPT-5.4-nano
|
||||
response = completion(
|
||||
model="openai/gpt-5.4-nano",
|
||||
messages=[{"role": "user", "content": "What is 2 + 2?"}],
|
||||
)
|
||||
print(response.choices[0].message.content)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
## Notes
|
||||
|
||||
- Both models support function calling, vision, and tool-use — see the [OpenAI provider docs](../../docs/providers/openai) for advanced usage.
|
||||
- GPT-5.4-nano is the most cost-effective option for simple tasks; GPT-5.4-mini offers a balance of speed and capability.
|
||||
|
|
@ -506,12 +506,15 @@ Request body will be in the Anthropic messages API format. **litellm follows the
|
|||
A system prompt providing context or specific instructions to the model.
|
||||
- **temperature** (number):
|
||||
Controls randomness in the model's responses. Valid range: `0 < temperature < 1`.
|
||||
- **thinking** (object):
|
||||
- **thinking** (object):
|
||||
Configuration for enabling extended thinking. If enabled, it includes:
|
||||
- **budget_tokens** (integer):
|
||||
- **budget_tokens** (integer):
|
||||
Minimum of 1024 tokens (and less than `max_tokens`).
|
||||
- **type** (enum):
|
||||
- **type** (enum):
|
||||
E.g., `"enabled"`.
|
||||
- **summary** (string, optional):
|
||||
Enables the summary style for thinking blocks. Possible values: `"auto"`, `"concise"`, `"detailed"`, `"disabled"`.
|
||||
When routing to non-Anthropic providers (e.g., `openai/gpt-5.1`), the `summary` value is preserved and forwarded to the downstream API.
|
||||
- **tool_choice** (object):
|
||||
Instructs how the model should utilize any provided tools.
|
||||
- **tools** (array of objects):
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@ import TabItem from '@theme/TabItem';
|
|||
Supported Providers:
|
||||
- OpenAI (`openai/`)
|
||||
- Anthropic API (`anthropic/`)
|
||||
- Google AI Studio (`gemini/`)
|
||||
- Vertex AI (`vertex_ai/`, `vertex_ai_beta/`)
|
||||
- Bedrock (`bedrock/`, `bedrock/invoke/`, `bedrock/converse`) ([All models bedrock supports prompt caching on](https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-caching.html))
|
||||
- Deepseek API (`deepseek/`)
|
||||
|
||||
|
|
@ -257,7 +259,7 @@ Anthropic charges for cache writes.
|
|||
|
||||
Specify the content to cache with `"cache_control": {"type": "ephemeral"}`.
|
||||
|
||||
If you pass that in for any other llm provider, it will be ignored.
|
||||
This same format also works for [Gemini / Vertex AI](#google-ai-studio--vertex-ai-gemini-example). For other providers, it will be ignored.
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="sdk" label="SDK">
|
||||
|
|
@ -356,6 +358,208 @@ print(response.usage)
|
|||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
### Google AI Studio / Vertex AI (Gemini) Example
|
||||
|
||||
Use the same Anthropic-style `cache_control` format — LiteLLM automatically translates it to Google's [context caching API](https://ai.google.dev/api/caching).
|
||||
|
||||
**How it works under the hood:**
|
||||
1. Messages with `cache_control` are separated and sent to Google's `cachedContents` API
|
||||
2. The cached content ID is then passed as `cachedContent` in the Gemini request body
|
||||
3. Works across all three providers: `gemini/` (Google AI Studio), `vertex_ai/`, and `vertex_ai_beta/`
|
||||
4. Requires a minimum of **1024 tokens** in the cached content — below that, caching is silently skipped
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="sdk" label="SDK">
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
import os
|
||||
|
||||
os.environ["GEMINI_API_KEY"] = ""
|
||||
|
||||
response = completion(
|
||||
model="gemini/gemini-2.5-flash",
|
||||
messages=[
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "You are an AI assistant tasked with analyzing legal documents.",
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Here is the full text of a complex legal agreement" * 400,
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "what are the key terms and conditions in this agreement?",
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
print(response.usage)
|
||||
```
|
||||
</TabItem>
|
||||
<TabItem value="proxy" label="PROXY">
|
||||
|
||||
1. Setup config.yaml
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: gemini-2.5-flash
|
||||
litellm_params:
|
||||
model: gemini/gemini-2.5-flash
|
||||
api_key: os.environ/GEMINI_API_KEY
|
||||
```
|
||||
|
||||
2. Start proxy
|
||||
|
||||
```bash
|
||||
litellm --config /path/to/config.yaml
|
||||
```
|
||||
|
||||
3. Test it!
|
||||
|
||||
```python
|
||||
from openai import OpenAI
|
||||
|
||||
client = OpenAI(
|
||||
api_key="LITELLM_PROXY_KEY", # sk-1234
|
||||
base_url="LITELLM_PROXY_BASE", # http://0.0.0.0:4000
|
||||
)
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model="gemini-2.5-flash",
|
||||
messages=[
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "You are an AI assistant tasked with analyzing legal documents.",
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Here is the full text of a complex legal agreement" * 400,
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "what are the key terms and conditions in this agreement?",
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
print(response.usage)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
#### Vertex AI
|
||||
|
||||
For Vertex AI, use `vertex_ai/` prefix:
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="sdk" label="SDK">
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
|
||||
response = completion(
|
||||
model="vertex_ai/gemini-2.5-flash",
|
||||
vertex_project="my-gcp-project",
|
||||
vertex_location="us-central1",
|
||||
messages=[
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "You are an AI assistant tasked with analyzing legal documents.",
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Here is the full text of a complex legal agreement" * 400,
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "what are the key terms and conditions in this agreement?",
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
print(response.usage)
|
||||
```
|
||||
</TabItem>
|
||||
<TabItem value="proxy" label="PROXY">
|
||||
|
||||
1. Setup config.yaml
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: gemini-2.5-flash
|
||||
litellm_params:
|
||||
model: vertex_ai/gemini-2.5-flash
|
||||
vertex_project: my-gcp-project
|
||||
vertex_location: us-central1
|
||||
```
|
||||
|
||||
2. Start proxy
|
||||
|
||||
```bash
|
||||
litellm --config /path/to/config.yaml
|
||||
```
|
||||
|
||||
3. Test it!
|
||||
|
||||
```python
|
||||
from openai import OpenAI
|
||||
|
||||
client = OpenAI(
|
||||
api_key="LITELLM_PROXY_KEY", # sk-1234
|
||||
base_url="LITELLM_PROXY_BASE", # http://0.0.0.0:4000
|
||||
)
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model="gemini-2.5-flash",
|
||||
messages=[
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "You are an AI assistant tasked with analyzing legal documents.",
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Here is the full text of a complex legal agreement" * 400,
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "what are the key terms and conditions in this agreement?",
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
print(response.usage)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
### Deepeek Example
|
||||
|
||||
Works the same as OpenAI.
|
||||
|
|
|
|||
48
docs/my-website/docs/prompt_management.md
Normal file
48
docs/my-website/docs/prompt_management.md
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
---
|
||||
title: Prompt Management with Responses API
|
||||
---
|
||||
|
||||
# Prompt Management with Responses API
|
||||
|
||||
Use LiteLLM Prompt Management with `/v1/responses` by passing `prompt_id` and optional `prompt_variables`.
|
||||
|
||||
## Basic Usage
|
||||
|
||||
```bash
|
||||
curl -X POST "http://localhost:4000/v1/responses" \
|
||||
-H "Authorization: Bearer sk-1234" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "gpt-4o",
|
||||
"prompt_id": "my-responses-prompt",
|
||||
"prompt_variables": {"topic": "large language models"},
|
||||
"input": []
|
||||
}'
|
||||
```
|
||||
|
||||
## Multi-turn Follow-up in `input`
|
||||
|
||||
To send follow-up turns in one request, pass message history in `input`.
|
||||
|
||||
```bash
|
||||
curl -X POST "http://localhost:4000/v1/responses" \
|
||||
-H "Authorization: Bearer sk-1234" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "gpt-4o",
|
||||
"prompt_id": "my-responses-prompt",
|
||||
"prompt_variables": {"topic": "large language models"},
|
||||
"input": [
|
||||
{"role": "user", "content": "Topic is LLMs. Start short."},
|
||||
{"role": "assistant", "content": "Sure, go ahead."},
|
||||
{"role": "user", "content": "Now give me 3 bullets and include pricing caveat."}
|
||||
]
|
||||
}'
|
||||
```
|
||||
|
||||
## Notes
|
||||
|
||||
- Prompt template messages are merged with your `input` messages.
|
||||
- Prompt variable substitution applies to prompt message content.
|
||||
- Tool call payload fields are not substituted by prompt variables.
|
||||
- For follow-ups with `previous_response_id`, include `prompt_id` again if you want prompt management applied on that turn.
|
||||
|
|
@ -54,6 +54,7 @@ response = completion(
|
|||
- stream
|
||||
- tools
|
||||
- tool_choice
|
||||
- include_server_side_tool_invocations
|
||||
- functions
|
||||
- response_format
|
||||
- n
|
||||
|
|
@ -856,7 +857,112 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
|
|||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
### URL Context
|
||||
### Context Circulation (Server-Side Tool Combination)
|
||||
|
||||
Context circulation allows Gemini 3+ models to combine **built-in tools** (like Google Search) with **your custom functions** in the same request. Without it, Gemini returns an error if you try to use both.
|
||||
|
||||
When enabled, Gemini can execute Google Search server-side, use those results to decide whether to call your custom functions, and return the full chain of reasoning.
|
||||
|
||||
**How it works:**
|
||||
1. You pass `include_server_side_tool_invocations=True` along with both Google Search and your function tools
|
||||
2. Gemini executes server-side tools internally and returns `toolCall`/`toolResponse` parts alongside any `functionCall` parts
|
||||
3. LiteLLM extracts the server-side invocations into `provider_specific_fields["server_side_tool_invocations"]`
|
||||
4. On subsequent turns, include the full assistant message in your conversation history — LiteLLM re-injects the server-side parts automatically
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="sdk" label="SDK">
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
|
||||
response = completion(
|
||||
model="gemini/gemini-3-flash-preview",
|
||||
messages=[{"role": "user", "content": "What's the weather in Buenos Aires? If it's raining, schedule a meeting."}],
|
||||
tools=[
|
||||
{"type": "web_search_preview"}, # Google Search (server-side)
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "schedule_meeting",
|
||||
"description": "Schedule a meeting",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"reason": {"type": "string"}},
|
||||
"required": ["reason"],
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
include_server_side_tool_invocations=True,
|
||||
)
|
||||
|
||||
msg = response.choices[0].message
|
||||
|
||||
# Server-side tool results are in provider_specific_fields
|
||||
psf = msg.provider_specific_fields or {}
|
||||
for invocation in psf.get("server_side_tool_invocations", []):
|
||||
print(invocation["tool_type"]) # e.g. "GOOGLE_SEARCH_WEB"
|
||||
print(invocation["id"])
|
||||
print(invocation["args"]) # e.g. {"queries": ["weather Buenos Aires"]}
|
||||
print(invocation["response"]) # Search results from Google
|
||||
|
||||
# For multi-turn: just append the full message to history
|
||||
messages.append(msg)
|
||||
messages.append({"role": "user", "content": "Thanks!"})
|
||||
# LiteLLM automatically re-injects the server-side parts + thought signatures
|
||||
response2 = completion(
|
||||
model="gemini/gemini-3-flash-preview",
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
include_server_side_tool_invocations=True,
|
||||
)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="proxy" label="PROXY">
|
||||
|
||||
1. Setup config.yaml
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: gemini-3-flash
|
||||
litellm_params:
|
||||
model: gemini/gemini-3-flash-preview
|
||||
api_key: os.environ/GEMINI_API_KEY
|
||||
```
|
||||
|
||||
2. Start Proxy
|
||||
```bash
|
||||
$ litellm --config /path/to/config.yaml
|
||||
```
|
||||
|
||||
3. Make Request
|
||||
```bash
|
||||
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H 'Authorization: Bearer sk-1234' \
|
||||
-d '{
|
||||
"model": "gemini-3-flash",
|
||||
"messages": [{"role": "user", "content": "What is the weather in Buenos Aires?"}],
|
||||
"tools": [
|
||||
{"type": "web_search_preview"},
|
||||
{"type": "function", "function": {"name": "schedule_meeting", "description": "Schedule a meeting", "parameters": {"type": "object", "properties": {"reason": {"type": "string"}}}}}
|
||||
],
|
||||
"include_server_side_tool_invocations": true
|
||||
}'
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
:::info
|
||||
|
||||
- Context circulation requires **Gemini 3+** models
|
||||
- Server-side tool invocations (`toolCall`/`toolResponse`) are **not** included in `tool_calls` — they are in `provider_specific_fields["server_side_tool_invocations"]` because they were already executed by Google, not by your code
|
||||
- `thought_signatures` are automatically preserved alongside server-side invocations for multi-turn coherence
|
||||
|
||||
:::
|
||||
|
||||
### URL Context
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="sdk" label="SDK">
|
||||
|
|
|
|||
|
|
@ -361,8 +361,9 @@ router_settings:
|
|||
| redis_url | str | URL for Redis server. **Known performance issue with Redis URL.** |
|
||||
| cache_responses | boolean | Flag to enable caching LLM Responses, if cache set under `router_settings`. If true, caches responses. Defaults to False. |
|
||||
| router_general_settings | RouterGeneralSettings | [SDK-Only] Router general settings - contains optimizations like 'async_only_mode'. [Docs](../routing.md#router-general-settings) |
|
||||
| optional_pre_call_checks | List[str] | List of pre-call checks to add to the router. Supported: `router_budget_limiting`, `prompt_caching`, `responses_api_deployment_check`, `encrypted_content_affinity`, `deployment_affinity`, `session_affinity`, `forward_client_headers_by_model_group` |
|
||||
| optional_pre_call_checks | List[str] | List of pre-call checks to add to the router. Supported: `router_budget_limiting`, `prompt_caching`, `responses_api_deployment_check`, `encrypted_content_affinity` (requires LiteLLM >= 1.82.3), `deployment_affinity`, `session_affinity`, `forward_client_headers_by_model_group` |
|
||||
| deployment_affinity_ttl_seconds | int | TTL (seconds) for user-key → deployment affinity mapping when `deployment_affinity` is enabled (configured at Router init / proxy startup). Defaults to `3600` (1 hour). |
|
||||
| model_group_affinity_config | Dict[str, List[str]] | Per-model-group affinity flags. Keys are model group names; values are lists of checks to enable (`deployment_affinity`, `responses_api_deployment_check`, `session_affinity`). Groups not listed fall back to the global `optional_pre_call_checks`. [Docs](../response_api.md#per-model-group-affinity-configuration) |
|
||||
| ignore_invalid_deployments | boolean | If true, ignores invalid deployments. Default for proxy is True - to prevent invalid models from blocking other models from being loaded. |
|
||||
| search_tools | List[SearchToolTypedDict] | List of search tool configurations for Search API integration. Each tool specifies a search_tool_name and litellm_params with search_provider, api_key, api_base, etc. [Further Docs](../search/index.md) |
|
||||
| guardrail_list | List[GuardrailTypedDict] | List of guardrail configurations for guardrail load balancing. Enables load balancing across multiple guardrail deployments with the same guardrail_name. [Further Docs](./guardrails/guardrail_load_balancing.md) |
|
||||
|
|
@ -813,7 +814,7 @@ router_settings:
|
|||
| LITELLM_MODE | Operating mode for LiteLLM (e.g., production, development)
|
||||
| LITELLM_NON_ROOT | Flag to run LiteLLM in non-root mode for enhanced security in Docker containers
|
||||
| LITELLM_RATE_LIMIT_WINDOW_SIZE | Rate limit window size for LiteLLM. Default is 60
|
||||
| LITELLM_REASONING_AUTO_SUMMARY | If set to "true", automatically enables detailed reasoning summaries for reasoning models (e.g., o1, o3-mini, deepseek-reasoner). When enabled, adds `summary: "detailed"` to reasoning effort configurations. Default is "false"
|
||||
| LITELLM_REASONING_AUTO_SUMMARY | If set to "true", automatically enables detailed reasoning summaries (`summary: "detailed"`) for reasoning models across all translation paths (Anthropic adapter, Responses API, etc.). Default is "false"
|
||||
| LITELLM_SALT_KEY | Salt key for encryption in LiteLLM
|
||||
| LITELLM_SSL_CIPHERS | SSL/TLS cipher configuration for faster handshakes. Controls cipher suite preferences for OpenSSL connections.
|
||||
| LITELLM_SECRET_AWS_KMS_LITELLM_LICENSE | AWS KMS encrypted license for LiteLLM
|
||||
|
|
@ -952,6 +953,8 @@ router_settings:
|
|||
| QDRANT_URL | Connection URL for Qdrant database
|
||||
| QDRANT_VECTOR_SIZE | Vector size for Qdrant operations. Default is 1536
|
||||
| REDIS_CONNECTION_POOL_TIMEOUT | Timeout in seconds for Redis connection pool. Default is 5
|
||||
| REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD | Number of consecutive failures before the Redis circuit breaker opens. Default is 5
|
||||
| REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT | Time in seconds before the Redis circuit breaker attempts recovery after opening. Default is 60
|
||||
| REDIS_CLUSTER_NODES | JSON-formatted list of Redis cluster startup nodes for Redis Cluster mode. Example: `[{"host": "node1", "port": 6379}]`
|
||||
| REDIS_HOST | Hostname for Redis server
|
||||
| REDIS_PASSWORD | Password for Redis service
|
||||
|
|
|
|||
139
docs/my-website/docs/proxy/guardrails/akto.md
Normal file
139
docs/my-website/docs/proxy/guardrails/akto.md
Normal file
|
|
@ -0,0 +1,139 @@
|
|||
# Akto
|
||||
|
||||
## Overview
|
||||
[Akto](https://www.akto.io/) provides API security guardrails and data ingestion for LLM traffic.
|
||||
|
||||
Akto now uses a **two-entry guardrail pattern** in LiteLLM:
|
||||
- `akto-validate` (`pre_call`) for request validation
|
||||
- `akto-ingest` (`post_call`) for request/response ingestion
|
||||
|
||||
There is no `on_flagged` setting anymore.
|
||||
|
||||
Use these as two separate guardrails in `config.yaml`:
|
||||
- `guardrail_name: "akto-validate"`
|
||||
- `guardrail_name: "akto-ingest"`
|
||||
|
||||
## 1. Get Your Akto Credentials
|
||||
|
||||
Set up the Akto Guardrail API Service and grab:
|
||||
- `AKTO_GUARDRAIL_API_BASE` — your Guardrail API Base URL
|
||||
- `AKTO_API_KEY` — your API key
|
||||
|
||||
## 2. Configure in `config.yaml`
|
||||
|
||||
### Block + Ingest (recommended)
|
||||
|
||||
Use both entries below. This gives you:
|
||||
- pre-call block decision
|
||||
- post-call ingestion for allowed traffic
|
||||
|
||||
Keep these as two separate entries (`akto-validate` and `akto-ingest`).
|
||||
|
||||
```yaml
|
||||
guardrails:
|
||||
- guardrail_name: "akto-validate"
|
||||
litellm_params:
|
||||
guardrail: akto
|
||||
mode: pre_call
|
||||
akto_base_url: os.environ/AKTO_GUARDRAIL_API_BASE
|
||||
akto_api_key: os.environ/AKTO_API_KEY
|
||||
default_on: true
|
||||
unreachable_fallback: fail_closed # optional: fail_open | fail_closed (default: fail_closed)
|
||||
guardrail_timeout: 5 # optional, default: 5
|
||||
akto_account_id: "1000000" # optional, env fallback: AKTO_ACCOUNT_ID
|
||||
akto_vxlan_id: "0" # optional, env fallback: AKTO_VXLAN_ID
|
||||
|
||||
- guardrail_name: "akto-ingest"
|
||||
litellm_params:
|
||||
guardrail: akto
|
||||
mode: post_call
|
||||
akto_base_url: os.environ/AKTO_GUARDRAIL_API_BASE
|
||||
akto_api_key: os.environ/AKTO_API_KEY
|
||||
default_on: true
|
||||
```
|
||||
|
||||
### Monitor-only mode
|
||||
|
||||
If you only want logging/ingestion and no blocking, keep only `akto-ingest`.
|
||||
|
||||
```yaml
|
||||
guardrails:
|
||||
- guardrail_name: "akto-ingest"
|
||||
litellm_params:
|
||||
guardrail: akto
|
||||
mode: post_call
|
||||
akto_base_url: os.environ/AKTO_GUARDRAIL_API_BASE
|
||||
akto_api_key: os.environ/AKTO_API_KEY
|
||||
default_on: true
|
||||
```
|
||||
|
||||
## 3. Test It
|
||||
|
||||
```shell
|
||||
curl -i http://localhost:4000/v1/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer <your litellm key>" \
|
||||
-d '{
|
||||
"model": "gpt-3.5-turbo",
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello, how are you?"}
|
||||
]
|
||||
}'
|
||||
```
|
||||
|
||||
If a request gets blocked:
|
||||
|
||||
```json
|
||||
{
|
||||
"error": {
|
||||
"message": "Prompt injection detected",
|
||||
"type": "None",
|
||||
"param": "None",
|
||||
"code": "403"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## 4. How It Works
|
||||
|
||||
**Block + Ingest mode:**
|
||||
```
|
||||
Request → LiteLLM → Akto guardrail check
|
||||
→ Allowed → forward to LLM → ingest response
|
||||
→ Blocked → ingest blocked marker → 403 error
|
||||
```
|
||||
|
||||
**Monitor-only mode:**
|
||||
```
|
||||
Request → LiteLLM → forward to LLM → get response
|
||||
→ Send to Akto (guardrails + ingest) → log only
|
||||
```
|
||||
|
||||
## 5. Event behavior
|
||||
|
||||
| Entry | LiteLLM hook | Akto call behavior |
|
||||
|------|---|---|
|
||||
| `akto-validate` | `pre_call` | Awaited call with `guardrails=true`, `ingest_data=false` |
|
||||
| `akto-ingest` | `post_call` | Fire-and-forget call with `guardrails=true`, `ingest_data=true` |
|
||||
|
||||
When blocked in `pre_call`, LiteLLM sends one fire-and-forget ingest payload with blocked metadata and returns `403`.
|
||||
|
||||
## 6. Parameters
|
||||
|
||||
| Parameter | Env Variable | Default | Description |
|
||||
|-----------|-------------|---------|-------------|
|
||||
| `akto_base_url` | `AKTO_GUARDRAIL_API_BASE` | *required* | Akto Guardrail API Base URL |
|
||||
| `akto_api_key` | `AKTO_API_KEY` | *required* | API key (sent as `Authorization` header) |
|
||||
| `akto_account_id` | `AKTO_ACCOUNT_ID` | `1000000` | Akto account id included in payload |
|
||||
| `akto_vxlan_id` | `AKTO_VXLAN_ID` | `0` | Akto vxlan id included in payload |
|
||||
| `unreachable_fallback` | — | `fail_closed` | `fail_open` or `fail_closed` |
|
||||
| `guardrail_timeout` | — | `5` | Timeout in seconds |
|
||||
| `default_on` | — | `true` (recommended) | Enables the guardrail entry by default |
|
||||
|
||||
## 7. Error Handling
|
||||
|
||||
| Scenario | `fail_closed` (default) | `fail_open` |
|
||||
|----------|------------------------|-------------|
|
||||
| Akto unreachable | ❌ Blocked (503) | ✅ Passes through |
|
||||
| Akto returns error | ❌ Blocked (503) | ✅ Passes through |
|
||||
| Guardrail says no | ❌ Blocked (403) | ❌ Blocked (403) |
|
||||
|
|
@ -117,6 +117,14 @@ guardrails:
|
|||
|
||||
:::
|
||||
|
||||
:::note Streaming and post_call guardrails
|
||||
|
||||
For **streaming responses**, `post_call` guardrails run on the fully assembled response **after** all chunks have been delivered to the client. This means `post_call` guardrails on streaming are **audit-only** — they can inspect and log the complete response, but cannot block content delivery. Guardrail results are recorded in `guardrail_information` within the logging payload for compliance and auditing.
|
||||
|
||||
To filter or block streaming content in real-time, use `async_post_call_streaming_iterator_hook` instead, which processes chunks as they arrive.
|
||||
|
||||
:::
|
||||
|
||||
<details>
|
||||
<summary>Advanced: Multiple modes with individual event hooks</summary>
|
||||
|
||||
|
|
@ -655,8 +663,8 @@ class myCustomGuardrail(CustomGuardrail):
|
|||
| `apply_guardrail` | Simple method to check and optionally modify text | ✅ | INPUT or OUTPUT | ✅ | ✅ | ✅ |
|
||||
| `async_pre_call_hook` | A hook that runs before the LLM API call | ✅ | INPUT | ✅ | ❌ | ✅ |
|
||||
| `async_moderation_hook` | A hook that runs during the LLM API call| ✅ | INPUT | ❌ | ❌ | ✅ |
|
||||
| `async_post_call_success_hook` | A hook that runs after a successful LLM API call| ✅ | INPUT, OUTPUT | ❌ | ✅ | ✅ |
|
||||
| `async_post_call_streaming_iterator_hook` | A hook that processes streaming responses | ✅ | OUTPUT | ❌ | ✅ | ✅ |
|
||||
| `async_post_call_success_hook` | A hook that runs after a successful LLM API call. For streaming, runs on the assembled response after delivery (audit-only, cannot block). | ✅ | INPUT, OUTPUT | ❌ | ✅ | ✅ (non-streaming only) |
|
||||
| `async_post_call_streaming_iterator_hook` | A hook that processes streaming responses in real-time (can filter/block chunks) | ✅ | OUTPUT | ❌ | ✅ | ✅ |
|
||||
|
||||
|
||||
## Frequently Asked Questions
|
||||
|
|
|
|||
190
docs/my-website/docs/proxy/high_availability_control_plane.md
Normal file
190
docs/my-website/docs/proxy/high_availability_control_plane.md
Normal file
|
|
@ -0,0 +1,190 @@
|
|||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
import { ControlPlaneArchitecture } from '@site/src/components/ControlPlaneArchitecture';
|
||||
|
||||
# [BETA] High Availability Control Plane
|
||||
|
||||
Deploy a single LiteLLM UI that manages multiple independent LiteLLM proxy instances, each with its own database, Redis, and master key.
|
||||
|
||||
:::info
|
||||
|
||||
This is an Enterprise feature.
|
||||
|
||||
[Enterprise Pricing](https://www.litellm.ai/#pricing)
|
||||
|
||||
[Get free 7-day trial key](https://www.litellm.ai/enterprise#trial)
|
||||
|
||||
:::
|
||||
|
||||
## Why This Architecture?
|
||||
|
||||
In the [standard multi-region setup](./control_plane_and_data_plane.md), all instances share a single database and master key. This works, but introduces a shared dependency. If the database goes down, every instance is affected.
|
||||
|
||||
The **High Availability Control Plane** takes a different approach:
|
||||
|
||||
| | Shared Database (Standard) | High Availability Control Plane |
|
||||
|---|---|---|
|
||||
| **Database** | Single shared DB for all instances | Each instance has its own DB |
|
||||
| **Redis** | Shared Redis | Each instance has its own Redis |
|
||||
| **Master Key** | Same key across all instances | Each instance has its own key |
|
||||
| **Failure isolation** | DB outage affects all instances | Failure is isolated to one instance |
|
||||
| **User management** | Centralized, one user table | Independent, each worker manages its own users |
|
||||
| **UI** | One UI per admin instance | Single control plane UI manages all workers |
|
||||
|
||||
### Benefits
|
||||
|
||||
- **True high availability**: no shared infrastructure means no single point of failure
|
||||
- **Blast radius containment**: a misconfiguration or outage on one worker doesn't affect others
|
||||
- **Regional isolation**: workers can run in different regions with data residency requirements
|
||||
- **Simpler operations**: each worker is a self-contained LiteLLM deployment
|
||||
|
||||
## Architecture
|
||||
|
||||
<ControlPlaneArchitecture />
|
||||
|
||||
The **control plane** is a LiteLLM instance that serves the admin UI and knows about all the workers. It does not proxy LLM requests, it is purely for administration.
|
||||
|
||||
Each **worker** is a fully independent LiteLLM proxy that handles LLM requests for its region or team. Workers have their own users, keys, teams, and budgets.
|
||||
|
||||
## Setup
|
||||
|
||||
### 1. Control Plane Configuration
|
||||
|
||||
The control plane needs a `worker_registry` that lists all worker instances.
|
||||
|
||||
```yaml title="cp_config.yaml"
|
||||
model_list: []
|
||||
|
||||
general_settings:
|
||||
master_key: sk-1234
|
||||
database_url: os.environ/DATABASE_URL
|
||||
|
||||
worker_registry:
|
||||
- worker_id: "worker-a"
|
||||
name: "Worker A"
|
||||
url: "http://localhost:4001"
|
||||
- worker_id: "worker-b"
|
||||
name: "Worker B"
|
||||
url: "http://localhost:4002"
|
||||
```
|
||||
|
||||
Start the control plane:
|
||||
|
||||
```bash
|
||||
litellm --config cp_config.yaml --port 4000
|
||||
```
|
||||
|
||||
### 2. Worker Configuration
|
||||
|
||||
Each worker needs `control_plane_url` in its `general_settings` to enable cross-origin authentication from the control plane UI.
|
||||
|
||||
`PROXY_BASE_URL` must also be set for each worker so that SSO callback redirects resolve correctly.
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="worker-a" label="Worker A">
|
||||
|
||||
```yaml title="worker_a_config.yaml"
|
||||
model_list: []
|
||||
|
||||
general_settings:
|
||||
master_key: sk-worker-a-1234
|
||||
database_url: os.environ/WORKER_A_DATABASE_URL
|
||||
control_plane_url: "http://localhost:4000"
|
||||
```
|
||||
|
||||
```bash
|
||||
PROXY_BASE_URL=http://localhost:4001 litellm --config worker_a_config.yaml --port 4001
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="worker-b" label="Worker B">
|
||||
|
||||
```yaml title="worker_b_config.yaml"
|
||||
model_list: []
|
||||
|
||||
general_settings:
|
||||
master_key: sk-worker-b-1234
|
||||
database_url: os.environ/WORKER_B_DATABASE_URL
|
||||
control_plane_url: "http://localhost:4000"
|
||||
```
|
||||
|
||||
```bash
|
||||
PROXY_BASE_URL=http://localhost:4002 litellm --config worker_b_config.yaml --port 4002
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
:::important
|
||||
Each worker must have its own `master_key` and `database_url`. The whole point of this architecture is that workers are independent.
|
||||
:::
|
||||
|
||||
### 3. SSO Configuration (Optional)
|
||||
|
||||
SSO is configured on the **control plane** instance the same way as a standard LiteLLM proxy. See the [SSO setup guide](./admin_ui_sso.md) for full instructions.
|
||||
|
||||
If using SSO, make sure to register each worker URL and the control plane URL as allowed callback URLs in your SSO provider's dashboard.
|
||||
|
||||
## How It Works
|
||||
|
||||
### Login Flow
|
||||
|
||||
1. User visits the control plane UI (`http://localhost:4000/ui`)
|
||||
2. The login page shows a **worker selector** dropdown listing all registered workers
|
||||
3. User selects a worker (e.g. "Worker A") and logs in with username/password or SSO
|
||||
4. The UI authenticates against the **selected worker** using the `/v3/login` endpoint
|
||||
5. On success, the UI stores the worker's JWT and points all subsequent API calls at the worker
|
||||
6. The user can now manage keys, teams, models, and budgets on that worker, all from the control plane UI
|
||||
|
||||
### Switching Workers
|
||||
|
||||
Once logged in, users can switch workers from the **navbar dropdown** without leaving the UI. Switching redirects back to the login page to authenticate against the new worker.
|
||||
|
||||
### Discovery
|
||||
|
||||
The control plane exposes a `/.well-known/litellm-ui-config` endpoint that the UI reads on load. This endpoint returns:
|
||||
- `is_control_plane: true`
|
||||
- The list of workers with their IDs, names, and URLs
|
||||
|
||||
This is how the login page knows to show the worker selector.
|
||||
|
||||
## Local Testing
|
||||
|
||||
To try this out locally, start each instance in a separate terminal:
|
||||
|
||||
```bash
|
||||
# Terminal 1: Control Plane
|
||||
litellm --config cp_config.yaml --port 4000
|
||||
|
||||
# Terminal 2: Worker A
|
||||
PROXY_BASE_URL=http://localhost:4001 litellm --config worker_a_config.yaml --port 4001
|
||||
|
||||
# Terminal 3: Worker B
|
||||
PROXY_BASE_URL=http://localhost:4002 litellm --config worker_b_config.yaml --port 4002
|
||||
```
|
||||
|
||||
Then open `http://localhost:4000/ui`. You should see the worker selector on the login page.
|
||||
|
||||
## Configuration Reference
|
||||
|
||||
### Control Plane Settings
|
||||
|
||||
| Field | Location | Description |
|
||||
|---|---|---|
|
||||
| `worker_registry` | Top-level config | List of worker instances |
|
||||
| `worker_registry[].worker_id` | Required | Unique identifier for the worker |
|
||||
| `worker_registry[].name` | Required | Display name shown in the UI |
|
||||
| `worker_registry[].url` | Required | Full URL of the worker instance |
|
||||
|
||||
### Worker Settings
|
||||
|
||||
| Field | Location | Description |
|
||||
|---|---|---|
|
||||
| `general_settings.control_plane_url` | Required | URL of the control plane instance. Enables `/v3/login` and `/v3/login/exchange` endpoints on this worker. |
|
||||
| `PROXY_BASE_URL` | Environment variable | The worker's own external URL. Required for SSO callback redirects. |
|
||||
|
||||
## Related Documentation
|
||||
|
||||
- [Standard Multi-Region Setup](./control_plane_and_data_plane.md) - shared-database architecture for admin/worker split
|
||||
- [SSO Setup](./admin_ui_sso.md) - configuring SSO for the admin UI
|
||||
- [Production Deployment](./prod.md) - production best practices
|
||||
|
|
@ -352,7 +352,7 @@ If `order=1` deployment is unavailable (e.g., rate-limited), the router falls ba
|
|||
|
||||
When load balancing OpenAI's Responses API across deployments with **different API keys** (e.g., different Azure regions or organizations), encrypted content items (like `rs_...` reasoning items) can only be decrypted by the originating API key.
|
||||
|
||||
**Solution:** Use the `encrypted_content_affinity` pre-call check to automatically route follow-up requests containing encrypted items to the correct deployment:
|
||||
**Solution:** Use the `encrypted_content_affinity` pre-call check (requires LiteLLM >= 1.82.3) to automatically route follow-up requests containing encrypted items to the correct deployment:
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
|
|
|
|||
|
|
@ -311,7 +311,7 @@ litellm_settings:
|
|||
1. **At Startup**: When the proxy starts, it reads the `prompts` field from `config.yaml`
|
||||
2. **Initialization**: Each prompt is initialized based on its `prompt_integration` type
|
||||
3. **In-Memory Storage**: Prompts are stored in the `IN_MEMORY_PROMPT_REGISTRY`
|
||||
4. **Access**: Use these prompts via the `/v1/chat/completions` endpoint with `prompt_id` in the request
|
||||
4. **Access**: Use these prompts via `/v1/chat/completions` or `/v1/responses` with `prompt_id` in the request
|
||||
|
||||
### Using Config-Loaded Prompts
|
||||
|
||||
|
|
@ -331,6 +331,23 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
|
|||
}'
|
||||
```
|
||||
|
||||
You can also use the same `prompt_id` with the Responses API:
|
||||
|
||||
```bash
|
||||
curl -L -X POST 'http://0.0.0.0:4000/v1/responses' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H 'Authorization: Bearer sk-1234' \
|
||||
-d '{
|
||||
"model": "gpt-4o",
|
||||
"prompt_id": "coding_assistant",
|
||||
"prompt_variables": {
|
||||
"language": "python",
|
||||
"task": "create a web scraper"
|
||||
},
|
||||
"input": []
|
||||
}'
|
||||
```
|
||||
|
||||
### Prompt Schema Reference
|
||||
|
||||
Each prompt in the `prompts` list requires:
|
||||
|
|
|
|||
|
|
@ -594,9 +594,26 @@ Expected Response
|
|||
|
||||
:::tip gpt-5.4: reasoning_effort + function tools
|
||||
|
||||
LiteLLM drops `reasoning_effort` from `gpt-5.4` requests to `litellm.completion()` that include tools, since that combination is supported in the Responses API.
|
||||
When `gpt-5.4+` requests to `litellm.completion()` include both `reasoning_effort` and `tools`, LiteLLM **automatically routes** the request through the Responses API bridge. This works for both **OpenAI** (`openai/gpt-5.4`) and **Azure** (`azure/gpt-5.4`) providers — no extra configuration needed.
|
||||
|
||||
If you need reasoning **and** tools together, use `openai/responses/gpt-5.4` to route through the Responses API instead. See [Responses API Bridge](/docs/providers/openai#openai-chat-completion-to-responses-api-bridge) for details.
|
||||
You can also route explicitly via `openai/responses/gpt-5.4` or `azure/responses/gpt-5.4`. See [Responses API Bridge](/docs/providers/openai#openai-chat-completion-to-responses-api-bridge) for details.
|
||||
|
||||
**Azure custom deployment names:** Auto-routing relies on the deployment name matching the `gpt-5.4*` pattern. If you use a custom deployment name (e.g. `"my-reasoning-model"`), enable routing via:
|
||||
|
||||
**SDK:**
|
||||
```python
|
||||
litellm.completion(model="azure/responses/my-reasoning-model", ...)
|
||||
```
|
||||
|
||||
**Proxy config:**
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: my-reasoning-model
|
||||
litellm_params:
|
||||
model: azure/my-reasoning-model
|
||||
model_info:
|
||||
mode: responses
|
||||
```
|
||||
|
||||
:::
|
||||
|
||||
|
|
@ -683,3 +700,69 @@ response = litellm.completion(
|
|||
reasoning_effort={"effort": "low", "summary": "detailed"}, # Explicit control
|
||||
)
|
||||
```
|
||||
|
||||
### Summary Preservation via `/v1/messages` Adapter
|
||||
|
||||
When using the Anthropic `/v1/messages` adapter to route non-Claude models (e.g., `openai/gpt-5.1`), the `thinking.summary` value is preserved and forwarded to the downstream provider. For example:
|
||||
|
||||
```python
|
||||
import litellm
|
||||
|
||||
response = await litellm.anthropic.messages.acreate(
|
||||
model="openai/gpt-5.1",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
max_tokens=8096,
|
||||
thinking={"type": "enabled", "budget_tokens": 5000, "summary": "concise"},
|
||||
)
|
||||
# The summary="concise" is preserved when routing to OpenAI's Responses API
|
||||
```
|
||||
|
||||
### Enabling Default Summary Injection for `/v1/messages` Adapter
|
||||
|
||||
When the Anthropic `/v1/messages` adapter translates `thinking` parameters to OpenAI `reasoning_effort` for non-Claude models, you can opt-in to automatic `summary="detailed"` injection using the `reasoning_auto_summary` flag. This ensures that reasoning text is returned in the response (matching the Anthropic thinking behavior).
|
||||
|
||||
To **enable** this default injection, use the `reasoning_auto_summary` flag:
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="sdk" label="SDK">
|
||||
|
||||
```python
|
||||
import litellm
|
||||
|
||||
# Enable default summary="detailed" injection
|
||||
litellm.reasoning_auto_summary = True
|
||||
|
||||
response = await litellm.anthropic.messages.acreate(
|
||||
model="openai/gpt-5.1",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
max_tokens=8096,
|
||||
thinking={"type": "enabled", "budget_tokens": 5000},
|
||||
)
|
||||
# summary="detailed" will be automatically added to reasoning_effort
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="env" label="Environment Variable">
|
||||
|
||||
```bash
|
||||
export LITELLM_REASONING_AUTO_SUMMARY=true
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="proxy" label="Proxy Config">
|
||||
|
||||
```yaml
|
||||
litellm_settings:
|
||||
reasoning_auto_summary: true
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
:::info
|
||||
|
||||
This flag only affects the automatic injection of `summary="detailed"` when no user-provided summary is present. If you explicitly pass `thinking.summary` (e.g., `"concise"` or `"auto"`), your value is always preserved regardless of this flag.
|
||||
|
||||
:::
|
||||
|
|
|
|||
|
|
@ -1160,12 +1160,12 @@ follow_up = await router.aresponses(
|
|||
To enable session continuity for Responses API in your LiteLLM proxy, set `optional_pre_call_checks` in your proxy config.yaml.
|
||||
|
||||
- `responses_api_deployment_check`: high priority routing when `previous_response_id` is provided
|
||||
- `encrypted_content_affinity`: **[Recommended]** content-aware routing for encrypted items (e.g., `rs_...` reasoning items)
|
||||
- `encrypted_content_affinity`: **[Recommended]** content-aware routing for encrypted items (e.g., `rs_...` reasoning items) (**requires LiteLLM >= 1.82.3**)
|
||||
- `session_affinity`: sticky sessions based on session id (takes priority over `deployment_affinity`)
|
||||
- `deployment_affinity`: sticky sessions based on user key (applies even without `previous_response_id`)
|
||||
|
||||
:::tip Recommended: Use `encrypted_content_affinity`
|
||||
For Responses API with load balancing across deployments with **different API keys**, use `encrypted_content_affinity` instead of `deployment_affinity`. It only pins requests that contain encrypted content, avoiding quota reduction while preventing `invalid_encrypted_content` errors.
|
||||
For Responses API with load balancing across deployments with **different API keys**, use `encrypted_content_affinity` instead of `deployment_affinity`. It only pins requests that contain encrypted content, avoiding quota reduction while preventing `invalid_encrypted_content` errors. (Requires LiteLLM >= 1.82.3.)
|
||||
:::
|
||||
|
||||
Notes:
|
||||
|
|
@ -1364,6 +1364,85 @@ litellm --config config.yaml
|
|||
| `deployment_affinity` | Simple sticky sessions | All requests from same API key | ❌ Reduces quota by # of users |
|
||||
|
||||
|
||||
## Per-Model-Group Affinity Configuration
|
||||
|
||||
By default, `optional_pre_call_checks` applies globally to all model groups. Use `model_group_affinity_config` when you want different affinity behavior per model group — for example, enabling stickiness only for models spread across providers (Azure + Bedrock) while leaving single-provider groups free to load-balance.
|
||||
|
||||
Groups not listed fall back to the global `optional_pre_call_checks` settings.
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="python-sdk" label="Python SDK">
|
||||
|
||||
```python
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-4",
|
||||
"litellm_params": {"model": "azure/gpt-4", "api_key": "...", "api_base": "https://endpoint1.openai.azure.com"},
|
||||
},
|
||||
{
|
||||
"model_name": "gpt-4",
|
||||
"litellm_params": {"model": "bedrock/anthropic.claude-v2", "aws_region_name": "us-east-1"},
|
||||
},
|
||||
{
|
||||
"model_name": "text-embedding-ada-002",
|
||||
"litellm_params": {"model": "azure/text-embedding-ada-002", "api_key": "...", "api_base": "https://endpoint1.openai.azure.com"},
|
||||
},
|
||||
{
|
||||
"model_name": "text-embedding-ada-002",
|
||||
"litellm_params": {"model": "azure/text-embedding-ada-002", "api_key": "...", "api_base": "https://endpoint2.openai.azure.com"},
|
||||
},
|
||||
],
|
||||
# gpt-4: cross-provider (Azure + Bedrock) — enable deployment affinity
|
||||
# text-embedding-ada-002: same provider — no affinity, let it load balance freely
|
||||
model_group_affinity_config={
|
||||
"gpt-4": ["deployment_affinity", "responses_api_deployment_check"],
|
||||
},
|
||||
)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="proxy-server" label="Proxy Server">
|
||||
|
||||
```yaml title="config.yaml"
|
||||
model_list:
|
||||
- model_name: gpt-4
|
||||
litellm_params:
|
||||
model: azure/gpt-4
|
||||
api_key: os.environ/AZURE_API_KEY_1
|
||||
api_base: https://endpoint1.openai.azure.com
|
||||
|
||||
- model_name: gpt-4
|
||||
litellm_params:
|
||||
model: bedrock/anthropic.claude-v2
|
||||
aws_region_name: us-east-1
|
||||
|
||||
- model_name: text-embedding-ada-002
|
||||
litellm_params:
|
||||
model: azure/text-embedding-ada-002
|
||||
api_key: os.environ/AZURE_API_KEY_1
|
||||
api_base: https://endpoint1.openai.azure.com
|
||||
|
||||
- model_name: text-embedding-ada-002
|
||||
litellm_params:
|
||||
model: azure/text-embedding-ada-002
|
||||
api_key: os.environ/AZURE_API_KEY_2
|
||||
api_base: https://endpoint2.openai.azure.com
|
||||
|
||||
router_settings:
|
||||
# gpt-4: cross-provider — enable stickiness
|
||||
# text-embedding-ada-002: not listed — load balances freely
|
||||
model_group_affinity_config:
|
||||
"gpt-4":
|
||||
- deployment_affinity
|
||||
- responses_api_deployment_check
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
**Supported values:** `deployment_affinity`, `responses_api_deployment_check`, `session_affinity`
|
||||
|
||||
## Calling non-Responses API endpoints (`/responses` to `/chat/completions` Bridge)
|
||||
|
||||
LiteLLM allows you to call non-Responses API models via a bridge to LiteLLM's `/chat/completions` endpoint. This is useful for calling Anthropic, Gemini and even non-Responses API OpenAI models.
|
||||
|
|
@ -1556,6 +1635,12 @@ curl -X POST "http://localhost:4000/v1/responses" \
|
|||
}'
|
||||
```
|
||||
|
||||
## File Search (Vector Stores)
|
||||
|
||||
For full `file_search` usage (native + emulated fallback), SDK/Proxy examples, architecture diagram, and Q&A, see:
|
||||
|
||||
- [`File Search in the Responses API — E2E Testing Guide`](/docs/tutorials/file_search_responses_api)
|
||||
|
||||
## Session Management
|
||||
|
||||
LiteLLM Proxy supports session management for all supported models. This allows you to store and fetch conversation history (state) in LiteLLM Proxy.
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ Click **+ Add New Plugin** to register a plugin in your marketplace.
|
|||
Enter the plugin information:
|
||||
|
||||
- **Name**: Plugin identifier (kebab-case, e.g., `my-plugin`)
|
||||
- **Source Type**: Choose GitHub or URL
|
||||
- **Source Type**: Choose GitHub, Git URL, or Git Subdir
|
||||
- **Repository/URL**: The git source (e.g., `org/repo` for GitHub)
|
||||
- **Version**: Semantic version (optional)
|
||||
- **Description**: What the plugin does
|
||||
|
|
@ -216,6 +216,22 @@ curl -X DELETE http://localhost:4000/claude-code/plugins/my-plugin \
|
|||
|
||||
Use this format for GitLab, Bitbucket, or self-hosted git repositories.
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="git-subdir" label="Git Subdir">
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "my-plugin",
|
||||
"source": {
|
||||
"source": "git-subdir",
|
||||
"url": "https://github.com/org/repo.git",
|
||||
"path": "plugins/my-plugin"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Use this format when your plugin lives in a subdirectory of a git repository. The `path` field must be a relative path of slash-separated segments (alphanumeric, dots, hyphens, underscores only).
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
|
|
|
|||
241
docs/my-website/docs/tutorials/file_search_responses_api.md
Normal file
241
docs/my-website/docs/tutorials/file_search_responses_api.md
Normal file
|
|
@ -0,0 +1,241 @@
|
|||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
# File Search in the Responses API
|
||||
|
||||
LiteLLM now supports `file_search` in the Responses API across both:
|
||||
- providers that support it natively (like OpenAI / Azure), and
|
||||
- providers that do not (like Anthropic, Bedrock, and other non-native providers) via emulation.
|
||||
|
||||
## What this is
|
||||
|
||||
`file_search` lets models retrieve grounded context from your vector stores and answer with citations.
|
||||
LiteLLM keeps one OpenAI-compatible output shape while routing requests through either native passthrough or an emulated fallback.
|
||||
|
||||
Two paths are covered:
|
||||
|
||||
| Path | When it runs | What LiteLLM does |
|
||||
| --- | --- | --- |
|
||||
| **Native passthrough** | Provider natively supports `file_search` (OpenAI, Azure) | Decodes unified vector store ID → forwards to provider as-is |
|
||||
| **Emulated fallback** | Provider doesn't support `file_search` (Anthropic, Bedrock, etc.) | Converts to a function tool → intercepts tool call → runs vector search → synthesizes OpenAI-format output |
|
||||
|
||||
In `tools[].vector_store_ids`, LiteLLM accepts both provider-native IDs (e.g. `vs_...`) **and** **managed vector store unified IDs** (URL-safe base64 strings from the proxy managed-vector flow), e.g. `litellm.responses(..., tools=[{"type": "file_search", "vector_store_ids": ["bGl0ZWxsbV9wcm94eT..."]}])`.
|
||||
|
||||
## Usage
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="proxy" label="LiteLLM Proxy" default>
|
||||
|
||||
### 1. Setup `config.yaml`
|
||||
|
||||
```yaml title="config.yaml"
|
||||
model_list:
|
||||
- model_name: gpt-4.1
|
||||
litellm_params:
|
||||
model: openai/gpt-4.1
|
||||
api_key: os.environ/OPENAI_API_KEY
|
||||
|
||||
- model_name: claude-sonnet
|
||||
litellm_params:
|
||||
model: anthropic/claude-sonnet-4-5
|
||||
api_key: os.environ/ANTHROPIC_API_KEY
|
||||
```
|
||||
|
||||
### 2. Start the proxy
|
||||
|
||||
```bash
|
||||
litellm --config config.yaml
|
||||
```
|
||||
|
||||
### 3. Call Responses API with `file_search`
|
||||
|
||||
```python title="Proxy call"
|
||||
from openai import OpenAI
|
||||
|
||||
client = OpenAI(base_url="http://localhost:4000", api_key="sk-your-proxy-key")
|
||||
|
||||
response = client.responses.create(
|
||||
model="claude-sonnet", # swap to "gpt-4.1" for native path
|
||||
input="What does LiteLLM support?",
|
||||
tools=[{
|
||||
"type": "file_search",
|
||||
"vector_store_ids": ["vs_abc123"]
|
||||
}],
|
||||
include=["file_search_call.results"],
|
||||
)
|
||||
|
||||
print(response.output)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="sdk" label="LiteLLM SDK">
|
||||
|
||||
### 1. Install + set keys
|
||||
|
||||
```bash
|
||||
pip install litellm
|
||||
export OPENAI_API_KEY="sk-..."
|
||||
export ANTHROPIC_API_KEY="sk-ant-..."
|
||||
```
|
||||
|
||||
### 2. Call Responses API with `file_search`
|
||||
|
||||
```python title="SDK call"
|
||||
import litellm
|
||||
|
||||
response = litellm.responses(
|
||||
model="anthropic/claude-sonnet-4-5", # swap to openai/gpt-4.1 for native path
|
||||
input="What does LiteLLM support?",
|
||||
tools=[{
|
||||
"type": "file_search",
|
||||
"vector_store_ids": ["vs_abc123"]
|
||||
}],
|
||||
include=["file_search_call.results"],
|
||||
)
|
||||
|
||||
print(response.output)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
### Behavior Matrix
|
||||
|
||||
| Path | SDK model | Proxy model | Behavior |
|
||||
| --- | --- | --- | --- |
|
||||
| Native passthrough | `openai/gpt-4.1` | `gpt-4.1` | Provider executes native `file_search` |
|
||||
| Emulated fallback | `anthropic/claude-sonnet-4-5` | `claude-sonnet` | LiteLLM converts to function tool and synthesizes OpenAI-format output |
|
||||
|
||||
|
||||
|
||||
## Architecture Diagram
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
A[Client SDK or Proxy Caller] --> B[LiteLLM Responses API]
|
||||
B --> C{Provider supports native file_search?}
|
||||
|
||||
C -->|Yes| D[Native passthrough path]
|
||||
D --> D1[Decode unified vector_store_id if needed]
|
||||
D1 --> D2[Forward request to provider unchanged]
|
||||
D2 --> D3[Provider performs file_search]
|
||||
D3 --> Z[OpenAI-compatible output]
|
||||
|
||||
C -->|No| E[Emulated fallback path]
|
||||
E --> E1[Convert file_search to litellm_file_search function tool]
|
||||
E1 --> E2[First model call returns tool call with one or more queries]
|
||||
E2 --> E3[LiteLLM executes vector search for each query]
|
||||
E3 --> E4[Second model call with tool_result context]
|
||||
E4 --> E5[Synthesize file_search_call + message + citations]
|
||||
E5 --> Z[OpenAI-compatible output]
|
||||
```
|
||||
|
||||
|
||||
|
||||
## Prerequisites
|
||||
|
||||
```bash
|
||||
pip install 'litellm[proxy]'
|
||||
export OPENAI_API_KEY="sk-..." # for native path
|
||||
export ANTHROPIC_API_KEY="sk-ant-..." # for emulated path
|
||||
```
|
||||
|
||||
|
||||
|
||||
## Example response shape
|
||||
|
||||
## Validating the Output Format
|
||||
|
||||
Regardless of which path ran, the response always follows the OpenAI Responses API format:
|
||||
|
||||
```json
|
||||
{
|
||||
"output": [
|
||||
{
|
||||
"type": "file_search_call",
|
||||
"id": "fs_abc123",
|
||||
"status": "completed",
|
||||
"queries": ["What does LiteLLM support?"],
|
||||
"search_results": null
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{
|
||||
"type": "output_text",
|
||||
"text": "LiteLLM is a unified interface...",
|
||||
"annotations": [
|
||||
{
|
||||
"type": "file_citation",
|
||||
"index": 150,
|
||||
"file_id": "file-xxxx",
|
||||
"filename": "knowledge.txt"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**Validation script:**
|
||||
|
||||
```python showLineNumbers title="Validate response structure"
|
||||
def validate_file_search_response(response):
|
||||
"""Assert that response follows OpenAI file_search output format."""
|
||||
output = response.output
|
||||
assert len(output) >= 2, "Expected at least 2 output items"
|
||||
|
||||
# First item: file_search_call
|
||||
fs_call = output[0]
|
||||
fs_type = fs_call["type"] if isinstance(fs_call, dict) else fs_call.type
|
||||
assert fs_type == "file_search_call", f"Expected file_search_call, got {fs_type}"
|
||||
|
||||
fs_status = fs_call["status"] if isinstance(fs_call, dict) else fs_call.status
|
||||
assert fs_status == "completed"
|
||||
|
||||
# Second item: message
|
||||
msg = output[1]
|
||||
msg_type = msg["type"] if isinstance(msg, dict) else msg.type
|
||||
assert msg_type == "message"
|
||||
|
||||
content = msg["content"] if isinstance(msg, dict) else msg.content
|
||||
assert len(content) > 0
|
||||
text_block = content[0]
|
||||
text = text_block["text"] if isinstance(text_block, dict) else text_block.text
|
||||
assert isinstance(text, str) and len(text) > 0
|
||||
|
||||
print("✅ Response structure valid")
|
||||
print(f" Queries: {fs_call['queries'] if isinstance(fs_call, dict) else fs_call.queries}")
|
||||
print(f" Answer length: {len(text)} chars")
|
||||
annotations = text_block["annotations"] if isinstance(text_block, dict) else text_block.annotations
|
||||
print(f" Citations: {len(annotations)}")
|
||||
|
||||
validate_file_search_response(response)
|
||||
```
|
||||
|
||||
|
||||
|
||||
## Q&A
|
||||
|
||||
- **Why do I see `UnsupportedParamsError`?** This usually means `file_search` was passed to a provider that does not support it natively and emulation could not route correctly. Check:
|
||||
- The model string is valid (for example, `anthropic/claude-sonnet-4-5`).
|
||||
- `custom_llm_provider` resolves correctly so LiteLLM can load the provider config.
|
||||
- **Why does vector search return no results?** Common causes:
|
||||
- The vector store ID is wrong or has no files attached.
|
||||
- In LiteLLM-managed stores, file ingestion is not complete (`status != completed`).
|
||||
- The query is too narrow; try a broader query.
|
||||
- **Why am I getting `403 Access denied` on vector store calls?** The caller does not have access to that vector store.
|
||||
- The store may belong to another team.
|
||||
- Use an admin/proxy key if your setup requires cross-team access.
|
||||
- **Why are `annotations` empty in emulated mode?** `file_citation` annotations require `file_id` metadata in search results. If your vector backend does not return file-level metadata, the answer text is still generated but citations can be empty.
|
||||
|
||||
|
||||
|
||||
## What to check next
|
||||
|
||||
- [File Search reference in Responses API docs](/docs/response_api#file-search-vector-stores) — full API reference
|
||||
- [Vector Store management](/docs/vector_store_files) — create and manage vector stores
|
||||
- [Managed vector stores](/docs/providers/bedrock_vector_store) — provider-specific setup
|
||||
151
docs/my-website/docs/tutorials/vertex_ai_pay_go.md
Normal file
151
docs/my-website/docs/tutorials/vertex_ai_pay_go.md
Normal file
|
|
@ -0,0 +1,151 @@
|
|||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
# Vertex AI PayGo and Priority
|
||||
|
||||
## Priority PayGo
|
||||
|
||||
LiteLLM supports Priority PayGo.
|
||||
Send a priority header, get priority queueing, and pay priority token rates.
|
||||
|
||||
:::info Which models support Priority PayGo?
|
||||
As of this writing: `gemini/gemini-2.5-pro`, `vertex_ai/gemini-3-pro-preview`, `vertex_ai/gemini-3.1-pro-preview`, `vertex_ai/gemini-3-flash-preview`, and their variants.
|
||||
Check `supports_service_tier: true` in LiteLLM's [model pricing JSON](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json).
|
||||
:::
|
||||
|
||||
### Send a priority request
|
||||
|
||||
Use this header:
|
||||
|
||||
`X-Vertex-AI-LLM-Shared-Request-Type: priority`
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="litellm-sdk" label="LiteLLM SDK">
|
||||
|
||||
```python
|
||||
import litellm
|
||||
|
||||
response = litellm.completion(
|
||||
model="vertex_ai/gemini-3-pro-preview",
|
||||
messages=[{"role": "user", "content": "Summarize the Gettysburg Address."}],
|
||||
vertex_project="YOUR_PROJECT_ID",
|
||||
vertex_location="us-central1",
|
||||
extra_headers={"X-Vertex-AI-LLM-Shared-Request-Type": "priority"},
|
||||
)
|
||||
|
||||
print(response.choices[0].message.content)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="proxy-config" label="Proxy config">
|
||||
|
||||
```yaml title="config.yaml"
|
||||
model_list:
|
||||
- model_name: gemini-priority
|
||||
litellm_params:
|
||||
model: vertex_ai/gemini-3-pro-preview
|
||||
vertex_project: "YOUR_PROJECT_ID"
|
||||
vertex_location: "us-central1"
|
||||
vertex_credentials: os.environ/GOOGLE_APPLICATION_CREDENTIALS
|
||||
extra_headers:
|
||||
X-Vertex-AI-LLM-Shared-Request-Type: priority
|
||||
```
|
||||
|
||||
```bash
|
||||
curl http://localhost:4000/v1/chat/completions \
|
||||
-H "Authorization: Bearer sk-your-key" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"model": "gemini-priority", "messages": [{"role": "user", "content": "Hello"}]}'
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="pass-through" label="Pass-through mode">
|
||||
|
||||
Use `x-pass-` so LiteLLM forwards provider-specific headers.
|
||||
|
||||
```bash
|
||||
MODEL_ID="gemini-3-pro-preview-0325"
|
||||
PROJECT_ID="YOUR_PROJECT_ID"
|
||||
|
||||
curl -X POST \
|
||||
"${LITELLM_PROXY_BASE_URL}/vertex_ai/v1/projects/${PROJECT_ID}/locations/global/publishers/google/models/${MODEL_ID}:generateContent" \
|
||||
-H "Authorization: Bearer sk-your-litellm-key" \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "x-pass-X-Vertex-AI-LLM-Shared-Request-Type: priority" \
|
||||
-d '{"contents": [{"role": "user", "parts": [{"text": "Hello!"}]}]}'
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
### How cost tracking works
|
||||
|
||||
|
||||
|
||||
**`trafficType` → `service_tier` mapping**
|
||||
|
||||
| `usageMetadata.trafficType` | `service_tier` | Pricing keys used |
|
||||
|---|---|---|
|
||||
| `ON_DEMAND` | `None` | `input_cost_per_token` |
|
||||
| `ON_DEMAND_PRIORITY` | `"priority"` | `input_cost_per_token_priority` |
|
||||
| `FLEX` / `BATCH` | `"flex"` | `input_cost_per_token_flex` |
|
||||
|
||||
If a tier-specific key is missing, LiteLLM falls back to standard pricing keys.
|
||||
|
||||
---
|
||||
|
||||
## Standard PayGo vs Provisioned Throughput
|
||||
|
||||
This is a different header from priority routing:
|
||||
|
||||
| Header value | Behavior |
|
||||
|---|---|
|
||||
| `X-Vertex-AI-LLM-Request-Type: shared` | Force standard PayGo (bypass PT) |
|
||||
| `X-Vertex-AI-LLM-Request-Type: dedicated` | Force Provisioned Throughput only (`429` if exhausted) |
|
||||
|
||||
### Native route example
|
||||
|
||||
```python
|
||||
import litellm
|
||||
|
||||
response = litellm.completion(
|
||||
model="vertex_ai/gemini-2.0-flash",
|
||||
messages=[{"role": "user", "content": "Hello!"}],
|
||||
vertex_project="YOUR_PROJECT_ID",
|
||||
vertex_location="us-central1",
|
||||
extra_headers={"X-Vertex-AI-LLM-Request-Type": "shared"},
|
||||
)
|
||||
```
|
||||
|
||||
### Pass-through example
|
||||
|
||||
```bash
|
||||
MODEL_ID="gemini-2.0-flash-001"
|
||||
PROJECT_ID="YOUR_PROJECT_ID"
|
||||
|
||||
curl -X POST \
|
||||
"${LITELLM_PROXY_BASE_URL}/vertex_ai/v1/projects/${PROJECT_ID}/locations/global/publishers/google/models/${MODEL_ID}:generateContent" \
|
||||
-H "Authorization: Bearer sk-your-litellm-key" \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "x-pass-X-Vertex-AI-LLM-Request-Type: shared" \
|
||||
-d '{
|
||||
"contents": [{"role": "user", "parts": [{"text": "Hello!"}]}]
|
||||
}'
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
**Q: What does `403 Permission denied` or `IAM_PERMISSION_DENIED` mean?**
|
||||
A: The service account or Application Default Credentials (ADC) user does not have the `roles/aiplatform.user` role. To resolve this, re-run the `gcloud projects add-iam-policy-binding`.
|
||||
|
||||
**Q: What should I do if I get a `429 Quota exceeded` error?**
|
||||
A: This means you've hit the per-region QPM (queries per minute) or TPM (tokens per minute) quota. You can:
|
||||
- Request a quota increase from the [GCP Quotas console](https://console.cloud.google.com/iam-admin/quotas)
|
||||
- Add more regions to your LiteLLM configuration for load balancing
|
||||
- Upgrade to [Provisioned Throughput](https://cloud.google.com/vertex-ai/generative-ai/docs/provisioned-throughput) for guaranteed capacity
|
||||
|
||||
**Q: How do I fix the `VERTEXAI_PROJECT not set` error?**
|
||||
A: Either pass the `vertex_project` parameter explicitly in your LiteLLM call, or set the `VERTEXAI_PROJECT` environment variable before running your code.
|
||||
|
||||
|
|
@ -430,6 +430,7 @@ const sidebars = {
|
|||
"proxy/architecture",
|
||||
"proxy/multi_tenant_architecture",
|
||||
"proxy/control_plane_and_data_plane",
|
||||
"proxy/high_availability_control_plane",
|
||||
"proxy/db_deadlocks",
|
||||
"proxy/db_info",
|
||||
"proxy/image_handling",
|
||||
|
|
@ -584,6 +585,7 @@ const sidebars = {
|
|||
label: "Spend Tracking",
|
||||
items: [
|
||||
"proxy/cost_tracking",
|
||||
"tutorials/vertex_ai_pay_go",
|
||||
"proxy/request_tags",
|
||||
"proxy/custom_pricing",
|
||||
"proxy/pricing_calculator",
|
||||
|
|
@ -737,6 +739,7 @@ const sidebars = {
|
|||
"proxy/realtime_webrtc",
|
||||
"rerank",
|
||||
"response_api",
|
||||
"prompt_management",
|
||||
"response_api_compact",
|
||||
{
|
||||
type: "category",
|
||||
|
|
@ -1433,6 +1436,7 @@ const learnSidebar = {
|
|||
},
|
||||
items: [
|
||||
"tutorials/prompt_caching",
|
||||
"tutorials/file_search_responses_api",
|
||||
"tutorials/anthropic_file_usage",
|
||||
"tutorials/gemini_realtime_with_audio",
|
||||
"tutorials/litellm_proxy_aporia",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,95 @@
|
|||
import React from 'react';
|
||||
import styles from './styles.module.css';
|
||||
|
||||
/* ────────────────────── Shared small pieces ────────────────────── */
|
||||
|
||||
function InfraChip({ color, label }: { color: string; label: string }) {
|
||||
const dotClass =
|
||||
color === 'green'
|
||||
? styles.infraDotGreen
|
||||
: color === 'blue'
|
||||
? styles.infraDotBlue
|
||||
: styles.infraDotOrange;
|
||||
|
||||
return (
|
||||
<span className={styles.infraChip}>
|
||||
<span className={`${styles.infraDot} ${dotClass}`} />
|
||||
{label}
|
||||
</span>
|
||||
);
|
||||
}
|
||||
|
||||
/* ────────────────────── Architecture tab ────────────────────── */
|
||||
|
||||
function ArchitectureView() {
|
||||
return (
|
||||
<div className={styles.diagram}>
|
||||
{/* User */}
|
||||
<div className={styles.userRow}>
|
||||
<div className={styles.userIcon}>👤</div>
|
||||
<span className={styles.userLabel}>Admin</span>
|
||||
</div>
|
||||
|
||||
<div className={styles.connectorDown} />
|
||||
|
||||
{/* Control Plane */}
|
||||
<div className={`${styles.node} ${styles.nodeControlPlane}`}>
|
||||
<div className={styles.nodeHeader}>
|
||||
<span className={styles.nodeTitle}>Control Plane</span>
|
||||
<span className={`${styles.badge} ${styles.badgeBlue}`}>UI</span>
|
||||
</div>
|
||||
<div className={styles.nodeSubtitle}>cp.example.com</div>
|
||||
<div className={styles.infraRow}>
|
||||
<InfraChip color="green" label="Own DB" />
|
||||
<InfraChip color="orange" label="Own Redis" />
|
||||
<InfraChip color="blue" label="Own Key" />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Branch connector */}
|
||||
<div className={styles.connectorBranch}>
|
||||
<div className={`${styles.branchLeg} ${styles.branchLegLeft}`} />
|
||||
<div className={`${styles.branchLeg} ${styles.branchLegRight}`} />
|
||||
</div>
|
||||
|
||||
{/* Workers */}
|
||||
<div className={styles.workersRow}>
|
||||
<div className={`${styles.node} ${styles.nodeWorker} ${styles.nodeWorkerA}`}>
|
||||
<div className={styles.nodeHeader}>
|
||||
<span className={styles.nodeTitle}>Worker A</span>
|
||||
<span className={`${styles.badge} ${styles.badgeGreen}`}>US East</span>
|
||||
</div>
|
||||
<div className={styles.nodeSubtitle}>worker-a.example.com</div>
|
||||
<div className={styles.infraRow}>
|
||||
<InfraChip color="green" label="Own DB" />
|
||||
<InfraChip color="orange" label="Own Redis" />
|
||||
<InfraChip color="blue" label="Own Key" />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className={`${styles.node} ${styles.nodeWorker} ${styles.nodeWorkerB}`}>
|
||||
<div className={styles.nodeHeader}>
|
||||
<span className={styles.nodeTitle}>Worker B</span>
|
||||
<span className={`${styles.badge} ${styles.badgePurple}`}>EU West</span>
|
||||
</div>
|
||||
<div className={styles.nodeSubtitle}>worker-b.example.com</div>
|
||||
<div className={styles.infraRow}>
|
||||
<InfraChip color="green" label="Own DB" />
|
||||
<InfraChip color="orange" label="Own Redis" />
|
||||
<InfraChip color="blue" label="Own Key" />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
/* ────────────────────── Main component ────────────────────── */
|
||||
|
||||
export default function ControlPlaneArchitecture() {
|
||||
return (
|
||||
<div className={styles.wrapper}>
|
||||
<ArchitectureView />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1 @@
|
|||
export { default as ControlPlaneArchitecture } from './ControlPlaneArchitecture';
|
||||
|
|
@ -0,0 +1,517 @@
|
|||
/* ── Custom properties ── */
|
||||
:root {
|
||||
--cp-bg: #ffffff;
|
||||
--cp-border: #e5e7eb;
|
||||
--cp-text: #1a1a2e;
|
||||
--cp-text-secondary: #6b7280;
|
||||
--cp-text-muted: #9ca3af;
|
||||
--cp-accent: #3b82f6;
|
||||
--cp-accent-light: #dbeafe;
|
||||
--cp-accent-glow: rgba(59, 130, 246, 0.15);
|
||||
--cp-green: #10b981;
|
||||
--cp-green-light: #d1fae5;
|
||||
--cp-green-glow: rgba(16, 185, 129, 0.15);
|
||||
--cp-orange: #f59e0b;
|
||||
--cp-orange-light: #fef3c7;
|
||||
--cp-purple: #8b5cf6;
|
||||
--cp-purple-light: #ede9fe;
|
||||
--cp-red: #ef4444;
|
||||
--cp-red-light: #fee2e2;
|
||||
--cp-card-bg: #f9fafb;
|
||||
--cp-infra-bg: #f1f5f9;
|
||||
--cp-infra-border: #cbd5e1;
|
||||
--cp-connector: #d1d5db;
|
||||
--cp-dot-size: 8px;
|
||||
}
|
||||
|
||||
[data-theme='dark'] {
|
||||
--cp-bg: #111827;
|
||||
--cp-border: #374151;
|
||||
--cp-text: #e5e7eb;
|
||||
--cp-text-secondary: #9ca3af;
|
||||
--cp-text-muted: #6b7280;
|
||||
--cp-accent: #60a5fa;
|
||||
--cp-accent-light: #1e3a5f;
|
||||
--cp-accent-glow: rgba(96, 165, 250, 0.2);
|
||||
--cp-green: #34d399;
|
||||
--cp-green-light: #064e3b;
|
||||
--cp-green-glow: rgba(52, 211, 153, 0.2);
|
||||
--cp-orange: #fbbf24;
|
||||
--cp-orange-light: #78350f;
|
||||
--cp-purple: #a78bfa;
|
||||
--cp-purple-light: #3b0764;
|
||||
--cp-red: #f87171;
|
||||
--cp-red-light: #451a1a;
|
||||
--cp-card-bg: #1f2937;
|
||||
--cp-infra-bg: #1e293b;
|
||||
--cp-infra-border: #475569;
|
||||
--cp-connector: #4b5563;
|
||||
}
|
||||
|
||||
/* ── Wrapper ── */
|
||||
.wrapper {
|
||||
margin: 1.5rem 0;
|
||||
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
|
||||
}
|
||||
|
||||
/* ── Tab bar ── */
|
||||
.tabs {
|
||||
display: flex;
|
||||
gap: 0;
|
||||
margin-bottom: 1.5rem;
|
||||
border-bottom: 2px solid var(--cp-border);
|
||||
}
|
||||
|
||||
.tab {
|
||||
padding: 0.6rem 1.25rem;
|
||||
font-size: 0.85rem;
|
||||
font-weight: 600;
|
||||
color: var(--cp-text-secondary);
|
||||
background: none;
|
||||
border: none;
|
||||
border-bottom: 2px solid transparent;
|
||||
margin-bottom: -2px;
|
||||
cursor: pointer;
|
||||
transition: color 0.2s, border-color 0.2s;
|
||||
}
|
||||
|
||||
.tab:hover {
|
||||
color: var(--cp-text);
|
||||
}
|
||||
|
||||
.tabActive {
|
||||
color: var(--cp-accent);
|
||||
border-bottom-color: var(--cp-accent);
|
||||
}
|
||||
|
||||
/* ── Architecture diagram ── */
|
||||
.diagram {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
gap: 0;
|
||||
}
|
||||
|
||||
/* ── User icon ── */
|
||||
.userRow {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
margin-bottom: 0.5rem;
|
||||
}
|
||||
|
||||
.userIcon {
|
||||
width: 40px;
|
||||
height: 40px;
|
||||
border-radius: 50%;
|
||||
background: var(--cp-accent-light);
|
||||
border: 2px solid var(--cp-accent);
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
font-size: 1.1rem;
|
||||
}
|
||||
|
||||
.userLabel {
|
||||
font-size: 0.75rem;
|
||||
color: var(--cp-text-secondary);
|
||||
margin-top: 0.3rem;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
/* ── Connectors ── */
|
||||
.connectorDown {
|
||||
width: 2px;
|
||||
height: 28px;
|
||||
background: var(--cp-connector);
|
||||
position: relative;
|
||||
}
|
||||
|
||||
.connectorDown::after {
|
||||
content: '';
|
||||
position: absolute;
|
||||
bottom: -4px;
|
||||
left: 50%;
|
||||
transform: translateX(-50%);
|
||||
width: 0;
|
||||
height: 0;
|
||||
border-left: 5px solid transparent;
|
||||
border-right: 5px solid transparent;
|
||||
border-top: 5px solid var(--cp-connector);
|
||||
}
|
||||
|
||||
.connectorBranch {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
justify-content: center;
|
||||
position: relative;
|
||||
width: 100%;
|
||||
max-width: 700px;
|
||||
height: 36px;
|
||||
}
|
||||
|
||||
.connectorBranch::before {
|
||||
content: '';
|
||||
position: absolute;
|
||||
top: 0;
|
||||
left: 50%;
|
||||
width: 2px;
|
||||
height: 12px;
|
||||
background: var(--cp-connector);
|
||||
transform: translateX(-50%);
|
||||
}
|
||||
|
||||
.connectorBranch::after {
|
||||
content: '';
|
||||
position: absolute;
|
||||
top: 12px;
|
||||
left: calc(25% + 12px);
|
||||
right: calc(25% + 12px);
|
||||
height: 2px;
|
||||
background: var(--cp-connector);
|
||||
}
|
||||
|
||||
.branchLeg {
|
||||
position: absolute;
|
||||
top: 12px;
|
||||
width: 2px;
|
||||
height: 24px;
|
||||
background: var(--cp-connector);
|
||||
}
|
||||
|
||||
.branchLeg::after {
|
||||
content: '';
|
||||
position: absolute;
|
||||
bottom: -4px;
|
||||
left: 50%;
|
||||
transform: translateX(-50%);
|
||||
width: 0;
|
||||
height: 0;
|
||||
border-left: 5px solid transparent;
|
||||
border-right: 5px solid transparent;
|
||||
border-top: 5px solid var(--cp-connector);
|
||||
}
|
||||
|
||||
.branchLegLeft {
|
||||
left: calc(25% + 12px);
|
||||
}
|
||||
|
||||
.branchLegRight {
|
||||
right: calc(25% + 12px);
|
||||
}
|
||||
|
||||
/* ── Node cards ── */
|
||||
.node {
|
||||
border: 2px solid var(--cp-border);
|
||||
border-radius: 12px;
|
||||
background: var(--cp-card-bg);
|
||||
padding: 1rem 1.25rem;
|
||||
text-align: center;
|
||||
transition: border-color 0.3s, box-shadow 0.3s;
|
||||
position: relative;
|
||||
}
|
||||
|
||||
.nodeControlPlane {
|
||||
border-color: var(--cp-accent);
|
||||
box-shadow: 0 0 0 3px var(--cp-accent-glow);
|
||||
min-width: 280px;
|
||||
}
|
||||
|
||||
.nodeWorker {
|
||||
min-width: 220px;
|
||||
}
|
||||
|
||||
.nodeWorkerA {
|
||||
border-color: var(--cp-green);
|
||||
box-shadow: 0 0 0 3px var(--cp-green-glow);
|
||||
}
|
||||
|
||||
.nodeWorkerB {
|
||||
border-color: var(--cp-purple);
|
||||
box-shadow: 0 0 0 3px rgba(139, 92, 246, 0.15);
|
||||
}
|
||||
|
||||
[data-theme='dark'] .nodeWorkerB {
|
||||
box-shadow: 0 0 0 3px rgba(167, 139, 250, 0.2);
|
||||
}
|
||||
|
||||
.nodeHeader {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
gap: 0.5rem;
|
||||
margin-bottom: 0.5rem;
|
||||
}
|
||||
|
||||
.nodeIcon {
|
||||
font-size: 1.1rem;
|
||||
}
|
||||
|
||||
.nodeTitle {
|
||||
font-size: 0.95rem;
|
||||
font-weight: 700;
|
||||
color: var(--cp-text);
|
||||
}
|
||||
|
||||
.nodeSubtitle {
|
||||
font-size: 0.75rem;
|
||||
color: var(--cp-text-secondary);
|
||||
margin-bottom: 0.75rem;
|
||||
}
|
||||
|
||||
.badge {
|
||||
display: inline-block;
|
||||
font-size: 0.65rem;
|
||||
font-weight: 600;
|
||||
padding: 0.15rem 0.5rem;
|
||||
border-radius: 9999px;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.04em;
|
||||
}
|
||||
|
||||
.badgeBlue {
|
||||
background: var(--cp-accent-light);
|
||||
color: var(--cp-accent);
|
||||
}
|
||||
|
||||
.badgeGreen {
|
||||
background: var(--cp-green-light);
|
||||
color: var(--cp-green);
|
||||
}
|
||||
|
||||
.badgePurple {
|
||||
background: var(--cp-purple-light);
|
||||
color: var(--cp-purple);
|
||||
}
|
||||
|
||||
/* ── Infrastructure chips ── */
|
||||
.infraRow {
|
||||
display: flex;
|
||||
gap: 0.4rem;
|
||||
justify-content: center;
|
||||
flex-wrap: wrap;
|
||||
margin-top: 0.5rem;
|
||||
}
|
||||
|
||||
.infraChip {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.3rem;
|
||||
font-size: 0.7rem;
|
||||
font-weight: 500;
|
||||
color: var(--cp-text-secondary);
|
||||
background: var(--cp-infra-bg);
|
||||
border: 1px solid var(--cp-infra-border);
|
||||
border-radius: 6px;
|
||||
padding: 0.2rem 0.5rem;
|
||||
}
|
||||
|
||||
.infraDot {
|
||||
width: 6px;
|
||||
height: 6px;
|
||||
border-radius: 50%;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.infraDotGreen {
|
||||
background: var(--cp-green);
|
||||
}
|
||||
|
||||
.infraDotBlue {
|
||||
background: var(--cp-accent);
|
||||
}
|
||||
|
||||
.infraDotOrange {
|
||||
background: var(--cp-orange);
|
||||
}
|
||||
|
||||
/* ── Workers row ── */
|
||||
.workersRow {
|
||||
display: flex;
|
||||
gap: 2rem;
|
||||
justify-content: center;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
/* ── Animated flow ── */
|
||||
.flowLabel {
|
||||
font-size: 0.7rem;
|
||||
color: var(--cp-accent);
|
||||
font-weight: 600;
|
||||
position: absolute;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
/* ── Comparison view ── */
|
||||
.comparisonGrid {
|
||||
display: grid;
|
||||
grid-template-columns: 1fr 1fr;
|
||||
gap: 1.5rem;
|
||||
margin-top: 0.5rem;
|
||||
}
|
||||
|
||||
.comparisonColumn {
|
||||
border: 2px solid var(--cp-border);
|
||||
border-radius: 12px;
|
||||
padding: 1.25rem;
|
||||
background: var(--cp-card-bg);
|
||||
}
|
||||
|
||||
.comparisonColumnOld {
|
||||
border-color: var(--cp-red);
|
||||
}
|
||||
|
||||
.comparisonColumnNew {
|
||||
border-color: var(--cp-green);
|
||||
}
|
||||
|
||||
.comparisonTitle {
|
||||
font-size: 0.9rem;
|
||||
font-weight: 700;
|
||||
color: var(--cp-text);
|
||||
text-align: center;
|
||||
margin-bottom: 1rem;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
gap: 0.4rem;
|
||||
}
|
||||
|
||||
.comparisonTitleOld {
|
||||
color: var(--cp-red);
|
||||
}
|
||||
|
||||
.comparisonTitleNew {
|
||||
color: var(--cp-green);
|
||||
}
|
||||
|
||||
/* ── Mini diagram inside comparison ── */
|
||||
.miniDiagram {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
.miniNode {
|
||||
border: 1.5px solid var(--cp-border);
|
||||
border-radius: 8px;
|
||||
background: var(--cp-bg);
|
||||
padding: 0.5rem 0.75rem;
|
||||
text-align: center;
|
||||
font-size: 0.75rem;
|
||||
font-weight: 600;
|
||||
color: var(--cp-text);
|
||||
width: 100%;
|
||||
max-width: 180px;
|
||||
}
|
||||
|
||||
.miniNodeHighlight {
|
||||
border-color: var(--cp-accent);
|
||||
background: var(--cp-accent-light);
|
||||
}
|
||||
|
||||
.miniNodeDanger {
|
||||
border-color: var(--cp-red);
|
||||
background: var(--cp-red-light);
|
||||
}
|
||||
|
||||
.miniNodeSuccess {
|
||||
border-color: var(--cp-green);
|
||||
background: var(--cp-green-light);
|
||||
}
|
||||
|
||||
.miniConnector {
|
||||
width: 1.5px;
|
||||
height: 16px;
|
||||
background: var(--cp-connector);
|
||||
}
|
||||
|
||||
.miniWorkersRow {
|
||||
display: flex;
|
||||
gap: 0.5rem;
|
||||
justify-content: center;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.miniWorkerStack {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
gap: 0.3rem;
|
||||
flex: 1;
|
||||
max-width: 140px;
|
||||
}
|
||||
|
||||
.miniInfra {
|
||||
font-size: 0.65rem;
|
||||
color: var(--cp-text-muted);
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.miniInfraShared {
|
||||
color: var(--cp-red);
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.miniInfraOwn {
|
||||
color: var(--cp-green);
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
/* ── Callout box ── */
|
||||
.callout {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
gap: 0.6rem;
|
||||
padding: 0.75rem 1rem;
|
||||
border-radius: 8px;
|
||||
margin-top: 1rem;
|
||||
font-size: 0.8rem;
|
||||
color: var(--cp-text);
|
||||
line-height: 1.5;
|
||||
}
|
||||
|
||||
.calloutDanger {
|
||||
background: var(--cp-red-light);
|
||||
border: 1px solid var(--cp-red);
|
||||
}
|
||||
|
||||
.calloutSuccess {
|
||||
background: var(--cp-green-light);
|
||||
border: 1px solid var(--cp-green);
|
||||
}
|
||||
|
||||
.calloutIcon {
|
||||
font-size: 1rem;
|
||||
flex-shrink: 0;
|
||||
margin-top: 0.1rem;
|
||||
}
|
||||
|
||||
/* ── Responsive ── */
|
||||
@media (max-width: 768px) {
|
||||
.comparisonGrid {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
|
||||
.workersRow {
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.nodeControlPlane {
|
||||
min-width: auto;
|
||||
width: 100%;
|
||||
max-width: 300px;
|
||||
}
|
||||
|
||||
.nodeWorker {
|
||||
min-width: auto;
|
||||
width: 100%;
|
||||
max-width: 260px;
|
||||
}
|
||||
|
||||
.connectorBranch {
|
||||
display: none;
|
||||
}
|
||||
}
|
||||
63
docs/my-website/static/img/vertex_cost_tracking_flow.svg
Normal file
63
docs/my-website/static/img/vertex_cost_tracking_flow.svg
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
<svg width="100%" viewBox="0 0 680 560" xmlns="http://www.w3.org/2000/svg">
|
||||
<defs>
|
||||
<marker id="arrow" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="6" markerHeight="6" orient="auto-start-reverse">
|
||||
<path d="M2 1L8 5L2 9" fill="none" stroke="context-stroke" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
|
||||
</marker>
|
||||
</defs>
|
||||
|
||||
<!-- Step 1: HTTP Request -->
|
||||
<g style="fill:rgb(0, 0, 0);stroke:none;color:rgb(255, 255, 255);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto">
|
||||
<rect x="190" y="30" width="300" height="56" rx="8" stroke-width="0.5" style="fill:rgb(12, 68, 124);stroke:rgb(133, 183, 235);color:rgb(255, 255, 255);stroke-width:0.5px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto"/>
|
||||
<text x="340" y="52" text-anchor="middle" dominant-baseline="central" style="fill:rgb(181, 212, 244);stroke:none;color:rgb(255, 255, 255);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:14px;font-weight:500;text-anchor:middle;dominant-baseline:central">HTTP request</text>
|
||||
<text x="340" y="70" text-anchor="middle" dominant-baseline="central" style="fill:rgb(133, 183, 235);stroke:none;color:rgb(255, 255, 255);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:12px;font-weight:400;text-anchor:middle;dominant-baseline:central">X-Vertex-AI-LLM-Shared-Request-Type: priority</text>
|
||||
</g>
|
||||
|
||||
<!-- Arrow 1 -->
|
||||
<line x1="340" y1="86" x2="340" y2="120" marker-end="url(#arrow)" style="fill:none;stroke:rgb(156, 154, 146);color:rgb(255, 255, 255);stroke-width:1.5px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto"/>
|
||||
<text x="356" y="108" dominant-baseline="central" style="fill:rgb(194, 192, 182);stroke:none;color:rgb(255, 255, 255);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:12px;font-weight:400;text-anchor:start;dominant-baseline:central">Vertex AI</text>
|
||||
|
||||
<!-- Step 2: Vertex response -->
|
||||
<g style="fill:rgb(0, 0, 0);stroke:none;color:rgb(255, 255, 255);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto">
|
||||
<rect x="190" y="120" width="300" height="56" rx="8" stroke-width="0.5" style="fill:rgb(8, 80, 65);stroke:rgb(93, 202, 165);color:rgb(255, 255, 255);stroke-width:0.5px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto"/>
|
||||
<text x="340" y="142" text-anchor="middle" dominant-baseline="central" style="fill:rgb(159, 225, 203);stroke:none;color:rgb(255, 255, 255);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:14px;font-weight:500;text-anchor:middle;dominant-baseline:central">Vertex response</text>
|
||||
<text x="340" y="160" text-anchor="middle" dominant-baseline="central" style="fill:rgb(93, 202, 165);stroke:none;color:rgb(255, 255, 255);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:12px;font-weight:400;text-anchor:middle;dominant-baseline:central">usageMetadata.trafficType = ON_DEMAND_PRIORITY</text>
|
||||
</g>
|
||||
|
||||
<!-- Arrow 2 -->
|
||||
<line x1="340" y1="176" x2="340" y2="210" marker-end="url(#arrow)" style="fill:none;stroke:rgb(156, 154, 146);color:rgb(255, 255, 255);stroke-width:1.5px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto"/>
|
||||
|
||||
<!-- Step 3: LiteLLM hidden params -->
|
||||
<g style="fill:rgb(0, 0, 0);stroke:none;color:rgb(255, 255, 255);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto">
|
||||
<rect x="190" y="210" width="300" height="56" rx="8" stroke-width="0.5" style="fill:rgb(60, 52, 137);stroke:rgb(175, 169, 236);color:rgb(255, 255, 255);stroke-width:0.5px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto"/>
|
||||
<text x="340" y="232" text-anchor="middle" dominant-baseline="central" style="fill:rgb(206, 203, 246);stroke:none;color:rgb(255, 255, 255);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:14px;font-weight:500;text-anchor:middle;dominant-baseline:central">LiteLLM stores it</text>
|
||||
<text x="340" y="250" text-anchor="middle" dominant-baseline="central" style="fill:rgb(175, 169, 236);stroke:none;color:rgb(255, 255, 255);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:12px;font-weight:400;text-anchor:middle;dominant-baseline:central">_hidden_params.provider_specific_fields.traffic_type</text>
|
||||
</g>
|
||||
|
||||
<!-- Arrow 3 -->
|
||||
<line x1="340" y1="266" x2="340" y2="300" marker-end="url(#arrow)" style="fill:none;stroke:rgb(156, 154, 146);color:rgb(255, 255, 255);stroke-width:1.5px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto"/>
|
||||
|
||||
<!-- Step 4: completion_cost() -->
|
||||
<g style="fill:rgb(0, 0, 0);stroke:none;color:rgb(255, 255, 255);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto">
|
||||
<rect x="190" y="300" width="300" height="56" rx="8" stroke-width="0.5" style="fill:rgb(99, 56, 6);stroke:rgb(239, 159, 39);color:rgb(255, 255, 255);stroke-width:0.5px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto"/>
|
||||
<text x="340" y="322" text-anchor="middle" dominant-baseline="central" style="fill:rgb(250, 199, 117);stroke:none;color:rgb(255, 255, 255);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:14px;font-weight:500;text-anchor:middle;dominant-baseline:central">completion_cost()</text>
|
||||
<text x="340" y="340" text-anchor="middle" dominant-baseline="central" style="fill:rgb(239, 159, 39);stroke:none;color:rgb(255, 255, 255);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:12px;font-weight:400;text-anchor:middle;dominant-baseline:central">Maps traffic_type → service_tier = "priority"</text>
|
||||
</g>
|
||||
|
||||
<!-- Arrow 4 -->
|
||||
<line x1="340" y1="356" x2="340" y2="390" marker-end="url(#arrow)" style="fill:none;stroke:rgb(156, 154, 146);color:rgb(255, 255, 255);stroke-width:1.5px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto"/>
|
||||
|
||||
<!-- Step 5: Pricing lookup -->
|
||||
<g style="fill:rgb(0, 0, 0);stroke:none;color:rgb(255, 255, 255);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto">
|
||||
<rect x="190" y="390" width="300" height="56" rx="8" stroke-width="0.5" style="fill:rgb(113, 43, 19);stroke:rgb(240, 153, 123);color:rgb(255, 255, 255);stroke-width:0.5px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto"/>
|
||||
<text x="340" y="412" text-anchor="middle" dominant-baseline="central" style="fill:rgb(245, 196, 179);stroke:none;color:rgb(255, 255, 255);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:14px;font-weight:500;text-anchor:middle;dominant-baseline:central">Pricing lookup</text>
|
||||
<text x="340" y="430" text-anchor="middle" dominant-baseline="central" style="fill:rgb(240, 153, 123);stroke:none;color:rgb(255, 255, 255);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:12px;font-weight:400;text-anchor:middle;dominant-baseline:central">input/output_cost_per_token_priority</text>
|
||||
</g>
|
||||
|
||||
<!-- Step numbers in left margin -->
|
||||
<text x="172" y="58" text-anchor="end" dominant-baseline="central" style="fill:rgb(194, 192, 182);stroke:none;color:rgb(255, 255, 255);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:12px;font-weight:400;text-anchor:end;dominant-baseline:central">①</text>
|
||||
<text x="172" y="148" text-anchor="end" dominant-baseline="central" style="fill:rgb(194, 192, 182);stroke:none;color:rgb(255, 255, 255);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:12px;font-weight:400;text-anchor:end;dominant-baseline:central">②</text>
|
||||
<text x="172" y="238" text-anchor="end" dominant-baseline="central" style="fill:rgb(194, 192, 182);stroke:none;color:rgb(255, 255, 255);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:12px;font-weight:400;text-anchor:end;dominant-baseline:central">③</text>
|
||||
<text x="172" y="328" text-anchor="end" dominant-baseline="central" style="fill:rgb(194, 192, 182);stroke:none;color:rgb(255, 255, 255);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:12px;font-weight:400;text-anchor:end;dominant-baseline:central">④</text>
|
||||
<text x="172" y="418" text-anchor="end" dominant-baseline="central" style="fill:rgb(194, 192, 182);stroke:none;color:rgb(255, 255, 255);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:12px;font-weight:400;text-anchor:end;dominant-baseline:central">⑤</text>
|
||||
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 12 KiB |
|
|
@ -29,7 +29,7 @@ from litellm.proxy.openai_files_endpoints.common_utils import (
|
|||
get_models_from_unified_file_id,
|
||||
normalize_mime_type_for_provider,
|
||||
)
|
||||
from litellm.types.llms.openai import (
|
||||
from litellm.types.llms.openai import ( # pyright: ignore[reportAttributeAccessIssue]
|
||||
AllMessageValues,
|
||||
AsyncCursorPage,
|
||||
ChatCompletionFileObject,
|
||||
|
|
@ -442,25 +442,33 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
elif call_type == CallTypes.aresponses.value or call_type == CallTypes.responses.value:
|
||||
# Handle managed files in responses API input and tools
|
||||
file_ids = []
|
||||
|
||||
|
||||
# Extract file IDs from input parameter
|
||||
input_data = data.get("input")
|
||||
if input_data:
|
||||
file_ids.extend(self.get_file_ids_from_responses_input(input_data))
|
||||
|
||||
|
||||
# Extract file IDs from tools parameter (e.g., code_interpreter container)
|
||||
tools = data.get("tools")
|
||||
if tools:
|
||||
file_ids.extend(self.get_file_ids_from_responses_tools(tools))
|
||||
|
||||
|
||||
if file_ids:
|
||||
# Check user has access to all managed files
|
||||
await self.check_file_ids_access(file_ids, user_api_key_dict)
|
||||
|
||||
|
||||
model_file_id_mapping = await self.get_model_file_id_mapping(
|
||||
file_ids, user_api_key_dict.parent_otel_span
|
||||
)
|
||||
data["model_file_id_mapping"] = model_file_id_mapping
|
||||
|
||||
# Check access for file_search vector_store_ids
|
||||
if tools:
|
||||
unified_vs_ids = self.get_vector_store_ids_from_file_search_tools(tools)
|
||||
if unified_vs_ids:
|
||||
await self.check_vector_store_ids_access(
|
||||
unified_vs_ids, user_api_key_dict
|
||||
)
|
||||
elif call_type == CallTypes.afile_content.value:
|
||||
retrieve_file_id = cast(Optional[str], data.get("file_id"))
|
||||
potential_file_id = (
|
||||
|
|
@ -704,6 +712,101 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
|
||||
return file_ids
|
||||
|
||||
def get_vector_store_ids_from_file_search_tools(
|
||||
self, tools: List[Dict[str, Any]]
|
||||
) -> List[str]:
|
||||
"""
|
||||
Extract unified vector_store_ids from file_search tools.
|
||||
|
||||
Only returns IDs that are LiteLLM-managed (base64 unified IDs).
|
||||
Native provider IDs are skipped — they have no LiteLLM access record.
|
||||
"""
|
||||
from litellm.llms.base_llm.managed_resources.utils import (
|
||||
is_base64_encoded_unified_id,
|
||||
)
|
||||
|
||||
vs_ids: List[str] = []
|
||||
if not isinstance(tools, list):
|
||||
return vs_ids
|
||||
|
||||
for tool in tools:
|
||||
if not isinstance(tool, dict) or tool.get("type") != "file_search":
|
||||
continue
|
||||
vector_store_ids = tool.get("vector_store_ids")
|
||||
if not isinstance(vector_store_ids, list):
|
||||
continue
|
||||
for vs_id in vector_store_ids:
|
||||
if isinstance(vs_id, str) and is_base64_encoded_unified_id(vs_id):
|
||||
vs_ids.append(vs_id)
|
||||
|
||||
return vs_ids
|
||||
|
||||
async def check_vector_store_ids_access(
|
||||
self,
|
||||
vector_store_ids: List[str],
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
) -> None:
|
||||
"""
|
||||
Verify the caller's team can access each LiteLLM-managed vector store.
|
||||
|
||||
Batch-fetches vector stores from DB and checks team_id.
|
||||
Raises HTTPException(403) on the first access violation.
|
||||
Non-managed (native) IDs should already be filtered out before calling this.
|
||||
"""
|
||||
from litellm.llms.base_llm.managed_resources.utils import (
|
||||
extract_unified_uuid_from_unified_id,
|
||||
)
|
||||
from litellm.proxy.auth.auth_checks import (
|
||||
get_managed_vector_store_rows_by_uuids,
|
||||
)
|
||||
from litellm.proxy.proxy_server import (
|
||||
prisma_client,
|
||||
proxy_logging_obj,
|
||||
user_api_key_cache,
|
||||
)
|
||||
|
||||
if not vector_store_ids or prisma_client is None:
|
||||
return
|
||||
|
||||
# Map each unified ID to its internal UUID for a single batch DB fetch
|
||||
uuid_to_unified: Dict[str, str] = {}
|
||||
for vs_id in vector_store_ids:
|
||||
uuid = extract_unified_uuid_from_unified_id(vs_id)
|
||||
if uuid:
|
||||
uuid_to_unified[uuid] = vs_id
|
||||
|
||||
if not uuid_to_unified:
|
||||
return
|
||||
|
||||
rows = await get_managed_vector_store_rows_by_uuids(
|
||||
uuids=list(uuid_to_unified.keys()),
|
||||
prisma_client=prisma_client,
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
|
||||
found_uuids = {row.vector_store_id for row in rows}
|
||||
|
||||
for uuid, original_id in uuid_to_unified.items():
|
||||
if uuid not in found_uuids:
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail=f"Vector store '{original_id}' not found or access denied.",
|
||||
)
|
||||
|
||||
caller_team_id = user_api_key_dict.team_id
|
||||
for row in rows:
|
||||
vs_team_id = getattr(row, "team_id", None)
|
||||
if vs_team_id is not None and vs_team_id != caller_team_id:
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail=(
|
||||
f"Team '{caller_team_id}' does not have access to vector "
|
||||
f"store '{row.vector_store_id}'. The store belongs to team "
|
||||
f"'{vs_team_id}'."
|
||||
),
|
||||
)
|
||||
|
||||
async def get_model_file_id_mapping(
|
||||
self, file_ids: List[str], litellm_parent_otel_span: Span
|
||||
) -> dict:
|
||||
|
|
@ -954,7 +1057,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
)
|
||||
else:
|
||||
file_object = await litellm.afile_retrieve(
|
||||
custom_llm_provider=model_name.split("/")[0] if model_name and "/" in model_name else "openai",
|
||||
custom_llm_provider=model_name.split("/")[0] if model_name and "/" in model_name else "openai", # type: ignore[arg-type]
|
||||
file_id=original_file_id,
|
||||
)
|
||||
verbose_logger.debug(
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[tool.poetry]
|
||||
name = "litellm-enterprise"
|
||||
version = "0.1.34"
|
||||
version = "0.1.35"
|
||||
description = "Package for LiteLLM Enterprise features"
|
||||
authors = ["BerriAI"]
|
||||
readme = "README.md"
|
||||
|
|
@ -22,7 +22,7 @@ requires = ["poetry-core"]
|
|||
build-backend = "poetry.core.masonry.api"
|
||||
|
||||
[tool.commitizen]
|
||||
version = "0.1.33"
|
||||
version = "0.1.35"
|
||||
version_files = [
|
||||
"pyproject.toml:version",
|
||||
"../requirements.txt:litellm-enterprise==",
|
||||
|
|
|
|||
BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.60-py3-none-any.whl
vendored
Normal file
BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.60-py3-none-any.whl
vendored
Normal file
Binary file not shown.
BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.60.tar.gz
vendored
Normal file
BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.60.tar.gz
vendored
Normal file
Binary file not shown.
|
|
@ -1,9 +1,9 @@
|
|||
-- CreateIndex
|
||||
CREATE INDEX "LiteLLM_TeamTable_organization_id_idx" ON "LiteLLM_TeamTable"("organization_id");
|
||||
CREATE INDEX IF NOT EXISTS "LiteLLM_TeamTable_organization_id_idx" ON "LiteLLM_TeamTable"("organization_id");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "LiteLLM_TeamTable_team_alias_idx" ON "LiteLLM_TeamTable"("team_alias");
|
||||
CREATE INDEX IF NOT EXISTS "LiteLLM_TeamTable_team_alias_idx" ON "LiteLLM_TeamTable"("team_alias");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "LiteLLM_TeamTable_created_at_idx" ON "LiteLLM_TeamTable"("created_at");
|
||||
CREATE INDEX IF NOT EXISTS "LiteLLM_TeamTable_created_at_idx" ON "LiteLLM_TeamTable"("created_at");
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[tool.poetry]
|
||||
name = "litellm-proxy-extras"
|
||||
version = "0.4.58"
|
||||
version = "0.4.60"
|
||||
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
|
||||
authors = ["BerriAI"]
|
||||
readme = "README.md"
|
||||
|
|
@ -22,7 +22,7 @@ requires = ["poetry-core"]
|
|||
build-backend = "poetry.core.masonry.api"
|
||||
|
||||
[tool.commitizen]
|
||||
version = "0.4.58"
|
||||
version = "0.4.60"
|
||||
version_files = [
|
||||
"pyproject.toml:version",
|
||||
"../requirements.txt:litellm-proxy-extras==",
|
||||
|
|
|
|||
|
|
@ -97,6 +97,7 @@ input_callback: List[CALLBACK_TYPES] = []
|
|||
success_callback: List[CALLBACK_TYPES] = []
|
||||
failure_callback: List[CALLBACK_TYPES] = []
|
||||
service_callback: List[CALLBACK_TYPES] = []
|
||||
audit_log_callbacks: List[CALLBACK_TYPES] = []
|
||||
# logging_callback_manager is lazy-loaded via __getattr__
|
||||
_custom_logger_compatible_callbacks_literal = Literal[
|
||||
"lago",
|
||||
|
|
|
|||
|
|
@ -52,6 +52,17 @@ def _build_secret_patterns() -> re.Pattern:
|
|||
r"(?<=://)[^\s'\"]*:[^\s'\"@]+(?=@)",
|
||||
# Databricks personal access tokens
|
||||
r"dapi[0-9a-f]{32}",
|
||||
# ── Key-name-based redaction ──
|
||||
# Catches secrets inside dicts/config dumps by matching on the KEY name
|
||||
# regardless of what the value looks like.
|
||||
# e.g. 'master_key': 'any-value-here', "database_url": "postgres://..."
|
||||
r"(?:master_key|database_url|db_url|connection_string|"
|
||||
r"private_key|signing_key|encryption_key|"
|
||||
r"auth_token|access_token|refresh_token|"
|
||||
r"slack_webhook_url|webhook_url|"
|
||||
r"database_connection_string|"
|
||||
r"huggingface_token|jwt_secret)"
|
||||
r"""['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+""",
|
||||
]
|
||||
return re.compile("|".join(patterns), re.IGNORECASE)
|
||||
|
||||
|
|
@ -272,7 +283,7 @@ verbose_proxy_logger = logging.getLogger("LiteLLM Proxy")
|
|||
verbose_router_logger = logging.getLogger("LiteLLM Router")
|
||||
verbose_logger = logging.getLogger("LiteLLM")
|
||||
|
||||
# Add the handler to the logger
|
||||
# Add the handler to the loggers
|
||||
verbose_router_logger.addHandler(handler)
|
||||
verbose_proxy_logger.addHandler(handler)
|
||||
verbose_logger.addHandler(handler)
|
||||
|
|
|
|||
|
|
@ -222,8 +222,12 @@ def _get_redis_client_logic(**env_overrides):
|
|||
"REDIS_CLUSTER_NODES"
|
||||
)
|
||||
|
||||
# If startup_nodes resolved to None (not set by kwarg or env), remove the key
|
||||
# entirely so callers can rely on key presence as a reliable cluster-mode signal.
|
||||
if _startup_nodes is not None and isinstance(_startup_nodes, str):
|
||||
redis_kwargs["startup_nodes"] = json.loads(_startup_nodes)
|
||||
elif _startup_nodes is None:
|
||||
redis_kwargs.pop("startup_nodes", None)
|
||||
|
||||
_sentinel_nodes: Optional[Union[str, list]] = redis_kwargs.get("sentinel_nodes", None) or get_secret( # type: ignore
|
||||
"REDIS_SENTINEL_NODES"
|
||||
|
|
@ -273,10 +277,14 @@ def _get_redis_client_logic(**env_overrides):
|
|||
redis_kwargs["ssl_ca_certs"] = _gcp_ssl_ca_certs
|
||||
|
||||
if "url" in redis_kwargs and redis_kwargs["url"] is not None:
|
||||
redis_kwargs.pop("host", None)
|
||||
redis_kwargs.pop("port", None)
|
||||
redis_kwargs.pop("db", None)
|
||||
redis_kwargs.pop("password", None)
|
||||
# Only strip host/port/db/password when not routing to a cluster.
|
||||
# When startup_nodes is also present the cluster path takes priority and
|
||||
# needs the password for authentication.
|
||||
if not redis_kwargs.get("startup_nodes"):
|
||||
redis_kwargs.pop("host", None)
|
||||
redis_kwargs.pop("port", None)
|
||||
redis_kwargs.pop("db", None)
|
||||
redis_kwargs.pop("password", None)
|
||||
elif "startup_nodes" in redis_kwargs and redis_kwargs["startup_nodes"] is not None:
|
||||
pass
|
||||
elif (
|
||||
|
|
@ -368,6 +376,10 @@ def _init_async_redis_sentinel(redis_kwargs) -> async_redis.Redis:
|
|||
|
||||
def get_redis_client(**env_overrides):
|
||||
redis_kwargs = _get_redis_client_logic(**env_overrides)
|
||||
|
||||
if "startup_nodes" in redis_kwargs:
|
||||
return init_redis_cluster(redis_kwargs)
|
||||
|
||||
if "url" in redis_kwargs and redis_kwargs["url"] is not None:
|
||||
args = _get_redis_url_kwargs()
|
||||
url_kwargs = {}
|
||||
|
|
@ -377,9 +389,6 @@ def get_redis_client(**env_overrides):
|
|||
|
||||
return redis.Redis.from_url(**url_kwargs)
|
||||
|
||||
if "startup_nodes" in redis_kwargs or get_secret("REDIS_CLUSTER_NODES") is not None: # type: ignore
|
||||
return init_redis_cluster(redis_kwargs)
|
||||
|
||||
# Check for Redis Sentinel
|
||||
if "sentinel_nodes" in redis_kwargs and "service_name" in redis_kwargs:
|
||||
return _init_redis_sentinel(redis_kwargs)
|
||||
|
|
@ -392,21 +401,6 @@ def get_redis_async_client(
|
|||
**env_overrides,
|
||||
) -> Union[async_redis.Redis, async_redis.RedisCluster]:
|
||||
redis_kwargs = _get_redis_client_logic(**env_overrides)
|
||||
if "url" in redis_kwargs and redis_kwargs["url"] is not None:
|
||||
if connection_pool is not None:
|
||||
return async_redis.Redis(connection_pool=connection_pool)
|
||||
args = _get_redis_url_kwargs(client=async_redis.Redis.from_url)
|
||||
url_kwargs = {}
|
||||
for arg in redis_kwargs:
|
||||
if arg in args:
|
||||
url_kwargs[arg] = redis_kwargs[arg]
|
||||
else:
|
||||
verbose_logger.debug(
|
||||
"REDIS: ignoring argument: {}. Not an allowed async_redis.Redis.from_url arg.".format(
|
||||
arg
|
||||
)
|
||||
)
|
||||
return async_redis.Redis.from_url(**url_kwargs)
|
||||
|
||||
if "startup_nodes" in redis_kwargs:
|
||||
from redis.cluster import ClusterNode
|
||||
|
|
@ -469,6 +463,22 @@ def get_redis_async_client(
|
|||
|
||||
return cluster_client
|
||||
|
||||
if "url" in redis_kwargs and redis_kwargs["url"] is not None:
|
||||
if connection_pool is not None:
|
||||
return async_redis.Redis(connection_pool=connection_pool)
|
||||
args = _get_redis_url_kwargs(client=async_redis.Redis.from_url)
|
||||
url_kwargs = {}
|
||||
for arg in redis_kwargs:
|
||||
if arg in args:
|
||||
url_kwargs[arg] = redis_kwargs[arg]
|
||||
else:
|
||||
verbose_logger.debug(
|
||||
"REDIS: ignoring argument: {}. Not an allowed async_redis.Redis.from_url arg.".format(
|
||||
arg
|
||||
)
|
||||
)
|
||||
return async_redis.Redis.from_url(**url_kwargs)
|
||||
|
||||
# Check for Redis Sentinel
|
||||
if "sentinel_nodes" in redis_kwargs and "service_name" in redis_kwargs:
|
||||
return _init_async_redis_sentinel(redis_kwargs)
|
||||
|
|
@ -482,9 +492,15 @@ def get_redis_async_client(
|
|||
)
|
||||
|
||||
|
||||
def get_redis_connection_pool(**env_overrides):
|
||||
def get_redis_connection_pool(
|
||||
**env_overrides,
|
||||
) -> Optional[async_redis.BlockingConnectionPool]:
|
||||
redis_kwargs = _get_redis_client_logic(**env_overrides)
|
||||
verbose_logger.debug("get_redis_connection_pool: redis_kwargs", redis_kwargs)
|
||||
|
||||
if "startup_nodes" in redis_kwargs:
|
||||
return None
|
||||
|
||||
if "url" in redis_kwargs and redis_kwargs["url"] is not None:
|
||||
pool_kwargs = {
|
||||
"timeout": REDIS_CONNECTION_POOL_TIMEOUT,
|
||||
|
|
@ -504,7 +520,6 @@ def get_redis_connection_pool(**env_overrides):
|
|||
connection_class = async_redis.SSLConnection
|
||||
redis_kwargs.pop("ssl", None)
|
||||
redis_kwargs["connection_class"] = connection_class
|
||||
redis_kwargs.pop("startup_nodes", None)
|
||||
return async_redis.BlockingConnectionPool(
|
||||
timeout=REDIS_CONNECTION_POOL_TIMEOUT, **redis_kwargs
|
||||
)
|
||||
|
|
|
|||
|
|
@ -885,7 +885,7 @@ def list_batches(
|
|||
async def acancel_batch(
|
||||
batch_id: str,
|
||||
model: Optional[str] = None,
|
||||
custom_llm_provider: Literal["openai", "azure"] = "openai",
|
||||
custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
|
||||
metadata: Optional[Dict[str, str]] = None,
|
||||
extra_headers: Optional[Dict[str, str]] = None,
|
||||
extra_body: Optional[Dict[str, str]] = None,
|
||||
|
|
@ -931,7 +931,7 @@ async def acancel_batch(
|
|||
def cancel_batch(
|
||||
batch_id: str,
|
||||
model: Optional[str] = None,
|
||||
custom_llm_provider: Union[Literal["openai", "azure"], str] = "openai",
|
||||
custom_llm_provider: Union[Literal["openai", "azure", "vertex_ai"], str] = "openai",
|
||||
metadata: Optional[Dict[str, str]] = None,
|
||||
extra_headers: Optional[Dict[str, str]] = None,
|
||||
extra_body: Optional[Dict[str, str]] = None,
|
||||
|
|
@ -1048,9 +1048,35 @@ def cancel_batch(
|
|||
cancel_batch_data=_cancel_batch_request,
|
||||
litellm_params=litellm_params,
|
||||
)
|
||||
elif custom_llm_provider == "vertex_ai":
|
||||
api_base = optional_params.api_base or None
|
||||
vertex_ai_project = (
|
||||
optional_params.vertex_project
|
||||
or litellm.vertex_project
|
||||
or get_secret_str("VERTEXAI_PROJECT")
|
||||
)
|
||||
vertex_ai_location = (
|
||||
optional_params.vertex_location
|
||||
or litellm.vertex_location
|
||||
or get_secret_str("VERTEXAI_LOCATION")
|
||||
)
|
||||
vertex_credentials = optional_params.vertex_credentials or get_secret_str(
|
||||
"VERTEXAI_CREDENTIALS"
|
||||
)
|
||||
|
||||
response = vertex_ai_batches_instance.cancel_batch(
|
||||
_is_async=_is_async,
|
||||
batch_id=batch_id,
|
||||
api_base=api_base,
|
||||
vertex_project=vertex_ai_project,
|
||||
vertex_location=vertex_ai_location,
|
||||
vertex_credentials=vertex_credentials,
|
||||
timeout=timeout,
|
||||
max_retries=optional_params.max_retries,
|
||||
)
|
||||
else:
|
||||
raise litellm.exceptions.BadRequestError(
|
||||
message="LiteLLM doesn't support {} for 'cancel_batch'. Only 'openai' and 'azure' are supported.".format(
|
||||
message="LiteLLM doesn't support {} for 'cancel_batch'. Only 'openai', 'azure', and 'vertex_ai' are supported.".format(
|
||||
custom_llm_provider
|
||||
),
|
||||
model="n/a",
|
||||
|
|
|
|||
|
|
@ -393,16 +393,17 @@ class DualCache(BaseCache):
|
|||
parent_otel_span: Optional[Span] = None,
|
||||
local_only: bool = False,
|
||||
**kwargs,
|
||||
) -> float:
|
||||
) -> Optional[float]:
|
||||
"""
|
||||
Key - the key in cache
|
||||
|
||||
Value - float - the value you want to increment by
|
||||
|
||||
Returns - float - the incremented value
|
||||
Returns - the incremented value, or None if no cache backend is
|
||||
available (in_memory_cache is None and Redis failed/is absent).
|
||||
"""
|
||||
result: Optional[float] = None
|
||||
try:
|
||||
result: float = value
|
||||
if self.in_memory_cache is not None:
|
||||
result = await self.in_memory_cache.async_increment(
|
||||
key, value, **kwargs
|
||||
|
|
@ -418,7 +419,11 @@ class DualCache(BaseCache):
|
|||
|
||||
return result
|
||||
except Exception as e:
|
||||
raise e # don't log if exception is raised
|
||||
verbose_logger.warning(
|
||||
"Redis async_increment_cache failed, falling back to in-memory result: %s",
|
||||
e,
|
||||
)
|
||||
return result
|
||||
|
||||
async def async_increment_cache_pipeline(
|
||||
self,
|
||||
|
|
@ -427,8 +432,8 @@ class DualCache(BaseCache):
|
|||
parent_otel_span: Optional[Span] = None,
|
||||
**kwargs,
|
||||
) -> Optional[List[float]]:
|
||||
result: Optional[List[float]] = None
|
||||
try:
|
||||
result: Optional[List[float]] = None
|
||||
if self.in_memory_cache is not None:
|
||||
result = await self.in_memory_cache.async_increment_pipeline(
|
||||
increment_list=increment_list,
|
||||
|
|
@ -443,7 +448,11 @@ class DualCache(BaseCache):
|
|||
|
||||
return result
|
||||
except Exception as e:
|
||||
raise e # don't log if exception is raised
|
||||
verbose_logger.warning(
|
||||
"Redis async_increment_cache_pipeline failed, falling back to in-memory result: %s",
|
||||
e,
|
||||
)
|
||||
return result
|
||||
|
||||
async def async_set_cache_sadd(
|
||||
self, key, value: List, local_only: bool = False, **kwargs
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ Has 4 primary methods:
|
|||
|
||||
import ast
|
||||
import asyncio
|
||||
import functools
|
||||
import hashlib
|
||||
import inspect
|
||||
import json
|
||||
|
|
@ -19,7 +20,11 @@ from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Union, cast
|
|||
|
||||
import litellm
|
||||
from litellm._logging import print_verbose, verbose_logger
|
||||
from litellm.constants import DEFAULT_REDIS_MAJOR_VERSION
|
||||
from litellm.constants import (
|
||||
DEFAULT_REDIS_MAJOR_VERSION,
|
||||
REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD,
|
||||
REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import _get_parent_otel_span_from_kwargs
|
||||
from litellm.litellm_core_utils.coroutine_checker import coroutine_checker
|
||||
from litellm.types.caching import (
|
||||
|
|
@ -89,6 +94,91 @@ def _get_call_stack_info(num_frames: int = 2) -> str:
|
|||
return "unknown"
|
||||
|
||||
|
||||
class RedisCircuitBreaker:
|
||||
"""
|
||||
Tracks Redis health for a RedisCache instance.
|
||||
|
||||
States:
|
||||
CLOSED - normal, Redis is called
|
||||
OPEN - Redis is down, raise immediately (no network call)
|
||||
HALF_OPEN - recovery probe: allow one request through
|
||||
|
||||
Transitions:
|
||||
CLOSED -> OPEN after failure_threshold consecutive failures
|
||||
OPEN -> HALF_OPEN after recovery_timeout seconds
|
||||
HALF_OPEN -> CLOSED on success
|
||||
HALF_OPEN -> OPEN on failure (resets timer)
|
||||
"""
|
||||
|
||||
CLOSED = "closed"
|
||||
OPEN = "open"
|
||||
HALF_OPEN = "half_open"
|
||||
|
||||
def __init__(self, failure_threshold: int, recovery_timeout: int) -> None:
|
||||
self.failure_threshold = failure_threshold
|
||||
self.recovery_timeout = recovery_timeout
|
||||
self._failure_count = 0
|
||||
self._opened_at: Optional[float] = None
|
||||
self._state = self.CLOSED
|
||||
|
||||
def is_open(self) -> bool:
|
||||
"""Returns True if Redis calls should be skipped."""
|
||||
if self._state == self.HALF_OPEN:
|
||||
# Probe already in flight — fast-fail all concurrent requests.
|
||||
# Only the one call that caused the OPEN→HALF_OPEN transition
|
||||
# (which returned False) is the designated probe.
|
||||
return True
|
||||
if self._state == self.OPEN:
|
||||
if time.time() - (self._opened_at or 0) > self.recovery_timeout:
|
||||
self._state = self.HALF_OPEN
|
||||
return False # this caller is the designated probe
|
||||
return True
|
||||
return False
|
||||
|
||||
def record_failure(self) -> None:
|
||||
self._failure_count += 1
|
||||
self._opened_at = time.time()
|
||||
if self._failure_count >= self.failure_threshold:
|
||||
if self._state != self.OPEN:
|
||||
verbose_logger.warning(
|
||||
"Redis circuit breaker OPENED after %d consecutive failures — "
|
||||
"fast-failing Redis calls for %ds",
|
||||
self._failure_count,
|
||||
self.recovery_timeout,
|
||||
)
|
||||
self._state = self.OPEN
|
||||
|
||||
def record_success(self) -> None:
|
||||
if self._state == self.HALF_OPEN:
|
||||
verbose_logger.info("Redis circuit breaker CLOSED — Redis recovered")
|
||||
self._failure_count = 0
|
||||
self._state = self.CLOSED
|
||||
|
||||
|
||||
def _redis_circuit_breaker_guard(method): # type: ignore
|
||||
"""
|
||||
Decorator for RedisCache async methods.
|
||||
Checks the circuit breaker before each call; records success/failure after.
|
||||
Does not apply to ping/disconnect/test_connection (health/teardown must always run).
|
||||
"""
|
||||
|
||||
@functools.wraps(method)
|
||||
async def wrapper(self, *args, **kwargs): # type: ignore
|
||||
if self._circuit_breaker.is_open():
|
||||
raise Exception(
|
||||
f"Redis circuit breaker is open — skipping {method.__name__}"
|
||||
)
|
||||
try:
|
||||
result = await method(self, *args, **kwargs)
|
||||
self._circuit_breaker.record_success()
|
||||
return result
|
||||
except Exception:
|
||||
self._circuit_breaker.record_failure()
|
||||
raise
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
class RedisCache(BaseCache):
|
||||
# if users don't provider one, use the default litellm cache
|
||||
|
||||
|
|
@ -150,6 +240,11 @@ class RedisCache(BaseCache):
|
|||
except Exception:
|
||||
pass
|
||||
|
||||
self._circuit_breaker = RedisCircuitBreaker(
|
||||
failure_threshold=REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD,
|
||||
recovery_timeout=REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT,
|
||||
)
|
||||
|
||||
self._setup_health_pings()
|
||||
|
||||
if litellm.default_redis_ttl is not None:
|
||||
|
|
@ -375,6 +470,7 @@ class RedisCache(BaseCache):
|
|||
)
|
||||
raise e
|
||||
|
||||
@_redis_circuit_breaker_guard
|
||||
async def async_scan_iter(self, pattern: str, count: int = 100) -> list:
|
||||
start_time = time.time()
|
||||
try:
|
||||
|
|
@ -451,6 +547,7 @@ class RedisCache(BaseCache):
|
|||
verbose_logger.error(f"Error registering Redis script: {str(e)}")
|
||||
raise e
|
||||
|
||||
@_redis_circuit_breaker_guard
|
||||
async def async_set_cache(self, key, value, **kwargs):
|
||||
from redis.asyncio import Redis
|
||||
|
||||
|
|
@ -560,6 +657,7 @@ class RedisCache(BaseCache):
|
|||
results = await pipe.execute()
|
||||
return results
|
||||
|
||||
@_redis_circuit_breaker_guard
|
||||
async def async_set_cache_pipeline(
|
||||
self, cache_list: List[Tuple[Any, Any]], ttl: Optional[float] = None, **kwargs
|
||||
):
|
||||
|
|
@ -636,6 +734,7 @@ class RedisCache(BaseCache):
|
|||
except Exception:
|
||||
raise
|
||||
|
||||
@_redis_circuit_breaker_guard
|
||||
async def async_set_cache_sadd(
|
||||
self, key, value: List, ttl: Optional[float], **kwargs
|
||||
):
|
||||
|
|
@ -708,6 +807,7 @@ class RedisCache(BaseCache):
|
|||
value,
|
||||
)
|
||||
|
||||
@_redis_circuit_breaker_guard
|
||||
async def batch_cache_write(self, key, value, **kwargs):
|
||||
print_verbose(
|
||||
f"in batch cache writing for redis buffer size={len(self.redis_batch_writing_buffer)}",
|
||||
|
|
@ -717,6 +817,7 @@ class RedisCache(BaseCache):
|
|||
if len(self.redis_batch_writing_buffer) >= self.redis_flush_size:
|
||||
await self.flush_cache_buffer() # logging done in here
|
||||
|
||||
@_redis_circuit_breaker_guard
|
||||
async def async_increment(
|
||||
self,
|
||||
key,
|
||||
|
|
@ -894,6 +995,7 @@ class RedisCache(BaseCache):
|
|||
verbose_logger.error(f"Error occurred in batch get cache - {str(e)}")
|
||||
return key_value_dict
|
||||
|
||||
@_redis_circuit_breaker_guard
|
||||
async def async_get_cache(
|
||||
self, key, parent_otel_span: Optional[Span] = None, **kwargs
|
||||
):
|
||||
|
|
@ -944,6 +1046,7 @@ class RedisCache(BaseCache):
|
|||
f"litellm.caching.caching: async get() - Got exception from REDIS: {str(e)}"
|
||||
)
|
||||
|
||||
@_redis_circuit_breaker_guard
|
||||
async def async_batch_get_cache(
|
||||
self,
|
||||
key_list: Union[List[str], List[Optional[str]]],
|
||||
|
|
@ -1087,6 +1190,7 @@ class RedisCache(BaseCache):
|
|||
)
|
||||
raise e
|
||||
|
||||
@_redis_circuit_breaker_guard
|
||||
async def delete_cache_keys(self, keys):
|
||||
# typed as Any, redis python lib has incomplete type stubs for RedisCluster and does not include `delete`
|
||||
_redis_client: Any = self.init_async_client()
|
||||
|
|
@ -1151,6 +1255,7 @@ class RedisCache(BaseCache):
|
|||
"error": str(e),
|
||||
}
|
||||
|
||||
@_redis_circuit_breaker_guard
|
||||
async def async_delete_cache(self, key: str):
|
||||
# typed as Any, redis python lib has incomplete type stubs for RedisCluster and does not include `delete`
|
||||
_redis_client: Any = self.init_async_client()
|
||||
|
|
@ -1184,6 +1289,7 @@ class RedisCache(BaseCache):
|
|||
)
|
||||
return [r for r in results if isinstance(r, float)]
|
||||
|
||||
@_redis_circuit_breaker_guard
|
||||
async def async_increment_pipeline(
|
||||
self, increment_list: List[RedisPipelineIncrementOperation], **kwargs
|
||||
) -> Optional[List[float]]:
|
||||
|
|
@ -1247,6 +1353,7 @@ class RedisCache(BaseCache):
|
|||
)
|
||||
raise e
|
||||
|
||||
@_redis_circuit_breaker_guard
|
||||
async def async_get_ttl(self, key: str) -> Optional[int]:
|
||||
"""
|
||||
Get the remaining TTL of a key in Redis
|
||||
|
|
@ -1270,6 +1377,7 @@ class RedisCache(BaseCache):
|
|||
verbose_logger.debug(f"Redis TTL Error: {e}")
|
||||
return None
|
||||
|
||||
@_redis_circuit_breaker_guard
|
||||
async def async_rpush(
|
||||
self,
|
||||
key: str,
|
||||
|
|
@ -1336,6 +1444,7 @@ class RedisCache(BaseCache):
|
|||
raise r
|
||||
return results
|
||||
|
||||
@_redis_circuit_breaker_guard
|
||||
async def async_rpush_pipeline(
|
||||
self,
|
||||
rpush_list: List[RedisPipelineRpushOperation],
|
||||
|
|
@ -1405,6 +1514,7 @@ class RedisCache(BaseCache):
|
|||
|
||||
return result
|
||||
|
||||
@_redis_circuit_breaker_guard
|
||||
async def async_lpop(
|
||||
self,
|
||||
key: str,
|
||||
|
|
@ -1534,6 +1644,7 @@ class RedisCache(BaseCache):
|
|||
decoded_results.append(None)
|
||||
return decoded_results
|
||||
|
||||
@_redis_circuit_breaker_guard
|
||||
async def async_lpop_pipeline(
|
||||
self,
|
||||
lpop_list: List[RedisPipelineLpopOperation],
|
||||
|
|
|
|||
|
|
@ -696,6 +696,20 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
verbose_logger.debug(
|
||||
f"Chat provider: image -> {converted}"
|
||||
)
|
||||
elif item_type == "file":
|
||||
# Map Chat Completion file to Responses API input_file
|
||||
# {"type": "file", "file": {"file_data": "...", "filename": "..."}}
|
||||
# -> {"type": "input_file", "file_data": "...", "filename": "..."}
|
||||
file_data = item.get("file", {})
|
||||
converted = {"type": "input_file"}
|
||||
if isinstance(file_data, dict):
|
||||
for key in ["file_id", "file_data", "filename"]:
|
||||
if key in file_data:
|
||||
converted[key] = file_data[key]
|
||||
result.append(converted)
|
||||
verbose_logger.debug(
|
||||
f"Chat provider: file -> {converted}"
|
||||
)
|
||||
elif item_type in [
|
||||
"input_text",
|
||||
"input_image",
|
||||
|
|
|
|||
|
|
@ -350,6 +350,12 @@ AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI = int(
|
|||
)
|
||||
REDIS_SOCKET_TIMEOUT = float(os.getenv("REDIS_SOCKET_TIMEOUT", 0.1))
|
||||
REDIS_CONNECTION_POOL_TIMEOUT = int(os.getenv("REDIS_CONNECTION_POOL_TIMEOUT", 5))
|
||||
REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD = int(
|
||||
os.getenv("REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD", 5)
|
||||
)
|
||||
REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT = int(
|
||||
os.getenv("REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT", 60)
|
||||
)
|
||||
# Default Redis major version to assume when version cannot be determined
|
||||
# Using 7 as it's the modern version that supports LPOP with count parameter
|
||||
DEFAULT_REDIS_MAJOR_VERSION = int(os.getenv("DEFAULT_REDIS_MAJOR_VERSION", 7))
|
||||
|
|
|
|||
|
|
@ -60,13 +60,20 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
# Create a deep copy of messages to avoid modifying the original list
|
||||
processed_messages = copy.deepcopy(messages)
|
||||
|
||||
# Process message-level cache controls
|
||||
# Separate message-level and non-message-level injection points
|
||||
remaining_points = []
|
||||
for point in injection_points:
|
||||
if point.get("location") == "message":
|
||||
point = cast(CacheControlMessageInjectionPoint, point)
|
||||
processed_messages = self._process_message_injection(
|
||||
point=point, messages=processed_messages
|
||||
)
|
||||
else:
|
||||
remaining_points.append(point)
|
||||
|
||||
# Pass through non-message injection points for provider-specific handling
|
||||
if remaining_points:
|
||||
non_default_params["cache_control_injection_points"] = remaining_points
|
||||
|
||||
return model, processed_messages, non_default_params
|
||||
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ from litellm.types.utils import (
|
|||
LLMResponseTypes,
|
||||
ModelResponse,
|
||||
ModelResponseStream,
|
||||
StandardAuditLogPayload,
|
||||
StandardCallbackDynamicParams,
|
||||
StandardLoggingPayload,
|
||||
)
|
||||
|
|
@ -177,6 +178,10 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
|
|||
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
|
||||
pass
|
||||
|
||||
async def async_log_audit_log_event(self, audit_log: "StandardAuditLogPayload"):
|
||||
"""Called when an audit log is created. Override in subclasses to handle."""
|
||||
pass
|
||||
|
||||
#### PROMPT MANAGEMENT HOOKS ####
|
||||
|
||||
async def async_get_chat_completion_prompt(
|
||||
|
|
|
|||
|
|
@ -83,7 +83,28 @@ class LangsmithLogger(CustomBatchLogger):
|
|||
if _batch_size:
|
||||
self.batch_size = int(_batch_size)
|
||||
self.log_queue: List[LangsmithQueueObject] = []
|
||||
asyncio.create_task(self.periodic_flush())
|
||||
self._flush_task: Optional[
|
||||
asyncio.Task[Any]
|
||||
] = self._start_periodic_flush_task()
|
||||
|
||||
def _start_periodic_flush_task(self) -> Optional[asyncio.Task[Any]]:
|
||||
"""Start the periodic flush task only when an event loop is already running."""
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
verbose_logger.debug(
|
||||
"Langsmith logger init: no running event loop, skipping periodic flush task startup"
|
||||
)
|
||||
return None
|
||||
|
||||
return loop.create_task(self.periodic_flush())
|
||||
|
||||
def _ensure_periodic_flush_task(self) -> None:
|
||||
# This helper is intentionally synchronous. In asyncio's cooperative
|
||||
# execution model, there is no await between the check and assignment,
|
||||
# so one caller cannot interleave here and create a duplicate task.
|
||||
if self._flush_task is None or self._flush_task.done():
|
||||
self._flush_task = self._start_periodic_flush_task()
|
||||
|
||||
def get_credentials_from_env(
|
||||
self,
|
||||
|
|
@ -266,6 +287,7 @@ class LangsmithLogger(CustomBatchLogger):
|
|||
|
||||
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
|
||||
try:
|
||||
self._ensure_periodic_flush_task()
|
||||
sampling_rate = self._get_sampling_rate_to_use_for_request(kwargs=kwargs)
|
||||
random_sample = random.random()
|
||||
if random_sample > sampling_rate:
|
||||
|
|
@ -307,17 +329,18 @@ class LangsmithLogger(CustomBatchLogger):
|
|||
)
|
||||
|
||||
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
|
||||
sampling_rate = self._get_sampling_rate_to_use_for_request(kwargs=kwargs)
|
||||
random_sample = random.random()
|
||||
if random_sample > sampling_rate:
|
||||
verbose_logger.info(
|
||||
"Skipping Langsmith logging. Sampling rate={}, random_sample={}".format(
|
||||
sampling_rate, random_sample
|
||||
)
|
||||
)
|
||||
return # Skip logging
|
||||
verbose_logger.info("Langsmith Failure Event Logging!")
|
||||
try:
|
||||
self._ensure_periodic_flush_task()
|
||||
sampling_rate = self._get_sampling_rate_to_use_for_request(kwargs=kwargs)
|
||||
random_sample = random.random()
|
||||
if random_sample > sampling_rate:
|
||||
verbose_logger.info(
|
||||
"Skipping Langsmith logging. Sampling rate={}, random_sample={}".format(
|
||||
sampling_rate, random_sample
|
||||
)
|
||||
)
|
||||
return # Skip logging
|
||||
verbose_logger.info("Langsmith Failure Event Logging!")
|
||||
credentials = self._get_credentials_to_use_for_request(kwargs=kwargs)
|
||||
data = self._prepare_log_data(
|
||||
kwargs=kwargs,
|
||||
|
|
|
|||
|
|
@ -931,10 +931,14 @@ class PrometheusLogger(CustomLogger):
|
|||
user_api_key_auth_metadata: Optional[dict] = standard_logging_payload[
|
||||
"metadata"
|
||||
].get("user_api_key_auth_metadata")
|
||||
spend_logs_metadata: Optional[dict] = standard_logging_payload["metadata"].get(
|
||||
"spend_logs_metadata"
|
||||
)
|
||||
|
||||
combined_metadata: Dict[str, Any] = {
|
||||
**(_requester_metadata if _requester_metadata else {}),
|
||||
**(user_api_key_auth_metadata if user_api_key_auth_metadata else {}),
|
||||
**(spend_logs_metadata if spend_logs_metadata else {}),
|
||||
}
|
||||
if standard_logging_payload is not None and isinstance(
|
||||
standard_logging_payload, dict
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ from litellm.llms.custom_httpx.http_handler import (
|
|||
httpxSpecialProvider,
|
||||
)
|
||||
from litellm.types.integrations.s3_v2 import s3BatchLoggingElement
|
||||
from litellm.types.utils import StandardLoggingPayload
|
||||
from litellm.types.utils import StandardAuditLogPayload, StandardLoggingPayload
|
||||
|
||||
from .custom_batch_logger import CustomBatchLogger
|
||||
|
||||
|
|
@ -248,6 +248,38 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
|
|||
)
|
||||
pass
|
||||
|
||||
async def async_log_audit_log_event(
|
||||
self, audit_log: StandardAuditLogPayload
|
||||
) -> None:
|
||||
"""Batch audit logs and upload to S3 under audit_logs/ prefix."""
|
||||
try:
|
||||
from datetime import timezone
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
audit_log_id = audit_log.get("id", "unknown")
|
||||
|
||||
s3_path = cast(Optional[str], self.s3_path) or ""
|
||||
s3_path = s3_path.rstrip("/") + "/" if s3_path else ""
|
||||
|
||||
s3_object_key = (
|
||||
f"{s3_path}audit_logs/"
|
||||
f"{now.strftime('%Y-%m-%d')}/"
|
||||
f"{now.strftime('%H-%M-%S')}_{audit_log_id}.json"
|
||||
)
|
||||
|
||||
element = s3BatchLoggingElement(
|
||||
payload=dict(audit_log),
|
||||
s3_object_key=s3_object_key,
|
||||
s3_object_download_filename=f"audit-{audit_log_id}.json",
|
||||
)
|
||||
|
||||
self.log_queue.append(element)
|
||||
|
||||
if len(self.log_queue) >= self.batch_size:
|
||||
await self.flush_queue()
|
||||
except Exception as e:
|
||||
verbose_logger.exception("S3 audit log error: %s", e)
|
||||
|
||||
async def _async_log_event_base(self, kwargs, response_obj, start_time, end_time):
|
||||
try:
|
||||
verbose_logger.debug(
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ server-side using litellm router's search tools.
|
|||
|
||||
import asyncio
|
||||
import math
|
||||
import uuid
|
||||
from typing import Any, Dict, List, Optional, Tuple, Union, cast
|
||||
|
||||
import litellm
|
||||
|
|
@ -27,7 +28,9 @@ from litellm.integrations.websearch_interception.transformation import (
|
|||
from litellm.types.integrations.websearch_interception import (
|
||||
WebSearchInterceptionConfig,
|
||||
)
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.utils import LlmProviders
|
||||
from litellm.utils import ProviderConfigManager
|
||||
|
||||
|
||||
class WebSearchInterceptionLogger(CustomLogger):
|
||||
|
|
@ -67,6 +70,111 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
self.search_tool_name = search_tool_name
|
||||
self._request_has_websearch = False # Track if current request has web search
|
||||
|
||||
async def try_short_circuit_search(
|
||||
self,
|
||||
model: str,
|
||||
messages: List[Dict],
|
||||
tools: Optional[List[Dict]],
|
||||
custom_llm_provider: Optional[str],
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Short-circuit web-search-only requests by executing the search directly.
|
||||
|
||||
Claude Code sends web search as a separate, standalone /v1/messages
|
||||
request with a simple prompt and only web_search tool(s). For providers
|
||||
that don't natively support web search (e.g. github_copilot), there is
|
||||
no need to route this through the backend LLM — we can detect the
|
||||
pattern, execute the search via Tavily/Perplexity, and return a
|
||||
synthetic Anthropic response immediately.
|
||||
|
||||
Args:
|
||||
model: Model name from the request
|
||||
messages: Messages list from the request
|
||||
tools: Tools list from the request
|
||||
custom_llm_provider: Provider name
|
||||
|
||||
Returns:
|
||||
An AnthropicMessagesResponse dict if short-circuited, or None to
|
||||
continue normal processing.
|
||||
"""
|
||||
if not tools:
|
||||
return None
|
||||
|
||||
# Check if provider is in enabled list
|
||||
provider_str = custom_llm_provider or ""
|
||||
if (
|
||||
self.enabled_providers is not None
|
||||
and provider_str not in self.enabled_providers
|
||||
):
|
||||
return None
|
||||
|
||||
# Only short-circuit for providers without native Anthropic Messages
|
||||
# support. Providers that have a BaseAnthropicMessagesConfig (bedrock,
|
||||
# vertex_ai, azure_ai, anthropic) already use the agentic loop, which
|
||||
# includes a follow-up LLM call to synthesize the answer from search
|
||||
# results. Short-circuiting those would skip that synthesis step and
|
||||
# return raw search text — a regression for existing users.
|
||||
try:
|
||||
provider_enum = LlmProviders(provider_str)
|
||||
anthropic_config = (
|
||||
ProviderConfigManager.get_provider_anthropic_messages_config(
|
||||
model=model, provider=provider_enum
|
||||
)
|
||||
)
|
||||
if anthropic_config is not None:
|
||||
verbose_logger.debug(
|
||||
f"WebSearchInterception: Skipping short-circuit for {provider_str} "
|
||||
"(provider has native Anthropic Messages support, using agentic loop)"
|
||||
)
|
||||
return None
|
||||
except (ValueError, Exception):
|
||||
pass # unknown provider enum → safe to short-circuit
|
||||
|
||||
# All tools must be web search tools
|
||||
if not all(is_web_search_tool(t) for t in tools):
|
||||
return None
|
||||
|
||||
# Extract search query from the last user message
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
get_last_user_message,
|
||||
)
|
||||
|
||||
query = get_last_user_message(cast(List[AllMessageValues], messages))
|
||||
if not query:
|
||||
return None
|
||||
|
||||
verbose_logger.debug(
|
||||
"WebSearchInterception: Short-circuit search detected "
|
||||
f"(provider={provider_str}, query='{query}')"
|
||||
)
|
||||
|
||||
# Execute search
|
||||
try:
|
||||
search_result_text = await self._execute_search(query)
|
||||
except Exception as e:
|
||||
verbose_logger.error(
|
||||
f"WebSearchInterception: Short-circuit search failed: {e}"
|
||||
)
|
||||
search_result_text = f"Search failed: {e}"
|
||||
|
||||
# Build synthetic Anthropic response
|
||||
response: Dict[str, Any] = {
|
||||
"id": f"msg_{str(uuid.uuid4())}",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": model,
|
||||
"content": [{"type": "text", "text": search_result_text}],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 0, "output_tokens": 0},
|
||||
}
|
||||
|
||||
verbose_logger.debug(
|
||||
"WebSearchInterception: Short-circuit search completed, "
|
||||
f"returning synthetic response ({len(search_result_text)} chars)"
|
||||
)
|
||||
return response
|
||||
|
||||
async def async_pre_call_deployment_hook(
|
||||
self, kwargs: Dict[str, Any], call_type: Optional[Any]
|
||||
) -> Optional[dict]:
|
||||
|
|
|
|||
|
|
@ -64,6 +64,7 @@ _FINISH_REASON_MAP: dict[str, OpenAIChatCompletionFinishReason] = {
|
|||
"end_turn": "stop",
|
||||
"max_tokens": "length",
|
||||
"tool_use": "tool_calls",
|
||||
"refusal": "content_filter",
|
||||
"compaction": "length",
|
||||
# Cohere
|
||||
"COMPLETE": "stop",
|
||||
|
|
|
|||
|
|
@ -354,9 +354,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
)
|
||||
self.function_id = function_id
|
||||
self.streaming_chunks: List[Any] = [] # for generating complete stream response
|
||||
self.sync_streaming_chunks: List[
|
||||
Any
|
||||
] = [] # for generating complete stream response
|
||||
self.sync_streaming_chunks: List[Any] = (
|
||||
[]
|
||||
) # for generating complete stream response
|
||||
self.log_raw_request_response = log_raw_request_response
|
||||
|
||||
# Initialize dynamic callbacks
|
||||
|
|
@ -801,9 +801,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
prompt_spec=prompt_spec,
|
||||
dynamic_callback_params=dynamic_callback_params,
|
||||
):
|
||||
self.model_call_details[
|
||||
"prompt_integration"
|
||||
] = logger.__class__.__name__
|
||||
self.model_call_details["prompt_integration"] = (
|
||||
logger.__class__.__name__
|
||||
)
|
||||
return logger
|
||||
except Exception:
|
||||
# If check fails, continue to next logger
|
||||
|
|
@ -871,9 +871,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
if anthropic_cache_control_logger := AnthropicCacheControlHook.get_custom_logger_for_anthropic_cache_control_hook(
|
||||
non_default_params
|
||||
):
|
||||
self.model_call_details[
|
||||
"prompt_integration"
|
||||
] = anthropic_cache_control_logger.__class__.__name__
|
||||
self.model_call_details["prompt_integration"] = (
|
||||
anthropic_cache_control_logger.__class__.__name__
|
||||
)
|
||||
return anthropic_cache_control_logger
|
||||
|
||||
#########################################################
|
||||
|
|
@ -885,9 +885,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
internal_usage_cache=None,
|
||||
llm_router=None,
|
||||
)
|
||||
self.model_call_details[
|
||||
"prompt_integration"
|
||||
] = vector_store_custom_logger.__class__.__name__
|
||||
self.model_call_details["prompt_integration"] = (
|
||||
vector_store_custom_logger.__class__.__name__
|
||||
)
|
||||
# Add to global callbacks so post-call hooks are invoked
|
||||
if (
|
||||
vector_store_custom_logger
|
||||
|
|
@ -947,9 +947,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
model
|
||||
): # if model name was changes pre-call, overwrite the initial model call name with the new one
|
||||
self.model_call_details["model"] = model
|
||||
self.model_call_details["litellm_params"][
|
||||
"api_base"
|
||||
] = self._get_masked_api_base(additional_args.get("api_base", ""))
|
||||
self.model_call_details["litellm_params"]["api_base"] = (
|
||||
self._get_masked_api_base(additional_args.get("api_base", ""))
|
||||
)
|
||||
|
||||
def pre_call(self, input, api_key, model=None, additional_args={}): # noqa: PLR0915
|
||||
# Log the exact input to the LLM API
|
||||
|
|
@ -978,9 +978,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
try:
|
||||
# [Non-blocking Extra Debug Information in metadata]
|
||||
if turn_off_message_logging is True:
|
||||
_metadata[
|
||||
"raw_request"
|
||||
] = "redacted by litellm. \
|
||||
_metadata["raw_request"] = "redacted by litellm. \
|
||||
'litellm.turn_off_message_logging=True'"
|
||||
else:
|
||||
curl_command = self._get_request_curl_command(
|
||||
|
|
@ -992,35 +990,31 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
|
||||
_metadata["raw_request"] = str(curl_command)
|
||||
# split up, so it's easier to parse in the UI
|
||||
self.model_call_details[
|
||||
"raw_request_typed_dict"
|
||||
] = RawRequestTypedDict(
|
||||
raw_request_api_base=str(
|
||||
additional_args.get("api_base") or ""
|
||||
),
|
||||
raw_request_body=self._get_raw_request_body(
|
||||
additional_args.get("complete_input_dict", {})
|
||||
),
|
||||
# NOTE: setting ignore_sensitive_headers to True will cause
|
||||
# the Authorization header to be leaked when calls to the health
|
||||
# endpoint are made and fail.
|
||||
raw_request_headers=self._get_masked_headers(
|
||||
additional_args.get("headers", {}) or {},
|
||||
),
|
||||
error=None,
|
||||
self.model_call_details["raw_request_typed_dict"] = (
|
||||
RawRequestTypedDict(
|
||||
raw_request_api_base=str(
|
||||
additional_args.get("api_base") or ""
|
||||
),
|
||||
raw_request_body=self._get_raw_request_body(
|
||||
additional_args.get("complete_input_dict", {})
|
||||
),
|
||||
# NOTE: setting ignore_sensitive_headers to True will cause
|
||||
# the Authorization header to be leaked when calls to the health
|
||||
# endpoint are made and fail.
|
||||
raw_request_headers=self._get_masked_headers(
|
||||
additional_args.get("headers", {}) or {},
|
||||
),
|
||||
error=None,
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
self.model_call_details[
|
||||
"raw_request_typed_dict"
|
||||
] = RawRequestTypedDict(
|
||||
error=str(e),
|
||||
)
|
||||
_metadata[
|
||||
"raw_request"
|
||||
] = "Unable to Log \
|
||||
raw request: {}".format(
|
||||
str(e)
|
||||
self.model_call_details["raw_request_typed_dict"] = (
|
||||
RawRequestTypedDict(
|
||||
error=str(e),
|
||||
)
|
||||
)
|
||||
_metadata["raw_request"] = "Unable to Log \
|
||||
raw request: {}".format(str(e))
|
||||
if getattr(self, "logger_fn", None) and callable(self.logger_fn):
|
||||
try:
|
||||
self.logger_fn(
|
||||
|
|
@ -1320,13 +1314,13 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
for callback in callbacks:
|
||||
try:
|
||||
if isinstance(callback, CustomLogger):
|
||||
response: Optional[
|
||||
MCPPostCallResponseObject
|
||||
] = await callback.async_post_mcp_tool_call_hook(
|
||||
kwargs=kwargs,
|
||||
response_obj=post_mcp_tool_call_response_obj,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
response: Optional[MCPPostCallResponseObject] = (
|
||||
await callback.async_post_mcp_tool_call_hook(
|
||||
kwargs=kwargs,
|
||||
response_obj=post_mcp_tool_call_response_obj,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
)
|
||||
)
|
||||
######################################################################
|
||||
# if any of the callbacks modify the response, use the modified response
|
||||
|
|
@ -1527,9 +1521,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
verbose_logger.debug(
|
||||
f"response_cost_failure_debug_information: {debug_info}"
|
||||
)
|
||||
self.model_call_details[
|
||||
"response_cost_failure_debug_information"
|
||||
] = debug_info
|
||||
self.model_call_details["response_cost_failure_debug_information"] = (
|
||||
debug_info
|
||||
)
|
||||
return None
|
||||
|
||||
try:
|
||||
|
|
@ -1555,9 +1549,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
verbose_logger.debug(
|
||||
f"response_cost_failure_debug_information: {debug_info}"
|
||||
)
|
||||
self.model_call_details[
|
||||
"response_cost_failure_debug_information"
|
||||
] = debug_info
|
||||
self.model_call_details["response_cost_failure_debug_information"] = (
|
||||
debug_info
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
|
|
@ -1686,6 +1680,30 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
)
|
||||
return logging_result
|
||||
|
||||
def _merge_hidden_params_from_response_into_metadata(
|
||||
self, logging_result: Any
|
||||
) -> None:
|
||||
"""
|
||||
Copy response._hidden_params into litellm_params.metadata['hidden_params'].
|
||||
|
||||
Non-streaming success uses _process_hidden_params_and_response_cost (skipped when
|
||||
stream=True). Streaming assembles the full response later; without this merge,
|
||||
OTEL/callbacks that read metadata.hidden_params miss cost-related fields.
|
||||
"""
|
||||
if logging_result is None:
|
||||
return
|
||||
hidden_params = getattr(logging_result, "_hidden_params", None)
|
||||
if not hidden_params:
|
||||
return
|
||||
if self.model_call_details.get("litellm_params") is None:
|
||||
return
|
||||
self.model_call_details["litellm_params"].setdefault("metadata", {})
|
||||
if self.model_call_details["litellm_params"]["metadata"] is None:
|
||||
self.model_call_details["litellm_params"]["metadata"] = {}
|
||||
self.model_call_details["litellm_params"]["metadata"]["hidden_params"] = (
|
||||
getattr(logging_result, "_hidden_params", {})
|
||||
)
|
||||
|
||||
def _process_hidden_params_and_response_cost(
|
||||
self,
|
||||
logging_result,
|
||||
|
|
@ -1713,9 +1731,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
result=logging_result
|
||||
)
|
||||
|
||||
self.model_call_details[
|
||||
"standard_logging_object"
|
||||
] = self._build_standard_logging_payload(logging_result, start_time, end_time)
|
||||
self.model_call_details["standard_logging_object"] = (
|
||||
self._build_standard_logging_payload(logging_result, start_time, end_time)
|
||||
)
|
||||
|
||||
if (
|
||||
standard_logging_payload := self.model_call_details.get(
|
||||
|
|
@ -1793,9 +1811,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
end_time = datetime.datetime.now()
|
||||
if self.completion_start_time is None:
|
||||
self.completion_start_time = end_time
|
||||
self.model_call_details[
|
||||
"completion_start_time"
|
||||
] = self.completion_start_time
|
||||
self.model_call_details["completion_start_time"] = (
|
||||
self.completion_start_time
|
||||
)
|
||||
|
||||
self.model_call_details["log_event_type"] = "successful_api_call"
|
||||
self.model_call_details["end_time"] = end_time
|
||||
|
|
@ -1832,10 +1850,10 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
end_time=end_time,
|
||||
)
|
||||
elif isinstance(result, dict) or isinstance(result, list):
|
||||
self.model_call_details[
|
||||
"standard_logging_object"
|
||||
] = self._build_standard_logging_payload(
|
||||
result, start_time, end_time
|
||||
self.model_call_details["standard_logging_object"] = (
|
||||
self._build_standard_logging_payload(
|
||||
result, start_time, end_time
|
||||
)
|
||||
)
|
||||
if (
|
||||
standard_logging_payload := self.model_call_details.get(
|
||||
|
|
@ -1844,9 +1862,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
) is not None:
|
||||
emit_standard_logging_payload(standard_logging_payload)
|
||||
elif standard_logging_object is not None:
|
||||
self.model_call_details[
|
||||
"standard_logging_object"
|
||||
] = standard_logging_object
|
||||
self.model_call_details["standard_logging_object"] = (
|
||||
standard_logging_object
|
||||
)
|
||||
else:
|
||||
self.model_call_details["response_cost"] = None
|
||||
|
||||
|
|
@ -2004,17 +2022,20 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
verbose_logger.debug(
|
||||
"Logging Details LiteLLM-Success Call streaming complete"
|
||||
)
|
||||
self.model_call_details[
|
||||
"complete_streaming_response"
|
||||
] = complete_streaming_response
|
||||
self.model_call_details[
|
||||
"response_cost"
|
||||
] = self._response_cost_calculator(result=complete_streaming_response)
|
||||
self.model_call_details["complete_streaming_response"] = (
|
||||
complete_streaming_response
|
||||
)
|
||||
self.model_call_details["response_cost"] = (
|
||||
self._response_cost_calculator(result=complete_streaming_response)
|
||||
)
|
||||
self._merge_hidden_params_from_response_into_metadata(
|
||||
complete_streaming_response
|
||||
)
|
||||
## STANDARDIZED LOGGING PAYLOAD
|
||||
self.model_call_details[
|
||||
"standard_logging_object"
|
||||
] = self._build_standard_logging_payload(
|
||||
complete_streaming_response, start_time, end_time
|
||||
self.model_call_details["standard_logging_object"] = (
|
||||
self._build_standard_logging_payload(
|
||||
complete_streaming_response, start_time, end_time
|
||||
)
|
||||
)
|
||||
if (
|
||||
standard_logging_payload := self.model_call_details.get(
|
||||
|
|
@ -2348,10 +2369,10 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
)
|
||||
else:
|
||||
if self.stream and complete_streaming_response:
|
||||
self.model_call_details[
|
||||
"complete_response"
|
||||
] = self.model_call_details.get(
|
||||
"complete_streaming_response", {}
|
||||
self.model_call_details["complete_response"] = (
|
||||
self.model_call_details.get(
|
||||
"complete_streaming_response", {}
|
||||
)
|
||||
)
|
||||
result = self.model_call_details["complete_response"]
|
||||
openMeterLogger.log_success_event(
|
||||
|
|
@ -2375,10 +2396,10 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
)
|
||||
else:
|
||||
if self.stream and complete_streaming_response:
|
||||
self.model_call_details[
|
||||
"complete_response"
|
||||
] = self.model_call_details.get(
|
||||
"complete_streaming_response", {}
|
||||
self.model_call_details["complete_response"] = (
|
||||
self.model_call_details.get(
|
||||
"complete_streaming_response", {}
|
||||
)
|
||||
)
|
||||
result = self.model_call_details["complete_response"]
|
||||
|
||||
|
|
@ -2517,9 +2538,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
if complete_streaming_response is not None:
|
||||
print_verbose("Async success callbacks: Got a complete streaming response")
|
||||
|
||||
self.model_call_details[
|
||||
"async_complete_streaming_response"
|
||||
] = complete_streaming_response
|
||||
self.model_call_details["async_complete_streaming_response"] = (
|
||||
complete_streaming_response
|
||||
)
|
||||
|
||||
try:
|
||||
if self.model_call_details.get("cache_hit", False) is True:
|
||||
|
|
@ -2530,10 +2551,10 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
model_call_details=self.model_call_details
|
||||
)
|
||||
# base_model defaults to None if not set on model_info
|
||||
self.model_call_details[
|
||||
"response_cost"
|
||||
] = self._response_cost_calculator(
|
||||
result=complete_streaming_response
|
||||
self.model_call_details["response_cost"] = (
|
||||
self._response_cost_calculator(
|
||||
result=complete_streaming_response
|
||||
)
|
||||
)
|
||||
|
||||
verbose_logger.debug(
|
||||
|
|
@ -2545,11 +2566,15 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
)
|
||||
self.model_call_details["response_cost"] = None
|
||||
|
||||
self._merge_hidden_params_from_response_into_metadata(
|
||||
complete_streaming_response
|
||||
)
|
||||
|
||||
## STANDARDIZED LOGGING PAYLOAD
|
||||
self.model_call_details[
|
||||
"standard_logging_object"
|
||||
] = self._build_standard_logging_payload(
|
||||
complete_streaming_response, start_time, end_time
|
||||
self.model_call_details["standard_logging_object"] = (
|
||||
self._build_standard_logging_payload(
|
||||
complete_streaming_response, start_time, end_time
|
||||
)
|
||||
)
|
||||
|
||||
# print standard logging payload
|
||||
|
|
@ -2576,9 +2601,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
# _success_handler_helper_fn
|
||||
if self.model_call_details.get("standard_logging_object") is None:
|
||||
## STANDARDIZED LOGGING PAYLOAD
|
||||
self.model_call_details[
|
||||
"standard_logging_object"
|
||||
] = self._build_standard_logging_payload(result, start_time, end_time)
|
||||
self.model_call_details["standard_logging_object"] = (
|
||||
self._build_standard_logging_payload(result, start_time, end_time)
|
||||
)
|
||||
|
||||
# print standard logging payload
|
||||
if (
|
||||
|
|
@ -2821,18 +2846,18 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
|
||||
## STANDARDIZED LOGGING PAYLOAD
|
||||
|
||||
self.model_call_details[
|
||||
"standard_logging_object"
|
||||
] = get_standard_logging_object_payload(
|
||||
kwargs=self.model_call_details,
|
||||
init_response_obj={},
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
logging_obj=self,
|
||||
status="failure",
|
||||
error_str=str(exception),
|
||||
original_exception=exception,
|
||||
standard_built_in_tools_params=self.standard_built_in_tools_params,
|
||||
self.model_call_details["standard_logging_object"] = (
|
||||
get_standard_logging_object_payload(
|
||||
kwargs=self.model_call_details,
|
||||
init_response_obj={},
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
logging_obj=self,
|
||||
status="failure",
|
||||
error_str=str(exception),
|
||||
original_exception=exception,
|
||||
standard_built_in_tools_params=self.standard_built_in_tools_params,
|
||||
)
|
||||
)
|
||||
return start_time, end_time
|
||||
|
||||
|
|
@ -3800,9 +3825,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
|
|||
service_name=arize_config.project_name,
|
||||
)
|
||||
|
||||
os.environ[
|
||||
"OTEL_EXPORTER_OTLP_TRACES_HEADERS"
|
||||
] = f"space_id={arize_config.space_key or arize_config.space_id},api_key={arize_config.api_key}"
|
||||
os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = (
|
||||
f"space_id={arize_config.space_key or arize_config.space_id},api_key={arize_config.api_key}"
|
||||
)
|
||||
for callback in _in_memory_loggers:
|
||||
if (
|
||||
isinstance(callback, ArizeLogger)
|
||||
|
|
@ -3828,13 +3853,13 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
|
|||
existing_attrs = os.environ.get("OTEL_RESOURCE_ATTRIBUTES", "")
|
||||
# Add openinference.project.name attribute
|
||||
if existing_attrs:
|
||||
os.environ[
|
||||
"OTEL_RESOURCE_ATTRIBUTES"
|
||||
] = f"{existing_attrs},openinference.project.name={arize_phoenix_config.project_name}"
|
||||
os.environ["OTEL_RESOURCE_ATTRIBUTES"] = (
|
||||
f"{existing_attrs},openinference.project.name={arize_phoenix_config.project_name}"
|
||||
)
|
||||
else:
|
||||
os.environ[
|
||||
"OTEL_RESOURCE_ATTRIBUTES"
|
||||
] = f"openinference.project.name={arize_phoenix_config.project_name}"
|
||||
os.environ["OTEL_RESOURCE_ATTRIBUTES"] = (
|
||||
f"openinference.project.name={arize_phoenix_config.project_name}"
|
||||
)
|
||||
|
||||
# Set Phoenix project name from environment variable
|
||||
phoenix_project_name = os.environ.get("PHOENIX_PROJECT_NAME", None)
|
||||
|
|
@ -3842,19 +3867,19 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
|
|||
existing_attrs = os.environ.get("OTEL_RESOURCE_ATTRIBUTES", "")
|
||||
# Add openinference.project.name attribute
|
||||
if existing_attrs:
|
||||
os.environ[
|
||||
"OTEL_RESOURCE_ATTRIBUTES"
|
||||
] = f"{existing_attrs},openinference.project.name={phoenix_project_name}"
|
||||
os.environ["OTEL_RESOURCE_ATTRIBUTES"] = (
|
||||
f"{existing_attrs},openinference.project.name={phoenix_project_name}"
|
||||
)
|
||||
else:
|
||||
os.environ[
|
||||
"OTEL_RESOURCE_ATTRIBUTES"
|
||||
] = f"openinference.project.name={phoenix_project_name}"
|
||||
os.environ["OTEL_RESOURCE_ATTRIBUTES"] = (
|
||||
f"openinference.project.name={phoenix_project_name}"
|
||||
)
|
||||
|
||||
# auth can be disabled on local deployments of arize phoenix
|
||||
if arize_phoenix_config.otlp_auth_headers is not None:
|
||||
os.environ[
|
||||
"OTEL_EXPORTER_OTLP_TRACES_HEADERS"
|
||||
] = arize_phoenix_config.otlp_auth_headers
|
||||
os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = (
|
||||
arize_phoenix_config.otlp_auth_headers
|
||||
)
|
||||
|
||||
for callback in _in_memory_loggers:
|
||||
if (
|
||||
|
|
@ -4041,9 +4066,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
|
|||
exporter="otlp_http",
|
||||
endpoint="https://langtrace.ai/api/trace",
|
||||
)
|
||||
os.environ[
|
||||
"OTEL_EXPORTER_OTLP_TRACES_HEADERS"
|
||||
] = f"api_key={os.getenv('LANGTRACE_API_KEY')}"
|
||||
os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = (
|
||||
f"api_key={os.getenv('LANGTRACE_API_KEY')}"
|
||||
)
|
||||
for callback in _in_memory_loggers:
|
||||
if (
|
||||
isinstance(callback, OpenTelemetry)
|
||||
|
|
@ -4967,10 +4992,10 @@ class StandardLoggingPayloadSetup:
|
|||
for key in StandardLoggingHiddenParams.__annotations__.keys():
|
||||
if key in hidden_params:
|
||||
if key == "additional_headers":
|
||||
clean_hidden_params[
|
||||
"additional_headers"
|
||||
] = StandardLoggingPayloadSetup.get_additional_headers(
|
||||
hidden_params[key]
|
||||
clean_hidden_params["additional_headers"] = (
|
||||
StandardLoggingPayloadSetup.get_additional_headers(
|
||||
hidden_params[key]
|
||||
)
|
||||
)
|
||||
else:
|
||||
clean_hidden_params[key] = hidden_params[key] # type: ignore
|
||||
|
|
@ -5609,9 +5634,9 @@ def scrub_sensitive_keys_in_metadata(litellm_params: Optional[dict]):
|
|||
):
|
||||
for k, v in metadata["user_api_key_metadata"].items():
|
||||
if k == "logging": # prevent logging user logging keys
|
||||
cleaned_user_api_key_metadata[
|
||||
k
|
||||
] = "scrubbed_by_litellm_for_sensitive_keys"
|
||||
cleaned_user_api_key_metadata[k] = (
|
||||
"scrubbed_by_litellm_for_sensitive_keys"
|
||||
)
|
||||
else:
|
||||
cleaned_user_api_key_metadata[k] = v
|
||||
|
||||
|
|
|
|||
|
|
@ -257,6 +257,15 @@ def detect_first_expected_role(
|
|||
return None
|
||||
|
||||
|
||||
def _counts_for_alternation(message: AllMessageValues) -> bool:
|
||||
role = message.get("role")
|
||||
if role == "user":
|
||||
return True
|
||||
if role == "assistant":
|
||||
return not bool(message.get("tool_calls"))
|
||||
return False
|
||||
|
||||
|
||||
def _insert_user_continue_message(
|
||||
messages: List[AllMessageValues],
|
||||
user_continue_message: Optional[ChatCompletionUserMessage],
|
||||
|
|
@ -269,8 +278,8 @@ def _insert_user_continue_message(
|
|||
2. Final assistant message
|
||||
3. Consecutive assistant messages
|
||||
|
||||
Only inserts messages between consecutive assistant messages,
|
||||
ignoring all other role types.
|
||||
Skips tool messages and assistant messages with tool calls in the
|
||||
alternation check, matching strict templates like llama.cpp.
|
||||
"""
|
||||
if not messages:
|
||||
return messages
|
||||
|
|
@ -278,25 +287,42 @@ def _insert_user_continue_message(
|
|||
result_messages = messages.copy() # Don't modify the input list
|
||||
continue_message = user_continue_message or DEFAULT_USER_CONTINUE_MESSAGE
|
||||
|
||||
# Handle first message if it's an assistant message
|
||||
# Handle first message if it's an assistant message — always prepend
|
||||
# user_continue regardless of tool_calls, to preserve backward compatibility.
|
||||
if result_messages[0]["role"] == "assistant":
|
||||
result_messages.insert(0, continue_message)
|
||||
|
||||
# Handle consecutive assistant messages and final message
|
||||
i = 1 # Start from second message since we handled first message
|
||||
# Handle consecutive assistant messages in the counted sequence
|
||||
i = 1
|
||||
while i < len(result_messages):
|
||||
curr_message = result_messages[i]
|
||||
prev_message = result_messages[i - 1]
|
||||
|
||||
# Only check for consecutive assistant messages
|
||||
# Ignore all other role types
|
||||
if curr_message["role"] == "assistant" and prev_message["role"] == "assistant":
|
||||
result_messages.insert(i, continue_message)
|
||||
i += 2 # Skip over the message we just inserted
|
||||
else:
|
||||
inserted_continue_message = False
|
||||
if (
|
||||
_counts_for_alternation(curr_message)
|
||||
and curr_message["role"] == "assistant"
|
||||
):
|
||||
# Preserve old behavior for malformed adjacent assistant sequences like
|
||||
# assistant(tool_calls) -> assistant(no-tool-calls) with no tool message.
|
||||
if i > 0 and result_messages[i - 1].get("role") == "assistant":
|
||||
result_messages.insert(i, continue_message)
|
||||
i += 2
|
||||
inserted_continue_message = True
|
||||
else:
|
||||
j = i - 1
|
||||
while j >= 0:
|
||||
previous_message = result_messages[j]
|
||||
if _counts_for_alternation(previous_message):
|
||||
if previous_message["role"] == "assistant":
|
||||
result_messages.insert(i, continue_message)
|
||||
i += 2
|
||||
inserted_continue_message = True
|
||||
break
|
||||
j -= 1
|
||||
if not inserted_continue_message:
|
||||
i += 1
|
||||
|
||||
# Handle final message
|
||||
# Handle final message — append user_continue after any trailing assistant,
|
||||
# including ones with tool_calls, to preserve backward compatibility.
|
||||
if result_messages[-1]["role"] == "assistant" and ensure_alternating_roles:
|
||||
result_messages.append(continue_message)
|
||||
|
||||
|
|
@ -311,34 +337,24 @@ def _insert_assistant_continue_message(
|
|||
"""
|
||||
Add assistant continuation messages between consecutive user messages.
|
||||
|
||||
Args:
|
||||
messages: List of message dictionaries
|
||||
assistant_continue_message: Optional custom assistant message
|
||||
ensure_alternating_roles: Whether to enforce alternating roles
|
||||
|
||||
Returns:
|
||||
Modified list of messages with inserted assistant messages
|
||||
Only checks directly adjacent messages to preserve backward compatibility.
|
||||
"""
|
||||
if not ensure_alternating_roles or len(messages) <= 1:
|
||||
return messages
|
||||
|
||||
# Create a new list to store modified messages
|
||||
continue_message = assistant_continue_message or DEFAULT_ASSISTANT_CONTINUE_MESSAGE
|
||||
|
||||
modified_messages: List[AllMessageValues] = []
|
||||
|
||||
for i, message in enumerate(messages):
|
||||
modified_messages.append(message)
|
||||
|
||||
# Check if we need to insert an assistant message
|
||||
if (
|
||||
i < len(messages) - 1 # Not the last message
|
||||
and message.get("role") == "user" # Current is user
|
||||
i < len(messages) - 1
|
||||
and message.get("role") == "user"
|
||||
and messages[i + 1].get("role") == "user"
|
||||
): # Next is user
|
||||
# Insert assistant message
|
||||
continue_message = (
|
||||
assistant_continue_message or DEFAULT_ASSISTANT_CONTINUE_MESSAGE
|
||||
)
|
||||
):
|
||||
modified_messages.append(message)
|
||||
modified_messages.append(continue_message)
|
||||
else:
|
||||
modified_messages.append(message)
|
||||
|
||||
return modified_messages
|
||||
|
||||
|
|
@ -536,6 +552,61 @@ def update_responses_input_with_model_file_ids(
|
|||
return updated_input
|
||||
|
||||
|
||||
def _decode_vector_store_ids_in_tools(
|
||||
tools: Optional[List[Dict[str, Any]]],
|
||||
) -> Optional[List[Dict[str, Any]]]:
|
||||
"""
|
||||
Decodes unified (LiteLLM-managed) vector_store_ids in file_search tools to
|
||||
provider-native IDs. Non-unified IDs are passed through unchanged.
|
||||
|
||||
This runs unconditionally — no file-ID mapping is required.
|
||||
"""
|
||||
if not tools or not isinstance(tools, list):
|
||||
return tools
|
||||
|
||||
from litellm.llms.base_llm.managed_resources.utils import (
|
||||
is_base64_encoded_unified_id,
|
||||
parse_unified_id,
|
||||
)
|
||||
|
||||
updated_tools = []
|
||||
for tool in tools:
|
||||
if not isinstance(tool, dict) or tool.get("type") != "file_search":
|
||||
updated_tools.append(tool)
|
||||
continue
|
||||
|
||||
vector_store_ids = tool.get("vector_store_ids")
|
||||
if not isinstance(vector_store_ids, list):
|
||||
updated_tools.append(tool)
|
||||
continue
|
||||
|
||||
decoded_ids = []
|
||||
for vs_id in vector_store_ids:
|
||||
if not isinstance(vs_id, str) or not is_base64_encoded_unified_id(vs_id):
|
||||
decoded_ids.append(vs_id)
|
||||
continue
|
||||
|
||||
parsed = parse_unified_id(vs_id)
|
||||
provider_resource_id = (
|
||||
parsed.get("provider_resource_id") if parsed else None
|
||||
)
|
||||
|
||||
if not provider_resource_id:
|
||||
verbose_logger.warning(
|
||||
"file_search tool contains unified vector_store_id '%s' that could "
|
||||
"not be decoded to a provider resource ID — passing original ID. "
|
||||
"Ensure the vector store was created via LiteLLM.",
|
||||
vs_id,
|
||||
)
|
||||
decoded_ids.append(vs_id)
|
||||
else:
|
||||
decoded_ids.append(provider_resource_id)
|
||||
|
||||
updated_tools.append({**tool, "vector_store_ids": decoded_ids})
|
||||
|
||||
return updated_tools
|
||||
|
||||
|
||||
def update_responses_tools_with_model_file_ids(
|
||||
tools: Optional[List[Dict[str, Any]]],
|
||||
model_id: Optional[str] = None,
|
||||
|
|
@ -544,7 +615,8 @@ def update_responses_tools_with_model_file_ids(
|
|||
"""
|
||||
Updates responses API tools with provider-specific file IDs.
|
||||
|
||||
Handles code_interpreter tools with container.file_ids.
|
||||
Pass 1 (always): decode unified vector_store_ids in file_search tools.
|
||||
Pass 2 (needs mapping): map code_interpreter container file_ids to provider IDs.
|
||||
|
||||
Args:
|
||||
tools: The responses API tools parameter
|
||||
|
|
@ -555,6 +627,10 @@ def update_responses_tools_with_model_file_ids(
|
|||
if not tools or not isinstance(tools, list):
|
||||
return tools
|
||||
|
||||
# Pass 1: decode unified vector_store_ids (no mapping needed)
|
||||
tools = _decode_vector_store_ids_in_tools(tools) or tools
|
||||
|
||||
# Pass 2: map code_interpreter file IDs (requires mapping)
|
||||
if not model_file_id_mapping or not model_id:
|
||||
return tools
|
||||
|
||||
|
|
|
|||
|
|
@ -1498,17 +1498,49 @@ def convert_to_gemini_tool_call_result( # noqa: PLR0915
|
|||
from litellm.types.llms.vertex_ai import BlobType
|
||||
|
||||
content_str: str = ""
|
||||
inline_data: Optional[BlobType] = None
|
||||
inline_data_list: List[BlobType] = []
|
||||
|
||||
if "content" in message:
|
||||
if isinstance(message["content"], str):
|
||||
content_str = message["content"]
|
||||
# Detect data-URL images (e.g. from Anthropic tool_result with a single image block
|
||||
# that was serialised as a plain string by translate_anthropic_messages_to_openai)
|
||||
# and promote them to inline_data so Gemini receives actual image bytes.
|
||||
if content_str[:5].lower() == "data:" and ";base64," in content_str:
|
||||
try:
|
||||
mime_rest = content_str[5:].split(";base64,", 1)
|
||||
if len(mime_rest) == 2 and mime_rest[0].startswith("image/"):
|
||||
# Strip any extra parameters (e.g. ";charset=UTF-8") from the MIME segment
|
||||
clean_mime = mime_rest[0].split(";")[0].strip()
|
||||
inline_data_list.append(
|
||||
BlobType(data=mime_rest[1], mime_type=clean_mime)
|
||||
)
|
||||
content_str = ""
|
||||
except Exception as e:
|
||||
verbose_logger.warning(
|
||||
f"Failed to parse data URL in tool response: {e}"
|
||||
)
|
||||
elif isinstance(message["content"], List):
|
||||
content_list = message["content"]
|
||||
for content in content_list:
|
||||
content_type = content.get("type", "")
|
||||
if content_type == "text":
|
||||
content_str += content.get("text", "")
|
||||
elif content_type == "image":
|
||||
# Anthropic-native image block: {"type": "image", "source": {"type": "base64", ...}}
|
||||
source = content.get("source", {})
|
||||
if isinstance(source, dict) and source.get("type") == "base64":
|
||||
try:
|
||||
inline_data_list.append(
|
||||
BlobType(
|
||||
data=source.get("data", ""),
|
||||
mime_type=source.get("media_type", "image/jpeg"),
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_logger.warning(
|
||||
f"Failed to process Anthropic image block in tool response: {e}"
|
||||
)
|
||||
elif content_type in ("input_image", "image_url"):
|
||||
# Extract image for inline_data (for Computer Use screenshots and tool results)
|
||||
image_url_data = content.get("image_url", "")
|
||||
|
|
@ -1524,9 +1556,11 @@ def convert_to_gemini_tool_call_result( # noqa: PLR0915
|
|||
image_obj = convert_to_anthropic_image_obj(
|
||||
image_url, format=None
|
||||
)
|
||||
inline_data = BlobType(
|
||||
data=image_obj["data"],
|
||||
mime_type=image_obj["media_type"],
|
||||
inline_data_list.append(
|
||||
BlobType(
|
||||
data=image_obj["data"],
|
||||
mime_type=image_obj["media_type"],
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_logger.warning(
|
||||
|
|
@ -1551,9 +1585,11 @@ def convert_to_gemini_tool_call_result( # noqa: PLR0915
|
|||
file_obj = convert_to_anthropic_image_obj(
|
||||
file_data, format=None
|
||||
)
|
||||
inline_data = BlobType(
|
||||
data=file_obj["data"],
|
||||
mime_type=file_obj["media_type"],
|
||||
inline_data_list.append(
|
||||
BlobType(
|
||||
data=file_obj["data"],
|
||||
mime_type=file_obj["media_type"],
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_logger.warning(
|
||||
|
|
@ -1607,13 +1643,12 @@ def convert_to_gemini_tool_call_result( # noqa: PLR0915
|
|||
# Create part with function_response, and optionally inline_data for images (Computer Use)
|
||||
_part: VertexPartType = {"function_response": _function_response}
|
||||
|
||||
# For Computer Use, if we have an image, we need separate parts:
|
||||
# For Computer Use, if we have images/files, we need separate parts:
|
||||
# - One part with function_response
|
||||
# - One part with inline_data
|
||||
# - One part per inline_data item
|
||||
# Gemini's PartType is a oneof, so we can't have both in the same part
|
||||
if inline_data:
|
||||
image_part: VertexPartType = {"inline_data": inline_data}
|
||||
return [_part, image_part]
|
||||
if inline_data_list:
|
||||
return [_part] + [{"inline_data": d} for d in inline_data_list]
|
||||
|
||||
return _part
|
||||
|
||||
|
|
|
|||
|
|
@ -57,6 +57,22 @@ IMAGE_ATTRIBUTE = "images"
|
|||
TOOL_CALLS_ATTRIBUTE = "tool_calls"
|
||||
FUNCTION_CALL_ATTRIBUTE = "function_call"
|
||||
|
||||
_SYNC_ITER_EXHAUSTED = object()
|
||||
|
||||
|
||||
def _next_sync_or_exhausted(it: Any) -> Any:
|
||||
"""
|
||||
Call next(it) from a thread and return _SYNC_ITER_EXHAUSTED on StopIteration.
|
||||
|
||||
asyncio.to_thread re-raises thread exceptions inside a coroutine, where PEP 479
|
||||
converts StopIteration to RuntimeError before any except clause can catch it.
|
||||
Returning a sentinel instead keeps StopIteration out of the coroutine boundary.
|
||||
"""
|
||||
try:
|
||||
return next(it)
|
||||
except StopIteration:
|
||||
return _SYNC_ITER_EXHAUSTED
|
||||
|
||||
|
||||
def is_async_iterable(obj: Any) -> bool:
|
||||
"""
|
||||
|
|
@ -2090,7 +2106,9 @@ class CustomStreamWrapper:
|
|||
):
|
||||
chunk = self.completion_stream
|
||||
else:
|
||||
chunk = next(self.completion_stream) # type: ignore[arg-type]
|
||||
chunk = await asyncio.to_thread(_next_sync_or_exhausted, self.completion_stream) # type: ignore[arg-type]
|
||||
if chunk is _SYNC_ITER_EXHAUSTED:
|
||||
raise StopAsyncIteration
|
||||
if chunk is not None and chunk != b"":
|
||||
processed_chunk = self.chunk_creator(chunk=chunk)
|
||||
if processed_chunk is None:
|
||||
|
|
@ -2150,22 +2168,36 @@ class CustomStreamWrapper:
|
|||
self.sent_stream_usage = True
|
||||
return response
|
||||
|
||||
asyncio.create_task(
|
||||
self.logging_obj.async_success_handler(
|
||||
_deferred_cb = getattr(
|
||||
self.logging_obj,
|
||||
"_on_deferred_stream_complete",
|
||||
None,
|
||||
)
|
||||
if _deferred_cb is not None:
|
||||
# Proxy has post-call guardrails — let the closure
|
||||
# run guardrails on the assembled response, then
|
||||
# fire logging with guardrail_information populated.
|
||||
self.logging_obj._on_deferred_stream_complete = None # type: ignore[attr-defined]
|
||||
asyncio.create_task(
|
||||
_deferred_cb(complete_streaming_response, cache_hit)
|
||||
)
|
||||
else:
|
||||
asyncio.create_task(
|
||||
self.logging_obj.async_success_handler(
|
||||
complete_streaming_response,
|
||||
cache_hit=cache_hit,
|
||||
start_time=None,
|
||||
end_time=None,
|
||||
)
|
||||
)
|
||||
|
||||
executor.submit(
|
||||
self.logging_obj.success_handler,
|
||||
complete_streaming_response,
|
||||
cache_hit=cache_hit,
|
||||
start_time=None,
|
||||
end_time=None,
|
||||
)
|
||||
)
|
||||
|
||||
executor.submit(
|
||||
self.logging_obj.success_handler,
|
||||
complete_streaming_response,
|
||||
cache_hit=cache_hit,
|
||||
start_time=None,
|
||||
end_time=None,
|
||||
)
|
||||
|
||||
raise StopAsyncIteration # Re-raise StopIteration
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -111,6 +111,7 @@ class A2AGuardrailHandler(BaseTranslation):
|
|||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: Optional["LiteLLMLoggingObj"] = None,
|
||||
user_api_key_dict: Optional["UserAPIKeyAuth"] = None,
|
||||
request_data: Optional[dict] = None,
|
||||
) -> Any:
|
||||
"""
|
||||
Process A2A output response by applying guardrails to text content.
|
||||
|
|
@ -166,13 +167,21 @@ class A2AGuardrailHandler(BaseTranslation):
|
|||
return response
|
||||
|
||||
# Step 2: Apply guardrail to all texts in batch
|
||||
# Create a request_data dict with response info and user API key metadata
|
||||
request_data: dict = {"response": response_dict}
|
||||
# Use the real request_data if provided (proxy path), otherwise
|
||||
# create a standalone dict (SDK / direct-call path).
|
||||
if request_data is None:
|
||||
request_data = {"response": response_dict}
|
||||
else:
|
||||
if "response" not in request_data:
|
||||
request_data["response"] = response_dict
|
||||
|
||||
# Add user API key metadata with prefixed keys
|
||||
user_metadata = self.transform_user_api_key_dict_to_metadata(user_api_key_dict)
|
||||
if user_metadata:
|
||||
request_data["litellm_metadata"] = user_metadata
|
||||
if "litellm_metadata" not in request_data:
|
||||
user_metadata = self.transform_user_api_key_dict_to_metadata(
|
||||
user_api_key_dict
|
||||
)
|
||||
if user_metadata:
|
||||
request_data["litellm_metadata"] = user_metadata
|
||||
|
||||
inputs = GenericGuardrailAPIInputs(texts=texts_to_check)
|
||||
|
||||
|
|
@ -213,6 +222,7 @@ class A2AGuardrailHandler(BaseTranslation):
|
|||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: Optional["LiteLLMLoggingObj"] = None,
|
||||
user_api_key_dict: Optional["UserAPIKeyAuth"] = None,
|
||||
request_data: Optional[dict] = None,
|
||||
) -> List[Any]:
|
||||
"""
|
||||
Process A2A streaming output by applying guardrails to accumulated text.
|
||||
|
|
@ -258,10 +268,18 @@ class A2AGuardrailHandler(BaseTranslation):
|
|||
if not combined_text:
|
||||
return responses_so_far
|
||||
|
||||
request_data: dict = {"responses_so_far": responses_so_far}
|
||||
user_metadata = self.transform_user_api_key_dict_to_metadata(user_api_key_dict)
|
||||
if user_metadata:
|
||||
request_data["litellm_metadata"] = user_metadata
|
||||
if request_data is None:
|
||||
request_data = {"responses_so_far": responses_so_far}
|
||||
else:
|
||||
if "responses_so_far" not in request_data:
|
||||
request_data["responses_so_far"] = responses_so_far
|
||||
|
||||
if "litellm_metadata" not in request_data:
|
||||
user_metadata = self.transform_user_api_key_dict_to_metadata(
|
||||
user_api_key_dict
|
||||
)
|
||||
if user_metadata:
|
||||
request_data["litellm_metadata"] = user_metadata
|
||||
|
||||
inputs = GenericGuardrailAPIInputs(texts=[combined_text])
|
||||
guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
|
||||
|
|
|
|||
|
|
@ -252,6 +252,7 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: Optional[Any] = None,
|
||||
user_api_key_dict: Optional[Any] = None,
|
||||
request_data: Optional[dict] = None,
|
||||
) -> Any:
|
||||
"""
|
||||
Process output response by applying guardrails to text content and tool calls.
|
||||
|
|
@ -323,15 +324,21 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
|
||||
# Step 2: Apply guardrail to all texts in batch
|
||||
if texts_to_check or tool_calls_to_check:
|
||||
# Create a request_data dict with response info and user API key metadata
|
||||
request_data: dict = {"response": response}
|
||||
# Use the real request_data if provided (proxy path), otherwise
|
||||
# create a standalone dict (SDK / direct-call path).
|
||||
if request_data is None:
|
||||
request_data = {"response": response}
|
||||
else:
|
||||
if "response" not in request_data:
|
||||
request_data["response"] = response
|
||||
|
||||
# Add user API key metadata with prefixed keys
|
||||
user_metadata = self.transform_user_api_key_dict_to_metadata(
|
||||
user_api_key_dict
|
||||
)
|
||||
if user_metadata:
|
||||
request_data["litellm_metadata"] = user_metadata
|
||||
if "litellm_metadata" not in request_data:
|
||||
user_metadata = self.transform_user_api_key_dict_to_metadata(
|
||||
user_api_key_dict
|
||||
)
|
||||
if user_metadata:
|
||||
request_data["litellm_metadata"] = user_metadata
|
||||
|
||||
inputs = GenericGuardrailAPIInputs(texts=texts_to_check)
|
||||
if images_to_check:
|
||||
|
|
@ -375,6 +382,7 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: Optional[Any] = None,
|
||||
user_api_key_dict: Optional[Any] = None,
|
||||
request_data: Optional[dict] = None,
|
||||
) -> List[Any]:
|
||||
"""
|
||||
Process output streaming response by applying guardrails to text content.
|
||||
|
|
@ -413,7 +421,7 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
|
||||
_guardrailed_inputs = await guardrail_to_apply.apply_guardrail( # allow rejecting the response, if invalid
|
||||
inputs=guardrail_inputs,
|
||||
request_data={},
|
||||
request_data=request_data if request_data is not None else {},
|
||||
input_type="response",
|
||||
logging_obj=litellm_logging_obj,
|
||||
)
|
||||
|
|
@ -426,7 +434,7 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
string_so_far = self.get_streaming_string_so_far(responses_so_far)
|
||||
_guardrailed_inputs = await guardrail_to_apply.apply_guardrail( # allow rejecting the response, if invalid
|
||||
inputs={"texts": [string_so_far]},
|
||||
request_data={},
|
||||
request_data=request_data if request_data is not None else {},
|
||||
input_type="response",
|
||||
logging_obj=litellm_logging_obj,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -15,6 +15,9 @@ import litellm
|
|||
from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import (
|
||||
AnthropicAdapter,
|
||||
)
|
||||
from litellm.llms.anthropic.experimental_pass_through.utils import (
|
||||
is_reasoning_auto_summary_enabled,
|
||||
)
|
||||
from litellm.types.llms.anthropic_messages.anthropic_response import (
|
||||
AnthropicMessagesResponse,
|
||||
)
|
||||
|
|
@ -44,8 +47,9 @@ class LiteLLMMessagesToCompletionTransformationHandler:
|
|||
|
||||
For OpenAI models, Chat Completions typically does not return reasoning text
|
||||
(only token accounting). To return a thinking-like content block in the
|
||||
Anthropic response format, we route the request through OpenAI's Responses API
|
||||
and request a reasoning summary.
|
||||
Anthropic response format, we route the request through OpenAI's Responses API.
|
||||
If the user provides a `summary` field in the thinking dict, it is passed
|
||||
through to the OpenAI reasoning params (opt-in per OpenAI spec).
|
||||
"""
|
||||
custom_llm_provider = completion_kwargs.get("custom_llm_provider")
|
||||
if custom_llm_provider is None:
|
||||
|
|
@ -78,20 +82,29 @@ class LiteLLMMessagesToCompletionTransformationHandler:
|
|||
# Prefix model with "responses/" to route to OpenAI Responses API
|
||||
completion_kwargs["model"] = f"responses/{model}"
|
||||
|
||||
auto_summary = is_reasoning_auto_summary_enabled()
|
||||
|
||||
reasoning_effort = completion_kwargs.get("reasoning_effort")
|
||||
summary = thinking.get("summary")
|
||||
if isinstance(reasoning_effort, str) and reasoning_effort:
|
||||
completion_kwargs["reasoning_effort"] = {
|
||||
"effort": reasoning_effort,
|
||||
"summary": "detailed",
|
||||
}
|
||||
reasoning_dict: Dict[str, Any] = {"effort": reasoning_effort}
|
||||
if summary:
|
||||
reasoning_dict["summary"] = summary
|
||||
elif auto_summary:
|
||||
reasoning_dict["summary"] = "detailed"
|
||||
completion_kwargs["reasoning_effort"] = reasoning_dict
|
||||
elif isinstance(reasoning_effort, dict):
|
||||
if (
|
||||
"summary" not in reasoning_effort
|
||||
and "generate_summary" not in reasoning_effort
|
||||
):
|
||||
updated_reasoning_effort = dict(reasoning_effort)
|
||||
updated_reasoning_effort["summary"] = "detailed"
|
||||
completion_kwargs["reasoning_effort"] = updated_reasoning_effort
|
||||
effective_summary = (
|
||||
summary if summary else ("detailed" if auto_summary else None)
|
||||
)
|
||||
if effective_summary:
|
||||
updated_reasoning_effort = dict(reasoning_effort)
|
||||
updated_reasoning_effort["summary"] = effective_summary
|
||||
completion_kwargs["reasoning_effort"] = updated_reasoning_effort
|
||||
|
||||
@staticmethod
|
||||
def _prepare_completion_kwargs(
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import copy
|
||||
import hashlib
|
||||
import json
|
||||
from typing import (
|
||||
|
|
@ -13,6 +14,10 @@ from typing import (
|
|||
cast,
|
||||
)
|
||||
|
||||
from litellm.llms.anthropic.experimental_pass_through.utils import (
|
||||
is_reasoning_auto_summary_enabled,
|
||||
)
|
||||
|
||||
# OpenAI has a 64-character limit for function/tool names
|
||||
# Anthropic does not have this limit, so we need to truncate long names
|
||||
OPENAI_MAX_TOOL_NAME_LENGTH = 64
|
||||
|
|
@ -733,6 +738,24 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
thinking
|
||||
)
|
||||
if reasoning_effort:
|
||||
summary = (
|
||||
thinking.get("summary") if isinstance(thinking, dict) else None
|
||||
)
|
||||
auto_summary = is_reasoning_auto_summary_enabled()
|
||||
if summary:
|
||||
return {
|
||||
"reasoning_effort": {
|
||||
"effort": reasoning_effort,
|
||||
"summary": summary,
|
||||
}
|
||||
}
|
||||
elif auto_summary:
|
||||
return {
|
||||
"reasoning_effort": {
|
||||
"effort": reasoning_effort,
|
||||
"summary": "detailed",
|
||||
}
|
||||
}
|
||||
return {"reasoning_effort": reasoning_effort}
|
||||
return {}
|
||||
|
||||
|
|
@ -833,6 +856,11 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
if not schema:
|
||||
return None
|
||||
|
||||
# Deep copy to avoid mutating the original schema
|
||||
schema = copy.deepcopy(schema)
|
||||
# OpenAI strict mode requires additionalProperties: false on every object
|
||||
self._add_additional_properties_false(schema)
|
||||
|
||||
# Convert to OpenAI response_format structure
|
||||
return {
|
||||
"type": "json_schema",
|
||||
|
|
@ -843,6 +871,46 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
},
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _add_additional_properties_false(schema: dict) -> None:
|
||||
"""
|
||||
Recursively ensure object schemas comply with OpenAI strict mode.
|
||||
|
||||
OpenAI's strict mode requires:
|
||||
1. 'additionalProperties': false at every object nesting level
|
||||
2. All property keys listed in 'required'
|
||||
"""
|
||||
if not isinstance(schema, dict):
|
||||
return
|
||||
|
||||
if schema.get("type") == "object" and "properties" in schema:
|
||||
schema["additionalProperties"] = False
|
||||
schema["required"] = list(schema["properties"].keys())
|
||||
for prop in schema["properties"].values():
|
||||
LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(prop)
|
||||
|
||||
# Handle array items
|
||||
if "items" in schema:
|
||||
LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(
|
||||
schema["items"]
|
||||
)
|
||||
|
||||
# Handle anyOf/oneOf/allOf
|
||||
for key in ("anyOf", "oneOf", "allOf"):
|
||||
if key in schema:
|
||||
for sub_schema in schema[key]:
|
||||
LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(
|
||||
sub_schema
|
||||
)
|
||||
|
||||
# Handle $defs / definitions
|
||||
for key in ("$defs", "definitions"):
|
||||
if key in schema:
|
||||
for def_schema in schema[key].values():
|
||||
LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(
|
||||
def_schema
|
||||
)
|
||||
|
||||
def _add_system_message_to_messages(
|
||||
self,
|
||||
new_messages: List[AllMessageValues],
|
||||
|
|
@ -878,6 +946,144 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
ChatCompletionSystemMessage(role="system", content=openai_system_content), # type: ignore
|
||||
)
|
||||
|
||||
def _translate_metadata_to_openai(
|
||||
self,
|
||||
anthropic_message_request: AnthropicMessagesRequest,
|
||||
new_kwargs: ChatCompletionRequest,
|
||||
) -> None:
|
||||
"""Translate metadata fields from Anthropic request to OpenAI request."""
|
||||
if "metadata" in anthropic_message_request:
|
||||
metadata = anthropic_message_request["metadata"]
|
||||
if metadata and "user_id" in metadata:
|
||||
new_kwargs["user"] = metadata["user_id"]
|
||||
|
||||
if "litellm_metadata" in anthropic_message_request:
|
||||
# metadata will be passed to litellm.acompletion(), it's a litellm_param
|
||||
new_kwargs["metadata"] = anthropic_message_request.pop("litellm_metadata")
|
||||
|
||||
def _translate_tool_choice_to_openai(
|
||||
self,
|
||||
anthropic_message_request: AnthropicMessagesRequest,
|
||||
new_kwargs: ChatCompletionRequest,
|
||||
) -> None:
|
||||
"""Translate Anthropic tool_choice to OpenAI format."""
|
||||
if "tool_choice" not in anthropic_message_request:
|
||||
return
|
||||
tool_choice = anthropic_message_request["tool_choice"]
|
||||
if not tool_choice:
|
||||
return
|
||||
new_kwargs["tool_choice"] = self.translate_anthropic_tool_choice_to_openai(
|
||||
tool_choice=cast(AnthropicMessagesToolChoice, tool_choice)
|
||||
)
|
||||
|
||||
def _translate_tools_to_openai(
|
||||
self,
|
||||
anthropic_message_request: AnthropicMessagesRequest,
|
||||
new_kwargs: ChatCompletionRequest,
|
||||
) -> Dict[str, str]:
|
||||
"""Translate tools and extract web_search_options when needed."""
|
||||
if "tools" not in anthropic_message_request:
|
||||
return {}
|
||||
|
||||
tools = anthropic_message_request["tools"]
|
||||
if not tools:
|
||||
return {}
|
||||
|
||||
web_search_tools: List[AllAnthropicToolsValues] = []
|
||||
regular_tools: List[AllAnthropicToolsValues] = []
|
||||
for tool in tools:
|
||||
cast_tool = cast(Dict[str, Any], tool)
|
||||
if self._is_web_search_tool(cast_tool):
|
||||
web_search_tools.append(cast(AllAnthropicToolsValues, tool))
|
||||
else:
|
||||
regular_tools.append(cast(AllAnthropicToolsValues, tool))
|
||||
|
||||
if web_search_tools:
|
||||
new_kwargs["web_search_options"] = {} # type: ignore
|
||||
|
||||
if not regular_tools:
|
||||
return {}
|
||||
|
||||
translated_tools, tool_name_mapping = self.translate_anthropic_tools_to_openai(
|
||||
tools=regular_tools,
|
||||
model=new_kwargs.get("model"),
|
||||
)
|
||||
new_kwargs["tools"] = translated_tools
|
||||
return tool_name_mapping
|
||||
|
||||
def _translate_thinking_to_openai(
|
||||
self,
|
||||
anthropic_message_request: AnthropicMessagesRequest,
|
||||
new_kwargs: ChatCompletionRequest,
|
||||
) -> None:
|
||||
"""Translate Anthropic thinking to either thinking or reasoning_effort."""
|
||||
if "thinking" not in anthropic_message_request:
|
||||
return
|
||||
|
||||
thinking = anthropic_message_request["thinking"]
|
||||
if not thinking:
|
||||
return
|
||||
|
||||
model = new_kwargs.get("model", "")
|
||||
if self.is_anthropic_claude_model(model):
|
||||
new_kwargs["thinking"] = thinking # type: ignore
|
||||
return
|
||||
|
||||
reasoning_effort = self.translate_anthropic_thinking_to_reasoning_effort(
|
||||
cast(Dict[str, Any], thinking)
|
||||
)
|
||||
if not reasoning_effort:
|
||||
return
|
||||
|
||||
summary = thinking.get("summary") if isinstance(thinking, dict) else None
|
||||
auto_summary = is_reasoning_auto_summary_enabled()
|
||||
if summary:
|
||||
new_kwargs["reasoning_effort"] = cast(
|
||||
Any,
|
||||
{
|
||||
"effort": reasoning_effort,
|
||||
"summary": summary,
|
||||
},
|
||||
)
|
||||
elif auto_summary:
|
||||
new_kwargs["reasoning_effort"] = cast(
|
||||
Any,
|
||||
{
|
||||
"effort": reasoning_effort,
|
||||
"summary": "detailed",
|
||||
},
|
||||
)
|
||||
else:
|
||||
new_kwargs["reasoning_effort"] = reasoning_effort
|
||||
|
||||
def _translate_output_format_to_openai(
|
||||
self,
|
||||
anthropic_message_request: AnthropicMessagesRequest,
|
||||
new_kwargs: ChatCompletionRequest,
|
||||
) -> None:
|
||||
"""Translate output_format to response_format when applicable."""
|
||||
if "output_format" not in anthropic_message_request:
|
||||
return
|
||||
output_format = anthropic_message_request["output_format"]
|
||||
if not output_format:
|
||||
return
|
||||
response_format = self.translate_anthropic_output_format_to_openai(
|
||||
output_format=output_format
|
||||
)
|
||||
if response_format:
|
||||
new_kwargs["response_format"] = response_format
|
||||
|
||||
def _copy_untranslated_anthropic_params(
|
||||
self,
|
||||
anthropic_message_request: AnthropicMessagesRequest,
|
||||
new_kwargs: ChatCompletionRequest,
|
||||
) -> None:
|
||||
"""Copy through anthropic params that do not require translation."""
|
||||
translatable_params = self.translatable_anthropic_params()
|
||||
for k, v in anthropic_message_request.items():
|
||||
if k not in translatable_params: # pass remaining params as is
|
||||
new_kwargs[k] = v # type: ignore
|
||||
|
||||
def translate_anthropic_to_openai(
|
||||
self, anthropic_message_request: AnthropicMessagesRequest
|
||||
) -> Tuple[ChatCompletionRequest, Dict[str, str]]:
|
||||
|
|
@ -918,83 +1124,35 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
"model": anthropic_message_request["model"],
|
||||
"messages": new_messages,
|
||||
}
|
||||
## CONVERT METADATA (user_id)
|
||||
if "metadata" in anthropic_message_request:
|
||||
metadata = anthropic_message_request["metadata"]
|
||||
if metadata and "user_id" in metadata:
|
||||
new_kwargs["user"] = metadata["user_id"]
|
||||
|
||||
# Pass litellm proxy specific metadata
|
||||
if "litellm_metadata" in anthropic_message_request:
|
||||
# metadata will be passed to litellm.acompletion(), it's a litellm_param
|
||||
new_kwargs["metadata"] = anthropic_message_request.pop("litellm_metadata")
|
||||
|
||||
## CONVERT METADATA (user_id + litellm metadata)
|
||||
self._translate_metadata_to_openai(
|
||||
anthropic_message_request=anthropic_message_request,
|
||||
new_kwargs=new_kwargs,
|
||||
)
|
||||
## CONVERT TOOL CHOICE
|
||||
if "tool_choice" in anthropic_message_request:
|
||||
tool_choice = anthropic_message_request["tool_choice"]
|
||||
if tool_choice:
|
||||
new_kwargs[
|
||||
"tool_choice"
|
||||
] = self.translate_anthropic_tool_choice_to_openai(
|
||||
tool_choice=cast(AnthropicMessagesToolChoice, tool_choice)
|
||||
)
|
||||
self._translate_tool_choice_to_openai(
|
||||
anthropic_message_request=anthropic_message_request,
|
||||
new_kwargs=new_kwargs,
|
||||
)
|
||||
## CONVERT TOOLS
|
||||
if "tools" in anthropic_message_request:
|
||||
tools = anthropic_message_request["tools"]
|
||||
if tools:
|
||||
# Separate web search tools from regular tools
|
||||
web_search_tools = []
|
||||
regular_tools = []
|
||||
for tool in tools:
|
||||
if self._is_web_search_tool(cast(Dict[str, Any], tool)):
|
||||
web_search_tools.append(tool)
|
||||
else:
|
||||
regular_tools.append(tool)
|
||||
|
||||
# If web search tools are present, add web_search_options parameter
|
||||
if web_search_tools:
|
||||
new_kwargs["web_search_options"] = {} # type: ignore
|
||||
|
||||
# Only translate regular tools (non-web-search)
|
||||
if regular_tools:
|
||||
(
|
||||
new_kwargs["tools"],
|
||||
tool_name_mapping,
|
||||
) = self.translate_anthropic_tools_to_openai(
|
||||
tools=cast(List[AllAnthropicToolsValues], regular_tools),
|
||||
model=new_kwargs.get("model"),
|
||||
)
|
||||
|
||||
tool_name_mapping = self._translate_tools_to_openai(
|
||||
anthropic_message_request=anthropic_message_request,
|
||||
new_kwargs=new_kwargs,
|
||||
)
|
||||
## CONVERT THINKING
|
||||
if "thinking" in anthropic_message_request:
|
||||
thinking = anthropic_message_request["thinking"]
|
||||
if thinking:
|
||||
model = new_kwargs.get("model", "")
|
||||
if self.is_anthropic_claude_model(model):
|
||||
new_kwargs["thinking"] = thinking # type: ignore
|
||||
else:
|
||||
reasoning_effort = (
|
||||
self.translate_anthropic_thinking_to_reasoning_effort(
|
||||
cast(Dict[str, Any], thinking)
|
||||
)
|
||||
)
|
||||
if reasoning_effort:
|
||||
new_kwargs["reasoning_effort"] = reasoning_effort
|
||||
|
||||
self._translate_thinking_to_openai(
|
||||
anthropic_message_request=anthropic_message_request,
|
||||
new_kwargs=new_kwargs,
|
||||
)
|
||||
## CONVERT OUTPUT_FORMAT to RESPONSE_FORMAT
|
||||
if "output_format" in anthropic_message_request:
|
||||
output_format = anthropic_message_request["output_format"]
|
||||
if output_format:
|
||||
response_format = self.translate_anthropic_output_format_to_openai(
|
||||
output_format=output_format
|
||||
)
|
||||
if response_format:
|
||||
new_kwargs["response_format"] = response_format
|
||||
|
||||
translatable_params = self.translatable_anthropic_params()
|
||||
for k, v in anthropic_message_request.items():
|
||||
if k not in translatable_params: # pass remaining params as is
|
||||
new_kwargs[k] = v # type: ignore
|
||||
self._translate_output_format_to_openai(
|
||||
anthropic_message_request=anthropic_message_request,
|
||||
new_kwargs=new_kwargs,
|
||||
)
|
||||
self._copy_untranslated_anthropic_params(
|
||||
anthropic_message_request=anthropic_message_request,
|
||||
new_kwargs=new_kwargs,
|
||||
)
|
||||
|
||||
return new_kwargs, tool_name_mapping
|
||||
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@
|
|||
import asyncio
|
||||
import contextvars
|
||||
from functools import partial
|
||||
from typing import Any, AsyncIterator, Coroutine, Dict, List, Optional, Union
|
||||
from typing import Any, AsyncIterator, Coroutine, Dict, List, Optional, Union, cast
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
|
|
@ -114,6 +114,55 @@ async def _execute_pre_request_hooks(
|
|||
return request_kwargs
|
||||
|
||||
|
||||
async def _try_websearch_short_circuit(
|
||||
model: str,
|
||||
messages: List[Dict],
|
||||
tools: Optional[List[Dict]],
|
||||
custom_llm_provider: Optional[str],
|
||||
stream: Optional[bool],
|
||||
) -> Optional[Union[AnthropicMessagesResponse, AsyncIterator]]:
|
||||
"""
|
||||
Attempt to short-circuit a web-search-only request.
|
||||
|
||||
Claude Code sends web search as a separate, standalone /v1/messages
|
||||
request. For providers that don't natively support web search (e.g.
|
||||
github_copilot), we detect this pattern, execute the search via
|
||||
Tavily/Perplexity, and return a synthetic Anthropic response — bypassing
|
||||
the backend LLM entirely.
|
||||
|
||||
Returns the synthetic response if short-circuited, or None to continue
|
||||
normal processing.
|
||||
"""
|
||||
if not litellm.callbacks:
|
||||
return None
|
||||
|
||||
from litellm.integrations.websearch_interception.handler import (
|
||||
WebSearchInterceptionLogger,
|
||||
)
|
||||
|
||||
for callback in litellm.callbacks:
|
||||
if not isinstance(callback, WebSearchInterceptionLogger):
|
||||
continue
|
||||
|
||||
response = await callback.try_short_circuit_search(
|
||||
model=model,
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
if response is not None:
|
||||
anthropic_response = cast(AnthropicMessagesResponse, response)
|
||||
if stream:
|
||||
from litellm.llms.anthropic.experimental_pass_through.messages.fake_stream_iterator import (
|
||||
FakeAnthropicMessagesStreamIterator,
|
||||
)
|
||||
|
||||
return FakeAnthropicMessagesStreamIterator(anthropic_response)
|
||||
return anthropic_response
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@client
|
||||
async def anthropic_messages(
|
||||
max_tokens: int,
|
||||
|
|
@ -138,6 +187,12 @@ async def anthropic_messages(
|
|||
"""
|
||||
Async: Make llm api request in Anthropic /messages API spec
|
||||
"""
|
||||
# Save original stream flag before pre-request hooks can convert it.
|
||||
# The websearch interception hook converts stream=True → stream=False
|
||||
# for the agentic loop, but the short-circuit path needs to know
|
||||
# whether the caller originally requested streaming.
|
||||
original_stream = stream
|
||||
|
||||
# Execute pre-request hooks to allow CustomLoggers to modify request
|
||||
request_kwargs = await _execute_pre_request_hooks(
|
||||
model=model,
|
||||
|
|
@ -151,11 +206,38 @@ async def anthropic_messages(
|
|||
# Extract modified parameters
|
||||
tools = request_kwargs.pop("tools", tools)
|
||||
stream = request_kwargs.pop("stream", stream)
|
||||
# Propagate the provider derived inside pre-request hooks, if not already set.
|
||||
# The litellm_params dict may have been overwritten by **kwargs in
|
||||
# _execute_pre_request_hooks, so fall back to get_llm_provider() if needed.
|
||||
if not custom_llm_provider:
|
||||
custom_llm_provider = request_kwargs.get("litellm_params", {}).get(
|
||||
"custom_llm_provider"
|
||||
)
|
||||
if not custom_llm_provider:
|
||||
try:
|
||||
_, custom_llm_provider, _, _ = litellm.get_llm_provider(model=model)
|
||||
except Exception:
|
||||
pass
|
||||
# Remove litellm_params from kwargs (only needed for hooks)
|
||||
request_kwargs.pop("litellm_params", None)
|
||||
# Merge back any other modifications
|
||||
kwargs.update(request_kwargs)
|
||||
|
||||
# Short-circuit web-search-only requests: detect the pattern, execute
|
||||
# search directly via Tavily/Perplexity, and return a synthetic response
|
||||
# without ever touching the backend LLM or the adapter path.
|
||||
# Use original_stream (not the hook-converted stream) so streaming
|
||||
# callers get SSE events instead of a plain dict.
|
||||
short_circuit_response = await _try_websearch_short_circuit(
|
||||
model=model,
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
stream=original_stream,
|
||||
)
|
||||
if short_circuit_response is not None:
|
||||
return short_circuit_response
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
kwargs["is_async"] = True
|
||||
|
||||
|
|
|
|||
|
|
@ -8,6 +8,9 @@ path used for OpenAI and Azure models.
|
|||
import json
|
||||
from typing import Any, Dict, List, Optional, Union, cast
|
||||
|
||||
from litellm.llms.anthropic.experimental_pass_through.utils import (
|
||||
is_reasoning_auto_summary_enabled,
|
||||
)
|
||||
from litellm.types.llms.anthropic import (
|
||||
AllAnthropicToolsValues,
|
||||
AnthopicMessagesAssistantMessageParam,
|
||||
|
|
@ -94,7 +97,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
|
|||
)
|
||||
elif btype == "image":
|
||||
url = self._translate_anthropic_image_source_to_url(
|
||||
block.get("source", {})
|
||||
cast(dict, block.get("source", {}))
|
||||
)
|
||||
if url:
|
||||
user_parts.append(
|
||||
|
|
@ -267,7 +270,14 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
|
|||
effort = "low"
|
||||
else:
|
||||
effort = "minimal"
|
||||
return {"effort": effort, "summary": "detailed"}
|
||||
auto_summary = is_reasoning_auto_summary_enabled()
|
||||
result: Dict[str, Any] = {"effort": effort}
|
||||
summary = thinking.get("summary")
|
||||
if summary:
|
||||
result["summary"] = summary
|
||||
elif auto_summary:
|
||||
result["summary"] = "detailed"
|
||||
return result
|
||||
|
||||
def translate_request(
|
||||
self,
|
||||
|
|
|
|||
11
litellm/llms/anthropic/experimental_pass_through/utils.py
Normal file
11
litellm/llms/anthropic/experimental_pass_through/utils.py
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
import os
|
||||
|
||||
import litellm
|
||||
|
||||
|
||||
def is_reasoning_auto_summary_enabled() -> bool:
|
||||
"""Check whether the default 'summary: detailed' injection is enabled (opt-in)."""
|
||||
return (
|
||||
litellm.reasoning_auto_summary
|
||||
or os.getenv("LITELLM_REASONING_AUTO_SUMMARY", "false").lower() == "true"
|
||||
)
|
||||
|
|
@ -131,14 +131,9 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
|
|||
if result_effort == "none" and not supports_none:
|
||||
result.pop("reasoning_effort")
|
||||
|
||||
# Azure Chat Completions: gpt-5.4+ does not support tools + reasoning together.
|
||||
# Drop reasoning_effort when both are present (OpenAI routes to Responses API; Azure does not).
|
||||
if self.is_model_gpt_5_4_plus_model(model):
|
||||
has_tools = bool(
|
||||
non_default_params.get("tools") or optional_params.get("tools")
|
||||
)
|
||||
if has_tools and result_effort not in (None, "none"):
|
||||
result.pop("reasoning_effort", None)
|
||||
# Azure gpt-5.4+ with tools + reasoning_effort is now routed to the
|
||||
# Responses API bridge (same as OpenAI), so we no longer need to drop
|
||||
# reasoning_effort here. See: responses_api_bridge_check() in main.py.
|
||||
|
||||
return result
|
||||
|
||||
|
|
|
|||
|
|
@ -97,14 +97,59 @@ class AzureAIAgentsHandler:
|
|||
# -------------------------------------------------------------------------
|
||||
# Response Helpers
|
||||
# -------------------------------------------------------------------------
|
||||
def _extract_content_from_messages(self, messages_data: dict) -> str:
|
||||
"""Extract assistant content from the messages response."""
|
||||
def _extract_content_from_messages(
|
||||
self, messages_data: dict
|
||||
) -> Tuple[str, Optional[List[Dict[str, Any]]]]:
|
||||
"""Extract assistant content and annotations from the messages response.
|
||||
|
||||
Returns (content, annotations) where annotations is a list of
|
||||
OpenAI-compatible ChatCompletionAnnotation dicts, or None.
|
||||
"""
|
||||
for msg in messages_data.get("data", []):
|
||||
if msg.get("role") == "assistant":
|
||||
for content_item in msg.get("content", []):
|
||||
if content_item.get("type") == "text":
|
||||
return content_item.get("text", {}).get("value", "")
|
||||
return ""
|
||||
text_obj = content_item.get("text", {})
|
||||
content = text_obj.get("value", "")
|
||||
raw_annotations = text_obj.get("annotations")
|
||||
annotations = self._transform_annotations(raw_annotations)
|
||||
return content, annotations
|
||||
return "", None
|
||||
|
||||
def _transform_annotations(
|
||||
self,
|
||||
raw_annotations: Optional[List[Dict[str, Any]]],
|
||||
) -> Optional[List[Dict[str, Any]]]:
|
||||
"""Transform Azure AI Foundry annotations to OpenAI-compatible format.
|
||||
|
||||
Azure AI returns annotations like:
|
||||
{"type": "url_citation", "text": "[1]", "start_index": 10,
|
||||
"end_index": 13, "url_citation": {"url": "...", "title": "..."}}
|
||||
|
||||
OpenAI expects:
|
||||
{"type": "url_citation", "url_citation": {"url": "...", "title": "...",
|
||||
"start_index": 10, "end_index": 13}}
|
||||
"""
|
||||
if not raw_annotations:
|
||||
return None
|
||||
|
||||
result: List[Dict[str, Any]] = []
|
||||
for ann in raw_annotations:
|
||||
ann_type = ann.get("type")
|
||||
if ann_type == "url_citation":
|
||||
url_citation = dict(ann.get("url_citation", {}))
|
||||
# Azure puts start/end_index at annotation level; OpenAI
|
||||
# expects them inside url_citation
|
||||
if "start_index" in ann and "start_index" not in url_citation:
|
||||
url_citation["start_index"] = ann["start_index"]
|
||||
if "end_index" in ann and "end_index" not in url_citation:
|
||||
url_citation["end_index"] = ann["end_index"]
|
||||
result.append({"type": "url_citation", "url_citation": url_citation})
|
||||
else:
|
||||
# Pass through unknown annotation types as-is
|
||||
result.append(ann)
|
||||
|
||||
return result if result else None
|
||||
|
||||
def _build_model_response(
|
||||
self,
|
||||
|
|
@ -113,15 +158,23 @@ class AzureAIAgentsHandler:
|
|||
model_response: ModelResponse,
|
||||
thread_id: str,
|
||||
messages: List[Dict[str, Any]],
|
||||
annotations: Optional[List[Dict[str, Any]]] = None,
|
||||
) -> ModelResponse:
|
||||
"""Build the ModelResponse from agent output."""
|
||||
from litellm.types.utils import Choices, Message, Usage
|
||||
|
||||
message_kwargs: Dict[str, Any] = {
|
||||
"content": content,
|
||||
"role": "assistant",
|
||||
}
|
||||
if annotations:
|
||||
message_kwargs["annotations"] = annotations
|
||||
|
||||
model_response.choices = [
|
||||
Choices(
|
||||
finish_reason="stop",
|
||||
index=0,
|
||||
message=Message(content=content, role="assistant"),
|
||||
message=Message(**message_kwargs),
|
||||
)
|
||||
]
|
||||
model_response.model = model
|
||||
|
|
@ -250,7 +303,7 @@ class AzureAIAgentsHandler:
|
|||
)
|
||||
|
||||
# Execute the agent flow
|
||||
thread_id, content = self._execute_agent_flow_sync(
|
||||
thread_id, content, annotations = self._execute_agent_flow_sync(
|
||||
make_request=make_request,
|
||||
api_base=api_base,
|
||||
api_version=api_version,
|
||||
|
|
@ -261,7 +314,7 @@ class AzureAIAgentsHandler:
|
|||
)
|
||||
|
||||
return self._build_model_response(
|
||||
model, content, model_response, thread_id, messages
|
||||
model, content, model_response, thread_id, messages, annotations
|
||||
)
|
||||
|
||||
def _execute_agent_flow_sync(
|
||||
|
|
@ -273,8 +326,8 @@ class AzureAIAgentsHandler:
|
|||
thread_id: Optional[str],
|
||||
messages: List[Dict[str, Any]],
|
||||
optional_params: dict,
|
||||
) -> Tuple[str, str]:
|
||||
"""Execute the agent flow synchronously. Returns (thread_id, content)."""
|
||||
) -> Tuple[str, str, Optional[List[Dict[str, Any]]]]:
|
||||
"""Execute the agent flow synchronously. Returns (thread_id, content, annotations)."""
|
||||
|
||||
# Step 1: Create thread if not provided
|
||||
if not thread_id:
|
||||
|
|
@ -347,8 +400,8 @@ class AzureAIAgentsHandler:
|
|||
)
|
||||
self._check_response(response, [200], "Failed to get messages")
|
||||
|
||||
content = self._extract_content_from_messages(response.json())
|
||||
return thread_id, content
|
||||
content, annotations = self._extract_content_from_messages(response.json())
|
||||
return thread_id, content, annotations
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Async Completion
|
||||
|
|
@ -399,7 +452,7 @@ class AzureAIAgentsHandler:
|
|||
)
|
||||
|
||||
# Execute the agent flow
|
||||
thread_id, content = await self._execute_agent_flow_async(
|
||||
thread_id, content, annotations = await self._execute_agent_flow_async(
|
||||
make_request=make_request,
|
||||
api_base=api_base,
|
||||
api_version=api_version,
|
||||
|
|
@ -410,7 +463,7 @@ class AzureAIAgentsHandler:
|
|||
)
|
||||
|
||||
return self._build_model_response(
|
||||
model, content, model_response, thread_id, messages
|
||||
model, content, model_response, thread_id, messages, annotations
|
||||
)
|
||||
|
||||
async def _execute_agent_flow_async(
|
||||
|
|
@ -422,8 +475,8 @@ class AzureAIAgentsHandler:
|
|||
thread_id: Optional[str],
|
||||
messages: List[Dict[str, Any]],
|
||||
optional_params: dict,
|
||||
) -> Tuple[str, str]:
|
||||
"""Execute the agent flow asynchronously. Returns (thread_id, content)."""
|
||||
) -> Tuple[str, str, Optional[List[Dict[str, Any]]]]:
|
||||
"""Execute the agent flow asynchronously. Returns (thread_id, content, annotations)."""
|
||||
|
||||
# Step 1: Create thread if not provided
|
||||
if not thread_id:
|
||||
|
|
@ -496,8 +549,8 @@ class AzureAIAgentsHandler:
|
|||
)
|
||||
self._check_response(response, [200], "Failed to get messages")
|
||||
|
||||
content = self._extract_content_from_messages(response.json())
|
||||
return thread_id, content
|
||||
content, annotations = self._extract_content_from_messages(response.json())
|
||||
return thread_id, content, annotations
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Streaming Completion (Native SSE)
|
||||
|
|
@ -585,6 +638,7 @@ class AzureAIAgentsHandler:
|
|||
response_id = f"chatcmpl-{uuid.uuid4().hex[:8]}"
|
||||
created = int(time.time())
|
||||
thread_id = None
|
||||
collected_annotations: Optional[List[Dict[str, Any]]] = None
|
||||
|
||||
current_event = None
|
||||
|
||||
|
|
@ -600,6 +654,9 @@ class AzureAIAgentsHandler:
|
|||
|
||||
if data_str == "[DONE]":
|
||||
# Send final chunk with finish_reason
|
||||
final_delta_kwargs: Dict[str, Any] = {"content": None}
|
||||
if collected_annotations:
|
||||
final_delta_kwargs["annotations"] = collected_annotations
|
||||
final_chunk = ModelResponseStream(
|
||||
id=response_id,
|
||||
created=created,
|
||||
|
|
@ -609,7 +666,7 @@ class AzureAIAgentsHandler:
|
|||
StreamingChoices(
|
||||
finish_reason="stop",
|
||||
index=0,
|
||||
delta=Delta(content=None),
|
||||
delta=Delta(**final_delta_kwargs),
|
||||
)
|
||||
],
|
||||
)
|
||||
|
|
@ -628,6 +685,19 @@ class AzureAIAgentsHandler:
|
|||
thread_id = data["id"]
|
||||
verbose_logger.debug(f"Stream created thread: {thread_id}")
|
||||
|
||||
# Extract annotations from completed message
|
||||
if current_event == "thread.message.completed":
|
||||
for content_item in data.get("content", []):
|
||||
if content_item.get("type") == "text":
|
||||
raw_annotations = content_item.get("text", {}).get(
|
||||
"annotations"
|
||||
)
|
||||
transformed = self._transform_annotations(raw_annotations)
|
||||
if transformed:
|
||||
if collected_annotations is None:
|
||||
collected_annotations = []
|
||||
collected_annotations.extend(transformed)
|
||||
|
||||
# Process message deltas - this is where the actual content comes
|
||||
if current_event == "thread.message.delta":
|
||||
delta_content = data.get("delta", {}).get("content", [])
|
||||
|
|
|
|||
|
|
@ -73,6 +73,7 @@ class BaseTranslation(ABC):
|
|||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: Optional["LiteLLMLoggingObj"] = None,
|
||||
user_api_key_dict: Optional["UserAPIKeyAuth"] = None,
|
||||
request_data: Optional[dict] = None,
|
||||
) -> Any:
|
||||
"""
|
||||
Process output response with guardrails.
|
||||
|
|
@ -91,6 +92,7 @@ class BaseTranslation(ABC):
|
|||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: Optional["LiteLLMLoggingObj"] = None,
|
||||
user_api_key_dict: Optional["UserAPIKeyAuth"] = None,
|
||||
request_data: Optional[dict] = None,
|
||||
) -> Any:
|
||||
"""
|
||||
Process output streaming response with guardrails.
|
||||
|
|
|
|||
|
|
@ -54,6 +54,14 @@ class BaseResponsesAPIConfig(ABC):
|
|||
and v is not None
|
||||
}
|
||||
|
||||
def supports_native_file_search(self) -> bool:
|
||||
"""Return True if this provider handles the file_search tool natively.
|
||||
|
||||
Override in provider subclasses that support file_search without
|
||||
LiteLLM emulation (e.g. OpenAI, Azure OpenAI).
|
||||
"""
|
||||
return False
|
||||
|
||||
@abstractmethod
|
||||
def get_supported_openai_params(self, model: str) -> list:
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -1446,6 +1446,16 @@ class AmazonConverseConfig(BaseConfig):
|
|||
original_tools, model, headers, additional_request_params
|
||||
)
|
||||
|
||||
# Append cachePoint to tools if cache_control_injection_points has tool_config
|
||||
cache_injection_points = additional_request_params.pop(
|
||||
"cache_control_injection_points", None
|
||||
)
|
||||
if cache_injection_points and len(bedrock_tools) > 0:
|
||||
for point in cache_injection_points:
|
||||
if point.get("location") == "tool_config":
|
||||
bedrock_tools.append({"cachePoint": {"type": "default"}})
|
||||
break
|
||||
|
||||
bedrock_tool_config: Optional[ToolConfigBlock] = None
|
||||
if len(bedrock_tools) > 0:
|
||||
tool_choice_values: ToolChoiceValuesBlock = inference_params.pop(
|
||||
|
|
|
|||
|
|
@ -64,8 +64,15 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig):
|
|||
verbose_logger.debug(f"Transformed request: {bedrock_request}")
|
||||
|
||||
# Get endpoint URL using simplified function
|
||||
api_base = litellm_params.get("api_base", None)
|
||||
aws_bedrock_runtime_endpoint = litellm_params.get(
|
||||
"aws_bedrock_runtime_endpoint", None
|
||||
)
|
||||
endpoint_url = self.get_bedrock_count_tokens_endpoint(
|
||||
resolved_model, aws_region_name
|
||||
model=resolved_model,
|
||||
aws_region_name=aws_region_name,
|
||||
api_base=api_base,
|
||||
aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint,
|
||||
)
|
||||
|
||||
verbose_logger.debug(f"Making request to: {endpoint_url}")
|
||||
|
|
|
|||
|
|
@ -177,7 +177,11 @@ class BedrockCountTokensConfig(BaseAWSLLM):
|
|||
return {"input": {"invokeModel": {"body": json.dumps(body_data)}}}
|
||||
|
||||
def get_bedrock_count_tokens_endpoint(
|
||||
self, model: str, aws_region_name: str
|
||||
self,
|
||||
model: str,
|
||||
aws_region_name: str,
|
||||
api_base: Optional[str] = None,
|
||||
aws_bedrock_runtime_endpoint: Optional[str] = None,
|
||||
) -> str:
|
||||
"""
|
||||
Construct the AWS Bedrock CountTokens API endpoint using existing LiteLLM functions.
|
||||
|
|
@ -185,6 +189,8 @@ class BedrockCountTokensConfig(BaseAWSLLM):
|
|||
Args:
|
||||
model: The resolved model ID from router lookup
|
||||
aws_region_name: AWS region (e.g., "eu-west-1")
|
||||
api_base: Optional custom API base URL (takes highest priority)
|
||||
aws_bedrock_runtime_endpoint: Optional custom Bedrock runtime endpoint
|
||||
|
||||
Returns:
|
||||
Complete endpoint URL for CountTokens API
|
||||
|
|
@ -196,7 +202,11 @@ class BedrockCountTokensConfig(BaseAWSLLM):
|
|||
if model_id.startswith("bedrock/"):
|
||||
model_id = model_id[8:] # Remove "bedrock/" prefix
|
||||
|
||||
base_url = f"https://bedrock-runtime.{aws_region_name}.amazonaws.com"
|
||||
base_url, _ = self.get_runtime_endpoint(
|
||||
api_base=api_base,
|
||||
aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint,
|
||||
aws_region_name=aws_region_name,
|
||||
)
|
||||
endpoint = f"{base_url}/model/{model_id}/count-tokens"
|
||||
|
||||
return endpoint
|
||||
|
|
|
|||
|
|
@ -83,6 +83,7 @@ class CohereRerankHandler(BaseTranslation):
|
|||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: Optional[Any] = None,
|
||||
user_api_key_dict: Optional[Any] = None,
|
||||
request_data: Optional[dict] = None,
|
||||
) -> Any:
|
||||
"""
|
||||
Process output response - not applicable for rerank.
|
||||
|
|
|
|||
|
|
@ -185,11 +185,16 @@ class FireworksAIConfig(OpenAIGPTConfig):
|
|||
): # allow user to toggle this feature.
|
||||
return content
|
||||
if isinstance(content["image_url"], str):
|
||||
content["image_url"] = f"{content['image_url']}#transform=inline"
|
||||
# Skip base64 data URLs — appending #transform=inline corrupts the
|
||||
# base64 payload and causes an "Incorrect padding" decode error on
|
||||
# the Fireworks side. Data URLs are already inlined by definition.
|
||||
# Lower-case before checking: URI schemes are case-insensitive (RFC 3986).
|
||||
if not content["image_url"].lower().startswith("data:"):
|
||||
content["image_url"] = f"{content['image_url']}#transform=inline"
|
||||
elif isinstance(content["image_url"], dict):
|
||||
content["image_url"][
|
||||
"url"
|
||||
] = f"{content['image_url']['url']}#transform=inline"
|
||||
url = content["image_url"]["url"]
|
||||
if not url.lower().startswith("data:"):
|
||||
content["image_url"]["url"] = f"{url}#transform=inline"
|
||||
return content
|
||||
|
||||
def _transform_tools(
|
||||
|
|
|
|||
|
|
@ -88,12 +88,15 @@ class GoogleImageGenConfig(BaseImageGenerationConfig):
|
|||
tokens_details = usage_metadata.get("promptTokensDetails", [])
|
||||
for details in tokens_details:
|
||||
if isinstance(details, dict):
|
||||
modality = details.get("modality")
|
||||
token_count = details.get("tokenCount", 0)
|
||||
modality = str(details.get("modality", "")).upper()
|
||||
raw_token_count = details.get(
|
||||
"tokenCount", details.get("token_count", 0)
|
||||
)
|
||||
token_count = raw_token_count if isinstance(raw_token_count, int) else 0
|
||||
if modality == "TEXT":
|
||||
input_tokens_details.text_tokens = token_count
|
||||
input_tokens_details.text_tokens += token_count
|
||||
elif modality == "IMAGE":
|
||||
input_tokens_details.image_tokens = token_count
|
||||
input_tokens_details.image_tokens += token_count
|
||||
|
||||
return ImageUsage(
|
||||
input_tokens=usage_metadata.get("promptTokenCount", 0),
|
||||
|
|
|
|||
|
|
@ -148,5 +148,12 @@ class MistralAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
|
|||
|
||||
text = response_json.get("text") or ""
|
||||
response = TranscriptionResponse(text=text)
|
||||
|
||||
# Preserve Mistral-specific fields (e.g. diarization segments)
|
||||
if "segments" in response_json:
|
||||
response["segments"] = response_json["segments"]
|
||||
if "language" in response_json:
|
||||
response["language"] = response_json["language"]
|
||||
|
||||
response._hidden_params = response_json
|
||||
return response
|
||||
|
|
|
|||
|
|
@ -91,6 +91,7 @@ class OCRHandler(BaseTranslation):
|
|||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: Optional[Any] = None,
|
||||
user_api_key_dict: Optional[Any] = None,
|
||||
request_data: Optional[dict] = None,
|
||||
) -> Any:
|
||||
"""
|
||||
Process OCR output by applying guardrails to extracted page text.
|
||||
|
|
@ -127,14 +128,27 @@ class OCRHandler(BaseTranslation):
|
|||
if model:
|
||||
inputs["model"] = model
|
||||
|
||||
# Use the real request_data if provided (proxy path), otherwise
|
||||
# create a standalone dict (SDK / direct-call path).
|
||||
if request_data is None:
|
||||
request_data = {}
|
||||
|
||||
# Add user metadata if available
|
||||
if user_api_key_dict is not None:
|
||||
metadata = self.transform_user_api_key_dict_to_metadata(user_api_key_dict)
|
||||
inputs.update(metadata) # type: ignore
|
||||
user_metadata = self.transform_user_api_key_dict_to_metadata(
|
||||
user_api_key_dict
|
||||
)
|
||||
if user_metadata:
|
||||
# Preserve original behavior: inject metadata into inputs for
|
||||
# third-party guardrail providers that read it from there
|
||||
inputs.update(user_metadata) # type: ignore
|
||||
# Also store in request_data for the logging pipeline
|
||||
if "litellm_metadata" not in request_data:
|
||||
request_data["litellm_metadata"] = user_metadata
|
||||
|
||||
guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
|
||||
inputs=inputs,
|
||||
request_data={},
|
||||
request_data=request_data,
|
||||
input_type="response",
|
||||
logging_obj=litellm_logging_obj,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -36,6 +36,8 @@ class MistralOCRConfig(BaseOCRConfig):
|
|||
- image_min_size: Minimum size of images to include
|
||||
- bbox_annotation_format: Format for bounding box annotations
|
||||
- document_annotation_format: Format for document annotations
|
||||
- extract_header: Whether to extract document header
|
||||
- extract_footer: Whether to extract document footer
|
||||
"""
|
||||
return [
|
||||
"pages",
|
||||
|
|
@ -44,6 +46,8 @@ class MistralOCRConfig(BaseOCRConfig):
|
|||
"image_min_size",
|
||||
"bbox_annotation_format",
|
||||
"document_annotation_format",
|
||||
"extract_header",
|
||||
"extract_footer",
|
||||
]
|
||||
|
||||
def map_ocr_params(
|
||||
|
|
|
|||
|
|
@ -155,9 +155,11 @@ class MoonshotChatConfig(OpenAIGPTConfig):
|
|||
message that contains tool_calls (multi-turn tool-calling flows).
|
||||
|
||||
For each such message that is missing the field:
|
||||
1. Promote provider_specific_fields["reasoning_content"] if present and non-empty
|
||||
1. Check if reasoning_content exists at the top level (for Pydantic models
|
||||
that have the attribute but don't support 'in' operator)
|
||||
2. Promote provider_specific_fields["reasoning_content"] if present and non-empty
|
||||
(this is where LiteLLM stores it from a previous response)
|
||||
2. Otherwise inject a single space — the minimum value the API accepts
|
||||
3. Otherwise inject a single space — the minimum value the API accepts
|
||||
Messages that already carry the field, or are not assistant/tool-call messages,
|
||||
are appended as-is (no copy made).
|
||||
"""
|
||||
|
|
@ -166,7 +168,9 @@ class MoonshotChatConfig(OpenAIGPTConfig):
|
|||
if (
|
||||
msg.get("role") == "assistant"
|
||||
and msg.get("tool_calls")
|
||||
and "reasoning_content" not in msg
|
||||
and not msg.get(
|
||||
"reasoning_content"
|
||||
) # Check using .get() which works for both dicts and Pydantic models
|
||||
):
|
||||
patched = dict(cast(dict, msg))
|
||||
provider_fields = patched.get("provider_specific_fields") or {}
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
from typing import Optional, Union
|
||||
|
||||
import litellm
|
||||
from litellm.utils import _supports_factory
|
||||
from litellm.utils import _is_explicitly_disabled_factory, _supports_factory
|
||||
|
||||
from .gpt_transformation import OpenAIGPTConfig
|
||||
|
||||
|
|
@ -113,6 +113,25 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
|
|||
key=f"supports_{level}_reasoning_effort",
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _is_reasoning_effort_level_explicitly_disabled(
|
||||
cls, model: str, level: str
|
||||
) -> bool:
|
||||
"""Return True only when the model map explicitly sets the capability to False.
|
||||
|
||||
Unlike ``_supports_reasoning_effort_level`` (which requires an explicit True),
|
||||
this method returns True only when ``supports_{level}_reasoning_effort`` is
|
||||
explicitly set to ``False`` in the model map. A missing key is treated as
|
||||
supported (i.e. this method returns False = not disabled).
|
||||
|
||||
Use this for opt-out checks where unknown models should be allowed through.
|
||||
"""
|
||||
return _is_explicitly_disabled_factory(
|
||||
model=model,
|
||||
custom_llm_provider=None,
|
||||
key=f"supports_{level}_reasoning_effort",
|
||||
)
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list:
|
||||
if self.is_model_gpt_5_search_model(model):
|
||||
return [
|
||||
|
|
@ -200,14 +219,32 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
|
|||
if "reasoning_effort" in optional_params:
|
||||
optional_params["reasoning_effort"] = normalized
|
||||
|
||||
if effective_effort is not None and effective_effort == "xhigh":
|
||||
if not self._supports_reasoning_effort_level(model, "xhigh"):
|
||||
if effective_effort == "xhigh":
|
||||
# xhigh is an opt-in capability: only allow if model explicitly supports it.
|
||||
if not self._supports_reasoning_effort_level(model, effective_effort):
|
||||
if litellm.drop_params or drop_params:
|
||||
non_default_params.pop("reasoning_effort", None)
|
||||
optional_params.pop("reasoning_effort", None)
|
||||
else:
|
||||
raise litellm.utils.UnsupportedParamsError(
|
||||
message=(
|
||||
"reasoning_effort='xhigh' is only supported for gpt-5.1-codex-max, gpt-5.2, and gpt-5.4+ models."
|
||||
f"reasoning_effort={effective_effort} is not supported for this model."
|
||||
),
|
||||
status_code=400,
|
||||
)
|
||||
elif effective_effort == "minimal":
|
||||
# minimal is opt-out: unknown models pass through; only block when
|
||||
# the model map explicitly sets supports_minimal_reasoning_effort=false.
|
||||
if self._is_reasoning_effort_level_explicitly_disabled(
|
||||
model, effective_effort
|
||||
):
|
||||
if litellm.drop_params or drop_params:
|
||||
non_default_params.pop("reasoning_effort", None)
|
||||
optional_params.pop("reasoning_effort", None)
|
||||
else:
|
||||
raise litellm.utils.UnsupportedParamsError(
|
||||
message=(
|
||||
f"reasoning_effort={effective_effort} is not supported for this model."
|
||||
),
|
||||
status_code=400,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ from typing import (
|
|||
Any,
|
||||
AsyncIterator,
|
||||
Coroutine,
|
||||
Dict,
|
||||
Iterator,
|
||||
List,
|
||||
Literal,
|
||||
|
|
@ -805,8 +806,8 @@ class OpenAIChatCompletionStreamingHandler(BaseModelResponseIterator):
|
|||
choices = chunk.get("choices", [])
|
||||
choices = self._map_reasoning_to_reasoning_content(choices)
|
||||
|
||||
kwargs = {
|
||||
"id": chunk["id"],
|
||||
kwargs: Dict[str, Any] = {
|
||||
"id": chunk.get("id"),
|
||||
"object": "chat.completion.chunk",
|
||||
"created": chunk.get("created"),
|
||||
"model": chunk.get("model"),
|
||||
|
|
|
|||
|
|
@ -86,9 +86,9 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
if tool_calls_to_check:
|
||||
inputs["tool_calls"] = tool_calls_to_check # type: ignore
|
||||
if messages:
|
||||
inputs[
|
||||
"structured_messages"
|
||||
] = messages # pass the openai /chat/completions messages to the guardrail, as-is
|
||||
inputs["structured_messages"] = (
|
||||
messages # pass the openai /chat/completions messages to the guardrail, as-is
|
||||
)
|
||||
# Pass tools (function definitions) to the guardrail
|
||||
tools = data.get("tools")
|
||||
if tools:
|
||||
|
|
@ -260,6 +260,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: Optional[Any] = None,
|
||||
user_api_key_dict: Optional[Any] = None,
|
||||
request_data: Optional[dict] = None,
|
||||
) -> Any:
|
||||
"""
|
||||
Process output response by applying guardrails to text content.
|
||||
|
|
@ -308,15 +309,21 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
|
||||
# Step 2: Apply guardrail to all texts and tool calls in batch
|
||||
if texts_to_check or tool_calls_to_check:
|
||||
# Create a request_data dict with response info and user API key metadata
|
||||
request_data: dict = {"response": response}
|
||||
# Use the real request_data if provided (proxy path), otherwise
|
||||
# create a standalone dict (SDK / direct-call path).
|
||||
if request_data is None:
|
||||
request_data = {"response": response}
|
||||
else:
|
||||
if "response" not in request_data:
|
||||
request_data["response"] = response
|
||||
|
||||
# Add user API key metadata with prefixed keys
|
||||
user_metadata = self.transform_user_api_key_dict_to_metadata(
|
||||
user_api_key_dict
|
||||
)
|
||||
if user_metadata:
|
||||
request_data["litellm_metadata"] = user_metadata
|
||||
if "litellm_metadata" not in request_data:
|
||||
user_metadata = self.transform_user_api_key_dict_to_metadata(
|
||||
user_api_key_dict
|
||||
)
|
||||
if user_metadata:
|
||||
request_data["litellm_metadata"] = user_metadata
|
||||
|
||||
inputs = GenericGuardrailAPIInputs(texts=texts_to_check)
|
||||
if images_to_check:
|
||||
|
|
@ -364,6 +371,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: Optional[Any] = None,
|
||||
user_api_key_dict: Optional[Any] = None,
|
||||
request_data: Optional[dict] = None,
|
||||
) -> List["ModelResponseStream"]:
|
||||
"""
|
||||
Process output streaming responses by applying guardrails to text content.
|
||||
|
|
@ -402,6 +410,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
guardrail_to_apply=guardrail_to_apply,
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
request_data=request_data,
|
||||
)
|
||||
|
||||
return responses_so_far
|
||||
|
|
@ -436,15 +445,21 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
|
||||
# Step 3: Apply guardrail to all combined texts in batch
|
||||
if texts_to_check:
|
||||
# Create a request_data dict with response info and user API key metadata
|
||||
request_data: dict = {"responses": responses_so_far}
|
||||
# Use the real request_data if provided (proxy path), otherwise
|
||||
# create a standalone dict (SDK / direct-call path).
|
||||
if request_data is None:
|
||||
request_data = {"responses": responses_so_far}
|
||||
else:
|
||||
if "responses" not in request_data:
|
||||
request_data["responses"] = responses_so_far
|
||||
|
||||
# Add user API key metadata with prefixed keys
|
||||
user_metadata = self.transform_user_api_key_dict_to_metadata(
|
||||
user_api_key_dict
|
||||
)
|
||||
if user_metadata:
|
||||
request_data["litellm_metadata"] = user_metadata
|
||||
if "litellm_metadata" not in request_data:
|
||||
user_metadata = self.transform_user_api_key_dict_to_metadata(
|
||||
user_api_key_dict
|
||||
)
|
||||
if user_metadata:
|
||||
request_data["litellm_metadata"] = user_metadata
|
||||
|
||||
inputs = GenericGuardrailAPIInputs(texts=texts_to_check)
|
||||
if images_to_check:
|
||||
|
|
|
|||
|
|
@ -125,6 +125,7 @@ class OpenAITextCompletionHandler(BaseTranslation):
|
|||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: Optional[Any] = None,
|
||||
user_api_key_dict: Optional[Any] = None,
|
||||
request_data: Optional[dict] = None,
|
||||
) -> Any:
|
||||
"""
|
||||
Process output response by applying guardrails to completion text.
|
||||
|
|
@ -155,15 +156,21 @@ class OpenAITextCompletionHandler(BaseTranslation):
|
|||
|
||||
# Apply guardrails in batch
|
||||
if texts_to_check:
|
||||
# Create a request_data dict with response info and user API key metadata
|
||||
request_data: dict = {"response": response}
|
||||
# Use the real request_data if provided (proxy path), otherwise
|
||||
# create a standalone dict (SDK / direct-call path).
|
||||
if request_data is None:
|
||||
request_data = {"response": response}
|
||||
else:
|
||||
if "response" not in request_data:
|
||||
request_data["response"] = response
|
||||
|
||||
# Add user API key metadata with prefixed keys
|
||||
user_metadata = self.transform_user_api_key_dict_to_metadata(
|
||||
user_api_key_dict
|
||||
)
|
||||
if user_metadata:
|
||||
request_data["litellm_metadata"] = user_metadata
|
||||
if "litellm_metadata" not in request_data:
|
||||
user_metadata = self.transform_user_api_key_dict_to_metadata(
|
||||
user_api_key_dict
|
||||
)
|
||||
if user_metadata:
|
||||
request_data["litellm_metadata"] = user_metadata
|
||||
|
||||
inputs = GenericGuardrailAPIInputs(texts=texts_to_check)
|
||||
# Include model information from the response if available
|
||||
|
|
|
|||
|
|
@ -155,6 +155,7 @@ class OpenAIEmbeddingsHandler(BaseTranslation):
|
|||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: Optional[Any] = None,
|
||||
user_api_key_dict: Optional[Any] = None,
|
||||
request_data: Optional[dict] = None,
|
||||
) -> Any:
|
||||
"""
|
||||
Process output response - embeddings responses contain vectors, not text.
|
||||
|
|
|
|||
|
|
@ -87,6 +87,7 @@ class OpenAIImageGenerationHandler(BaseTranslation):
|
|||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: Optional[Any] = None,
|
||||
user_api_key_dict: Optional[Any] = None,
|
||||
request_data: Optional[dict] = None,
|
||||
) -> Any:
|
||||
"""
|
||||
Process output response - typically not needed for image generation.
|
||||
|
|
|
|||
|
|
@ -347,6 +347,7 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: Optional[Any] = None,
|
||||
user_api_key_dict: Optional[Any] = None,
|
||||
request_data: Optional[dict] = None,
|
||||
) -> Any:
|
||||
"""
|
||||
Process output response by applying guardrails to text content and tool calls.
|
||||
|
|
@ -402,15 +403,21 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
|
||||
# Step 2: Apply guardrail to all texts in batch
|
||||
if texts_to_check or tool_calls_to_check:
|
||||
# Create a request_data dict with response info and user API key metadata
|
||||
request_data: dict = {"response": response}
|
||||
# Use the real request_data if provided (proxy path), otherwise
|
||||
# create a standalone dict (SDK / direct-call path).
|
||||
if request_data is None:
|
||||
request_data = {"response": response}
|
||||
else:
|
||||
if "response" not in request_data:
|
||||
request_data["response"] = response
|
||||
|
||||
# Add user API key metadata with prefixed keys
|
||||
user_metadata = self.transform_user_api_key_dict_to_metadata(
|
||||
user_api_key_dict
|
||||
)
|
||||
if user_metadata:
|
||||
request_data["litellm_metadata"] = user_metadata
|
||||
if "litellm_metadata" not in request_data:
|
||||
user_metadata = self.transform_user_api_key_dict_to_metadata(
|
||||
user_api_key_dict
|
||||
)
|
||||
if user_metadata:
|
||||
request_data["litellm_metadata"] = user_metadata
|
||||
|
||||
inputs = GenericGuardrailAPIInputs(texts=texts_to_check)
|
||||
if images_to_check:
|
||||
|
|
@ -454,6 +461,7 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: Optional[Any] = None,
|
||||
user_api_key_dict: Optional[Any] = None,
|
||||
request_data: Optional[dict] = None,
|
||||
) -> List[Any]:
|
||||
"""
|
||||
Process output streaming response by applying guardrails to text content.
|
||||
|
|
@ -481,7 +489,7 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
inputs["model"] = model_response_stream.model
|
||||
_guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
|
||||
inputs=inputs,
|
||||
request_data={},
|
||||
request_data=request_data if request_data is not None else {},
|
||||
input_type="response",
|
||||
logging_obj=litellm_logging_obj,
|
||||
)
|
||||
|
|
@ -512,7 +520,7 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
if tool_calls or text:
|
||||
_guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
|
||||
inputs=guardrail_inputs,
|
||||
request_data={},
|
||||
request_data=request_data if request_data is not None else {},
|
||||
input_type="response",
|
||||
logging_obj=litellm_logging_obj,
|
||||
)
|
||||
|
|
@ -537,7 +545,7 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
inputs["model"] = response_model
|
||||
_guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
|
||||
inputs=inputs,
|
||||
request_data={},
|
||||
request_data=request_data if request_data is not None else {},
|
||||
input_type="response",
|
||||
logging_obj=litellm_logging_obj,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -32,6 +32,9 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
|
|||
def custom_llm_provider(self) -> LlmProviders:
|
||||
return LlmProviders.OPENAI
|
||||
|
||||
def supports_native_file_search(self) -> bool:
|
||||
return True
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list:
|
||||
"""
|
||||
All OpenAI Responses API params are supported
|
||||
|
|
|
|||
|
|
@ -85,6 +85,7 @@ class OpenAITextToSpeechHandler(BaseTranslation):
|
|||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: Optional[Any] = None,
|
||||
user_api_key_dict: Optional[Any] = None,
|
||||
request_data: Optional[dict] = None,
|
||||
) -> Any:
|
||||
"""
|
||||
Process output - not applicable for text-to-speech.
|
||||
|
|
|
|||
|
|
@ -58,6 +58,7 @@ class OpenAIAudioTranscriptionHandler(BaseTranslation):
|
|||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: Optional[Any] = None,
|
||||
user_api_key_dict: Optional[Any] = None,
|
||||
request_data: Optional[dict] = None,
|
||||
) -> Any:
|
||||
"""
|
||||
Process output transcription by applying guardrails to transcribed text.
|
||||
|
|
@ -79,15 +80,21 @@ class OpenAIAudioTranscriptionHandler(BaseTranslation):
|
|||
|
||||
if isinstance(response.text, str):
|
||||
original_text = response.text
|
||||
# Create a request_data dict with response info and user API key metadata
|
||||
request_data: dict = {"response": response}
|
||||
# Use the real request_data if provided (proxy path), otherwise
|
||||
# create a standalone dict (SDK / direct-call path).
|
||||
if request_data is None:
|
||||
request_data = {"response": response}
|
||||
else:
|
||||
if "response" not in request_data:
|
||||
request_data["response"] = response
|
||||
|
||||
# Add user API key metadata with prefixed keys
|
||||
user_metadata = self.transform_user_api_key_dict_to_metadata(
|
||||
user_api_key_dict
|
||||
)
|
||||
if user_metadata:
|
||||
request_data["litellm_metadata"] = user_metadata
|
||||
if "litellm_metadata" not in request_data:
|
||||
user_metadata = self.transform_user_api_key_dict_to_metadata(
|
||||
user_api_key_dict
|
||||
)
|
||||
if user_metadata:
|
||||
request_data["litellm_metadata"] = user_metadata
|
||||
|
||||
inputs = GenericGuardrailAPIInputs(texts=[original_text])
|
||||
# Include model information from the response if available
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ More information on our website: https://endpoints.ai.cloud.ovh.net
|
|||
from typing import Optional, Union, List
|
||||
|
||||
import httpx
|
||||
from litellm.utils import ModelResponseStream, get_model_info
|
||||
from litellm.utils import ModelResponseStream, _get_model_info_helper
|
||||
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.llms.ovhcloud.utils import OVHCloudException
|
||||
|
|
@ -28,13 +28,15 @@ class OVHCloudChatConfig(OpenAIGPTConfig):
|
|||
"""
|
||||
supports_function_calling: Optional[bool] = None
|
||||
try:
|
||||
model_info = get_model_info(model, custom_llm_provider="ovhcloud")
|
||||
model_info = _get_model_info_helper(model, custom_llm_provider="ovhcloud")
|
||||
supports_function_calling = model_info.get(
|
||||
"supports_function_calling", False
|
||||
"supports_function_calling", None
|
||||
)
|
||||
if supports_function_calling is None:
|
||||
supports_function_calling = False
|
||||
except Exception as e:
|
||||
verbose_logger.debug(f"Error getting supported OpenAI params: {e}")
|
||||
pass
|
||||
supports_function_calling = False
|
||||
|
||||
optional_params = super().get_supported_openai_params(model)
|
||||
if supports_function_calling is not True:
|
||||
|
|
|
|||
|
|
@ -139,6 +139,7 @@ class PassThroughEndpointHandler(BaseTranslation):
|
|||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: Optional["LiteLLMLoggingObj"] = None,
|
||||
user_api_key_dict: Optional[Any] = None,
|
||||
request_data: Optional[dict] = None,
|
||||
) -> Any:
|
||||
"""
|
||||
Process output response by applying guardrails to targeted fields.
|
||||
|
|
@ -171,17 +172,27 @@ class PassThroughEndpointHandler(BaseTranslation):
|
|||
if not text_to_check:
|
||||
return response
|
||||
|
||||
# Create a request_data dict with response info and user API key metadata
|
||||
request_data: dict = (
|
||||
{"response": response}
|
||||
if not isinstance(response, dict)
|
||||
else response.copy()
|
||||
)
|
||||
# Use the real request_data if provided (proxy path), otherwise
|
||||
# create a standalone dict (SDK / direct-call path).
|
||||
if request_data is None:
|
||||
request_data = (
|
||||
{"response": response}
|
||||
if not isinstance(response, dict)
|
||||
else response.copy()
|
||||
)
|
||||
else:
|
||||
if "response" not in request_data:
|
||||
request_data["response"] = (
|
||||
response if not isinstance(response, dict) else response.copy()
|
||||
)
|
||||
|
||||
# Add user API key metadata with prefixed keys
|
||||
user_metadata = self.transform_user_api_key_dict_to_metadata(user_api_key_dict)
|
||||
if user_metadata:
|
||||
request_data["litellm_metadata"] = user_metadata
|
||||
if "litellm_metadata" not in request_data:
|
||||
user_metadata = self.transform_user_api_key_dict_to_metadata(
|
||||
user_api_key_dict
|
||||
)
|
||||
if user_metadata:
|
||||
request_data["litellm_metadata"] = user_metadata
|
||||
|
||||
# Apply guardrail (pass-through doesn't modify the text, just checks it)
|
||||
inputs = GenericGuardrailAPIInputs(texts=[text_to_check])
|
||||
|
|
|
|||
|
|
@ -376,3 +376,148 @@ class VertexAIBatchPrediction(VertexLLM):
|
|||
response=_json_response
|
||||
)
|
||||
return vertex_batch_response
|
||||
|
||||
def cancel_batch(
|
||||
self,
|
||||
_is_async: bool,
|
||||
batch_id: str,
|
||||
api_base: Optional[str],
|
||||
vertex_credentials: Optional[VERTEX_CREDENTIALS_TYPES],
|
||||
vertex_project: Optional[str],
|
||||
vertex_location: Optional[str],
|
||||
timeout: Union[float, httpx.Timeout],
|
||||
max_retries: Optional[int],
|
||||
) -> Union[LiteLLMBatch, Coroutine[Any, Any, LiteLLMBatch]]:
|
||||
access_token, project_id = self._ensure_access_token(
|
||||
credentials=vertex_credentials,
|
||||
project_id=vertex_project,
|
||||
custom_llm_provider="vertex_ai",
|
||||
)
|
||||
|
||||
default_api_base = self.create_vertex_batch_url(
|
||||
vertex_location=vertex_location or "us-central1",
|
||||
vertex_project=vertex_project or project_id,
|
||||
)
|
||||
|
||||
retrieve_api_base_default = f"{default_api_base}/{batch_id}"
|
||||
cancel_api_base_default = f"{retrieve_api_base_default}:cancel"
|
||||
|
||||
_, api_base = self._check_custom_proxy(
|
||||
api_base=api_base,
|
||||
custom_llm_provider="vertex_ai",
|
||||
gemini_api_key=None,
|
||||
endpoint="cancel",
|
||||
stream=None,
|
||||
auth_header=None,
|
||||
url=cancel_api_base_default,
|
||||
model=None,
|
||||
vertex_project=vertex_project or project_id,
|
||||
vertex_location=vertex_location or "us-central1",
|
||||
vertex_api_version="v1",
|
||||
)
|
||||
|
||||
if api_base.endswith(":cancel"):
|
||||
retrieve_api_base = api_base.removesuffix(":cancel")
|
||||
else:
|
||||
retrieve_api_base = api_base.rsplit(":cancel", 1)[0].rstrip("/")
|
||||
|
||||
headers = {
|
||||
"Content-Type": "application/json; charset=utf-8",
|
||||
"Authorization": f"Bearer {access_token}",
|
||||
}
|
||||
|
||||
if _is_async is True:
|
||||
return self._async_cancel_batch(
|
||||
api_base=api_base,
|
||||
retrieve_api_base=retrieve_api_base,
|
||||
headers=headers,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
sync_handler = _get_httpx_client()
|
||||
try:
|
||||
response = sync_handler.post(
|
||||
url=api_base,
|
||||
headers=headers,
|
||||
data=json.dumps({}),
|
||||
timeout=timeout,
|
||||
)
|
||||
except httpx.HTTPStatusError as e:
|
||||
litellm.verbose_logger.error(
|
||||
"Vertex AI batch cancel failed: status=%s, body=%s",
|
||||
e.response.status_code,
|
||||
e.response.text[:1000],
|
||||
)
|
||||
raise
|
||||
|
||||
if response.status_code != 200:
|
||||
raise Exception(f"Error: {response.status_code} {response.text}")
|
||||
|
||||
# HTTPHandler.get() does not accept a timeout parameter
|
||||
retrieve_response = sync_handler.get(
|
||||
url=retrieve_api_base,
|
||||
headers=headers,
|
||||
)
|
||||
if retrieve_response.status_code != 200:
|
||||
litellm.verbose_logger.error(
|
||||
"Vertex AI batch retrieve-after-cancel failed: status=%s, body=%s",
|
||||
retrieve_response.status_code,
|
||||
retrieve_response.text[:1000],
|
||||
)
|
||||
raise Exception(
|
||||
f"Error: {retrieve_response.status_code} {retrieve_response.text}"
|
||||
)
|
||||
|
||||
_json_response = retrieve_response.json()
|
||||
vertex_batch_response = VertexAIBatchTransformation.transform_vertex_ai_batch_response_to_openai_batch_response(
|
||||
response=_json_response
|
||||
)
|
||||
return vertex_batch_response
|
||||
|
||||
async def _async_cancel_batch(
|
||||
self,
|
||||
api_base: str,
|
||||
retrieve_api_base: str,
|
||||
headers: Dict[str, str],
|
||||
timeout: Union[float, httpx.Timeout] = 600.0,
|
||||
) -> LiteLLMBatch:
|
||||
client = get_async_httpx_client(
|
||||
llm_provider=litellm.LlmProviders.VERTEX_AI,
|
||||
)
|
||||
try:
|
||||
response = await client.post(
|
||||
url=api_base,
|
||||
headers=headers,
|
||||
data=json.dumps({}),
|
||||
timeout=timeout,
|
||||
)
|
||||
except httpx.HTTPStatusError as e:
|
||||
litellm.verbose_logger.error(
|
||||
"Vertex AI batch cancel failed: status=%s, body=%s",
|
||||
e.response.status_code,
|
||||
e.response.text[:1000],
|
||||
)
|
||||
raise
|
||||
if response.status_code != 200:
|
||||
raise Exception(f"Error: {response.status_code} {response.text}")
|
||||
|
||||
# AsyncHTTPHandler.get() does not accept a timeout parameter
|
||||
retrieve_response = await client.get(
|
||||
url=retrieve_api_base,
|
||||
headers=headers,
|
||||
)
|
||||
if retrieve_response.status_code != 200:
|
||||
litellm.verbose_logger.error(
|
||||
"Vertex AI batch retrieve-after-cancel failed: status=%s, body=%s",
|
||||
retrieve_response.status_code,
|
||||
retrieve_response.text[:1000],
|
||||
)
|
||||
raise Exception(
|
||||
f"Error: {retrieve_response.status_code} {retrieve_response.text}"
|
||||
)
|
||||
|
||||
_json_response = retrieve_response.json()
|
||||
vertex_batch_response = VertexAIBatchTransformation.transform_vertex_ai_batch_response_to_openai_batch_response(
|
||||
response=_json_response
|
||||
)
|
||||
return vertex_batch_response
|
||||
|
|
|
|||
|
|
@ -51,6 +51,7 @@ class ContextCachingEndpoints(VertexBase):
|
|||
vertex_project: Optional[str],
|
||||
vertex_location: Optional[str],
|
||||
vertex_auth_header: Optional[str],
|
||||
model: Optional[str] = None,
|
||||
) -> Tuple[Optional[str], str]:
|
||||
"""
|
||||
Internal function. Returns the token and url for the call.
|
||||
|
|
@ -89,7 +90,7 @@ class ContextCachingEndpoints(VertexBase):
|
|||
stream=None,
|
||||
auth_header=auth_header,
|
||||
url=url,
|
||||
model=None,
|
||||
model=model,
|
||||
vertex_project=vertex_project,
|
||||
vertex_location=vertex_location,
|
||||
vertex_api_version="v1beta1"
|
||||
|
|
@ -109,6 +110,7 @@ class ContextCachingEndpoints(VertexBase):
|
|||
vertex_project: Optional[str],
|
||||
vertex_location: Optional[str],
|
||||
vertex_auth_header: Optional[str],
|
||||
model: Optional[str] = None,
|
||||
) -> Optional[str]:
|
||||
"""
|
||||
Checks if content already cached.
|
||||
|
|
@ -128,6 +130,7 @@ class ContextCachingEndpoints(VertexBase):
|
|||
vertex_project=vertex_project,
|
||||
vertex_location=vertex_location,
|
||||
vertex_auth_header=vertex_auth_header,
|
||||
model=model,
|
||||
)
|
||||
|
||||
page_token: Optional[str] = None
|
||||
|
|
@ -201,6 +204,7 @@ class ContextCachingEndpoints(VertexBase):
|
|||
vertex_project: Optional[str],
|
||||
vertex_location: Optional[str],
|
||||
vertex_auth_header: Optional[str],
|
||||
model: Optional[str] = None,
|
||||
) -> Optional[str]:
|
||||
"""
|
||||
Checks if content already cached.
|
||||
|
|
@ -220,6 +224,7 @@ class ContextCachingEndpoints(VertexBase):
|
|||
vertex_project=vertex_project,
|
||||
vertex_location=vertex_location,
|
||||
vertex_auth_header=vertex_auth_header,
|
||||
model=model,
|
||||
)
|
||||
|
||||
page_token: Optional[str] = None
|
||||
|
|
@ -342,6 +347,7 @@ class ContextCachingEndpoints(VertexBase):
|
|||
vertex_project=vertex_project,
|
||||
vertex_location=vertex_location,
|
||||
vertex_auth_header=vertex_auth_header,
|
||||
model=model,
|
||||
)
|
||||
|
||||
headers = {
|
||||
|
|
@ -377,6 +383,7 @@ class ContextCachingEndpoints(VertexBase):
|
|||
vertex_project=vertex_project,
|
||||
vertex_location=vertex_location,
|
||||
vertex_auth_header=vertex_auth_header,
|
||||
model=model,
|
||||
)
|
||||
if google_cache_name:
|
||||
return non_cached_messages, optional_params, google_cache_name
|
||||
|
|
@ -488,6 +495,7 @@ class ContextCachingEndpoints(VertexBase):
|
|||
vertex_project=vertex_project,
|
||||
vertex_location=vertex_location,
|
||||
vertex_auth_header=vertex_auth_header,
|
||||
model=model,
|
||||
)
|
||||
|
||||
headers = {
|
||||
|
|
@ -520,6 +528,7 @@ class ContextCachingEndpoints(VertexBase):
|
|||
vertex_project=vertex_project,
|
||||
vertex_location=vertex_location,
|
||||
vertex_auth_header=vertex_auth_header,
|
||||
model=model,
|
||||
)
|
||||
|
||||
if google_cache_name:
|
||||
|
|
|
|||
|
|
@ -540,6 +540,41 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915
|
|||
assistant_content.append(gemini_tool_call_part)
|
||||
last_message_with_tool_calls = assistant_msg
|
||||
|
||||
## HANDLE SERVER-SIDE TOOL INVOCATIONS (context circulation)
|
||||
_psf = assistant_msg.get("provider_specific_fields")
|
||||
if isinstance(_psf, dict):
|
||||
_ss_invocations = _psf.get("server_side_tool_invocations")
|
||||
if isinstance(_ss_invocations, list):
|
||||
for invocation in _ss_invocations:
|
||||
# Re-inject toolCall part
|
||||
tc_part: Dict[str, Any] = {
|
||||
"toolCall": {
|
||||
"toolType": invocation.get("tool_type"),
|
||||
"id": invocation.get("id"),
|
||||
"args": invocation.get("args"),
|
||||
}
|
||||
}
|
||||
if "thought_signature" in invocation:
|
||||
tc_part["thoughtSignature"] = invocation[
|
||||
"thought_signature"
|
||||
]
|
||||
assistant_content.append(tc_part) # type: ignore
|
||||
|
||||
# Re-inject toolResponse part if response is present
|
||||
if "response" in invocation:
|
||||
tr_dict: Dict[str, Any] = {
|
||||
"id": invocation.get("id"),
|
||||
"response": invocation.get("response"),
|
||||
}
|
||||
if invocation.get("tool_type"):
|
||||
tr_dict["toolType"] = invocation["tool_type"]
|
||||
tr_part: Dict[str, Any] = {"toolResponse": tr_dict}
|
||||
if "thought_signature" in invocation:
|
||||
tr_part["thoughtSignature"] = invocation[
|
||||
"thought_signature"
|
||||
]
|
||||
assistant_content.append(tr_part) # type: ignore
|
||||
|
||||
msg_i += 1
|
||||
|
||||
if assistant_content:
|
||||
|
|
@ -666,6 +701,9 @@ def _transform_request_body( # noqa: PLR0915
|
|||
)
|
||||
tools: Optional[Tools] = optional_params.pop("tools", None)
|
||||
tool_choice: Optional[ToolConfig] = optional_params.pop("tool_choice", None)
|
||||
include_server_side_tool_invocations: bool = optional_params.pop(
|
||||
"include_server_side_tool_invocations", False
|
||||
)
|
||||
safety_settings: Optional[List[SafetSettingsConfig]] = optional_params.pop(
|
||||
"safety_settings", None
|
||||
) # type: ignore
|
||||
|
|
@ -715,6 +753,10 @@ def _transform_request_body( # noqa: PLR0915
|
|||
data["tools"] = tools
|
||||
if tool_choice is not None:
|
||||
data["toolConfig"] = tool_choice
|
||||
if include_server_side_tool_invocations:
|
||||
if "toolConfig" not in data:
|
||||
data["toolConfig"] = {}
|
||||
data["toolConfig"]["includeServerSideToolInvocations"] = True
|
||||
if safety_settings is not None:
|
||||
data["safetySettings"] = safety_settings
|
||||
if generation_config is not None and len(generation_config) > 0:
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from typing import (
|
|||
Dict,
|
||||
List,
|
||||
Literal,
|
||||
Mapping,
|
||||
Optional,
|
||||
Tuple,
|
||||
Type,
|
||||
|
|
@ -316,6 +317,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
"audio",
|
||||
"parallel_tool_calls",
|
||||
"web_search_options",
|
||||
"include_server_side_tool_invocations",
|
||||
]
|
||||
|
||||
# Add penalty parameters only for non-preview models
|
||||
|
|
@ -1119,6 +1121,8 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
optional_params = self._add_tools_to_optional_params(
|
||||
optional_params, [_tools]
|
||||
)
|
||||
elif param == "include_server_side_tool_invocations" and value is True:
|
||||
optional_params["include_server_side_tool_invocations"] = True
|
||||
if litellm.vertex_ai_safety_settings is not None:
|
||||
optional_params["safety_settings"] = litellm.vertex_ai_safety_settings
|
||||
|
||||
|
|
@ -1360,6 +1364,67 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
signatures.append(signature)
|
||||
return signatures if signatures else None
|
||||
|
||||
@staticmethod
|
||||
def _extract_server_side_tool_invocations(
|
||||
parts: List[HttpxPartType],
|
||||
) -> Optional[List[Dict[str, Any]]]:
|
||||
"""Extract server-side tool invocations (toolCall/toolResponse) from parts.
|
||||
|
||||
These are returned by Gemini when context circulation is enabled
|
||||
(includeServerSideToolInvocations=true). They represent tools executed
|
||||
server-side (e.g. Google Search) and must be circulated back in
|
||||
subsequent turns for multi-turn coherence.
|
||||
|
||||
Returns:
|
||||
List of server-side invocation dicts if any found, None otherwise.
|
||||
"""
|
||||
invocations: List[Dict[str, Any]] = []
|
||||
# Index toolCalls by id so we can pair them with responses
|
||||
tool_calls_by_id: Dict[str, Dict[str, Any]] = {}
|
||||
tool_responses_by_id: Dict[str, Dict[str, Any]] = {}
|
||||
|
||||
for part in parts:
|
||||
if "toolCall" in part:
|
||||
tc = part["toolCall"]
|
||||
entry: Dict[str, Any] = {
|
||||
"tool_type": tc.get("toolType"),
|
||||
"id": tc.get("id"),
|
||||
"args": tc.get("args"),
|
||||
}
|
||||
signature = part.get("thoughtSignature")
|
||||
if signature is not None:
|
||||
entry["thought_signature"] = signature
|
||||
tool_calls_by_id[tc.get("id", "")] = entry
|
||||
|
||||
elif "toolResponse" in part:
|
||||
tr = part["toolResponse"]
|
||||
entry = {
|
||||
"id": tr.get("id"),
|
||||
"tool_type": tr.get("toolType"),
|
||||
"response": tr.get("response"),
|
||||
}
|
||||
signature = part.get("thoughtSignature")
|
||||
if signature is not None:
|
||||
entry["thought_signature"] = signature
|
||||
tool_responses_by_id[tr.get("id", "")] = entry
|
||||
|
||||
# Merge calls with their responses
|
||||
for call_id, call_entry in tool_calls_by_id.items():
|
||||
merged = dict(call_entry)
|
||||
resp = tool_responses_by_id.pop(call_id, None)
|
||||
if resp is not None:
|
||||
merged["response"] = resp.get("response")
|
||||
# Keep response signature if call didn't have one
|
||||
if "thought_signature" not in merged and "thought_signature" in resp:
|
||||
merged["thought_signature"] = resp["thought_signature"]
|
||||
invocations.append(merged)
|
||||
|
||||
# Any orphan responses (shouldn't happen, but be safe)
|
||||
for resp_id, resp_entry in tool_responses_by_id.items():
|
||||
invocations.append(resp_entry)
|
||||
|
||||
return invocations if invocations else None
|
||||
|
||||
def _extract_image_response_from_parts(
|
||||
self, parts: List[HttpxPartType]
|
||||
) -> Optional[List[ImageURLListItem]]:
|
||||
|
|
@ -1632,6 +1697,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
response_tokens: Optional[int] = None
|
||||
response_tokens_details: Optional[CompletionTokensDetailsWrapper] = None
|
||||
usage_metadata = completion_response["usageMetadata"]
|
||||
|
||||
def _get_token_count(detail: Mapping[str, Any]) -> int:
|
||||
raw_token_count = detail.get("tokenCount", detail.get("token_count", 0))
|
||||
return raw_token_count if isinstance(raw_token_count, int) else 0
|
||||
|
||||
if "cachedContentTokenCount" in usage_metadata:
|
||||
cached_tokens = usage_metadata["cachedContentTokenCount"]
|
||||
|
||||
|
|
@ -1641,10 +1711,16 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
if "responseTokensDetails" in usage_metadata:
|
||||
response_tokens_details = CompletionTokensDetailsWrapper()
|
||||
for detail in usage_metadata["responseTokensDetails"]:
|
||||
if detail["modality"] == "TEXT":
|
||||
response_tokens_details.text_tokens = detail.get("tokenCount", 0)
|
||||
elif detail["modality"] == "AUDIO":
|
||||
response_tokens_details.audio_tokens = detail.get("tokenCount", 0)
|
||||
modality = str(detail.get("modality", "")).upper()
|
||||
token_count = _get_token_count(detail)
|
||||
if modality == "TEXT":
|
||||
response_tokens_details.text_tokens = (
|
||||
response_tokens_details.text_tokens or 0
|
||||
) + token_count
|
||||
elif modality == "AUDIO":
|
||||
response_tokens_details.audio_tokens = (
|
||||
response_tokens_details.audio_tokens or 0
|
||||
) + token_count
|
||||
|
||||
#########################################################
|
||||
|
||||
|
|
@ -1653,16 +1729,24 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
if response_tokens_details is None:
|
||||
response_tokens_details = CompletionTokensDetailsWrapper()
|
||||
for detail in usage_metadata["candidatesTokensDetails"]:
|
||||
modality = detail.get("modality")
|
||||
token_count = detail.get("tokenCount", 0)
|
||||
modality = str(detail.get("modality", "")).upper()
|
||||
token_count = _get_token_count(detail)
|
||||
if modality == "TEXT":
|
||||
response_tokens_details.text_tokens = token_count
|
||||
response_tokens_details.text_tokens = (
|
||||
response_tokens_details.text_tokens or 0
|
||||
) + token_count
|
||||
elif modality == "AUDIO":
|
||||
response_tokens_details.audio_tokens = token_count
|
||||
response_tokens_details.audio_tokens = (
|
||||
response_tokens_details.audio_tokens or 0
|
||||
) + token_count
|
||||
elif modality == "IMAGE":
|
||||
response_tokens_details.image_tokens = token_count
|
||||
response_tokens_details.image_tokens = (
|
||||
response_tokens_details.image_tokens or 0
|
||||
) + token_count
|
||||
elif modality == "VIDEO":
|
||||
response_tokens_details.video_tokens = token_count
|
||||
response_tokens_details.video_tokens = (
|
||||
response_tokens_details.video_tokens or 0
|
||||
) + token_count
|
||||
|
||||
# Calculate text_tokens if not explicitly provided in candidatesTokensDetails
|
||||
# candidatesTokenCount includes all modalities, so: text = total - (image + audio + video)
|
||||
|
|
@ -1686,14 +1770,16 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
## Parse promptTokensDetails (total tokens by modality, includes cached + non-cached)
|
||||
if "promptTokensDetails" in usage_metadata:
|
||||
for detail in usage_metadata["promptTokensDetails"]:
|
||||
if detail["modality"] == "AUDIO":
|
||||
prompt_audio_tokens = detail.get("tokenCount", 0)
|
||||
elif detail["modality"] == "TEXT":
|
||||
prompt_text_tokens = detail.get("tokenCount", 0)
|
||||
elif detail["modality"] == "IMAGE":
|
||||
prompt_image_tokens = detail.get("tokenCount", 0)
|
||||
elif detail["modality"] == "VIDEO":
|
||||
prompt_video_tokens = detail.get("tokenCount", 0)
|
||||
modality = str(detail.get("modality", "")).upper()
|
||||
token_count = _get_token_count(detail)
|
||||
if modality == "AUDIO":
|
||||
prompt_audio_tokens = (prompt_audio_tokens or 0) + token_count
|
||||
elif modality == "TEXT":
|
||||
prompt_text_tokens = (prompt_text_tokens or 0) + token_count
|
||||
elif modality == "IMAGE":
|
||||
prompt_image_tokens = (prompt_image_tokens or 0) + token_count
|
||||
elif modality == "VIDEO":
|
||||
prompt_video_tokens = (prompt_video_tokens or 0) + token_count
|
||||
|
||||
## Parse cacheTokensDetails (breakdown of cached tokens by modality)
|
||||
## When explicit caching is used, Gemini provides this field to show which modalities were cached
|
||||
|
|
@ -1704,14 +1790,16 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
|
||||
if "cacheTokensDetails" in usage_metadata:
|
||||
for detail in usage_metadata["cacheTokensDetails"]:
|
||||
if detail["modality"] == "AUDIO":
|
||||
cached_audio_tokens = detail.get("tokenCount", 0)
|
||||
elif detail["modality"] == "TEXT":
|
||||
cached_text_tokens = detail.get("tokenCount", 0)
|
||||
elif detail["modality"] == "IMAGE":
|
||||
cached_image_tokens = detail.get("tokenCount", 0)
|
||||
elif detail["modality"] == "VIDEO":
|
||||
cached_video_tokens = detail.get("tokenCount", 0)
|
||||
modality = str(detail.get("modality", "")).upper()
|
||||
token_count = _get_token_count(detail)
|
||||
if modality == "AUDIO":
|
||||
cached_audio_tokens = (cached_audio_tokens or 0) + token_count
|
||||
elif modality == "TEXT":
|
||||
cached_text_tokens = (cached_text_tokens or 0) + token_count
|
||||
elif modality == "IMAGE":
|
||||
cached_image_tokens = (cached_image_tokens or 0) + token_count
|
||||
elif modality == "VIDEO":
|
||||
cached_video_tokens = (cached_video_tokens or 0) + token_count
|
||||
|
||||
## Calculate non-cached tokens by subtracting cached from total (per modality)
|
||||
## This is necessary because promptTokensDetails includes both cached and non-cached tokens
|
||||
|
|
@ -2018,6 +2106,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
thinking_blocks: Optional[List[ChatCompletionThinkingBlock]] = None
|
||||
reasoning_content: Optional[str] = None
|
||||
thought_signatures: Optional[Any] = None
|
||||
server_side_tool_invocations: Optional[List[Dict[str, Any]]] = None
|
||||
|
||||
for idx, candidate in enumerate(_candidates):
|
||||
if "content" not in candidate:
|
||||
|
|
@ -2068,6 +2157,13 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
)
|
||||
)
|
||||
|
||||
# Extract server-side tool invocations (context circulation)
|
||||
server_side_tool_invocations = (
|
||||
VertexGeminiConfig._extract_server_side_tool_invocations(
|
||||
parts=candidate["content"]["parts"]
|
||||
)
|
||||
)
|
||||
|
||||
if audio_response is not None:
|
||||
cast(Dict[str, Any], chat_completion_message)[
|
||||
"audio"
|
||||
|
|
@ -2139,6 +2235,12 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
chat_completion_message["provider_specific_fields"] = {}
|
||||
chat_completion_message["provider_specific_fields"]["thought_signatures"] = thought_signatures # type: ignore
|
||||
|
||||
# Store server-side tool invocations in provider_specific_fields
|
||||
if server_side_tool_invocations is not None:
|
||||
if "provider_specific_fields" not in chat_completion_message:
|
||||
chat_completion_message["provider_specific_fields"] = {}
|
||||
chat_completion_message["provider_specific_fields"]["server_side_tool_invocations"] = server_side_tool_invocations # type: ignore
|
||||
|
||||
if isinstance(model_response, ModelResponseStream):
|
||||
choice = VertexGeminiConfig._create_streaming_choice(
|
||||
chat_completion_message=chat_completion_message,
|
||||
|
|
|
|||
|
|
@ -152,6 +152,8 @@ def transform_openai_input_gemini_content(
|
|||
gemini_params = optional_params.copy()
|
||||
if "dimensions" in gemini_params:
|
||||
gemini_params["outputDimensionality"] = gemini_params.pop("dimensions")
|
||||
if "task_type" in gemini_params:
|
||||
gemini_params["taskType"] = gemini_params.pop("task_type")
|
||||
|
||||
requests: List[EmbedContentRequest] = []
|
||||
if isinstance(input, str):
|
||||
|
|
@ -196,6 +198,8 @@ def transform_openai_input_gemini_embed_content(
|
|||
gemini_params = optional_params.copy()
|
||||
if "dimensions" in gemini_params:
|
||||
gemini_params["outputDimensionality"] = gemini_params.pop("dimensions")
|
||||
if "task_type" in gemini_params:
|
||||
gemini_params["taskType"] = gemini_params.pop("task_type")
|
||||
|
||||
input_list = [input] if isinstance(input, str) else input
|
||||
parts: List[PartType] = []
|
||||
|
|
|
|||
|
|
@ -105,12 +105,27 @@ class VertexAIPartnerModelsTokenCounter(VertexBase):
|
|||
# Extract Vertex AI credentials and settings
|
||||
vertex_credentials = self.get_vertex_ai_credentials(litellm_params)
|
||||
vertex_project = self.get_vertex_ai_project(litellm_params)
|
||||
vertex_location = self.get_vertex_ai_location(litellm_params)
|
||||
|
||||
# Map empty location/cluade models to a supported region for count-tokens endpoint
|
||||
# Check for count_tokens specific location override
|
||||
vertex_count_tokens_location = litellm_params.get(
|
||||
"vertex_count_tokens_location"
|
||||
)
|
||||
vertex_location_raw = self.get_vertex_ai_location(litellm_params)
|
||||
|
||||
# Determine final location with precedence:
|
||||
# 1. vertex_count_tokens_location (if provided)
|
||||
# 2. vertex_location (if provided)
|
||||
# 3. Default to us-east5 for Claude models when no location is set
|
||||
# Supported regions: us-east5, europe-west1, asia-southeast1
|
||||
# https://docs.cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude/count-tokens
|
||||
if not vertex_location or "claude" in model.lower():
|
||||
vertex_location = "us-central1"
|
||||
if vertex_count_tokens_location:
|
||||
vertex_location: str = vertex_count_tokens_location
|
||||
elif vertex_location_raw:
|
||||
vertex_location = vertex_location_raw
|
||||
elif "claude" in model.lower():
|
||||
vertex_location = "us-east5"
|
||||
else:
|
||||
vertex_location = "us-east5"
|
||||
|
||||
# Get access token and resolved project ID
|
||||
access_token, project_id = await self._ensure_access_token_async(
|
||||
|
|
|
|||
|
|
@ -955,16 +955,6 @@ def responses_api_bridge_check(
|
|||
model_info["mode"] = "responses"
|
||||
model = model.replace("responses/", "")
|
||||
|
||||
# OpenAI gpt-5.4+ chat-completions calls with both tools + reasoning_effort
|
||||
# must be bridged to Responses API.
|
||||
if (
|
||||
custom_llm_provider == "openai"
|
||||
and OpenAIGPT5Config.is_model_gpt_5_4_plus_model(model)
|
||||
and tools
|
||||
and reasoning_effort is not None
|
||||
):
|
||||
model_info["mode"] = "responses"
|
||||
model = model.replace("responses/", "")
|
||||
except Exception as e:
|
||||
verbose_logger.debug("Error getting model info: {}".format(e))
|
||||
|
||||
|
|
@ -974,6 +964,19 @@ def responses_api_bridge_check(
|
|||
model = model.replace("responses/", "")
|
||||
mode = "responses"
|
||||
model_info["mode"] = mode
|
||||
|
||||
# OpenAI/Azure gpt-5.4+ chat-completions calls with both tools + reasoning_effort
|
||||
# must be bridged to Responses API.
|
||||
if (
|
||||
custom_llm_provider in ("openai", "azure")
|
||||
and OpenAIGPT5Config.is_model_gpt_5_4_plus_model(model)
|
||||
and tools
|
||||
and reasoning_effort is not None
|
||||
and model_info.get("mode") != "responses"
|
||||
):
|
||||
model_info["mode"] = "responses"
|
||||
model = model.replace("responses/", "")
|
||||
|
||||
return model_info, model
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -3435,7 +3435,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_service_tier": true,
|
||||
"supports_vision": true,
|
||||
"supports_none_reasoning_effort": true
|
||||
"supports_none_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"azure/gpt-5.1-chat-2025-11-13": {
|
||||
"cache_read_input_token_cost": 1.25e-07,
|
||||
|
|
@ -6152,7 +6153,8 @@
|
|||
"max_query_tokens": 4096,
|
||||
"max_tokens": 32768,
|
||||
"mode": "rerank",
|
||||
"output_cost_per_token": 0.0
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-cohere-rerank-4-0-in-microsoft-foundry/4477076"
|
||||
},
|
||||
"azure_ai/cohere-rerank-v4.0-fast": {
|
||||
"input_cost_per_query": 0.002,
|
||||
|
|
@ -6163,7 +6165,8 @@
|
|||
"max_query_tokens": 4096,
|
||||
"max_tokens": 32768,
|
||||
"mode": "rerank",
|
||||
"output_cost_per_token": 0.0
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-cohere-rerank-4-0-in-microsoft-foundry/4477076"
|
||||
},
|
||||
"azure_ai/deepseek-v3.2": {
|
||||
"input_cost_per_token": 5.8e-07,
|
||||
|
|
@ -6173,6 +6176,7 @@
|
|||
"max_tokens": 163840,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.68e-06,
|
||||
"source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-deepseek-v3-2-and-deepseek-v3-2-speciale-in-microsoft-foundry/4477549",
|
||||
"supports_assistant_prefill": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
|
|
@ -6187,6 +6191,7 @@
|
|||
"max_tokens": 163840,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.68e-06,
|
||||
"source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-deepseek-v3-2-and-deepseek-v3-2-speciale-in-microsoft-foundry/4477549",
|
||||
"supports_assistant_prefill": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
|
|
@ -16936,6 +16941,18 @@
|
|||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"gpt-4-0314": {
|
||||
"deprecation_date": "2026-03-26",
|
||||
"input_cost_per_token": 3e-05,
|
||||
"litellm_provider": "openai",
|
||||
"max_input_tokens": 8192,
|
||||
"max_output_tokens": 4096,
|
||||
"max_tokens": 4096,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 6e-05,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"gpt-4-0613": {
|
||||
"deprecation_date": "2025-06-06",
|
||||
"input_cost_per_token": 3e-05,
|
||||
|
|
@ -18289,7 +18306,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": false
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5.1": {
|
||||
"cache_read_input_token_cost": 1.25e-07,
|
||||
|
|
@ -18328,7 +18346,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": true,
|
||||
"supports_xhigh_reasoning_effort": false
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5.1-2025-11-13": {
|
||||
"cache_read_input_token_cost": 1.25e-07,
|
||||
|
|
@ -18367,7 +18386,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": true,
|
||||
"supports_xhigh_reasoning_effort": false
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5.1-chat-latest": {
|
||||
"cache_read_input_token_cost": 1.25e-07,
|
||||
|
|
@ -18405,7 +18425,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": true,
|
||||
"supports_xhigh_reasoning_effort": false
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5.2": {
|
||||
"cache_read_input_token_cost": 1.75e-07,
|
||||
|
|
@ -18445,7 +18466,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": true,
|
||||
"supports_xhigh_reasoning_effort": true
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5.2-2025-12-11": {
|
||||
"cache_read_input_token_cost": 1.75e-07,
|
||||
|
|
@ -18485,7 +18507,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": true,
|
||||
"supports_xhigh_reasoning_effort": true
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5.2-chat-latest": {
|
||||
"cache_read_input_token_cost": 1.75e-07,
|
||||
|
|
@ -18522,7 +18545,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": false
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5.3-chat-latest": {
|
||||
"cache_read_input_token_cost": 1.75e-07,
|
||||
|
|
@ -18559,7 +18583,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": false
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5.2-pro": {
|
||||
"input_cost_per_token": 2.1e-05,
|
||||
|
|
@ -18592,7 +18617,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": true
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5.2-pro-2025-12-11": {
|
||||
"input_cost_per_token": 2.1e-05,
|
||||
|
|
@ -18625,7 +18651,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": true
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5.4": {
|
||||
"cache_read_input_token_cost": 2.5e-07,
|
||||
|
|
@ -18674,7 +18701,8 @@
|
|||
"supports_service_tier": true,
|
||||
"supports_vision": true,
|
||||
"supports_none_reasoning_effort": true,
|
||||
"supports_xhigh_reasoning_effort": true
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5.4-2026-03-05": {
|
||||
"cache_read_input_token_cost": 2.5e-07,
|
||||
|
|
@ -18769,7 +18797,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": true
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5.4-pro-2026-03-05": {
|
||||
"cache_read_input_token_cost": 3e-06,
|
||||
|
|
@ -18817,7 +18846,94 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": true
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5.4-mini": {
|
||||
"cache_read_input_token_cost": 7.5e-08,
|
||||
"cache_read_input_token_cost_flex": 1e-08,
|
||||
"cache_read_input_token_cost_batches": 3.8e-08,
|
||||
"input_cost_per_token": 7.5e-07,
|
||||
"input_cost_per_token_flex": 3.75e-07,
|
||||
"input_cost_per_token_batches": 3.75e-07,
|
||||
"litellm_provider": "openai",
|
||||
"max_input_tokens": 272000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4.5e-06,
|
||||
"output_cost_per_token_flex": 2.25e-06,
|
||||
"output_cost_per_token_batches": 2.25e-06,
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
"/v1/batch",
|
||||
"/v1/responses"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_native_streaming": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_service_tier": true,
|
||||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": false
|
||||
},
|
||||
"gpt-5.4-nano": {
|
||||
"cache_read_input_token_cost": 2e-08,
|
||||
"cache_read_input_token_cost_flex": 1e-08,
|
||||
"cache_read_input_token_cost_batches": 1e-08,
|
||||
"input_cost_per_token": 2e-07,
|
||||
"input_cost_per_token_flex": 1e-07,
|
||||
"input_cost_per_token_batches": 1e-07,
|
||||
"litellm_provider": "openai",
|
||||
"max_input_tokens": 272000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.25e-06,
|
||||
"output_cost_per_token_flex": 6.25e-07,
|
||||
"output_cost_per_token_batches": 6.25e-07,
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
"/v1/batch",
|
||||
"/v1/responses"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_native_streaming": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_service_tier": true,
|
||||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": false
|
||||
},
|
||||
"gpt-5-pro": {
|
||||
"input_cost_per_token": 1.5e-05,
|
||||
|
|
@ -18852,7 +18968,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": false
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5-pro-2025-10-06": {
|
||||
"input_cost_per_token": 1.5e-05,
|
||||
|
|
@ -18887,7 +19004,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": false
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5-2025-08-07": {
|
||||
"cache_read_input_token_cost": 1.25e-07,
|
||||
|
|
@ -18929,7 +19047,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": false
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5-chat": {
|
||||
"cache_read_input_token_cost": 1.25e-07,
|
||||
|
|
@ -18963,7 +19082,8 @@
|
|||
"supports_tool_choice": false,
|
||||
"supports_vision": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": false
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5-chat-latest": {
|
||||
"cache_read_input_token_cost": 1.25e-07,
|
||||
|
|
@ -18997,7 +19117,8 @@
|
|||
"supports_tool_choice": false,
|
||||
"supports_vision": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": false
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5-codex": {
|
||||
"cache_read_input_token_cost": 1.25e-07,
|
||||
|
|
@ -19030,7 +19151,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": false
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5.1-codex": {
|
||||
"cache_read_input_token_cost": 1.25e-07,
|
||||
|
|
@ -19066,7 +19188,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": false
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5.1-codex-max": {
|
||||
"cache_read_input_token_cost": 1.25e-07,
|
||||
|
|
@ -19099,7 +19222,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": true
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5.1-codex-mini": {
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
|
|
@ -19135,7 +19259,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": false
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5.2-codex": {
|
||||
"cache_read_input_token_cost": 1.75e-07,
|
||||
|
|
@ -19171,7 +19296,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": true
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5.3-codex": {
|
||||
"cache_read_input_token_cost": 1.75e-07,
|
||||
|
|
@ -19207,7 +19333,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": false
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5-mini": {
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
|
|
@ -19249,7 +19376,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": false
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5-mini-2025-08-07": {
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
|
|
@ -19291,7 +19419,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": false
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5-nano": {
|
||||
"cache_read_input_token_cost": 5e-09,
|
||||
|
|
@ -19330,7 +19459,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": false
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5-nano-2025-08-07": {
|
||||
"cache_read_input_token_cost": 5e-09,
|
||||
|
|
@ -19368,7 +19498,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": false
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-image-1": {
|
||||
"cache_read_input_image_token_cost": 2.5e-06,
|
||||
|
|
@ -30506,7 +30637,7 @@
|
|||
"output_cost_per_token": 5.4e-06,
|
||||
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models",
|
||||
"supported_regions": [
|
||||
"us-west2"
|
||||
"us-central1"
|
||||
],
|
||||
"supports_assistant_prefill": true,
|
||||
"supports_function_calling": true,
|
||||
|
|
@ -30526,7 +30657,7 @@
|
|||
"output_cost_per_token_batches": 8.4e-07,
|
||||
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models",
|
||||
"supported_regions": [
|
||||
"us-west2"
|
||||
"global"
|
||||
],
|
||||
"supports_assistant_prefill": true,
|
||||
"supports_function_calling": true,
|
||||
|
|
@ -30543,6 +30674,9 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 5.4e-06,
|
||||
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models",
|
||||
"supported_regions": [
|
||||
"us-central1"
|
||||
],
|
||||
"supports_assistant_prefill": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
|
|
@ -31167,7 +31301,10 @@
|
|||
"input_cost_per_token": 3e-07,
|
||||
"output_cost_per_token": 1.2e-06,
|
||||
"ocr_cost_per_page": 0.0003,
|
||||
"source": "https://cloud.google.com/vertex-ai/pricing"
|
||||
"source": "https://cloud.google.com/vertex-ai/pricing",
|
||||
"supported_regions": [
|
||||
"us-central1"
|
||||
]
|
||||
},
|
||||
"vertex_ai/openai/gpt-oss-120b-maas": {
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
|
|
@ -36386,7 +36523,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": false
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5-search-api-2025-10-14": {
|
||||
"cache_read_input_token_cost": 1.25e-07,
|
||||
|
|
|
|||
|
|
@ -92,6 +92,7 @@ class MCPGuardrailTranslationHandler(BaseTranslation):
|
|||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: Optional[Any] = None,
|
||||
user_api_key_dict: Optional[Any] = None,
|
||||
request_data: Optional[dict] = None,
|
||||
) -> Any:
|
||||
verbose_proxy_logger.debug(
|
||||
"MCP Guardrail: Output processing not implemented for MCP tools",
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -1,27 +1,28 @@
|
|||
1:"$Sreact.fragment"
|
||||
2:I[347257,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"ClientPageRoot"]
|
||||
3:I[952683,["/litellm-asset-prefix/_next/static/chunks/a7f104aa2cc7f3f0.js","/litellm-asset-prefix/_next/static/chunks/e8ed72789c2b42ff.js","/litellm-asset-prefix/_next/static/chunks/e627c7aa5ead52b3.js","/litellm-asset-prefix/_next/static/chunks/142704439974f6b3.js","/litellm-asset-prefix/_next/static/chunks/30539b80ac15aad2.js","/litellm-asset-prefix/_next/static/chunks/7d82a1cebfdb679c.js","/litellm-asset-prefix/_next/static/chunks/2d471965761a22ff.js","/litellm-asset-prefix/_next/static/chunks/1fe0596a309ad6cf.js","/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","/litellm-asset-prefix/_next/static/chunks/d64d74932cb225a3.js","/litellm-asset-prefix/_next/static/chunks/8c13023d89b01566.js","/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","/litellm-asset-prefix/_next/static/chunks/74ce31aa0fb2adc9.js","/litellm-asset-prefix/_next/static/chunks/cdf98a03da656604.js","/litellm-asset-prefix/_next/static/chunks/cac89fc12fb6ef7e.js","/litellm-asset-prefix/_next/static/chunks/d0d828f9a0668699.js","/litellm-asset-prefix/_next/static/chunks/1a04d31843c96649.js","/litellm-asset-prefix/_next/static/chunks/ed90bf177ad61e18.js","/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","/litellm-asset-prefix/_next/static/chunks/6c4c97f1ea6e7d77.js","/litellm-asset-prefix/_next/static/chunks/9d6e5aad99b19216.js","/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","/litellm-asset-prefix/_next/static/chunks/acbeac1b0fde1fdf.js","/litellm-asset-prefix/_next/static/chunks/a89452659b6e1d90.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/348b31083769a7c4.js","/litellm-asset-prefix/_next/static/chunks/a85adee4198d5478.js","/litellm-asset-prefix/_next/static/chunks/c8eee6971ca36303.js","/litellm-asset-prefix/_next/static/chunks/67ddb5107368a659.js","/litellm-asset-prefix/_next/static/chunks/b6cdb9a433f054f3.js","/litellm-asset-prefix/_next/static/chunks/22970a12064ba16b.js","/litellm-asset-prefix/_next/static/chunks/4348e537165edb3b.js","/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","/litellm-asset-prefix/_next/static/chunks/d069df5baead6d90.js","/litellm-asset-prefix/_next/static/chunks/0dda11815be4f78b.js","/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","/litellm-asset-prefix/_next/static/chunks/fc4d54eb6afe7984.js","/litellm-asset-prefix/_next/static/chunks/40f766ecc87dbf9a.js","/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","/litellm-asset-prefix/_next/static/chunks/06ebe9b0e9cdf241.js","/litellm-asset-prefix/_next/static/chunks/df6546cd8a44d3b3.js","/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","/litellm-asset-prefix/_next/static/chunks/23bf955e8672ce98.js","/litellm-asset-prefix/_next/static/chunks/8dda507c226082ca.js","/litellm-asset-prefix/_next/static/chunks/54e29148cb2f2582.js","/litellm-asset-prefix/_next/static/chunks/3675074b1d85e268.js","/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","/litellm-asset-prefix/_next/static/chunks/bd94e2fe34d8a187.js"],"default"]
|
||||
17:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"OutletBoundary"]
|
||||
18:"$Sreact.suspense"
|
||||
3:I[952683,["/litellm-asset-prefix/_next/static/chunks/112ad77f3dd2e3cd.js","/litellm-asset-prefix/_next/static/chunks/65f709264734a9bf.js","/litellm-asset-prefix/_next/static/chunks/e627c7aa5ead52b3.js","/litellm-asset-prefix/_next/static/chunks/142704439974f6b3.js","/litellm-asset-prefix/_next/static/chunks/30539b80ac15aad2.js","/litellm-asset-prefix/_next/static/chunks/99d715502d5069f4.js","/litellm-asset-prefix/_next/static/chunks/53a707a5829899ed.js","/litellm-asset-prefix/_next/static/chunks/1fe0596a309ad6cf.js","/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","/litellm-asset-prefix/_next/static/chunks/ed90bf177ad61e18.js","/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","/litellm-asset-prefix/_next/static/chunks/b4bd164f5553a31d.js","/litellm-asset-prefix/_next/static/chunks/adb8beb738574863.js","/litellm-asset-prefix/_next/static/chunks/b6cdb9a433f054f3.js","/litellm-asset-prefix/_next/static/chunks/af8668386d7005fe.js","/litellm-asset-prefix/_next/static/chunks/9606513e20bc3d4f.js","/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","/litellm-asset-prefix/_next/static/chunks/d512ca3b7169bef6.js","/litellm-asset-prefix/_next/static/chunks/e1e3f652dbc5be03.js","/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","/litellm-asset-prefix/_next/static/chunks/f3e0cbc0e84e0a5d.js","/litellm-asset-prefix/_next/static/chunks/8c13023d89b01566.js","/litellm-asset-prefix/_next/static/chunks/338e84191fe615bf.js","/litellm-asset-prefix/_next/static/chunks/c53c9c7afec96700.js","/litellm-asset-prefix/_next/static/chunks/5f4170980a69ffa3.js","/litellm-asset-prefix/_next/static/chunks/c74f3813068add76.js","/litellm-asset-prefix/_next/static/chunks/99109c78121231a0.js","/litellm-asset-prefix/_next/static/chunks/5929da573d876909.js","/litellm-asset-prefix/_next/static/chunks/58461a445becf104.js","/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","/litellm-asset-prefix/_next/static/chunks/d0d828f9a0668699.js","/litellm-asset-prefix/_next/static/chunks/ce9cf9f407f4b359.js","/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","/litellm-asset-prefix/_next/static/chunks/6b2bc4046c4cbfc8.js","/litellm-asset-prefix/_next/static/chunks/5400ee883dfa8c43.js","/litellm-asset-prefix/_next/static/chunks/bd94e2fe34d8a187.js","/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","/litellm-asset-prefix/_next/static/chunks/7a2dc852f68481ea.js","/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","/litellm-asset-prefix/_next/static/chunks/f9c24d6e7ec43046.js","/litellm-asset-prefix/_next/static/chunks/7c797521435cb59c.js","/litellm-asset-prefix/_next/static/chunks/5855ff7033bd4d2e.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/1da362a651d209bd.js","/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","/litellm-asset-prefix/_next/static/chunks/0b8ec8bf90ea9721.js"],"default"]
|
||||
18:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"OutletBoundary"]
|
||||
19:"$Sreact.suspense"
|
||||
:HL["/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","style"]
|
||||
0:{"buildId":"aKKihXXKRJWLQThZgi8Rq","rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/e627c7aa5ead52b3.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/142704439974f6b3.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/30539b80ac15aad2.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/7d82a1cebfdb679c.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/2d471965761a22ff.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/1fe0596a309ad6cf.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/d64d74932cb225a3.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/8c13023d89b01566.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/74ce31aa0fb2adc9.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/cdf98a03da656604.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/cac89fc12fb6ef7e.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/d0d828f9a0668699.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/1a04d31843c96649.js","async":true}],["$","script","script-18",{"src":"/litellm-asset-prefix/_next/static/chunks/ed90bf177ad61e18.js","async":true}],["$","script","script-19",{"src":"/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","async":true}],["$","script","script-20",{"src":"/litellm-asset-prefix/_next/static/chunks/6c4c97f1ea6e7d77.js","async":true}],["$","script","script-21",{"src":"/litellm-asset-prefix/_next/static/chunks/9d6e5aad99b19216.js","async":true}],["$","script","script-22",{"src":"/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","async":true}],["$","script","script-23",{"src":"/litellm-asset-prefix/_next/static/chunks/acbeac1b0fde1fdf.js","async":true}],["$","script","script-24",{"src":"/litellm-asset-prefix/_next/static/chunks/a89452659b6e1d90.js","async":true}],["$","script","script-25",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true}],["$","script","script-26",{"src":"/litellm-asset-prefix/_next/static/chunks/348b31083769a7c4.js","async":true}],["$","script","script-27",{"src":"/litellm-asset-prefix/_next/static/chunks/a85adee4198d5478.js","async":true}],["$","script","script-28",{"src":"/litellm-asset-prefix/_next/static/chunks/c8eee6971ca36303.js","async":true}],["$","script","script-29",{"src":"/litellm-asset-prefix/_next/static/chunks/67ddb5107368a659.js","async":true}],["$","script","script-30",{"src":"/litellm-asset-prefix/_next/static/chunks/b6cdb9a433f054f3.js","async":true}],["$","script","script-31",{"src":"/litellm-asset-prefix/_next/static/chunks/22970a12064ba16b.js","async":true}],["$","script","script-32",{"src":"/litellm-asset-prefix/_next/static/chunks/4348e537165edb3b.js","async":true}],["$","script","script-33",{"src":"/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","async":true}],"$L6","$L7","$L8","$L9","$La","$Lb","$Lc","$Ld","$Le","$Lf","$L10","$L11","$L12","$L13","$L14","$L15"],"$L16"]}],"loading":null,"isPartial":false}
|
||||
0:{"buildId":"Hp-LQxDEAEt-JSJFExm-i","rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/e627c7aa5ead52b3.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/142704439974f6b3.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/30539b80ac15aad2.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/99d715502d5069f4.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/53a707a5829899ed.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/1fe0596a309ad6cf.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/ed90bf177ad61e18.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/b4bd164f5553a31d.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/adb8beb738574863.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/b6cdb9a433f054f3.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/af8668386d7005fe.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/9606513e20bc3d4f.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","async":true}],["$","script","script-18",{"src":"/litellm-asset-prefix/_next/static/chunks/d512ca3b7169bef6.js","async":true}],["$","script","script-19",{"src":"/litellm-asset-prefix/_next/static/chunks/e1e3f652dbc5be03.js","async":true}],["$","script","script-20",{"src":"/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","async":true}],["$","script","script-21",{"src":"/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","async":true}],["$","script","script-22",{"src":"/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","async":true}],["$","script","script-23",{"src":"/litellm-asset-prefix/_next/static/chunks/f3e0cbc0e84e0a5d.js","async":true}],["$","script","script-24",{"src":"/litellm-asset-prefix/_next/static/chunks/8c13023d89b01566.js","async":true}],["$","script","script-25",{"src":"/litellm-asset-prefix/_next/static/chunks/338e84191fe615bf.js","async":true}],["$","script","script-26",{"src":"/litellm-asset-prefix/_next/static/chunks/c53c9c7afec96700.js","async":true}],["$","script","script-27",{"src":"/litellm-asset-prefix/_next/static/chunks/5f4170980a69ffa3.js","async":true}],["$","script","script-28",{"src":"/litellm-asset-prefix/_next/static/chunks/c74f3813068add76.js","async":true}],["$","script","script-29",{"src":"/litellm-asset-prefix/_next/static/chunks/99109c78121231a0.js","async":true}],["$","script","script-30",{"src":"/litellm-asset-prefix/_next/static/chunks/5929da573d876909.js","async":true}],["$","script","script-31",{"src":"/litellm-asset-prefix/_next/static/chunks/58461a445becf104.js","async":true}],["$","script","script-32",{"src":"/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","async":true}],["$","script","script-33",{"src":"/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","async":true}],"$L6","$L7","$L8","$L9","$La","$Lb","$Lc","$Ld","$Le","$Lf","$L10","$L11","$L12","$L13","$L14","$L15","$L16"],"$L17"]}],"loading":null,"isPartial":false}
|
||||
4:{}
|
||||
5:"$0:rsc:props:children:0:props:serverProvidedParams:params"
|
||||
6:["$","script","script-34",{"src":"/litellm-asset-prefix/_next/static/chunks/d069df5baead6d90.js","async":true}]
|
||||
7:["$","script","script-35",{"src":"/litellm-asset-prefix/_next/static/chunks/0dda11815be4f78b.js","async":true}]
|
||||
6:["$","script","script-34",{"src":"/litellm-asset-prefix/_next/static/chunks/d0d828f9a0668699.js","async":true}]
|
||||
7:["$","script","script-35",{"src":"/litellm-asset-prefix/_next/static/chunks/ce9cf9f407f4b359.js","async":true}]
|
||||
8:["$","script","script-36",{"src":"/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","async":true}]
|
||||
9:["$","script","script-37",{"src":"/litellm-asset-prefix/_next/static/chunks/fc4d54eb6afe7984.js","async":true}]
|
||||
a:["$","script","script-38",{"src":"/litellm-asset-prefix/_next/static/chunks/40f766ecc87dbf9a.js","async":true}]
|
||||
b:["$","script","script-39",{"src":"/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","async":true}]
|
||||
c:["$","script","script-40",{"src":"/litellm-asset-prefix/_next/static/chunks/06ebe9b0e9cdf241.js","async":true}]
|
||||
d:["$","script","script-41",{"src":"/litellm-asset-prefix/_next/static/chunks/df6546cd8a44d3b3.js","async":true}]
|
||||
e:["$","script","script-42",{"src":"/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","async":true}]
|
||||
f:["$","script","script-43",{"src":"/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","async":true}]
|
||||
10:["$","script","script-44",{"src":"/litellm-asset-prefix/_next/static/chunks/23bf955e8672ce98.js","async":true}]
|
||||
11:["$","script","script-45",{"src":"/litellm-asset-prefix/_next/static/chunks/8dda507c226082ca.js","async":true}]
|
||||
12:["$","script","script-46",{"src":"/litellm-asset-prefix/_next/static/chunks/54e29148cb2f2582.js","async":true}]
|
||||
13:["$","script","script-47",{"src":"/litellm-asset-prefix/_next/static/chunks/3675074b1d85e268.js","async":true}]
|
||||
14:["$","script","script-48",{"src":"/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","async":true}]
|
||||
15:["$","script","script-49",{"src":"/litellm-asset-prefix/_next/static/chunks/bd94e2fe34d8a187.js","async":true}]
|
||||
16:["$","$L17",null,{"children":["$","$18",null,{"name":"Next.MetadataOutlet","children":"$@19"}]}]
|
||||
19:null
|
||||
9:["$","script","script-37",{"src":"/litellm-asset-prefix/_next/static/chunks/6b2bc4046c4cbfc8.js","async":true}]
|
||||
a:["$","script","script-38",{"src":"/litellm-asset-prefix/_next/static/chunks/5400ee883dfa8c43.js","async":true}]
|
||||
b:["$","script","script-39",{"src":"/litellm-asset-prefix/_next/static/chunks/bd94e2fe34d8a187.js","async":true}]
|
||||
c:["$","script","script-40",{"src":"/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","async":true}]
|
||||
d:["$","script","script-41",{"src":"/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","async":true}]
|
||||
e:["$","script","script-42",{"src":"/litellm-asset-prefix/_next/static/chunks/7a2dc852f68481ea.js","async":true}]
|
||||
f:["$","script","script-43",{"src":"/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","async":true}]
|
||||
10:["$","script","script-44",{"src":"/litellm-asset-prefix/_next/static/chunks/f9c24d6e7ec43046.js","async":true}]
|
||||
11:["$","script","script-45",{"src":"/litellm-asset-prefix/_next/static/chunks/7c797521435cb59c.js","async":true}]
|
||||
12:["$","script","script-46",{"src":"/litellm-asset-prefix/_next/static/chunks/5855ff7033bd4d2e.js","async":true}]
|
||||
13:["$","script","script-47",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true}]
|
||||
14:["$","script","script-48",{"src":"/litellm-asset-prefix/_next/static/chunks/1da362a651d209bd.js","async":true}]
|
||||
15:["$","script","script-49",{"src":"/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","async":true}]
|
||||
16:["$","script","script-50",{"src":"/litellm-asset-prefix/_next/static/chunks/0b8ec8bf90ea9721.js","async":true}]
|
||||
17:["$","$L18",null,{"children":["$","$19",null,{"name":"Next.MetadataOutlet","children":"$@1a"}]}]
|
||||
1a:null
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -3,4 +3,4 @@
|
|||
3:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"MetadataBoundary"]
|
||||
4:"$Sreact.suspense"
|
||||
5:I[27201,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"IconMark"]
|
||||
0:{"buildId":"aKKihXXKRJWLQThZgi8Rq","rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"./favicon.ico"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"loading":null,"isPartial":false}
|
||||
0:{"buildId":"Hp-LQxDEAEt-JSJFExm-i","rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"./favicon.ico"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"loading":null,"isPartial":false}
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
1:"$Sreact.fragment"
|
||||
2:I[867271,["/litellm-asset-prefix/_next/static/chunks/a7f104aa2cc7f3f0.js","/litellm-asset-prefix/_next/static/chunks/e8ed72789c2b42ff.js"],"default"]
|
||||
3:I[71195,["/litellm-asset-prefix/_next/static/chunks/a7f104aa2cc7f3f0.js","/litellm-asset-prefix/_next/static/chunks/e8ed72789c2b42ff.js"],"default"]
|
||||
2:I[867271,["/litellm-asset-prefix/_next/static/chunks/112ad77f3dd2e3cd.js","/litellm-asset-prefix/_next/static/chunks/65f709264734a9bf.js"],"default"]
|
||||
3:I[71195,["/litellm-asset-prefix/_next/static/chunks/112ad77f3dd2e3cd.js","/litellm-asset-prefix/_next/static/chunks/65f709264734a9bf.js"],"default"]
|
||||
4:I[339756,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
|
||||
5:I[837457,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
|
||||
:HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"]
|
||||
:HL["/litellm-asset-prefix/_next/static/chunks/8dc3b559a2e76f88.css","style"]
|
||||
0:{"buildId":"aKKihXXKRJWLQThZgi8Rq","rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/8dc3b559a2e76f88.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/a7f104aa2cc7f3f0.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/e8ed72789c2b42ff.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]]}],"loading":null,"isPartial":false}
|
||||
:HL["/litellm-asset-prefix/_next/static/chunks/cab8d46a8c32ec36.css","style"]
|
||||
0:{"buildId":"Hp-LQxDEAEt-JSJFExm-i","rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/cab8d46a8c32ec36.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/112ad77f3dd2e3cd.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/65f709264734a9bf.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]]}],"loading":null,"isPartial":false}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
:HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"]
|
||||
:HL["/litellm-asset-prefix/_next/static/chunks/8dc3b559a2e76f88.css","style"]
|
||||
:HL["/litellm-asset-prefix/_next/static/chunks/cab8d46a8c32ec36.css","style"]
|
||||
:HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.3a6ba036.woff2","font",{"crossOrigin":"","type":"font/woff2"}]
|
||||
:HL["/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","style"]
|
||||
0:{"buildId":"aKKihXXKRJWLQThZgi8Rq","tree":{"name":"","paramType":null,"paramKey":"","hasRuntimePrefetch":false,"slots":{"children":{"name":"__PAGE__","paramType":null,"paramKey":"__PAGE__","hasRuntimePrefetch":false,"slots":null,"isRootLayout":false}},"isRootLayout":true},"staleTime":300}
|
||||
0:{"buildId":"Hp-LQxDEAEt-JSJFExm-i","tree":{"name":"","paramType":null,"paramKey":"","hasRuntimePrefetch":false,"slots":{"children":{"name":"__PAGE__","paramType":null,"paramKey":"__PAGE__","hasRuntimePrefetch":false,"slots":null,"isRootLayout":false}},"isRootLayout":true},"staleTime":300}
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -1 +0,0 @@
|
|||
(globalThis.TURBOPACK||(globalThis.TURBOPACK=[])).push(["object"==typeof document?document.currentScript:void 0,233525,(e,c,u)=>{"use strict";Object.defineProperty(u,"__esModule",{value:!0}),Object.defineProperty(u,"warnOnce",{enumerable:!0,get:function(){return r}});let r=e=>{}},349356,e=>{e.v({AElig:"Æ",AMP:"&",Aacute:"Á",Acirc:"Â",Agrave:"À",Aring:"Å",Atilde:"Ã",Auml:"Ä",COPY:"©",Ccedil:"Ç",ETH:"Ð",Eacute:"É",Ecirc:"Ê",Egrave:"È",Euml:"Ë",GT:">",Iacute:"Í",Icirc:"Î",Igrave:"Ì",Iuml:"Ï",LT:"<",Ntilde:"Ñ",Oacute:"Ó",Ocirc:"Ô",Ograve:"Ò",Oslash:"Ø",Otilde:"Õ",Ouml:"Ö",QUOT:'"',REG:"®",THORN:"Þ",Uacute:"Ú",Ucirc:"Û",Ugrave:"Ù",Uuml:"Ü",Yacute:"Ý",aacute:"á",acirc:"â",acute:"´",aelig:"æ",agrave:"à",amp:"&",aring:"å",atilde:"ã",auml:"ä",brvbar:"¦",ccedil:"ç",cedil:"¸",cent:"¢",copy:"©",curren:"¤",deg:"°",divide:"÷",eacute:"é",ecirc:"ê",egrave:"è",eth:"ð",euml:"ë",frac12:"½",frac14:"¼",frac34:"¾",gt:">",iacute:"í",icirc:"î",iexcl:"¡",igrave:"ì",iquest:"¿",iuml:"ï",laquo:"«",lt:"<",macr:"¯",micro:"µ",middot:"·",nbsp:" ",not:"¬",ntilde:"ñ",oacute:"ó",ocirc:"ô",ograve:"ò",ordf:"ª",ordm:"º",oslash:"ø",otilde:"õ",ouml:"ö",para:"¶",plusmn:"±",pound:"£",quot:'"',raquo:"»",reg:"®",sect:"§",shy:"",sup1:"¹",sup2:"²",sup3:"³",szlig:"ß",thorn:"þ",times:"×",uacute:"ú",ucirc:"û",ugrave:"ù",uml:"¨",uuml:"ü",yacute:"ý",yen:"¥",yuml:"ÿ"})},137429,e=>{e.v({0:"<22>",128:"€",130:"‚",131:"ƒ",132:"„",133:"…",134:"†",135:"‡",136:"ˆ",137:"‰",138:"Š",139:"‹",140:"Œ",142:"Ž",145:"‘",146:"’",147:"“",148:"”",149:"•",150:"–",151:"—",152:"˜",153:"™",154:"š",155:"›",156:"œ",158:"ž",159:"Ÿ"})},928685,e=>{"use strict";var c=e.i(38953);e.s(["SearchOutlined",()=>c.default])},86408,e=>{"use strict";var c=e.i(843476),u=e.i(271645),r=e.i(618566),t=e.i(934879);function a(){let e=(0,r.useSearchParams)().get("key"),[a,i]=(0,u.useState)(null);return console.log("PublicModelHubTable accessToken:",a),(0,u.useEffect)(()=>{e&&i(e)},[e]),(0,c.jsx)(t.default,{accessToken:a,publicPage:!0,premiumUser:!1,userRole:null})}function i(){return(0,c.jsx)(u.Suspense,{fallback:(0,c.jsx)("div",{className:"flex items-center justify-center min-h-screen",children:"Loading..."}),children:(0,c.jsx)(a,{})})}e.s(["default",()=>i])}]);
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue