mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-16 23:41:43 +00:00
merge: resolve conflicts with origin/main
Keep both our GCP credential provider tests and the new cluster tests added in main. Merge imports (json, redis, get_redis_client, get_redis_connection_pool) from both branches.
This commit is contained in:
Ishaan Jaffer
commit
5df2d5c36a
781 changed files with 21346 additions and 6000 deletions
106
docs/my-website/blog/gpt_5_4_mini_nano/index.md
Normal file
106
docs/my-website/blog/gpt_5_4_mini_nano/index.md
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
---
|
||||
slug: gpt_5_4_mini_nano
|
||||
title: "Day 0 Support: GPT-5.4-mini and GPT-5.4-nano"
|
||||
date: 2026-03-17T10:00:00
|
||||
authors:
|
||||
- name: Sameer Kankute
|
||||
title: SWE @ LiteLLM (LLM Translation)
|
||||
url: https://www.linkedin.com/in/sameer-kankute/
|
||||
image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
|
||||
- name: Krrish Dholakia
|
||||
title: "CEO, LiteLLM"
|
||||
url: https://www.linkedin.com/in/krish-d/
|
||||
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
|
||||
- name: Ishaan Jaff
|
||||
title: "CTO, LiteLLM"
|
||||
url: https://www.linkedin.com/in/reffajnaahsi/
|
||||
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
|
||||
description: "GPT-5.4-mini and GPT-5.4-nano model support in LiteLLM"
|
||||
tags: [openai, gpt-5.4-mini, gpt-5.4-nano, completion]
|
||||
hide_table_of_contents: false
|
||||
---
|
||||
|
||||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
LiteLLM now supports GPT-5.4-mini and GPT-5.4-nano — cost-effective models for simple completions and high-throughput workloads.
|
||||
|
||||
:::note
|
||||
If you're on **v1.82.3-stable** or above, you don't need any update to use these models.
|
||||
:::
|
||||
|
||||
## Usage
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="proxy" label="LiteLLM Proxy">
|
||||
|
||||
**1. Setup config.yaml**
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: gpt-5.4-mini
|
||||
litellm_params:
|
||||
model: openai/gpt-5.4-mini
|
||||
api_key: os.environ/OPENAI_API_KEY
|
||||
- model_name: gpt-5.4-nano
|
||||
litellm_params:
|
||||
model: openai/gpt-5.4-nano
|
||||
api_key: os.environ/OPENAI_API_KEY
|
||||
```
|
||||
|
||||
**2. Start the proxy**
|
||||
|
||||
```bash
|
||||
litellm --config /path/to/config.yaml
|
||||
```
|
||||
|
||||
**3. Test it**
|
||||
|
||||
```bash
|
||||
# GPT-5.4-mini
|
||||
curl -X POST "http://localhost:4000/v1/chat/completions" \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer $LITELLM_KEY" \
|
||||
-d '{
|
||||
"model": "gpt-5.4-mini",
|
||||
"messages": [{"role": "user", "content": "What is the capital of France?"}]
|
||||
}'
|
||||
|
||||
# GPT-5.4-nano
|
||||
curl -X POST "http://localhost:4000/v1/chat/completions" \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer $LITELLM_KEY" \
|
||||
-d '{
|
||||
"model": "gpt-5.4-nano",
|
||||
"messages": [{"role": "user", "content": "What is 2 + 2?"}]
|
||||
}'
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="sdk" label="LiteLLM SDK">
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
|
||||
# GPT-5.4-mini
|
||||
response = completion(
|
||||
model="openai/gpt-5.4-mini",
|
||||
messages=[{"role": "user", "content": "What is the capital of France?"}],
|
||||
)
|
||||
print(response.choices[0].message.content)
|
||||
|
||||
# GPT-5.4-nano
|
||||
response = completion(
|
||||
model="openai/gpt-5.4-nano",
|
||||
messages=[{"role": "user", "content": "What is 2 + 2?"}],
|
||||
)
|
||||
print(response.choices[0].message.content)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
## Notes
|
||||
|
||||
- Both models support function calling, vision, and tool-use — see the [OpenAI provider docs](../../docs/providers/openai) for advanced usage.
|
||||
- GPT-5.4-nano is the most cost-effective option for simple tasks; GPT-5.4-mini offers a balance of speed and capability.
|
||||
|
|
@ -506,12 +506,15 @@ Request body will be in the Anthropic messages API format. **litellm follows the
|
|||
A system prompt providing context or specific instructions to the model.
|
||||
- **temperature** (number):
|
||||
Controls randomness in the model's responses. Valid range: `0 < temperature < 1`.
|
||||
- **thinking** (object):
|
||||
- **thinking** (object):
|
||||
Configuration for enabling extended thinking. If enabled, it includes:
|
||||
- **budget_tokens** (integer):
|
||||
- **budget_tokens** (integer):
|
||||
Minimum of 1024 tokens (and less than `max_tokens`).
|
||||
- **type** (enum):
|
||||
- **type** (enum):
|
||||
E.g., `"enabled"`.
|
||||
- **summary** (string, optional):
|
||||
Enables the summary style for thinking blocks. Possible values: `"auto"`, `"concise"`, `"detailed"`, `"disabled"`.
|
||||
When routing to non-Anthropic providers (e.g., `openai/gpt-5.1`), the `summary` value is preserved and forwarded to the downstream API.
|
||||
- **tool_choice** (object):
|
||||
Instructs how the model should utilize any provided tools.
|
||||
- **tools** (array of objects):
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@ import TabItem from '@theme/TabItem';
|
|||
Supported Providers:
|
||||
- OpenAI (`openai/`)
|
||||
- Anthropic API (`anthropic/`)
|
||||
- Google AI Studio (`gemini/`)
|
||||
- Vertex AI (`vertex_ai/`, `vertex_ai_beta/`)
|
||||
- Bedrock (`bedrock/`, `bedrock/invoke/`, `bedrock/converse`) ([All models bedrock supports prompt caching on](https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-caching.html))
|
||||
- Deepseek API (`deepseek/`)
|
||||
|
||||
|
|
@ -257,7 +259,7 @@ Anthropic charges for cache writes.
|
|||
|
||||
Specify the content to cache with `"cache_control": {"type": "ephemeral"}`.
|
||||
|
||||
If you pass that in for any other llm provider, it will be ignored.
|
||||
This same format also works for [Gemini / Vertex AI](#google-ai-studio--vertex-ai-gemini-example). For other providers, it will be ignored.
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="sdk" label="SDK">
|
||||
|
|
@ -356,6 +358,208 @@ print(response.usage)
|
|||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
### Google AI Studio / Vertex AI (Gemini) Example
|
||||
|
||||
Use the same Anthropic-style `cache_control` format — LiteLLM automatically translates it to Google's [context caching API](https://ai.google.dev/api/caching).
|
||||
|
||||
**How it works under the hood:**
|
||||
1. Messages with `cache_control` are separated and sent to Google's `cachedContents` API
|
||||
2. The cached content ID is then passed as `cachedContent` in the Gemini request body
|
||||
3. Works across all three providers: `gemini/` (Google AI Studio), `vertex_ai/`, and `vertex_ai_beta/`
|
||||
4. Requires a minimum of **1024 tokens** in the cached content — below that, caching is silently skipped
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="sdk" label="SDK">
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
import os
|
||||
|
||||
os.environ["GEMINI_API_KEY"] = ""
|
||||
|
||||
response = completion(
|
||||
model="gemini/gemini-2.5-flash",
|
||||
messages=[
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "You are an AI assistant tasked with analyzing legal documents.",
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Here is the full text of a complex legal agreement" * 400,
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "what are the key terms and conditions in this agreement?",
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
print(response.usage)
|
||||
```
|
||||
</TabItem>
|
||||
<TabItem value="proxy" label="PROXY">
|
||||
|
||||
1. Setup config.yaml
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: gemini-2.5-flash
|
||||
litellm_params:
|
||||
model: gemini/gemini-2.5-flash
|
||||
api_key: os.environ/GEMINI_API_KEY
|
||||
```
|
||||
|
||||
2. Start proxy
|
||||
|
||||
```bash
|
||||
litellm --config /path/to/config.yaml
|
||||
```
|
||||
|
||||
3. Test it!
|
||||
|
||||
```python
|
||||
from openai import OpenAI
|
||||
|
||||
client = OpenAI(
|
||||
api_key="LITELLM_PROXY_KEY", # sk-1234
|
||||
base_url="LITELLM_PROXY_BASE", # http://0.0.0.0:4000
|
||||
)
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model="gemini-2.5-flash",
|
||||
messages=[
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "You are an AI assistant tasked with analyzing legal documents.",
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Here is the full text of a complex legal agreement" * 400,
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "what are the key terms and conditions in this agreement?",
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
print(response.usage)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
#### Vertex AI
|
||||
|
||||
For Vertex AI, use `vertex_ai/` prefix:
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="sdk" label="SDK">
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
|
||||
response = completion(
|
||||
model="vertex_ai/gemini-2.5-flash",
|
||||
vertex_project="my-gcp-project",
|
||||
vertex_location="us-central1",
|
||||
messages=[
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "You are an AI assistant tasked with analyzing legal documents.",
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Here is the full text of a complex legal agreement" * 400,
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "what are the key terms and conditions in this agreement?",
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
print(response.usage)
|
||||
```
|
||||
</TabItem>
|
||||
<TabItem value="proxy" label="PROXY">
|
||||
|
||||
1. Setup config.yaml
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: gemini-2.5-flash
|
||||
litellm_params:
|
||||
model: vertex_ai/gemini-2.5-flash
|
||||
vertex_project: my-gcp-project
|
||||
vertex_location: us-central1
|
||||
```
|
||||
|
||||
2. Start proxy
|
||||
|
||||
```bash
|
||||
litellm --config /path/to/config.yaml
|
||||
```
|
||||
|
||||
3. Test it!
|
||||
|
||||
```python
|
||||
from openai import OpenAI
|
||||
|
||||
client = OpenAI(
|
||||
api_key="LITELLM_PROXY_KEY", # sk-1234
|
||||
base_url="LITELLM_PROXY_BASE", # http://0.0.0.0:4000
|
||||
)
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model="gemini-2.5-flash",
|
||||
messages=[
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "You are an AI assistant tasked with analyzing legal documents.",
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Here is the full text of a complex legal agreement" * 400,
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "what are the key terms and conditions in this agreement?",
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
print(response.usage)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
### Deepeek Example
|
||||
|
||||
Works the same as OpenAI.
|
||||
|
|
|
|||
48
docs/my-website/docs/prompt_management.md
Normal file
48
docs/my-website/docs/prompt_management.md
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
---
|
||||
title: Prompt Management with Responses API
|
||||
---
|
||||
|
||||
# Prompt Management with Responses API
|
||||
|
||||
Use LiteLLM Prompt Management with `/v1/responses` by passing `prompt_id` and optional `prompt_variables`.
|
||||
|
||||
## Basic Usage
|
||||
|
||||
```bash
|
||||
curl -X POST "http://localhost:4000/v1/responses" \
|
||||
-H "Authorization: Bearer sk-1234" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "gpt-4o",
|
||||
"prompt_id": "my-responses-prompt",
|
||||
"prompt_variables": {"topic": "large language models"},
|
||||
"input": []
|
||||
}'
|
||||
```
|
||||
|
||||
## Multi-turn Follow-up in `input`
|
||||
|
||||
To send follow-up turns in one request, pass message history in `input`.
|
||||
|
||||
```bash
|
||||
curl -X POST "http://localhost:4000/v1/responses" \
|
||||
-H "Authorization: Bearer sk-1234" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "gpt-4o",
|
||||
"prompt_id": "my-responses-prompt",
|
||||
"prompt_variables": {"topic": "large language models"},
|
||||
"input": [
|
||||
{"role": "user", "content": "Topic is LLMs. Start short."},
|
||||
{"role": "assistant", "content": "Sure, go ahead."},
|
||||
{"role": "user", "content": "Now give me 3 bullets and include pricing caveat."}
|
||||
]
|
||||
}'
|
||||
```
|
||||
|
||||
## Notes
|
||||
|
||||
- Prompt template messages are merged with your `input` messages.
|
||||
- Prompt variable substitution applies to prompt message content.
|
||||
- Tool call payload fields are not substituted by prompt variables.
|
||||
- For follow-ups with `previous_response_id`, include `prompt_id` again if you want prompt management applied on that turn.
|
||||
|
|
@ -54,6 +54,7 @@ response = completion(
|
|||
- stream
|
||||
- tools
|
||||
- tool_choice
|
||||
- include_server_side_tool_invocations
|
||||
- functions
|
||||
- response_format
|
||||
- n
|
||||
|
|
@ -856,7 +857,112 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
|
|||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
### URL Context
|
||||
### Context Circulation (Server-Side Tool Combination)
|
||||
|
||||
Context circulation allows Gemini 3+ models to combine **built-in tools** (like Google Search) with **your custom functions** in the same request. Without it, Gemini returns an error if you try to use both.
|
||||
|
||||
When enabled, Gemini can execute Google Search server-side, use those results to decide whether to call your custom functions, and return the full chain of reasoning.
|
||||
|
||||
**How it works:**
|
||||
1. You pass `include_server_side_tool_invocations=True` along with both Google Search and your function tools
|
||||
2. Gemini executes server-side tools internally and returns `toolCall`/`toolResponse` parts alongside any `functionCall` parts
|
||||
3. LiteLLM extracts the server-side invocations into `provider_specific_fields["server_side_tool_invocations"]`
|
||||
4. On subsequent turns, include the full assistant message in your conversation history — LiteLLM re-injects the server-side parts automatically
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="sdk" label="SDK">
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
|
||||
response = completion(
|
||||
model="gemini/gemini-3-flash-preview",
|
||||
messages=[{"role": "user", "content": "What's the weather in Buenos Aires? If it's raining, schedule a meeting."}],
|
||||
tools=[
|
||||
{"type": "web_search_preview"}, # Google Search (server-side)
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "schedule_meeting",
|
||||
"description": "Schedule a meeting",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"reason": {"type": "string"}},
|
||||
"required": ["reason"],
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
include_server_side_tool_invocations=True,
|
||||
)
|
||||
|
||||
msg = response.choices[0].message
|
||||
|
||||
# Server-side tool results are in provider_specific_fields
|
||||
psf = msg.provider_specific_fields or {}
|
||||
for invocation in psf.get("server_side_tool_invocations", []):
|
||||
print(invocation["tool_type"]) # e.g. "GOOGLE_SEARCH_WEB"
|
||||
print(invocation["id"])
|
||||
print(invocation["args"]) # e.g. {"queries": ["weather Buenos Aires"]}
|
||||
print(invocation["response"]) # Search results from Google
|
||||
|
||||
# For multi-turn: just append the full message to history
|
||||
messages.append(msg)
|
||||
messages.append({"role": "user", "content": "Thanks!"})
|
||||
# LiteLLM automatically re-injects the server-side parts + thought signatures
|
||||
response2 = completion(
|
||||
model="gemini/gemini-3-flash-preview",
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
include_server_side_tool_invocations=True,
|
||||
)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="proxy" label="PROXY">
|
||||
|
||||
1. Setup config.yaml
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: gemini-3-flash
|
||||
litellm_params:
|
||||
model: gemini/gemini-3-flash-preview
|
||||
api_key: os.environ/GEMINI_API_KEY
|
||||
```
|
||||
|
||||
2. Start Proxy
|
||||
```bash
|
||||
$ litellm --config /path/to/config.yaml
|
||||
```
|
||||
|
||||
3. Make Request
|
||||
```bash
|
||||
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H 'Authorization: Bearer sk-1234' \
|
||||
-d '{
|
||||
"model": "gemini-3-flash",
|
||||
"messages": [{"role": "user", "content": "What is the weather in Buenos Aires?"}],
|
||||
"tools": [
|
||||
{"type": "web_search_preview"},
|
||||
{"type": "function", "function": {"name": "schedule_meeting", "description": "Schedule a meeting", "parameters": {"type": "object", "properties": {"reason": {"type": "string"}}}}}
|
||||
],
|
||||
"include_server_side_tool_invocations": true
|
||||
}'
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
:::info
|
||||
|
||||
- Context circulation requires **Gemini 3+** models
|
||||
- Server-side tool invocations (`toolCall`/`toolResponse`) are **not** included in `tool_calls` — they are in `provider_specific_fields["server_side_tool_invocations"]` because they were already executed by Google, not by your code
|
||||
- `thought_signatures` are automatically preserved alongside server-side invocations for multi-turn coherence
|
||||
|
||||
:::
|
||||
|
||||
### URL Context
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="sdk" label="SDK">
|
||||
|
|
|
|||
|
|
@ -361,8 +361,9 @@ router_settings:
|
|||
| redis_url | str | URL for Redis server. **Known performance issue with Redis URL.** |
|
||||
| cache_responses | boolean | Flag to enable caching LLM Responses, if cache set under `router_settings`. If true, caches responses. Defaults to False. |
|
||||
| router_general_settings | RouterGeneralSettings | [SDK-Only] Router general settings - contains optimizations like 'async_only_mode'. [Docs](../routing.md#router-general-settings) |
|
||||
| optional_pre_call_checks | List[str] | List of pre-call checks to add to the router. Supported: `router_budget_limiting`, `prompt_caching`, `responses_api_deployment_check`, `encrypted_content_affinity`, `deployment_affinity`, `session_affinity`, `forward_client_headers_by_model_group` |
|
||||
| optional_pre_call_checks | List[str] | List of pre-call checks to add to the router. Supported: `router_budget_limiting`, `prompt_caching`, `responses_api_deployment_check`, `encrypted_content_affinity` (requires LiteLLM >= 1.82.3), `deployment_affinity`, `session_affinity`, `forward_client_headers_by_model_group` |
|
||||
| deployment_affinity_ttl_seconds | int | TTL (seconds) for user-key → deployment affinity mapping when `deployment_affinity` is enabled (configured at Router init / proxy startup). Defaults to `3600` (1 hour). |
|
||||
| model_group_affinity_config | Dict[str, List[str]] | Per-model-group affinity flags. Keys are model group names; values are lists of checks to enable (`deployment_affinity`, `responses_api_deployment_check`, `session_affinity`). Groups not listed fall back to the global `optional_pre_call_checks`. [Docs](../response_api.md#per-model-group-affinity-configuration) |
|
||||
| ignore_invalid_deployments | boolean | If true, ignores invalid deployments. Default for proxy is True - to prevent invalid models from blocking other models from being loaded. |
|
||||
| search_tools | List[SearchToolTypedDict] | List of search tool configurations for Search API integration. Each tool specifies a search_tool_name and litellm_params with search_provider, api_key, api_base, etc. [Further Docs](../search/index.md) |
|
||||
| guardrail_list | List[GuardrailTypedDict] | List of guardrail configurations for guardrail load balancing. Enables load balancing across multiple guardrail deployments with the same guardrail_name. [Further Docs](./guardrails/guardrail_load_balancing.md) |
|
||||
|
|
@ -813,7 +814,7 @@ router_settings:
|
|||
| LITELLM_MODE | Operating mode for LiteLLM (e.g., production, development)
|
||||
| LITELLM_NON_ROOT | Flag to run LiteLLM in non-root mode for enhanced security in Docker containers
|
||||
| LITELLM_RATE_LIMIT_WINDOW_SIZE | Rate limit window size for LiteLLM. Default is 60
|
||||
| LITELLM_REASONING_AUTO_SUMMARY | If set to "true", automatically enables detailed reasoning summaries for reasoning models (e.g., o1, o3-mini, deepseek-reasoner). When enabled, adds `summary: "detailed"` to reasoning effort configurations. Default is "false"
|
||||
| LITELLM_REASONING_AUTO_SUMMARY | If set to "true", automatically enables detailed reasoning summaries (`summary: "detailed"`) for reasoning models across all translation paths (Anthropic adapter, Responses API, etc.). Default is "false"
|
||||
| LITELLM_SALT_KEY | Salt key for encryption in LiteLLM
|
||||
| LITELLM_SSL_CIPHERS | SSL/TLS cipher configuration for faster handshakes. Controls cipher suite preferences for OpenSSL connections.
|
||||
| LITELLM_SECRET_AWS_KMS_LITELLM_LICENSE | AWS KMS encrypted license for LiteLLM
|
||||
|
|
@ -952,6 +953,8 @@ router_settings:
|
|||
| QDRANT_URL | Connection URL for Qdrant database
|
||||
| QDRANT_VECTOR_SIZE | Vector size for Qdrant operations. Default is 1536
|
||||
| REDIS_CONNECTION_POOL_TIMEOUT | Timeout in seconds for Redis connection pool. Default is 5
|
||||
| REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD | Number of consecutive failures before the Redis circuit breaker opens. Default is 5
|
||||
| REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT | Time in seconds before the Redis circuit breaker attempts recovery after opening. Default is 60
|
||||
| REDIS_CLUSTER_NODES | JSON-formatted list of Redis cluster startup nodes for Redis Cluster mode. Example: `[{"host": "node1", "port": 6379}]`
|
||||
| REDIS_HOST | Hostname for Redis server
|
||||
| REDIS_PASSWORD | Password for Redis service
|
||||
|
|
|
|||
139
docs/my-website/docs/proxy/guardrails/akto.md
Normal file
139
docs/my-website/docs/proxy/guardrails/akto.md
Normal file
|
|
@ -0,0 +1,139 @@
|
|||
# Akto
|
||||
|
||||
## Overview
|
||||
[Akto](https://www.akto.io/) provides API security guardrails and data ingestion for LLM traffic.
|
||||
|
||||
Akto now uses a **two-entry guardrail pattern** in LiteLLM:
|
||||
- `akto-validate` (`pre_call`) for request validation
|
||||
- `akto-ingest` (`post_call`) for request/response ingestion
|
||||
|
||||
There is no `on_flagged` setting anymore.
|
||||
|
||||
Use these as two separate guardrails in `config.yaml`:
|
||||
- `guardrail_name: "akto-validate"`
|
||||
- `guardrail_name: "akto-ingest"`
|
||||
|
||||
## 1. Get Your Akto Credentials
|
||||
|
||||
Set up the Akto Guardrail API Service and grab:
|
||||
- `AKTO_GUARDRAIL_API_BASE` — your Guardrail API Base URL
|
||||
- `AKTO_API_KEY` — your API key
|
||||
|
||||
## 2. Configure in `config.yaml`
|
||||
|
||||
### Block + Ingest (recommended)
|
||||
|
||||
Use both entries below. This gives you:
|
||||
- pre-call block decision
|
||||
- post-call ingestion for allowed traffic
|
||||
|
||||
Keep these as two separate entries (`akto-validate` and `akto-ingest`).
|
||||
|
||||
```yaml
|
||||
guardrails:
|
||||
- guardrail_name: "akto-validate"
|
||||
litellm_params:
|
||||
guardrail: akto
|
||||
mode: pre_call
|
||||
akto_base_url: os.environ/AKTO_GUARDRAIL_API_BASE
|
||||
akto_api_key: os.environ/AKTO_API_KEY
|
||||
default_on: true
|
||||
unreachable_fallback: fail_closed # optional: fail_open | fail_closed (default: fail_closed)
|
||||
guardrail_timeout: 5 # optional, default: 5
|
||||
akto_account_id: "1000000" # optional, env fallback: AKTO_ACCOUNT_ID
|
||||
akto_vxlan_id: "0" # optional, env fallback: AKTO_VXLAN_ID
|
||||
|
||||
- guardrail_name: "akto-ingest"
|
||||
litellm_params:
|
||||
guardrail: akto
|
||||
mode: post_call
|
||||
akto_base_url: os.environ/AKTO_GUARDRAIL_API_BASE
|
||||
akto_api_key: os.environ/AKTO_API_KEY
|
||||
default_on: true
|
||||
```
|
||||
|
||||
### Monitor-only mode
|
||||
|
||||
If you only want logging/ingestion and no blocking, keep only `akto-ingest`.
|
||||
|
||||
```yaml
|
||||
guardrails:
|
||||
- guardrail_name: "akto-ingest"
|
||||
litellm_params:
|
||||
guardrail: akto
|
||||
mode: post_call
|
||||
akto_base_url: os.environ/AKTO_GUARDRAIL_API_BASE
|
||||
akto_api_key: os.environ/AKTO_API_KEY
|
||||
default_on: true
|
||||
```
|
||||
|
||||
## 3. Test It
|
||||
|
||||
```shell
|
||||
curl -i http://localhost:4000/v1/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer <your litellm key>" \
|
||||
-d '{
|
||||
"model": "gpt-3.5-turbo",
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello, how are you?"}
|
||||
]
|
||||
}'
|
||||
```
|
||||
|
||||
If a request gets blocked:
|
||||
|
||||
```json
|
||||
{
|
||||
"error": {
|
||||
"message": "Prompt injection detected",
|
||||
"type": "None",
|
||||
"param": "None",
|
||||
"code": "403"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## 4. How It Works
|
||||
|
||||
**Block + Ingest mode:**
|
||||
```
|
||||
Request → LiteLLM → Akto guardrail check
|
||||
→ Allowed → forward to LLM → ingest response
|
||||
→ Blocked → ingest blocked marker → 403 error
|
||||
```
|
||||
|
||||
**Monitor-only mode:**
|
||||
```
|
||||
Request → LiteLLM → forward to LLM → get response
|
||||
→ Send to Akto (guardrails + ingest) → log only
|
||||
```
|
||||
|
||||
## 5. Event behavior
|
||||
|
||||
| Entry | LiteLLM hook | Akto call behavior |
|
||||
|------|---|---|
|
||||
| `akto-validate` | `pre_call` | Awaited call with `guardrails=true`, `ingest_data=false` |
|
||||
| `akto-ingest` | `post_call` | Fire-and-forget call with `guardrails=true`, `ingest_data=true` |
|
||||
|
||||
When blocked in `pre_call`, LiteLLM sends one fire-and-forget ingest payload with blocked metadata and returns `403`.
|
||||
|
||||
## 6. Parameters
|
||||
|
||||
| Parameter | Env Variable | Default | Description |
|
||||
|-----------|-------------|---------|-------------|
|
||||
| `akto_base_url` | `AKTO_GUARDRAIL_API_BASE` | *required* | Akto Guardrail API Base URL |
|
||||
| `akto_api_key` | `AKTO_API_KEY` | *required* | API key (sent as `Authorization` header) |
|
||||
| `akto_account_id` | `AKTO_ACCOUNT_ID` | `1000000` | Akto account id included in payload |
|
||||
| `akto_vxlan_id` | `AKTO_VXLAN_ID` | `0` | Akto vxlan id included in payload |
|
||||
| `unreachable_fallback` | — | `fail_closed` | `fail_open` or `fail_closed` |
|
||||
| `guardrail_timeout` | — | `5` | Timeout in seconds |
|
||||
| `default_on` | — | `true` (recommended) | Enables the guardrail entry by default |
|
||||
|
||||
## 7. Error Handling
|
||||
|
||||
| Scenario | `fail_closed` (default) | `fail_open` |
|
||||
|----------|------------------------|-------------|
|
||||
| Akto unreachable | ❌ Blocked (503) | ✅ Passes through |
|
||||
| Akto returns error | ❌ Blocked (503) | ✅ Passes through |
|
||||
| Guardrail says no | ❌ Blocked (403) | ❌ Blocked (403) |
|
||||
|
|
@ -117,6 +117,14 @@ guardrails:
|
|||
|
||||
:::
|
||||
|
||||
:::note Streaming and post_call guardrails
|
||||
|
||||
For **streaming responses**, `post_call` guardrails run on the fully assembled response **after** all chunks have been delivered to the client. This means `post_call` guardrails on streaming are **audit-only** — they can inspect and log the complete response, but cannot block content delivery. Guardrail results are recorded in `guardrail_information` within the logging payload for compliance and auditing.
|
||||
|
||||
To filter or block streaming content in real-time, use `async_post_call_streaming_iterator_hook` instead, which processes chunks as they arrive.
|
||||
|
||||
:::
|
||||
|
||||
<details>
|
||||
<summary>Advanced: Multiple modes with individual event hooks</summary>
|
||||
|
||||
|
|
@ -655,8 +663,8 @@ class myCustomGuardrail(CustomGuardrail):
|
|||
| `apply_guardrail` | Simple method to check and optionally modify text | ✅ | INPUT or OUTPUT | ✅ | ✅ | ✅ |
|
||||
| `async_pre_call_hook` | A hook that runs before the LLM API call | ✅ | INPUT | ✅ | ❌ | ✅ |
|
||||
| `async_moderation_hook` | A hook that runs during the LLM API call| ✅ | INPUT | ❌ | ❌ | ✅ |
|
||||
| `async_post_call_success_hook` | A hook that runs after a successful LLM API call| ✅ | INPUT, OUTPUT | ❌ | ✅ | ✅ |
|
||||
| `async_post_call_streaming_iterator_hook` | A hook that processes streaming responses | ✅ | OUTPUT | ❌ | ✅ | ✅ |
|
||||
| `async_post_call_success_hook` | A hook that runs after a successful LLM API call. For streaming, runs on the assembled response after delivery (audit-only, cannot block). | ✅ | INPUT, OUTPUT | ❌ | ✅ | ✅ (non-streaming only) |
|
||||
| `async_post_call_streaming_iterator_hook` | A hook that processes streaming responses in real-time (can filter/block chunks) | ✅ | OUTPUT | ❌ | ✅ | ✅ |
|
||||
|
||||
|
||||
## Frequently Asked Questions
|
||||
|
|
|
|||
190
docs/my-website/docs/proxy/high_availability_control_plane.md
Normal file
190
docs/my-website/docs/proxy/high_availability_control_plane.md
Normal file
|
|
@ -0,0 +1,190 @@
|
|||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
import { ControlPlaneArchitecture } from '@site/src/components/ControlPlaneArchitecture';
|
||||
|
||||
# [BETA] High Availability Control Plane
|
||||
|
||||
Deploy a single LiteLLM UI that manages multiple independent LiteLLM proxy instances, each with its own database, Redis, and master key.
|
||||
|
||||
:::info
|
||||
|
||||
This is an Enterprise feature.
|
||||
|
||||
[Enterprise Pricing](https://www.litellm.ai/#pricing)
|
||||
|
||||
[Get free 7-day trial key](https://www.litellm.ai/enterprise#trial)
|
||||
|
||||
:::
|
||||
|
||||
## Why This Architecture?
|
||||
|
||||
In the [standard multi-region setup](./control_plane_and_data_plane.md), all instances share a single database and master key. This works, but introduces a shared dependency. If the database goes down, every instance is affected.
|
||||
|
||||
The **High Availability Control Plane** takes a different approach:
|
||||
|
||||
| | Shared Database (Standard) | High Availability Control Plane |
|
||||
|---|---|---|
|
||||
| **Database** | Single shared DB for all instances | Each instance has its own DB |
|
||||
| **Redis** | Shared Redis | Each instance has its own Redis |
|
||||
| **Master Key** | Same key across all instances | Each instance has its own key |
|
||||
| **Failure isolation** | DB outage affects all instances | Failure is isolated to one instance |
|
||||
| **User management** | Centralized, one user table | Independent, each worker manages its own users |
|
||||
| **UI** | One UI per admin instance | Single control plane UI manages all workers |
|
||||
|
||||
### Benefits
|
||||
|
||||
- **True high availability**: no shared infrastructure means no single point of failure
|
||||
- **Blast radius containment**: a misconfiguration or outage on one worker doesn't affect others
|
||||
- **Regional isolation**: workers can run in different regions with data residency requirements
|
||||
- **Simpler operations**: each worker is a self-contained LiteLLM deployment
|
||||
|
||||
## Architecture
|
||||
|
||||
<ControlPlaneArchitecture />
|
||||
|
||||
The **control plane** is a LiteLLM instance that serves the admin UI and knows about all the workers. It does not proxy LLM requests, it is purely for administration.
|
||||
|
||||
Each **worker** is a fully independent LiteLLM proxy that handles LLM requests for its region or team. Workers have their own users, keys, teams, and budgets.
|
||||
|
||||
## Setup
|
||||
|
||||
### 1. Control Plane Configuration
|
||||
|
||||
The control plane needs a `worker_registry` that lists all worker instances.
|
||||
|
||||
```yaml title="cp_config.yaml"
|
||||
model_list: []
|
||||
|
||||
general_settings:
|
||||
master_key: sk-1234
|
||||
database_url: os.environ/DATABASE_URL
|
||||
|
||||
worker_registry:
|
||||
- worker_id: "worker-a"
|
||||
name: "Worker A"
|
||||
url: "http://localhost:4001"
|
||||
- worker_id: "worker-b"
|
||||
name: "Worker B"
|
||||
url: "http://localhost:4002"
|
||||
```
|
||||
|
||||
Start the control plane:
|
||||
|
||||
```bash
|
||||
litellm --config cp_config.yaml --port 4000
|
||||
```
|
||||
|
||||
### 2. Worker Configuration
|
||||
|
||||
Each worker needs `control_plane_url` in its `general_settings` to enable cross-origin authentication from the control plane UI.
|
||||
|
||||
`PROXY_BASE_URL` must also be set for each worker so that SSO callback redirects resolve correctly.
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="worker-a" label="Worker A">
|
||||
|
||||
```yaml title="worker_a_config.yaml"
|
||||
model_list: []
|
||||
|
||||
general_settings:
|
||||
master_key: sk-worker-a-1234
|
||||
database_url: os.environ/WORKER_A_DATABASE_URL
|
||||
control_plane_url: "http://localhost:4000"
|
||||
```
|
||||
|
||||
```bash
|
||||
PROXY_BASE_URL=http://localhost:4001 litellm --config worker_a_config.yaml --port 4001
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="worker-b" label="Worker B">
|
||||
|
||||
```yaml title="worker_b_config.yaml"
|
||||
model_list: []
|
||||
|
||||
general_settings:
|
||||
master_key: sk-worker-b-1234
|
||||
database_url: os.environ/WORKER_B_DATABASE_URL
|
||||
control_plane_url: "http://localhost:4000"
|
||||
```
|
||||
|
||||
```bash
|
||||
PROXY_BASE_URL=http://localhost:4002 litellm --config worker_b_config.yaml --port 4002
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
:::important
|
||||
Each worker must have its own `master_key` and `database_url`. The whole point of this architecture is that workers are independent.
|
||||
:::
|
||||
|
||||
### 3. SSO Configuration (Optional)
|
||||
|
||||
SSO is configured on the **control plane** instance the same way as a standard LiteLLM proxy. See the [SSO setup guide](./admin_ui_sso.md) for full instructions.
|
||||
|
||||
If using SSO, make sure to register each worker URL and the control plane URL as allowed callback URLs in your SSO provider's dashboard.
|
||||
|
||||
## How It Works
|
||||
|
||||
### Login Flow
|
||||
|
||||
1. User visits the control plane UI (`http://localhost:4000/ui`)
|
||||
2. The login page shows a **worker selector** dropdown listing all registered workers
|
||||
3. User selects a worker (e.g. "Worker A") and logs in with username/password or SSO
|
||||
4. The UI authenticates against the **selected worker** using the `/v3/login` endpoint
|
||||
5. On success, the UI stores the worker's JWT and points all subsequent API calls at the worker
|
||||
6. The user can now manage keys, teams, models, and budgets on that worker, all from the control plane UI
|
||||
|
||||
### Switching Workers
|
||||
|
||||
Once logged in, users can switch workers from the **navbar dropdown** without leaving the UI. Switching redirects back to the login page to authenticate against the new worker.
|
||||
|
||||
### Discovery
|
||||
|
||||
The control plane exposes a `/.well-known/litellm-ui-config` endpoint that the UI reads on load. This endpoint returns:
|
||||
- `is_control_plane: true`
|
||||
- The list of workers with their IDs, names, and URLs
|
||||
|
||||
This is how the login page knows to show the worker selector.
|
||||
|
||||
## Local Testing
|
||||
|
||||
To try this out locally, start each instance in a separate terminal:
|
||||
|
||||
```bash
|
||||
# Terminal 1: Control Plane
|
||||
litellm --config cp_config.yaml --port 4000
|
||||
|
||||
# Terminal 2: Worker A
|
||||
PROXY_BASE_URL=http://localhost:4001 litellm --config worker_a_config.yaml --port 4001
|
||||
|
||||
# Terminal 3: Worker B
|
||||
PROXY_BASE_URL=http://localhost:4002 litellm --config worker_b_config.yaml --port 4002
|
||||
```
|
||||
|
||||
Then open `http://localhost:4000/ui`. You should see the worker selector on the login page.
|
||||
|
||||
## Configuration Reference
|
||||
|
||||
### Control Plane Settings
|
||||
|
||||
| Field | Location | Description |
|
||||
|---|---|---|
|
||||
| `worker_registry` | Top-level config | List of worker instances |
|
||||
| `worker_registry[].worker_id` | Required | Unique identifier for the worker |
|
||||
| `worker_registry[].name` | Required | Display name shown in the UI |
|
||||
| `worker_registry[].url` | Required | Full URL of the worker instance |
|
||||
|
||||
### Worker Settings
|
||||
|
||||
| Field | Location | Description |
|
||||
|---|---|---|
|
||||
| `general_settings.control_plane_url` | Required | URL of the control plane instance. Enables `/v3/login` and `/v3/login/exchange` endpoints on this worker. |
|
||||
| `PROXY_BASE_URL` | Environment variable | The worker's own external URL. Required for SSO callback redirects. |
|
||||
|
||||
## Related Documentation
|
||||
|
||||
- [Standard Multi-Region Setup](./control_plane_and_data_plane.md) - shared-database architecture for admin/worker split
|
||||
- [SSO Setup](./admin_ui_sso.md) - configuring SSO for the admin UI
|
||||
- [Production Deployment](./prod.md) - production best practices
|
||||
|
|
@ -352,7 +352,7 @@ If `order=1` deployment is unavailable (e.g., rate-limited), the router falls ba
|
|||
|
||||
When load balancing OpenAI's Responses API across deployments with **different API keys** (e.g., different Azure regions or organizations), encrypted content items (like `rs_...` reasoning items) can only be decrypted by the originating API key.
|
||||
|
||||
**Solution:** Use the `encrypted_content_affinity` pre-call check to automatically route follow-up requests containing encrypted items to the correct deployment:
|
||||
**Solution:** Use the `encrypted_content_affinity` pre-call check (requires LiteLLM >= 1.82.3) to automatically route follow-up requests containing encrypted items to the correct deployment:
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
|
|
|
|||
|
|
@ -311,7 +311,7 @@ litellm_settings:
|
|||
1. **At Startup**: When the proxy starts, it reads the `prompts` field from `config.yaml`
|
||||
2. **Initialization**: Each prompt is initialized based on its `prompt_integration` type
|
||||
3. **In-Memory Storage**: Prompts are stored in the `IN_MEMORY_PROMPT_REGISTRY`
|
||||
4. **Access**: Use these prompts via the `/v1/chat/completions` endpoint with `prompt_id` in the request
|
||||
4. **Access**: Use these prompts via `/v1/chat/completions` or `/v1/responses` with `prompt_id` in the request
|
||||
|
||||
### Using Config-Loaded Prompts
|
||||
|
||||
|
|
@ -331,6 +331,23 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
|
|||
}'
|
||||
```
|
||||
|
||||
You can also use the same `prompt_id` with the Responses API:
|
||||
|
||||
```bash
|
||||
curl -L -X POST 'http://0.0.0.0:4000/v1/responses' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H 'Authorization: Bearer sk-1234' \
|
||||
-d '{
|
||||
"model": "gpt-4o",
|
||||
"prompt_id": "coding_assistant",
|
||||
"prompt_variables": {
|
||||
"language": "python",
|
||||
"task": "create a web scraper"
|
||||
},
|
||||
"input": []
|
||||
}'
|
||||
```
|
||||
|
||||
### Prompt Schema Reference
|
||||
|
||||
Each prompt in the `prompts` list requires:
|
||||
|
|
|
|||
|
|
@ -594,9 +594,26 @@ Expected Response
|
|||
|
||||
:::tip gpt-5.4: reasoning_effort + function tools
|
||||
|
||||
LiteLLM drops `reasoning_effort` from `gpt-5.4` requests to `litellm.completion()` that include tools, since that combination is supported in the Responses API.
|
||||
When `gpt-5.4+` requests to `litellm.completion()` include both `reasoning_effort` and `tools`, LiteLLM **automatically routes** the request through the Responses API bridge. This works for both **OpenAI** (`openai/gpt-5.4`) and **Azure** (`azure/gpt-5.4`) providers — no extra configuration needed.
|
||||
|
||||
If you need reasoning **and** tools together, use `openai/responses/gpt-5.4` to route through the Responses API instead. See [Responses API Bridge](/docs/providers/openai#openai-chat-completion-to-responses-api-bridge) for details.
|
||||
You can also route explicitly via `openai/responses/gpt-5.4` or `azure/responses/gpt-5.4`. See [Responses API Bridge](/docs/providers/openai#openai-chat-completion-to-responses-api-bridge) for details.
|
||||
|
||||
**Azure custom deployment names:** Auto-routing relies on the deployment name matching the `gpt-5.4*` pattern. If you use a custom deployment name (e.g. `"my-reasoning-model"`), enable routing via:
|
||||
|
||||
**SDK:**
|
||||
```python
|
||||
litellm.completion(model="azure/responses/my-reasoning-model", ...)
|
||||
```
|
||||
|
||||
**Proxy config:**
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: my-reasoning-model
|
||||
litellm_params:
|
||||
model: azure/my-reasoning-model
|
||||
model_info:
|
||||
mode: responses
|
||||
```
|
||||
|
||||
:::
|
||||
|
||||
|
|
@ -683,3 +700,69 @@ response = litellm.completion(
|
|||
reasoning_effort={"effort": "low", "summary": "detailed"}, # Explicit control
|
||||
)
|
||||
```
|
||||
|
||||
### Summary Preservation via `/v1/messages` Adapter
|
||||
|
||||
When using the Anthropic `/v1/messages` adapter to route non-Claude models (e.g., `openai/gpt-5.1`), the `thinking.summary` value is preserved and forwarded to the downstream provider. For example:
|
||||
|
||||
```python
|
||||
import litellm
|
||||
|
||||
response = await litellm.anthropic.messages.acreate(
|
||||
model="openai/gpt-5.1",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
max_tokens=8096,
|
||||
thinking={"type": "enabled", "budget_tokens": 5000, "summary": "concise"},
|
||||
)
|
||||
# The summary="concise" is preserved when routing to OpenAI's Responses API
|
||||
```
|
||||
|
||||
### Enabling Default Summary Injection for `/v1/messages` Adapter
|
||||
|
||||
When the Anthropic `/v1/messages` adapter translates `thinking` parameters to OpenAI `reasoning_effort` for non-Claude models, you can opt-in to automatic `summary="detailed"` injection using the `reasoning_auto_summary` flag. This ensures that reasoning text is returned in the response (matching the Anthropic thinking behavior).
|
||||
|
||||
To **enable** this default injection, use the `reasoning_auto_summary` flag:
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="sdk" label="SDK">
|
||||
|
||||
```python
|
||||
import litellm
|
||||
|
||||
# Enable default summary="detailed" injection
|
||||
litellm.reasoning_auto_summary = True
|
||||
|
||||
response = await litellm.anthropic.messages.acreate(
|
||||
model="openai/gpt-5.1",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
max_tokens=8096,
|
||||
thinking={"type": "enabled", "budget_tokens": 5000},
|
||||
)
|
||||
# summary="detailed" will be automatically added to reasoning_effort
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="env" label="Environment Variable">
|
||||
|
||||
```bash
|
||||
export LITELLM_REASONING_AUTO_SUMMARY=true
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="proxy" label="Proxy Config">
|
||||
|
||||
```yaml
|
||||
litellm_settings:
|
||||
reasoning_auto_summary: true
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
:::info
|
||||
|
||||
This flag only affects the automatic injection of `summary="detailed"` when no user-provided summary is present. If you explicitly pass `thinking.summary` (e.g., `"concise"` or `"auto"`), your value is always preserved regardless of this flag.
|
||||
|
||||
:::
|
||||
|
|
|
|||
|
|
@ -1160,12 +1160,12 @@ follow_up = await router.aresponses(
|
|||
To enable session continuity for Responses API in your LiteLLM proxy, set `optional_pre_call_checks` in your proxy config.yaml.
|
||||
|
||||
- `responses_api_deployment_check`: high priority routing when `previous_response_id` is provided
|
||||
- `encrypted_content_affinity`: **[Recommended]** content-aware routing for encrypted items (e.g., `rs_...` reasoning items)
|
||||
- `encrypted_content_affinity`: **[Recommended]** content-aware routing for encrypted items (e.g., `rs_...` reasoning items) (**requires LiteLLM >= 1.82.3**)
|
||||
- `session_affinity`: sticky sessions based on session id (takes priority over `deployment_affinity`)
|
||||
- `deployment_affinity`: sticky sessions based on user key (applies even without `previous_response_id`)
|
||||
|
||||
:::tip Recommended: Use `encrypted_content_affinity`
|
||||
For Responses API with load balancing across deployments with **different API keys**, use `encrypted_content_affinity` instead of `deployment_affinity`. It only pins requests that contain encrypted content, avoiding quota reduction while preventing `invalid_encrypted_content` errors.
|
||||
For Responses API with load balancing across deployments with **different API keys**, use `encrypted_content_affinity` instead of `deployment_affinity`. It only pins requests that contain encrypted content, avoiding quota reduction while preventing `invalid_encrypted_content` errors. (Requires LiteLLM >= 1.82.3.)
|
||||
:::
|
||||
|
||||
Notes:
|
||||
|
|
@ -1364,6 +1364,85 @@ litellm --config config.yaml
|
|||
| `deployment_affinity` | Simple sticky sessions | All requests from same API key | ❌ Reduces quota by # of users |
|
||||
|
||||
|
||||
## Per-Model-Group Affinity Configuration
|
||||
|
||||
By default, `optional_pre_call_checks` applies globally to all model groups. Use `model_group_affinity_config` when you want different affinity behavior per model group — for example, enabling stickiness only for models spread across providers (Azure + Bedrock) while leaving single-provider groups free to load-balance.
|
||||
|
||||
Groups not listed fall back to the global `optional_pre_call_checks` settings.
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="python-sdk" label="Python SDK">
|
||||
|
||||
```python
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-4",
|
||||
"litellm_params": {"model": "azure/gpt-4", "api_key": "...", "api_base": "https://endpoint1.openai.azure.com"},
|
||||
},
|
||||
{
|
||||
"model_name": "gpt-4",
|
||||
"litellm_params": {"model": "bedrock/anthropic.claude-v2", "aws_region_name": "us-east-1"},
|
||||
},
|
||||
{
|
||||
"model_name": "text-embedding-ada-002",
|
||||
"litellm_params": {"model": "azure/text-embedding-ada-002", "api_key": "...", "api_base": "https://endpoint1.openai.azure.com"},
|
||||
},
|
||||
{
|
||||
"model_name": "text-embedding-ada-002",
|
||||
"litellm_params": {"model": "azure/text-embedding-ada-002", "api_key": "...", "api_base": "https://endpoint2.openai.azure.com"},
|
||||
},
|
||||
],
|
||||
# gpt-4: cross-provider (Azure + Bedrock) — enable deployment affinity
|
||||
# text-embedding-ada-002: same provider — no affinity, let it load balance freely
|
||||
model_group_affinity_config={
|
||||
"gpt-4": ["deployment_affinity", "responses_api_deployment_check"],
|
||||
},
|
||||
)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="proxy-server" label="Proxy Server">
|
||||
|
||||
```yaml title="config.yaml"
|
||||
model_list:
|
||||
- model_name: gpt-4
|
||||
litellm_params:
|
||||
model: azure/gpt-4
|
||||
api_key: os.environ/AZURE_API_KEY_1
|
||||
api_base: https://endpoint1.openai.azure.com
|
||||
|
||||
- model_name: gpt-4
|
||||
litellm_params:
|
||||
model: bedrock/anthropic.claude-v2
|
||||
aws_region_name: us-east-1
|
||||
|
||||
- model_name: text-embedding-ada-002
|
||||
litellm_params:
|
||||
model: azure/text-embedding-ada-002
|
||||
api_key: os.environ/AZURE_API_KEY_1
|
||||
api_base: https://endpoint1.openai.azure.com
|
||||
|
||||
- model_name: text-embedding-ada-002
|
||||
litellm_params:
|
||||
model: azure/text-embedding-ada-002
|
||||
api_key: os.environ/AZURE_API_KEY_2
|
||||
api_base: https://endpoint2.openai.azure.com
|
||||
|
||||
router_settings:
|
||||
# gpt-4: cross-provider — enable stickiness
|
||||
# text-embedding-ada-002: not listed — load balances freely
|
||||
model_group_affinity_config:
|
||||
"gpt-4":
|
||||
- deployment_affinity
|
||||
- responses_api_deployment_check
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
**Supported values:** `deployment_affinity`, `responses_api_deployment_check`, `session_affinity`
|
||||
|
||||
## Calling non-Responses API endpoints (`/responses` to `/chat/completions` Bridge)
|
||||
|
||||
LiteLLM allows you to call non-Responses API models via a bridge to LiteLLM's `/chat/completions` endpoint. This is useful for calling Anthropic, Gemini and even non-Responses API OpenAI models.
|
||||
|
|
@ -1556,6 +1635,12 @@ curl -X POST "http://localhost:4000/v1/responses" \
|
|||
}'
|
||||
```
|
||||
|
||||
## File Search (Vector Stores)
|
||||
|
||||
For full `file_search` usage (native + emulated fallback), SDK/Proxy examples, architecture diagram, and Q&A, see:
|
||||
|
||||
- [`File Search in the Responses API — E2E Testing Guide`](/docs/tutorials/file_search_responses_api)
|
||||
|
||||
## Session Management
|
||||
|
||||
LiteLLM Proxy supports session management for all supported models. This allows you to store and fetch conversation history (state) in LiteLLM Proxy.
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ Click **+ Add New Plugin** to register a plugin in your marketplace.
|
|||
Enter the plugin information:
|
||||
|
||||
- **Name**: Plugin identifier (kebab-case, e.g., `my-plugin`)
|
||||
- **Source Type**: Choose GitHub or URL
|
||||
- **Source Type**: Choose GitHub, Git URL, or Git Subdir
|
||||
- **Repository/URL**: The git source (e.g., `org/repo` for GitHub)
|
||||
- **Version**: Semantic version (optional)
|
||||
- **Description**: What the plugin does
|
||||
|
|
@ -216,6 +216,22 @@ curl -X DELETE http://localhost:4000/claude-code/plugins/my-plugin \
|
|||
|
||||
Use this format for GitLab, Bitbucket, or self-hosted git repositories.
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="git-subdir" label="Git Subdir">
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "my-plugin",
|
||||
"source": {
|
||||
"source": "git-subdir",
|
||||
"url": "https://github.com/org/repo.git",
|
||||
"path": "plugins/my-plugin"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Use this format when your plugin lives in a subdirectory of a git repository. The `path` field must be a relative path of slash-separated segments (alphanumeric, dots, hyphens, underscores only).
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
|
|
|
|||
241
docs/my-website/docs/tutorials/file_search_responses_api.md
Normal file
241
docs/my-website/docs/tutorials/file_search_responses_api.md
Normal file
|
|
@ -0,0 +1,241 @@
|
|||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
# File Search in the Responses API
|
||||
|
||||
LiteLLM now supports `file_search` in the Responses API across both:
|
||||
- providers that support it natively (like OpenAI / Azure), and
|
||||
- providers that do not (like Anthropic, Bedrock, and other non-native providers) via emulation.
|
||||
|
||||
## What this is
|
||||
|
||||
`file_search` lets models retrieve grounded context from your vector stores and answer with citations.
|
||||
LiteLLM keeps one OpenAI-compatible output shape while routing requests through either native passthrough or an emulated fallback.
|
||||
|
||||
Two paths are covered:
|
||||
|
||||
| Path | When it runs | What LiteLLM does |
|
||||
| --- | --- | --- |
|
||||
| **Native passthrough** | Provider natively supports `file_search` (OpenAI, Azure) | Decodes unified vector store ID → forwards to provider as-is |
|
||||
| **Emulated fallback** | Provider doesn't support `file_search` (Anthropic, Bedrock, etc.) | Converts to a function tool → intercepts tool call → runs vector search → synthesizes OpenAI-format output |
|
||||
|
||||
In `tools[].vector_store_ids`, LiteLLM accepts both provider-native IDs (e.g. `vs_...`) **and** **managed vector store unified IDs** (URL-safe base64 strings from the proxy managed-vector flow), e.g. `litellm.responses(..., tools=[{"type": "file_search", "vector_store_ids": ["bGl0ZWxsbV9wcm94eT..."]}])`.
|
||||
|
||||
## Usage
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="proxy" label="LiteLLM Proxy" default>
|
||||
|
||||
### 1. Setup `config.yaml`
|
||||
|
||||
```yaml title="config.yaml"
|
||||
model_list:
|
||||
- model_name: gpt-4.1
|
||||
litellm_params:
|
||||
model: openai/gpt-4.1
|
||||
api_key: os.environ/OPENAI_API_KEY
|
||||
|
||||
- model_name: claude-sonnet
|
||||
litellm_params:
|
||||
model: anthropic/claude-sonnet-4-5
|
||||
api_key: os.environ/ANTHROPIC_API_KEY
|
||||
```
|
||||
|
||||
### 2. Start the proxy
|
||||
|
||||
```bash
|
||||
litellm --config config.yaml
|
||||
```
|
||||
|
||||
### 3. Call Responses API with `file_search`
|
||||
|
||||
```python title="Proxy call"
|
||||
from openai import OpenAI
|
||||
|
||||
client = OpenAI(base_url="http://localhost:4000", api_key="sk-your-proxy-key")
|
||||
|
||||
response = client.responses.create(
|
||||
model="claude-sonnet", # swap to "gpt-4.1" for native path
|
||||
input="What does LiteLLM support?",
|
||||
tools=[{
|
||||
"type": "file_search",
|
||||
"vector_store_ids": ["vs_abc123"]
|
||||
}],
|
||||
include=["file_search_call.results"],
|
||||
)
|
||||
|
||||
print(response.output)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="sdk" label="LiteLLM SDK">
|
||||
|
||||
### 1. Install + set keys
|
||||
|
||||
```bash
|
||||
pip install litellm
|
||||
export OPENAI_API_KEY="sk-..."
|
||||
export ANTHROPIC_API_KEY="sk-ant-..."
|
||||
```
|
||||
|
||||
### 2. Call Responses API with `file_search`
|
||||
|
||||
```python title="SDK call"
|
||||
import litellm
|
||||
|
||||
response = litellm.responses(
|
||||
model="anthropic/claude-sonnet-4-5", # swap to openai/gpt-4.1 for native path
|
||||
input="What does LiteLLM support?",
|
||||
tools=[{
|
||||
"type": "file_search",
|
||||
"vector_store_ids": ["vs_abc123"]
|
||||
}],
|
||||
include=["file_search_call.results"],
|
||||
)
|
||||
|
||||
print(response.output)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
### Behavior Matrix
|
||||
|
||||
| Path | SDK model | Proxy model | Behavior |
|
||||
| --- | --- | --- | --- |
|
||||
| Native passthrough | `openai/gpt-4.1` | `gpt-4.1` | Provider executes native `file_search` |
|
||||
| Emulated fallback | `anthropic/claude-sonnet-4-5` | `claude-sonnet` | LiteLLM converts to function tool and synthesizes OpenAI-format output |
|
||||
|
||||
|
||||
|
||||
## Architecture Diagram
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
A[Client SDK or Proxy Caller] --> B[LiteLLM Responses API]
|
||||
B --> C{Provider supports native file_search?}
|
||||
|
||||
C -->|Yes| D[Native passthrough path]
|
||||
D --> D1[Decode unified vector_store_id if needed]
|
||||
D1 --> D2[Forward request to provider unchanged]
|
||||
D2 --> D3[Provider performs file_search]
|
||||
D3 --> Z[OpenAI-compatible output]
|
||||
|
||||
C -->|No| E[Emulated fallback path]
|
||||
E --> E1[Convert file_search to litellm_file_search function tool]
|
||||
E1 --> E2[First model call returns tool call with one or more queries]
|
||||
E2 --> E3[LiteLLM executes vector search for each query]
|
||||
E3 --> E4[Second model call with tool_result context]
|
||||
E4 --> E5[Synthesize file_search_call + message + citations]
|
||||
E5 --> Z[OpenAI-compatible output]
|
||||
```
|
||||
|
||||
|
||||
|
||||
## Prerequisites
|
||||
|
||||
```bash
|
||||
pip install 'litellm[proxy]'
|
||||
export OPENAI_API_KEY="sk-..." # for native path
|
||||
export ANTHROPIC_API_KEY="sk-ant-..." # for emulated path
|
||||
```
|
||||
|
||||
|
||||
|
||||
## Example response shape
|
||||
|
||||
## Validating the Output Format
|
||||
|
||||
Regardless of which path ran, the response always follows the OpenAI Responses API format:
|
||||
|
||||
```json
|
||||
{
|
||||
"output": [
|
||||
{
|
||||
"type": "file_search_call",
|
||||
"id": "fs_abc123",
|
||||
"status": "completed",
|
||||
"queries": ["What does LiteLLM support?"],
|
||||
"search_results": null
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{
|
||||
"type": "output_text",
|
||||
"text": "LiteLLM is a unified interface...",
|
||||
"annotations": [
|
||||
{
|
||||
"type": "file_citation",
|
||||
"index": 150,
|
||||
"file_id": "file-xxxx",
|
||||
"filename": "knowledge.txt"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**Validation script:**
|
||||
|
||||
```python showLineNumbers title="Validate response structure"
|
||||
def validate_file_search_response(response):
|
||||
"""Assert that response follows OpenAI file_search output format."""
|
||||
output = response.output
|
||||
assert len(output) >= 2, "Expected at least 2 output items"
|
||||
|
||||
# First item: file_search_call
|
||||
fs_call = output[0]
|
||||
fs_type = fs_call["type"] if isinstance(fs_call, dict) else fs_call.type
|
||||
assert fs_type == "file_search_call", f"Expected file_search_call, got {fs_type}"
|
||||
|
||||
fs_status = fs_call["status"] if isinstance(fs_call, dict) else fs_call.status
|
||||
assert fs_status == "completed"
|
||||
|
||||
# Second item: message
|
||||
msg = output[1]
|
||||
msg_type = msg["type"] if isinstance(msg, dict) else msg.type
|
||||
assert msg_type == "message"
|
||||
|
||||
content = msg["content"] if isinstance(msg, dict) else msg.content
|
||||
assert len(content) > 0
|
||||
text_block = content[0]
|
||||
text = text_block["text"] if isinstance(text_block, dict) else text_block.text
|
||||
assert isinstance(text, str) and len(text) > 0
|
||||
|
||||
print("✅ Response structure valid")
|
||||
print(f" Queries: {fs_call['queries'] if isinstance(fs_call, dict) else fs_call.queries}")
|
||||
print(f" Answer length: {len(text)} chars")
|
||||
annotations = text_block["annotations"] if isinstance(text_block, dict) else text_block.annotations
|
||||
print(f" Citations: {len(annotations)}")
|
||||
|
||||
validate_file_search_response(response)
|
||||
```
|
||||
|
||||
|
||||
|
||||
## Q&A
|
||||
|
||||
- **Why do I see `UnsupportedParamsError`?** This usually means `file_search` was passed to a provider that does not support it natively and emulation could not route correctly. Check:
|
||||
- The model string is valid (for example, `anthropic/claude-sonnet-4-5`).
|
||||
- `custom_llm_provider` resolves correctly so LiteLLM can load the provider config.
|
||||
- **Why does vector search return no results?** Common causes:
|
||||
- The vector store ID is wrong or has no files attached.
|
||||
- In LiteLLM-managed stores, file ingestion is not complete (`status != completed`).
|
||||
- The query is too narrow; try a broader query.
|
||||
- **Why am I getting `403 Access denied` on vector store calls?** The caller does not have access to that vector store.
|
||||
- The store may belong to another team.
|
||||
- Use an admin/proxy key if your setup requires cross-team access.
|
||||
- **Why are `annotations` empty in emulated mode?** `file_citation` annotations require `file_id` metadata in search results. If your vector backend does not return file-level metadata, the answer text is still generated but citations can be empty.
|
||||
|
||||
|
||||
|
||||
## What to check next
|
||||
|
||||
- [File Search reference in Responses API docs](/docs/response_api#file-search-vector-stores) — full API reference
|
||||
- [Vector Store management](/docs/vector_store_files) — create and manage vector stores
|
||||
- [Managed vector stores](/docs/providers/bedrock_vector_store) — provider-specific setup
|
||||
151
docs/my-website/docs/tutorials/vertex_ai_pay_go.md
Normal file
151
docs/my-website/docs/tutorials/vertex_ai_pay_go.md
Normal file
|
|
@ -0,0 +1,151 @@
|
|||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
# Vertex AI PayGo and Priority
|
||||
|
||||
## Priority PayGo
|
||||
|
||||
LiteLLM supports Priority PayGo.
|
||||
Send a priority header, get priority queueing, and pay priority token rates.
|
||||
|
||||
:::info Which models support Priority PayGo?
|
||||
As of this writing: `gemini/gemini-2.5-pro`, `vertex_ai/gemini-3-pro-preview`, `vertex_ai/gemini-3.1-pro-preview`, `vertex_ai/gemini-3-flash-preview`, and their variants.
|
||||
Check `supports_service_tier: true` in LiteLLM's [model pricing JSON](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json).
|
||||
:::
|
||||
|
||||
### Send a priority request
|
||||
|
||||
Use this header:
|
||||
|
||||
`X-Vertex-AI-LLM-Shared-Request-Type: priority`
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="litellm-sdk" label="LiteLLM SDK">
|
||||
|
||||
```python
|
||||
import litellm
|
||||
|
||||
response = litellm.completion(
|
||||
model="vertex_ai/gemini-3-pro-preview",
|
||||
messages=[{"role": "user", "content": "Summarize the Gettysburg Address."}],
|
||||
vertex_project="YOUR_PROJECT_ID",
|
||||
vertex_location="us-central1",
|
||||
extra_headers={"X-Vertex-AI-LLM-Shared-Request-Type": "priority"},
|
||||
)
|
||||
|
||||
print(response.choices[0].message.content)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="proxy-config" label="Proxy config">
|
||||
|
||||
```yaml title="config.yaml"
|
||||
model_list:
|
||||
- model_name: gemini-priority
|
||||
litellm_params:
|
||||
model: vertex_ai/gemini-3-pro-preview
|
||||
vertex_project: "YOUR_PROJECT_ID"
|
||||
vertex_location: "us-central1"
|
||||
vertex_credentials: os.environ/GOOGLE_APPLICATION_CREDENTIALS
|
||||
extra_headers:
|
||||
X-Vertex-AI-LLM-Shared-Request-Type: priority
|
||||
```
|
||||
|
||||
```bash
|
||||
curl http://localhost:4000/v1/chat/completions \
|
||||
-H "Authorization: Bearer sk-your-key" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"model": "gemini-priority", "messages": [{"role": "user", "content": "Hello"}]}'
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="pass-through" label="Pass-through mode">
|
||||
|
||||
Use `x-pass-` so LiteLLM forwards provider-specific headers.
|
||||
|
||||
```bash
|
||||
MODEL_ID="gemini-3-pro-preview-0325"
|
||||
PROJECT_ID="YOUR_PROJECT_ID"
|
||||
|
||||
curl -X POST \
|
||||
"${LITELLM_PROXY_BASE_URL}/vertex_ai/v1/projects/${PROJECT_ID}/locations/global/publishers/google/models/${MODEL_ID}:generateContent" \
|
||||
-H "Authorization: Bearer sk-your-litellm-key" \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "x-pass-X-Vertex-AI-LLM-Shared-Request-Type: priority" \
|
||||
-d '{"contents": [{"role": "user", "parts": [{"text": "Hello!"}]}]}'
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
### How cost tracking works
|
||||
|
||||
|
||||
|
||||
**`trafficType` → `service_tier` mapping**
|
||||
|
||||
| `usageMetadata.trafficType` | `service_tier` | Pricing keys used |
|
||||
|---|---|---|
|
||||
| `ON_DEMAND` | `None` | `input_cost_per_token` |
|
||||
| `ON_DEMAND_PRIORITY` | `"priority"` | `input_cost_per_token_priority` |
|
||||
| `FLEX` / `BATCH` | `"flex"` | `input_cost_per_token_flex` |
|
||||
|
||||
If a tier-specific key is missing, LiteLLM falls back to standard pricing keys.
|
||||
|
||||
---
|
||||
|
||||
## Standard PayGo vs Provisioned Throughput
|
||||
|
||||
This is a different header from priority routing:
|
||||
|
||||
| Header value | Behavior |
|
||||
|---|---|
|
||||
| `X-Vertex-AI-LLM-Request-Type: shared` | Force standard PayGo (bypass PT) |
|
||||
| `X-Vertex-AI-LLM-Request-Type: dedicated` | Force Provisioned Throughput only (`429` if exhausted) |
|
||||
|
||||
### Native route example
|
||||
|
||||
```python
|
||||
import litellm
|
||||
|
||||
response = litellm.completion(
|
||||
model="vertex_ai/gemini-2.0-flash",
|
||||
messages=[{"role": "user", "content": "Hello!"}],
|
||||
vertex_project="YOUR_PROJECT_ID",
|
||||
vertex_location="us-central1",
|
||||
extra_headers={"X-Vertex-AI-LLM-Request-Type": "shared"},
|
||||
)
|
||||
```
|
||||
|
||||
### Pass-through example
|
||||
|
||||
```bash
|
||||
MODEL_ID="gemini-2.0-flash-001"
|
||||
PROJECT_ID="YOUR_PROJECT_ID"
|
||||
|
||||
curl -X POST \
|
||||
"${LITELLM_PROXY_BASE_URL}/vertex_ai/v1/projects/${PROJECT_ID}/locations/global/publishers/google/models/${MODEL_ID}:generateContent" \
|
||||
-H "Authorization: Bearer sk-your-litellm-key" \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "x-pass-X-Vertex-AI-LLM-Request-Type: shared" \
|
||||
-d '{
|
||||
"contents": [{"role": "user", "parts": [{"text": "Hello!"}]}]
|
||||
}'
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
**Q: What does `403 Permission denied` or `IAM_PERMISSION_DENIED` mean?**
|
||||
A: The service account or Application Default Credentials (ADC) user does not have the `roles/aiplatform.user` role. To resolve this, re-run the `gcloud projects add-iam-policy-binding`.
|
||||
|
||||
**Q: What should I do if I get a `429 Quota exceeded` error?**
|
||||
A: This means you've hit the per-region QPM (queries per minute) or TPM (tokens per minute) quota. You can:
|
||||
- Request a quota increase from the [GCP Quotas console](https://console.cloud.google.com/iam-admin/quotas)
|
||||
- Add more regions to your LiteLLM configuration for load balancing
|
||||
- Upgrade to [Provisioned Throughput](https://cloud.google.com/vertex-ai/generative-ai/docs/provisioned-throughput) for guaranteed capacity
|
||||
|
||||
**Q: How do I fix the `VERTEXAI_PROJECT not set` error?**
|
||||
A: Either pass the `vertex_project` parameter explicitly in your LiteLLM call, or set the `VERTEXAI_PROJECT` environment variable before running your code.
|
||||
|
||||
|
|
@ -430,6 +430,7 @@ const sidebars = {
|
|||
"proxy/architecture",
|
||||
"proxy/multi_tenant_architecture",
|
||||
"proxy/control_plane_and_data_plane",
|
||||
"proxy/high_availability_control_plane",
|
||||
"proxy/db_deadlocks",
|
||||
"proxy/db_info",
|
||||
"proxy/image_handling",
|
||||
|
|
@ -584,6 +585,7 @@ const sidebars = {
|
|||
label: "Spend Tracking",
|
||||
items: [
|
||||
"proxy/cost_tracking",
|
||||
"tutorials/vertex_ai_pay_go",
|
||||
"proxy/request_tags",
|
||||
"proxy/custom_pricing",
|
||||
"proxy/pricing_calculator",
|
||||
|
|
@ -737,6 +739,7 @@ const sidebars = {
|
|||
"proxy/realtime_webrtc",
|
||||
"rerank",
|
||||
"response_api",
|
||||
"prompt_management",
|
||||
"response_api_compact",
|
||||
{
|
||||
type: "category",
|
||||
|
|
@ -1433,6 +1436,7 @@ const learnSidebar = {
|
|||
},
|
||||
items: [
|
||||
"tutorials/prompt_caching",
|
||||
"tutorials/file_search_responses_api",
|
||||
"tutorials/anthropic_file_usage",
|
||||
"tutorials/gemini_realtime_with_audio",
|
||||
"tutorials/litellm_proxy_aporia",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,95 @@
|
|||
import React from 'react';
|
||||
import styles from './styles.module.css';
|
||||
|
||||
/* ────────────────────── Shared small pieces ────────────────────── */
|
||||
|
||||
function InfraChip({ color, label }: { color: string; label: string }) {
|
||||
const dotClass =
|
||||
color === 'green'
|
||||
? styles.infraDotGreen
|
||||
: color === 'blue'
|
||||
? styles.infraDotBlue
|
||||
: styles.infraDotOrange;
|
||||
|
||||
return (
|
||||
<span className={styles.infraChip}>
|
||||
<span className={`${styles.infraDot} ${dotClass}`} />
|
||||
{label}
|
||||
</span>
|
||||
);
|
||||
}
|
||||
|
||||
/* ────────────────────── Architecture tab ────────────────────── */
|
||||
|
||||
function ArchitectureView() {
|
||||
return (
|
||||
<div className={styles.diagram}>
|
||||
{/* User */}
|
||||
<div className={styles.userRow}>
|
||||
<div className={styles.userIcon}>👤</div>
|
||||
<span className={styles.userLabel}>Admin</span>
|
||||
</div>
|
||||
|
||||
<div className={styles.connectorDown} />
|
||||
|
||||
{/* Control Plane */}
|
||||
<div className={`${styles.node} ${styles.nodeControlPlane}`}>
|
||||
<div className={styles.nodeHeader}>
|
||||
<span className={styles.nodeTitle}>Control Plane</span>
|
||||
<span className={`${styles.badge} ${styles.badgeBlue}`}>UI</span>
|
||||
</div>
|
||||
<div className={styles.nodeSubtitle}>cp.example.com</div>
|
||||
<div className={styles.infraRow}>
|
||||
<InfraChip color="green" label="Own DB" />
|
||||
<InfraChip color="orange" label="Own Redis" />
|
||||
<InfraChip color="blue" label="Own Key" />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Branch connector */}
|
||||
<div className={styles.connectorBranch}>
|
||||
<div className={`${styles.branchLeg} ${styles.branchLegLeft}`} />
|
||||
<div className={`${styles.branchLeg} ${styles.branchLegRight}`} />
|
||||
</div>
|
||||
|
||||
{/* Workers */}
|
||||
<div className={styles.workersRow}>
|
||||
<div className={`${styles.node} ${styles.nodeWorker} ${styles.nodeWorkerA}`}>
|
||||
<div className={styles.nodeHeader}>
|
||||
<span className={styles.nodeTitle}>Worker A</span>
|
||||
<span className={`${styles.badge} ${styles.badgeGreen}`}>US East</span>
|
||||
</div>
|
||||
<div className={styles.nodeSubtitle}>worker-a.example.com</div>
|
||||
<div className={styles.infraRow}>
|
||||
<InfraChip color="green" label="Own DB" />
|
||||
<InfraChip color="orange" label="Own Redis" />
|
||||
<InfraChip color="blue" label="Own Key" />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className={`${styles.node} ${styles.nodeWorker} ${styles.nodeWorkerB}`}>
|
||||
<div className={styles.nodeHeader}>
|
||||
<span className={styles.nodeTitle}>Worker B</span>
|
||||
<span className={`${styles.badge} ${styles.badgePurple}`}>EU West</span>
|
||||
</div>
|
||||
<div className={styles.nodeSubtitle}>worker-b.example.com</div>
|
||||
<div className={styles.infraRow}>
|
||||
<InfraChip color="green" label="Own DB" />
|
||||
<InfraChip color="orange" label="Own Redis" />
|
||||
<InfraChip color="blue" label="Own Key" />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
/* ────────────────────── Main component ────────────────────── */
|
||||
|
||||
export default function ControlPlaneArchitecture() {
|
||||
return (
|
||||
<div className={styles.wrapper}>
|
||||
<ArchitectureView />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1 @@
|
|||
export { default as ControlPlaneArchitecture } from './ControlPlaneArchitecture';
|
||||
|
|
@ -0,0 +1,517 @@
|
|||
/* ── Custom properties ── */
|
||||
:root {
|
||||
--cp-bg: #ffffff;
|
||||
--cp-border: #e5e7eb;
|
||||
--cp-text: #1a1a2e;
|
||||
--cp-text-secondary: #6b7280;
|
||||
--cp-text-muted: #9ca3af;
|
||||
--cp-accent: #3b82f6;
|
||||
--cp-accent-light: #dbeafe;
|
||||
--cp-accent-glow: rgba(59, 130, 246, 0.15);
|
||||
--cp-green: #10b981;
|
||||
--cp-green-light: #d1fae5;
|
||||
--cp-green-glow: rgba(16, 185, 129, 0.15);
|
||||
--cp-orange: #f59e0b;
|
||||
--cp-orange-light: #fef3c7;
|
||||
--cp-purple: #8b5cf6;
|
||||
--cp-purple-light: #ede9fe;
|
||||
--cp-red: #ef4444;
|
||||
--cp-red-light: #fee2e2;
|
||||
--cp-card-bg: #f9fafb;
|
||||
--cp-infra-bg: #f1f5f9;
|
||||
--cp-infra-border: #cbd5e1;
|
||||
--cp-connector: #d1d5db;
|
||||
--cp-dot-size: 8px;
|
||||
}
|
||||
|
||||
[data-theme='dark'] {
|
||||
--cp-bg: #111827;
|
||||
--cp-border: #374151;
|
||||
--cp-text: #e5e7eb;
|
||||
--cp-text-secondary: #9ca3af;
|
||||
--cp-text-muted: #6b7280;
|
||||
--cp-accent: #60a5fa;
|
||||
--cp-accent-light: #1e3a5f;
|
||||
--cp-accent-glow: rgba(96, 165, 250, 0.2);
|
||||
--cp-green: #34d399;
|
||||
--cp-green-light: #064e3b;
|
||||
--cp-green-glow: rgba(52, 211, 153, 0.2);
|
||||
--cp-orange: #fbbf24;
|
||||
--cp-orange-light: #78350f;
|
||||
--cp-purple: #a78bfa;
|
||||
--cp-purple-light: #3b0764;
|
||||
--cp-red: #f87171;
|
||||
--cp-red-light: #451a1a;
|
||||
--cp-card-bg: #1f2937;
|
||||
--cp-infra-bg: #1e293b;
|
||||
--cp-infra-border: #475569;
|
||||
--cp-connector: #4b5563;
|
||||
}
|
||||
|
||||
/* ── Wrapper ── */
|
||||
.wrapper {
|
||||
margin: 1.5rem 0;
|
||||
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
|
||||
}
|
||||
|
||||
/* ── Tab bar ── */
|
||||
.tabs {
|
||||
display: flex;
|
||||
gap: 0;
|
||||
margin-bottom: 1.5rem;
|
||||
border-bottom: 2px solid var(--cp-border);
|
||||
}
|
||||
|
||||
.tab {
|
||||
padding: 0.6rem 1.25rem;
|
||||
font-size: 0.85rem;
|
||||
font-weight: 600;
|
||||
color: var(--cp-text-secondary);
|
||||
background: none;
|
||||
border: none;
|
||||
border-bottom: 2px solid transparent;
|
||||
margin-bottom: -2px;
|
||||
cursor: pointer;
|
||||
transition: color 0.2s, border-color 0.2s;
|
||||
}
|
||||
|
||||
.tab:hover {
|
||||
color: var(--cp-text);
|
||||
}
|
||||
|
||||
.tabActive {
|
||||
color: var(--cp-accent);
|
||||
border-bottom-color: var(--cp-accent);
|
||||
}
|
||||
|
||||
/* ── Architecture diagram ── */
|
||||
.diagram {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
gap: 0;
|
||||
}
|
||||
|
||||
/* ── User icon ── */
|
||||
.userRow {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
margin-bottom: 0.5rem;
|
||||
}
|
||||
|
||||
.userIcon {
|
||||
width: 40px;
|
||||
height: 40px;
|
||||
border-radius: 50%;
|
||||
background: var(--cp-accent-light);
|
||||
border: 2px solid var(--cp-accent);
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
font-size: 1.1rem;
|
||||
}
|
||||
|
||||
.userLabel {
|
||||
font-size: 0.75rem;
|
||||
color: var(--cp-text-secondary);
|
||||
margin-top: 0.3rem;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
/* ── Connectors ── */
|
||||
.connectorDown {
|
||||
width: 2px;
|
||||
height: 28px;
|
||||
background: var(--cp-connector);
|
||||
position: relative;
|
||||
}
|
||||
|
||||
.connectorDown::after {
|
||||
content: '';
|
||||
position: absolute;
|
||||
bottom: -4px;
|
||||
left: 50%;
|
||||
transform: translateX(-50%);
|
||||
width: 0;
|
||||
height: 0;
|
||||
border-left: 5px solid transparent;
|
||||
border-right: 5px solid transparent;
|
||||
border-top: 5px solid var(--cp-connector);
|
||||
}
|
||||
|
||||
.connectorBranch {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
justify-content: center;
|
||||
position: relative;
|
||||
width: 100%;
|
||||
max-width: 700px;
|
||||
height: 36px;
|
||||
}
|
||||
|
||||
.connectorBranch::before {
|
||||
content: '';
|
||||
position: absolute;
|
||||
top: 0;
|
||||
left: 50%;
|
||||
width: 2px;
|
||||
height: 12px;
|
||||
background: var(--cp-connector);
|
||||
transform: translateX(-50%);
|
||||
}
|
||||
|
||||
.connectorBranch::after {
|
||||
content: '';
|
||||
position: absolute;
|
||||
top: 12px;
|
||||
left: calc(25% + 12px);
|
||||
right: calc(25% + 12px);
|
||||
height: 2px;
|
||||
background: var(--cp-connector);
|
||||
}
|
||||
|
||||
.branchLeg {
|
||||
position: absolute;
|
||||
top: 12px;
|
||||
width: 2px;
|
||||
height: 24px;
|
||||
background: var(--cp-connector);
|
||||
}
|
||||
|
||||
.branchLeg::after {
|
||||
content: '';
|
||||
position: absolute;
|
||||
bottom: -4px;
|
||||
left: 50%;
|
||||
transform: translateX(-50%);
|
||||
width: 0;
|
||||
height: 0;
|
||||
border-left: 5px solid transparent;
|
||||
border-right: 5px solid transparent;
|
||||
border-top: 5px solid var(--cp-connector);
|
||||
}
|
||||
|
||||
.branchLegLeft {
|
||||
left: calc(25% + 12px);
|
||||
}
|
||||
|
||||
.branchLegRight {
|
||||
right: calc(25% + 12px);
|
||||
}
|
||||
|
||||
/* ── Node cards ── */
|
||||
.node {
|
||||
border: 2px solid var(--cp-border);
|
||||
border-radius: 12px;
|
||||
background: var(--cp-card-bg);
|
||||
padding: 1rem 1.25rem;
|
||||
text-align: center;
|
||||
transition: border-color 0.3s, box-shadow 0.3s;
|
||||
position: relative;
|
||||
}
|
||||
|
||||
.nodeControlPlane {
|
||||
border-color: var(--cp-accent);
|
||||
box-shadow: 0 0 0 3px var(--cp-accent-glow);
|
||||
min-width: 280px;
|
||||
}
|
||||
|
||||
.nodeWorker {
|
||||
min-width: 220px;
|
||||
}
|
||||
|
||||
.nodeWorkerA {
|
||||
border-color: var(--cp-green);
|
||||
box-shadow: 0 0 0 3px var(--cp-green-glow);
|
||||
}
|
||||
|
||||
.nodeWorkerB {
|
||||
border-color: var(--cp-purple);
|
||||
box-shadow: 0 0 0 3px rgba(139, 92, 246, 0.15);
|
||||
}
|
||||
|
||||
[data-theme='dark'] .nodeWorkerB {
|
||||
box-shadow: 0 0 0 3px rgba(167, 139, 250, 0.2);
|
||||
}
|
||||
|
||||
.nodeHeader {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
gap: 0.5rem;
|
||||
margin-bottom: 0.5rem;
|
||||
}
|
||||
|
||||
.nodeIcon {
|
||||
font-size: 1.1rem;
|
||||
}
|
||||
|
||||
.nodeTitle {
|
||||
font-size: 0.95rem;
|
||||
font-weight: 700;
|
||||
color: var(--cp-text);
|
||||
}
|
||||
|
||||
.nodeSubtitle {
|
||||
font-size: 0.75rem;
|
||||
color: var(--cp-text-secondary);
|
||||
margin-bottom: 0.75rem;
|
||||
}
|
||||
|
||||
.badge {
|
||||
display: inline-block;
|
||||
font-size: 0.65rem;
|
||||
font-weight: 600;
|
||||
padding: 0.15rem 0.5rem;
|
||||
border-radius: 9999px;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.04em;
|
||||
}
|
||||
|
||||
.badgeBlue {
|
||||
background: var(--cp-accent-light);
|
||||
color: var(--cp-accent);
|
||||
}
|
||||
|
||||
.badgeGreen {
|
||||
background: var(--cp-green-light);
|
||||
color: var(--cp-green);
|
||||
}
|
||||
|
||||
.badgePurple {
|
||||
background: var(--cp-purple-light);
|
||||
color: var(--cp-purple);
|
||||
}
|
||||
|
||||
/* ── Infrastructure chips ── */
|
||||
.infraRow {
|
||||
display: flex;
|
||||
gap: 0.4rem;
|
||||
justify-content: center;
|
||||
flex-wrap: wrap;
|
||||
margin-top: 0.5rem;
|
||||
}
|
||||
|
||||
.infraChip {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.3rem;
|
||||
font-size: 0.7rem;
|
||||
font-weight: 500;
|
||||
color: var(--cp-text-secondary);
|
||||
background: var(--cp-infra-bg);
|
||||
border: 1px solid var(--cp-infra-border);
|
||||
border-radius: 6px;
|
||||
padding: 0.2rem 0.5rem;
|
||||
}
|
||||
|
||||
.infraDot {
|
||||
width: 6px;
|
||||
height: 6px;
|
||||
border-radius: 50%;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.infraDotGreen {
|
||||
background: var(--cp-green);
|
||||
}
|
||||
|
||||
.infraDotBlue {
|
||||
background: var(--cp-accent);
|
||||
}
|
||||
|
||||
.infraDotOrange {
|
||||
background: var(--cp-orange);
|
||||
}
|
||||
|
||||
/* ── Workers row ── */
|
||||
.workersRow {
|
||||
display: flex;
|
||||
gap: 2rem;
|
||||
justify-content: center;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
/* ── Animated flow ── */
|
||||
.flowLabel {
|
||||
font-size: 0.7rem;
|
||||
color: var(--cp-accent);
|
||||
font-weight: 600;
|
||||
position: absolute;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
/* ── Comparison view ── */
|
||||
.comparisonGrid {
|
||||
display: grid;
|
||||
grid-template-columns: 1fr 1fr;
|
||||
gap: 1.5rem;
|
||||
margin-top: 0.5rem;
|
||||
}
|
||||
|
||||
.comparisonColumn {
|
||||
border: 2px solid var(--cp-border);
|
||||
border-radius: 12px;
|
||||
padding: 1.25rem;
|
||||
background: var(--cp-card-bg);
|
||||
}
|
||||
|
||||
.comparisonColumnOld {
|
||||
border-color: var(--cp-red);
|
||||
}
|
||||
|
||||
.comparisonColumnNew {
|
||||
border-color: var(--cp-green);
|
||||
}
|
||||
|
||||
.comparisonTitle {
|
||||
font-size: 0.9rem;
|
||||
font-weight: 700;
|
||||
color: var(--cp-text);
|
||||
text-align: center;
|
||||
margin-bottom: 1rem;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
gap: 0.4rem;
|
||||
}
|
||||
|
||||
.comparisonTitleOld {
|
||||
color: var(--cp-red);
|
||||
}
|
||||
|
||||
.comparisonTitleNew {
|
||||
color: var(--cp-green);
|
||||
}
|
||||
|
||||
/* ── Mini diagram inside comparison ── */
|
||||
.miniDiagram {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
.miniNode {
|
||||
border: 1.5px solid var(--cp-border);
|
||||
border-radius: 8px;
|
||||
background: var(--cp-bg);
|
||||
padding: 0.5rem 0.75rem;
|
||||
text-align: center;
|
||||
font-size: 0.75rem;
|
||||
font-weight: 600;
|
||||
color: var(--cp-text);
|
||||
width: 100%;
|
||||
max-width: 180px;
|
||||
}
|
||||
|
||||
.miniNodeHighlight {
|
||||
border-color: var(--cp-accent);
|
||||
background: var(--cp-accent-light);
|
||||
}
|
||||
|
||||
.miniNodeDanger {
|
||||
border-color: var(--cp-red);
|
||||
background: var(--cp-red-light);
|
||||
}
|
||||
|
||||
.miniNodeSuccess {
|
||||
border-color: var(--cp-green);
|
||||
background: var(--cp-green-light);
|
||||
}
|
||||
|
||||
.miniConnector {
|
||||
width: 1.5px;
|
||||
height: 16px;
|
||||
background: var(--cp-connector);
|
||||
}
|
||||
|
||||
.miniWorkersRow {
|
||||
display: flex;
|
||||
gap: 0.5rem;
|
||||
justify-content: center;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.miniWorkerStack {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
gap: 0.3rem;
|
||||
flex: 1;
|
||||
max-width: 140px;
|
||||
}
|
||||
|
||||
.miniInfra {
|
||||
font-size: 0.65rem;
|
||||
color: var(--cp-text-muted);
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.miniInfraShared {
|
||||
color: var(--cp-red);
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.miniInfraOwn {
|
||||
color: var(--cp-green);
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
/* ── Callout box ── */
|
||||
.callout {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
gap: 0.6rem;
|
||||
padding: 0.75rem 1rem;
|
||||
border-radius: 8px;
|
||||
margin-top: 1rem;
|
||||
font-size: 0.8rem;
|
||||
color: var(--cp-text);
|
||||
line-height: 1.5;
|
||||
}
|
||||
|
||||
.calloutDanger {
|
||||
background: var(--cp-red-light);
|
||||
border: 1px solid var(--cp-red);
|
||||
}
|
||||
|
||||
.calloutSuccess {
|
||||
background: var(--cp-green-light);
|
||||
border: 1px solid var(--cp-green);
|
||||
}
|
||||
|
||||
.calloutIcon {
|
||||
font-size: 1rem;
|
||||
flex-shrink: 0;
|
||||
margin-top: 0.1rem;
|
||||
}
|
||||
|
||||
/* ── Responsive ── */
|
||||
@media (max-width: 768px) {
|
||||
.comparisonGrid {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
|
||||
.workersRow {
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.nodeControlPlane {
|
||||
min-width: auto;
|
||||
width: 100%;
|
||||
max-width: 300px;
|
||||
}
|
||||
|
||||
.nodeWorker {
|
||||
min-width: auto;
|
||||
width: 100%;
|
||||
max-width: 260px;
|
||||
}
|
||||
|
||||
.connectorBranch {
|
||||
display: none;
|
||||
}
|
||||
}
|
||||
63
docs/my-website/static/img/vertex_cost_tracking_flow.svg
Normal file
63
docs/my-website/static/img/vertex_cost_tracking_flow.svg
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
<svg width="100%" viewBox="0 0 680 560" xmlns="http://www.w3.org/2000/svg">
|
||||
<defs>
|
||||
<marker id="arrow" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="6" markerHeight="6" orient="auto-start-reverse">
|
||||
<path d="M2 1L8 5L2 9" fill="none" stroke="context-stroke" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
|
||||
</marker>
|
||||
</defs>
|
||||
|
||||
<!-- Step 1: HTTP Request -->
|
||||
<g style="fill:rgb(0, 0, 0);stroke:none;color:rgb(255, 255, 255);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto">
|
||||
<rect x="190" y="30" width="300" height="56" rx="8" stroke-width="0.5" style="fill:rgb(12, 68, 124);stroke:rgb(133, 183, 235);color:rgb(255, 255, 255);stroke-width:0.5px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto"/>
|
||||
<text x="340" y="52" text-anchor="middle" dominant-baseline="central" style="fill:rgb(181, 212, 244);stroke:none;color:rgb(255, 255, 255);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:14px;font-weight:500;text-anchor:middle;dominant-baseline:central">HTTP request</text>
|
||||
<text x="340" y="70" text-anchor="middle" dominant-baseline="central" style="fill:rgb(133, 183, 235);stroke:none;color:rgb(255, 255, 255);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:12px;font-weight:400;text-anchor:middle;dominant-baseline:central">X-Vertex-AI-LLM-Shared-Request-Type: priority</text>
|
||||
</g>
|
||||
|
||||
<!-- Arrow 1 -->
|
||||
<line x1="340" y1="86" x2="340" y2="120" marker-end="url(#arrow)" style="fill:none;stroke:rgb(156, 154, 146);color:rgb(255, 255, 255);stroke-width:1.5px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto"/>
|
||||
<text x="356" y="108" dominant-baseline="central" style="fill:rgb(194, 192, 182);stroke:none;color:rgb(255, 255, 255);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:12px;font-weight:400;text-anchor:start;dominant-baseline:central">Vertex AI</text>
|
||||
|
||||
<!-- Step 2: Vertex response -->
|
||||
<g style="fill:rgb(0, 0, 0);stroke:none;color:rgb(255, 255, 255);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto">
|
||||
<rect x="190" y="120" width="300" height="56" rx="8" stroke-width="0.5" style="fill:rgb(8, 80, 65);stroke:rgb(93, 202, 165);color:rgb(255, 255, 255);stroke-width:0.5px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto"/>
|
||||
<text x="340" y="142" text-anchor="middle" dominant-baseline="central" style="fill:rgb(159, 225, 203);stroke:none;color:rgb(255, 255, 255);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:14px;font-weight:500;text-anchor:middle;dominant-baseline:central">Vertex response</text>
|
||||
<text x="340" y="160" text-anchor="middle" dominant-baseline="central" style="fill:rgb(93, 202, 165);stroke:none;color:rgb(255, 255, 255);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:12px;font-weight:400;text-anchor:middle;dominant-baseline:central">usageMetadata.trafficType = ON_DEMAND_PRIORITY</text>
|
||||
</g>
|
||||
|
||||
<!-- Arrow 2 -->
|
||||
<line x1="340" y1="176" x2="340" y2="210" marker-end="url(#arrow)" style="fill:none;stroke:rgb(156, 154, 146);color:rgb(255, 255, 255);stroke-width:1.5px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto"/>
|
||||
|
||||
<!-- Step 3: LiteLLM hidden params -->
|
||||
<g style="fill:rgb(0, 0, 0);stroke:none;color:rgb(255, 255, 255);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto">
|
||||
<rect x="190" y="210" width="300" height="56" rx="8" stroke-width="0.5" style="fill:rgb(60, 52, 137);stroke:rgb(175, 169, 236);color:rgb(255, 255, 255);stroke-width:0.5px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto"/>
|
||||
<text x="340" y="232" text-anchor="middle" dominant-baseline="central" style="fill:rgb(206, 203, 246);stroke:none;color:rgb(255, 255, 255);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:14px;font-weight:500;text-anchor:middle;dominant-baseline:central">LiteLLM stores it</text>
|
||||
<text x="340" y="250" text-anchor="middle" dominant-baseline="central" style="fill:rgb(175, 169, 236);stroke:none;color:rgb(255, 255, 255);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:12px;font-weight:400;text-anchor:middle;dominant-baseline:central">_hidden_params.provider_specific_fields.traffic_type</text>
|
||||
</g>
|
||||
|
||||
<!-- Arrow 3 -->
|
||||
<line x1="340" y1="266" x2="340" y2="300" marker-end="url(#arrow)" style="fill:none;stroke:rgb(156, 154, 146);color:rgb(255, 255, 255);stroke-width:1.5px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto"/>
|
||||
|
||||
<!-- Step 4: completion_cost() -->
|
||||
<g style="fill:rgb(0, 0, 0);stroke:none;color:rgb(255, 255, 255);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto">
|
||||
<rect x="190" y="300" width="300" height="56" rx="8" stroke-width="0.5" style="fill:rgb(99, 56, 6);stroke:rgb(239, 159, 39);color:rgb(255, 255, 255);stroke-width:0.5px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto"/>
|
||||
<text x="340" y="322" text-anchor="middle" dominant-baseline="central" style="fill:rgb(250, 199, 117);stroke:none;color:rgb(255, 255, 255);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:14px;font-weight:500;text-anchor:middle;dominant-baseline:central">completion_cost()</text>
|
||||
<text x="340" y="340" text-anchor="middle" dominant-baseline="central" style="fill:rgb(239, 159, 39);stroke:none;color:rgb(255, 255, 255);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:12px;font-weight:400;text-anchor:middle;dominant-baseline:central">Maps traffic_type → service_tier = "priority"</text>
|
||||
</g>
|
||||
|
||||
<!-- Arrow 4 -->
|
||||
<line x1="340" y1="356" x2="340" y2="390" marker-end="url(#arrow)" style="fill:none;stroke:rgb(156, 154, 146);color:rgb(255, 255, 255);stroke-width:1.5px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto"/>
|
||||
|
||||
<!-- Step 5: Pricing lookup -->
|
||||
<g style="fill:rgb(0, 0, 0);stroke:none;color:rgb(255, 255, 255);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto">
|
||||
<rect x="190" y="390" width="300" height="56" rx="8" stroke-width="0.5" style="fill:rgb(113, 43, 19);stroke:rgb(240, 153, 123);color:rgb(255, 255, 255);stroke-width:0.5px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:16px;font-weight:400;text-anchor:start;dominant-baseline:auto"/>
|
||||
<text x="340" y="412" text-anchor="middle" dominant-baseline="central" style="fill:rgb(245, 196, 179);stroke:none;color:rgb(255, 255, 255);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:14px;font-weight:500;text-anchor:middle;dominant-baseline:central">Pricing lookup</text>
|
||||
<text x="340" y="430" text-anchor="middle" dominant-baseline="central" style="fill:rgb(240, 153, 123);stroke:none;color:rgb(255, 255, 255);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:12px;font-weight:400;text-anchor:middle;dominant-baseline:central">input/output_cost_per_token_priority</text>
|
||||
</g>
|
||||
|
||||
<!-- Step numbers in left margin -->
|
||||
<text x="172" y="58" text-anchor="end" dominant-baseline="central" style="fill:rgb(194, 192, 182);stroke:none;color:rgb(255, 255, 255);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:12px;font-weight:400;text-anchor:end;dominant-baseline:central">①</text>
|
||||
<text x="172" y="148" text-anchor="end" dominant-baseline="central" style="fill:rgb(194, 192, 182);stroke:none;color:rgb(255, 255, 255);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:12px;font-weight:400;text-anchor:end;dominant-baseline:central">②</text>
|
||||
<text x="172" y="238" text-anchor="end" dominant-baseline="central" style="fill:rgb(194, 192, 182);stroke:none;color:rgb(255, 255, 255);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:12px;font-weight:400;text-anchor:end;dominant-baseline:central">③</text>
|
||||
<text x="172" y="328" text-anchor="end" dominant-baseline="central" style="fill:rgb(194, 192, 182);stroke:none;color:rgb(255, 255, 255);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:12px;font-weight:400;text-anchor:end;dominant-baseline:central">④</text>
|
||||
<text x="172" y="418" text-anchor="end" dominant-baseline="central" style="fill:rgb(194, 192, 182);stroke:none;color:rgb(255, 255, 255);stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;opacity:1;font-family:"Anthropic Sans", -apple-system, "system-ui", "Segoe UI", sans-serif;font-size:12px;font-weight:400;text-anchor:end;dominant-baseline:central">⑤</text>
|
||||
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 12 KiB |
|
|
@ -29,7 +29,7 @@ from litellm.proxy.openai_files_endpoints.common_utils import (
|
|||
get_models_from_unified_file_id,
|
||||
normalize_mime_type_for_provider,
|
||||
)
|
||||
from litellm.types.llms.openai import (
|
||||
from litellm.types.llms.openai import ( # pyright: ignore[reportAttributeAccessIssue]
|
||||
AllMessageValues,
|
||||
AsyncCursorPage,
|
||||
ChatCompletionFileObject,
|
||||
|
|
@ -442,25 +442,33 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
elif call_type == CallTypes.aresponses.value or call_type == CallTypes.responses.value:
|
||||
# Handle managed files in responses API input and tools
|
||||
file_ids = []
|
||||
|
||||
|
||||
# Extract file IDs from input parameter
|
||||
input_data = data.get("input")
|
||||
if input_data:
|
||||
file_ids.extend(self.get_file_ids_from_responses_input(input_data))
|
||||
|
||||
|
||||
# Extract file IDs from tools parameter (e.g., code_interpreter container)
|
||||
tools = data.get("tools")
|
||||
if tools:
|
||||
file_ids.extend(self.get_file_ids_from_responses_tools(tools))
|
||||
|
||||
|
||||
if file_ids:
|
||||
# Check user has access to all managed files
|
||||
await self.check_file_ids_access(file_ids, user_api_key_dict)
|
||||
|
||||
|
||||
model_file_id_mapping = await self.get_model_file_id_mapping(
|
||||
file_ids, user_api_key_dict.parent_otel_span
|
||||
)
|
||||
data["model_file_id_mapping"] = model_file_id_mapping
|
||||
|
||||
# Check access for file_search vector_store_ids
|
||||
if tools:
|
||||
unified_vs_ids = self.get_vector_store_ids_from_file_search_tools(tools)
|
||||
if unified_vs_ids:
|
||||
await self.check_vector_store_ids_access(
|
||||
unified_vs_ids, user_api_key_dict
|
||||
)
|
||||
elif call_type == CallTypes.afile_content.value:
|
||||
retrieve_file_id = cast(Optional[str], data.get("file_id"))
|
||||
potential_file_id = (
|
||||
|
|
@ -704,6 +712,101 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
|
||||
return file_ids
|
||||
|
||||
def get_vector_store_ids_from_file_search_tools(
|
||||
self, tools: List[Dict[str, Any]]
|
||||
) -> List[str]:
|
||||
"""
|
||||
Extract unified vector_store_ids from file_search tools.
|
||||
|
||||
Only returns IDs that are LiteLLM-managed (base64 unified IDs).
|
||||
Native provider IDs are skipped — they have no LiteLLM access record.
|
||||
"""
|
||||
from litellm.llms.base_llm.managed_resources.utils import (
|
||||
is_base64_encoded_unified_id,
|
||||
)
|
||||
|
||||
vs_ids: List[str] = []
|
||||
if not isinstance(tools, list):
|
||||
return vs_ids
|
||||
|
||||
for tool in tools:
|
||||
if not isinstance(tool, dict) or tool.get("type") != "file_search":
|
||||
continue
|
||||
vector_store_ids = tool.get("vector_store_ids")
|
||||
if not isinstance(vector_store_ids, list):
|
||||
continue
|
||||
for vs_id in vector_store_ids:
|
||||
if isinstance(vs_id, str) and is_base64_encoded_unified_id(vs_id):
|
||||
vs_ids.append(vs_id)
|
||||
|
||||
return vs_ids
|
||||
|
||||
async def check_vector_store_ids_access(
|
||||
self,
|
||||
vector_store_ids: List[str],
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
) -> None:
|
||||
"""
|
||||
Verify the caller's team can access each LiteLLM-managed vector store.
|
||||
|
||||
Batch-fetches vector stores from DB and checks team_id.
|
||||
Raises HTTPException(403) on the first access violation.
|
||||
Non-managed (native) IDs should already be filtered out before calling this.
|
||||
"""
|
||||
from litellm.llms.base_llm.managed_resources.utils import (
|
||||
extract_unified_uuid_from_unified_id,
|
||||
)
|
||||
from litellm.proxy.auth.auth_checks import (
|
||||
get_managed_vector_store_rows_by_uuids,
|
||||
)
|
||||
from litellm.proxy.proxy_server import (
|
||||
prisma_client,
|
||||
proxy_logging_obj,
|
||||
user_api_key_cache,
|
||||
)
|
||||
|
||||
if not vector_store_ids or prisma_client is None:
|
||||
return
|
||||
|
||||
# Map each unified ID to its internal UUID for a single batch DB fetch
|
||||
uuid_to_unified: Dict[str, str] = {}
|
||||
for vs_id in vector_store_ids:
|
||||
uuid = extract_unified_uuid_from_unified_id(vs_id)
|
||||
if uuid:
|
||||
uuid_to_unified[uuid] = vs_id
|
||||
|
||||
if not uuid_to_unified:
|
||||
return
|
||||
|
||||
rows = await get_managed_vector_store_rows_by_uuids(
|
||||
uuids=list(uuid_to_unified.keys()),
|
||||
prisma_client=prisma_client,
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
|
||||
found_uuids = {row.vector_store_id for row in rows}
|
||||
|
||||
for uuid, original_id in uuid_to_unified.items():
|
||||
if uuid not in found_uuids:
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail=f"Vector store '{original_id}' not found or access denied.",
|
||||
)
|
||||
|
||||
caller_team_id = user_api_key_dict.team_id
|
||||
for row in rows:
|
||||
vs_team_id = getattr(row, "team_id", None)
|
||||
if vs_team_id is not None and vs_team_id != caller_team_id:
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail=(
|
||||
f"Team '{caller_team_id}' does not have access to vector "
|
||||
f"store '{row.vector_store_id}'. The store belongs to team "
|
||||
f"'{vs_team_id}'."
|
||||
),
|
||||
)
|
||||
|
||||
async def get_model_file_id_mapping(
|
||||
self, file_ids: List[str], litellm_parent_otel_span: Span
|
||||
) -> dict:
|
||||
|
|
@ -954,7 +1057,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
)
|
||||
else:
|
||||
file_object = await litellm.afile_retrieve(
|
||||
custom_llm_provider=model_name.split("/")[0] if model_name and "/" in model_name else "openai",
|
||||
custom_llm_provider=model_name.split("/")[0] if model_name and "/" in model_name else "openai", # type: ignore[arg-type]
|
||||
file_id=original_file_id,
|
||||
)
|
||||
verbose_logger.debug(
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[tool.poetry]
|
||||
name = "litellm-enterprise"
|
||||
version = "0.1.34"
|
||||
version = "0.1.35"
|
||||
description = "Package for LiteLLM Enterprise features"
|
||||
authors = ["BerriAI"]
|
||||
readme = "README.md"
|
||||
|
|
@ -22,7 +22,7 @@ requires = ["poetry-core"]
|
|||
build-backend = "poetry.core.masonry.api"
|
||||
|
||||
[tool.commitizen]
|
||||
version = "0.1.33"
|
||||
version = "0.1.35"
|
||||
version_files = [
|
||||
"pyproject.toml:version",
|
||||
"../requirements.txt:litellm-enterprise==",
|
||||
|
|
|
|||
BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.60-py3-none-any.whl
vendored
Normal file
BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.60-py3-none-any.whl
vendored
Normal file
Binary file not shown.
BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.60.tar.gz
vendored
Normal file
BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.60.tar.gz
vendored
Normal file
Binary file not shown.
|
|
@ -1,9 +1,9 @@
|
|||
-- CreateIndex
|
||||
CREATE INDEX "LiteLLM_TeamTable_organization_id_idx" ON "LiteLLM_TeamTable"("organization_id");
|
||||
CREATE INDEX IF NOT EXISTS "LiteLLM_TeamTable_organization_id_idx" ON "LiteLLM_TeamTable"("organization_id");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "LiteLLM_TeamTable_team_alias_idx" ON "LiteLLM_TeamTable"("team_alias");
|
||||
CREATE INDEX IF NOT EXISTS "LiteLLM_TeamTable_team_alias_idx" ON "LiteLLM_TeamTable"("team_alias");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "LiteLLM_TeamTable_created_at_idx" ON "LiteLLM_TeamTable"("created_at");
|
||||
CREATE INDEX IF NOT EXISTS "LiteLLM_TeamTable_created_at_idx" ON "LiteLLM_TeamTable"("created_at");
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[tool.poetry]
|
||||
name = "litellm-proxy-extras"
|
||||
version = "0.4.58"
|
||||
version = "0.4.60"
|
||||
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
|
||||
authors = ["BerriAI"]
|
||||
readme = "README.md"
|
||||
|
|
@ -22,7 +22,7 @@ requires = ["poetry-core"]
|
|||
build-backend = "poetry.core.masonry.api"
|
||||
|
||||
[tool.commitizen]
|
||||
version = "0.4.58"
|
||||
version = "0.4.60"
|
||||
version_files = [
|
||||
"pyproject.toml:version",
|
||||
"../requirements.txt:litellm-proxy-extras==",
|
||||
|
|
|
|||
|
|
@ -97,6 +97,7 @@ input_callback: List[CALLBACK_TYPES] = []
|
|||
success_callback: List[CALLBACK_TYPES] = []
|
||||
failure_callback: List[CALLBACK_TYPES] = []
|
||||
service_callback: List[CALLBACK_TYPES] = []
|
||||
audit_log_callbacks: List[CALLBACK_TYPES] = []
|
||||
# logging_callback_manager is lazy-loaded via __getattr__
|
||||
_custom_logger_compatible_callbacks_literal = Literal[
|
||||
"lago",
|
||||
|
|
|
|||
|
|
@ -52,6 +52,17 @@ def _build_secret_patterns() -> re.Pattern:
|
|||
r"(?<=://)[^\s'\"]*:[^\s'\"@]+(?=@)",
|
||||
# Databricks personal access tokens
|
||||
r"dapi[0-9a-f]{32}",
|
||||
# ── Key-name-based redaction ──
|
||||
# Catches secrets inside dicts/config dumps by matching on the KEY name
|
||||
# regardless of what the value looks like.
|
||||
# e.g. 'master_key': 'any-value-here', "database_url": "postgres://..."
|
||||
r"(?:master_key|database_url|db_url|connection_string|"
|
||||
r"private_key|signing_key|encryption_key|"
|
||||
r"auth_token|access_token|refresh_token|"
|
||||
r"slack_webhook_url|webhook_url|"
|
||||
r"database_connection_string|"
|
||||
r"huggingface_token|jwt_secret)"
|
||||
r"""['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+""",
|
||||
]
|
||||
return re.compile("|".join(patterns), re.IGNORECASE)
|
||||
|
||||
|
|
@ -272,7 +283,7 @@ verbose_proxy_logger = logging.getLogger("LiteLLM Proxy")
|
|||
verbose_router_logger = logging.getLogger("LiteLLM Router")
|
||||
verbose_logger = logging.getLogger("LiteLLM")
|
||||
|
||||
# Add the handler to the logger
|
||||
# Add the handler to the loggers
|
||||
verbose_router_logger.addHandler(handler)
|
||||
verbose_proxy_logger.addHandler(handler)
|
||||
verbose_logger.addHandler(handler)
|
||||
|
|
|
|||
|
|
@ -199,8 +199,12 @@ def _get_redis_client_logic(**env_overrides):
|
|||
"REDIS_CLUSTER_NODES"
|
||||
)
|
||||
|
||||
# If startup_nodes resolved to None (not set by kwarg or env), remove the key
|
||||
# entirely so callers can rely on key presence as a reliable cluster-mode signal.
|
||||
if _startup_nodes is not None and isinstance(_startup_nodes, str):
|
||||
redis_kwargs["startup_nodes"] = json.loads(_startup_nodes)
|
||||
elif _startup_nodes is None:
|
||||
redis_kwargs.pop("startup_nodes", None)
|
||||
|
||||
_sentinel_nodes: Optional[Union[str, list]] = redis_kwargs.get("sentinel_nodes", None) or get_secret( # type: ignore
|
||||
"REDIS_SENTINEL_NODES"
|
||||
|
|
@ -250,10 +254,14 @@ def _get_redis_client_logic(**env_overrides):
|
|||
redis_kwargs["ssl_ca_certs"] = _gcp_ssl_ca_certs
|
||||
|
||||
if "url" in redis_kwargs and redis_kwargs["url"] is not None:
|
||||
redis_kwargs.pop("host", None)
|
||||
redis_kwargs.pop("port", None)
|
||||
redis_kwargs.pop("db", None)
|
||||
redis_kwargs.pop("password", None)
|
||||
# Only strip host/port/db/password when not routing to a cluster.
|
||||
# When startup_nodes is also present the cluster path takes priority and
|
||||
# needs the password for authentication.
|
||||
if not redis_kwargs.get("startup_nodes"):
|
||||
redis_kwargs.pop("host", None)
|
||||
redis_kwargs.pop("port", None)
|
||||
redis_kwargs.pop("db", None)
|
||||
redis_kwargs.pop("password", None)
|
||||
elif "startup_nodes" in redis_kwargs and redis_kwargs["startup_nodes"] is not None:
|
||||
pass
|
||||
elif (
|
||||
|
|
@ -345,6 +353,10 @@ def _init_async_redis_sentinel(redis_kwargs) -> async_redis.Redis:
|
|||
|
||||
def get_redis_client(**env_overrides):
|
||||
redis_kwargs = _get_redis_client_logic(**env_overrides)
|
||||
|
||||
if "startup_nodes" in redis_kwargs:
|
||||
return init_redis_cluster(redis_kwargs)
|
||||
|
||||
if "url" in redis_kwargs and redis_kwargs["url"] is not None:
|
||||
args = _get_redis_url_kwargs()
|
||||
url_kwargs = {}
|
||||
|
|
@ -354,9 +366,6 @@ def get_redis_client(**env_overrides):
|
|||
|
||||
return redis.Redis.from_url(**url_kwargs)
|
||||
|
||||
if "startup_nodes" in redis_kwargs or get_secret("REDIS_CLUSTER_NODES") is not None: # type: ignore
|
||||
return init_redis_cluster(redis_kwargs)
|
||||
|
||||
# Check for Redis Sentinel
|
||||
if "sentinel_nodes" in redis_kwargs and "service_name" in redis_kwargs:
|
||||
return _init_redis_sentinel(redis_kwargs)
|
||||
|
|
@ -369,21 +378,6 @@ def get_redis_async_client(
|
|||
**env_overrides,
|
||||
) -> Union[async_redis.Redis, async_redis.RedisCluster]:
|
||||
redis_kwargs = _get_redis_client_logic(**env_overrides)
|
||||
if "url" in redis_kwargs and redis_kwargs["url"] is not None:
|
||||
if connection_pool is not None:
|
||||
return async_redis.Redis(connection_pool=connection_pool)
|
||||
args = _get_redis_url_kwargs(client=async_redis.Redis.from_url)
|
||||
url_kwargs = {}
|
||||
for arg in redis_kwargs:
|
||||
if arg in args:
|
||||
url_kwargs[arg] = redis_kwargs[arg]
|
||||
else:
|
||||
verbose_logger.debug(
|
||||
"REDIS: ignoring argument: {}. Not an allowed async_redis.Redis.from_url arg.".format(
|
||||
arg
|
||||
)
|
||||
)
|
||||
return async_redis.Redis.from_url(**url_kwargs)
|
||||
|
||||
if "startup_nodes" in redis_kwargs:
|
||||
from redis.cluster import ClusterNode
|
||||
|
|
@ -418,6 +412,22 @@ def get_redis_async_client(
|
|||
|
||||
return cluster_client
|
||||
|
||||
if "url" in redis_kwargs and redis_kwargs["url"] is not None:
|
||||
if connection_pool is not None:
|
||||
return async_redis.Redis(connection_pool=connection_pool)
|
||||
args = _get_redis_url_kwargs(client=async_redis.Redis.from_url)
|
||||
url_kwargs = {}
|
||||
for arg in redis_kwargs:
|
||||
if arg in args:
|
||||
url_kwargs[arg] = redis_kwargs[arg]
|
||||
else:
|
||||
verbose_logger.debug(
|
||||
"REDIS: ignoring argument: {}. Not an allowed async_redis.Redis.from_url arg.".format(
|
||||
arg
|
||||
)
|
||||
)
|
||||
return async_redis.Redis.from_url(**url_kwargs)
|
||||
|
||||
# Check for Redis Sentinel
|
||||
if "sentinel_nodes" in redis_kwargs and "service_name" in redis_kwargs:
|
||||
return _init_async_redis_sentinel(redis_kwargs)
|
||||
|
|
@ -431,9 +441,15 @@ def get_redis_async_client(
|
|||
)
|
||||
|
||||
|
||||
def get_redis_connection_pool(**env_overrides):
|
||||
def get_redis_connection_pool(
|
||||
**env_overrides,
|
||||
) -> Optional[async_redis.BlockingConnectionPool]:
|
||||
redis_kwargs = _get_redis_client_logic(**env_overrides)
|
||||
verbose_logger.debug("get_redis_connection_pool: redis_kwargs", redis_kwargs)
|
||||
|
||||
if "startup_nodes" in redis_kwargs:
|
||||
return None
|
||||
|
||||
if "url" in redis_kwargs and redis_kwargs["url"] is not None:
|
||||
pool_kwargs = {
|
||||
"timeout": REDIS_CONNECTION_POOL_TIMEOUT,
|
||||
|
|
@ -453,7 +469,6 @@ def get_redis_connection_pool(**env_overrides):
|
|||
connection_class = async_redis.SSLConnection
|
||||
redis_kwargs.pop("ssl", None)
|
||||
redis_kwargs["connection_class"] = connection_class
|
||||
redis_kwargs.pop("startup_nodes", None)
|
||||
return async_redis.BlockingConnectionPool(
|
||||
timeout=REDIS_CONNECTION_POOL_TIMEOUT, **redis_kwargs
|
||||
)
|
||||
|
|
|
|||
|
|
@ -885,7 +885,7 @@ def list_batches(
|
|||
async def acancel_batch(
|
||||
batch_id: str,
|
||||
model: Optional[str] = None,
|
||||
custom_llm_provider: Literal["openai", "azure"] = "openai",
|
||||
custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
|
||||
metadata: Optional[Dict[str, str]] = None,
|
||||
extra_headers: Optional[Dict[str, str]] = None,
|
||||
extra_body: Optional[Dict[str, str]] = None,
|
||||
|
|
@ -931,7 +931,7 @@ async def acancel_batch(
|
|||
def cancel_batch(
|
||||
batch_id: str,
|
||||
model: Optional[str] = None,
|
||||
custom_llm_provider: Union[Literal["openai", "azure"], str] = "openai",
|
||||
custom_llm_provider: Union[Literal["openai", "azure", "vertex_ai"], str] = "openai",
|
||||
metadata: Optional[Dict[str, str]] = None,
|
||||
extra_headers: Optional[Dict[str, str]] = None,
|
||||
extra_body: Optional[Dict[str, str]] = None,
|
||||
|
|
@ -1048,9 +1048,35 @@ def cancel_batch(
|
|||
cancel_batch_data=_cancel_batch_request,
|
||||
litellm_params=litellm_params,
|
||||
)
|
||||
elif custom_llm_provider == "vertex_ai":
|
||||
api_base = optional_params.api_base or None
|
||||
vertex_ai_project = (
|
||||
optional_params.vertex_project
|
||||
or litellm.vertex_project
|
||||
or get_secret_str("VERTEXAI_PROJECT")
|
||||
)
|
||||
vertex_ai_location = (
|
||||
optional_params.vertex_location
|
||||
or litellm.vertex_location
|
||||
or get_secret_str("VERTEXAI_LOCATION")
|
||||
)
|
||||
vertex_credentials = optional_params.vertex_credentials or get_secret_str(
|
||||
"VERTEXAI_CREDENTIALS"
|
||||
)
|
||||
|
||||
response = vertex_ai_batches_instance.cancel_batch(
|
||||
_is_async=_is_async,
|
||||
batch_id=batch_id,
|
||||
api_base=api_base,
|
||||
vertex_project=vertex_ai_project,
|
||||
vertex_location=vertex_ai_location,
|
||||
vertex_credentials=vertex_credentials,
|
||||
timeout=timeout,
|
||||
max_retries=optional_params.max_retries,
|
||||
)
|
||||
else:
|
||||
raise litellm.exceptions.BadRequestError(
|
||||
message="LiteLLM doesn't support {} for 'cancel_batch'. Only 'openai' and 'azure' are supported.".format(
|
||||
message="LiteLLM doesn't support {} for 'cancel_batch'. Only 'openai', 'azure', and 'vertex_ai' are supported.".format(
|
||||
custom_llm_provider
|
||||
),
|
||||
model="n/a",
|
||||
|
|
|
|||
|
|
@ -393,16 +393,17 @@ class DualCache(BaseCache):
|
|||
parent_otel_span: Optional[Span] = None,
|
||||
local_only: bool = False,
|
||||
**kwargs,
|
||||
) -> float:
|
||||
) -> Optional[float]:
|
||||
"""
|
||||
Key - the key in cache
|
||||
|
||||
Value - float - the value you want to increment by
|
||||
|
||||
Returns - float - the incremented value
|
||||
Returns - the incremented value, or None if no cache backend is
|
||||
available (in_memory_cache is None and Redis failed/is absent).
|
||||
"""
|
||||
result: Optional[float] = None
|
||||
try:
|
||||
result: float = value
|
||||
if self.in_memory_cache is not None:
|
||||
result = await self.in_memory_cache.async_increment(
|
||||
key, value, **kwargs
|
||||
|
|
@ -418,7 +419,11 @@ class DualCache(BaseCache):
|
|||
|
||||
return result
|
||||
except Exception as e:
|
||||
raise e # don't log if exception is raised
|
||||
verbose_logger.warning(
|
||||
"Redis async_increment_cache failed, falling back to in-memory result: %s",
|
||||
e,
|
||||
)
|
||||
return result
|
||||
|
||||
async def async_increment_cache_pipeline(
|
||||
self,
|
||||
|
|
@ -427,8 +432,8 @@ class DualCache(BaseCache):
|
|||
parent_otel_span: Optional[Span] = None,
|
||||
**kwargs,
|
||||
) -> Optional[List[float]]:
|
||||
result: Optional[List[float]] = None
|
||||
try:
|
||||
result: Optional[List[float]] = None
|
||||
if self.in_memory_cache is not None:
|
||||
result = await self.in_memory_cache.async_increment_pipeline(
|
||||
increment_list=increment_list,
|
||||
|
|
@ -443,7 +448,11 @@ class DualCache(BaseCache):
|
|||
|
||||
return result
|
||||
except Exception as e:
|
||||
raise e # don't log if exception is raised
|
||||
verbose_logger.warning(
|
||||
"Redis async_increment_cache_pipeline failed, falling back to in-memory result: %s",
|
||||
e,
|
||||
)
|
||||
return result
|
||||
|
||||
async def async_set_cache_sadd(
|
||||
self, key, value: List, local_only: bool = False, **kwargs
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ Has 4 primary methods:
|
|||
|
||||
import ast
|
||||
import asyncio
|
||||
import functools
|
||||
import hashlib
|
||||
import inspect
|
||||
import json
|
||||
|
|
@ -19,7 +20,11 @@ from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Union, cast
|
|||
|
||||
import litellm
|
||||
from litellm._logging import print_verbose, verbose_logger
|
||||
from litellm.constants import DEFAULT_REDIS_MAJOR_VERSION
|
||||
from litellm.constants import (
|
||||
DEFAULT_REDIS_MAJOR_VERSION,
|
||||
REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD,
|
||||
REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import _get_parent_otel_span_from_kwargs
|
||||
from litellm.litellm_core_utils.coroutine_checker import coroutine_checker
|
||||
from litellm.types.caching import (
|
||||
|
|
@ -89,6 +94,91 @@ def _get_call_stack_info(num_frames: int = 2) -> str:
|
|||
return "unknown"
|
||||
|
||||
|
||||
class RedisCircuitBreaker:
|
||||
"""
|
||||
Tracks Redis health for a RedisCache instance.
|
||||
|
||||
States:
|
||||
CLOSED - normal, Redis is called
|
||||
OPEN - Redis is down, raise immediately (no network call)
|
||||
HALF_OPEN - recovery probe: allow one request through
|
||||
|
||||
Transitions:
|
||||
CLOSED -> OPEN after failure_threshold consecutive failures
|
||||
OPEN -> HALF_OPEN after recovery_timeout seconds
|
||||
HALF_OPEN -> CLOSED on success
|
||||
HALF_OPEN -> OPEN on failure (resets timer)
|
||||
"""
|
||||
|
||||
CLOSED = "closed"
|
||||
OPEN = "open"
|
||||
HALF_OPEN = "half_open"
|
||||
|
||||
def __init__(self, failure_threshold: int, recovery_timeout: int) -> None:
|
||||
self.failure_threshold = failure_threshold
|
||||
self.recovery_timeout = recovery_timeout
|
||||
self._failure_count = 0
|
||||
self._opened_at: Optional[float] = None
|
||||
self._state = self.CLOSED
|
||||
|
||||
def is_open(self) -> bool:
|
||||
"""Returns True if Redis calls should be skipped."""
|
||||
if self._state == self.HALF_OPEN:
|
||||
# Probe already in flight — fast-fail all concurrent requests.
|
||||
# Only the one call that caused the OPEN→HALF_OPEN transition
|
||||
# (which returned False) is the designated probe.
|
||||
return True
|
||||
if self._state == self.OPEN:
|
||||
if time.time() - (self._opened_at or 0) > self.recovery_timeout:
|
||||
self._state = self.HALF_OPEN
|
||||
return False # this caller is the designated probe
|
||||
return True
|
||||
return False
|
||||
|
||||
def record_failure(self) -> None:
|
||||
self._failure_count += 1
|
||||
self._opened_at = time.time()
|
||||
if self._failure_count >= self.failure_threshold:
|
||||
if self._state != self.OPEN:
|
||||
verbose_logger.warning(
|
||||
"Redis circuit breaker OPENED after %d consecutive failures — "
|
||||
"fast-failing Redis calls for %ds",
|
||||
self._failure_count,
|
||||
self.recovery_timeout,
|
||||
)
|
||||
self._state = self.OPEN
|
||||
|
||||
def record_success(self) -> None:
|
||||
if self._state == self.HALF_OPEN:
|
||||
verbose_logger.info("Redis circuit breaker CLOSED — Redis recovered")
|
||||
self._failure_count = 0
|
||||
self._state = self.CLOSED
|
||||
|
||||
|
||||
def _redis_circuit_breaker_guard(method): # type: ignore
|
||||
"""
|
||||
Decorator for RedisCache async methods.
|
||||
Checks the circuit breaker before each call; records success/failure after.
|
||||
Does not apply to ping/disconnect/test_connection (health/teardown must always run).
|
||||
"""
|
||||
|
||||
@functools.wraps(method)
|
||||
async def wrapper(self, *args, **kwargs): # type: ignore
|
||||
if self._circuit_breaker.is_open():
|
||||
raise Exception(
|
||||
f"Redis circuit breaker is open — skipping {method.__name__}"
|
||||
)
|
||||
try:
|
||||
result = await method(self, *args, **kwargs)
|
||||
self._circuit_breaker.record_success()
|
||||
return result
|
||||
except Exception:
|
||||
self._circuit_breaker.record_failure()
|
||||
raise
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
class RedisCache(BaseCache):
|
||||
# if users don't provider one, use the default litellm cache
|
||||
|
||||
|
|
@ -150,6 +240,11 @@ class RedisCache(BaseCache):
|
|||
except Exception:
|
||||
pass
|
||||
|
||||
self._circuit_breaker = RedisCircuitBreaker(
|
||||
failure_threshold=REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD,
|
||||
recovery_timeout=REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT,
|
||||
)
|
||||
|
||||
self._setup_health_pings()
|
||||
|
||||
if litellm.default_redis_ttl is not None:
|
||||
|
|
@ -375,6 +470,7 @@ class RedisCache(BaseCache):
|
|||
)
|
||||
raise e
|
||||
|
||||
@_redis_circuit_breaker_guard
|
||||
async def async_scan_iter(self, pattern: str, count: int = 100) -> list:
|
||||
start_time = time.time()
|
||||
try:
|
||||
|
|
@ -451,6 +547,7 @@ class RedisCache(BaseCache):
|
|||
verbose_logger.error(f"Error registering Redis script: {str(e)}")
|
||||
raise e
|
||||
|
||||
@_redis_circuit_breaker_guard
|
||||
async def async_set_cache(self, key, value, **kwargs):
|
||||
from redis.asyncio import Redis
|
||||
|
||||
|
|
@ -560,6 +657,7 @@ class RedisCache(BaseCache):
|
|||
results = await pipe.execute()
|
||||
return results
|
||||
|
||||
@_redis_circuit_breaker_guard
|
||||
async def async_set_cache_pipeline(
|
||||
self, cache_list: List[Tuple[Any, Any]], ttl: Optional[float] = None, **kwargs
|
||||
):
|
||||
|
|
@ -636,6 +734,7 @@ class RedisCache(BaseCache):
|
|||
except Exception:
|
||||
raise
|
||||
|
||||
@_redis_circuit_breaker_guard
|
||||
async def async_set_cache_sadd(
|
||||
self, key, value: List, ttl: Optional[float], **kwargs
|
||||
):
|
||||
|
|
@ -708,6 +807,7 @@ class RedisCache(BaseCache):
|
|||
value,
|
||||
)
|
||||
|
||||
@_redis_circuit_breaker_guard
|
||||
async def batch_cache_write(self, key, value, **kwargs):
|
||||
print_verbose(
|
||||
f"in batch cache writing for redis buffer size={len(self.redis_batch_writing_buffer)}",
|
||||
|
|
@ -717,6 +817,7 @@ class RedisCache(BaseCache):
|
|||
if len(self.redis_batch_writing_buffer) >= self.redis_flush_size:
|
||||
await self.flush_cache_buffer() # logging done in here
|
||||
|
||||
@_redis_circuit_breaker_guard
|
||||
async def async_increment(
|
||||
self,
|
||||
key,
|
||||
|
|
@ -894,6 +995,7 @@ class RedisCache(BaseCache):
|
|||
verbose_logger.error(f"Error occurred in batch get cache - {str(e)}")
|
||||
return key_value_dict
|
||||
|
||||
@_redis_circuit_breaker_guard
|
||||
async def async_get_cache(
|
||||
self, key, parent_otel_span: Optional[Span] = None, **kwargs
|
||||
):
|
||||
|
|
@ -944,6 +1046,7 @@ class RedisCache(BaseCache):
|
|||
f"litellm.caching.caching: async get() - Got exception from REDIS: {str(e)}"
|
||||
)
|
||||
|
||||
@_redis_circuit_breaker_guard
|
||||
async def async_batch_get_cache(
|
||||
self,
|
||||
key_list: Union[List[str], List[Optional[str]]],
|
||||
|
|
@ -1087,6 +1190,7 @@ class RedisCache(BaseCache):
|
|||
)
|
||||
raise e
|
||||
|
||||
@_redis_circuit_breaker_guard
|
||||
async def delete_cache_keys(self, keys):
|
||||
# typed as Any, redis python lib has incomplete type stubs for RedisCluster and does not include `delete`
|
||||
_redis_client: Any = self.init_async_client()
|
||||
|
|
@ -1151,6 +1255,7 @@ class RedisCache(BaseCache):
|
|||
"error": str(e),
|
||||
}
|
||||
|
||||
@_redis_circuit_breaker_guard
|
||||
async def async_delete_cache(self, key: str):
|
||||
# typed as Any, redis python lib has incomplete type stubs for RedisCluster and does not include `delete`
|
||||
_redis_client: Any = self.init_async_client()
|
||||
|
|
@ -1184,6 +1289,7 @@ class RedisCache(BaseCache):
|
|||
)
|
||||
return [r for r in results if isinstance(r, float)]
|
||||
|
||||
@_redis_circuit_breaker_guard
|
||||
async def async_increment_pipeline(
|
||||
self, increment_list: List[RedisPipelineIncrementOperation], **kwargs
|
||||
) -> Optional[List[float]]:
|
||||
|
|
@ -1247,6 +1353,7 @@ class RedisCache(BaseCache):
|
|||
)
|
||||
raise e
|
||||
|
||||
@_redis_circuit_breaker_guard
|
||||
async def async_get_ttl(self, key: str) -> Optional[int]:
|
||||
"""
|
||||
Get the remaining TTL of a key in Redis
|
||||
|
|
@ -1270,6 +1377,7 @@ class RedisCache(BaseCache):
|
|||
verbose_logger.debug(f"Redis TTL Error: {e}")
|
||||
return None
|
||||
|
||||
@_redis_circuit_breaker_guard
|
||||
async def async_rpush(
|
||||
self,
|
||||
key: str,
|
||||
|
|
@ -1336,6 +1444,7 @@ class RedisCache(BaseCache):
|
|||
raise r
|
||||
return results
|
||||
|
||||
@_redis_circuit_breaker_guard
|
||||
async def async_rpush_pipeline(
|
||||
self,
|
||||
rpush_list: List[RedisPipelineRpushOperation],
|
||||
|
|
@ -1405,6 +1514,7 @@ class RedisCache(BaseCache):
|
|||
|
||||
return result
|
||||
|
||||
@_redis_circuit_breaker_guard
|
||||
async def async_lpop(
|
||||
self,
|
||||
key: str,
|
||||
|
|
@ -1534,6 +1644,7 @@ class RedisCache(BaseCache):
|
|||
decoded_results.append(None)
|
||||
return decoded_results
|
||||
|
||||
@_redis_circuit_breaker_guard
|
||||
async def async_lpop_pipeline(
|
||||
self,
|
||||
lpop_list: List[RedisPipelineLpopOperation],
|
||||
|
|
|
|||
|
|
@ -696,6 +696,20 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
verbose_logger.debug(
|
||||
f"Chat provider: image -> {converted}"
|
||||
)
|
||||
elif item_type == "file":
|
||||
# Map Chat Completion file to Responses API input_file
|
||||
# {"type": "file", "file": {"file_data": "...", "filename": "..."}}
|
||||
# -> {"type": "input_file", "file_data": "...", "filename": "..."}
|
||||
file_data = item.get("file", {})
|
||||
converted = {"type": "input_file"}
|
||||
if isinstance(file_data, dict):
|
||||
for key in ["file_id", "file_data", "filename"]:
|
||||
if key in file_data:
|
||||
converted[key] = file_data[key]
|
||||
result.append(converted)
|
||||
verbose_logger.debug(
|
||||
f"Chat provider: file -> {converted}"
|
||||
)
|
||||
elif item_type in [
|
||||
"input_text",
|
||||
"input_image",
|
||||
|
|
|
|||
|
|
@ -350,6 +350,12 @@ AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI = int(
|
|||
)
|
||||
REDIS_SOCKET_TIMEOUT = float(os.getenv("REDIS_SOCKET_TIMEOUT", 0.1))
|
||||
REDIS_CONNECTION_POOL_TIMEOUT = int(os.getenv("REDIS_CONNECTION_POOL_TIMEOUT", 5))
|
||||
REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD = int(
|
||||
os.getenv("REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD", 5)
|
||||
)
|
||||
REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT = int(
|
||||
os.getenv("REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT", 60)
|
||||
)
|
||||
# Default Redis major version to assume when version cannot be determined
|
||||
# Using 7 as it's the modern version that supports LPOP with count parameter
|
||||
DEFAULT_REDIS_MAJOR_VERSION = int(os.getenv("DEFAULT_REDIS_MAJOR_VERSION", 7))
|
||||
|
|
|
|||
|
|
@ -60,13 +60,20 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
# Create a deep copy of messages to avoid modifying the original list
|
||||
processed_messages = copy.deepcopy(messages)
|
||||
|
||||
# Process message-level cache controls
|
||||
# Separate message-level and non-message-level injection points
|
||||
remaining_points = []
|
||||
for point in injection_points:
|
||||
if point.get("location") == "message":
|
||||
point = cast(CacheControlMessageInjectionPoint, point)
|
||||
processed_messages = self._process_message_injection(
|
||||
point=point, messages=processed_messages
|
||||
)
|
||||
else:
|
||||
remaining_points.append(point)
|
||||
|
||||
# Pass through non-message injection points for provider-specific handling
|
||||
if remaining_points:
|
||||
non_default_params["cache_control_injection_points"] = remaining_points
|
||||
|
||||
return model, processed_messages, non_default_params
|
||||
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ from litellm.types.utils import (
|
|||
LLMResponseTypes,
|
||||
ModelResponse,
|
||||
ModelResponseStream,
|
||||
StandardAuditLogPayload,
|
||||
StandardCallbackDynamicParams,
|
||||
StandardLoggingPayload,
|
||||
)
|
||||
|
|
@ -177,6 +178,10 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
|
|||
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
|
||||
pass
|
||||
|
||||
async def async_log_audit_log_event(self, audit_log: "StandardAuditLogPayload"):
|
||||
"""Called when an audit log is created. Override in subclasses to handle."""
|
||||
pass
|
||||
|
||||
#### PROMPT MANAGEMENT HOOKS ####
|
||||
|
||||
async def async_get_chat_completion_prompt(
|
||||
|
|
|
|||
|
|
@ -83,7 +83,28 @@ class LangsmithLogger(CustomBatchLogger):
|
|||
if _batch_size:
|
||||
self.batch_size = int(_batch_size)
|
||||
self.log_queue: List[LangsmithQueueObject] = []
|
||||
asyncio.create_task(self.periodic_flush())
|
||||
self._flush_task: Optional[
|
||||
asyncio.Task[Any]
|
||||
] = self._start_periodic_flush_task()
|
||||
|
||||
def _start_periodic_flush_task(self) -> Optional[asyncio.Task[Any]]:
|
||||
"""Start the periodic flush task only when an event loop is already running."""
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
verbose_logger.debug(
|
||||
"Langsmith logger init: no running event loop, skipping periodic flush task startup"
|
||||
)
|
||||
return None
|
||||
|
||||
return loop.create_task(self.periodic_flush())
|
||||
|
||||
def _ensure_periodic_flush_task(self) -> None:
|
||||
# This helper is intentionally synchronous. In asyncio's cooperative
|
||||
# execution model, there is no await between the check and assignment,
|
||||
# so one caller cannot interleave here and create a duplicate task.
|
||||
if self._flush_task is None or self._flush_task.done():
|
||||
self._flush_task = self._start_periodic_flush_task()
|
||||
|
||||
def get_credentials_from_env(
|
||||
self,
|
||||
|
|
@ -266,6 +287,7 @@ class LangsmithLogger(CustomBatchLogger):
|
|||
|
||||
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
|
||||
try:
|
||||
self._ensure_periodic_flush_task()
|
||||
sampling_rate = self._get_sampling_rate_to_use_for_request(kwargs=kwargs)
|
||||
random_sample = random.random()
|
||||
if random_sample > sampling_rate:
|
||||
|
|
@ -307,17 +329,18 @@ class LangsmithLogger(CustomBatchLogger):
|
|||
)
|
||||
|
||||
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
|
||||
sampling_rate = self._get_sampling_rate_to_use_for_request(kwargs=kwargs)
|
||||
random_sample = random.random()
|
||||
if random_sample > sampling_rate:
|
||||
verbose_logger.info(
|
||||
"Skipping Langsmith logging. Sampling rate={}, random_sample={}".format(
|
||||
sampling_rate, random_sample
|
||||
)
|
||||
)
|
||||
return # Skip logging
|
||||
verbose_logger.info("Langsmith Failure Event Logging!")
|
||||
try:
|
||||
self._ensure_periodic_flush_task()
|
||||
sampling_rate = self._get_sampling_rate_to_use_for_request(kwargs=kwargs)
|
||||
random_sample = random.random()
|
||||
if random_sample > sampling_rate:
|
||||
verbose_logger.info(
|
||||
"Skipping Langsmith logging. Sampling rate={}, random_sample={}".format(
|
||||
sampling_rate, random_sample
|
||||
)
|
||||
)
|
||||
return # Skip logging
|
||||
verbose_logger.info("Langsmith Failure Event Logging!")
|
||||
credentials = self._get_credentials_to_use_for_request(kwargs=kwargs)
|
||||
data = self._prepare_log_data(
|
||||
kwargs=kwargs,
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ from litellm.llms.custom_httpx.http_handler import (
|
|||
httpxSpecialProvider,
|
||||
)
|
||||
from litellm.types.integrations.s3_v2 import s3BatchLoggingElement
|
||||
from litellm.types.utils import StandardLoggingPayload
|
||||
from litellm.types.utils import StandardAuditLogPayload, StandardLoggingPayload
|
||||
|
||||
from .custom_batch_logger import CustomBatchLogger
|
||||
|
||||
|
|
@ -248,6 +248,38 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
|
|||
)
|
||||
pass
|
||||
|
||||
async def async_log_audit_log_event(
|
||||
self, audit_log: StandardAuditLogPayload
|
||||
) -> None:
|
||||
"""Batch audit logs and upload to S3 under audit_logs/ prefix."""
|
||||
try:
|
||||
from datetime import timezone
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
audit_log_id = audit_log.get("id", "unknown")
|
||||
|
||||
s3_path = cast(Optional[str], self.s3_path) or ""
|
||||
s3_path = s3_path.rstrip("/") + "/" if s3_path else ""
|
||||
|
||||
s3_object_key = (
|
||||
f"{s3_path}audit_logs/"
|
||||
f"{now.strftime('%Y-%m-%d')}/"
|
||||
f"{now.strftime('%H-%M-%S')}_{audit_log_id}.json"
|
||||
)
|
||||
|
||||
element = s3BatchLoggingElement(
|
||||
payload=dict(audit_log),
|
||||
s3_object_key=s3_object_key,
|
||||
s3_object_download_filename=f"audit-{audit_log_id}.json",
|
||||
)
|
||||
|
||||
self.log_queue.append(element)
|
||||
|
||||
if len(self.log_queue) >= self.batch_size:
|
||||
await self.flush_queue()
|
||||
except Exception as e:
|
||||
verbose_logger.exception("S3 audit log error: %s", e)
|
||||
|
||||
async def _async_log_event_base(self, kwargs, response_obj, start_time, end_time):
|
||||
try:
|
||||
verbose_logger.debug(
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ server-side using litellm router's search tools.
|
|||
|
||||
import asyncio
|
||||
import math
|
||||
import uuid
|
||||
from typing import Any, Dict, List, Optional, Tuple, Union, cast
|
||||
|
||||
import litellm
|
||||
|
|
@ -27,7 +28,9 @@ from litellm.integrations.websearch_interception.transformation import (
|
|||
from litellm.types.integrations.websearch_interception import (
|
||||
WebSearchInterceptionConfig,
|
||||
)
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.utils import LlmProviders
|
||||
from litellm.utils import ProviderConfigManager
|
||||
|
||||
|
||||
class WebSearchInterceptionLogger(CustomLogger):
|
||||
|
|
@ -67,6 +70,111 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
self.search_tool_name = search_tool_name
|
||||
self._request_has_websearch = False # Track if current request has web search
|
||||
|
||||
async def try_short_circuit_search(
|
||||
self,
|
||||
model: str,
|
||||
messages: List[Dict],
|
||||
tools: Optional[List[Dict]],
|
||||
custom_llm_provider: Optional[str],
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Short-circuit web-search-only requests by executing the search directly.
|
||||
|
||||
Claude Code sends web search as a separate, standalone /v1/messages
|
||||
request with a simple prompt and only web_search tool(s). For providers
|
||||
that don't natively support web search (e.g. github_copilot), there is
|
||||
no need to route this through the backend LLM — we can detect the
|
||||
pattern, execute the search via Tavily/Perplexity, and return a
|
||||
synthetic Anthropic response immediately.
|
||||
|
||||
Args:
|
||||
model: Model name from the request
|
||||
messages: Messages list from the request
|
||||
tools: Tools list from the request
|
||||
custom_llm_provider: Provider name
|
||||
|
||||
Returns:
|
||||
An AnthropicMessagesResponse dict if short-circuited, or None to
|
||||
continue normal processing.
|
||||
"""
|
||||
if not tools:
|
||||
return None
|
||||
|
||||
# Check if provider is in enabled list
|
||||
provider_str = custom_llm_provider or ""
|
||||
if (
|
||||
self.enabled_providers is not None
|
||||
and provider_str not in self.enabled_providers
|
||||
):
|
||||
return None
|
||||
|
||||
# Only short-circuit for providers without native Anthropic Messages
|
||||
# support. Providers that have a BaseAnthropicMessagesConfig (bedrock,
|
||||
# vertex_ai, azure_ai, anthropic) already use the agentic loop, which
|
||||
# includes a follow-up LLM call to synthesize the answer from search
|
||||
# results. Short-circuiting those would skip that synthesis step and
|
||||
# return raw search text — a regression for existing users.
|
||||
try:
|
||||
provider_enum = LlmProviders(provider_str)
|
||||
anthropic_config = (
|
||||
ProviderConfigManager.get_provider_anthropic_messages_config(
|
||||
model=model, provider=provider_enum
|
||||
)
|
||||
)
|
||||
if anthropic_config is not None:
|
||||
verbose_logger.debug(
|
||||
f"WebSearchInterception: Skipping short-circuit for {provider_str} "
|
||||
"(provider has native Anthropic Messages support, using agentic loop)"
|
||||
)
|
||||
return None
|
||||
except (ValueError, Exception):
|
||||
pass # unknown provider enum → safe to short-circuit
|
||||
|
||||
# All tools must be web search tools
|
||||
if not all(is_web_search_tool(t) for t in tools):
|
||||
return None
|
||||
|
||||
# Extract search query from the last user message
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
get_last_user_message,
|
||||
)
|
||||
|
||||
query = get_last_user_message(cast(List[AllMessageValues], messages))
|
||||
if not query:
|
||||
return None
|
||||
|
||||
verbose_logger.debug(
|
||||
"WebSearchInterception: Short-circuit search detected "
|
||||
f"(provider={provider_str}, query='{query}')"
|
||||
)
|
||||
|
||||
# Execute search
|
||||
try:
|
||||
search_result_text = await self._execute_search(query)
|
||||
except Exception as e:
|
||||
verbose_logger.error(
|
||||
f"WebSearchInterception: Short-circuit search failed: {e}"
|
||||
)
|
||||
search_result_text = f"Search failed: {e}"
|
||||
|
||||
# Build synthetic Anthropic response
|
||||
response: Dict[str, Any] = {
|
||||
"id": f"msg_{str(uuid.uuid4())}",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": model,
|
||||
"content": [{"type": "text", "text": search_result_text}],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 0, "output_tokens": 0},
|
||||
}
|
||||
|
||||
verbose_logger.debug(
|
||||
"WebSearchInterception: Short-circuit search completed, "
|
||||
f"returning synthetic response ({len(search_result_text)} chars)"
|
||||
)
|
||||
return response
|
||||
|
||||
async def async_pre_call_deployment_hook(
|
||||
self, kwargs: Dict[str, Any], call_type: Optional[Any]
|
||||
) -> Optional[dict]:
|
||||
|
|
|
|||
|
|
@ -64,6 +64,7 @@ _FINISH_REASON_MAP: dict[str, OpenAIChatCompletionFinishReason] = {
|
|||
"end_turn": "stop",
|
||||
"max_tokens": "length",
|
||||
"tool_use": "tool_calls",
|
||||
"refusal": "content_filter",
|
||||
"compaction": "length",
|
||||
# Cohere
|
||||
"COMPLETE": "stop",
|
||||
|
|
|
|||
|
|
@ -354,9 +354,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
)
|
||||
self.function_id = function_id
|
||||
self.streaming_chunks: List[Any] = [] # for generating complete stream response
|
||||
self.sync_streaming_chunks: List[
|
||||
Any
|
||||
] = [] # for generating complete stream response
|
||||
self.sync_streaming_chunks: List[Any] = (
|
||||
[]
|
||||
) # for generating complete stream response
|
||||
self.log_raw_request_response = log_raw_request_response
|
||||
|
||||
# Initialize dynamic callbacks
|
||||
|
|
@ -801,9 +801,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
prompt_spec=prompt_spec,
|
||||
dynamic_callback_params=dynamic_callback_params,
|
||||
):
|
||||
self.model_call_details[
|
||||
"prompt_integration"
|
||||
] = logger.__class__.__name__
|
||||
self.model_call_details["prompt_integration"] = (
|
||||
logger.__class__.__name__
|
||||
)
|
||||
return logger
|
||||
except Exception:
|
||||
# If check fails, continue to next logger
|
||||
|
|
@ -871,9 +871,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
if anthropic_cache_control_logger := AnthropicCacheControlHook.get_custom_logger_for_anthropic_cache_control_hook(
|
||||
non_default_params
|
||||
):
|
||||
self.model_call_details[
|
||||
"prompt_integration"
|
||||
] = anthropic_cache_control_logger.__class__.__name__
|
||||
self.model_call_details["prompt_integration"] = (
|
||||
anthropic_cache_control_logger.__class__.__name__
|
||||
)
|
||||
return anthropic_cache_control_logger
|
||||
|
||||
#########################################################
|
||||
|
|
@ -885,9 +885,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
internal_usage_cache=None,
|
||||
llm_router=None,
|
||||
)
|
||||
self.model_call_details[
|
||||
"prompt_integration"
|
||||
] = vector_store_custom_logger.__class__.__name__
|
||||
self.model_call_details["prompt_integration"] = (
|
||||
vector_store_custom_logger.__class__.__name__
|
||||
)
|
||||
# Add to global callbacks so post-call hooks are invoked
|
||||
if (
|
||||
vector_store_custom_logger
|
||||
|
|
@ -947,9 +947,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
model
|
||||
): # if model name was changes pre-call, overwrite the initial model call name with the new one
|
||||
self.model_call_details["model"] = model
|
||||
self.model_call_details["litellm_params"][
|
||||
"api_base"
|
||||
] = self._get_masked_api_base(additional_args.get("api_base", ""))
|
||||
self.model_call_details["litellm_params"]["api_base"] = (
|
||||
self._get_masked_api_base(additional_args.get("api_base", ""))
|
||||
)
|
||||
|
||||
def pre_call(self, input, api_key, model=None, additional_args={}): # noqa: PLR0915
|
||||
# Log the exact input to the LLM API
|
||||
|
|
@ -978,9 +978,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
try:
|
||||
# [Non-blocking Extra Debug Information in metadata]
|
||||
if turn_off_message_logging is True:
|
||||
_metadata[
|
||||
"raw_request"
|
||||
] = "redacted by litellm. \
|
||||
_metadata["raw_request"] = "redacted by litellm. \
|
||||
'litellm.turn_off_message_logging=True'"
|
||||
else:
|
||||
curl_command = self._get_request_curl_command(
|
||||
|
|
@ -992,35 +990,31 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
|
||||
_metadata["raw_request"] = str(curl_command)
|
||||
# split up, so it's easier to parse in the UI
|
||||
self.model_call_details[
|
||||
"raw_request_typed_dict"
|
||||
] = RawRequestTypedDict(
|
||||
raw_request_api_base=str(
|
||||
additional_args.get("api_base") or ""
|
||||
),
|
||||
raw_request_body=self._get_raw_request_body(
|
||||
additional_args.get("complete_input_dict", {})
|
||||
),
|
||||
# NOTE: setting ignore_sensitive_headers to True will cause
|
||||
# the Authorization header to be leaked when calls to the health
|
||||
# endpoint are made and fail.
|
||||
raw_request_headers=self._get_masked_headers(
|
||||
additional_args.get("headers", {}) or {},
|
||||
),
|
||||
error=None,
|
||||
self.model_call_details["raw_request_typed_dict"] = (
|
||||
RawRequestTypedDict(
|
||||
raw_request_api_base=str(
|
||||
additional_args.get("api_base") or ""
|
||||
),
|
||||
raw_request_body=self._get_raw_request_body(
|
||||
additional_args.get("complete_input_dict", {})
|
||||
),
|
||||
# NOTE: setting ignore_sensitive_headers to True will cause
|
||||
# the Authorization header to be leaked when calls to the health
|
||||
# endpoint are made and fail.
|
||||
raw_request_headers=self._get_masked_headers(
|
||||
additional_args.get("headers", {}) or {},
|
||||
),
|
||||
error=None,
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
self.model_call_details[
|
||||
"raw_request_typed_dict"
|
||||
] = RawRequestTypedDict(
|
||||
error=str(e),
|
||||
)
|
||||
_metadata[
|
||||
"raw_request"
|
||||
] = "Unable to Log \
|
||||
raw request: {}".format(
|
||||
str(e)
|
||||
self.model_call_details["raw_request_typed_dict"] = (
|
||||
RawRequestTypedDict(
|
||||
error=str(e),
|
||||
)
|
||||
)
|
||||
_metadata["raw_request"] = "Unable to Log \
|
||||
raw request: {}".format(str(e))
|
||||
if getattr(self, "logger_fn", None) and callable(self.logger_fn):
|
||||
try:
|
||||
self.logger_fn(
|
||||
|
|
@ -1320,13 +1314,13 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
for callback in callbacks:
|
||||
try:
|
||||
if isinstance(callback, CustomLogger):
|
||||
response: Optional[
|
||||
MCPPostCallResponseObject
|
||||
] = await callback.async_post_mcp_tool_call_hook(
|
||||
kwargs=kwargs,
|
||||
response_obj=post_mcp_tool_call_response_obj,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
response: Optional[MCPPostCallResponseObject] = (
|
||||
await callback.async_post_mcp_tool_call_hook(
|
||||
kwargs=kwargs,
|
||||
response_obj=post_mcp_tool_call_response_obj,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
)
|
||||
)
|
||||
######################################################################
|
||||
# if any of the callbacks modify the response, use the modified response
|
||||
|
|
@ -1527,9 +1521,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
verbose_logger.debug(
|
||||
f"response_cost_failure_debug_information: {debug_info}"
|
||||
)
|
||||
self.model_call_details[
|
||||
"response_cost_failure_debug_information"
|
||||
] = debug_info
|
||||
self.model_call_details["response_cost_failure_debug_information"] = (
|
||||
debug_info
|
||||
)
|
||||
return None
|
||||
|
||||
try:
|
||||
|
|
@ -1555,9 +1549,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
verbose_logger.debug(
|
||||
f"response_cost_failure_debug_information: {debug_info}"
|
||||
)
|
||||
self.model_call_details[
|
||||
"response_cost_failure_debug_information"
|
||||
] = debug_info
|
||||
self.model_call_details["response_cost_failure_debug_information"] = (
|
||||
debug_info
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
|
|
@ -1686,6 +1680,30 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
)
|
||||
return logging_result
|
||||
|
||||
def _merge_hidden_params_from_response_into_metadata(
|
||||
self, logging_result: Any
|
||||
) -> None:
|
||||
"""
|
||||
Copy response._hidden_params into litellm_params.metadata['hidden_params'].
|
||||
|
||||
Non-streaming success uses _process_hidden_params_and_response_cost (skipped when
|
||||
stream=True). Streaming assembles the full response later; without this merge,
|
||||
OTEL/callbacks that read metadata.hidden_params miss cost-related fields.
|
||||
"""
|
||||
if logging_result is None:
|
||||
return
|
||||
hidden_params = getattr(logging_result, "_hidden_params", None)
|
||||
if not hidden_params:
|
||||
return
|
||||
if self.model_call_details.get("litellm_params") is None:
|
||||
return
|
||||
self.model_call_details["litellm_params"].setdefault("metadata", {})
|
||||
if self.model_call_details["litellm_params"]["metadata"] is None:
|
||||
self.model_call_details["litellm_params"]["metadata"] = {}
|
||||
self.model_call_details["litellm_params"]["metadata"]["hidden_params"] = (
|
||||
getattr(logging_result, "_hidden_params", {})
|
||||
)
|
||||
|
||||
def _process_hidden_params_and_response_cost(
|
||||
self,
|
||||
logging_result,
|
||||
|
|
@ -1713,9 +1731,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
result=logging_result
|
||||
)
|
||||
|
||||
self.model_call_details[
|
||||
"standard_logging_object"
|
||||
] = self._build_standard_logging_payload(logging_result, start_time, end_time)
|
||||
self.model_call_details["standard_logging_object"] = (
|
||||
self._build_standard_logging_payload(logging_result, start_time, end_time)
|
||||
)
|
||||
|
||||
if (
|
||||
standard_logging_payload := self.model_call_details.get(
|
||||
|
|
@ -1793,9 +1811,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
end_time = datetime.datetime.now()
|
||||
if self.completion_start_time is None:
|
||||
self.completion_start_time = end_time
|
||||
self.model_call_details[
|
||||
"completion_start_time"
|
||||
] = self.completion_start_time
|
||||
self.model_call_details["completion_start_time"] = (
|
||||
self.completion_start_time
|
||||
)
|
||||
|
||||
self.model_call_details["log_event_type"] = "successful_api_call"
|
||||
self.model_call_details["end_time"] = end_time
|
||||
|
|
@ -1832,10 +1850,10 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
end_time=end_time,
|
||||
)
|
||||
elif isinstance(result, dict) or isinstance(result, list):
|
||||
self.model_call_details[
|
||||
"standard_logging_object"
|
||||
] = self._build_standard_logging_payload(
|
||||
result, start_time, end_time
|
||||
self.model_call_details["standard_logging_object"] = (
|
||||
self._build_standard_logging_payload(
|
||||
result, start_time, end_time
|
||||
)
|
||||
)
|
||||
if (
|
||||
standard_logging_payload := self.model_call_details.get(
|
||||
|
|
@ -1844,9 +1862,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
) is not None:
|
||||
emit_standard_logging_payload(standard_logging_payload)
|
||||
elif standard_logging_object is not None:
|
||||
self.model_call_details[
|
||||
"standard_logging_object"
|
||||
] = standard_logging_object
|
||||
self.model_call_details["standard_logging_object"] = (
|
||||
standard_logging_object
|
||||
)
|
||||
else:
|
||||
self.model_call_details["response_cost"] = None
|
||||
|
||||
|
|
@ -2004,17 +2022,20 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
verbose_logger.debug(
|
||||
"Logging Details LiteLLM-Success Call streaming complete"
|
||||
)
|
||||
self.model_call_details[
|
||||
"complete_streaming_response"
|
||||
] = complete_streaming_response
|
||||
self.model_call_details[
|
||||
"response_cost"
|
||||
] = self._response_cost_calculator(result=complete_streaming_response)
|
||||
self.model_call_details["complete_streaming_response"] = (
|
||||
complete_streaming_response
|
||||
)
|
||||
self.model_call_details["response_cost"] = (
|
||||
self._response_cost_calculator(result=complete_streaming_response)
|
||||
)
|
||||
self._merge_hidden_params_from_response_into_metadata(
|
||||
complete_streaming_response
|
||||
)
|
||||
## STANDARDIZED LOGGING PAYLOAD
|
||||
self.model_call_details[
|
||||
"standard_logging_object"
|
||||
] = self._build_standard_logging_payload(
|
||||
complete_streaming_response, start_time, end_time
|
||||
self.model_call_details["standard_logging_object"] = (
|
||||
self._build_standard_logging_payload(
|
||||
complete_streaming_response, start_time, end_time
|
||||
)
|
||||
)
|
||||
if (
|
||||
standard_logging_payload := self.model_call_details.get(
|
||||
|
|
@ -2348,10 +2369,10 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
)
|
||||
else:
|
||||
if self.stream and complete_streaming_response:
|
||||
self.model_call_details[
|
||||
"complete_response"
|
||||
] = self.model_call_details.get(
|
||||
"complete_streaming_response", {}
|
||||
self.model_call_details["complete_response"] = (
|
||||
self.model_call_details.get(
|
||||
"complete_streaming_response", {}
|
||||
)
|
||||
)
|
||||
result = self.model_call_details["complete_response"]
|
||||
openMeterLogger.log_success_event(
|
||||
|
|
@ -2375,10 +2396,10 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
)
|
||||
else:
|
||||
if self.stream and complete_streaming_response:
|
||||
self.model_call_details[
|
||||
"complete_response"
|
||||
] = self.model_call_details.get(
|
||||
"complete_streaming_response", {}
|
||||
self.model_call_details["complete_response"] = (
|
||||
self.model_call_details.get(
|
||||
"complete_streaming_response", {}
|
||||
)
|
||||
)
|
||||
result = self.model_call_details["complete_response"]
|
||||
|
||||
|
|
@ -2517,9 +2538,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
if complete_streaming_response is not None:
|
||||
print_verbose("Async success callbacks: Got a complete streaming response")
|
||||
|
||||
self.model_call_details[
|
||||
"async_complete_streaming_response"
|
||||
] = complete_streaming_response
|
||||
self.model_call_details["async_complete_streaming_response"] = (
|
||||
complete_streaming_response
|
||||
)
|
||||
|
||||
try:
|
||||
if self.model_call_details.get("cache_hit", False) is True:
|
||||
|
|
@ -2530,10 +2551,10 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
model_call_details=self.model_call_details
|
||||
)
|
||||
# base_model defaults to None if not set on model_info
|
||||
self.model_call_details[
|
||||
"response_cost"
|
||||
] = self._response_cost_calculator(
|
||||
result=complete_streaming_response
|
||||
self.model_call_details["response_cost"] = (
|
||||
self._response_cost_calculator(
|
||||
result=complete_streaming_response
|
||||
)
|
||||
)
|
||||
|
||||
verbose_logger.debug(
|
||||
|
|
@ -2545,11 +2566,15 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
)
|
||||
self.model_call_details["response_cost"] = None
|
||||
|
||||
self._merge_hidden_params_from_response_into_metadata(
|
||||
complete_streaming_response
|
||||
)
|
||||
|
||||
## STANDARDIZED LOGGING PAYLOAD
|
||||
self.model_call_details[
|
||||
"standard_logging_object"
|
||||
] = self._build_standard_logging_payload(
|
||||
complete_streaming_response, start_time, end_time
|
||||
self.model_call_details["standard_logging_object"] = (
|
||||
self._build_standard_logging_payload(
|
||||
complete_streaming_response, start_time, end_time
|
||||
)
|
||||
)
|
||||
|
||||
# print standard logging payload
|
||||
|
|
@ -2576,9 +2601,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
# _success_handler_helper_fn
|
||||
if self.model_call_details.get("standard_logging_object") is None:
|
||||
## STANDARDIZED LOGGING PAYLOAD
|
||||
self.model_call_details[
|
||||
"standard_logging_object"
|
||||
] = self._build_standard_logging_payload(result, start_time, end_time)
|
||||
self.model_call_details["standard_logging_object"] = (
|
||||
self._build_standard_logging_payload(result, start_time, end_time)
|
||||
)
|
||||
|
||||
# print standard logging payload
|
||||
if (
|
||||
|
|
@ -2821,18 +2846,18 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
|
||||
## STANDARDIZED LOGGING PAYLOAD
|
||||
|
||||
self.model_call_details[
|
||||
"standard_logging_object"
|
||||
] = get_standard_logging_object_payload(
|
||||
kwargs=self.model_call_details,
|
||||
init_response_obj={},
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
logging_obj=self,
|
||||
status="failure",
|
||||
error_str=str(exception),
|
||||
original_exception=exception,
|
||||
standard_built_in_tools_params=self.standard_built_in_tools_params,
|
||||
self.model_call_details["standard_logging_object"] = (
|
||||
get_standard_logging_object_payload(
|
||||
kwargs=self.model_call_details,
|
||||
init_response_obj={},
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
logging_obj=self,
|
||||
status="failure",
|
||||
error_str=str(exception),
|
||||
original_exception=exception,
|
||||
standard_built_in_tools_params=self.standard_built_in_tools_params,
|
||||
)
|
||||
)
|
||||
return start_time, end_time
|
||||
|
||||
|
|
@ -3800,9 +3825,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
|
|||
service_name=arize_config.project_name,
|
||||
)
|
||||
|
||||
os.environ[
|
||||
"OTEL_EXPORTER_OTLP_TRACES_HEADERS"
|
||||
] = f"space_id={arize_config.space_key or arize_config.space_id},api_key={arize_config.api_key}"
|
||||
os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = (
|
||||
f"space_id={arize_config.space_key or arize_config.space_id},api_key={arize_config.api_key}"
|
||||
)
|
||||
for callback in _in_memory_loggers:
|
||||
if (
|
||||
isinstance(callback, ArizeLogger)
|
||||
|
|
@ -3828,13 +3853,13 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
|
|||
existing_attrs = os.environ.get("OTEL_RESOURCE_ATTRIBUTES", "")
|
||||
# Add openinference.project.name attribute
|
||||
if existing_attrs:
|
||||
os.environ[
|
||||
"OTEL_RESOURCE_ATTRIBUTES"
|
||||
] = f"{existing_attrs},openinference.project.name={arize_phoenix_config.project_name}"
|
||||
os.environ["OTEL_RESOURCE_ATTRIBUTES"] = (
|
||||
f"{existing_attrs},openinference.project.name={arize_phoenix_config.project_name}"
|
||||
)
|
||||
else:
|
||||
os.environ[
|
||||
"OTEL_RESOURCE_ATTRIBUTES"
|
||||
] = f"openinference.project.name={arize_phoenix_config.project_name}"
|
||||
os.environ["OTEL_RESOURCE_ATTRIBUTES"] = (
|
||||
f"openinference.project.name={arize_phoenix_config.project_name}"
|
||||
)
|
||||
|
||||
# Set Phoenix project name from environment variable
|
||||
phoenix_project_name = os.environ.get("PHOENIX_PROJECT_NAME", None)
|
||||
|
|
@ -3842,19 +3867,19 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
|
|||
existing_attrs = os.environ.get("OTEL_RESOURCE_ATTRIBUTES", "")
|
||||
# Add openinference.project.name attribute
|
||||
if existing_attrs:
|
||||
os.environ[
|
||||
"OTEL_RESOURCE_ATTRIBUTES"
|
||||
] = f"{existing_attrs},openinference.project.name={phoenix_project_name}"
|
||||
os.environ["OTEL_RESOURCE_ATTRIBUTES"] = (
|
||||
f"{existing_attrs},openinference.project.name={phoenix_project_name}"
|
||||
)
|
||||
else:
|
||||
os.environ[
|
||||
"OTEL_RESOURCE_ATTRIBUTES"
|
||||
] = f"openinference.project.name={phoenix_project_name}"
|
||||
os.environ["OTEL_RESOURCE_ATTRIBUTES"] = (
|
||||
f"openinference.project.name={phoenix_project_name}"
|
||||
)
|
||||
|
||||
# auth can be disabled on local deployments of arize phoenix
|
||||
if arize_phoenix_config.otlp_auth_headers is not None:
|
||||
os.environ[
|
||||
"OTEL_EXPORTER_OTLP_TRACES_HEADERS"
|
||||
] = arize_phoenix_config.otlp_auth_headers
|
||||
os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = (
|
||||
arize_phoenix_config.otlp_auth_headers
|
||||
)
|
||||
|
||||
for callback in _in_memory_loggers:
|
||||
if (
|
||||
|
|
@ -4041,9 +4066,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
|
|||
exporter="otlp_http",
|
||||
endpoint="https://langtrace.ai/api/trace",
|
||||
)
|
||||
os.environ[
|
||||
"OTEL_EXPORTER_OTLP_TRACES_HEADERS"
|
||||
] = f"api_key={os.getenv('LANGTRACE_API_KEY')}"
|
||||
os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = (
|
||||
f"api_key={os.getenv('LANGTRACE_API_KEY')}"
|
||||
)
|
||||
for callback in _in_memory_loggers:
|
||||
if (
|
||||
isinstance(callback, OpenTelemetry)
|
||||
|
|
@ -4967,10 +4992,10 @@ class StandardLoggingPayloadSetup:
|
|||
for key in StandardLoggingHiddenParams.__annotations__.keys():
|
||||
if key in hidden_params:
|
||||
if key == "additional_headers":
|
||||
clean_hidden_params[
|
||||
"additional_headers"
|
||||
] = StandardLoggingPayloadSetup.get_additional_headers(
|
||||
hidden_params[key]
|
||||
clean_hidden_params["additional_headers"] = (
|
||||
StandardLoggingPayloadSetup.get_additional_headers(
|
||||
hidden_params[key]
|
||||
)
|
||||
)
|
||||
else:
|
||||
clean_hidden_params[key] = hidden_params[key] # type: ignore
|
||||
|
|
@ -5609,9 +5634,9 @@ def scrub_sensitive_keys_in_metadata(litellm_params: Optional[dict]):
|
|||
):
|
||||
for k, v in metadata["user_api_key_metadata"].items():
|
||||
if k == "logging": # prevent logging user logging keys
|
||||
cleaned_user_api_key_metadata[
|
||||
k
|
||||
] = "scrubbed_by_litellm_for_sensitive_keys"
|
||||
cleaned_user_api_key_metadata[k] = (
|
||||
"scrubbed_by_litellm_for_sensitive_keys"
|
||||
)
|
||||
else:
|
||||
cleaned_user_api_key_metadata[k] = v
|
||||
|
||||
|
|
|
|||
|
|
@ -257,6 +257,15 @@ def detect_first_expected_role(
|
|||
return None
|
||||
|
||||
|
||||
def _counts_for_alternation(message: AllMessageValues) -> bool:
|
||||
role = message.get("role")
|
||||
if role == "user":
|
||||
return True
|
||||
if role == "assistant":
|
||||
return not bool(message.get("tool_calls"))
|
||||
return False
|
||||
|
||||
|
||||
def _insert_user_continue_message(
|
||||
messages: List[AllMessageValues],
|
||||
user_continue_message: Optional[ChatCompletionUserMessage],
|
||||
|
|
@ -269,8 +278,8 @@ def _insert_user_continue_message(
|
|||
2. Final assistant message
|
||||
3. Consecutive assistant messages
|
||||
|
||||
Only inserts messages between consecutive assistant messages,
|
||||
ignoring all other role types.
|
||||
Skips tool messages and assistant messages with tool calls in the
|
||||
alternation check, matching strict templates like llama.cpp.
|
||||
"""
|
||||
if not messages:
|
||||
return messages
|
||||
|
|
@ -278,25 +287,42 @@ def _insert_user_continue_message(
|
|||
result_messages = messages.copy() # Don't modify the input list
|
||||
continue_message = user_continue_message or DEFAULT_USER_CONTINUE_MESSAGE
|
||||
|
||||
# Handle first message if it's an assistant message
|
||||
# Handle first message if it's an assistant message — always prepend
|
||||
# user_continue regardless of tool_calls, to preserve backward compatibility.
|
||||
if result_messages[0]["role"] == "assistant":
|
||||
result_messages.insert(0, continue_message)
|
||||
|
||||
# Handle consecutive assistant messages and final message
|
||||
i = 1 # Start from second message since we handled first message
|
||||
# Handle consecutive assistant messages in the counted sequence
|
||||
i = 1
|
||||
while i < len(result_messages):
|
||||
curr_message = result_messages[i]
|
||||
prev_message = result_messages[i - 1]
|
||||
|
||||
# Only check for consecutive assistant messages
|
||||
# Ignore all other role types
|
||||
if curr_message["role"] == "assistant" and prev_message["role"] == "assistant":
|
||||
result_messages.insert(i, continue_message)
|
||||
i += 2 # Skip over the message we just inserted
|
||||
else:
|
||||
inserted_continue_message = False
|
||||
if (
|
||||
_counts_for_alternation(curr_message)
|
||||
and curr_message["role"] == "assistant"
|
||||
):
|
||||
# Preserve old behavior for malformed adjacent assistant sequences like
|
||||
# assistant(tool_calls) -> assistant(no-tool-calls) with no tool message.
|
||||
if i > 0 and result_messages[i - 1].get("role") == "assistant":
|
||||
result_messages.insert(i, continue_message)
|
||||
i += 2
|
||||
inserted_continue_message = True
|
||||
else:
|
||||
j = i - 1
|
||||
while j >= 0:
|
||||
previous_message = result_messages[j]
|
||||
if _counts_for_alternation(previous_message):
|
||||
if previous_message["role"] == "assistant":
|
||||
result_messages.insert(i, continue_message)
|
||||
i += 2
|
||||
inserted_continue_message = True
|
||||
break
|
||||
j -= 1
|
||||
if not inserted_continue_message:
|
||||
i += 1
|
||||
|
||||
# Handle final message
|
||||
# Handle final message — append user_continue after any trailing assistant,
|
||||
# including ones with tool_calls, to preserve backward compatibility.
|
||||
if result_messages[-1]["role"] == "assistant" and ensure_alternating_roles:
|
||||
result_messages.append(continue_message)
|
||||
|
||||
|
|
@ -311,34 +337,24 @@ def _insert_assistant_continue_message(
|
|||
"""
|
||||
Add assistant continuation messages between consecutive user messages.
|
||||
|
||||
Args:
|
||||
messages: List of message dictionaries
|
||||
assistant_continue_message: Optional custom assistant message
|
||||
ensure_alternating_roles: Whether to enforce alternating roles
|
||||
|
||||
Returns:
|
||||
Modified list of messages with inserted assistant messages
|
||||
Only checks directly adjacent messages to preserve backward compatibility.
|
||||
"""
|
||||
if not ensure_alternating_roles or len(messages) <= 1:
|
||||
return messages
|
||||
|
||||
# Create a new list to store modified messages
|
||||
continue_message = assistant_continue_message or DEFAULT_ASSISTANT_CONTINUE_MESSAGE
|
||||
|
||||
modified_messages: List[AllMessageValues] = []
|
||||
|
||||
for i, message in enumerate(messages):
|
||||
modified_messages.append(message)
|
||||
|
||||
# Check if we need to insert an assistant message
|
||||
if (
|
||||
i < len(messages) - 1 # Not the last message
|
||||
and message.get("role") == "user" # Current is user
|
||||
i < len(messages) - 1
|
||||
and message.get("role") == "user"
|
||||
and messages[i + 1].get("role") == "user"
|
||||
): # Next is user
|
||||
# Insert assistant message
|
||||
continue_message = (
|
||||
assistant_continue_message or DEFAULT_ASSISTANT_CONTINUE_MESSAGE
|
||||
)
|
||||
):
|
||||
modified_messages.append(message)
|
||||
modified_messages.append(continue_message)
|
||||
else:
|
||||
modified_messages.append(message)
|
||||
|
||||
return modified_messages
|
||||
|
||||
|
|
@ -536,6 +552,61 @@ def update_responses_input_with_model_file_ids(
|
|||
return updated_input
|
||||
|
||||
|
||||
def _decode_vector_store_ids_in_tools(
|
||||
tools: Optional[List[Dict[str, Any]]],
|
||||
) -> Optional[List[Dict[str, Any]]]:
|
||||
"""
|
||||
Decodes unified (LiteLLM-managed) vector_store_ids in file_search tools to
|
||||
provider-native IDs. Non-unified IDs are passed through unchanged.
|
||||
|
||||
This runs unconditionally — no file-ID mapping is required.
|
||||
"""
|
||||
if not tools or not isinstance(tools, list):
|
||||
return tools
|
||||
|
||||
from litellm.llms.base_llm.managed_resources.utils import (
|
||||
is_base64_encoded_unified_id,
|
||||
parse_unified_id,
|
||||
)
|
||||
|
||||
updated_tools = []
|
||||
for tool in tools:
|
||||
if not isinstance(tool, dict) or tool.get("type") != "file_search":
|
||||
updated_tools.append(tool)
|
||||
continue
|
||||
|
||||
vector_store_ids = tool.get("vector_store_ids")
|
||||
if not isinstance(vector_store_ids, list):
|
||||
updated_tools.append(tool)
|
||||
continue
|
||||
|
||||
decoded_ids = []
|
||||
for vs_id in vector_store_ids:
|
||||
if not isinstance(vs_id, str) or not is_base64_encoded_unified_id(vs_id):
|
||||
decoded_ids.append(vs_id)
|
||||
continue
|
||||
|
||||
parsed = parse_unified_id(vs_id)
|
||||
provider_resource_id = (
|
||||
parsed.get("provider_resource_id") if parsed else None
|
||||
)
|
||||
|
||||
if not provider_resource_id:
|
||||
verbose_logger.warning(
|
||||
"file_search tool contains unified vector_store_id '%s' that could "
|
||||
"not be decoded to a provider resource ID — passing original ID. "
|
||||
"Ensure the vector store was created via LiteLLM.",
|
||||
vs_id,
|
||||
)
|
||||
decoded_ids.append(vs_id)
|
||||
else:
|
||||
decoded_ids.append(provider_resource_id)
|
||||
|
||||
updated_tools.append({**tool, "vector_store_ids": decoded_ids})
|
||||
|
||||
return updated_tools
|
||||
|
||||
|
||||
def update_responses_tools_with_model_file_ids(
|
||||
tools: Optional[List[Dict[str, Any]]],
|
||||
model_id: Optional[str] = None,
|
||||
|
|
@ -544,7 +615,8 @@ def update_responses_tools_with_model_file_ids(
|
|||
"""
|
||||
Updates responses API tools with provider-specific file IDs.
|
||||
|
||||
Handles code_interpreter tools with container.file_ids.
|
||||
Pass 1 (always): decode unified vector_store_ids in file_search tools.
|
||||
Pass 2 (needs mapping): map code_interpreter container file_ids to provider IDs.
|
||||
|
||||
Args:
|
||||
tools: The responses API tools parameter
|
||||
|
|
@ -555,6 +627,10 @@ def update_responses_tools_with_model_file_ids(
|
|||
if not tools or not isinstance(tools, list):
|
||||
return tools
|
||||
|
||||
# Pass 1: decode unified vector_store_ids (no mapping needed)
|
||||
tools = _decode_vector_store_ids_in_tools(tools) or tools
|
||||
|
||||
# Pass 2: map code_interpreter file IDs (requires mapping)
|
||||
if not model_file_id_mapping or not model_id:
|
||||
return tools
|
||||
|
||||
|
|
|
|||
|
|
@ -1498,17 +1498,49 @@ def convert_to_gemini_tool_call_result( # noqa: PLR0915
|
|||
from litellm.types.llms.vertex_ai import BlobType
|
||||
|
||||
content_str: str = ""
|
||||
inline_data: Optional[BlobType] = None
|
||||
inline_data_list: List[BlobType] = []
|
||||
|
||||
if "content" in message:
|
||||
if isinstance(message["content"], str):
|
||||
content_str = message["content"]
|
||||
# Detect data-URL images (e.g. from Anthropic tool_result with a single image block
|
||||
# that was serialised as a plain string by translate_anthropic_messages_to_openai)
|
||||
# and promote them to inline_data so Gemini receives actual image bytes.
|
||||
if content_str[:5].lower() == "data:" and ";base64," in content_str:
|
||||
try:
|
||||
mime_rest = content_str[5:].split(";base64,", 1)
|
||||
if len(mime_rest) == 2 and mime_rest[0].startswith("image/"):
|
||||
# Strip any extra parameters (e.g. ";charset=UTF-8") from the MIME segment
|
||||
clean_mime = mime_rest[0].split(";")[0].strip()
|
||||
inline_data_list.append(
|
||||
BlobType(data=mime_rest[1], mime_type=clean_mime)
|
||||
)
|
||||
content_str = ""
|
||||
except Exception as e:
|
||||
verbose_logger.warning(
|
||||
f"Failed to parse data URL in tool response: {e}"
|
||||
)
|
||||
elif isinstance(message["content"], List):
|
||||
content_list = message["content"]
|
||||
for content in content_list:
|
||||
content_type = content.get("type", "")
|
||||
if content_type == "text":
|
||||
content_str += content.get("text", "")
|
||||
elif content_type == "image":
|
||||
# Anthropic-native image block: {"type": "image", "source": {"type": "base64", ...}}
|
||||
source = content.get("source", {})
|
||||
if isinstance(source, dict) and source.get("type") == "base64":
|
||||
try:
|
||||
inline_data_list.append(
|
||||
BlobType(
|
||||
data=source.get("data", ""),
|
||||
mime_type=source.get("media_type", "image/jpeg"),
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_logger.warning(
|
||||
f"Failed to process Anthropic image block in tool response: {e}"
|
||||
)
|
||||
elif content_type in ("input_image", "image_url"):
|
||||
# Extract image for inline_data (for Computer Use screenshots and tool results)
|
||||
image_url_data = content.get("image_url", "")
|
||||
|
|
@ -1524,9 +1556,11 @@ def convert_to_gemini_tool_call_result( # noqa: PLR0915
|
|||
image_obj = convert_to_anthropic_image_obj(
|
||||
image_url, format=None
|
||||
)
|
||||
inline_data = BlobType(
|
||||
data=image_obj["data"],
|
||||
mime_type=image_obj["media_type"],
|
||||
inline_data_list.append(
|
||||
BlobType(
|
||||
data=image_obj["data"],
|
||||
mime_type=image_obj["media_type"],
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_logger.warning(
|
||||
|
|
@ -1551,9 +1585,11 @@ def convert_to_gemini_tool_call_result( # noqa: PLR0915
|
|||
file_obj = convert_to_anthropic_image_obj(
|
||||
file_data, format=None
|
||||
)
|
||||
inline_data = BlobType(
|
||||
data=file_obj["data"],
|
||||
mime_type=file_obj["media_type"],
|
||||
inline_data_list.append(
|
||||
BlobType(
|
||||
data=file_obj["data"],
|
||||
mime_type=file_obj["media_type"],
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_logger.warning(
|
||||
|
|
@ -1607,13 +1643,12 @@ def convert_to_gemini_tool_call_result( # noqa: PLR0915
|
|||
# Create part with function_response, and optionally inline_data for images (Computer Use)
|
||||
_part: VertexPartType = {"function_response": _function_response}
|
||||
|
||||
# For Computer Use, if we have an image, we need separate parts:
|
||||
# For Computer Use, if we have images/files, we need separate parts:
|
||||
# - One part with function_response
|
||||
# - One part with inline_data
|
||||
# - One part per inline_data item
|
||||
# Gemini's PartType is a oneof, so we can't have both in the same part
|
||||
if inline_data:
|
||||
image_part: VertexPartType = {"inline_data": inline_data}
|
||||
return [_part, image_part]
|
||||
if inline_data_list:
|
||||
return [_part] + [{"inline_data": d} for d in inline_data_list]
|
||||
|
||||
return _part
|
||||
|
||||
|
|
|
|||
|
|
@ -57,6 +57,22 @@ IMAGE_ATTRIBUTE = "images"
|
|||
TOOL_CALLS_ATTRIBUTE = "tool_calls"
|
||||
FUNCTION_CALL_ATTRIBUTE = "function_call"
|
||||
|
||||
_SYNC_ITER_EXHAUSTED = object()
|
||||
|
||||
|
||||
def _next_sync_or_exhausted(it: Any) -> Any:
|
||||
"""
|
||||
Call next(it) from a thread and return _SYNC_ITER_EXHAUSTED on StopIteration.
|
||||
|
||||
asyncio.to_thread re-raises thread exceptions inside a coroutine, where PEP 479
|
||||
converts StopIteration to RuntimeError before any except clause can catch it.
|
||||
Returning a sentinel instead keeps StopIteration out of the coroutine boundary.
|
||||
"""
|
||||
try:
|
||||
return next(it)
|
||||
except StopIteration:
|
||||
return _SYNC_ITER_EXHAUSTED
|
||||
|
||||
|
||||
def is_async_iterable(obj: Any) -> bool:
|
||||
"""
|
||||
|
|
@ -2090,7 +2106,9 @@ class CustomStreamWrapper:
|
|||
):
|
||||
chunk = self.completion_stream
|
||||
else:
|
||||
chunk = next(self.completion_stream) # type: ignore[arg-type]
|
||||
chunk = await asyncio.to_thread(_next_sync_or_exhausted, self.completion_stream) # type: ignore[arg-type]
|
||||
if chunk is _SYNC_ITER_EXHAUSTED:
|
||||
raise StopAsyncIteration
|
||||
if chunk is not None and chunk != b"":
|
||||
processed_chunk = self.chunk_creator(chunk=chunk)
|
||||
if processed_chunk is None:
|
||||
|
|
@ -2150,22 +2168,36 @@ class CustomStreamWrapper:
|
|||
self.sent_stream_usage = True
|
||||
return response
|
||||
|
||||
asyncio.create_task(
|
||||
self.logging_obj.async_success_handler(
|
||||
_deferred_cb = getattr(
|
||||
self.logging_obj,
|
||||
"_on_deferred_stream_complete",
|
||||
None,
|
||||
)
|
||||
if _deferred_cb is not None:
|
||||
# Proxy has post-call guardrails — let the closure
|
||||
# run guardrails on the assembled response, then
|
||||
# fire logging with guardrail_information populated.
|
||||
self.logging_obj._on_deferred_stream_complete = None # type: ignore[attr-defined]
|
||||
asyncio.create_task(
|
||||
_deferred_cb(complete_streaming_response, cache_hit)
|
||||
)
|
||||
else:
|
||||
asyncio.create_task(
|
||||
self.logging_obj.async_success_handler(
|
||||
complete_streaming_response,
|
||||
cache_hit=cache_hit,
|
||||
start_time=None,
|
||||
end_time=None,
|
||||
)
|
||||
)
|
||||
|
||||
executor.submit(
|
||||
self.logging_obj.success_handler,
|
||||
complete_streaming_response,
|
||||
cache_hit=cache_hit,
|
||||
start_time=None,
|
||||
end_time=None,
|
||||
)
|
||||
)
|
||||
|
||||
executor.submit(
|
||||
self.logging_obj.success_handler,
|
||||
complete_streaming_response,
|
||||
cache_hit=cache_hit,
|
||||
start_time=None,
|
||||
end_time=None,
|
||||
)
|
||||
|
||||
raise StopAsyncIteration # Re-raise StopIteration
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -111,6 +111,7 @@ class A2AGuardrailHandler(BaseTranslation):
|
|||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: Optional["LiteLLMLoggingObj"] = None,
|
||||
user_api_key_dict: Optional["UserAPIKeyAuth"] = None,
|
||||
request_data: Optional[dict] = None,
|
||||
) -> Any:
|
||||
"""
|
||||
Process A2A output response by applying guardrails to text content.
|
||||
|
|
@ -166,13 +167,21 @@ class A2AGuardrailHandler(BaseTranslation):
|
|||
return response
|
||||
|
||||
# Step 2: Apply guardrail to all texts in batch
|
||||
# Create a request_data dict with response info and user API key metadata
|
||||
request_data: dict = {"response": response_dict}
|
||||
# Use the real request_data if provided (proxy path), otherwise
|
||||
# create a standalone dict (SDK / direct-call path).
|
||||
if request_data is None:
|
||||
request_data = {"response": response_dict}
|
||||
else:
|
||||
if "response" not in request_data:
|
||||
request_data["response"] = response_dict
|
||||
|
||||
# Add user API key metadata with prefixed keys
|
||||
user_metadata = self.transform_user_api_key_dict_to_metadata(user_api_key_dict)
|
||||
if user_metadata:
|
||||
request_data["litellm_metadata"] = user_metadata
|
||||
if "litellm_metadata" not in request_data:
|
||||
user_metadata = self.transform_user_api_key_dict_to_metadata(
|
||||
user_api_key_dict
|
||||
)
|
||||
if user_metadata:
|
||||
request_data["litellm_metadata"] = user_metadata
|
||||
|
||||
inputs = GenericGuardrailAPIInputs(texts=texts_to_check)
|
||||
|
||||
|
|
@ -213,6 +222,7 @@ class A2AGuardrailHandler(BaseTranslation):
|
|||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: Optional["LiteLLMLoggingObj"] = None,
|
||||
user_api_key_dict: Optional["UserAPIKeyAuth"] = None,
|
||||
request_data: Optional[dict] = None,
|
||||
) -> List[Any]:
|
||||
"""
|
||||
Process A2A streaming output by applying guardrails to accumulated text.
|
||||
|
|
@ -258,10 +268,18 @@ class A2AGuardrailHandler(BaseTranslation):
|
|||
if not combined_text:
|
||||
return responses_so_far
|
||||
|
||||
request_data: dict = {"responses_so_far": responses_so_far}
|
||||
user_metadata = self.transform_user_api_key_dict_to_metadata(user_api_key_dict)
|
||||
if user_metadata:
|
||||
request_data["litellm_metadata"] = user_metadata
|
||||
if request_data is None:
|
||||
request_data = {"responses_so_far": responses_so_far}
|
||||
else:
|
||||
if "responses_so_far" not in request_data:
|
||||
request_data["responses_so_far"] = responses_so_far
|
||||
|
||||
if "litellm_metadata" not in request_data:
|
||||
user_metadata = self.transform_user_api_key_dict_to_metadata(
|
||||
user_api_key_dict
|
||||
)
|
||||
if user_metadata:
|
||||
request_data["litellm_metadata"] = user_metadata
|
||||
|
||||
inputs = GenericGuardrailAPIInputs(texts=[combined_text])
|
||||
guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
|
||||
|
|
|
|||
|
|
@ -252,6 +252,7 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: Optional[Any] = None,
|
||||
user_api_key_dict: Optional[Any] = None,
|
||||
request_data: Optional[dict] = None,
|
||||
) -> Any:
|
||||
"""
|
||||
Process output response by applying guardrails to text content and tool calls.
|
||||
|
|
@ -323,15 +324,21 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
|
||||
# Step 2: Apply guardrail to all texts in batch
|
||||
if texts_to_check or tool_calls_to_check:
|
||||
# Create a request_data dict with response info and user API key metadata
|
||||
request_data: dict = {"response": response}
|
||||
# Use the real request_data if provided (proxy path), otherwise
|
||||
# create a standalone dict (SDK / direct-call path).
|
||||
if request_data is None:
|
||||
request_data = {"response": response}
|
||||
else:
|
||||
if "response" not in request_data:
|
||||
request_data["response"] = response
|
||||
|
||||
# Add user API key metadata with prefixed keys
|
||||
user_metadata = self.transform_user_api_key_dict_to_metadata(
|
||||
user_api_key_dict
|
||||
)
|
||||
if user_metadata:
|
||||
request_data["litellm_metadata"] = user_metadata
|
||||
if "litellm_metadata" not in request_data:
|
||||
user_metadata = self.transform_user_api_key_dict_to_metadata(
|
||||
user_api_key_dict
|
||||
)
|
||||
if user_metadata:
|
||||
request_data["litellm_metadata"] = user_metadata
|
||||
|
||||
inputs = GenericGuardrailAPIInputs(texts=texts_to_check)
|
||||
if images_to_check:
|
||||
|
|
@ -375,6 +382,7 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: Optional[Any] = None,
|
||||
user_api_key_dict: Optional[Any] = None,
|
||||
request_data: Optional[dict] = None,
|
||||
) -> List[Any]:
|
||||
"""
|
||||
Process output streaming response by applying guardrails to text content.
|
||||
|
|
@ -413,7 +421,7 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
|
||||
_guardrailed_inputs = await guardrail_to_apply.apply_guardrail( # allow rejecting the response, if invalid
|
||||
inputs=guardrail_inputs,
|
||||
request_data={},
|
||||
request_data=request_data if request_data is not None else {},
|
||||
input_type="response",
|
||||
logging_obj=litellm_logging_obj,
|
||||
)
|
||||
|
|
@ -426,7 +434,7 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
string_so_far = self.get_streaming_string_so_far(responses_so_far)
|
||||
_guardrailed_inputs = await guardrail_to_apply.apply_guardrail( # allow rejecting the response, if invalid
|
||||
inputs={"texts": [string_so_far]},
|
||||
request_data={},
|
||||
request_data=request_data if request_data is not None else {},
|
||||
input_type="response",
|
||||
logging_obj=litellm_logging_obj,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -15,6 +15,9 @@ import litellm
|
|||
from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import (
|
||||
AnthropicAdapter,
|
||||
)
|
||||
from litellm.llms.anthropic.experimental_pass_through.utils import (
|
||||
is_reasoning_auto_summary_enabled,
|
||||
)
|
||||
from litellm.types.llms.anthropic_messages.anthropic_response import (
|
||||
AnthropicMessagesResponse,
|
||||
)
|
||||
|
|
@ -44,8 +47,9 @@ class LiteLLMMessagesToCompletionTransformationHandler:
|
|||
|
||||
For OpenAI models, Chat Completions typically does not return reasoning text
|
||||
(only token accounting). To return a thinking-like content block in the
|
||||
Anthropic response format, we route the request through OpenAI's Responses API
|
||||
and request a reasoning summary.
|
||||
Anthropic response format, we route the request through OpenAI's Responses API.
|
||||
If the user provides a `summary` field in the thinking dict, it is passed
|
||||
through to the OpenAI reasoning params (opt-in per OpenAI spec).
|
||||
"""
|
||||
custom_llm_provider = completion_kwargs.get("custom_llm_provider")
|
||||
if custom_llm_provider is None:
|
||||
|
|
@ -78,20 +82,29 @@ class LiteLLMMessagesToCompletionTransformationHandler:
|
|||
# Prefix model with "responses/" to route to OpenAI Responses API
|
||||
completion_kwargs["model"] = f"responses/{model}"
|
||||
|
||||
auto_summary = is_reasoning_auto_summary_enabled()
|
||||
|
||||
reasoning_effort = completion_kwargs.get("reasoning_effort")
|
||||
summary = thinking.get("summary")
|
||||
if isinstance(reasoning_effort, str) and reasoning_effort:
|
||||
completion_kwargs["reasoning_effort"] = {
|
||||
"effort": reasoning_effort,
|
||||
"summary": "detailed",
|
||||
}
|
||||
reasoning_dict: Dict[str, Any] = {"effort": reasoning_effort}
|
||||
if summary:
|
||||
reasoning_dict["summary"] = summary
|
||||
elif auto_summary:
|
||||
reasoning_dict["summary"] = "detailed"
|
||||
completion_kwargs["reasoning_effort"] = reasoning_dict
|
||||
elif isinstance(reasoning_effort, dict):
|
||||
if (
|
||||
"summary" not in reasoning_effort
|
||||
and "generate_summary" not in reasoning_effort
|
||||
):
|
||||
updated_reasoning_effort = dict(reasoning_effort)
|
||||
updated_reasoning_effort["summary"] = "detailed"
|
||||
completion_kwargs["reasoning_effort"] = updated_reasoning_effort
|
||||
effective_summary = (
|
||||
summary if summary else ("detailed" if auto_summary else None)
|
||||
)
|
||||
if effective_summary:
|
||||
updated_reasoning_effort = dict(reasoning_effort)
|
||||
updated_reasoning_effort["summary"] = effective_summary
|
||||
completion_kwargs["reasoning_effort"] = updated_reasoning_effort
|
||||
|
||||
@staticmethod
|
||||
def _prepare_completion_kwargs(
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import copy
|
||||
import hashlib
|
||||
import json
|
||||
from typing import (
|
||||
|
|
@ -13,6 +14,10 @@ from typing import (
|
|||
cast,
|
||||
)
|
||||
|
||||
from litellm.llms.anthropic.experimental_pass_through.utils import (
|
||||
is_reasoning_auto_summary_enabled,
|
||||
)
|
||||
|
||||
# OpenAI has a 64-character limit for function/tool names
|
||||
# Anthropic does not have this limit, so we need to truncate long names
|
||||
OPENAI_MAX_TOOL_NAME_LENGTH = 64
|
||||
|
|
@ -733,6 +738,24 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
thinking
|
||||
)
|
||||
if reasoning_effort:
|
||||
summary = (
|
||||
thinking.get("summary") if isinstance(thinking, dict) else None
|
||||
)
|
||||
auto_summary = is_reasoning_auto_summary_enabled()
|
||||
if summary:
|
||||
return {
|
||||
"reasoning_effort": {
|
||||
"effort": reasoning_effort,
|
||||
"summary": summary,
|
||||
}
|
||||
}
|
||||
elif auto_summary:
|
||||
return {
|
||||
"reasoning_effort": {
|
||||
"effort": reasoning_effort,
|
||||
"summary": "detailed",
|
||||
}
|
||||
}
|
||||
return {"reasoning_effort": reasoning_effort}
|
||||
return {}
|
||||
|
||||
|
|
@ -833,6 +856,11 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
if not schema:
|
||||
return None
|
||||
|
||||
# Deep copy to avoid mutating the original schema
|
||||
schema = copy.deepcopy(schema)
|
||||
# OpenAI strict mode requires additionalProperties: false on every object
|
||||
self._add_additional_properties_false(schema)
|
||||
|
||||
# Convert to OpenAI response_format structure
|
||||
return {
|
||||
"type": "json_schema",
|
||||
|
|
@ -843,6 +871,46 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
},
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _add_additional_properties_false(schema: dict) -> None:
|
||||
"""
|
||||
Recursively ensure object schemas comply with OpenAI strict mode.
|
||||
|
||||
OpenAI's strict mode requires:
|
||||
1. 'additionalProperties': false at every object nesting level
|
||||
2. All property keys listed in 'required'
|
||||
"""
|
||||
if not isinstance(schema, dict):
|
||||
return
|
||||
|
||||
if schema.get("type") == "object" and "properties" in schema:
|
||||
schema["additionalProperties"] = False
|
||||
schema["required"] = list(schema["properties"].keys())
|
||||
for prop in schema["properties"].values():
|
||||
LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(prop)
|
||||
|
||||
# Handle array items
|
||||
if "items" in schema:
|
||||
LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(
|
||||
schema["items"]
|
||||
)
|
||||
|
||||
# Handle anyOf/oneOf/allOf
|
||||
for key in ("anyOf", "oneOf", "allOf"):
|
||||
if key in schema:
|
||||
for sub_schema in schema[key]:
|
||||
LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(
|
||||
sub_schema
|
||||
)
|
||||
|
||||
# Handle $defs / definitions
|
||||
for key in ("$defs", "definitions"):
|
||||
if key in schema:
|
||||
for def_schema in schema[key].values():
|
||||
LiteLLMAnthropicMessagesAdapter._add_additional_properties_false(
|
||||
def_schema
|
||||
)
|
||||
|
||||
def _add_system_message_to_messages(
|
||||
self,
|
||||
new_messages: List[AllMessageValues],
|
||||
|
|
@ -878,6 +946,144 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
ChatCompletionSystemMessage(role="system", content=openai_system_content), # type: ignore
|
||||
)
|
||||
|
||||
def _translate_metadata_to_openai(
|
||||
self,
|
||||
anthropic_message_request: AnthropicMessagesRequest,
|
||||
new_kwargs: ChatCompletionRequest,
|
||||
) -> None:
|
||||
"""Translate metadata fields from Anthropic request to OpenAI request."""
|
||||
if "metadata" in anthropic_message_request:
|
||||
metadata = anthropic_message_request["metadata"]
|
||||
if metadata and "user_id" in metadata:
|
||||
new_kwargs["user"] = metadata["user_id"]
|
||||
|
||||
if "litellm_metadata" in anthropic_message_request:
|
||||
# metadata will be passed to litellm.acompletion(), it's a litellm_param
|
||||
new_kwargs["metadata"] = anthropic_message_request.pop("litellm_metadata")
|
||||
|
||||
def _translate_tool_choice_to_openai(
|
||||
self,
|
||||
anthropic_message_request: AnthropicMessagesRequest,
|
||||
new_kwargs: ChatCompletionRequest,
|
||||
) -> None:
|
||||
"""Translate Anthropic tool_choice to OpenAI format."""
|
||||
if "tool_choice" not in anthropic_message_request:
|
||||
return
|
||||
tool_choice = anthropic_message_request["tool_choice"]
|
||||
if not tool_choice:
|
||||
return
|
||||
new_kwargs["tool_choice"] = self.translate_anthropic_tool_choice_to_openai(
|
||||
tool_choice=cast(AnthropicMessagesToolChoice, tool_choice)
|
||||
)
|
||||
|
||||
def _translate_tools_to_openai(
|
||||
self,
|
||||
anthropic_message_request: AnthropicMessagesRequest,
|
||||
new_kwargs: ChatCompletionRequest,
|
||||
) -> Dict[str, str]:
|
||||
"""Translate tools and extract web_search_options when needed."""
|
||||
if "tools" not in anthropic_message_request:
|
||||
return {}
|
||||
|
||||
tools = anthropic_message_request["tools"]
|
||||
if not tools:
|
||||
return {}
|
||||
|
||||
web_search_tools: List[AllAnthropicToolsValues] = []
|
||||
regular_tools: List[AllAnthropicToolsValues] = []
|
||||
for tool in tools:
|
||||
cast_tool = cast(Dict[str, Any], tool)
|
||||
if self._is_web_search_tool(cast_tool):
|
||||
web_search_tools.append(cast(AllAnthropicToolsValues, tool))
|
||||
else:
|
||||
regular_tools.append(cast(AllAnthropicToolsValues, tool))
|
||||
|
||||
if web_search_tools:
|
||||
new_kwargs["web_search_options"] = {} # type: ignore
|
||||
|
||||
if not regular_tools:
|
||||
return {}
|
||||
|
||||
translated_tools, tool_name_mapping = self.translate_anthropic_tools_to_openai(
|
||||
tools=regular_tools,
|
||||
model=new_kwargs.get("model"),
|
||||
)
|
||||
new_kwargs["tools"] = translated_tools
|
||||
return tool_name_mapping
|
||||
|
||||
def _translate_thinking_to_openai(
|
||||
self,
|
||||
anthropic_message_request: AnthropicMessagesRequest,
|
||||
new_kwargs: ChatCompletionRequest,
|
||||
) -> None:
|
||||
"""Translate Anthropic thinking to either thinking or reasoning_effort."""
|
||||
if "thinking" not in anthropic_message_request:
|
||||
return
|
||||
|
||||
thinking = anthropic_message_request["thinking"]
|
||||
if not thinking:
|
||||
return
|
||||
|
||||
model = new_kwargs.get("model", "")
|
||||
if self.is_anthropic_claude_model(model):
|
||||
new_kwargs["thinking"] = thinking # type: ignore
|
||||
return
|
||||
|
||||
reasoning_effort = self.translate_anthropic_thinking_to_reasoning_effort(
|
||||
cast(Dict[str, Any], thinking)
|
||||
)
|
||||
if not reasoning_effort:
|
||||
return
|
||||
|
||||
summary = thinking.get("summary") if isinstance(thinking, dict) else None
|
||||
auto_summary = is_reasoning_auto_summary_enabled()
|
||||
if summary:
|
||||
new_kwargs["reasoning_effort"] = cast(
|
||||
Any,
|
||||
{
|
||||
"effort": reasoning_effort,
|
||||
"summary": summary,
|
||||
},
|
||||
)
|
||||
elif auto_summary:
|
||||
new_kwargs["reasoning_effort"] = cast(
|
||||
Any,
|
||||
{
|
||||
"effort": reasoning_effort,
|
||||
"summary": "detailed",
|
||||
},
|
||||
)
|
||||
else:
|
||||
new_kwargs["reasoning_effort"] = reasoning_effort
|
||||
|
||||
def _translate_output_format_to_openai(
|
||||
self,
|
||||
anthropic_message_request: AnthropicMessagesRequest,
|
||||
new_kwargs: ChatCompletionRequest,
|
||||
) -> None:
|
||||
"""Translate output_format to response_format when applicable."""
|
||||
if "output_format" not in anthropic_message_request:
|
||||
return
|
||||
output_format = anthropic_message_request["output_format"]
|
||||
if not output_format:
|
||||
return
|
||||
response_format = self.translate_anthropic_output_format_to_openai(
|
||||
output_format=output_format
|
||||
)
|
||||
if response_format:
|
||||
new_kwargs["response_format"] = response_format
|
||||
|
||||
def _copy_untranslated_anthropic_params(
|
||||
self,
|
||||
anthropic_message_request: AnthropicMessagesRequest,
|
||||
new_kwargs: ChatCompletionRequest,
|
||||
) -> None:
|
||||
"""Copy through anthropic params that do not require translation."""
|
||||
translatable_params = self.translatable_anthropic_params()
|
||||
for k, v in anthropic_message_request.items():
|
||||
if k not in translatable_params: # pass remaining params as is
|
||||
new_kwargs[k] = v # type: ignore
|
||||
|
||||
def translate_anthropic_to_openai(
|
||||
self, anthropic_message_request: AnthropicMessagesRequest
|
||||
) -> Tuple[ChatCompletionRequest, Dict[str, str]]:
|
||||
|
|
@ -918,83 +1124,35 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
"model": anthropic_message_request["model"],
|
||||
"messages": new_messages,
|
||||
}
|
||||
## CONVERT METADATA (user_id)
|
||||
if "metadata" in anthropic_message_request:
|
||||
metadata = anthropic_message_request["metadata"]
|
||||
if metadata and "user_id" in metadata:
|
||||
new_kwargs["user"] = metadata["user_id"]
|
||||
|
||||
# Pass litellm proxy specific metadata
|
||||
if "litellm_metadata" in anthropic_message_request:
|
||||
# metadata will be passed to litellm.acompletion(), it's a litellm_param
|
||||
new_kwargs["metadata"] = anthropic_message_request.pop("litellm_metadata")
|
||||
|
||||
## CONVERT METADATA (user_id + litellm metadata)
|
||||
self._translate_metadata_to_openai(
|
||||
anthropic_message_request=anthropic_message_request,
|
||||
new_kwargs=new_kwargs,
|
||||
)
|
||||
## CONVERT TOOL CHOICE
|
||||
if "tool_choice" in anthropic_message_request:
|
||||
tool_choice = anthropic_message_request["tool_choice"]
|
||||
if tool_choice:
|
||||
new_kwargs[
|
||||
"tool_choice"
|
||||
] = self.translate_anthropic_tool_choice_to_openai(
|
||||
tool_choice=cast(AnthropicMessagesToolChoice, tool_choice)
|
||||
)
|
||||
self._translate_tool_choice_to_openai(
|
||||
anthropic_message_request=anthropic_message_request,
|
||||
new_kwargs=new_kwargs,
|
||||
)
|
||||
## CONVERT TOOLS
|
||||
if "tools" in anthropic_message_request:
|
||||
tools = anthropic_message_request["tools"]
|
||||
if tools:
|
||||
# Separate web search tools from regular tools
|
||||
web_search_tools = []
|
||||
regular_tools = []
|
||||
for tool in tools:
|
||||
if self._is_web_search_tool(cast(Dict[str, Any], tool)):
|
||||
web_search_tools.append(tool)
|
||||
else:
|
||||
regular_tools.append(tool)
|
||||
|
||||
# If web search tools are present, add web_search_options parameter
|
||||
if web_search_tools:
|
||||
new_kwargs["web_search_options"] = {} # type: ignore
|
||||
|
||||
# Only translate regular tools (non-web-search)
|
||||
if regular_tools:
|
||||
(
|
||||
new_kwargs["tools"],
|
||||
tool_name_mapping,
|
||||
) = self.translate_anthropic_tools_to_openai(
|
||||
tools=cast(List[AllAnthropicToolsValues], regular_tools),
|
||||
model=new_kwargs.get("model"),
|
||||
)
|
||||
|
||||
tool_name_mapping = self._translate_tools_to_openai(
|
||||
anthropic_message_request=anthropic_message_request,
|
||||
new_kwargs=new_kwargs,
|
||||
)
|
||||
## CONVERT THINKING
|
||||
if "thinking" in anthropic_message_request:
|
||||
thinking = anthropic_message_request["thinking"]
|
||||
if thinking:
|
||||
model = new_kwargs.get("model", "")
|
||||
if self.is_anthropic_claude_model(model):
|
||||
new_kwargs["thinking"] = thinking # type: ignore
|
||||
else:
|
||||
reasoning_effort = (
|
||||
self.translate_anthropic_thinking_to_reasoning_effort(
|
||||
cast(Dict[str, Any], thinking)
|
||||
)
|
||||
)
|
||||
if reasoning_effort:
|
||||
new_kwargs["reasoning_effort"] = reasoning_effort
|
||||
|
||||
self._translate_thinking_to_openai(
|
||||
anthropic_message_request=anthropic_message_request,
|
||||
new_kwargs=new_kwargs,
|
||||
)
|
||||
## CONVERT OUTPUT_FORMAT to RESPONSE_FORMAT
|
||||
if "output_format" in anthropic_message_request:
|
||||
output_format = anthropic_message_request["output_format"]
|
||||
if output_format:
|
||||
response_format = self.translate_anthropic_output_format_to_openai(
|
||||
output_format=output_format
|
||||
)
|
||||
if response_format:
|
||||
new_kwargs["response_format"] = response_format
|
||||
|
||||
translatable_params = self.translatable_anthropic_params()
|
||||
for k, v in anthropic_message_request.items():
|
||||
if k not in translatable_params: # pass remaining params as is
|
||||
new_kwargs[k] = v # type: ignore
|
||||
self._translate_output_format_to_openai(
|
||||
anthropic_message_request=anthropic_message_request,
|
||||
new_kwargs=new_kwargs,
|
||||
)
|
||||
self._copy_untranslated_anthropic_params(
|
||||
anthropic_message_request=anthropic_message_request,
|
||||
new_kwargs=new_kwargs,
|
||||
)
|
||||
|
||||
return new_kwargs, tool_name_mapping
|
||||
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@
|
|||
import asyncio
|
||||
import contextvars
|
||||
from functools import partial
|
||||
from typing import Any, AsyncIterator, Coroutine, Dict, List, Optional, Union
|
||||
from typing import Any, AsyncIterator, Coroutine, Dict, List, Optional, Union, cast
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
|
|
@ -114,6 +114,55 @@ async def _execute_pre_request_hooks(
|
|||
return request_kwargs
|
||||
|
||||
|
||||
async def _try_websearch_short_circuit(
|
||||
model: str,
|
||||
messages: List[Dict],
|
||||
tools: Optional[List[Dict]],
|
||||
custom_llm_provider: Optional[str],
|
||||
stream: Optional[bool],
|
||||
) -> Optional[Union[AnthropicMessagesResponse, AsyncIterator]]:
|
||||
"""
|
||||
Attempt to short-circuit a web-search-only request.
|
||||
|
||||
Claude Code sends web search as a separate, standalone /v1/messages
|
||||
request. For providers that don't natively support web search (e.g.
|
||||
github_copilot), we detect this pattern, execute the search via
|
||||
Tavily/Perplexity, and return a synthetic Anthropic response — bypassing
|
||||
the backend LLM entirely.
|
||||
|
||||
Returns the synthetic response if short-circuited, or None to continue
|
||||
normal processing.
|
||||
"""
|
||||
if not litellm.callbacks:
|
||||
return None
|
||||
|
||||
from litellm.integrations.websearch_interception.handler import (
|
||||
WebSearchInterceptionLogger,
|
||||
)
|
||||
|
||||
for callback in litellm.callbacks:
|
||||
if not isinstance(callback, WebSearchInterceptionLogger):
|
||||
continue
|
||||
|
||||
response = await callback.try_short_circuit_search(
|
||||
model=model,
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
if response is not None:
|
||||
anthropic_response = cast(AnthropicMessagesResponse, response)
|
||||
if stream:
|
||||
from litellm.llms.anthropic.experimental_pass_through.messages.fake_stream_iterator import (
|
||||
FakeAnthropicMessagesStreamIterator,
|
||||
)
|
||||
|
||||
return FakeAnthropicMessagesStreamIterator(anthropic_response)
|
||||
return anthropic_response
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@client
|
||||
async def anthropic_messages(
|
||||
max_tokens: int,
|
||||
|
|
@ -138,6 +187,12 @@ async def anthropic_messages(
|
|||
"""
|
||||
Async: Make llm api request in Anthropic /messages API spec
|
||||
"""
|
||||
# Save original stream flag before pre-request hooks can convert it.
|
||||
# The websearch interception hook converts stream=True → stream=False
|
||||
# for the agentic loop, but the short-circuit path needs to know
|
||||
# whether the caller originally requested streaming.
|
||||
original_stream = stream
|
||||
|
||||
# Execute pre-request hooks to allow CustomLoggers to modify request
|
||||
request_kwargs = await _execute_pre_request_hooks(
|
||||
model=model,
|
||||
|
|
@ -151,11 +206,38 @@ async def anthropic_messages(
|
|||
# Extract modified parameters
|
||||
tools = request_kwargs.pop("tools", tools)
|
||||
stream = request_kwargs.pop("stream", stream)
|
||||
# Propagate the provider derived inside pre-request hooks, if not already set.
|
||||
# The litellm_params dict may have been overwritten by **kwargs in
|
||||
# _execute_pre_request_hooks, so fall back to get_llm_provider() if needed.
|
||||
if not custom_llm_provider:
|
||||
custom_llm_provider = request_kwargs.get("litellm_params", {}).get(
|
||||
"custom_llm_provider"
|
||||
)
|
||||
if not custom_llm_provider:
|
||||
try:
|
||||
_, custom_llm_provider, _, _ = litellm.get_llm_provider(model=model)
|
||||
except Exception:
|
||||
pass
|
||||
# Remove litellm_params from kwargs (only needed for hooks)
|
||||
request_kwargs.pop("litellm_params", None)
|
||||
# Merge back any other modifications
|
||||
kwargs.update(request_kwargs)
|
||||
|
||||
# Short-circuit web-search-only requests: detect the pattern, execute
|
||||
# search directly via Tavily/Perplexity, and return a synthetic response
|
||||
# without ever touching the backend LLM or the adapter path.
|
||||
# Use original_stream (not the hook-converted stream) so streaming
|
||||
# callers get SSE events instead of a plain dict.
|
||||
short_circuit_response = await _try_websearch_short_circuit(
|
||||
model=model,
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
stream=original_stream,
|
||||
)
|
||||
if short_circuit_response is not None:
|
||||
return short_circuit_response
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
kwargs["is_async"] = True
|
||||
|
||||
|
|
|
|||
|
|
@ -8,6 +8,9 @@ path used for OpenAI and Azure models.
|
|||
import json
|
||||
from typing import Any, Dict, List, Optional, Union, cast
|
||||
|
||||
from litellm.llms.anthropic.experimental_pass_through.utils import (
|
||||
is_reasoning_auto_summary_enabled,
|
||||
)
|
||||
from litellm.types.llms.anthropic import (
|
||||
AllAnthropicToolsValues,
|
||||
AnthopicMessagesAssistantMessageParam,
|
||||
|
|
@ -94,7 +97,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
|
|||
)
|
||||
elif btype == "image":
|
||||
url = self._translate_anthropic_image_source_to_url(
|
||||
block.get("source", {})
|
||||
cast(dict, block.get("source", {}))
|
||||
)
|
||||
if url:
|
||||
user_parts.append(
|
||||
|
|
@ -267,7 +270,14 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
|
|||
effort = "low"
|
||||
else:
|
||||
effort = "minimal"
|
||||
return {"effort": effort, "summary": "detailed"}
|
||||
auto_summary = is_reasoning_auto_summary_enabled()
|
||||
result: Dict[str, Any] = {"effort": effort}
|
||||
summary = thinking.get("summary")
|
||||
if summary:
|
||||
result["summary"] = summary
|
||||
elif auto_summary:
|
||||
result["summary"] = "detailed"
|
||||
return result
|
||||
|
||||
def translate_request(
|
||||
self,
|
||||
|
|
|
|||
11
litellm/llms/anthropic/experimental_pass_through/utils.py
Normal file
11
litellm/llms/anthropic/experimental_pass_through/utils.py
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
import os
|
||||
|
||||
import litellm
|
||||
|
||||
|
||||
def is_reasoning_auto_summary_enabled() -> bool:
|
||||
"""Check whether the default 'summary: detailed' injection is enabled (opt-in)."""
|
||||
return (
|
||||
litellm.reasoning_auto_summary
|
||||
or os.getenv("LITELLM_REASONING_AUTO_SUMMARY", "false").lower() == "true"
|
||||
)
|
||||
|
|
@ -131,14 +131,9 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
|
|||
if result_effort == "none" and not supports_none:
|
||||
result.pop("reasoning_effort")
|
||||
|
||||
# Azure Chat Completions: gpt-5.4+ does not support tools + reasoning together.
|
||||
# Drop reasoning_effort when both are present (OpenAI routes to Responses API; Azure does not).
|
||||
if self.is_model_gpt_5_4_plus_model(model):
|
||||
has_tools = bool(
|
||||
non_default_params.get("tools") or optional_params.get("tools")
|
||||
)
|
||||
if has_tools and result_effort not in (None, "none"):
|
||||
result.pop("reasoning_effort", None)
|
||||
# Azure gpt-5.4+ with tools + reasoning_effort is now routed to the
|
||||
# Responses API bridge (same as OpenAI), so we no longer need to drop
|
||||
# reasoning_effort here. See: responses_api_bridge_check() in main.py.
|
||||
|
||||
return result
|
||||
|
||||
|
|
|
|||
|
|
@ -97,14 +97,59 @@ class AzureAIAgentsHandler:
|
|||
# -------------------------------------------------------------------------
|
||||
# Response Helpers
|
||||
# -------------------------------------------------------------------------
|
||||
def _extract_content_from_messages(self, messages_data: dict) -> str:
|
||||
"""Extract assistant content from the messages response."""
|
||||
def _extract_content_from_messages(
|
||||
self, messages_data: dict
|
||||
) -> Tuple[str, Optional[List[Dict[str, Any]]]]:
|
||||
"""Extract assistant content and annotations from the messages response.
|
||||
|
||||
Returns (content, annotations) where annotations is a list of
|
||||
OpenAI-compatible ChatCompletionAnnotation dicts, or None.
|
||||
"""
|
||||
for msg in messages_data.get("data", []):
|
||||
if msg.get("role") == "assistant":
|
||||
for content_item in msg.get("content", []):
|
||||
if content_item.get("type") == "text":
|
||||
return content_item.get("text", {}).get("value", "")
|
||||
return ""
|
||||
text_obj = content_item.get("text", {})
|
||||
content = text_obj.get("value", "")
|
||||
raw_annotations = text_obj.get("annotations")
|
||||
annotations = self._transform_annotations(raw_annotations)
|
||||
return content, annotations
|
||||
return "", None
|
||||
|
||||
def _transform_annotations(
|
||||
self,
|
||||
raw_annotations: Optional[List[Dict[str, Any]]],
|
||||
) -> Optional[List[Dict[str, Any]]]:
|
||||
"""Transform Azure AI Foundry annotations to OpenAI-compatible format.
|
||||
|
||||
Azure AI returns annotations like:
|
||||
{"type": "url_citation", "text": "[1]", "start_index": 10,
|
||||
"end_index": 13, "url_citation": {"url": "...", "title": "..."}}
|
||||
|
||||
OpenAI expects:
|
||||
{"type": "url_citation", "url_citation": {"url": "...", "title": "...",
|
||||
"start_index": 10, "end_index": 13}}
|
||||
"""
|
||||
if not raw_annotations:
|
||||
return None
|
||||
|
||||
result: List[Dict[str, Any]] = []
|
||||
for ann in raw_annotations:
|
||||
ann_type = ann.get("type")
|
||||
if ann_type == "url_citation":
|
||||
url_citation = dict(ann.get("url_citation", {}))
|
||||
# Azure puts start/end_index at annotation level; OpenAI
|
||||
# expects them inside url_citation
|
||||
if "start_index" in ann and "start_index" not in url_citation:
|
||||
url_citation["start_index"] = ann["start_index"]
|
||||
if "end_index" in ann and "end_index" not in url_citation:
|
||||
url_citation["end_index"] = ann["end_index"]
|
||||
result.append({"type": "url_citation", "url_citation": url_citation})
|
||||
else:
|
||||
# Pass through unknown annotation types as-is
|
||||
result.append(ann)
|
||||
|
||||
return result if result else None
|
||||
|
||||
def _build_model_response(
|
||||
self,
|
||||
|
|
@ -113,15 +158,23 @@ class AzureAIAgentsHandler:
|
|||
model_response: ModelResponse,
|
||||
thread_id: str,
|
||||
messages: List[Dict[str, Any]],
|
||||
annotations: Optional[List[Dict[str, Any]]] = None,
|
||||
) -> ModelResponse:
|
||||
"""Build the ModelResponse from agent output."""
|
||||
from litellm.types.utils import Choices, Message, Usage
|
||||
|
||||
message_kwargs: Dict[str, Any] = {
|
||||
"content": content,
|
||||
"role": "assistant",
|
||||
}
|
||||
if annotations:
|
||||
message_kwargs["annotations"] = annotations
|
||||
|
||||
model_response.choices = [
|
||||
Choices(
|
||||
finish_reason="stop",
|
||||
index=0,
|
||||
message=Message(content=content, role="assistant"),
|
||||
message=Message(**message_kwargs),
|
||||
)
|
||||
]
|
||||
model_response.model = model
|
||||
|
|
@ -250,7 +303,7 @@ class AzureAIAgentsHandler:
|
|||
)
|
||||
|
||||
# Execute the agent flow
|
||||
thread_id, content = self._execute_agent_flow_sync(
|
||||
thread_id, content, annotations = self._execute_agent_flow_sync(
|
||||
make_request=make_request,
|
||||
api_base=api_base,
|
||||
api_version=api_version,
|
||||
|
|
@ -261,7 +314,7 @@ class AzureAIAgentsHandler:
|
|||
)
|
||||
|
||||
return self._build_model_response(
|
||||
model, content, model_response, thread_id, messages
|
||||
model, content, model_response, thread_id, messages, annotations
|
||||
)
|
||||
|
||||
def _execute_agent_flow_sync(
|
||||
|
|
@ -273,8 +326,8 @@ class AzureAIAgentsHandler:
|
|||
thread_id: Optional[str],
|
||||
messages: List[Dict[str, Any]],
|
||||
optional_params: dict,
|
||||
) -> Tuple[str, str]:
|
||||
"""Execute the agent flow synchronously. Returns (thread_id, content)."""
|
||||
) -> Tuple[str, str, Optional[List[Dict[str, Any]]]]:
|
||||
"""Execute the agent flow synchronously. Returns (thread_id, content, annotations)."""
|
||||
|
||||
# Step 1: Create thread if not provided
|
||||
if not thread_id:
|
||||
|
|
@ -347,8 +400,8 @@ class AzureAIAgentsHandler:
|
|||
)
|
||||
self._check_response(response, [200], "Failed to get messages")
|
||||
|
||||
content = self._extract_content_from_messages(response.json())
|
||||
return thread_id, content
|
||||
content, annotations = self._extract_content_from_messages(response.json())
|
||||
return thread_id, content, annotations
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Async Completion
|
||||
|
|
@ -399,7 +452,7 @@ class AzureAIAgentsHandler:
|
|||
)
|
||||
|
||||
# Execute the agent flow
|
||||
thread_id, content = await self._execute_agent_flow_async(
|
||||
thread_id, content, annotations = await self._execute_agent_flow_async(
|
||||
make_request=make_request,
|
||||
api_base=api_base,
|
||||
api_version=api_version,
|
||||
|
|
@ -410,7 +463,7 @@ class AzureAIAgentsHandler:
|
|||
)
|
||||
|
||||
return self._build_model_response(
|
||||
model, content, model_response, thread_id, messages
|
||||
model, content, model_response, thread_id, messages, annotations
|
||||
)
|
||||
|
||||
async def _execute_agent_flow_async(
|
||||
|
|
@ -422,8 +475,8 @@ class AzureAIAgentsHandler:
|
|||
thread_id: Optional[str],
|
||||
messages: List[Dict[str, Any]],
|
||||
optional_params: dict,
|
||||
) -> Tuple[str, str]:
|
||||
"""Execute the agent flow asynchronously. Returns (thread_id, content)."""
|
||||
) -> Tuple[str, str, Optional[List[Dict[str, Any]]]]:
|
||||
"""Execute the agent flow asynchronously. Returns (thread_id, content, annotations)."""
|
||||
|
||||
# Step 1: Create thread if not provided
|
||||
if not thread_id:
|
||||
|
|
@ -496,8 +549,8 @@ class AzureAIAgentsHandler:
|
|||
)
|
||||
self._check_response(response, [200], "Failed to get messages")
|
||||
|
||||
content = self._extract_content_from_messages(response.json())
|
||||
return thread_id, content
|
||||
content, annotations = self._extract_content_from_messages(response.json())
|
||||
return thread_id, content, annotations
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Streaming Completion (Native SSE)
|
||||
|
|
@ -585,6 +638,7 @@ class AzureAIAgentsHandler:
|
|||
response_id = f"chatcmpl-{uuid.uuid4().hex[:8]}"
|
||||
created = int(time.time())
|
||||
thread_id = None
|
||||
collected_annotations: Optional[List[Dict[str, Any]]] = None
|
||||
|
||||
current_event = None
|
||||
|
||||
|
|
@ -600,6 +654,9 @@ class AzureAIAgentsHandler:
|
|||
|
||||
if data_str == "[DONE]":
|
||||
# Send final chunk with finish_reason
|
||||
final_delta_kwargs: Dict[str, Any] = {"content": None}
|
||||
if collected_annotations:
|
||||
final_delta_kwargs["annotations"] = collected_annotations
|
||||
final_chunk = ModelResponseStream(
|
||||
id=response_id,
|
||||
created=created,
|
||||
|
|
@ -609,7 +666,7 @@ class AzureAIAgentsHandler:
|
|||
StreamingChoices(
|
||||
finish_reason="stop",
|
||||
index=0,
|
||||
delta=Delta(content=None),
|
||||
delta=Delta(**final_delta_kwargs),
|
||||
)
|
||||
],
|
||||
)
|
||||
|
|
@ -628,6 +685,19 @@ class AzureAIAgentsHandler:
|
|||
thread_id = data["id"]
|
||||
verbose_logger.debug(f"Stream created thread: {thread_id}")
|
||||
|
||||
# Extract annotations from completed message
|
||||
if current_event == "thread.message.completed":
|
||||
for content_item in data.get("content", []):
|
||||
if content_item.get("type") == "text":
|
||||
raw_annotations = content_item.get("text", {}).get(
|
||||
"annotations"
|
||||
)
|
||||
transformed = self._transform_annotations(raw_annotations)
|
||||
if transformed:
|
||||
if collected_annotations is None:
|
||||
collected_annotations = []
|
||||
collected_annotations.extend(transformed)
|
||||
|
||||
# Process message deltas - this is where the actual content comes
|
||||
if current_event == "thread.message.delta":
|
||||
delta_content = data.get("delta", {}).get("content", [])
|
||||
|
|
|
|||
|
|
@ -73,6 +73,7 @@ class BaseTranslation(ABC):
|
|||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: Optional["LiteLLMLoggingObj"] = None,
|
||||
user_api_key_dict: Optional["UserAPIKeyAuth"] = None,
|
||||
request_data: Optional[dict] = None,
|
||||
) -> Any:
|
||||
"""
|
||||
Process output response with guardrails.
|
||||
|
|
@ -91,6 +92,7 @@ class BaseTranslation(ABC):
|
|||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: Optional["LiteLLMLoggingObj"] = None,
|
||||
user_api_key_dict: Optional["UserAPIKeyAuth"] = None,
|
||||
request_data: Optional[dict] = None,
|
||||
) -> Any:
|
||||
"""
|
||||
Process output streaming response with guardrails.
|
||||
|
|
|
|||
|
|
@ -54,6 +54,14 @@ class BaseResponsesAPIConfig(ABC):
|
|||
and v is not None
|
||||
}
|
||||
|
||||
def supports_native_file_search(self) -> bool:
|
||||
"""Return True if this provider handles the file_search tool natively.
|
||||
|
||||
Override in provider subclasses that support file_search without
|
||||
LiteLLM emulation (e.g. OpenAI, Azure OpenAI).
|
||||
"""
|
||||
return False
|
||||
|
||||
@abstractmethod
|
||||
def get_supported_openai_params(self, model: str) -> list:
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -1446,6 +1446,16 @@ class AmazonConverseConfig(BaseConfig):
|
|||
original_tools, model, headers, additional_request_params
|
||||
)
|
||||
|
||||
# Append cachePoint to tools if cache_control_injection_points has tool_config
|
||||
cache_injection_points = additional_request_params.pop(
|
||||
"cache_control_injection_points", None
|
||||
)
|
||||
if cache_injection_points and len(bedrock_tools) > 0:
|
||||
for point in cache_injection_points:
|
||||
if point.get("location") == "tool_config":
|
||||
bedrock_tools.append({"cachePoint": {"type": "default"}})
|
||||
break
|
||||
|
||||
bedrock_tool_config: Optional[ToolConfigBlock] = None
|
||||
if len(bedrock_tools) > 0:
|
||||
tool_choice_values: ToolChoiceValuesBlock = inference_params.pop(
|
||||
|
|
|
|||
|
|
@ -64,8 +64,15 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig):
|
|||
verbose_logger.debug(f"Transformed request: {bedrock_request}")
|
||||
|
||||
# Get endpoint URL using simplified function
|
||||
api_base = litellm_params.get("api_base", None)
|
||||
aws_bedrock_runtime_endpoint = litellm_params.get(
|
||||
"aws_bedrock_runtime_endpoint", None
|
||||
)
|
||||
endpoint_url = self.get_bedrock_count_tokens_endpoint(
|
||||
resolved_model, aws_region_name
|
||||
model=resolved_model,
|
||||
aws_region_name=aws_region_name,
|
||||
api_base=api_base,
|
||||
aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint,
|
||||
)
|
||||
|
||||
verbose_logger.debug(f"Making request to: {endpoint_url}")
|
||||
|
|
|
|||
|
|
@ -177,7 +177,11 @@ class BedrockCountTokensConfig(BaseAWSLLM):
|
|||
return {"input": {"invokeModel": {"body": json.dumps(body_data)}}}
|
||||
|
||||
def get_bedrock_count_tokens_endpoint(
|
||||
self, model: str, aws_region_name: str
|
||||
self,
|
||||
model: str,
|
||||
aws_region_name: str,
|
||||
api_base: Optional[str] = None,
|
||||
aws_bedrock_runtime_endpoint: Optional[str] = None,
|
||||
) -> str:
|
||||
"""
|
||||
Construct the AWS Bedrock CountTokens API endpoint using existing LiteLLM functions.
|
||||
|
|
@ -185,6 +189,8 @@ class BedrockCountTokensConfig(BaseAWSLLM):
|
|||
Args:
|
||||
model: The resolved model ID from router lookup
|
||||
aws_region_name: AWS region (e.g., "eu-west-1")
|
||||
api_base: Optional custom API base URL (takes highest priority)
|
||||
aws_bedrock_runtime_endpoint: Optional custom Bedrock runtime endpoint
|
||||
|
||||
Returns:
|
||||
Complete endpoint URL for CountTokens API
|
||||
|
|
@ -196,7 +202,11 @@ class BedrockCountTokensConfig(BaseAWSLLM):
|
|||
if model_id.startswith("bedrock/"):
|
||||
model_id = model_id[8:] # Remove "bedrock/" prefix
|
||||
|
||||
base_url = f"https://bedrock-runtime.{aws_region_name}.amazonaws.com"
|
||||
base_url, _ = self.get_runtime_endpoint(
|
||||
api_base=api_base,
|
||||
aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint,
|
||||
aws_region_name=aws_region_name,
|
||||
)
|
||||
endpoint = f"{base_url}/model/{model_id}/count-tokens"
|
||||
|
||||
return endpoint
|
||||
|
|
|
|||
|
|
@ -83,6 +83,7 @@ class CohereRerankHandler(BaseTranslation):
|
|||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: Optional[Any] = None,
|
||||
user_api_key_dict: Optional[Any] = None,
|
||||
request_data: Optional[dict] = None,
|
||||
) -> Any:
|
||||
"""
|
||||
Process output response - not applicable for rerank.
|
||||
|
|
|
|||
|
|
@ -185,11 +185,16 @@ class FireworksAIConfig(OpenAIGPTConfig):
|
|||
): # allow user to toggle this feature.
|
||||
return content
|
||||
if isinstance(content["image_url"], str):
|
||||
content["image_url"] = f"{content['image_url']}#transform=inline"
|
||||
# Skip base64 data URLs — appending #transform=inline corrupts the
|
||||
# base64 payload and causes an "Incorrect padding" decode error on
|
||||
# the Fireworks side. Data URLs are already inlined by definition.
|
||||
# Lower-case before checking: URI schemes are case-insensitive (RFC 3986).
|
||||
if not content["image_url"].lower().startswith("data:"):
|
||||
content["image_url"] = f"{content['image_url']}#transform=inline"
|
||||
elif isinstance(content["image_url"], dict):
|
||||
content["image_url"][
|
||||
"url"
|
||||
] = f"{content['image_url']['url']}#transform=inline"
|
||||
url = content["image_url"]["url"]
|
||||
if not url.lower().startswith("data:"):
|
||||
content["image_url"]["url"] = f"{url}#transform=inline"
|
||||
return content
|
||||
|
||||
def _transform_tools(
|
||||
|
|
|
|||
|
|
@ -88,12 +88,15 @@ class GoogleImageGenConfig(BaseImageGenerationConfig):
|
|||
tokens_details = usage_metadata.get("promptTokensDetails", [])
|
||||
for details in tokens_details:
|
||||
if isinstance(details, dict):
|
||||
modality = details.get("modality")
|
||||
token_count = details.get("tokenCount", 0)
|
||||
modality = str(details.get("modality", "")).upper()
|
||||
raw_token_count = details.get(
|
||||
"tokenCount", details.get("token_count", 0)
|
||||
)
|
||||
token_count = raw_token_count if isinstance(raw_token_count, int) else 0
|
||||
if modality == "TEXT":
|
||||
input_tokens_details.text_tokens = token_count
|
||||
input_tokens_details.text_tokens += token_count
|
||||
elif modality == "IMAGE":
|
||||
input_tokens_details.image_tokens = token_count
|
||||
input_tokens_details.image_tokens += token_count
|
||||
|
||||
return ImageUsage(
|
||||
input_tokens=usage_metadata.get("promptTokenCount", 0),
|
||||
|
|
|
|||
|
|
@ -148,5 +148,12 @@ class MistralAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
|
|||
|
||||
text = response_json.get("text") or ""
|
||||
response = TranscriptionResponse(text=text)
|
||||
|
||||
# Preserve Mistral-specific fields (e.g. diarization segments)
|
||||
if "segments" in response_json:
|
||||
response["segments"] = response_json["segments"]
|
||||
if "language" in response_json:
|
||||
response["language"] = response_json["language"]
|
||||
|
||||
response._hidden_params = response_json
|
||||
return response
|
||||
|
|
|
|||
|
|
@ -91,6 +91,7 @@ class OCRHandler(BaseTranslation):
|
|||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: Optional[Any] = None,
|
||||
user_api_key_dict: Optional[Any] = None,
|
||||
request_data: Optional[dict] = None,
|
||||
) -> Any:
|
||||
"""
|
||||
Process OCR output by applying guardrails to extracted page text.
|
||||
|
|
@ -127,14 +128,27 @@ class OCRHandler(BaseTranslation):
|
|||
if model:
|
||||
inputs["model"] = model
|
||||
|
||||
# Use the real request_data if provided (proxy path), otherwise
|
||||
# create a standalone dict (SDK / direct-call path).
|
||||
if request_data is None:
|
||||
request_data = {}
|
||||
|
||||
# Add user metadata if available
|
||||
if user_api_key_dict is not None:
|
||||
metadata = self.transform_user_api_key_dict_to_metadata(user_api_key_dict)
|
||||
inputs.update(metadata) # type: ignore
|
||||
user_metadata = self.transform_user_api_key_dict_to_metadata(
|
||||
user_api_key_dict
|
||||
)
|
||||
if user_metadata:
|
||||
# Preserve original behavior: inject metadata into inputs for
|
||||
# third-party guardrail providers that read it from there
|
||||
inputs.update(user_metadata) # type: ignore
|
||||
# Also store in request_data for the logging pipeline
|
||||
if "litellm_metadata" not in request_data:
|
||||
request_data["litellm_metadata"] = user_metadata
|
||||
|
||||
guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
|
||||
inputs=inputs,
|
||||
request_data={},
|
||||
request_data=request_data,
|
||||
input_type="response",
|
||||
logging_obj=litellm_logging_obj,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -36,6 +36,8 @@ class MistralOCRConfig(BaseOCRConfig):
|
|||
- image_min_size: Minimum size of images to include
|
||||
- bbox_annotation_format: Format for bounding box annotations
|
||||
- document_annotation_format: Format for document annotations
|
||||
- extract_header: Whether to extract document header
|
||||
- extract_footer: Whether to extract document footer
|
||||
"""
|
||||
return [
|
||||
"pages",
|
||||
|
|
@ -44,6 +46,8 @@ class MistralOCRConfig(BaseOCRConfig):
|
|||
"image_min_size",
|
||||
"bbox_annotation_format",
|
||||
"document_annotation_format",
|
||||
"extract_header",
|
||||
"extract_footer",
|
||||
]
|
||||
|
||||
def map_ocr_params(
|
||||
|
|
|
|||
|
|
@ -155,9 +155,11 @@ class MoonshotChatConfig(OpenAIGPTConfig):
|
|||
message that contains tool_calls (multi-turn tool-calling flows).
|
||||
|
||||
For each such message that is missing the field:
|
||||
1. Promote provider_specific_fields["reasoning_content"] if present and non-empty
|
||||
1. Check if reasoning_content exists at the top level (for Pydantic models
|
||||
that have the attribute but don't support 'in' operator)
|
||||
2. Promote provider_specific_fields["reasoning_content"] if present and non-empty
|
||||
(this is where LiteLLM stores it from a previous response)
|
||||
2. Otherwise inject a single space — the minimum value the API accepts
|
||||
3. Otherwise inject a single space — the minimum value the API accepts
|
||||
Messages that already carry the field, or are not assistant/tool-call messages,
|
||||
are appended as-is (no copy made).
|
||||
"""
|
||||
|
|
@ -166,7 +168,9 @@ class MoonshotChatConfig(OpenAIGPTConfig):
|
|||
if (
|
||||
msg.get("role") == "assistant"
|
||||
and msg.get("tool_calls")
|
||||
and "reasoning_content" not in msg
|
||||
and not msg.get(
|
||||
"reasoning_content"
|
||||
) # Check using .get() which works for both dicts and Pydantic models
|
||||
):
|
||||
patched = dict(cast(dict, msg))
|
||||
provider_fields = patched.get("provider_specific_fields") or {}
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
from typing import Optional, Union
|
||||
|
||||
import litellm
|
||||
from litellm.utils import _supports_factory
|
||||
from litellm.utils import _is_explicitly_disabled_factory, _supports_factory
|
||||
|
||||
from .gpt_transformation import OpenAIGPTConfig
|
||||
|
||||
|
|
@ -113,6 +113,25 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
|
|||
key=f"supports_{level}_reasoning_effort",
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _is_reasoning_effort_level_explicitly_disabled(
|
||||
cls, model: str, level: str
|
||||
) -> bool:
|
||||
"""Return True only when the model map explicitly sets the capability to False.
|
||||
|
||||
Unlike ``_supports_reasoning_effort_level`` (which requires an explicit True),
|
||||
this method returns True only when ``supports_{level}_reasoning_effort`` is
|
||||
explicitly set to ``False`` in the model map. A missing key is treated as
|
||||
supported (i.e. this method returns False = not disabled).
|
||||
|
||||
Use this for opt-out checks where unknown models should be allowed through.
|
||||
"""
|
||||
return _is_explicitly_disabled_factory(
|
||||
model=model,
|
||||
custom_llm_provider=None,
|
||||
key=f"supports_{level}_reasoning_effort",
|
||||
)
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list:
|
||||
if self.is_model_gpt_5_search_model(model):
|
||||
return [
|
||||
|
|
@ -200,14 +219,32 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
|
|||
if "reasoning_effort" in optional_params:
|
||||
optional_params["reasoning_effort"] = normalized
|
||||
|
||||
if effective_effort is not None and effective_effort == "xhigh":
|
||||
if not self._supports_reasoning_effort_level(model, "xhigh"):
|
||||
if effective_effort == "xhigh":
|
||||
# xhigh is an opt-in capability: only allow if model explicitly supports it.
|
||||
if not self._supports_reasoning_effort_level(model, effective_effort):
|
||||
if litellm.drop_params or drop_params:
|
||||
non_default_params.pop("reasoning_effort", None)
|
||||
optional_params.pop("reasoning_effort", None)
|
||||
else:
|
||||
raise litellm.utils.UnsupportedParamsError(
|
||||
message=(
|
||||
"reasoning_effort='xhigh' is only supported for gpt-5.1-codex-max, gpt-5.2, and gpt-5.4+ models."
|
||||
f"reasoning_effort={effective_effort} is not supported for this model."
|
||||
),
|
||||
status_code=400,
|
||||
)
|
||||
elif effective_effort == "minimal":
|
||||
# minimal is opt-out: unknown models pass through; only block when
|
||||
# the model map explicitly sets supports_minimal_reasoning_effort=false.
|
||||
if self._is_reasoning_effort_level_explicitly_disabled(
|
||||
model, effective_effort
|
||||
):
|
||||
if litellm.drop_params or drop_params:
|
||||
non_default_params.pop("reasoning_effort", None)
|
||||
optional_params.pop("reasoning_effort", None)
|
||||
else:
|
||||
raise litellm.utils.UnsupportedParamsError(
|
||||
message=(
|
||||
f"reasoning_effort={effective_effort} is not supported for this model."
|
||||
),
|
||||
status_code=400,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ from typing import (
|
|||
Any,
|
||||
AsyncIterator,
|
||||
Coroutine,
|
||||
Dict,
|
||||
Iterator,
|
||||
List,
|
||||
Literal,
|
||||
|
|
@ -805,8 +806,8 @@ class OpenAIChatCompletionStreamingHandler(BaseModelResponseIterator):
|
|||
choices = chunk.get("choices", [])
|
||||
choices = self._map_reasoning_to_reasoning_content(choices)
|
||||
|
||||
kwargs = {
|
||||
"id": chunk["id"],
|
||||
kwargs: Dict[str, Any] = {
|
||||
"id": chunk.get("id"),
|
||||
"object": "chat.completion.chunk",
|
||||
"created": chunk.get("created"),
|
||||
"model": chunk.get("model"),
|
||||
|
|
|
|||
|
|
@ -86,9 +86,9 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
if tool_calls_to_check:
|
||||
inputs["tool_calls"] = tool_calls_to_check # type: ignore
|
||||
if messages:
|
||||
inputs[
|
||||
"structured_messages"
|
||||
] = messages # pass the openai /chat/completions messages to the guardrail, as-is
|
||||
inputs["structured_messages"] = (
|
||||
messages # pass the openai /chat/completions messages to the guardrail, as-is
|
||||
)
|
||||
# Pass tools (function definitions) to the guardrail
|
||||
tools = data.get("tools")
|
||||
if tools:
|
||||
|
|
@ -260,6 +260,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: Optional[Any] = None,
|
||||
user_api_key_dict: Optional[Any] = None,
|
||||
request_data: Optional[dict] = None,
|
||||
) -> Any:
|
||||
"""
|
||||
Process output response by applying guardrails to text content.
|
||||
|
|
@ -308,15 +309,21 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
|
||||
# Step 2: Apply guardrail to all texts and tool calls in batch
|
||||
if texts_to_check or tool_calls_to_check:
|
||||
# Create a request_data dict with response info and user API key metadata
|
||||
request_data: dict = {"response": response}
|
||||
# Use the real request_data if provided (proxy path), otherwise
|
||||
# create a standalone dict (SDK / direct-call path).
|
||||
if request_data is None:
|
||||
request_data = {"response": response}
|
||||
else:
|
||||
if "response" not in request_data:
|
||||
request_data["response"] = response
|
||||
|
||||
# Add user API key metadata with prefixed keys
|
||||
user_metadata = self.transform_user_api_key_dict_to_metadata(
|
||||
user_api_key_dict
|
||||
)
|
||||
if user_metadata:
|
||||
request_data["litellm_metadata"] = user_metadata
|
||||
if "litellm_metadata" not in request_data:
|
||||
user_metadata = self.transform_user_api_key_dict_to_metadata(
|
||||
user_api_key_dict
|
||||
)
|
||||
if user_metadata:
|
||||
request_data["litellm_metadata"] = user_metadata
|
||||
|
||||
inputs = GenericGuardrailAPIInputs(texts=texts_to_check)
|
||||
if images_to_check:
|
||||
|
|
@ -364,6 +371,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: Optional[Any] = None,
|
||||
user_api_key_dict: Optional[Any] = None,
|
||||
request_data: Optional[dict] = None,
|
||||
) -> List["ModelResponseStream"]:
|
||||
"""
|
||||
Process output streaming responses by applying guardrails to text content.
|
||||
|
|
@ -402,6 +410,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
guardrail_to_apply=guardrail_to_apply,
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
request_data=request_data,
|
||||
)
|
||||
|
||||
return responses_so_far
|
||||
|
|
@ -436,15 +445,21 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
|
||||
# Step 3: Apply guardrail to all combined texts in batch
|
||||
if texts_to_check:
|
||||
# Create a request_data dict with response info and user API key metadata
|
||||
request_data: dict = {"responses": responses_so_far}
|
||||
# Use the real request_data if provided (proxy path), otherwise
|
||||
# create a standalone dict (SDK / direct-call path).
|
||||
if request_data is None:
|
||||
request_data = {"responses": responses_so_far}
|
||||
else:
|
||||
if "responses" not in request_data:
|
||||
request_data["responses"] = responses_so_far
|
||||
|
||||
# Add user API key metadata with prefixed keys
|
||||
user_metadata = self.transform_user_api_key_dict_to_metadata(
|
||||
user_api_key_dict
|
||||
)
|
||||
if user_metadata:
|
||||
request_data["litellm_metadata"] = user_metadata
|
||||
if "litellm_metadata" not in request_data:
|
||||
user_metadata = self.transform_user_api_key_dict_to_metadata(
|
||||
user_api_key_dict
|
||||
)
|
||||
if user_metadata:
|
||||
request_data["litellm_metadata"] = user_metadata
|
||||
|
||||
inputs = GenericGuardrailAPIInputs(texts=texts_to_check)
|
||||
if images_to_check:
|
||||
|
|
|
|||
|
|
@ -125,6 +125,7 @@ class OpenAITextCompletionHandler(BaseTranslation):
|
|||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: Optional[Any] = None,
|
||||
user_api_key_dict: Optional[Any] = None,
|
||||
request_data: Optional[dict] = None,
|
||||
) -> Any:
|
||||
"""
|
||||
Process output response by applying guardrails to completion text.
|
||||
|
|
@ -155,15 +156,21 @@ class OpenAITextCompletionHandler(BaseTranslation):
|
|||
|
||||
# Apply guardrails in batch
|
||||
if texts_to_check:
|
||||
# Create a request_data dict with response info and user API key metadata
|
||||
request_data: dict = {"response": response}
|
||||
# Use the real request_data if provided (proxy path), otherwise
|
||||
# create a standalone dict (SDK / direct-call path).
|
||||
if request_data is None:
|
||||
request_data = {"response": response}
|
||||
else:
|
||||
if "response" not in request_data:
|
||||
request_data["response"] = response
|
||||
|
||||
# Add user API key metadata with prefixed keys
|
||||
user_metadata = self.transform_user_api_key_dict_to_metadata(
|
||||
user_api_key_dict
|
||||
)
|
||||
if user_metadata:
|
||||
request_data["litellm_metadata"] = user_metadata
|
||||
if "litellm_metadata" not in request_data:
|
||||
user_metadata = self.transform_user_api_key_dict_to_metadata(
|
||||
user_api_key_dict
|
||||
)
|
||||
if user_metadata:
|
||||
request_data["litellm_metadata"] = user_metadata
|
||||
|
||||
inputs = GenericGuardrailAPIInputs(texts=texts_to_check)
|
||||
# Include model information from the response if available
|
||||
|
|
|
|||
|
|
@ -155,6 +155,7 @@ class OpenAIEmbeddingsHandler(BaseTranslation):
|
|||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: Optional[Any] = None,
|
||||
user_api_key_dict: Optional[Any] = None,
|
||||
request_data: Optional[dict] = None,
|
||||
) -> Any:
|
||||
"""
|
||||
Process output response - embeddings responses contain vectors, not text.
|
||||
|
|
|
|||
|
|
@ -87,6 +87,7 @@ class OpenAIImageGenerationHandler(BaseTranslation):
|
|||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: Optional[Any] = None,
|
||||
user_api_key_dict: Optional[Any] = None,
|
||||
request_data: Optional[dict] = None,
|
||||
) -> Any:
|
||||
"""
|
||||
Process output response - typically not needed for image generation.
|
||||
|
|
|
|||
|
|
@ -347,6 +347,7 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: Optional[Any] = None,
|
||||
user_api_key_dict: Optional[Any] = None,
|
||||
request_data: Optional[dict] = None,
|
||||
) -> Any:
|
||||
"""
|
||||
Process output response by applying guardrails to text content and tool calls.
|
||||
|
|
@ -402,15 +403,21 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
|
||||
# Step 2: Apply guardrail to all texts in batch
|
||||
if texts_to_check or tool_calls_to_check:
|
||||
# Create a request_data dict with response info and user API key metadata
|
||||
request_data: dict = {"response": response}
|
||||
# Use the real request_data if provided (proxy path), otherwise
|
||||
# create a standalone dict (SDK / direct-call path).
|
||||
if request_data is None:
|
||||
request_data = {"response": response}
|
||||
else:
|
||||
if "response" not in request_data:
|
||||
request_data["response"] = response
|
||||
|
||||
# Add user API key metadata with prefixed keys
|
||||
user_metadata = self.transform_user_api_key_dict_to_metadata(
|
||||
user_api_key_dict
|
||||
)
|
||||
if user_metadata:
|
||||
request_data["litellm_metadata"] = user_metadata
|
||||
if "litellm_metadata" not in request_data:
|
||||
user_metadata = self.transform_user_api_key_dict_to_metadata(
|
||||
user_api_key_dict
|
||||
)
|
||||
if user_metadata:
|
||||
request_data["litellm_metadata"] = user_metadata
|
||||
|
||||
inputs = GenericGuardrailAPIInputs(texts=texts_to_check)
|
||||
if images_to_check:
|
||||
|
|
@ -454,6 +461,7 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: Optional[Any] = None,
|
||||
user_api_key_dict: Optional[Any] = None,
|
||||
request_data: Optional[dict] = None,
|
||||
) -> List[Any]:
|
||||
"""
|
||||
Process output streaming response by applying guardrails to text content.
|
||||
|
|
@ -481,7 +489,7 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
inputs["model"] = model_response_stream.model
|
||||
_guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
|
||||
inputs=inputs,
|
||||
request_data={},
|
||||
request_data=request_data if request_data is not None else {},
|
||||
input_type="response",
|
||||
logging_obj=litellm_logging_obj,
|
||||
)
|
||||
|
|
@ -512,7 +520,7 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
if tool_calls or text:
|
||||
_guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
|
||||
inputs=guardrail_inputs,
|
||||
request_data={},
|
||||
request_data=request_data if request_data is not None else {},
|
||||
input_type="response",
|
||||
logging_obj=litellm_logging_obj,
|
||||
)
|
||||
|
|
@ -537,7 +545,7 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
inputs["model"] = response_model
|
||||
_guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
|
||||
inputs=inputs,
|
||||
request_data={},
|
||||
request_data=request_data if request_data is not None else {},
|
||||
input_type="response",
|
||||
logging_obj=litellm_logging_obj,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -32,6 +32,9 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
|
|||
def custom_llm_provider(self) -> LlmProviders:
|
||||
return LlmProviders.OPENAI
|
||||
|
||||
def supports_native_file_search(self) -> bool:
|
||||
return True
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list:
|
||||
"""
|
||||
All OpenAI Responses API params are supported
|
||||
|
|
|
|||
|
|
@ -85,6 +85,7 @@ class OpenAITextToSpeechHandler(BaseTranslation):
|
|||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: Optional[Any] = None,
|
||||
user_api_key_dict: Optional[Any] = None,
|
||||
request_data: Optional[dict] = None,
|
||||
) -> Any:
|
||||
"""
|
||||
Process output - not applicable for text-to-speech.
|
||||
|
|
|
|||
|
|
@ -58,6 +58,7 @@ class OpenAIAudioTranscriptionHandler(BaseTranslation):
|
|||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: Optional[Any] = None,
|
||||
user_api_key_dict: Optional[Any] = None,
|
||||
request_data: Optional[dict] = None,
|
||||
) -> Any:
|
||||
"""
|
||||
Process output transcription by applying guardrails to transcribed text.
|
||||
|
|
@ -79,15 +80,21 @@ class OpenAIAudioTranscriptionHandler(BaseTranslation):
|
|||
|
||||
if isinstance(response.text, str):
|
||||
original_text = response.text
|
||||
# Create a request_data dict with response info and user API key metadata
|
||||
request_data: dict = {"response": response}
|
||||
# Use the real request_data if provided (proxy path), otherwise
|
||||
# create a standalone dict (SDK / direct-call path).
|
||||
if request_data is None:
|
||||
request_data = {"response": response}
|
||||
else:
|
||||
if "response" not in request_data:
|
||||
request_data["response"] = response
|
||||
|
||||
# Add user API key metadata with prefixed keys
|
||||
user_metadata = self.transform_user_api_key_dict_to_metadata(
|
||||
user_api_key_dict
|
||||
)
|
||||
if user_metadata:
|
||||
request_data["litellm_metadata"] = user_metadata
|
||||
if "litellm_metadata" not in request_data:
|
||||
user_metadata = self.transform_user_api_key_dict_to_metadata(
|
||||
user_api_key_dict
|
||||
)
|
||||
if user_metadata:
|
||||
request_data["litellm_metadata"] = user_metadata
|
||||
|
||||
inputs = GenericGuardrailAPIInputs(texts=[original_text])
|
||||
# Include model information from the response if available
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ More information on our website: https://endpoints.ai.cloud.ovh.net
|
|||
from typing import Optional, Union, List
|
||||
|
||||
import httpx
|
||||
from litellm.utils import ModelResponseStream, get_model_info
|
||||
from litellm.utils import ModelResponseStream, _get_model_info_helper
|
||||
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.llms.ovhcloud.utils import OVHCloudException
|
||||
|
|
@ -28,13 +28,15 @@ class OVHCloudChatConfig(OpenAIGPTConfig):
|
|||
"""
|
||||
supports_function_calling: Optional[bool] = None
|
||||
try:
|
||||
model_info = get_model_info(model, custom_llm_provider="ovhcloud")
|
||||
model_info = _get_model_info_helper(model, custom_llm_provider="ovhcloud")
|
||||
supports_function_calling = model_info.get(
|
||||
"supports_function_calling", False
|
||||
"supports_function_calling", None
|
||||
)
|
||||
if supports_function_calling is None:
|
||||
supports_function_calling = False
|
||||
except Exception as e:
|
||||
verbose_logger.debug(f"Error getting supported OpenAI params: {e}")
|
||||
pass
|
||||
supports_function_calling = False
|
||||
|
||||
optional_params = super().get_supported_openai_params(model)
|
||||
if supports_function_calling is not True:
|
||||
|
|
|
|||
|
|
@ -139,6 +139,7 @@ class PassThroughEndpointHandler(BaseTranslation):
|
|||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: Optional["LiteLLMLoggingObj"] = None,
|
||||
user_api_key_dict: Optional[Any] = None,
|
||||
request_data: Optional[dict] = None,
|
||||
) -> Any:
|
||||
"""
|
||||
Process output response by applying guardrails to targeted fields.
|
||||
|
|
@ -171,17 +172,27 @@ class PassThroughEndpointHandler(BaseTranslation):
|
|||
if not text_to_check:
|
||||
return response
|
||||
|
||||
# Create a request_data dict with response info and user API key metadata
|
||||
request_data: dict = (
|
||||
{"response": response}
|
||||
if not isinstance(response, dict)
|
||||
else response.copy()
|
||||
)
|
||||
# Use the real request_data if provided (proxy path), otherwise
|
||||
# create a standalone dict (SDK / direct-call path).
|
||||
if request_data is None:
|
||||
request_data = (
|
||||
{"response": response}
|
||||
if not isinstance(response, dict)
|
||||
else response.copy()
|
||||
)
|
||||
else:
|
||||
if "response" not in request_data:
|
||||
request_data["response"] = (
|
||||
response if not isinstance(response, dict) else response.copy()
|
||||
)
|
||||
|
||||
# Add user API key metadata with prefixed keys
|
||||
user_metadata = self.transform_user_api_key_dict_to_metadata(user_api_key_dict)
|
||||
if user_metadata:
|
||||
request_data["litellm_metadata"] = user_metadata
|
||||
if "litellm_metadata" not in request_data:
|
||||
user_metadata = self.transform_user_api_key_dict_to_metadata(
|
||||
user_api_key_dict
|
||||
)
|
||||
if user_metadata:
|
||||
request_data["litellm_metadata"] = user_metadata
|
||||
|
||||
# Apply guardrail (pass-through doesn't modify the text, just checks it)
|
||||
inputs = GenericGuardrailAPIInputs(texts=[text_to_check])
|
||||
|
|
|
|||
|
|
@ -376,3 +376,148 @@ class VertexAIBatchPrediction(VertexLLM):
|
|||
response=_json_response
|
||||
)
|
||||
return vertex_batch_response
|
||||
|
||||
def cancel_batch(
|
||||
self,
|
||||
_is_async: bool,
|
||||
batch_id: str,
|
||||
api_base: Optional[str],
|
||||
vertex_credentials: Optional[VERTEX_CREDENTIALS_TYPES],
|
||||
vertex_project: Optional[str],
|
||||
vertex_location: Optional[str],
|
||||
timeout: Union[float, httpx.Timeout],
|
||||
max_retries: Optional[int],
|
||||
) -> Union[LiteLLMBatch, Coroutine[Any, Any, LiteLLMBatch]]:
|
||||
access_token, project_id = self._ensure_access_token(
|
||||
credentials=vertex_credentials,
|
||||
project_id=vertex_project,
|
||||
custom_llm_provider="vertex_ai",
|
||||
)
|
||||
|
||||
default_api_base = self.create_vertex_batch_url(
|
||||
vertex_location=vertex_location or "us-central1",
|
||||
vertex_project=vertex_project or project_id,
|
||||
)
|
||||
|
||||
retrieve_api_base_default = f"{default_api_base}/{batch_id}"
|
||||
cancel_api_base_default = f"{retrieve_api_base_default}:cancel"
|
||||
|
||||
_, api_base = self._check_custom_proxy(
|
||||
api_base=api_base,
|
||||
custom_llm_provider="vertex_ai",
|
||||
gemini_api_key=None,
|
||||
endpoint="cancel",
|
||||
stream=None,
|
||||
auth_header=None,
|
||||
url=cancel_api_base_default,
|
||||
model=None,
|
||||
vertex_project=vertex_project or project_id,
|
||||
vertex_location=vertex_location or "us-central1",
|
||||
vertex_api_version="v1",
|
||||
)
|
||||
|
||||
if api_base.endswith(":cancel"):
|
||||
retrieve_api_base = api_base.removesuffix(":cancel")
|
||||
else:
|
||||
retrieve_api_base = api_base.rsplit(":cancel", 1)[0].rstrip("/")
|
||||
|
||||
headers = {
|
||||
"Content-Type": "application/json; charset=utf-8",
|
||||
"Authorization": f"Bearer {access_token}",
|
||||
}
|
||||
|
||||
if _is_async is True:
|
||||
return self._async_cancel_batch(
|
||||
api_base=api_base,
|
||||
retrieve_api_base=retrieve_api_base,
|
||||
headers=headers,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
sync_handler = _get_httpx_client()
|
||||
try:
|
||||
response = sync_handler.post(
|
||||
url=api_base,
|
||||
headers=headers,
|
||||
data=json.dumps({}),
|
||||
timeout=timeout,
|
||||
)
|
||||
except httpx.HTTPStatusError as e:
|
||||
litellm.verbose_logger.error(
|
||||
"Vertex AI batch cancel failed: status=%s, body=%s",
|
||||
e.response.status_code,
|
||||
e.response.text[:1000],
|
||||
)
|
||||
raise
|
||||
|
||||
if response.status_code != 200:
|
||||
raise Exception(f"Error: {response.status_code} {response.text}")
|
||||
|
||||
# HTTPHandler.get() does not accept a timeout parameter
|
||||
retrieve_response = sync_handler.get(
|
||||
url=retrieve_api_base,
|
||||
headers=headers,
|
||||
)
|
||||
if retrieve_response.status_code != 200:
|
||||
litellm.verbose_logger.error(
|
||||
"Vertex AI batch retrieve-after-cancel failed: status=%s, body=%s",
|
||||
retrieve_response.status_code,
|
||||
retrieve_response.text[:1000],
|
||||
)
|
||||
raise Exception(
|
||||
f"Error: {retrieve_response.status_code} {retrieve_response.text}"
|
||||
)
|
||||
|
||||
_json_response = retrieve_response.json()
|
||||
vertex_batch_response = VertexAIBatchTransformation.transform_vertex_ai_batch_response_to_openai_batch_response(
|
||||
response=_json_response
|
||||
)
|
||||
return vertex_batch_response
|
||||
|
||||
async def _async_cancel_batch(
|
||||
self,
|
||||
api_base: str,
|
||||
retrieve_api_base: str,
|
||||
headers: Dict[str, str],
|
||||
timeout: Union[float, httpx.Timeout] = 600.0,
|
||||
) -> LiteLLMBatch:
|
||||
client = get_async_httpx_client(
|
||||
llm_provider=litellm.LlmProviders.VERTEX_AI,
|
||||
)
|
||||
try:
|
||||
response = await client.post(
|
||||
url=api_base,
|
||||
headers=headers,
|
||||
data=json.dumps({}),
|
||||
timeout=timeout,
|
||||
)
|
||||
except httpx.HTTPStatusError as e:
|
||||
litellm.verbose_logger.error(
|
||||
"Vertex AI batch cancel failed: status=%s, body=%s",
|
||||
e.response.status_code,
|
||||
e.response.text[:1000],
|
||||
)
|
||||
raise
|
||||
if response.status_code != 200:
|
||||
raise Exception(f"Error: {response.status_code} {response.text}")
|
||||
|
||||
# AsyncHTTPHandler.get() does not accept a timeout parameter
|
||||
retrieve_response = await client.get(
|
||||
url=retrieve_api_base,
|
||||
headers=headers,
|
||||
)
|
||||
if retrieve_response.status_code != 200:
|
||||
litellm.verbose_logger.error(
|
||||
"Vertex AI batch retrieve-after-cancel failed: status=%s, body=%s",
|
||||
retrieve_response.status_code,
|
||||
retrieve_response.text[:1000],
|
||||
)
|
||||
raise Exception(
|
||||
f"Error: {retrieve_response.status_code} {retrieve_response.text}"
|
||||
)
|
||||
|
||||
_json_response = retrieve_response.json()
|
||||
vertex_batch_response = VertexAIBatchTransformation.transform_vertex_ai_batch_response_to_openai_batch_response(
|
||||
response=_json_response
|
||||
)
|
||||
return vertex_batch_response
|
||||
|
|
|
|||
|
|
@ -51,6 +51,7 @@ class ContextCachingEndpoints(VertexBase):
|
|||
vertex_project: Optional[str],
|
||||
vertex_location: Optional[str],
|
||||
vertex_auth_header: Optional[str],
|
||||
model: Optional[str] = None,
|
||||
) -> Tuple[Optional[str], str]:
|
||||
"""
|
||||
Internal function. Returns the token and url for the call.
|
||||
|
|
@ -89,7 +90,7 @@ class ContextCachingEndpoints(VertexBase):
|
|||
stream=None,
|
||||
auth_header=auth_header,
|
||||
url=url,
|
||||
model=None,
|
||||
model=model,
|
||||
vertex_project=vertex_project,
|
||||
vertex_location=vertex_location,
|
||||
vertex_api_version="v1beta1"
|
||||
|
|
@ -109,6 +110,7 @@ class ContextCachingEndpoints(VertexBase):
|
|||
vertex_project: Optional[str],
|
||||
vertex_location: Optional[str],
|
||||
vertex_auth_header: Optional[str],
|
||||
model: Optional[str] = None,
|
||||
) -> Optional[str]:
|
||||
"""
|
||||
Checks if content already cached.
|
||||
|
|
@ -128,6 +130,7 @@ class ContextCachingEndpoints(VertexBase):
|
|||
vertex_project=vertex_project,
|
||||
vertex_location=vertex_location,
|
||||
vertex_auth_header=vertex_auth_header,
|
||||
model=model,
|
||||
)
|
||||
|
||||
page_token: Optional[str] = None
|
||||
|
|
@ -201,6 +204,7 @@ class ContextCachingEndpoints(VertexBase):
|
|||
vertex_project: Optional[str],
|
||||
vertex_location: Optional[str],
|
||||
vertex_auth_header: Optional[str],
|
||||
model: Optional[str] = None,
|
||||
) -> Optional[str]:
|
||||
"""
|
||||
Checks if content already cached.
|
||||
|
|
@ -220,6 +224,7 @@ class ContextCachingEndpoints(VertexBase):
|
|||
vertex_project=vertex_project,
|
||||
vertex_location=vertex_location,
|
||||
vertex_auth_header=vertex_auth_header,
|
||||
model=model,
|
||||
)
|
||||
|
||||
page_token: Optional[str] = None
|
||||
|
|
@ -342,6 +347,7 @@ class ContextCachingEndpoints(VertexBase):
|
|||
vertex_project=vertex_project,
|
||||
vertex_location=vertex_location,
|
||||
vertex_auth_header=vertex_auth_header,
|
||||
model=model,
|
||||
)
|
||||
|
||||
headers = {
|
||||
|
|
@ -377,6 +383,7 @@ class ContextCachingEndpoints(VertexBase):
|
|||
vertex_project=vertex_project,
|
||||
vertex_location=vertex_location,
|
||||
vertex_auth_header=vertex_auth_header,
|
||||
model=model,
|
||||
)
|
||||
if google_cache_name:
|
||||
return non_cached_messages, optional_params, google_cache_name
|
||||
|
|
@ -488,6 +495,7 @@ class ContextCachingEndpoints(VertexBase):
|
|||
vertex_project=vertex_project,
|
||||
vertex_location=vertex_location,
|
||||
vertex_auth_header=vertex_auth_header,
|
||||
model=model,
|
||||
)
|
||||
|
||||
headers = {
|
||||
|
|
@ -520,6 +528,7 @@ class ContextCachingEndpoints(VertexBase):
|
|||
vertex_project=vertex_project,
|
||||
vertex_location=vertex_location,
|
||||
vertex_auth_header=vertex_auth_header,
|
||||
model=model,
|
||||
)
|
||||
|
||||
if google_cache_name:
|
||||
|
|
|
|||
|
|
@ -540,6 +540,41 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915
|
|||
assistant_content.append(gemini_tool_call_part)
|
||||
last_message_with_tool_calls = assistant_msg
|
||||
|
||||
## HANDLE SERVER-SIDE TOOL INVOCATIONS (context circulation)
|
||||
_psf = assistant_msg.get("provider_specific_fields")
|
||||
if isinstance(_psf, dict):
|
||||
_ss_invocations = _psf.get("server_side_tool_invocations")
|
||||
if isinstance(_ss_invocations, list):
|
||||
for invocation in _ss_invocations:
|
||||
# Re-inject toolCall part
|
||||
tc_part: Dict[str, Any] = {
|
||||
"toolCall": {
|
||||
"toolType": invocation.get("tool_type"),
|
||||
"id": invocation.get("id"),
|
||||
"args": invocation.get("args"),
|
||||
}
|
||||
}
|
||||
if "thought_signature" in invocation:
|
||||
tc_part["thoughtSignature"] = invocation[
|
||||
"thought_signature"
|
||||
]
|
||||
assistant_content.append(tc_part) # type: ignore
|
||||
|
||||
# Re-inject toolResponse part if response is present
|
||||
if "response" in invocation:
|
||||
tr_dict: Dict[str, Any] = {
|
||||
"id": invocation.get("id"),
|
||||
"response": invocation.get("response"),
|
||||
}
|
||||
if invocation.get("tool_type"):
|
||||
tr_dict["toolType"] = invocation["tool_type"]
|
||||
tr_part: Dict[str, Any] = {"toolResponse": tr_dict}
|
||||
if "thought_signature" in invocation:
|
||||
tr_part["thoughtSignature"] = invocation[
|
||||
"thought_signature"
|
||||
]
|
||||
assistant_content.append(tr_part) # type: ignore
|
||||
|
||||
msg_i += 1
|
||||
|
||||
if assistant_content:
|
||||
|
|
@ -666,6 +701,9 @@ def _transform_request_body( # noqa: PLR0915
|
|||
)
|
||||
tools: Optional[Tools] = optional_params.pop("tools", None)
|
||||
tool_choice: Optional[ToolConfig] = optional_params.pop("tool_choice", None)
|
||||
include_server_side_tool_invocations: bool = optional_params.pop(
|
||||
"include_server_side_tool_invocations", False
|
||||
)
|
||||
safety_settings: Optional[List[SafetSettingsConfig]] = optional_params.pop(
|
||||
"safety_settings", None
|
||||
) # type: ignore
|
||||
|
|
@ -715,6 +753,10 @@ def _transform_request_body( # noqa: PLR0915
|
|||
data["tools"] = tools
|
||||
if tool_choice is not None:
|
||||
data["toolConfig"] = tool_choice
|
||||
if include_server_side_tool_invocations:
|
||||
if "toolConfig" not in data:
|
||||
data["toolConfig"] = {}
|
||||
data["toolConfig"]["includeServerSideToolInvocations"] = True
|
||||
if safety_settings is not None:
|
||||
data["safetySettings"] = safety_settings
|
||||
if generation_config is not None and len(generation_config) > 0:
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from typing import (
|
|||
Dict,
|
||||
List,
|
||||
Literal,
|
||||
Mapping,
|
||||
Optional,
|
||||
Tuple,
|
||||
Type,
|
||||
|
|
@ -316,6 +317,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
"audio",
|
||||
"parallel_tool_calls",
|
||||
"web_search_options",
|
||||
"include_server_side_tool_invocations",
|
||||
]
|
||||
|
||||
# Add penalty parameters only for non-preview models
|
||||
|
|
@ -1119,6 +1121,8 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
optional_params = self._add_tools_to_optional_params(
|
||||
optional_params, [_tools]
|
||||
)
|
||||
elif param == "include_server_side_tool_invocations" and value is True:
|
||||
optional_params["include_server_side_tool_invocations"] = True
|
||||
if litellm.vertex_ai_safety_settings is not None:
|
||||
optional_params["safety_settings"] = litellm.vertex_ai_safety_settings
|
||||
|
||||
|
|
@ -1360,6 +1364,67 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
signatures.append(signature)
|
||||
return signatures if signatures else None
|
||||
|
||||
@staticmethod
|
||||
def _extract_server_side_tool_invocations(
|
||||
parts: List[HttpxPartType],
|
||||
) -> Optional[List[Dict[str, Any]]]:
|
||||
"""Extract server-side tool invocations (toolCall/toolResponse) from parts.
|
||||
|
||||
These are returned by Gemini when context circulation is enabled
|
||||
(includeServerSideToolInvocations=true). They represent tools executed
|
||||
server-side (e.g. Google Search) and must be circulated back in
|
||||
subsequent turns for multi-turn coherence.
|
||||
|
||||
Returns:
|
||||
List of server-side invocation dicts if any found, None otherwise.
|
||||
"""
|
||||
invocations: List[Dict[str, Any]] = []
|
||||
# Index toolCalls by id so we can pair them with responses
|
||||
tool_calls_by_id: Dict[str, Dict[str, Any]] = {}
|
||||
tool_responses_by_id: Dict[str, Dict[str, Any]] = {}
|
||||
|
||||
for part in parts:
|
||||
if "toolCall" in part:
|
||||
tc = part["toolCall"]
|
||||
entry: Dict[str, Any] = {
|
||||
"tool_type": tc.get("toolType"),
|
||||
"id": tc.get("id"),
|
||||
"args": tc.get("args"),
|
||||
}
|
||||
signature = part.get("thoughtSignature")
|
||||
if signature is not None:
|
||||
entry["thought_signature"] = signature
|
||||
tool_calls_by_id[tc.get("id", "")] = entry
|
||||
|
||||
elif "toolResponse" in part:
|
||||
tr = part["toolResponse"]
|
||||
entry = {
|
||||
"id": tr.get("id"),
|
||||
"tool_type": tr.get("toolType"),
|
||||
"response": tr.get("response"),
|
||||
}
|
||||
signature = part.get("thoughtSignature")
|
||||
if signature is not None:
|
||||
entry["thought_signature"] = signature
|
||||
tool_responses_by_id[tr.get("id", "")] = entry
|
||||
|
||||
# Merge calls with their responses
|
||||
for call_id, call_entry in tool_calls_by_id.items():
|
||||
merged = dict(call_entry)
|
||||
resp = tool_responses_by_id.pop(call_id, None)
|
||||
if resp is not None:
|
||||
merged["response"] = resp.get("response")
|
||||
# Keep response signature if call didn't have one
|
||||
if "thought_signature" not in merged and "thought_signature" in resp:
|
||||
merged["thought_signature"] = resp["thought_signature"]
|
||||
invocations.append(merged)
|
||||
|
||||
# Any orphan responses (shouldn't happen, but be safe)
|
||||
for resp_id, resp_entry in tool_responses_by_id.items():
|
||||
invocations.append(resp_entry)
|
||||
|
||||
return invocations if invocations else None
|
||||
|
||||
def _extract_image_response_from_parts(
|
||||
self, parts: List[HttpxPartType]
|
||||
) -> Optional[List[ImageURLListItem]]:
|
||||
|
|
@ -1632,6 +1697,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
response_tokens: Optional[int] = None
|
||||
response_tokens_details: Optional[CompletionTokensDetailsWrapper] = None
|
||||
usage_metadata = completion_response["usageMetadata"]
|
||||
|
||||
def _get_token_count(detail: Mapping[str, Any]) -> int:
|
||||
raw_token_count = detail.get("tokenCount", detail.get("token_count", 0))
|
||||
return raw_token_count if isinstance(raw_token_count, int) else 0
|
||||
|
||||
if "cachedContentTokenCount" in usage_metadata:
|
||||
cached_tokens = usage_metadata["cachedContentTokenCount"]
|
||||
|
||||
|
|
@ -1641,10 +1711,16 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
if "responseTokensDetails" in usage_metadata:
|
||||
response_tokens_details = CompletionTokensDetailsWrapper()
|
||||
for detail in usage_metadata["responseTokensDetails"]:
|
||||
if detail["modality"] == "TEXT":
|
||||
response_tokens_details.text_tokens = detail.get("tokenCount", 0)
|
||||
elif detail["modality"] == "AUDIO":
|
||||
response_tokens_details.audio_tokens = detail.get("tokenCount", 0)
|
||||
modality = str(detail.get("modality", "")).upper()
|
||||
token_count = _get_token_count(detail)
|
||||
if modality == "TEXT":
|
||||
response_tokens_details.text_tokens = (
|
||||
response_tokens_details.text_tokens or 0
|
||||
) + token_count
|
||||
elif modality == "AUDIO":
|
||||
response_tokens_details.audio_tokens = (
|
||||
response_tokens_details.audio_tokens or 0
|
||||
) + token_count
|
||||
|
||||
#########################################################
|
||||
|
||||
|
|
@ -1653,16 +1729,24 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
if response_tokens_details is None:
|
||||
response_tokens_details = CompletionTokensDetailsWrapper()
|
||||
for detail in usage_metadata["candidatesTokensDetails"]:
|
||||
modality = detail.get("modality")
|
||||
token_count = detail.get("tokenCount", 0)
|
||||
modality = str(detail.get("modality", "")).upper()
|
||||
token_count = _get_token_count(detail)
|
||||
if modality == "TEXT":
|
||||
response_tokens_details.text_tokens = token_count
|
||||
response_tokens_details.text_tokens = (
|
||||
response_tokens_details.text_tokens or 0
|
||||
) + token_count
|
||||
elif modality == "AUDIO":
|
||||
response_tokens_details.audio_tokens = token_count
|
||||
response_tokens_details.audio_tokens = (
|
||||
response_tokens_details.audio_tokens or 0
|
||||
) + token_count
|
||||
elif modality == "IMAGE":
|
||||
response_tokens_details.image_tokens = token_count
|
||||
response_tokens_details.image_tokens = (
|
||||
response_tokens_details.image_tokens or 0
|
||||
) + token_count
|
||||
elif modality == "VIDEO":
|
||||
response_tokens_details.video_tokens = token_count
|
||||
response_tokens_details.video_tokens = (
|
||||
response_tokens_details.video_tokens or 0
|
||||
) + token_count
|
||||
|
||||
# Calculate text_tokens if not explicitly provided in candidatesTokensDetails
|
||||
# candidatesTokenCount includes all modalities, so: text = total - (image + audio + video)
|
||||
|
|
@ -1686,14 +1770,16 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
## Parse promptTokensDetails (total tokens by modality, includes cached + non-cached)
|
||||
if "promptTokensDetails" in usage_metadata:
|
||||
for detail in usage_metadata["promptTokensDetails"]:
|
||||
if detail["modality"] == "AUDIO":
|
||||
prompt_audio_tokens = detail.get("tokenCount", 0)
|
||||
elif detail["modality"] == "TEXT":
|
||||
prompt_text_tokens = detail.get("tokenCount", 0)
|
||||
elif detail["modality"] == "IMAGE":
|
||||
prompt_image_tokens = detail.get("tokenCount", 0)
|
||||
elif detail["modality"] == "VIDEO":
|
||||
prompt_video_tokens = detail.get("tokenCount", 0)
|
||||
modality = str(detail.get("modality", "")).upper()
|
||||
token_count = _get_token_count(detail)
|
||||
if modality == "AUDIO":
|
||||
prompt_audio_tokens = (prompt_audio_tokens or 0) + token_count
|
||||
elif modality == "TEXT":
|
||||
prompt_text_tokens = (prompt_text_tokens or 0) + token_count
|
||||
elif modality == "IMAGE":
|
||||
prompt_image_tokens = (prompt_image_tokens or 0) + token_count
|
||||
elif modality == "VIDEO":
|
||||
prompt_video_tokens = (prompt_video_tokens or 0) + token_count
|
||||
|
||||
## Parse cacheTokensDetails (breakdown of cached tokens by modality)
|
||||
## When explicit caching is used, Gemini provides this field to show which modalities were cached
|
||||
|
|
@ -1704,14 +1790,16 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
|
||||
if "cacheTokensDetails" in usage_metadata:
|
||||
for detail in usage_metadata["cacheTokensDetails"]:
|
||||
if detail["modality"] == "AUDIO":
|
||||
cached_audio_tokens = detail.get("tokenCount", 0)
|
||||
elif detail["modality"] == "TEXT":
|
||||
cached_text_tokens = detail.get("tokenCount", 0)
|
||||
elif detail["modality"] == "IMAGE":
|
||||
cached_image_tokens = detail.get("tokenCount", 0)
|
||||
elif detail["modality"] == "VIDEO":
|
||||
cached_video_tokens = detail.get("tokenCount", 0)
|
||||
modality = str(detail.get("modality", "")).upper()
|
||||
token_count = _get_token_count(detail)
|
||||
if modality == "AUDIO":
|
||||
cached_audio_tokens = (cached_audio_tokens or 0) + token_count
|
||||
elif modality == "TEXT":
|
||||
cached_text_tokens = (cached_text_tokens or 0) + token_count
|
||||
elif modality == "IMAGE":
|
||||
cached_image_tokens = (cached_image_tokens or 0) + token_count
|
||||
elif modality == "VIDEO":
|
||||
cached_video_tokens = (cached_video_tokens or 0) + token_count
|
||||
|
||||
## Calculate non-cached tokens by subtracting cached from total (per modality)
|
||||
## This is necessary because promptTokensDetails includes both cached and non-cached tokens
|
||||
|
|
@ -2018,6 +2106,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
thinking_blocks: Optional[List[ChatCompletionThinkingBlock]] = None
|
||||
reasoning_content: Optional[str] = None
|
||||
thought_signatures: Optional[Any] = None
|
||||
server_side_tool_invocations: Optional[List[Dict[str, Any]]] = None
|
||||
|
||||
for idx, candidate in enumerate(_candidates):
|
||||
if "content" not in candidate:
|
||||
|
|
@ -2068,6 +2157,13 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
)
|
||||
)
|
||||
|
||||
# Extract server-side tool invocations (context circulation)
|
||||
server_side_tool_invocations = (
|
||||
VertexGeminiConfig._extract_server_side_tool_invocations(
|
||||
parts=candidate["content"]["parts"]
|
||||
)
|
||||
)
|
||||
|
||||
if audio_response is not None:
|
||||
cast(Dict[str, Any], chat_completion_message)[
|
||||
"audio"
|
||||
|
|
@ -2139,6 +2235,12 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
chat_completion_message["provider_specific_fields"] = {}
|
||||
chat_completion_message["provider_specific_fields"]["thought_signatures"] = thought_signatures # type: ignore
|
||||
|
||||
# Store server-side tool invocations in provider_specific_fields
|
||||
if server_side_tool_invocations is not None:
|
||||
if "provider_specific_fields" not in chat_completion_message:
|
||||
chat_completion_message["provider_specific_fields"] = {}
|
||||
chat_completion_message["provider_specific_fields"]["server_side_tool_invocations"] = server_side_tool_invocations # type: ignore
|
||||
|
||||
if isinstance(model_response, ModelResponseStream):
|
||||
choice = VertexGeminiConfig._create_streaming_choice(
|
||||
chat_completion_message=chat_completion_message,
|
||||
|
|
@ -3001,6 +3103,16 @@ class ModelResponseIterator:
|
|||
)
|
||||
model_response.choices.append(choice)
|
||||
|
||||
# Also handle the case where the final chunk has empty
|
||||
# content (e.g. text:"") WITH finishReason. In this case
|
||||
# _process_candidates DOES create a choice, but maps
|
||||
# finishReason="STOP" to "stop" because the current chunk
|
||||
# has no tool_calls. Override if we saw tool_calls earlier.
|
||||
if self.has_seen_tool_calls:
|
||||
for choice in model_response.choices:
|
||||
if choice.finish_reason == "stop":
|
||||
choice.finish_reason = "tool_calls"
|
||||
|
||||
setattr(model_response, "vertex_ai_grounding_metadata", grounding_metadata) # type: ignore
|
||||
setattr(model_response, "vertex_ai_url_context_metadata", url_context_metadata) # type: ignore
|
||||
setattr(model_response, "vertex_ai_safety_ratings", safety_ratings) # type: ignore
|
||||
|
|
|
|||
|
|
@ -152,6 +152,8 @@ def transform_openai_input_gemini_content(
|
|||
gemini_params = optional_params.copy()
|
||||
if "dimensions" in gemini_params:
|
||||
gemini_params["outputDimensionality"] = gemini_params.pop("dimensions")
|
||||
if "task_type" in gemini_params:
|
||||
gemini_params["taskType"] = gemini_params.pop("task_type")
|
||||
|
||||
requests: List[EmbedContentRequest] = []
|
||||
if isinstance(input, str):
|
||||
|
|
@ -196,6 +198,8 @@ def transform_openai_input_gemini_embed_content(
|
|||
gemini_params = optional_params.copy()
|
||||
if "dimensions" in gemini_params:
|
||||
gemini_params["outputDimensionality"] = gemini_params.pop("dimensions")
|
||||
if "task_type" in gemini_params:
|
||||
gemini_params["taskType"] = gemini_params.pop("task_type")
|
||||
|
||||
input_list = [input] if isinstance(input, str) else input
|
||||
parts: List[PartType] = []
|
||||
|
|
|
|||
|
|
@ -105,12 +105,27 @@ class VertexAIPartnerModelsTokenCounter(VertexBase):
|
|||
# Extract Vertex AI credentials and settings
|
||||
vertex_credentials = self.get_vertex_ai_credentials(litellm_params)
|
||||
vertex_project = self.get_vertex_ai_project(litellm_params)
|
||||
vertex_location = self.get_vertex_ai_location(litellm_params)
|
||||
|
||||
# Map empty location/cluade models to a supported region for count-tokens endpoint
|
||||
# Check for count_tokens specific location override
|
||||
vertex_count_tokens_location = litellm_params.get(
|
||||
"vertex_count_tokens_location"
|
||||
)
|
||||
vertex_location_raw = self.get_vertex_ai_location(litellm_params)
|
||||
|
||||
# Determine final location with precedence:
|
||||
# 1. vertex_count_tokens_location (if provided)
|
||||
# 2. vertex_location (if provided)
|
||||
# 3. Default to us-east5 for Claude models when no location is set
|
||||
# Supported regions: us-east5, europe-west1, asia-southeast1
|
||||
# https://docs.cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude/count-tokens
|
||||
if not vertex_location or "claude" in model.lower():
|
||||
vertex_location = "us-central1"
|
||||
if vertex_count_tokens_location:
|
||||
vertex_location: str = vertex_count_tokens_location
|
||||
elif vertex_location_raw:
|
||||
vertex_location = vertex_location_raw
|
||||
elif "claude" in model.lower():
|
||||
vertex_location = "us-east5"
|
||||
else:
|
||||
vertex_location = "us-east5"
|
||||
|
||||
# Get access token and resolved project ID
|
||||
access_token, project_id = await self._ensure_access_token_async(
|
||||
|
|
|
|||
|
|
@ -955,16 +955,6 @@ def responses_api_bridge_check(
|
|||
model_info["mode"] = "responses"
|
||||
model = model.replace("responses/", "")
|
||||
|
||||
# OpenAI gpt-5.4+ chat-completions calls with both tools + reasoning_effort
|
||||
# must be bridged to Responses API.
|
||||
if (
|
||||
custom_llm_provider == "openai"
|
||||
and OpenAIGPT5Config.is_model_gpt_5_4_plus_model(model)
|
||||
and tools
|
||||
and reasoning_effort is not None
|
||||
):
|
||||
model_info["mode"] = "responses"
|
||||
model = model.replace("responses/", "")
|
||||
except Exception as e:
|
||||
verbose_logger.debug("Error getting model info: {}".format(e))
|
||||
|
||||
|
|
@ -974,6 +964,19 @@ def responses_api_bridge_check(
|
|||
model = model.replace("responses/", "")
|
||||
mode = "responses"
|
||||
model_info["mode"] = mode
|
||||
|
||||
# OpenAI/Azure gpt-5.4+ chat-completions calls with both tools + reasoning_effort
|
||||
# must be bridged to Responses API.
|
||||
if (
|
||||
custom_llm_provider in ("openai", "azure")
|
||||
and OpenAIGPT5Config.is_model_gpt_5_4_plus_model(model)
|
||||
and tools
|
||||
and reasoning_effort is not None
|
||||
and model_info.get("mode") != "responses"
|
||||
):
|
||||
model_info["mode"] = "responses"
|
||||
model = model.replace("responses/", "")
|
||||
|
||||
return model_info, model
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -3435,7 +3435,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_service_tier": true,
|
||||
"supports_vision": true,
|
||||
"supports_none_reasoning_effort": true
|
||||
"supports_none_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"azure/gpt-5.1-chat-2025-11-13": {
|
||||
"cache_read_input_token_cost": 1.25e-07,
|
||||
|
|
@ -6152,7 +6153,8 @@
|
|||
"max_query_tokens": 4096,
|
||||
"max_tokens": 32768,
|
||||
"mode": "rerank",
|
||||
"output_cost_per_token": 0.0
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-cohere-rerank-4-0-in-microsoft-foundry/4477076"
|
||||
},
|
||||
"azure_ai/cohere-rerank-v4.0-fast": {
|
||||
"input_cost_per_query": 0.002,
|
||||
|
|
@ -6163,7 +6165,8 @@
|
|||
"max_query_tokens": 4096,
|
||||
"max_tokens": 32768,
|
||||
"mode": "rerank",
|
||||
"output_cost_per_token": 0.0
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-cohere-rerank-4-0-in-microsoft-foundry/4477076"
|
||||
},
|
||||
"azure_ai/deepseek-v3.2": {
|
||||
"input_cost_per_token": 5.8e-07,
|
||||
|
|
@ -6173,6 +6176,7 @@
|
|||
"max_tokens": 163840,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.68e-06,
|
||||
"source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-deepseek-v3-2-and-deepseek-v3-2-speciale-in-microsoft-foundry/4477549",
|
||||
"supports_assistant_prefill": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
|
|
@ -6187,6 +6191,7 @@
|
|||
"max_tokens": 163840,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.68e-06,
|
||||
"source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-deepseek-v3-2-and-deepseek-v3-2-speciale-in-microsoft-foundry/4477549",
|
||||
"supports_assistant_prefill": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
|
|
@ -16936,6 +16941,18 @@
|
|||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"gpt-4-0314": {
|
||||
"deprecation_date": "2026-03-26",
|
||||
"input_cost_per_token": 3e-05,
|
||||
"litellm_provider": "openai",
|
||||
"max_input_tokens": 8192,
|
||||
"max_output_tokens": 4096,
|
||||
"max_tokens": 4096,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 6e-05,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"gpt-4-0613": {
|
||||
"deprecation_date": "2025-06-06",
|
||||
"input_cost_per_token": 3e-05,
|
||||
|
|
@ -18289,7 +18306,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": false
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5.1": {
|
||||
"cache_read_input_token_cost": 1.25e-07,
|
||||
|
|
@ -18328,7 +18346,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": true,
|
||||
"supports_xhigh_reasoning_effort": false
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5.1-2025-11-13": {
|
||||
"cache_read_input_token_cost": 1.25e-07,
|
||||
|
|
@ -18367,7 +18386,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": true,
|
||||
"supports_xhigh_reasoning_effort": false
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5.1-chat-latest": {
|
||||
"cache_read_input_token_cost": 1.25e-07,
|
||||
|
|
@ -18405,7 +18425,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": true,
|
||||
"supports_xhigh_reasoning_effort": false
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5.2": {
|
||||
"cache_read_input_token_cost": 1.75e-07,
|
||||
|
|
@ -18445,7 +18466,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": true,
|
||||
"supports_xhigh_reasoning_effort": true
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5.2-2025-12-11": {
|
||||
"cache_read_input_token_cost": 1.75e-07,
|
||||
|
|
@ -18485,7 +18507,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": true,
|
||||
"supports_xhigh_reasoning_effort": true
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5.2-chat-latest": {
|
||||
"cache_read_input_token_cost": 1.75e-07,
|
||||
|
|
@ -18522,7 +18545,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": false
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5.3-chat-latest": {
|
||||
"cache_read_input_token_cost": 1.75e-07,
|
||||
|
|
@ -18559,7 +18583,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": false
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5.2-pro": {
|
||||
"input_cost_per_token": 2.1e-05,
|
||||
|
|
@ -18592,7 +18617,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": true
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5.2-pro-2025-12-11": {
|
||||
"input_cost_per_token": 2.1e-05,
|
||||
|
|
@ -18625,7 +18651,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": true
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5.4": {
|
||||
"cache_read_input_token_cost": 2.5e-07,
|
||||
|
|
@ -18674,7 +18701,8 @@
|
|||
"supports_service_tier": true,
|
||||
"supports_vision": true,
|
||||
"supports_none_reasoning_effort": true,
|
||||
"supports_xhigh_reasoning_effort": true
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5.4-2026-03-05": {
|
||||
"cache_read_input_token_cost": 2.5e-07,
|
||||
|
|
@ -18769,7 +18797,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": true
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5.4-pro-2026-03-05": {
|
||||
"cache_read_input_token_cost": 3e-06,
|
||||
|
|
@ -18817,7 +18846,94 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": true
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5.4-mini": {
|
||||
"cache_read_input_token_cost": 7.5e-08,
|
||||
"cache_read_input_token_cost_flex": 1e-08,
|
||||
"cache_read_input_token_cost_batches": 3.8e-08,
|
||||
"input_cost_per_token": 7.5e-07,
|
||||
"input_cost_per_token_flex": 3.75e-07,
|
||||
"input_cost_per_token_batches": 3.75e-07,
|
||||
"litellm_provider": "openai",
|
||||
"max_input_tokens": 272000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4.5e-06,
|
||||
"output_cost_per_token_flex": 2.25e-06,
|
||||
"output_cost_per_token_batches": 2.25e-06,
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
"/v1/batch",
|
||||
"/v1/responses"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_native_streaming": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_service_tier": true,
|
||||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": false
|
||||
},
|
||||
"gpt-5.4-nano": {
|
||||
"cache_read_input_token_cost": 2e-08,
|
||||
"cache_read_input_token_cost_flex": 1e-08,
|
||||
"cache_read_input_token_cost_batches": 1e-08,
|
||||
"input_cost_per_token": 2e-07,
|
||||
"input_cost_per_token_flex": 1e-07,
|
||||
"input_cost_per_token_batches": 1e-07,
|
||||
"litellm_provider": "openai",
|
||||
"max_input_tokens": 272000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.25e-06,
|
||||
"output_cost_per_token_flex": 6.25e-07,
|
||||
"output_cost_per_token_batches": 6.25e-07,
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
"/v1/batch",
|
||||
"/v1/responses"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_native_streaming": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_service_tier": true,
|
||||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": false
|
||||
},
|
||||
"gpt-5-pro": {
|
||||
"input_cost_per_token": 1.5e-05,
|
||||
|
|
@ -18852,7 +18968,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": false
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5-pro-2025-10-06": {
|
||||
"input_cost_per_token": 1.5e-05,
|
||||
|
|
@ -18887,7 +19004,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": false
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5-2025-08-07": {
|
||||
"cache_read_input_token_cost": 1.25e-07,
|
||||
|
|
@ -18929,7 +19047,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": false
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5-chat": {
|
||||
"cache_read_input_token_cost": 1.25e-07,
|
||||
|
|
@ -18963,7 +19082,8 @@
|
|||
"supports_tool_choice": false,
|
||||
"supports_vision": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": false
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5-chat-latest": {
|
||||
"cache_read_input_token_cost": 1.25e-07,
|
||||
|
|
@ -18997,7 +19117,8 @@
|
|||
"supports_tool_choice": false,
|
||||
"supports_vision": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": false
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5-codex": {
|
||||
"cache_read_input_token_cost": 1.25e-07,
|
||||
|
|
@ -19030,7 +19151,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": false
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5.1-codex": {
|
||||
"cache_read_input_token_cost": 1.25e-07,
|
||||
|
|
@ -19066,7 +19188,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": false
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5.1-codex-max": {
|
||||
"cache_read_input_token_cost": 1.25e-07,
|
||||
|
|
@ -19099,7 +19222,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": true
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5.1-codex-mini": {
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
|
|
@ -19135,7 +19259,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": false
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5.2-codex": {
|
||||
"cache_read_input_token_cost": 1.75e-07,
|
||||
|
|
@ -19171,7 +19296,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": true
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5.3-codex": {
|
||||
"cache_read_input_token_cost": 1.75e-07,
|
||||
|
|
@ -19207,7 +19333,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": false
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5-mini": {
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
|
|
@ -19249,7 +19376,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": false
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5-mini-2025-08-07": {
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
|
|
@ -19291,7 +19419,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": false
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5-nano": {
|
||||
"cache_read_input_token_cost": 5e-09,
|
||||
|
|
@ -19330,7 +19459,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": false
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5-nano-2025-08-07": {
|
||||
"cache_read_input_token_cost": 5e-09,
|
||||
|
|
@ -19368,7 +19498,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": false
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-image-1": {
|
||||
"cache_read_input_image_token_cost": 2.5e-06,
|
||||
|
|
@ -30506,7 +30637,7 @@
|
|||
"output_cost_per_token": 5.4e-06,
|
||||
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models",
|
||||
"supported_regions": [
|
||||
"us-west2"
|
||||
"us-central1"
|
||||
],
|
||||
"supports_assistant_prefill": true,
|
||||
"supports_function_calling": true,
|
||||
|
|
@ -30526,7 +30657,7 @@
|
|||
"output_cost_per_token_batches": 8.4e-07,
|
||||
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models",
|
||||
"supported_regions": [
|
||||
"us-west2"
|
||||
"global"
|
||||
],
|
||||
"supports_assistant_prefill": true,
|
||||
"supports_function_calling": true,
|
||||
|
|
@ -30543,6 +30674,9 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 5.4e-06,
|
||||
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models",
|
||||
"supported_regions": [
|
||||
"us-central1"
|
||||
],
|
||||
"supports_assistant_prefill": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
|
|
@ -31167,7 +31301,10 @@
|
|||
"input_cost_per_token": 3e-07,
|
||||
"output_cost_per_token": 1.2e-06,
|
||||
"ocr_cost_per_page": 0.0003,
|
||||
"source": "https://cloud.google.com/vertex-ai/pricing"
|
||||
"source": "https://cloud.google.com/vertex-ai/pricing",
|
||||
"supported_regions": [
|
||||
"us-central1"
|
||||
]
|
||||
},
|
||||
"vertex_ai/openai/gpt-oss-120b-maas": {
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
|
|
@ -36386,7 +36523,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": false
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5-search-api-2025-10-14": {
|
||||
"cache_read_input_token_cost": 1.25e-07,
|
||||
|
|
|
|||
|
|
@ -92,6 +92,7 @@ class MCPGuardrailTranslationHandler(BaseTranslation):
|
|||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: Optional[Any] = None,
|
||||
user_api_key_dict: Optional[Any] = None,
|
||||
request_data: Optional[dict] = None,
|
||||
) -> Any:
|
||||
verbose_proxy_logger.debug(
|
||||
"MCP Guardrail: Output processing not implemented for MCP tools",
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -1,27 +1,28 @@
|
|||
1:"$Sreact.fragment"
|
||||
2:I[347257,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"ClientPageRoot"]
|
||||
3:I[952683,["/litellm-asset-prefix/_next/static/chunks/a7f104aa2cc7f3f0.js","/litellm-asset-prefix/_next/static/chunks/e8ed72789c2b42ff.js","/litellm-asset-prefix/_next/static/chunks/e627c7aa5ead52b3.js","/litellm-asset-prefix/_next/static/chunks/142704439974f6b3.js","/litellm-asset-prefix/_next/static/chunks/30539b80ac15aad2.js","/litellm-asset-prefix/_next/static/chunks/7d82a1cebfdb679c.js","/litellm-asset-prefix/_next/static/chunks/2d471965761a22ff.js","/litellm-asset-prefix/_next/static/chunks/1fe0596a309ad6cf.js","/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","/litellm-asset-prefix/_next/static/chunks/d64d74932cb225a3.js","/litellm-asset-prefix/_next/static/chunks/8c13023d89b01566.js","/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","/litellm-asset-prefix/_next/static/chunks/74ce31aa0fb2adc9.js","/litellm-asset-prefix/_next/static/chunks/cdf98a03da656604.js","/litellm-asset-prefix/_next/static/chunks/cac89fc12fb6ef7e.js","/litellm-asset-prefix/_next/static/chunks/d0d828f9a0668699.js","/litellm-asset-prefix/_next/static/chunks/1a04d31843c96649.js","/litellm-asset-prefix/_next/static/chunks/ed90bf177ad61e18.js","/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","/litellm-asset-prefix/_next/static/chunks/6c4c97f1ea6e7d77.js","/litellm-asset-prefix/_next/static/chunks/9d6e5aad99b19216.js","/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","/litellm-asset-prefix/_next/static/chunks/acbeac1b0fde1fdf.js","/litellm-asset-prefix/_next/static/chunks/a89452659b6e1d90.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/348b31083769a7c4.js","/litellm-asset-prefix/_next/static/chunks/a85adee4198d5478.js","/litellm-asset-prefix/_next/static/chunks/c8eee6971ca36303.js","/litellm-asset-prefix/_next/static/chunks/67ddb5107368a659.js","/litellm-asset-prefix/_next/static/chunks/b6cdb9a433f054f3.js","/litellm-asset-prefix/_next/static/chunks/22970a12064ba16b.js","/litellm-asset-prefix/_next/static/chunks/4348e537165edb3b.js","/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","/litellm-asset-prefix/_next/static/chunks/d069df5baead6d90.js","/litellm-asset-prefix/_next/static/chunks/0dda11815be4f78b.js","/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","/litellm-asset-prefix/_next/static/chunks/fc4d54eb6afe7984.js","/litellm-asset-prefix/_next/static/chunks/40f766ecc87dbf9a.js","/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","/litellm-asset-prefix/_next/static/chunks/06ebe9b0e9cdf241.js","/litellm-asset-prefix/_next/static/chunks/df6546cd8a44d3b3.js","/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","/litellm-asset-prefix/_next/static/chunks/23bf955e8672ce98.js","/litellm-asset-prefix/_next/static/chunks/8dda507c226082ca.js","/litellm-asset-prefix/_next/static/chunks/54e29148cb2f2582.js","/litellm-asset-prefix/_next/static/chunks/3675074b1d85e268.js","/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","/litellm-asset-prefix/_next/static/chunks/bd94e2fe34d8a187.js"],"default"]
|
||||
17:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"OutletBoundary"]
|
||||
18:"$Sreact.suspense"
|
||||
3:I[952683,["/litellm-asset-prefix/_next/static/chunks/112ad77f3dd2e3cd.js","/litellm-asset-prefix/_next/static/chunks/65f709264734a9bf.js","/litellm-asset-prefix/_next/static/chunks/e627c7aa5ead52b3.js","/litellm-asset-prefix/_next/static/chunks/142704439974f6b3.js","/litellm-asset-prefix/_next/static/chunks/30539b80ac15aad2.js","/litellm-asset-prefix/_next/static/chunks/99d715502d5069f4.js","/litellm-asset-prefix/_next/static/chunks/53a707a5829899ed.js","/litellm-asset-prefix/_next/static/chunks/1fe0596a309ad6cf.js","/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","/litellm-asset-prefix/_next/static/chunks/ed90bf177ad61e18.js","/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","/litellm-asset-prefix/_next/static/chunks/b4bd164f5553a31d.js","/litellm-asset-prefix/_next/static/chunks/adb8beb738574863.js","/litellm-asset-prefix/_next/static/chunks/b6cdb9a433f054f3.js","/litellm-asset-prefix/_next/static/chunks/af8668386d7005fe.js","/litellm-asset-prefix/_next/static/chunks/9606513e20bc3d4f.js","/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","/litellm-asset-prefix/_next/static/chunks/d512ca3b7169bef6.js","/litellm-asset-prefix/_next/static/chunks/e1e3f652dbc5be03.js","/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","/litellm-asset-prefix/_next/static/chunks/f3e0cbc0e84e0a5d.js","/litellm-asset-prefix/_next/static/chunks/8c13023d89b01566.js","/litellm-asset-prefix/_next/static/chunks/338e84191fe615bf.js","/litellm-asset-prefix/_next/static/chunks/c53c9c7afec96700.js","/litellm-asset-prefix/_next/static/chunks/5f4170980a69ffa3.js","/litellm-asset-prefix/_next/static/chunks/c74f3813068add76.js","/litellm-asset-prefix/_next/static/chunks/99109c78121231a0.js","/litellm-asset-prefix/_next/static/chunks/5929da573d876909.js","/litellm-asset-prefix/_next/static/chunks/58461a445becf104.js","/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","/litellm-asset-prefix/_next/static/chunks/d0d828f9a0668699.js","/litellm-asset-prefix/_next/static/chunks/ce9cf9f407f4b359.js","/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","/litellm-asset-prefix/_next/static/chunks/6b2bc4046c4cbfc8.js","/litellm-asset-prefix/_next/static/chunks/5400ee883dfa8c43.js","/litellm-asset-prefix/_next/static/chunks/bd94e2fe34d8a187.js","/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","/litellm-asset-prefix/_next/static/chunks/7a2dc852f68481ea.js","/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","/litellm-asset-prefix/_next/static/chunks/f9c24d6e7ec43046.js","/litellm-asset-prefix/_next/static/chunks/7c797521435cb59c.js","/litellm-asset-prefix/_next/static/chunks/5855ff7033bd4d2e.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/1da362a651d209bd.js","/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","/litellm-asset-prefix/_next/static/chunks/0b8ec8bf90ea9721.js"],"default"]
|
||||
18:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"OutletBoundary"]
|
||||
19:"$Sreact.suspense"
|
||||
:HL["/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","style"]
|
||||
0:{"buildId":"aKKihXXKRJWLQThZgi8Rq","rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/e627c7aa5ead52b3.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/142704439974f6b3.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/30539b80ac15aad2.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/7d82a1cebfdb679c.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/2d471965761a22ff.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/1fe0596a309ad6cf.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/d64d74932cb225a3.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/8c13023d89b01566.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/74ce31aa0fb2adc9.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/cdf98a03da656604.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/cac89fc12fb6ef7e.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/d0d828f9a0668699.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/1a04d31843c96649.js","async":true}],["$","script","script-18",{"src":"/litellm-asset-prefix/_next/static/chunks/ed90bf177ad61e18.js","async":true}],["$","script","script-19",{"src":"/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","async":true}],["$","script","script-20",{"src":"/litellm-asset-prefix/_next/static/chunks/6c4c97f1ea6e7d77.js","async":true}],["$","script","script-21",{"src":"/litellm-asset-prefix/_next/static/chunks/9d6e5aad99b19216.js","async":true}],["$","script","script-22",{"src":"/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","async":true}],["$","script","script-23",{"src":"/litellm-asset-prefix/_next/static/chunks/acbeac1b0fde1fdf.js","async":true}],["$","script","script-24",{"src":"/litellm-asset-prefix/_next/static/chunks/a89452659b6e1d90.js","async":true}],["$","script","script-25",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true}],["$","script","script-26",{"src":"/litellm-asset-prefix/_next/static/chunks/348b31083769a7c4.js","async":true}],["$","script","script-27",{"src":"/litellm-asset-prefix/_next/static/chunks/a85adee4198d5478.js","async":true}],["$","script","script-28",{"src":"/litellm-asset-prefix/_next/static/chunks/c8eee6971ca36303.js","async":true}],["$","script","script-29",{"src":"/litellm-asset-prefix/_next/static/chunks/67ddb5107368a659.js","async":true}],["$","script","script-30",{"src":"/litellm-asset-prefix/_next/static/chunks/b6cdb9a433f054f3.js","async":true}],["$","script","script-31",{"src":"/litellm-asset-prefix/_next/static/chunks/22970a12064ba16b.js","async":true}],["$","script","script-32",{"src":"/litellm-asset-prefix/_next/static/chunks/4348e537165edb3b.js","async":true}],["$","script","script-33",{"src":"/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","async":true}],"$L6","$L7","$L8","$L9","$La","$Lb","$Lc","$Ld","$Le","$Lf","$L10","$L11","$L12","$L13","$L14","$L15"],"$L16"]}],"loading":null,"isPartial":false}
|
||||
0:{"buildId":"Hp-LQxDEAEt-JSJFExm-i","rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/e627c7aa5ead52b3.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/142704439974f6b3.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/30539b80ac15aad2.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/99d715502d5069f4.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/53a707a5829899ed.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/1fe0596a309ad6cf.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/ed90bf177ad61e18.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/b4bd164f5553a31d.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/adb8beb738574863.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/b6cdb9a433f054f3.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/af8668386d7005fe.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/9606513e20bc3d4f.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","async":true}],["$","script","script-18",{"src":"/litellm-asset-prefix/_next/static/chunks/d512ca3b7169bef6.js","async":true}],["$","script","script-19",{"src":"/litellm-asset-prefix/_next/static/chunks/e1e3f652dbc5be03.js","async":true}],["$","script","script-20",{"src":"/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","async":true}],["$","script","script-21",{"src":"/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","async":true}],["$","script","script-22",{"src":"/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","async":true}],["$","script","script-23",{"src":"/litellm-asset-prefix/_next/static/chunks/f3e0cbc0e84e0a5d.js","async":true}],["$","script","script-24",{"src":"/litellm-asset-prefix/_next/static/chunks/8c13023d89b01566.js","async":true}],["$","script","script-25",{"src":"/litellm-asset-prefix/_next/static/chunks/338e84191fe615bf.js","async":true}],["$","script","script-26",{"src":"/litellm-asset-prefix/_next/static/chunks/c53c9c7afec96700.js","async":true}],["$","script","script-27",{"src":"/litellm-asset-prefix/_next/static/chunks/5f4170980a69ffa3.js","async":true}],["$","script","script-28",{"src":"/litellm-asset-prefix/_next/static/chunks/c74f3813068add76.js","async":true}],["$","script","script-29",{"src":"/litellm-asset-prefix/_next/static/chunks/99109c78121231a0.js","async":true}],["$","script","script-30",{"src":"/litellm-asset-prefix/_next/static/chunks/5929da573d876909.js","async":true}],["$","script","script-31",{"src":"/litellm-asset-prefix/_next/static/chunks/58461a445becf104.js","async":true}],["$","script","script-32",{"src":"/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","async":true}],["$","script","script-33",{"src":"/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","async":true}],"$L6","$L7","$L8","$L9","$La","$Lb","$Lc","$Ld","$Le","$Lf","$L10","$L11","$L12","$L13","$L14","$L15","$L16"],"$L17"]}],"loading":null,"isPartial":false}
|
||||
4:{}
|
||||
5:"$0:rsc:props:children:0:props:serverProvidedParams:params"
|
||||
6:["$","script","script-34",{"src":"/litellm-asset-prefix/_next/static/chunks/d069df5baead6d90.js","async":true}]
|
||||
7:["$","script","script-35",{"src":"/litellm-asset-prefix/_next/static/chunks/0dda11815be4f78b.js","async":true}]
|
||||
6:["$","script","script-34",{"src":"/litellm-asset-prefix/_next/static/chunks/d0d828f9a0668699.js","async":true}]
|
||||
7:["$","script","script-35",{"src":"/litellm-asset-prefix/_next/static/chunks/ce9cf9f407f4b359.js","async":true}]
|
||||
8:["$","script","script-36",{"src":"/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","async":true}]
|
||||
9:["$","script","script-37",{"src":"/litellm-asset-prefix/_next/static/chunks/fc4d54eb6afe7984.js","async":true}]
|
||||
a:["$","script","script-38",{"src":"/litellm-asset-prefix/_next/static/chunks/40f766ecc87dbf9a.js","async":true}]
|
||||
b:["$","script","script-39",{"src":"/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","async":true}]
|
||||
c:["$","script","script-40",{"src":"/litellm-asset-prefix/_next/static/chunks/06ebe9b0e9cdf241.js","async":true}]
|
||||
d:["$","script","script-41",{"src":"/litellm-asset-prefix/_next/static/chunks/df6546cd8a44d3b3.js","async":true}]
|
||||
e:["$","script","script-42",{"src":"/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","async":true}]
|
||||
f:["$","script","script-43",{"src":"/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","async":true}]
|
||||
10:["$","script","script-44",{"src":"/litellm-asset-prefix/_next/static/chunks/23bf955e8672ce98.js","async":true}]
|
||||
11:["$","script","script-45",{"src":"/litellm-asset-prefix/_next/static/chunks/8dda507c226082ca.js","async":true}]
|
||||
12:["$","script","script-46",{"src":"/litellm-asset-prefix/_next/static/chunks/54e29148cb2f2582.js","async":true}]
|
||||
13:["$","script","script-47",{"src":"/litellm-asset-prefix/_next/static/chunks/3675074b1d85e268.js","async":true}]
|
||||
14:["$","script","script-48",{"src":"/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","async":true}]
|
||||
15:["$","script","script-49",{"src":"/litellm-asset-prefix/_next/static/chunks/bd94e2fe34d8a187.js","async":true}]
|
||||
16:["$","$L17",null,{"children":["$","$18",null,{"name":"Next.MetadataOutlet","children":"$@19"}]}]
|
||||
19:null
|
||||
9:["$","script","script-37",{"src":"/litellm-asset-prefix/_next/static/chunks/6b2bc4046c4cbfc8.js","async":true}]
|
||||
a:["$","script","script-38",{"src":"/litellm-asset-prefix/_next/static/chunks/5400ee883dfa8c43.js","async":true}]
|
||||
b:["$","script","script-39",{"src":"/litellm-asset-prefix/_next/static/chunks/bd94e2fe34d8a187.js","async":true}]
|
||||
c:["$","script","script-40",{"src":"/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","async":true}]
|
||||
d:["$","script","script-41",{"src":"/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","async":true}]
|
||||
e:["$","script","script-42",{"src":"/litellm-asset-prefix/_next/static/chunks/7a2dc852f68481ea.js","async":true}]
|
||||
f:["$","script","script-43",{"src":"/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","async":true}]
|
||||
10:["$","script","script-44",{"src":"/litellm-asset-prefix/_next/static/chunks/f9c24d6e7ec43046.js","async":true}]
|
||||
11:["$","script","script-45",{"src":"/litellm-asset-prefix/_next/static/chunks/7c797521435cb59c.js","async":true}]
|
||||
12:["$","script","script-46",{"src":"/litellm-asset-prefix/_next/static/chunks/5855ff7033bd4d2e.js","async":true}]
|
||||
13:["$","script","script-47",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true}]
|
||||
14:["$","script","script-48",{"src":"/litellm-asset-prefix/_next/static/chunks/1da362a651d209bd.js","async":true}]
|
||||
15:["$","script","script-49",{"src":"/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","async":true}]
|
||||
16:["$","script","script-50",{"src":"/litellm-asset-prefix/_next/static/chunks/0b8ec8bf90ea9721.js","async":true}]
|
||||
17:["$","$L18",null,{"children":["$","$19",null,{"name":"Next.MetadataOutlet","children":"$@1a"}]}]
|
||||
1a:null
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -3,4 +3,4 @@
|
|||
3:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"MetadataBoundary"]
|
||||
4:"$Sreact.suspense"
|
||||
5:I[27201,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"IconMark"]
|
||||
0:{"buildId":"aKKihXXKRJWLQThZgi8Rq","rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"./favicon.ico"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"loading":null,"isPartial":false}
|
||||
0:{"buildId":"Hp-LQxDEAEt-JSJFExm-i","rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"./favicon.ico"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"loading":null,"isPartial":false}
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
1:"$Sreact.fragment"
|
||||
2:I[867271,["/litellm-asset-prefix/_next/static/chunks/a7f104aa2cc7f3f0.js","/litellm-asset-prefix/_next/static/chunks/e8ed72789c2b42ff.js"],"default"]
|
||||
3:I[71195,["/litellm-asset-prefix/_next/static/chunks/a7f104aa2cc7f3f0.js","/litellm-asset-prefix/_next/static/chunks/e8ed72789c2b42ff.js"],"default"]
|
||||
2:I[867271,["/litellm-asset-prefix/_next/static/chunks/112ad77f3dd2e3cd.js","/litellm-asset-prefix/_next/static/chunks/65f709264734a9bf.js"],"default"]
|
||||
3:I[71195,["/litellm-asset-prefix/_next/static/chunks/112ad77f3dd2e3cd.js","/litellm-asset-prefix/_next/static/chunks/65f709264734a9bf.js"],"default"]
|
||||
4:I[339756,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
|
||||
5:I[837457,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
|
||||
:HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"]
|
||||
:HL["/litellm-asset-prefix/_next/static/chunks/8dc3b559a2e76f88.css","style"]
|
||||
0:{"buildId":"aKKihXXKRJWLQThZgi8Rq","rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/8dc3b559a2e76f88.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/a7f104aa2cc7f3f0.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/e8ed72789c2b42ff.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]]}],"loading":null,"isPartial":false}
|
||||
:HL["/litellm-asset-prefix/_next/static/chunks/cab8d46a8c32ec36.css","style"]
|
||||
0:{"buildId":"Hp-LQxDEAEt-JSJFExm-i","rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/cab8d46a8c32ec36.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/112ad77f3dd2e3cd.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/65f709264734a9bf.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]]}],"loading":null,"isPartial":false}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
:HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"]
|
||||
:HL["/litellm-asset-prefix/_next/static/chunks/8dc3b559a2e76f88.css","style"]
|
||||
:HL["/litellm-asset-prefix/_next/static/chunks/cab8d46a8c32ec36.css","style"]
|
||||
:HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.3a6ba036.woff2","font",{"crossOrigin":"","type":"font/woff2"}]
|
||||
:HL["/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","style"]
|
||||
0:{"buildId":"aKKihXXKRJWLQThZgi8Rq","tree":{"name":"","paramType":null,"paramKey":"","hasRuntimePrefetch":false,"slots":{"children":{"name":"__PAGE__","paramType":null,"paramKey":"__PAGE__","hasRuntimePrefetch":false,"slots":null,"isRootLayout":false}},"isRootLayout":true},"staleTime":300}
|
||||
0:{"buildId":"Hp-LQxDEAEt-JSJFExm-i","tree":{"name":"","paramType":null,"paramKey":"","hasRuntimePrefetch":false,"slots":{"children":{"name":"__PAGE__","paramType":null,"paramKey":"__PAGE__","hasRuntimePrefetch":false,"slots":null,"isRootLayout":false}},"isRootLayout":true},"staleTime":300}
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -1 +0,0 @@
|
|||
(globalThis.TURBOPACK||(globalThis.TURBOPACK=[])).push(["object"==typeof document?document.currentScript:void 0,233525,(e,c,u)=>{"use strict";Object.defineProperty(u,"__esModule",{value:!0}),Object.defineProperty(u,"warnOnce",{enumerable:!0,get:function(){return r}});let r=e=>{}},349356,e=>{e.v({AElig:"Æ",AMP:"&",Aacute:"Á",Acirc:"Â",Agrave:"À",Aring:"Å",Atilde:"Ã",Auml:"Ä",COPY:"©",Ccedil:"Ç",ETH:"Ð",Eacute:"É",Ecirc:"Ê",Egrave:"È",Euml:"Ë",GT:">",Iacute:"Í",Icirc:"Î",Igrave:"Ì",Iuml:"Ï",LT:"<",Ntilde:"Ñ",Oacute:"Ó",Ocirc:"Ô",Ograve:"Ò",Oslash:"Ø",Otilde:"Õ",Ouml:"Ö",QUOT:'"',REG:"®",THORN:"Þ",Uacute:"Ú",Ucirc:"Û",Ugrave:"Ù",Uuml:"Ü",Yacute:"Ý",aacute:"á",acirc:"â",acute:"´",aelig:"æ",agrave:"à",amp:"&",aring:"å",atilde:"ã",auml:"ä",brvbar:"¦",ccedil:"ç",cedil:"¸",cent:"¢",copy:"©",curren:"¤",deg:"°",divide:"÷",eacute:"é",ecirc:"ê",egrave:"è",eth:"ð",euml:"ë",frac12:"½",frac14:"¼",frac34:"¾",gt:">",iacute:"í",icirc:"î",iexcl:"¡",igrave:"ì",iquest:"¿",iuml:"ï",laquo:"«",lt:"<",macr:"¯",micro:"µ",middot:"·",nbsp:" ",not:"¬",ntilde:"ñ",oacute:"ó",ocirc:"ô",ograve:"ò",ordf:"ª",ordm:"º",oslash:"ø",otilde:"õ",ouml:"ö",para:"¶",plusmn:"±",pound:"£",quot:'"',raquo:"»",reg:"®",sect:"§",shy:"",sup1:"¹",sup2:"²",sup3:"³",szlig:"ß",thorn:"þ",times:"×",uacute:"ú",ucirc:"û",ugrave:"ù",uml:"¨",uuml:"ü",yacute:"ý",yen:"¥",yuml:"ÿ"})},137429,e=>{e.v({0:"<22>",128:"€",130:"‚",131:"ƒ",132:"„",133:"…",134:"†",135:"‡",136:"ˆ",137:"‰",138:"Š",139:"‹",140:"Œ",142:"Ž",145:"‘",146:"’",147:"“",148:"”",149:"•",150:"–",151:"—",152:"˜",153:"™",154:"š",155:"›",156:"œ",158:"ž",159:"Ÿ"})},928685,e=>{"use strict";var c=e.i(38953);e.s(["SearchOutlined",()=>c.default])},86408,e=>{"use strict";var c=e.i(843476),u=e.i(271645),r=e.i(618566),t=e.i(934879);function a(){let e=(0,r.useSearchParams)().get("key"),[a,i]=(0,u.useState)(null);return console.log("PublicModelHubTable accessToken:",a),(0,u.useEffect)(()=>{e&&i(e)},[e]),(0,c.jsx)(t.default,{accessToken:a,publicPage:!0,premiumUser:!1,userRole:null})}function i(){return(0,c.jsx)(u.Suspense,{fallback:(0,c.jsx)("div",{className:"flex items-center justify-center min-h-screen",children:"Loading..."}),children:(0,c.jsx)(a,{})})}e.s(["default",()=>i])}]);
|
||||
File diff suppressed because one or more lines are too long
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue