mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-09 22:31:41 +00:00
Merge branch 'main' into litellm_oss_staging_01_31_2026
This commit is contained in:
Sameer Kankute
GitHub
commit
382aab0362
374 changed files with 8536 additions and 3662 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -95,6 +95,7 @@ update_model_cost_map.py
|
|||
tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
|
||||
scripts/test_vertex_ai_search.py
|
||||
LAZY_LOADING_IMPROVEMENTS.md
|
||||
STABILIZATION_TODO.md
|
||||
**/test-results
|
||||
**/playwright-report
|
||||
**/*.storageState.json
|
||||
|
|
|
|||
12
.trivyignore
Normal file
12
.trivyignore
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
# LiteLLM Trivy Ignore File
|
||||
# CVEs listed here are temporarily allowlisted pending fixes
|
||||
|
||||
# Next.js vulnerabilities in UI dashboard (next@14.2.35)
|
||||
# Allowlisted: 2026-01-31, 7-day fix timeline
|
||||
# Fix: Upgrade to Next.js 15.5.10+ or 16.1.5+
|
||||
|
||||
# HIGH: DoS via request deserialization
|
||||
GHSA-h25m-26qc-wcjf
|
||||
|
||||
# MEDIUM: Image Optimizer DoS
|
||||
CVE-2025-59471
|
||||
|
|
@ -47,7 +47,8 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
|
|||
USER root
|
||||
|
||||
# Install runtime dependencies (libsndfile needed for audio processing on ARM64)
|
||||
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile
|
||||
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
|
||||
npm install -g npm@latest tar@latest
|
||||
|
||||
WORKDIR /app
|
||||
# Copy the current directory contents into the container at /app
|
||||
|
|
|
|||
|
|
@ -81,10 +81,10 @@ run_trivy_scans() {
|
|||
echo "Running Trivy scans..."
|
||||
|
||||
echo "Scanning LiteLLM Docs..."
|
||||
trivy fs --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./docs/
|
||||
trivy fs --ignorefile .trivyignore --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./docs/
|
||||
|
||||
echo "Scanning LiteLLM UI..."
|
||||
trivy fs --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./ui/
|
||||
trivy fs --ignorefile .trivyignore --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./ui/
|
||||
|
||||
echo "Trivy scans completed successfully"
|
||||
}
|
||||
|
|
@ -137,6 +137,7 @@ run_grype_scans() {
|
|||
"CVE-2019-1010025" # glibc pthread heap address leak - awaiting patched Wolfi glibc build
|
||||
"CVE-2026-22184" # zlib untgz buffer overflow - untgz unused + no fixed Wolfi build yet
|
||||
"GHSA-58pv-8j8x-9vj2" # jaraco.context path traversal - setuptools vendored only (v5.3.0), not used in application code (using v6.1.0+)
|
||||
"GHSA-34x7-hfp2-rc4v" # node-tar hardlink path traversal - not applicable, tar CLI not exposed in application code
|
||||
"GHSA-r6q2-hw4h-h46w" # node-tar not used by application runtime, Linux-only container, not affect by macOS APFS-specific exploit
|
||||
"GHSA-8rrh-rw8j-w5fx" # wheel is from chainguard and will be handled by then TODO: Remove this after Chainguard updates the wheel
|
||||
"CVE-2025-59465" # We do not use Node in application runtime, only used for building Admin UI
|
||||
|
|
|
|||
|
|
@ -5,7 +5,8 @@ FROM ghcr.io/berriai/litellm:litellm_fwd_server_root_path-dev
|
|||
WORKDIR /app
|
||||
|
||||
# Install Node.js and npm (adjust version as needed)
|
||||
RUN apt-get update && apt-get install -y nodejs npm
|
||||
RUN apt-get update && apt-get install -y nodejs npm && \
|
||||
npm install -g npm@latest tar@latest
|
||||
|
||||
# Copy the UI source into the container
|
||||
COPY ./ui/litellm-dashboard /app/ui/litellm-dashboard
|
||||
|
|
|
|||
|
|
@ -49,7 +49,8 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
|
|||
USER root
|
||||
|
||||
# Install runtime dependencies
|
||||
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile
|
||||
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
|
||||
npm install -g npm@latest tar@latest
|
||||
|
||||
WORKDIR /app
|
||||
# Copy the current directory contents into the container at /app
|
||||
|
|
|
|||
|
|
@ -61,7 +61,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
|||
libatomic1 \
|
||||
nodejs \
|
||||
npm \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& npm install -g npm@latest tar@latest
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
|
|
|
|||
|
|
@ -104,7 +104,8 @@ RUN for i in 1 2 3; do \
|
|||
done \
|
||||
&& for i in 1 2 3; do \
|
||||
apk add --no-cache python3 py3-pip bash openssl tzdata nodejs npm supervisor && break || sleep 5; \
|
||||
done
|
||||
done \
|
||||
&& npm install -g npm@latest tar@latest
|
||||
|
||||
# Copy artifacts from builder
|
||||
COPY --from=builder /app/requirements.txt /app/requirements.txt
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ authors:
|
|||
title: "CTO, LiteLLM"
|
||||
url: https://www.linkedin.com/in/reffajnaahsi/
|
||||
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
|
||||
description: "Guide to Claude Opus 4.5 and advanced features in LiteLLM: Tool Search, Programmatic Tool Calling, and Effort Parameter."
|
||||
tags: [anthropic, claude, tool search, programmatic tool calling, effort, advanced features]
|
||||
hide_table_of_contents: false
|
||||
---
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ authors:
|
|||
title: "CTO, LiteLLM"
|
||||
url: https://www.linkedin.com/in/reffajnaahsi/
|
||||
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
|
||||
description: "Common questions and best practices for using gemini-3-pro-preview with LiteLLM Proxy and SDK."
|
||||
tags: [gemini, day 0 support, llms]
|
||||
hide_table_of_contents: false
|
||||
---
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ authors:
|
|||
title: "CTO, LiteLLM"
|
||||
url: https://www.linkedin.com/in/reffajnaahsi/
|
||||
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
|
||||
description: "Guide to using Gemini 3 Flash on LiteLLM Proxy and SDK with day 0 support."
|
||||
tags: [gemini, day 0 support, llms]
|
||||
hide_table_of_contents: false
|
||||
---
|
||||
|
|
|
|||
|
|
@ -5,19 +5,38 @@ Azure Model Router is a feature in Azure AI Foundry that automatically routes yo
|
|||
## Key Features
|
||||
|
||||
- **Automatic Model Selection**: Azure Model Router dynamically selects the best model for your request
|
||||
- **Cost Tracking**: LiteLLM automatically tracks costs based on the actual model used (e.g., `gpt-4.1-nano`), not the router endpoint
|
||||
- **Cost Tracking**: LiteLLM automatically tracks costs based on the actual model used (e.g., `gpt-4.1-nano`), plus the Model Router infrastructure fee
|
||||
- **Streaming Support**: Full support for streaming responses with accurate cost calculation
|
||||
- **Simple Configuration**: Easy to set up via UI or config file
|
||||
|
||||
## Model Naming Pattern
|
||||
|
||||
Use the pattern: `azure_ai/model_router/<deployment-name>`
|
||||
|
||||
**Components:**
|
||||
- `azure_ai` - The provider identifier
|
||||
- `model_router` - Indicates this is a Model Router deployment
|
||||
- `<deployment-name>` - Your actual deployment name from Azure AI Foundry (e.g., `azure-model-router`)
|
||||
|
||||
**Example:** `azure_ai/model_router/azure-model-router`
|
||||
|
||||
**How it works:**
|
||||
- LiteLLM automatically strips the `model_router/` prefix when sending requests to Azure
|
||||
- Only your deployment name (e.g., `azure-model-router`) is sent to the Azure API
|
||||
- The full path is preserved in responses and logs for proper cost tracking
|
||||
|
||||
## LiteLLM Python SDK
|
||||
|
||||
### Basic Usage
|
||||
|
||||
Use the pattern `azure_ai/model_router/<deployment-name>` where `<deployment-name>` is your Azure deployment name:
|
||||
|
||||
```python
|
||||
import litellm
|
||||
import os
|
||||
|
||||
response = litellm.completion(
|
||||
model="azure_ai/azure-model-router",
|
||||
model="azure_ai/model_router/azure-model-router", # Use your deployment name
|
||||
messages=[{"role": "user", "content": "Hello!"}],
|
||||
api_base="https://your-endpoint.cognitiveservices.azure.com/openai/v1/",
|
||||
api_key=os.getenv("AZURE_MODEL_ROUTER_API_KEY"),
|
||||
|
|
@ -26,6 +45,13 @@ response = litellm.completion(
|
|||
print(response)
|
||||
```
|
||||
|
||||
**Pattern Explanation:**
|
||||
- `azure_ai` - The provider
|
||||
- `model_router` - Indicates this is a model router deployment
|
||||
- `azure-model-router` - Your actual deployment name from Azure AI Foundry
|
||||
|
||||
LiteLLM will automatically strip the `model_router/` prefix when sending the request to Azure, so only `azure-model-router` is sent to the API.
|
||||
|
||||
### Streaming with Usage Tracking
|
||||
|
||||
```python
|
||||
|
|
@ -33,7 +59,7 @@ import litellm
|
|||
import os
|
||||
|
||||
response = await litellm.acompletion(
|
||||
model="azure_ai/azure-model-router",
|
||||
model="azure_ai/model_router/azure-model-router", # Use your deployment name
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
api_base="https://your-endpoint.cognitiveservices.azure.com/openai/v1/",
|
||||
api_key=os.getenv("AZURE_MODEL_ROUTER_API_KEY"),
|
||||
|
|
@ -51,13 +77,15 @@ async for chunk in response:
|
|||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: azure-model-router
|
||||
- model_name: azure-model-router # Public name for your users
|
||||
litellm_params:
|
||||
model: azure_ai/azure-model-router
|
||||
model: azure_ai/model_router/azure-model-router # Use your deployment name
|
||||
api_base: https://your-endpoint.cognitiveservices.azure.com/openai/v1/
|
||||
api_key: os.environ/AZURE_MODEL_ROUTER_API_KEY
|
||||
```
|
||||
|
||||
**Note:** Replace `azure-model-router` in the model path with your actual deployment name from Azure AI Foundry.
|
||||
|
||||
### Start Proxy
|
||||
|
||||
```bash
|
||||
|
|
@ -80,49 +108,42 @@ curl -X POST http://localhost:4000/chat/completions \
|
|||
|
||||
This walkthrough shows how to add an Azure Model Router endpoint to LiteLLM using the Admin Dashboard.
|
||||
|
||||
### Select Provider
|
||||
### Quick Start
|
||||
|
||||
1. Navigate to the **Models** page in the LiteLLM UI
|
||||
2. Select **"Azure AI Foundry (Studio)"** as the provider
|
||||
3. Enter your deployment name (e.g., `azure-model-router`)
|
||||
4. LiteLLM will automatically format it as `azure_ai/model_router/azure-model-router`
|
||||
5. Add your API base URL and API key
|
||||
6. Test and save
|
||||
|
||||
### Detailed Walkthrough
|
||||
|
||||
#### Step 1: Select Provider
|
||||
|
||||
Navigate to the Models page and select "Azure AI Foundry (Studio)" as the provider.
|
||||
|
||||
#### Navigate to Models Page
|
||||
##### Navigate to Models Page
|
||||
|
||||
|
||||
|
||||
#### Click Provider Dropdown
|
||||
##### Click Provider Dropdown
|
||||
|
||||
|
||||
|
||||
#### Choose Azure AI Foundry
|
||||
##### Choose Azure AI Foundry
|
||||
|
||||
|
||||
|
||||
### Configure Model Name
|
||||
#### Step 2: Enter Deployment Name
|
||||
|
||||
Set up the model name by entering `azure_ai/` followed by your model router deployment name from Azure.
|
||||
**New Simplified Method:** Just enter your deployment name directly in the text field. If your deployment name contains "model-router" or "model_router", LiteLLM will automatically format it as `azure_ai/model_router/<deployment-name>`.
|
||||
|
||||
#### Click Model Name Field
|
||||
**Example:**
|
||||
- Enter: `azure-model-router`
|
||||
- LiteLLM creates: `azure_ai/model_router/azure-model-router`
|
||||
|
||||
|
||||
|
||||
#### Select Custom Model Name
|
||||
|
||||
|
||||
|
||||
#### Enter LiteLLM Model Name
|
||||
|
||||
|
||||
|
||||
#### Click Custom Model Name Field
|
||||
|
||||
|
||||
|
||||
#### Type Model Prefix
|
||||
|
||||
Type `azure_ai/` as the prefix.
|
||||
|
||||
|
||||
|
||||
#### Copy Model Name from Azure Portal
|
||||
##### Copy Deployment Name from Azure Portal
|
||||
|
||||
Switch to Azure AI Foundry and copy your model router deployment name.
|
||||
|
||||
|
|
@ -130,73 +151,79 @@ Switch to Azure AI Foundry and copy your model router deployment name.
|
|||
|
||||
|
||||
|
||||
#### Paste Model Name
|
||||
##### Enter Deployment Name in LiteLLM
|
||||
|
||||
Paste to get `azure_ai/azure-model-router`.
|
||||
Paste your deployment name (e.g., `azure-model-router`) directly into the text field.
|
||||
|
||||
|
||||
|
||||
|
||||
### Configure API Base and Key
|
||||
**What happens behind the scenes:**
|
||||
- You enter: `azure-model-router`
|
||||
- LiteLLM automatically detects this is a model router deployment
|
||||
- The full model path becomes: `azure_ai/model_router/azure-model-router`
|
||||
- When making API calls, only `azure-model-router` is sent to Azure
|
||||
|
||||
#### Step 3: Configure API Base and Key
|
||||
|
||||
Copy the endpoint URL and API key from Azure portal.
|
||||
|
||||
#### Copy API Base URL from Azure
|
||||
##### Copy API Base URL from Azure
|
||||
|
||||
|
||||
|
||||
#### Enter API Base in LiteLLM
|
||||
##### Enter API Base in LiteLLM
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
#### Copy API Key from Azure
|
||||
##### Copy API Key from Azure
|
||||
|
||||
|
||||
|
||||
#### Enter API Key in LiteLLM
|
||||
##### Enter API Key in LiteLLM
|
||||
|
||||
|
||||
|
||||
### Test and Add Model
|
||||
#### Step 4: Test and Add Model
|
||||
|
||||
Verify your configuration works and save the model.
|
||||
|
||||
#### Test Connection
|
||||
##### Test Connection
|
||||
|
||||
|
||||
|
||||
#### Close Test Dialog
|
||||
##### Close Test Dialog
|
||||
|
||||
|
||||
|
||||
#### Add Model
|
||||
##### Add Model
|
||||
|
||||
|
||||
|
||||
### Verify in Playground
|
||||
#### Step 5: Verify in Playground
|
||||
|
||||
Test your model and verify cost tracking is working.
|
||||
|
||||
#### Open Playground
|
||||
##### Open Playground
|
||||
|
||||
|
||||
|
||||
#### Select Model
|
||||
##### Select Model
|
||||
|
||||
|
||||
|
||||
#### Send Test Message
|
||||
##### Send Test Message
|
||||
|
||||
|
||||
|
||||
#### View Logs
|
||||
##### View Logs
|
||||
|
||||
|
||||
|
||||
#### Verify Cost Tracking
|
||||
##### Verify Cost Tracking
|
||||
|
||||
Cost is tracked based on the actual model used (e.g., `gpt-4.1-nano`).
|
||||
Cost is tracked based on the actual model used (e.g., `gpt-4.1-nano`), plus a flat infrastructure cost of $0.14 per million input tokens for using the Model Router.
|
||||
|
||||
|
||||
|
||||
|
|
@ -205,28 +232,50 @@ Cost is tracked based on the actual model used (e.g., `gpt-4.1-nano`).
|
|||
LiteLLM automatically handles cost tracking for Azure Model Router by:
|
||||
|
||||
1. **Detecting the actual model**: When Azure Model Router routes your request to a specific model (e.g., `gpt-4.1-nano-2025-04-14`), LiteLLM extracts this from the response
|
||||
2. **Calculating accurate costs**: Costs are calculated based on the actual model used, not the router endpoint name
|
||||
2. **Calculating accurate costs**: Costs are calculated based on:
|
||||
- The actual model used (e.g., `gpt-4.1-nano` token costs)
|
||||
- Plus a flat infrastructure cost of **$0.14 per million input tokens** for using the Model Router
|
||||
3. **Streaming support**: Cost tracking works correctly for both streaming and non-streaming requests
|
||||
|
||||
### Cost Breakdown
|
||||
|
||||
When you use Azure Model Router, the total cost includes:
|
||||
|
||||
- **Model Cost**: Based on the actual model that handled your request (e.g., `gpt-4.1-nano`)
|
||||
- **Router Flat Cost**: $0.14 per million input tokens (Azure AI Foundry infrastructure fee)
|
||||
|
||||
### Example Response with Cost
|
||||
|
||||
```python
|
||||
import litellm
|
||||
|
||||
response = litellm.completion(
|
||||
model="azure_ai/azure-model-router",
|
||||
model="azure_ai/model_router/azure-model-router",
|
||||
messages=[{"role": "user", "content": "Hello!"}],
|
||||
api_base="https://your-endpoint.cognitiveservices.azure.com/openai/v1/",
|
||||
api_key="your-api-key",
|
||||
)
|
||||
|
||||
# The response will show the actual model used
|
||||
print(f"Model used: {response.model}") # e.g., "gpt-4.1-nano-2025-04-14"
|
||||
print(f"Model used: {response.model}") # e.g., "azure_ai/gpt-4.1-nano-2025-04-14"
|
||||
|
||||
# Get cost
|
||||
# Get cost (includes both model cost and router flat cost)
|
||||
from litellm import completion_cost
|
||||
cost = completion_cost(completion_response=response)
|
||||
print(f"Cost: ${cost}")
|
||||
print(f"Total cost: ${cost}")
|
||||
|
||||
# Access detailed cost breakdown
|
||||
if hasattr(response, '_hidden_params') and 'response_cost' in response._hidden_params:
|
||||
print(f"Response cost: ${response._hidden_params['response_cost']}")
|
||||
```
|
||||
|
||||
### Viewing Cost Breakdown in UI
|
||||
|
||||
When viewing logs in the LiteLLM UI, you'll see:
|
||||
- **Model Cost**: The cost for the actual model used
|
||||
- **Azure Model Router Flat Cost**: The $0.14/M input tokens infrastructure fee
|
||||
- **Total Cost**: Sum of both costs
|
||||
|
||||
This breakdown helps you understand exactly what you're paying for when using the Model Router.
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
# Sarvam.ai
|
||||
|
||||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
LiteLLM supports all the text models from [Sarvam ai](https://docs.sarvam.ai/api-reference-docs/chat/chat-completions)
|
||||
|
||||
## Usage
|
||||
|
|
|
|||
|
|
@ -312,6 +312,7 @@ Gemini models with audio output capabilities using the chat completions API.
|
|||
- Only supports `pcm16` audio format
|
||||
- Streaming not yet supported
|
||||
- Must set `modalities: ["audio"]`
|
||||
- When using via LiteLLM Proxy, must include `"allowed_openai_params": ["audio", "modalities"]` in the request body to enable audio parameters
|
||||
:::
|
||||
|
||||
### Quick Start
|
||||
|
|
@ -372,7 +373,8 @@ curl http://0.0.0.0:4000/v1/chat/completions \
|
|||
"model": "gemini-tts",
|
||||
"messages": [{"role": "user", "content": "Say hello in a friendly voice"}],
|
||||
"modalities": ["audio"],
|
||||
"audio": {"voice": "Kore", "format": "pcm16"}
|
||||
"audio": {"voice": "Kore", "format": "pcm16"},
|
||||
"allowed_openai_params": ["audio", "modalities"]
|
||||
}'
|
||||
```
|
||||
|
||||
|
|
@ -389,6 +391,7 @@ response = client.chat.completions.create(
|
|||
messages=[{"role": "user", "content": "Say hello in a friendly voice"}],
|
||||
modalities=["audio"],
|
||||
audio={"voice": "Kore", "format": "pcm16"},
|
||||
extra_body={"allowed_openai_params": ["audio", "modalities"]}
|
||||
)
|
||||
print(response)
|
||||
```
|
||||
|
|
|
|||
|
|
@ -507,6 +507,7 @@ router_settings:
|
|||
| DD_AGENT_HOST | Hostname or IP of DataDog agent (e.g., "localhost"). When set, logs are sent to agent instead of direct API
|
||||
| DD_AGENT_PORT | Port of DataDog agent for log intake. Default is 10518
|
||||
| DD_API_KEY | API key for Datadog integration
|
||||
| DD_APP_KEY | Application key for Datadog Cost Management integration. Required along with DD_API_KEY for cost metrics
|
||||
| DD_SITE | Site URL for Datadog (e.g., datadoghq.com)
|
||||
| DD_SOURCE | Source identifier for Datadog logs
|
||||
| DD_TRACER_STREAMING_CHUNK_YIELD_RESOURCE | Resource name for Datadog tracing of streaming chunk yields. Default is "streaming.chunk.yield"
|
||||
|
|
@ -643,6 +644,10 @@ router_settings:
|
|||
| GENERIC_USERINFO_ENDPOINT | Endpoint to fetch user information in generic OAuth
|
||||
| GENERIC_LOGGER_ENDPOINT | Endpoint URL for the Generic Logger callback to send logs to
|
||||
| GENERIC_LOGGER_HEADERS | JSON string of headers to include in Generic Logger callback requests
|
||||
| GENERIC_ROLE_MAPPINGS_DEFAULT_ROLE | Default LiteLLM role to assign when no role mapping matches in generic SSO. Used with GENERIC_ROLE_MAPPINGS_ROLES
|
||||
| GENERIC_ROLE_MAPPINGS_GROUP_CLAIM | The claim/attribute name in the SSO token that contains the user's groups. Used for role mapping
|
||||
| GENERIC_ROLE_MAPPINGS_ROLES | Python dict string mapping LiteLLM roles to SSO group names. Example: `{"proxy_admin": ["admin-group"], "internal_user": ["users"]}`
|
||||
| GENERIC_USER_ROLE_MAPPINGS | Alternative to GENERIC_ROLE_MAPPINGS_ROLES for configuring user role mappings from SSO
|
||||
| GEMINI_API_BASE | Base URL for Gemini API. Default is https://generativelanguage.googleapis.com
|
||||
| GALILEO_BASE_URL | Base URL for Galileo platform
|
||||
| GALILEO_PASSWORD | Password for Galileo authentication
|
||||
|
|
@ -735,6 +740,7 @@ router_settings:
|
|||
| LITELLM_CLI_JWT_EXPIRATION_HOURS | Expiration time in hours for CLI-generated JWT tokens. Default is 24 hours
|
||||
| LITELLM_DD_AGENT_HOST | Hostname or IP of DataDog agent for LiteLLM-specific logging. When set, logs are sent to agent instead of direct API
|
||||
| LITELLM_DD_AGENT_PORT | Port of DataDog agent for LiteLLM-specific log intake. Default is 10518
|
||||
| LITELLM_DD_LLM_OBS_PORT | Port for Datadog LLM Observability agent. Default is 8126
|
||||
| LITELLM_DONT_SHOW_FEEDBACK_BOX | Flag to hide feedback box in LiteLLM UI
|
||||
| LITELLM_DROP_PARAMS | Parameters to drop in LiteLLM requests
|
||||
| LITELLM_MODIFY_PARAMS | Parameters to modify in LiteLLM requests
|
||||
|
|
@ -830,6 +836,7 @@ router_settings:
|
|||
| OPENMETER_EVENT_TYPE | Type of events sent to OpenMeter
|
||||
| ONYX_API_BASE | Base URL for Onyx Security AI Guard service (defaults to https://ai-guard.onyx.security)
|
||||
| ONYX_API_KEY | API key for Onyx Security AI Guard service
|
||||
| ONYX_TIMEOUT | Timeout in seconds for Onyx Guard server requests. Default is 10
|
||||
| OTEL_ENDPOINT | OpenTelemetry endpoint for traces
|
||||
| OTEL_EXPORTER_OTLP_ENDPOINT | OpenTelemetry endpoint for traces
|
||||
| OTEL_ENVIRONMENT_NAME | Environment name for OpenTelemetry
|
||||
|
|
@ -892,6 +899,8 @@ router_settings:
|
|||
| ROUTER_MAX_FALLBACKS | Maximum number of fallbacks for router. Default is 5
|
||||
| RUNWAYML_DEFAULT_API_VERSION | Default API version for RunwayML service. Default is "2024-11-06"
|
||||
| RUNWAYML_POLLING_TIMEOUT | Timeout in seconds for RunwayML image generation polling. Default is 600 (10 minutes)
|
||||
| S3_VECTORS_DEFAULT_DIMENSION | Default vector dimension for S3 Vectors RAG ingestion. Default is 1024
|
||||
| S3_VECTORS_DEFAULT_DISTANCE_METRIC | Default distance metric for S3 Vectors RAG ingestion. Options: "cosine", "euclidean". Default is "cosine"
|
||||
| SECRET_MANAGER_REFRESH_INTERVAL | Refresh interval in seconds for secret manager. Default is 86400 (24 hours)
|
||||
| SEPARATE_HEALTH_APP | If set to '1', runs health endpoints on a separate ASGI app and port. Default: '0'.
|
||||
| SEPARATE_HEALTH_PORT | Port for the separate health endpoints app. Only used if SEPARATE_HEALTH_APP=1. Default: 4001.
|
||||
|
|
|
|||
|
|
@ -6,6 +6,16 @@ import TabItem from '@theme/TabItem';
|
|||
|
||||
See supported Embedding Providers & Models [here](https://docs.litellm.ai/docs/embedding/supported_embedding)
|
||||
|
||||
## Supported Input Formats
|
||||
|
||||
The `/v1/embeddings` endpoint follows the [OpenAI embeddings API specification](https://platform.openai.com/docs/api-reference/embeddings/create). The following input formats are supported:
|
||||
|
||||
| Format | Example |
|
||||
|--------|---------|
|
||||
| String | `"input": "Hello"` |
|
||||
| Array of strings | `"input": ["Hello", "World"]` |
|
||||
| Array of tokens (integers) | `"input": [1234, 5678, 9012]` |
|
||||
| Array of token arrays | `"input": [[1234, 5678], [9012, 3456]]` |
|
||||
|
||||
## Quick start
|
||||
Here's how to route between GPT-J embedding (sagemaker endpoint), Amazon Titan embedding (Bedrock) and Azure OpenAI embedding on the proxy server:
|
||||
|
|
|
|||
|
|
@ -25,7 +25,10 @@ View Spend, Token Usage, Key, Team Name for Each Request to LiteLLM
|
|||
|
||||
## Tracking - Request / Response Content in Logs Page
|
||||
|
||||
If you want to view request and response content on LiteLLM Logs, you need to opt in with this setting
|
||||
If you want to view request and response content on LiteLLM Logs, you can enable it in either place:
|
||||
|
||||
- **From the UI (no restart):** Use [UI Spend Log Settings](./ui_spend_log_settings.md) — open Logs → Settings → enable "Store Prompts in Spend Logs" → Save. Takes effect immediately and overrides config.
|
||||
- **From config:** Add this to your `proxy_config.yaml` (requires restart):
|
||||
|
||||
```yaml
|
||||
general_settings:
|
||||
|
|
@ -57,7 +60,10 @@ general_settings:
|
|||
|
||||
If you're storing spend logs, it might be a good idea to delete them regularly to keep the database fast.
|
||||
|
||||
LiteLLM lets you configure this in your `proxy_config.yaml`:
|
||||
You can set the retention period in either place:
|
||||
|
||||
- **From the UI (no restart):** [UI Spend Log Settings](./ui_spend_log_settings.md) — Logs → Settings → set Retention Period → Save.
|
||||
- **From config:** Add the following to your `proxy_config.yaml` (requires restart):
|
||||
|
||||
```yaml
|
||||
general_settings:
|
||||
|
|
|
|||
92
docs/my-website/docs/proxy/ui_spend_log_settings.md
Normal file
92
docs/my-website/docs/proxy/ui_spend_log_settings.md
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
import Image from '@theme/IdealImage';
|
||||
|
||||
# UI Spend Log Settings
|
||||
|
||||
Configure spend log behavior directly from the Admin UI—no config file edits or proxy restart required. This is especially useful for cloud deployments where updating the config is difficult or requires a long release process.
|
||||
|
||||
## Overview
|
||||
|
||||
Previously, spend log options (such as storing request/response content and retention period) had to be set in `proxy_config.yaml` under `general_settings`. Changing them required editing the config and restarting the proxy, which was a pain point for users-especially in cloud environments—who don't have easy access to the config or whose deployment process makes config updates slow.
|
||||
|
||||
<Image img={require('../../img/ui_spend_logs_settings.png')} />
|
||||
|
||||
**UI Spend Log Settings** lets you:
|
||||
|
||||
- **Store prompts in spend logs** – Enable or disable storing request and response content in the spend logs table (only affects logs created after you change the setting)
|
||||
- **Set retention period** – Configure how long spend logs are kept before automatic cleanup (e.g. `7d`, `30d`)
|
||||
- **Apply changes immediately** – No proxy restart needed; settings take effect for new requests as soon as you save
|
||||
|
||||
:::warning UI overrides config
|
||||
Settings changed in the UI **override** the values in your config file. For example, if `store_prompts_in_spend_logs` is explicitly set to `false` in `general_settings`, turning it on in the UI will still enable storing prompts. Use the UI when you want runtime control without redeploying.
|
||||
:::
|
||||
|
||||
## Settings You Can Configure
|
||||
|
||||
| Setting | Description |
|
||||
| ------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| **Store Prompts in Spend Logs** | When enabled, request messages and response content are stored for **new** spend logs so you can view them in the Logs UI. Logs created before you enabled this will not have request/response content. When disabled, only metadata (e.g. tokens, cost, model) is stored for new logs. |
|
||||
| **Retention Period** | Maximum time to keep spend logs before they are automatically deleted (e.g. `7d`, `30d`). Optional; if not set, logs are retained according to your config or default behavior. |
|
||||
|
||||
The same options can be set in config via [general_settings](./config_settings.md#general_settings---reference) (`store_prompts_in_spend_logs`, `maximum_spend_logs_retention_period`). Values set in the UI take precedence.
|
||||
|
||||
## How to Configure Spend Log Settings in the UI
|
||||
|
||||
### 1. Open the Logs page
|
||||
|
||||
Navigate to the Admin UI (e.g. `http://localhost:4000/ui` or your `PROXY_BASE_URL/ui`) and click **Logs**.
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
### 2. Open Logs settings
|
||||
|
||||
Click the **Settings** (gear) icon on the Logs page to open the spend log settings panel.
|
||||
|
||||
|
||||
|
||||
### 3. Enable Store Prompts in Spend Logs (optional)
|
||||
|
||||
Turn on **Store Prompts in Spend Logs** if you want request and response content to be stored for new requests and visible when you open those log entries. This only affects logs created after you enable it; existing logs will not gain request/response content. Leave it off if you only need metadata (tokens, cost, model, etc.).
|
||||
|
||||
|
||||
|
||||
### 4. Set the retention period (optional)
|
||||
|
||||
Optionally set the **Retention Period** (e.g. `7d`, `30d`) to control how long spend logs are kept before automatic cleanup. Uses the same format as the config option `maximum_spend_logs_retention_period`.
|
||||
|
||||
|
||||
|
||||
### 5. Save settings
|
||||
|
||||
Click **Save Settings**. Changes take effect immediately for new requests; no proxy restart is required. Existing logs are not updated.
|
||||
|
||||
|
||||
|
||||
### 6. Verify: view request and response in a log
|
||||
|
||||
After enabling **Store Prompts in Spend Logs**, make a new request through the proxy, then open that log entry (or any other log created after you enabled the setting). The log details view will include the request and response content. Logs that existed before you turned the setting on will not have this content.
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
## Use Cases
|
||||
|
||||
### Cloud and managed deployments
|
||||
|
||||
When the proxy runs in a managed or cloud environment, config may be in a separate repo, require a long release, or be controlled by another team. Using the UI lets you change spend log behavior (e.g. enable prompt storage for debugging or set retention) without going through that process.
|
||||
|
||||
### Quick toggles for debugging
|
||||
|
||||
Temporarily enable **Store Prompts in Spend Logs** to inspect request/response content on new requests when debugging, then turn it off again from the UI without editing config or restarting. Only logs created while the setting was on will contain the content.
|
||||
|
||||
### Retention without redeploying
|
||||
|
||||
Adjust how long spend logs are retained (e.g. shorten to reduce storage or extend for compliance) and have the new retention period and cleanup job take effect immediately.
|
||||
|
||||
## Related Documentation
|
||||
|
||||
- [Getting Started with UI Logs](./ui_logs.md) – Overview of what gets logged and config-based options
|
||||
- [Config Settings](./config_settings.md) – `store_prompts_in_spend_logs`, `disable_spend_logs`, `maximum_spend_logs_retention_period` in `general_settings`
|
||||
- [Spend Logs Deletion](./spend_logs_deletion.md) – How retention and cleanup work
|
||||
|
|
@ -2,7 +2,7 @@ import Image from '@theme/IdealImage';
|
|||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
# Claude Code Plugin Marketplace
|
||||
# Claude Code Plugin Marketplace (Managed Skills)
|
||||
|
||||
LiteLLM AI Gateway acts as a central registry for Claude Code plugins. Admins can govern which plugins are available across the organization, and engineers can discover and install approved plugins from a single source.
|
||||
|
||||
|
|
@ -252,7 +252,7 @@ curl -X POST http://localhost:4000/claude-code/plugins \
|
|||
}'
|
||||
```
|
||||
|
||||
### 3. Share with Your Team
|
||||
### 3. Use in Claude Code
|
||||
|
||||
Send engineers the marketplace URL:
|
||||
|
||||
|
|
|
|||
BIN
docs/my-website/img/ui_spend_logs_settings.png
Normal file
BIN
docs/my-website/img/ui_spend_logs_settings.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 343 KiB |
384
docs/my-website/release_notes/v1.81.6.md
Normal file
384
docs/my-website/release_notes/v1.81.6.md
Normal file
|
|
@ -0,0 +1,384 @@
|
|||
---
|
||||
title: "v1.81.6 - Logs v2 with Tool Call Tracing"
|
||||
slug: "v1-81-6"
|
||||
date: 2026-01-31T00:00:00
|
||||
authors:
|
||||
- name: Krrish Dholakia
|
||||
title: CEO, LiteLLM
|
||||
url: https://www.linkedin.com/in/krish-d/
|
||||
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
|
||||
- name: Ishaan Jaff
|
||||
title: CTO, LiteLLM
|
||||
url: https://www.linkedin.com/in/reffajnaahsi/
|
||||
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
|
||||
hide_table_of_contents: false
|
||||
---
|
||||
|
||||
## Deploy this version
|
||||
|
||||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
import Image from '@theme/IdealImage';
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="docker" label="Docker">
|
||||
|
||||
```bash
|
||||
docker run \
|
||||
-e STORE_MODEL_IN_DB=True \
|
||||
-p 4000:4000 \
|
||||
docker.litellm.ai/berriai/litellm:main-v1.81.6
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="pip" label="Pip">
|
||||
|
||||
```bash
|
||||
pip install litellm==1.81.6
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
## Key Highlights
|
||||
|
||||
Logs View v2 with Tool Call Tracing - Redesigned logs interface with side panel, structured tool visualization, and error message search for faster debugging.
|
||||
|
||||
Let's dive in.
|
||||
|
||||
### Logs View v2 with Tool Call Tracing
|
||||
|
||||
This release introduces comprehensive tool call tracing through LiteLLM's redesigned Logs View v2, enabling developers to debug and monitor AI agent workflows in production environments seamlessly.
|
||||
|
||||
This means you can now onboard use cases like tracing complex multi-step agent interactions, debugging tool execution failures, and monitoring MCP server calls while maintaining full visibility into request/response payloads with syntax highlighting.
|
||||
|
||||
Developers can access the new Logs View through LiteLLM's UI to inspect tool calls in structured format, search logs by error messages or request patterns, and correlate agent activities across sessions with collapsible side panel views.
|
||||
|
||||
{/* TODO: Add image from Slack (group_7219.png) - save as logs_v2_tool_tracing.png */}
|
||||
{/* <Image img={require('../../img/release_notes/logs_v2_tool_tracing.png')} style={{ maxWidth: '800px', width: '100%' }} /> */}
|
||||
|
||||
[Get Started](../../docs/proxy/ui_logs)
|
||||
|
||||
## New Models / Updated Models
|
||||
|
||||
#### New Model Support
|
||||
|
||||
| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features |
|
||||
| -------- | ----- | -------------- | ------------------- | -------------------- | -------- |
|
||||
| AWS Bedrock | `amazon.nova-2-pro-preview-20251202-v1:0` | 1M | $2.19 | $17.50 | Chat completions, vision, video, PDF, function calling, prompt caching, reasoning |
|
||||
| Google Vertex AI | `gemini-robotics-er-1.5-preview` | 1M | $0.30 | $2.50 | Chat completions, multimodal (text, image, video, audio), function calling, reasoning |
|
||||
| OpenRouter | `openrouter/xiaomi/mimo-v2-flash` | 262K | $0.09 | $0.29 | Chat completions, function calling, reasoning |
|
||||
| OpenRouter | `openrouter/moonshotai/kimi-k2.5` | - | - | - | Chat completions |
|
||||
| OpenRouter | `openrouter/z-ai/glm-4.7` | 202K | $0.40 | $1.50 | Chat completions, vision, function calling, reasoning |
|
||||
|
||||
#### Features
|
||||
|
||||
- **[AWS Bedrock](../../docs/providers/bedrock)**
|
||||
- Messages API Bedrock Converse caching and PDF support - [PR #19785](https://github.com/BerriAI/litellm/pull/19785)
|
||||
- Translate advanced-tool-use to Bedrock-specific headers for Claude Opus 4.5 - [PR #19841](https://github.com/BerriAI/litellm/pull/19841)
|
||||
- Support tool search header translation for Sonnet 4.5 - [PR #19871](https://github.com/BerriAI/litellm/pull/19871)
|
||||
- Filter unsupported beta headers for AWS Bedrock Invoke API - [PR #19877](https://github.com/BerriAI/litellm/pull/19877)
|
||||
- Nova grounding improvements - [PR #19598](https://github.com/BerriAI/litellm/pull/19598), [PR #20159](https://github.com/BerriAI/litellm/pull/20159)
|
||||
|
||||
- **[Anthropic](../../docs/providers/anthropic)**
|
||||
- Remove explicit cache_control null in tool_result content - [PR #19919](https://github.com/BerriAI/litellm/pull/19919)
|
||||
- Fix tool handling - [PR #19805](https://github.com/BerriAI/litellm/pull/19805)
|
||||
|
||||
- **[Google Gemini / Vertex AI](../../docs/providers/gemini)**
|
||||
- Add Gemini Robotics-ER 1.5 preview support - [PR #19845](https://github.com/BerriAI/litellm/pull/19845)
|
||||
- Support file retrieval in GoogleAIStudioFilesHandle - [PR #20018](https://github.com/BerriAI/litellm/pull/20018)
|
||||
- Add /delete endpoint support - [PR #20055](https://github.com/BerriAI/litellm/pull/20055)
|
||||
- Add custom_llm_provider as gemini translation - [PR #19988](https://github.com/BerriAI/litellm/pull/19988)
|
||||
- Subtract implicit cached tokens from text_tokens for correct cost calculation - [PR #19775](https://github.com/BerriAI/litellm/pull/19775)
|
||||
- Remove unsupported prompt-caching-scope-2026-01-05 header for vertex ai - [PR #20058](https://github.com/BerriAI/litellm/pull/20058)
|
||||
- Add disable flag for anthropic gemini cache translation - [PR #20052](https://github.com/BerriAI/litellm/pull/20052)
|
||||
- Convert image URLs to base64 in tool messages for Anthropic on Vertex AI - [PR #19896](https://github.com/BerriAI/litellm/pull/19896)
|
||||
|
||||
- **[xAI](../../docs/providers/xai)**
|
||||
- Add grok reasoning content support - [PR #19850](https://github.com/BerriAI/litellm/pull/19850)
|
||||
- Add websearch params support for Responses API - [PR #19915](https://github.com/BerriAI/litellm/pull/19915)
|
||||
- Add routing of xai chat completions to responses when web search options is present - [PR #20051](https://github.com/BerriAI/litellm/pull/20051)
|
||||
- Correct cached token cost calculation - [PR #19772](https://github.com/BerriAI/litellm/pull/19772)
|
||||
|
||||
- **[Azure OpenAI](../../docs/providers/azure)**
|
||||
- Use generic cost calculator for audio token pricing - [PR #19771](https://github.com/BerriAI/litellm/pull/19771)
|
||||
- Allow tool_choice for Azure GPT-5 chat models - [PR #19813](https://github.com/BerriAI/litellm/pull/19813)
|
||||
- Set gpt-5.2-codex mode to responses for Azure and OpenRouter - [PR #19770](https://github.com/BerriAI/litellm/pull/19770)
|
||||
|
||||
- **[OpenAI](../../docs/providers/openai)**
|
||||
- Fix max_input_tokens for gpt-5.2-codex - [PR #20009](https://github.com/BerriAI/litellm/pull/20009)
|
||||
- Fix gpt-image-1.5 cost calculation not including output image tokens - [PR #19515](https://github.com/BerriAI/litellm/pull/19515)
|
||||
|
||||
- **[Hosted VLLM](../../docs/providers/vllm)**
|
||||
- Support thinking parameter in anthropic_messages() and .completion() - [PR #19787](https://github.com/BerriAI/litellm/pull/19787)
|
||||
- Route through base_llm_http_handler to support ssl_verify - [PR #19893](https://github.com/BerriAI/litellm/pull/19893)
|
||||
- Fix vllm embedding format - [PR #20056](https://github.com/BerriAI/litellm/pull/20056)
|
||||
|
||||
- **[OCI GenAI](../../docs/providers/oci)**
|
||||
- Serialize imageUrl as object for OCI GenAI API - [PR #19661](https://github.com/BerriAI/litellm/pull/19661)
|
||||
|
||||
- **[Volcengine](../../docs/providers/volcano)**
|
||||
- Add context for volcengine models (deepseek-v3-2, glm-4-7, kimi-k2-thinking) - [PR #19335](https://github.com/BerriAI/litellm/pull/19335)
|
||||
|
||||
- **[Chinese Providers](../../docs/providers/)**
|
||||
- Add prompt caching and reasoning support for MiniMax, GLM, Xiaomi - [PR #19924](https://github.com/BerriAI/litellm/pull/19924)
|
||||
|
||||
- **[Vercel AI Gateway](../../docs/providers/vercel_ai_gateway)**
|
||||
- Add embeddings support - [PR #19660](https://github.com/BerriAI/litellm/pull/19660)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- **[Google](../../docs/providers/gemini)**
|
||||
- Fix gemini-robotics-er-1.5-preview entry - [PR #19974](https://github.com/BerriAI/litellm/pull/19974)
|
||||
|
||||
- **General**
|
||||
- Fix output_tokens_details.reasoning_tokens None - [PR #19914](https://github.com/BerriAI/litellm/pull/19914)
|
||||
- Fix stream_chunk_builder to preserve images from streaming chunks - [PR #19654](https://github.com/BerriAI/litellm/pull/19654)
|
||||
- Fix aspectRatio mapping in image edit - [PR #20053](https://github.com/BerriAI/litellm/pull/20053)
|
||||
- Handle unknown models in Azure AI cost calculator - [PR #20150](https://github.com/BerriAI/litellm/pull/20150)
|
||||
|
||||
- **[GigaChat](../../docs/providers/gigachat)**
|
||||
- Ensure function content is valid JSON - [PR #19232](https://github.com/BerriAI/litellm/pull/19232)
|
||||
|
||||
## LLM API Endpoints
|
||||
|
||||
#### Features
|
||||
|
||||
- **[Messages API (/messages)](../../docs/mcp)**
|
||||
- Add LiteLLM x Claude Agent SDK Integration - [PR #20035](https://github.com/BerriAI/litellm/pull/20035)
|
||||
|
||||
- **[A2A / MCP Gateway API (/a2a, /mcp)](../../docs/mcp)**
|
||||
- Add A2A agent header-based context propagation support - [PR #19504](https://github.com/BerriAI/litellm/pull/19504)
|
||||
- Enable progress notifications for MCP tool calls - [PR #19809](https://github.com/BerriAI/litellm/pull/19809)
|
||||
- Fix support for non-standard MCP URL patterns - [PR #19738](https://github.com/BerriAI/litellm/pull/19738)
|
||||
- Add backward compatibility for legacy A2A card formats (/.well-known/agent.json) - [PR #19949](https://github.com/BerriAI/litellm/pull/19949)
|
||||
- Add support for agent parameter in /interactions endpoint - [PR #19866](https://github.com/BerriAI/litellm/pull/19866)
|
||||
|
||||
- **[Responses API (/responses)](../../docs/response_api)**
|
||||
- Fix custom_llm_provider for provider-specific params - [PR #19798](https://github.com/BerriAI/litellm/pull/19798)
|
||||
- Extract input tokens details as dict in ResponseAPILoggingUtils - [PR #20046](https://github.com/BerriAI/litellm/pull/20046)
|
||||
|
||||
- **[Batch API (/batches)](../../docs/batches)**
|
||||
- Fix /batches to return encoded ids (from managed objects table) - [PR #19040](https://github.com/BerriAI/litellm/pull/19040)
|
||||
- Fix Batch and File user level permissions - [PR #19981](https://github.com/BerriAI/litellm/pull/19981)
|
||||
- Add cost tracking and usage object in retrieve_batch call type - [PR #19986](https://github.com/BerriAI/litellm/pull/19986)
|
||||
|
||||
- **[Embeddings API (/embeddings)](../../docs/embedding/supported_embedding)**
|
||||
- Add supported input formats documentation - [PR #20073](https://github.com/BerriAI/litellm/pull/20073)
|
||||
|
||||
- **[RAG API (/rag/ingest, /vector_store)](../../docs/rag_ingest)**
|
||||
- Add UI for /rag/ingest API - Upload docs, pdfs etc to create vector stores - [PR #19822](https://github.com/BerriAI/litellm/pull/19822)
|
||||
- Add support for using S3 Vectors as Vector Store Provider - [PR #19888](https://github.com/BerriAI/litellm/pull/19888)
|
||||
- Add s3_vectors as provider on /vector_store/search API + UI for creating + PDF support - [PR #19895](https://github.com/BerriAI/litellm/pull/19895)
|
||||
- Add permission management for users and teams on Vector Stores - [PR #19972](https://github.com/BerriAI/litellm/pull/19972)
|
||||
- Enable router support for completions in RAG query pipeline - [PR #19550](https://github.com/BerriAI/litellm/pull/19550)
|
||||
|
||||
- **[Search API (/search)](../../docs/search)**
|
||||
- Add /list endpoint to list what search tools exist in router - [PR #19969](https://github.com/BerriAI/litellm/pull/19969)
|
||||
- Fix router search tools v2 integration - [PR #19840](https://github.com/BerriAI/litellm/pull/19840)
|
||||
|
||||
- **[Passthrough Endpoints (/\{provider\}_passthrough)](../../docs/pass_through/intro)**
|
||||
- Add /openai_passthrough route for OpenAI passthrough requests - [PR #19989](https://github.com/BerriAI/litellm/pull/19989)
|
||||
- Add support for configuring role_mappings via environment variables - [PR #19498](https://github.com/BerriAI/litellm/pull/19498)
|
||||
- Add Vertex AI LLM credentials sensitive keyword "vertex_credentials" for masking - [PR #19551](https://github.com/BerriAI/litellm/pull/19551)
|
||||
- Fix prevention of provider-prefixed model name leaks in responses - [PR #19943](https://github.com/BerriAI/litellm/pull/19943)
|
||||
- Fix proxy support for slashes in Google Vertex generateContent model names - [PR #19737](https://github.com/BerriAI/litellm/pull/19737), [PR #19753](https://github.com/BerriAI/litellm/pull/19753)
|
||||
- Support model names with slashes in Vertex AI passthrough URLs - [PR #19944](https://github.com/BerriAI/litellm/pull/19944)
|
||||
- Fix regression in Vertex AI passthroughs for router models - [PR #19967](https://github.com/BerriAI/litellm/pull/19967)
|
||||
- Add regression tests for Vertex AI passthrough model names - [PR #19855](https://github.com/BerriAI/litellm/pull/19855)
|
||||
|
||||
#### Bugs
|
||||
|
||||
- **General**
|
||||
- Fix token calculations and refactor - [PR #19696](https://github.com/BerriAI/litellm/pull/19696)
|
||||
|
||||
## Management Endpoints / UI
|
||||
|
||||
#### Features
|
||||
|
||||
- **Proxy CLI Auth**
|
||||
- Add configurable CLI JWT expiration via environment variable - [PR #19780](https://github.com/BerriAI/litellm/pull/19780)
|
||||
- Fix team cli auth flow - [PR #19666](https://github.com/BerriAI/litellm/pull/19666)
|
||||
|
||||
- **Virtual Keys**
|
||||
- UI: Auto Truncation of Table Values - [PR #19718](https://github.com/BerriAI/litellm/pull/19718)
|
||||
- Fix Create Key: Expire Key Input Duration - [PR #19807](https://github.com/BerriAI/litellm/pull/19807)
|
||||
- Bulk Update Keys Endpoint - [PR #19886](https://github.com/BerriAI/litellm/pull/19886)
|
||||
|
||||
- **Logs View**
|
||||
- **v2 Logs view with side panel and improved UX** - [PR #20091](https://github.com/BerriAI/litellm/pull/20091)
|
||||
- New View to render "Tools" on Logs View - [PR #20093](https://github.com/BerriAI/litellm/pull/20093)
|
||||
- Add Pretty print view of request/response - [PR #20096](https://github.com/BerriAI/litellm/pull/20096)
|
||||
- Add error_message search in Spend Logs Endpoint - [PR #19960](https://github.com/BerriAI/litellm/pull/19960)
|
||||
- UI: Adding Error message search to ui spend logs - [PR #19963](https://github.com/BerriAI/litellm/pull/19963)
|
||||
- Spend Logs: Settings Modal - [PR #19918](https://github.com/BerriAI/litellm/pull/19918)
|
||||
- Fix error_code in Spend Logs metadata - [PR #20015](https://github.com/BerriAI/litellm/pull/20015)
|
||||
- Spend Logs: Show Current Store and Retention Status - [PR #20017](https://github.com/BerriAI/litellm/pull/20017)
|
||||
- Allow Dynamic Setting of store_prompts_in_spend_logs - [PR #19913](https://github.com/BerriAI/litellm/pull/19913)
|
||||
- [Docs: UI Spend Logs Settings](../../docs/proxy/ui_spend_log_settings) - [PR #20197](https://github.com/BerriAI/litellm/pull/20197)
|
||||
|
||||
- **Models + Endpoints**
|
||||
- Add sortBy and sortOrder params for /v2/model/info - [PR #19903](https://github.com/BerriAI/litellm/pull/19903)
|
||||
- Fix Sorting for /v2/model/info - [PR #19971](https://github.com/BerriAI/litellm/pull/19971)
|
||||
- UI: Model Page Server Sort - [PR #19908](https://github.com/BerriAI/litellm/pull/19908)
|
||||
|
||||
- **Usage & Analytics**
|
||||
- UI: Usage Export: Breakdown by Teams and Keys - [PR #19953](https://github.com/BerriAI/litellm/pull/19953)
|
||||
- UI: Usage: Model Breakdown Per Key - [PR #20039](https://github.com/BerriAI/litellm/pull/20039)
|
||||
|
||||
- **UI Improvements**
|
||||
- UI: Allow Admins to control what pages are visible on LeftNav - [PR #19907](https://github.com/BerriAI/litellm/pull/19907)
|
||||
- UI: Add Light/Dark Mode Switch for Development - [PR #19804](https://github.com/BerriAI/litellm/pull/19804)
|
||||
- UI: Dark Mode: Delete Resource Modal - [PR #20098](https://github.com/BerriAI/litellm/pull/20098)
|
||||
- UI: Tables: Reusable Table Sort Component - [PR #19970](https://github.com/BerriAI/litellm/pull/19970)
|
||||
- UI: New Badge Dot Render - [PR #20024](https://github.com/BerriAI/litellm/pull/20024)
|
||||
- UI: Feedback Prompts: Option To Hide Prompts - [PR #19831](https://github.com/BerriAI/litellm/pull/19831)
|
||||
- UI: Navbar: Fixed Default Logo + Bound Logo Box - [PR #20092](https://github.com/BerriAI/litellm/pull/20092)
|
||||
- UI: Navbar: User Dropdown - [PR #20095](https://github.com/BerriAI/litellm/pull/20095)
|
||||
- Change default key type from 'Default' to 'LLM API' - [PR #19516](https://github.com/BerriAI/litellm/pull/19516)
|
||||
|
||||
- **Team & User Management**
|
||||
- Fix /team/member_add User Email and ID Verifications - [PR #19814](https://github.com/BerriAI/litellm/pull/19814)
|
||||
- Fix SSO Email Case Sensitivity - [PR #19799](https://github.com/BerriAI/litellm/pull/19799)
|
||||
- UI: Internal User: Bulk Add - [PR #19721](https://github.com/BerriAI/litellm/pull/19721)
|
||||
|
||||
- **AI Gateway Features**
|
||||
- Add support for making silent LLM calls without logging - [PR #19544](https://github.com/BerriAI/litellm/pull/19544)
|
||||
- UI: Fix MCP tools instructions to display comma-separated strings - [PR #20101](https://github.com/BerriAI/litellm/pull/20101)
|
||||
|
||||
#### Bugs
|
||||
|
||||
- Fix Model Name During Fallback - [PR #20177](https://github.com/BerriAI/litellm/pull/20177)
|
||||
- Fix Health Endpoints when Callback Objects Defined - [PR #20182](https://github.com/BerriAI/litellm/pull/20182)
|
||||
- Fix Unable to reset user max budget to unlimited - [PR #19796](https://github.com/BerriAI/litellm/pull/19796)
|
||||
- Fix Password comparison with non-ASCII characters - [PR #19568](https://github.com/BerriAI/litellm/pull/19568)
|
||||
- Correct error message for DISABLE_ADMIN_ENDPOINTS - [PR #19861](https://github.com/BerriAI/litellm/pull/19861)
|
||||
- Prevent clearing content filter patterns when editing guardrail - [PR #19671](https://github.com/BerriAI/litellm/pull/19671)
|
||||
- Fix Prompt Studio history to load tools and system messages - [PR #19920](https://github.com/BerriAI/litellm/pull/19920)
|
||||
- Add WATSONX_ZENAPIKEY to WatsonX credentials - [PR #20086](https://github.com/BerriAI/litellm/pull/20086)
|
||||
- UI: Vector Store: Allow Config Defined Models to Be Selected - [PR #20031](https://github.com/BerriAI/litellm/pull/20031)
|
||||
|
||||
## Logging / Guardrail / Prompt Management Integrations
|
||||
|
||||
#### Features
|
||||
|
||||
- **[DataDog](../../docs/proxy/logging#datadog)**
|
||||
- Add agent support for LLM Observability - [PR #19574](https://github.com/BerriAI/litellm/pull/19574)
|
||||
- Add datadog cost management support and fix startup callback issue - [PR #19584](https://github.com/BerriAI/litellm/pull/19584)
|
||||
- Add datadog_llm_observability to /health/services allowed list - [PR #19952](https://github.com/BerriAI/litellm/pull/19952)
|
||||
- Check for agent mode before requiring DD_API_KEY/DD_SITE - [PR #20156](https://github.com/BerriAI/litellm/pull/20156)
|
||||
|
||||
- **[OpenTelemetry](../../docs/observability/opentelemetry_integration)**
|
||||
- Propagate JWT auth metadata to OTEL spans - [PR #19627](https://github.com/BerriAI/litellm/pull/19627)
|
||||
- Fix thread leak in dynamic header path - [PR #19946](https://github.com/BerriAI/litellm/pull/19946)
|
||||
|
||||
- **[Prometheus](../../docs/proxy/logging#prometheus)**
|
||||
- Add callbacks and labels - [PR #19708](https://github.com/BerriAI/litellm/pull/19708)
|
||||
- Add clientip and user agent in metrics - [PR #19717](https://github.com/BerriAI/litellm/pull/19717)
|
||||
- Add tpm-rpm limit metrics - [PR #19725](https://github.com/BerriAI/litellm/pull/19725)
|
||||
- Add model_id label to metrics - [PR #19678](https://github.com/BerriAI/litellm/pull/19678)
|
||||
- Safely handle None metadata in logging - [PR #19691](https://github.com/BerriAI/litellm/pull/19691)
|
||||
- Resolve high CPU when router_settings in DB by avoiding REGISTRY.collect() - [PR #20087](https://github.com/BerriAI/litellm/pull/20087)
|
||||
|
||||
- **[Langfuse](../../docs/proxy/logging#langfuse)**
|
||||
- Add litellm_callback_logging_failures_metric for Langfuse, Langfuse Otel and other Otel providers - [PR #19636](https://github.com/BerriAI/litellm/pull/19636)
|
||||
|
||||
- **General Logging**
|
||||
- Use return value from CustomLogger.async_post_call_success_hook - [PR #19670](https://github.com/BerriAI/litellm/pull/19670)
|
||||
- Add async_post_call_response_headers_hook to CustomLogger - [PR #20083](https://github.com/BerriAI/litellm/pull/20083)
|
||||
- Add mock client factory pattern and mock support for PostHog, Helicone, and Braintrust integrations - [PR #19707](https://github.com/BerriAI/litellm/pull/19707)
|
||||
|
||||
#### Guardrails
|
||||
|
||||
- **[Presidio](../../docs/proxy/guardrails/pii_masking_v2)**
|
||||
- Reuse HTTP connections to prevent performance degradation - [PR #19964](https://github.com/BerriAI/litellm/pull/19964)
|
||||
|
||||
- **Onyx**
|
||||
- Add timeout to onyx guardrail - [PR #19731](https://github.com/BerriAI/litellm/pull/19731)
|
||||
|
||||
- **General**
|
||||
- Add guardrail model argument feature - [PR #19619](https://github.com/BerriAI/litellm/pull/19619)
|
||||
- Fix guardrails issues with streaming-response regex - [PR #19901](https://github.com/BerriAI/litellm/pull/19901)
|
||||
- Remove enterprise requirement for guardrail monitoring (docs) - [PR #19833](https://github.com/BerriAI/litellm/pull/19833)
|
||||
|
||||
## Spend Tracking, Budgets and Rate Limiting
|
||||
|
||||
- Add event-driven coordination for global spend query to prevent cache stampede - [PR #20030](https://github.com/BerriAI/litellm/pull/20030)
|
||||
|
||||
## Performance / Loadbalancing / Reliability improvements
|
||||
|
||||
- **Resolve high CPU when router_settings in DB** - by avoiding REGISTRY.collect() in PrometheusServicesLogger - [PR #20087](https://github.com/BerriAI/litellm/pull/20087)
|
||||
- **Reuse HTTP connections in Presidio** - to prevent performance degradation - [PR #19964](https://github.com/BerriAI/litellm/pull/19964)
|
||||
- **Event-driven coordination for global spend query** - prevent cache stampede - [PR #20030](https://github.com/BerriAI/litellm/pull/20030)
|
||||
- Fix recursive Pydantic validation issue - [PR #19531](https://github.com/BerriAI/litellm/pull/19531)
|
||||
- Refactor argument handling into helper function to reduce code bloat - [PR #19720](https://github.com/BerriAI/litellm/pull/19720)
|
||||
- Optimize logo fetching and resolve MCP import blockers - [PR #19719](https://github.com/BerriAI/litellm/pull/19719)
|
||||
- Improve logo download performance using async HTTP client - [PR #20155](https://github.com/BerriAI/litellm/pull/20155)
|
||||
- Fix server root path configuration - [PR #19790](https://github.com/BerriAI/litellm/pull/19790)
|
||||
- Refactor: Extract transport context creation into separate method - [PR #19794](https://github.com/BerriAI/litellm/pull/19794)
|
||||
- Add native_background_mode configuration to override polling_via_cache for specific models - [PR #19899](https://github.com/BerriAI/litellm/pull/19899)
|
||||
- Initialize tiktoken environment at import time to enable offline usage - [PR #19882](https://github.com/BerriAI/litellm/pull/19882)
|
||||
- Improve tiktoken performance using local cache in lazy loading - [PR #19774](https://github.com/BerriAI/litellm/pull/19774)
|
||||
- Fix timeout errors in chat completion calls to be correctly reported in failure callbacks - [PR #19842](https://github.com/BerriAI/litellm/pull/19842)
|
||||
- Fix environment variable type handling for NUM_RETRIES - [PR #19507](https://github.com/BerriAI/litellm/pull/19507)
|
||||
- Use safe_deep_copy in silent experiment kwargs to prevent mutation - [PR #20170](https://github.com/BerriAI/litellm/pull/20170)
|
||||
- Improve error handling by inspecting BadRequestError after all other policy types - [PR #19878](https://github.com/BerriAI/litellm/pull/19878)
|
||||
|
||||
## Database Changes
|
||||
|
||||
### Schema Updates
|
||||
|
||||
| Table | Change Type | Description | PR | Migration |
|
||||
| ----- | ----------- | ----------- | -- | --------- |
|
||||
| `LiteLLM_ManagedVectorStoresTable` | New Columns | Added `team_id` and `user_id` fields for permission management | [PR #19972](https://github.com/BerriAI/litellm/pull/19972) | [Migration](https://github.com/BerriAI/litellm/blob/main/litellm-proxy-extras/litellm_proxy_extras/migrations/20260131150814_add_team_user_to_vector_stores/migration.sql) |
|
||||
|
||||
### Migration Improvements
|
||||
|
||||
- Fix Docker: Use correct schema path for Prisma generation - [PR #19631](https://github.com/BerriAI/litellm/pull/19631)
|
||||
- Resolve 'relation does not exist' migration errors in setup_database - [PR #19281](https://github.com/BerriAI/litellm/pull/19281)
|
||||
- Fix migration issue and improve Docker image stability - [PR #19843](https://github.com/BerriAI/litellm/pull/19843)
|
||||
- Run Prisma generate as nobody user in non-root Docker container for security - [PR #20000](https://github.com/BerriAI/litellm/pull/20000)
|
||||
- Bump litellm-proxy-extras version to 0.4.28 - [PR #20166](https://github.com/BerriAI/litellm/pull/20166)
|
||||
|
||||
## Documentation Updates
|
||||
|
||||
- **[Add Claude Agents SDK x LiteLLM Guide](../../docs/mcp)** - [PR #20036](https://github.com/BerriAI/litellm/pull/20036)
|
||||
- **[Add Cookbook: Using Claude Agent SDK + MCPs with LiteLLM](https://github.com/BerriAI/litellm/tree/main/cookbook)** - [PR #20081](https://github.com/BerriAI/litellm/pull/20081)
|
||||
- Fix A2A Python SDK URL in documentation - [PR #19832](https://github.com/BerriAI/litellm/pull/19832)
|
||||
- **[Add Sarvam usage documentation](../../docs/providers/sarvam)** - [PR #19844](https://github.com/BerriAI/litellm/pull/19844)
|
||||
- **[Add supported input formats for embeddings](../../docs/embedding/supported_embedding)** - [PR #20073](https://github.com/BerriAI/litellm/pull/20073)
|
||||
- **[UI Spend Logs Settings Docs](../../docs/proxy/ui_spend_log_settings)** - [PR #20197](https://github.com/BerriAI/litellm/pull/20197)
|
||||
- Add OpenAI Agents SDK to OSS Adopters list in README - [PR #19820](https://github.com/BerriAI/litellm/pull/19820)
|
||||
- Update docs: Remove enterprise requirement for guardrail monitoring - [PR #19833](https://github.com/BerriAI/litellm/pull/19833)
|
||||
- Add missing environment variable documentation - [PR #20138](https://github.com/BerriAI/litellm/pull/20138)
|
||||
- Improve documentation blog index page - [PR #20188](https://github.com/BerriAI/litellm/pull/20188)
|
||||
|
||||
## Infrastructure / Testing Improvements
|
||||
|
||||
- Add test coverage for Router.get_valid_args and improve code coverage reporting - [PR #19797](https://github.com/BerriAI/litellm/pull/19797)
|
||||
- Add validation of model cost map as CI job - [PR #19993](https://github.com/BerriAI/litellm/pull/19993)
|
||||
- Add Realtime API benchmarks - [PR #20074](https://github.com/BerriAI/litellm/pull/20074)
|
||||
- Add Init Containers support in community helm chart - [PR #19816](https://github.com/BerriAI/litellm/pull/19816)
|
||||
- Add libsndfile to main Dockerfile for ARM64 audio processing support - [PR #19776](https://github.com/BerriAI/litellm/pull/19776)
|
||||
|
||||
## New Contributors
|
||||
|
||||
* @ruanjf made their first contribution in https://github.com/BerriAI/litellm/pull/19551
|
||||
* @moh-dev-stack made their first contribution in https://github.com/BerriAI/litellm/pull/19507
|
||||
* @formorter made their first contribution in https://github.com/BerriAI/litellm/pull/19498
|
||||
* @priyam-that made their first contribution in https://github.com/BerriAI/litellm/pull/19516
|
||||
* @marcosgriselli made their first contribution in https://github.com/BerriAI/litellm/pull/19550
|
||||
* @natimofeev made their first contribution in https://github.com/BerriAI/litellm/pull/19232
|
||||
* @zifeo made their first contribution in https://github.com/BerriAI/litellm/pull/19805
|
||||
* @pragyasardana made their first contribution in https://github.com/BerriAI/litellm/pull/19816
|
||||
* @ryewilson made their first contribution in https://github.com/BerriAI/litellm/pull/19833
|
||||
* @lizhen921 made their first contribution in https://github.com/BerriAI/litellm/pull/19919
|
||||
* @boarder7395 made their first contribution in https://github.com/BerriAI/litellm/pull/19666
|
||||
* @rushilchugh01 made their first contribution in https://github.com/BerriAI/litellm/pull/19938
|
||||
* @cfchase made their first contribution in https://github.com/BerriAI/litellm/pull/19893
|
||||
* @ayim made their first contribution in https://github.com/BerriAI/litellm/pull/19872
|
||||
* @varunsripad123 made their first contribution in https://github.com/BerriAI/litellm/pull/20018
|
||||
* @nht1206 made their first contribution in https://github.com/BerriAI/litellm/pull/20046
|
||||
* @genga6 made their first contribution in https://github.com/BerriAI/litellm/pull/20009
|
||||
|
||||
**Full Changelog**: https://github.com/BerriAI/litellm/compare/v1.81.3.rc...v1.81.6
|
||||
|
|
@ -314,6 +314,7 @@ const sidebars = {
|
|||
label: "UI Logs",
|
||||
items: [
|
||||
"proxy/ui_logs",
|
||||
"proxy/ui_spend_log_settings",
|
||||
"proxy/ui_logs_sessions",
|
||||
"proxy/deleted_keys_teams"
|
||||
]
|
||||
|
|
|
|||
123
docs/my-website/src/theme/BlogListPage/index.js
Normal file
123
docs/my-website/src/theme/BlogListPage/index.js
Normal file
|
|
@ -0,0 +1,123 @@
|
|||
import React from 'react';
|
||||
import Layout from '@theme/Layout';
|
||||
import Link from '@docusaurus/Link';
|
||||
import styles from './styles.module.css';
|
||||
|
||||
const TAG_COLORS = {
|
||||
gemini: {bg: '#d2e3fc', text: '#174ea6', darkBg: '#1a3a5c', darkText: '#8ab4f8'},
|
||||
anthropic: {bg: '#fde0c4', text: '#b33d00', darkBg: '#4a2800', darkText: '#ffb74d'},
|
||||
claude: {bg: '#fde0c4', text: '#b33d00', darkBg: '#4a2800', darkText: '#ffb74d'},
|
||||
llms: {bg: '#c8e6c9', text: '#1b5e20', darkBg: '#1b3d1f', darkText: '#81c784'},
|
||||
};
|
||||
|
||||
function hashHue(str) {
|
||||
let hash = 0;
|
||||
for (let i = 0; i < str.length; i++) {
|
||||
hash = str.charCodeAt(i) + ((hash << 5) - hash);
|
||||
}
|
||||
return Math.abs(hash) % 360;
|
||||
}
|
||||
|
||||
function getTagColor(label) {
|
||||
const key = label.toLowerCase();
|
||||
for (const [k, v] of Object.entries(TAG_COLORS)) {
|
||||
if (key === k) return v;
|
||||
}
|
||||
const hue = hashHue(key);
|
||||
return {
|
||||
bg: `hsl(${hue}, 40%, 90%)`,
|
||||
text: `hsl(${hue}, 60%, 25%)`,
|
||||
darkBg: `hsl(${hue}, 40%, 20%)`,
|
||||
darkText: `hsl(${hue}, 50%, 75%)`,
|
||||
};
|
||||
}
|
||||
|
||||
function formatDate(dateStr) {
|
||||
const d = new Date(dateStr);
|
||||
const now = new Date();
|
||||
const diffDays = Math.floor((now - d) / (1000 * 60 * 60 * 24));
|
||||
if (diffDays <= 0) return 'Today';
|
||||
if (diffDays === 1) return '1d ago';
|
||||
if (diffDays < 30) return `${diffDays}d ago`;
|
||||
return d.toLocaleDateString('en-US', {month: 'short', day: 'numeric', year: 'numeric'});
|
||||
}
|
||||
|
||||
function BlogCard({post, featured}) {
|
||||
const {title, permalink, date, description, tags} = post;
|
||||
const visibleTags = (tags || []).slice(0, 3);
|
||||
|
||||
return (
|
||||
<Link to={permalink} className={styles.cardLink} aria-label={title}>
|
||||
<article className={featured ? styles.cardFeatured : styles.card}>
|
||||
<div className={styles.meta}>
|
||||
<time className={styles.time} dateTime={date}>{formatDate(date)}</time>
|
||||
{featured && <span className={styles.badge}>Latest</span>}
|
||||
</div>
|
||||
<h2 className={styles.title}>{title}</h2>
|
||||
{description && <p className={styles.desc}>{description}</p>}
|
||||
{visibleTags.length > 0 && (
|
||||
<div className={styles.tags}>
|
||||
{visibleTags.map(tag => {
|
||||
const c = getTagColor(tag.label);
|
||||
return (
|
||||
<span key={tag.label} className={styles.tag} style={{
|
||||
'--tag-bg': c.bg, '--tag-text': c.text,
|
||||
'--tag-bg-dark': c.darkBg, '--tag-text-dark': c.darkText,
|
||||
}}>{tag.label}</span>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
<div className={styles.arrow} aria-hidden="true">
|
||||
<svg width="16" height="16" viewBox="0 0 16 16" fill="none">
|
||||
<path d="M6 3l5 5-5 5" stroke="currentColor" strokeWidth="1.5" strokeLinecap="round" strokeLinejoin="round"/>
|
||||
</svg>
|
||||
</div>
|
||||
</article>
|
||||
</Link>
|
||||
);
|
||||
}
|
||||
|
||||
function Pagination({metadata}) {
|
||||
const {previousPage, nextPage} = metadata;
|
||||
if (!previousPage && !nextPage) return null;
|
||||
return (
|
||||
<nav className={styles.pagination} aria-label="Blog list pagination">
|
||||
{previousPage ? (
|
||||
<Link to={previousPage} className={styles.paginationLink}>← Newer posts</Link>
|
||||
) : <span />}
|
||||
{nextPage ? (
|
||||
<Link to={nextPage} className={styles.paginationLink}>Older posts →</Link>
|
||||
) : <span />}
|
||||
</nav>
|
||||
);
|
||||
}
|
||||
|
||||
export default function BlogListPage(props) {
|
||||
const items = props.items || [];
|
||||
const metadata = props.metadata || {};
|
||||
const [first, ...rest] = items;
|
||||
|
||||
return (
|
||||
<Layout
|
||||
title={metadata.blogTitle || 'Blog'}
|
||||
description={metadata.blogDescription || 'Guides, announcements, and best practices from the LiteLLM team.'}
|
||||
>
|
||||
<header className={styles.hero}>
|
||||
<h1 className={styles.heroTitle}>The LiteLLM Blog</h1>
|
||||
<p className={styles.heroSubtitle}>Guides, announcements, and best practices from the LiteLLM team.</p>
|
||||
</header>
|
||||
|
||||
<main className={styles.grid}>
|
||||
{first && (
|
||||
<BlogCard post={first.content.metadata} featured />
|
||||
)}
|
||||
{rest.map(({content}) => (
|
||||
<BlogCard key={content.metadata.permalink} post={content.metadata} />
|
||||
))}
|
||||
</main>
|
||||
|
||||
<Pagination metadata={metadata} />
|
||||
</Layout>
|
||||
);
|
||||
}
|
||||
163
docs/my-website/src/theme/BlogListPage/styles.module.css
Normal file
163
docs/my-website/src/theme/BlogListPage/styles.module.css
Normal file
|
|
@ -0,0 +1,163 @@
|
|||
.hero {
|
||||
max-width: 960px;
|
||||
margin: 0 auto;
|
||||
padding: 3rem 1.5rem 1rem;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.heroTitle {
|
||||
font-size: 2.25rem;
|
||||
font-weight: 700;
|
||||
margin-bottom: 0.25rem;
|
||||
letter-spacing: -0.02em;
|
||||
}
|
||||
|
||||
.heroSubtitle {
|
||||
color: var(--ifm-color-emphasis-600);
|
||||
font-size: 1.1rem;
|
||||
margin-bottom: 0;
|
||||
}
|
||||
|
||||
.grid {
|
||||
max-width: 960px;
|
||||
margin: 0 auto;
|
||||
padding: 1.5rem;
|
||||
display: grid;
|
||||
gap: 1rem;
|
||||
}
|
||||
|
||||
.cardLink {
|
||||
display: block;
|
||||
text-decoration: none;
|
||||
color: inherit;
|
||||
}
|
||||
|
||||
.card {
|
||||
position: relative;
|
||||
border: 1px solid var(--ifm-color-emphasis-200);
|
||||
border-radius: 12px;
|
||||
padding: 1.5rem;
|
||||
padding-right: 2.5rem;
|
||||
height: 100%;
|
||||
transition: border-color 0.15s, transform 0.15s, background 0.15s;
|
||||
background: var(--ifm-background-surface-color, var(--ifm-background-color));
|
||||
}
|
||||
|
||||
.card:hover {
|
||||
border-color: var(--ifm-color-primary);
|
||||
transform: translateY(-2px);
|
||||
background: var(--ifm-color-emphasis-100);
|
||||
}
|
||||
|
||||
.cardFeatured {
|
||||
composes: card;
|
||||
border-color: var(--ifm-color-primary-lighter);
|
||||
background: var(--ifm-color-emphasis-100);
|
||||
}
|
||||
|
||||
.meta {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
margin-bottom: 0.5rem;
|
||||
}
|
||||
|
||||
.time {
|
||||
font-size: 0.8rem;
|
||||
font-weight: 500;
|
||||
color: var(--ifm-color-emphasis-600);
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.04em;
|
||||
}
|
||||
|
||||
.badge {
|
||||
font-size: 0.65rem;
|
||||
font-weight: 600;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.06em;
|
||||
padding: 2px 8px;
|
||||
border-radius: 99px;
|
||||
background: var(--ifm-color-primary);
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.title {
|
||||
font-size: 1.15rem;
|
||||
font-weight: 600;
|
||||
margin: 0 0 0.4rem;
|
||||
line-height: 1.35;
|
||||
}
|
||||
|
||||
.desc {
|
||||
font-size: 0.88rem;
|
||||
color: var(--ifm-color-emphasis-700);
|
||||
line-height: 1.5;
|
||||
margin: 0 0 0.75rem;
|
||||
}
|
||||
|
||||
.tags {
|
||||
display: flex;
|
||||
gap: 6px;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
.tag {
|
||||
font-size: 0.7rem;
|
||||
font-weight: 500;
|
||||
padding: 2px 10px;
|
||||
border-radius: 99px;
|
||||
background: var(--tag-bg);
|
||||
color: var(--tag-text);
|
||||
}
|
||||
|
||||
:global([data-theme='dark']) .tag {
|
||||
background: var(--tag-bg-dark);
|
||||
color: var(--tag-text-dark);
|
||||
}
|
||||
|
||||
.arrow {
|
||||
position: absolute;
|
||||
right: 1rem;
|
||||
top: 50%;
|
||||
transform: translateY(-50%);
|
||||
color: var(--ifm-color-emphasis-400);
|
||||
transition: color 0.15s, transform 0.15s;
|
||||
}
|
||||
|
||||
.card:hover .arrow {
|
||||
color: var(--ifm-color-primary);
|
||||
transform: translateY(-50%) translateX(3px);
|
||||
}
|
||||
|
||||
.pagination {
|
||||
max-width: 960px;
|
||||
margin: 0 auto;
|
||||
padding: 1rem 1.5rem 3rem;
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
}
|
||||
|
||||
.paginationLink {
|
||||
font-size: 0.9rem;
|
||||
font-weight: 500;
|
||||
color: var(--ifm-color-primary);
|
||||
text-decoration: none;
|
||||
}
|
||||
|
||||
.paginationLink:hover {
|
||||
text-decoration: underline;
|
||||
}
|
||||
|
||||
@media (min-width: 640px) {
|
||||
.grid {
|
||||
grid-template-columns: repeat(2, 1fr);
|
||||
}
|
||||
|
||||
.grid .cardLink:first-child {
|
||||
grid-column: 1 / -1;
|
||||
}
|
||||
|
||||
.grid .cardLink:last-child:nth-child(even) {
|
||||
grid-column: 1 / -1;
|
||||
}
|
||||
}
|
||||
|
|
@ -282,6 +282,8 @@ async def get_vector_store_info(
|
|||
updated_at=vector_store.get("updated_at") or None,
|
||||
litellm_credential_name=vector_store.get("litellm_credential_name"),
|
||||
litellm_params=vector_store.get("litellm_params") or None,
|
||||
team_id=vector_store.get("team_id"),
|
||||
user_id=vector_store.get("user_id"),
|
||||
)
|
||||
return {"vector_store": vector_store_pydantic_obj}
|
||||
|
||||
|
|
|
|||
BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.28-py3-none-any.whl
vendored
Normal file
BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.28-py3-none-any.whl
vendored
Normal file
Binary file not shown.
BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.28.tar.gz
vendored
Normal file
BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.28.tar.gz
vendored
Normal file
Binary file not shown.
BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.29-py3-none-any.whl
vendored
Normal file
BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.29-py3-none-any.whl
vendored
Normal file
Binary file not shown.
BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.29.tar.gz
vendored
Normal file
BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.29.tar.gz
vendored
Normal file
Binary file not shown.
|
|
@ -0,0 +1,10 @@
|
|||
-- AlterTable
|
||||
ALTER TABLE "LiteLLM_ManagedVectorStoresTable" ADD COLUMN "team_id" TEXT,
|
||||
ADD COLUMN "user_id" TEXT;
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "LiteLLM_ManagedVectorStoresTable_team_id_idx" ON "LiteLLM_ManagedVectorStoresTable"("team_id");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX "LiteLLM_ManagedVectorStoresTable_user_id_idx" ON "LiteLLM_ManagedVectorStoresTable"("user_id");
|
||||
|
||||
|
|
@ -5,6 +5,7 @@ datasource client {
|
|||
|
||||
generator client {
|
||||
provider = "prisma-client-py"
|
||||
binaryTargets = ["native", "debian-openssl-1.1.x", "debian-openssl-3.0.x", "linux-musl", "linux-musl-openssl-3.0.x"]
|
||||
}
|
||||
|
||||
// Budget / Rate Limits for an org
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[tool.poetry]
|
||||
name = "litellm-proxy-extras"
|
||||
version = "0.4.27"
|
||||
version = "0.4.29"
|
||||
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
|
||||
authors = ["BerriAI"]
|
||||
readme = "README.md"
|
||||
|
|
@ -22,7 +22,7 @@ requires = ["poetry-core"]
|
|||
build-backend = "poetry.core.masonry.api"
|
||||
|
||||
[tool.commitizen]
|
||||
version = "0.4.27"
|
||||
version = "0.4.29"
|
||||
version_files = [
|
||||
"pyproject.toml:version",
|
||||
"../requirements.txt:litellm-proxy-extras==",
|
||||
|
|
|
|||
|
|
@ -876,7 +876,9 @@ async def acancel_batch(
|
|||
try:
|
||||
loop = asyncio.get_event_loop()
|
||||
kwargs["acancel_batch"] = True
|
||||
model = kwargs.pop("model", None)
|
||||
# Preserve model parameter - only pop from kwargs if it exists there
|
||||
# (to avoid passing it twice), otherwise keep the function parameter value
|
||||
model = kwargs.pop("model", None) or model
|
||||
|
||||
# Use a partial function to pass your keyword arguments
|
||||
func = partial(
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ from typing import (
|
|||
Optional,
|
||||
Tuple,
|
||||
Union,
|
||||
cast
|
||||
cast,
|
||||
)
|
||||
|
||||
from openai.types.responses.tool_param import FunctionToolParam
|
||||
|
|
@ -744,11 +744,18 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
if "tools" not in responses_api_request or responses_api_request["tools"] is None:
|
||||
responses_api_request["tools"] = []
|
||||
|
||||
# Get the tools list with proper type narrowing
|
||||
tools = responses_api_request["tools"]
|
||||
if tools is None:
|
||||
tools = []
|
||||
responses_api_request["tools"] = tools
|
||||
|
||||
web_search_tool: Dict[str, Any] = {"type": "web_search"}
|
||||
if isinstance(web_search_options, dict):
|
||||
web_search_tool.update(web_search_options)
|
||||
|
||||
responses_api_request["tools"].append(web_search_tool)
|
||||
# Cast to Any to match the expected union type for tools list items
|
||||
tools.append(cast(Any, web_search_tool))
|
||||
|
||||
def _transform_response_format_to_text_format(
|
||||
self, response_format: Union[Dict[str, Any], Any]
|
||||
|
|
|
|||
|
|
@ -36,6 +36,9 @@ from litellm.llms.anthropic.cost_calculation import (
|
|||
from litellm.llms.azure.cost_calculation import (
|
||||
cost_per_token as azure_openai_cost_per_token,
|
||||
)
|
||||
from litellm.llms.azure_ai.cost_calculator import (
|
||||
cost_per_token as azure_ai_cost_per_token,
|
||||
)
|
||||
from litellm.llms.base_llm.search.transformation import SearchResponse
|
||||
from litellm.llms.bedrock.cost_calculation import (
|
||||
cost_per_token as bedrock_cost_per_token,
|
||||
|
|
@ -138,6 +141,51 @@ def _cost_per_token_custom_pricing_helper(
|
|||
return None
|
||||
|
||||
|
||||
def _get_additional_costs(
|
||||
model: str,
|
||||
custom_llm_provider: Optional[str],
|
||||
prompt_tokens: int,
|
||||
completion_tokens: int,
|
||||
) -> Optional[dict]:
|
||||
"""
|
||||
Calculate additional costs beyond standard token costs.
|
||||
|
||||
This function delegates to provider-specific config classes to calculate
|
||||
any additional costs like routing fees, infrastructure costs, etc.
|
||||
|
||||
Args:
|
||||
model: The model name
|
||||
custom_llm_provider: The provider name (optional)
|
||||
prompt_tokens: Number of prompt tokens
|
||||
completion_tokens: Number of completion tokens
|
||||
|
||||
Returns:
|
||||
Optional dictionary with cost names and amounts, or None if no additional costs
|
||||
"""
|
||||
if not custom_llm_provider:
|
||||
return None
|
||||
|
||||
try:
|
||||
config_class = None
|
||||
if custom_llm_provider == "azure_ai":
|
||||
from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
|
||||
config_class = AzureFoundryModelInfo.get_azure_ai_config_for_model(model)
|
||||
# Add more providers here as needed
|
||||
# elif custom_llm_provider == "other_provider":
|
||||
# config_class = get_other_provider_config(model)
|
||||
|
||||
if config_class and hasattr(config_class, 'calculate_additional_costs'):
|
||||
return config_class.calculate_additional_costs(
|
||||
model=model,
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_logger.debug(f"Error calculating additional costs: {e}")
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _transcription_usage_has_token_details(
|
||||
usage_block: Optional[Usage],
|
||||
) -> bool:
|
||||
|
|
@ -427,8 +475,8 @@ def cost_per_token( # noqa: PLR0915
|
|||
|
||||
return dashscope_cost_per_token(model=model, usage=usage_block)
|
||||
elif custom_llm_provider == "azure_ai":
|
||||
return generic_cost_per_token(
|
||||
model=model, usage=usage_block, custom_llm_provider=custom_llm_provider
|
||||
return azure_ai_cost_per_token(
|
||||
model=model, usage=usage_block, response_time_ms=response_time_ms
|
||||
)
|
||||
else:
|
||||
model_info = _cached_get_model_info_helper(
|
||||
|
|
@ -805,6 +853,7 @@ def _store_cost_breakdown_in_logging_obj(
|
|||
completion_tokens_cost_usd_dollar: float,
|
||||
cost_for_built_in_tools_cost_usd_dollar: float,
|
||||
total_cost_usd_dollar: float,
|
||||
additional_costs: Optional[dict] = None,
|
||||
original_cost: Optional[float] = None,
|
||||
discount_percent: Optional[float] = None,
|
||||
discount_amount: Optional[float] = None,
|
||||
|
|
@ -821,6 +870,7 @@ def _store_cost_breakdown_in_logging_obj(
|
|||
completion_tokens_cost_usd_dollar: Cost of completion tokens (includes reasoning if applicable)
|
||||
cost_for_built_in_tools_cost_usd_dollar: Cost of built-in tools
|
||||
total_cost_usd_dollar: Total cost of request
|
||||
additional_costs: Free-form additional costs dict (e.g., {"azure_model_router_flat_cost": 0.00014})
|
||||
original_cost: Cost before discount
|
||||
discount_percent: Discount percentage applied (0.05 = 5%)
|
||||
discount_amount: Discount amount in USD
|
||||
|
|
@ -838,6 +888,7 @@ def _store_cost_breakdown_in_logging_obj(
|
|||
output_cost=completion_tokens_cost_usd_dollar,
|
||||
total_cost=total_cost_usd_dollar,
|
||||
cost_for_built_in_tools_cost_usd_dollar=cost_for_built_in_tools_cost_usd_dollar,
|
||||
additional_costs=additional_costs,
|
||||
original_cost=original_cost,
|
||||
discount_percent=discount_percent,
|
||||
discount_amount=discount_amount,
|
||||
|
|
@ -1335,6 +1386,15 @@ def completion_cost( # noqa: PLR0915
|
|||
service_tier=service_tier,
|
||||
response=completion_response,
|
||||
)
|
||||
|
||||
# Get additional costs from provider (e.g., routing fees, infrastructure costs)
|
||||
additional_costs = _get_additional_costs(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
)
|
||||
|
||||
_final_cost = (
|
||||
prompt_tokens_cost_usd_dollar + completion_tokens_cost_usd_dollar
|
||||
)
|
||||
|
|
@ -1374,6 +1434,7 @@ def completion_cost( # noqa: PLR0915
|
|||
completion_tokens_cost_usd_dollar=completion_tokens_cost_usd_dollar,
|
||||
cost_for_built_in_tools_cost_usd_dollar=cost_for_built_in_tools,
|
||||
total_cost_usd_dollar=_final_cost,
|
||||
additional_costs=additional_costs,
|
||||
original_cost=original_cost,
|
||||
discount_percent=discount_percent,
|
||||
discount_amount=discount_amount,
|
||||
|
|
|
|||
|
|
@ -11,10 +11,12 @@ from mcp import ClientSession, ReadResourceResult, Resource, StdioServerParamete
|
|||
from mcp.client.sse import sse_client
|
||||
from mcp.client.stdio import stdio_client
|
||||
|
||||
streamable_http_client: Optional[Any] = None
|
||||
try:
|
||||
from mcp.client.streamable_http import streamable_http_client # type: ignore
|
||||
import mcp.client.streamable_http as streamable_http_module # type: ignore
|
||||
streamable_http_client = getattr(streamable_http_module, "streamable_http_client", None)
|
||||
except ImportError:
|
||||
streamable_http_client = None
|
||||
pass
|
||||
from mcp.types import CallToolRequestParams as MCPCallToolRequestParams
|
||||
from mcp.types import CallToolResult as MCPCallToolResult
|
||||
from mcp.types import (
|
||||
|
|
@ -111,6 +113,12 @@ class MCPClient:
|
|||
), None
|
||||
|
||||
# HTTP transport (default)
|
||||
if streamable_http_client is None:
|
||||
raise ImportError(
|
||||
"streamable_http_client is not available. "
|
||||
"Please install mcp with HTTP support."
|
||||
)
|
||||
|
||||
headers = self._get_auth_headers()
|
||||
httpx_client_factory = self._create_httpx_client_factory()
|
||||
verbose_logger.debug(
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import asyncio
|
|||
import contextvars
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
import uuid as uuid_module
|
||||
from functools import partial
|
||||
from typing import Any, Coroutine, Dict, Literal, Optional, Union, cast
|
||||
|
||||
|
|
@ -451,7 +451,7 @@ def file_retrieve(
|
|||
stream=False,
|
||||
call_type="afile_retrieve" if _is_async else "file_retrieve",
|
||||
start_time=time.time(),
|
||||
litellm_call_id=kwargs.get("litellm_call_id", str(uuid.uuid4())),
|
||||
litellm_call_id=kwargs.get("litellm_call_id", str(uuid_module.uuid4())),
|
||||
function_id=str(kwargs.get("id") or ""),
|
||||
)
|
||||
|
||||
|
|
@ -660,7 +660,7 @@ def file_delete(
|
|||
stream=False,
|
||||
call_type="afile_delete" if _is_async else "file_delete",
|
||||
start_time=time.time(),
|
||||
litellm_call_id=kwargs.get("litellm_call_id", str(uuid.uuid4())),
|
||||
litellm_call_id=kwargs.get("litellm_call_id", str(uuid_module.uuid4())),
|
||||
function_id=str(kwargs.get("id") or ""),
|
||||
)
|
||||
|
||||
|
|
@ -793,7 +793,7 @@ def file_list(
|
|||
stream=False,
|
||||
call_type="afile_list" if _is_async else "file_list",
|
||||
start_time=time.time(),
|
||||
litellm_call_id=kwargs.get("litellm_call_id", str(uuid.uuid4())),
|
||||
litellm_call_id=kwargs.get("litellm_call_id", str(uuid_module.uuid4())),
|
||||
function_id=str(kwargs.get("id", "")),
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -55,14 +55,9 @@ class DataDogLLMObsLogger(CustomBatchLogger):
|
|||
create_mock_datadog_client()
|
||||
verbose_logger.debug("[DATADOG MOCK] DataDogLLMObs logger initialized in mock mode")
|
||||
|
||||
if os.getenv("DD_API_KEY", None) is None:
|
||||
raise Exception("DD_API_KEY is not set, set 'DD_API_KEY=<>'")
|
||||
if os.getenv("DD_SITE", None) is None:
|
||||
raise Exception(
|
||||
"DD_SITE is not set, set 'DD_SITE=<>', example sit = `us5.datadoghq.com`"
|
||||
)
|
||||
# Configure DataDog endpoint (Agent or Direct API)
|
||||
# Use LITELLM_DD_AGENT_HOST to avoid conflicts with ddtrace's DD_AGENT_HOST
|
||||
# Check for agent mode FIRST - agent mode doesn't require DD_API_KEY or DD_SITE
|
||||
dd_agent_host = os.getenv("LITELLM_DD_AGENT_HOST")
|
||||
|
||||
self.async_client = get_async_httpx_client(
|
||||
|
|
@ -73,6 +68,13 @@ class DataDogLLMObsLogger(CustomBatchLogger):
|
|||
if dd_agent_host:
|
||||
self._configure_dd_agent(dd_agent_host=dd_agent_host)
|
||||
else:
|
||||
# Only require DD_API_KEY and DD_SITE for direct API mode
|
||||
if os.getenv("DD_API_KEY", None) is None:
|
||||
raise Exception("DD_API_KEY is not set, set 'DD_API_KEY=<>'")
|
||||
if os.getenv("DD_SITE", None) is None:
|
||||
raise Exception(
|
||||
"DD_SITE is not set, set 'DD_SITE=<>', example sit = `us5.datadoghq.com`"
|
||||
)
|
||||
self._configure_dd_direct_api()
|
||||
|
||||
# Optional override for testing
|
||||
|
|
|
|||
|
|
@ -1635,7 +1635,7 @@ class OpenTelemetry(CustomLogger):
|
|||
)
|
||||
|
||||
except Exception as e:
|
||||
self.handle_callback_failure(callback_name= self.callback_name)
|
||||
self.handle_callback_failure(callback_name=self.callback_name or "opentelemetry")
|
||||
verbose_logger.exception(
|
||||
"OpenTelemetry logging error in set_attributes %s", str(e)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1297,6 +1297,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
output_cost: float,
|
||||
total_cost: float,
|
||||
cost_for_built_in_tools_cost_usd_dollar: float,
|
||||
additional_costs: Optional[dict] = None,
|
||||
original_cost: Optional[float] = None,
|
||||
discount_percent: Optional[float] = None,
|
||||
discount_amount: Optional[float] = None,
|
||||
|
|
@ -1312,6 +1313,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
output_cost: Cost of output/completion tokens
|
||||
cost_for_built_in_tools_cost_usd_dollar: Cost of built-in tools
|
||||
total_cost: Total cost of request
|
||||
additional_costs: Free-form additional costs dict (e.g., {"azure_model_router_flat_cost": 0.00014})
|
||||
original_cost: Cost before discount
|
||||
discount_percent: Discount percentage (0.05 = 5%)
|
||||
discount_amount: Discount amount in USD
|
||||
|
|
@ -1327,6 +1329,10 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
tool_usage_cost=cost_for_built_in_tools_cost_usd_dollar,
|
||||
)
|
||||
|
||||
# Store additional costs if provided (free-form dict for extensibility)
|
||||
if additional_costs and isinstance(additional_costs, dict) and len(additional_costs) > 0:
|
||||
self.cost_breakdown["additional_costs"] = additional_costs
|
||||
|
||||
# Store discount information if provided
|
||||
if original_cost is not None:
|
||||
self.cost_breakdown["original_cost"] = original_cost
|
||||
|
|
|
|||
|
|
@ -215,6 +215,9 @@ def _get_token_base_cost(
|
|||
cache_creation_tiered_key = (
|
||||
f"cache_creation_input_token_cost_above_{threshold_str}_tokens"
|
||||
)
|
||||
cache_creation_1hr_tiered_key = (
|
||||
f"cache_creation_input_token_cost_above_1hr_above_{threshold_str}_tokens"
|
||||
)
|
||||
cache_read_tiered_key = (
|
||||
f"cache_read_input_token_cost_above_{threshold_str}_tokens"
|
||||
)
|
||||
|
|
@ -229,6 +232,16 @@ def _get_token_base_cost(
|
|||
),
|
||||
)
|
||||
|
||||
if cache_creation_1hr_tiered_key in model_info:
|
||||
cache_creation_cost_above_1hr = cast(
|
||||
float,
|
||||
_get_cost_per_unit(
|
||||
model_info,
|
||||
cache_creation_1hr_tiered_key,
|
||||
cache_creation_cost_above_1hr,
|
||||
),
|
||||
)
|
||||
|
||||
if cache_read_tiered_key in model_info:
|
||||
cache_read_cost = cast(
|
||||
float,
|
||||
|
|
|
|||
|
|
@ -4456,6 +4456,32 @@ def add_cache_point_tool_block(tool: dict) -> Optional[BedrockToolBlock]:
|
|||
return None
|
||||
|
||||
|
||||
def _is_bedrock_tool_block(tool: dict) -> bool:
|
||||
"""
|
||||
Check if a tool is already a BedrockToolBlock.
|
||||
|
||||
BedrockToolBlock has one of: systemTool, toolSpec, or cachePoint.
|
||||
This is used to detect tools that are already in Bedrock format
|
||||
(e.g., systemTool for Nova grounding) vs OpenAI-style function tools
|
||||
that need transformation.
|
||||
|
||||
Args:
|
||||
tool: The tool dict to check
|
||||
|
||||
Returns:
|
||||
True if the tool is already a BedrockToolBlock, False otherwise
|
||||
|
||||
Examples:
|
||||
>>> _is_bedrock_tool_block({"systemTool": {"name": "nova_grounding"}})
|
||||
True
|
||||
>>> _is_bedrock_tool_block({"type": "function", "function": {...}})
|
||||
False
|
||||
"""
|
||||
return isinstance(tool, dict) and (
|
||||
"systemTool" in tool or "toolSpec" in tool or "cachePoint" in tool
|
||||
)
|
||||
|
||||
|
||||
def _bedrock_tools_pt(tools: List) -> List[BedrockToolBlock]:
|
||||
"""
|
||||
OpenAI tools looks like:
|
||||
|
|
@ -4509,7 +4535,13 @@ def _bedrock_tools_pt(tools: List) -> List[BedrockToolBlock]:
|
|||
|
||||
tool_block_list: List[BedrockToolBlock] = []
|
||||
for tool in tools:
|
||||
# Handle regular function tools
|
||||
# Check if tool is already a BedrockToolBlock (e.g., systemTool for Nova grounding)
|
||||
if _is_bedrock_tool_block(tool):
|
||||
# Already a BedrockToolBlock, pass it through
|
||||
tool_block_list.append(tool) # type: ignore
|
||||
continue
|
||||
|
||||
# Handle regular OpenAI-style function tools
|
||||
parameters = tool.get("function", {}).get(
|
||||
"parameters", {"type": "object", "properties": {}}
|
||||
)
|
||||
|
|
|
|||
4
litellm/llms/azure_ai/azure_model_router/__init__.py
Normal file
4
litellm/llms/azure_ai/azure_model_router/__init__.py
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
"""Azure AI Foundry Model Router support."""
|
||||
from .transformation import AzureModelRouterConfig
|
||||
|
||||
__all__ = ["AzureModelRouterConfig"]
|
||||
125
litellm/llms/azure_ai/azure_model_router/transformation.py
Normal file
125
litellm/llms/azure_ai/azure_model_router/transformation.py
Normal file
|
|
@ -0,0 +1,125 @@
|
|||
"""
|
||||
Transformation for Azure AI Foundry Model Router.
|
||||
|
||||
The Model Router is a special Azure AI deployment that automatically routes requests
|
||||
to the best available model. It has specific cost tracking requirements.
|
||||
"""
|
||||
from typing import Any, List, Optional
|
||||
|
||||
from httpx import Response
|
||||
|
||||
from litellm.llms.azure_ai.chat.transformation import AzureAIStudioConfig
|
||||
from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
||||
|
||||
class AzureModelRouterConfig(AzureAIStudioConfig):
|
||||
"""
|
||||
Configuration for Azure AI Foundry Model Router.
|
||||
|
||||
Handles:
|
||||
- Stripping model_router prefix before sending to Azure API
|
||||
- Preserving full model path in responses for cost tracking
|
||||
- Calculating flat infrastructure costs for Model Router
|
||||
"""
|
||||
|
||||
def transform_request(
|
||||
self,
|
||||
model: str,
|
||||
messages: List[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
headers: dict,
|
||||
) -> dict:
|
||||
"""
|
||||
Transform request for Model Router.
|
||||
|
||||
Strips the model_router/ prefix so only the deployment name is sent to Azure.
|
||||
Example: model_router/azure-model-router -> azure-model-router
|
||||
"""
|
||||
from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
|
||||
|
||||
# Get base model name (strips routing prefixes like model_router/)
|
||||
base_model: str = AzureFoundryModelInfo.get_base_model(model)
|
||||
|
||||
return super().transform_request(
|
||||
base_model, messages, optional_params, litellm_params, headers
|
||||
)
|
||||
|
||||
def transform_response(
|
||||
self,
|
||||
model: str,
|
||||
raw_response: Response,
|
||||
model_response: ModelResponse,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
request_data: dict,
|
||||
messages: List[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: Any,
|
||||
api_key: Optional[str] = None,
|
||||
json_mode: Optional[bool] = None,
|
||||
) -> ModelResponse:
|
||||
"""
|
||||
Transform response for Model Router.
|
||||
|
||||
Preserves the original model path (including model_router/ prefix) in the response
|
||||
for proper cost tracking and logging.
|
||||
"""
|
||||
from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
|
||||
|
||||
# Preserve the original model from litellm_params (includes routing prefixes like model_router/)
|
||||
# This ensures cost tracking and logging use the full model path
|
||||
original_model: str = litellm_params.get("model") or model
|
||||
if not original_model.startswith("azure_ai/"):
|
||||
# Add provider prefix if not already present
|
||||
model_response.model = f"azure_ai/{original_model}"
|
||||
else:
|
||||
model_response.model = original_model
|
||||
|
||||
# Get base model for the parent call (strips routing prefixes for API compatibility)
|
||||
base_model: str = AzureFoundryModelInfo.get_base_model(model)
|
||||
|
||||
return super().transform_response(
|
||||
model=base_model,
|
||||
raw_response=raw_response,
|
||||
model_response=model_response,
|
||||
logging_obj=logging_obj,
|
||||
request_data=request_data,
|
||||
messages=messages,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
encoding=encoding,
|
||||
api_key=api_key,
|
||||
json_mode=json_mode,
|
||||
)
|
||||
|
||||
def calculate_additional_costs(
|
||||
self, model: str, prompt_tokens: int, completion_tokens: int
|
||||
) -> Optional[dict]:
|
||||
"""
|
||||
Calculate additional costs for Azure Model Router.
|
||||
|
||||
Adds a flat infrastructure cost of $0.14 per M input tokens for using the Model Router.
|
||||
|
||||
Args:
|
||||
model: The model name (should be a model router model)
|
||||
prompt_tokens: Number of prompt tokens
|
||||
completion_tokens: Number of completion tokens
|
||||
|
||||
Returns:
|
||||
Dictionary with additional costs, or None if not applicable.
|
||||
"""
|
||||
from litellm.llms.azure_ai.cost_calculator import (
|
||||
calculate_azure_model_router_flat_cost,
|
||||
)
|
||||
|
||||
flat_cost = calculate_azure_model_router_flat_cost(
|
||||
model=model, prompt_tokens=prompt_tokens
|
||||
)
|
||||
|
||||
if flat_cost > 0:
|
||||
return {"Azure Model Router Flat Cost": flat_cost}
|
||||
|
||||
return None
|
||||
|
|
@ -13,14 +13,28 @@ class AzureFoundryModelInfo(BaseLLMModelInfo):
|
|||
self._model = model
|
||||
|
||||
@staticmethod
|
||||
def get_azure_ai_route(model: str) -> Literal["agents", "default"]:
|
||||
def get_azure_ai_route(model: str) -> Literal["agents", "model_router", "default"]:
|
||||
"""
|
||||
Get the Azure AI route for the given model.
|
||||
|
||||
Similar to BedrockModelInfo.get_bedrock_route().
|
||||
|
||||
Supported routes:
|
||||
- agents: azure_ai/agents/<agent_id>
|
||||
- model_router: azure_ai/model_router/<actual-model-name> or models with "model-router"/"model_router" in name
|
||||
- default: standard models
|
||||
"""
|
||||
if "agents/" in model:
|
||||
return "agents"
|
||||
# Detect model router by prefix (model_router/<name>) or by name containing "model-router"/"model_router"
|
||||
model_lower = model.lower()
|
||||
if (
|
||||
"model_router/" in model_lower
|
||||
or "model-router/" in model_lower
|
||||
or "model-router" in model_lower
|
||||
or "model_router" in model_lower
|
||||
):
|
||||
return "model_router"
|
||||
return "default"
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -75,8 +89,73 @@ class AzureFoundryModelInfo(BaseLLMModelInfo):
|
|||
#########################################################
|
||||
|
||||
@staticmethod
|
||||
def get_base_model(model: str) -> Optional[str]:
|
||||
raise NotImplementedError("Azure Foundry does not support base model")
|
||||
def strip_model_router_prefix(model: str) -> str:
|
||||
"""
|
||||
Strip the model_router prefix from model name.
|
||||
|
||||
Examples:
|
||||
- "model_router/gpt-4o" -> "gpt-4o"
|
||||
- "model-router/gpt-4o" -> "gpt-4o"
|
||||
- "gpt-4o" -> "gpt-4o"
|
||||
|
||||
Args:
|
||||
model: Model name potentially with model_router prefix
|
||||
|
||||
Returns:
|
||||
Model name without the prefix
|
||||
"""
|
||||
if "model_router/" in model:
|
||||
return model.split("model_router/", 1)[1]
|
||||
if "model-router/" in model:
|
||||
return model.split("model-router/", 1)[1]
|
||||
return model
|
||||
|
||||
@staticmethod
|
||||
def get_base_model(model: str) -> str:
|
||||
"""
|
||||
Get the base model name, stripping any Azure AI routing prefixes.
|
||||
|
||||
Args:
|
||||
model: Model name potentially with routing prefixes
|
||||
|
||||
Returns:
|
||||
Base model name
|
||||
"""
|
||||
# Strip model_router prefix if present
|
||||
model = AzureFoundryModelInfo.strip_model_router_prefix(model)
|
||||
return model
|
||||
|
||||
@staticmethod
|
||||
def get_azure_ai_config_for_model(model: str):
|
||||
"""
|
||||
Get the appropriate Azure AI config class for the given model.
|
||||
|
||||
Routes to specialized configs based on model type:
|
||||
- Model Router: AzureModelRouterConfig
|
||||
- Claude models: AzureAnthropicConfig
|
||||
- Default: AzureAIStudioConfig
|
||||
|
||||
Args:
|
||||
model: The model name
|
||||
|
||||
Returns:
|
||||
The appropriate config instance
|
||||
"""
|
||||
azure_ai_route = AzureFoundryModelInfo.get_azure_ai_route(model)
|
||||
|
||||
if azure_ai_route == "model_router":
|
||||
from litellm.llms.azure_ai.azure_model_router.transformation import (
|
||||
AzureModelRouterConfig,
|
||||
)
|
||||
return AzureModelRouterConfig()
|
||||
elif "claude" in model.lower():
|
||||
from litellm.llms.azure_ai.anthropic.transformation import (
|
||||
AzureAnthropicConfig,
|
||||
)
|
||||
return AzureAnthropicConfig()
|
||||
else:
|
||||
from litellm.llms.azure_ai.chat.transformation import AzureAIStudioConfig
|
||||
return AzureAIStudioConfig()
|
||||
|
||||
def validate_environment(
|
||||
self,
|
||||
|
|
|
|||
121
litellm/llms/azure_ai/cost_calculator.py
Normal file
121
litellm/llms/azure_ai/cost_calculator.py
Normal file
|
|
@ -0,0 +1,121 @@
|
|||
"""
|
||||
Azure AI cost calculation helper.
|
||||
Handles Azure AI Foundry Model Router flat cost and other Azure AI specific pricing.
|
||||
"""
|
||||
|
||||
from typing import Optional, Tuple
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
||||
from litellm.types.utils import Usage
|
||||
from litellm.utils import get_model_info
|
||||
|
||||
|
||||
def _is_azure_model_router(model: str) -> bool:
|
||||
"""
|
||||
Check if the model is Azure AI Foundry Model Router.
|
||||
|
||||
Detects patterns like:
|
||||
- "azure-model-router"
|
||||
- "model-router"
|
||||
- "model_router/<actual-model>"
|
||||
- "model-router/<actual-model>"
|
||||
|
||||
Args:
|
||||
model: The model name
|
||||
|
||||
Returns:
|
||||
bool: True if this is a model router model
|
||||
"""
|
||||
model_lower = model.lower()
|
||||
return (
|
||||
"model-router" in model_lower
|
||||
or "model_router" in model_lower
|
||||
or model_lower == "azure-model-router"
|
||||
)
|
||||
|
||||
|
||||
def calculate_azure_model_router_flat_cost(model: str, prompt_tokens: int) -> float:
|
||||
"""
|
||||
Calculate the flat cost for Azure AI Foundry Model Router.
|
||||
|
||||
Args:
|
||||
model: The model name (should be a model router model)
|
||||
prompt_tokens: Number of prompt tokens
|
||||
|
||||
Returns:
|
||||
float: The flat cost in USD, or 0.0 if not applicable
|
||||
"""
|
||||
if not _is_azure_model_router(model):
|
||||
return 0.0
|
||||
|
||||
# Get the model router pricing from model_prices_and_context_window.json
|
||||
# Use "model_router" as the key (without actual model name suffix)
|
||||
model_info = get_model_info(model="model_router", custom_llm_provider="azure_ai")
|
||||
router_flat_cost_per_token = model_info.get("input_cost_per_token", 0)
|
||||
|
||||
if router_flat_cost_per_token > 0:
|
||||
return prompt_tokens * router_flat_cost_per_token
|
||||
|
||||
return 0.0
|
||||
|
||||
|
||||
def cost_per_token(
|
||||
model: str, usage: Usage, response_time_ms: Optional[float] = 0.0
|
||||
) -> Tuple[float, float]:
|
||||
"""
|
||||
Calculate the cost per token for Azure AI models.
|
||||
|
||||
For Azure AI Foundry Model Router:
|
||||
- Adds a flat cost of $0.14 per million input tokens (from model_prices_and_context_window.json)
|
||||
- Plus the cost of the actual model used (handled by generic_cost_per_token)
|
||||
|
||||
Args:
|
||||
model: str, the model name without provider prefix
|
||||
usage: LiteLLM Usage block
|
||||
response_time_ms: Optional response time in milliseconds
|
||||
|
||||
Returns:
|
||||
Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd
|
||||
|
||||
Raises:
|
||||
ValueError: If the model is not found in the cost map and cost cannot be calculated
|
||||
(except for Model Router models where we return just the routing flat cost)
|
||||
"""
|
||||
prompt_cost = 0.0
|
||||
completion_cost = 0.0
|
||||
|
||||
# Calculate base cost using generic cost calculator
|
||||
# This may raise an exception if the model is not in the cost map
|
||||
try:
|
||||
prompt_cost, completion_cost = generic_cost_per_token(
|
||||
model=model,
|
||||
usage=usage,
|
||||
custom_llm_provider="azure_ai",
|
||||
)
|
||||
except Exception as e:
|
||||
# For Model Router, the model name (e.g., "azure-model-router") may not be in the cost map
|
||||
# because it's a routing service, not an actual model. In this case, we continue
|
||||
# to calculate just the routing flat cost.
|
||||
if not _is_azure_model_router(model):
|
||||
# Re-raise for non-router models - they should have pricing defined
|
||||
raise
|
||||
verbose_logger.debug(
|
||||
f"Azure AI Model Router: model '{model}' not in cost map, calculating routing flat cost only. Error: {e}"
|
||||
)
|
||||
|
||||
# Add flat cost for Azure Model Router
|
||||
# The flat cost is defined in model_prices_and_context_window.json for azure_ai/model_router
|
||||
if _is_azure_model_router(model):
|
||||
router_flat_cost = calculate_azure_model_router_flat_cost(model, usage.prompt_tokens)
|
||||
|
||||
if router_flat_cost > 0:
|
||||
verbose_logger.debug(
|
||||
f"Azure AI Model Router flat cost: ${router_flat_cost:.6f} "
|
||||
f"({usage.prompt_tokens} tokens × ${router_flat_cost / usage.prompt_tokens:.9f}/token)"
|
||||
)
|
||||
|
||||
# Add flat cost to prompt cost
|
||||
prompt_cost += router_flat_cost
|
||||
|
||||
return prompt_cost, completion_cost
|
||||
|
|
@ -437,3 +437,23 @@ class BaseConfig(ABC):
|
|||
By default, this is true for almost all providers.
|
||||
"""
|
||||
return True
|
||||
|
||||
def calculate_additional_costs(
|
||||
self, model: str, prompt_tokens: int, completion_tokens: int
|
||||
) -> Optional[dict]:
|
||||
"""
|
||||
Calculate any additional costs beyond standard token costs.
|
||||
|
||||
This is used for provider-specific infrastructure costs, routing fees, etc.
|
||||
|
||||
Args:
|
||||
model: The model name
|
||||
prompt_tokens: Number of prompt tokens
|
||||
completion_tokens: Number of completion tokens
|
||||
|
||||
Returns:
|
||||
Optional dictionary with cost names and amounts, e.g.:
|
||||
{"Infrastructure Fee": 0.001, "Routing Cost": 0.0005}
|
||||
Returns None if no additional costs apply.
|
||||
"""
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -806,6 +806,8 @@ class AmazonConverseConfig(BaseConfig):
|
|||
optional_params["serviceTier"] = {"type": bedrock_tier}
|
||||
|
||||
if param == "web_search_options" and isinstance(value, dict):
|
||||
# Note: we use `isinstance(value, dict)` instead of `value and isinstance(value, dict)`
|
||||
# because empty dict {} is falsy but is a valid way to enable Nova grounding
|
||||
grounding_tool = self._map_web_search_options(value, model)
|
||||
if grounding_tool is not None:
|
||||
optional_params = self._add_tools_to_optional_params(
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ class BedrockCohereEmbeddingConfig:
|
|||
pass
|
||||
|
||||
def get_supported_openai_params(self) -> List[str]:
|
||||
return ["encoding_format"]
|
||||
return ["encoding_format", "dimensions"]
|
||||
|
||||
def map_openai_params(
|
||||
self, non_default_params: dict, optional_params: dict
|
||||
|
|
@ -23,6 +23,8 @@ class BedrockCohereEmbeddingConfig:
|
|||
for k, v in non_default_params.items():
|
||||
if k == "encoding_format":
|
||||
optional_params["embedding_types"] = v
|
||||
elif k == "dimensions":
|
||||
optional_params["output_dimension"] = v
|
||||
return optional_params
|
||||
|
||||
def _is_v3_model(self, model: str) -> bool:
|
||||
|
|
|
|||
|
|
@ -50,9 +50,21 @@ try:
|
|||
except Exception:
|
||||
version = "0.0.0"
|
||||
|
||||
headers = {
|
||||
"User-Agent": f"litellm/{version}",
|
||||
}
|
||||
def get_default_headers() -> dict:
|
||||
"""
|
||||
Get default headers for HTTP requests.
|
||||
|
||||
- Default: `User-Agent: litellm/{version}`
|
||||
- Override: set `LITELLM_USER_AGENT` to fully override the header value.
|
||||
"""
|
||||
user_agent = os.environ.get("LITELLM_USER_AGENT")
|
||||
if user_agent is not None:
|
||||
return {"User-Agent": user_agent}
|
||||
|
||||
return {"User-Agent": f"litellm/{version}"}
|
||||
|
||||
# Initialize headers (User-Agent)
|
||||
headers = get_default_headers()
|
||||
|
||||
# https://www.python-httpx.org/advanced/timeouts
|
||||
_DEFAULT_TIMEOUT = httpx.Timeout(timeout=5.0, connect=5.0)
|
||||
|
|
@ -371,13 +383,16 @@ class AsyncHTTPHandler:
|
|||
shared_session=shared_session,
|
||||
)
|
||||
|
||||
# Get default headers (User-Agent, overridable via LITELLM_USER_AGENT)
|
||||
default_headers = get_default_headers()
|
||||
|
||||
return httpx.AsyncClient(
|
||||
transport=transport,
|
||||
event_hooks=event_hooks,
|
||||
timeout=timeout,
|
||||
verify=ssl_config,
|
||||
cert=cert,
|
||||
headers=headers,
|
||||
headers=default_headers,
|
||||
follow_redirects=True,
|
||||
)
|
||||
|
||||
|
|
@ -899,6 +914,9 @@ class HTTPHandler:
|
|||
# /path/to/client.pem
|
||||
cert = os.getenv("SSL_CERTIFICATE", litellm.ssl_certificate)
|
||||
|
||||
# Get default headers (User-Agent, overridable via LITELLM_USER_AGENT)
|
||||
default_headers = get_default_headers() if not disable_default_headers else None
|
||||
|
||||
if client is None:
|
||||
transport = self._create_sync_transport()
|
||||
|
||||
|
|
@ -908,7 +926,7 @@ class HTTPHandler:
|
|||
timeout=timeout,
|
||||
verify=ssl_config,
|
||||
cert=cert,
|
||||
headers=headers if not disable_default_headers else None,
|
||||
headers=default_headers,
|
||||
follow_redirects=True,
|
||||
)
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import os
|
||||
from typing import Optional, Union
|
||||
|
||||
import httpx
|
||||
|
|
@ -7,13 +8,22 @@ try:
|
|||
except Exception:
|
||||
version = "0.0.0"
|
||||
|
||||
headers = {
|
||||
"User-Agent": f"litellm/{version}",
|
||||
}
|
||||
def get_default_headers() -> dict:
|
||||
"""
|
||||
Get default headers for HTTP requests.
|
||||
|
||||
- Default: `User-Agent: litellm/{version}`
|
||||
- Override: set `LITELLM_USER_AGENT` to fully override the header value.
|
||||
"""
|
||||
user_agent = os.environ.get("LITELLM_USER_AGENT")
|
||||
if user_agent is not None:
|
||||
return {"User-Agent": user_agent}
|
||||
|
||||
return {"User-Agent": f"litellm/{version}"}
|
||||
|
||||
class HTTPHandler:
|
||||
def __init__(self, concurrent_limit=1000):
|
||||
headers = get_default_headers()
|
||||
# Create a client with a connection pool
|
||||
self.client = httpx.AsyncClient(
|
||||
limits=httpx.Limits(
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ Supports writing files to Google AI Studio Files API.
|
|||
For vertex ai, check out the vertex_ai/files/handler.py file.
|
||||
"""
|
||||
import time
|
||||
from typing import List, Optional
|
||||
from typing import Any, List, Literal, Optional
|
||||
|
||||
import httpx
|
||||
from openai.types.file_deleted import FileDeleted
|
||||
|
|
@ -17,6 +17,7 @@ from litellm.llms.base_llm.files.transformation import (
|
|||
)
|
||||
from litellm.types.llms.gemini import GeminiCreateFilesResponseObject
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
CreateFileRequest,
|
||||
HttpxBinaryResponseContent,
|
||||
OpenAICreateFileRequestOptionalParams,
|
||||
|
|
@ -37,22 +38,23 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig):
|
|||
|
||||
def validate_environment(
|
||||
self,
|
||||
api_key: Optional[str],
|
||||
headers: dict,
|
||||
headers: dict[Any, Any],
|
||||
model: str,
|
||||
messages: list,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
) -> dict:
|
||||
messages: List[AllMessageValues],
|
||||
optional_params: dict[Any, Any],
|
||||
litellm_params: dict[Any, Any],
|
||||
api_key: Optional[str] = None,
|
||||
api_base: Optional[str] = None,
|
||||
) -> dict[Any, Any]:
|
||||
"""
|
||||
Validate environment and add Gemini API key to headers.
|
||||
Google AI Studio uses x-goog-api-key header for authentication.
|
||||
"""
|
||||
api_key = self.get_api_key(api_key)
|
||||
if not api_key:
|
||||
resolved_api_key = self.get_api_key(api_key)
|
||||
if not resolved_api_key:
|
||||
raise ValueError("GEMINI_API_KEY is required for Google AI Studio file operations")
|
||||
|
||||
headers["x-goog-api-key"] = api_key
|
||||
headers["x-goog-api-key"] = resolved_api_key
|
||||
return headers
|
||||
|
||||
def get_complete_url(
|
||||
|
|
@ -236,11 +238,13 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig):
|
|||
|
||||
# Map Gemini state to OpenAI status
|
||||
gemini_state = response_json.get("state", "STATE_UNSPECIFIED")
|
||||
status = "uploaded" # Default
|
||||
# Explicitly type status as the Literal union
|
||||
if gemini_state == "ACTIVE":
|
||||
status = "processed"
|
||||
status: Literal["uploaded", "processed", "error"] = "processed"
|
||||
elif gemini_state == "FAILED":
|
||||
status = "error"
|
||||
else:
|
||||
status = "uploaded"
|
||||
|
||||
return OpenAIFileObject(
|
||||
id=response_json.get("uri", ""),
|
||||
|
|
@ -301,7 +305,7 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig):
|
|||
url = f"{api_base}/v1beta/{file_name}"
|
||||
|
||||
# Add API key as header (Google AI Studio uses x-goog-api-key header)
|
||||
params = {}
|
||||
params: dict = {}
|
||||
|
||||
return url, params
|
||||
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ def generate_iam_token(api_key=None, **params) -> str:
|
|||
get_secret_str("WX_API_KEY")
|
||||
or get_secret_str("WATSONX_API_KEY")
|
||||
or get_secret_str("WATSONX_APIKEY")
|
||||
or get_secret_str("WATSONX_ZENAPIKEY")
|
||||
)
|
||||
if api_key is None:
|
||||
raise ValueError("API key is required")
|
||||
|
|
@ -319,6 +320,7 @@ class IBMWatsonXMixin:
|
|||
or get_secret_str("WATSONX_APIKEY")
|
||||
or get_secret_str("WATSONX_API_KEY")
|
||||
or get_secret_str("WX_API_KEY")
|
||||
or get_secret_str("WATSONX_ZENAPIKEY")
|
||||
)
|
||||
|
||||
api_base = (
|
||||
|
|
|
|||
|
|
@ -1517,6 +1517,14 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"azure_ai/model_router": {
|
||||
"input_cost_per_token": 1.4e-07,
|
||||
"output_cost_per_token": 0,
|
||||
"litellm_provider": "azure_ai",
|
||||
"mode": "chat",
|
||||
"source": "https://azure.microsoft.com/en-us/pricing/details/ai-services/",
|
||||
"comment": "Flat cost of $0.14 per M input tokens for Azure AI Foundry Model Router infrastructure. Use pattern: azure_ai/model_router/<deployment-name> where deployment-name is your Azure deployment (e.g., azure-model-router)"
|
||||
},
|
||||
"azure/eu/gpt-4o-2024-08-06": {
|
||||
"deprecation_date": "2026-02-27",
|
||||
"cache_read_input_token_cost": 1.375e-06,
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ import datetime
|
|||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from typing import Any, Dict, List, Literal, Optional, Set, Tuple, Union, cast, Callable
|
||||
from typing import Any, Callable, Dict, List, Literal, Optional, Set, Tuple, Union, cast
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
|
@ -30,7 +30,6 @@ from pydantic import AnyUrl
|
|||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.types.utils import CallTypes
|
||||
from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException
|
||||
from litellm.experimental_mcp_client.client import MCPClient
|
||||
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
|
||||
|
|
@ -63,20 +62,23 @@ from litellm.types.mcp_server.mcp_server_manager import (
|
|||
MCPOAuthMetadata,
|
||||
MCPServer,
|
||||
)
|
||||
from litellm.types.utils import CallTypes
|
||||
|
||||
try:
|
||||
from mcp.shared.tool_name_validation import SEP_986_URL, validate_tool_name # type: ignore
|
||||
from mcp.shared.tool_name_validation import ( # type: ignore
|
||||
SEP_986_URL,
|
||||
validate_tool_name,
|
||||
)
|
||||
except ImportError:
|
||||
from pydantic import BaseModel
|
||||
SEP_986_URL = "https://github.com/modelcontextprotocol/protocol/blob/main/proposals/0001-tool-name-validation.md"
|
||||
|
||||
def validate_tool_name(name: str):
|
||||
from pydantic import BaseModel
|
||||
class ToolNameValidationResult(BaseModel):
|
||||
is_valid: bool = True
|
||||
warnings: list = []
|
||||
|
||||
class MockResult(BaseModel):
|
||||
is_valid: bool = True
|
||||
warnings: list = []
|
||||
|
||||
return MockResult()
|
||||
def validate_tool_name(name: str) -> ToolNameValidationResult: # type: ignore[misc]
|
||||
return ToolNameValidationResult()
|
||||
|
||||
|
||||
# Probe includes characters on both sides of the separator to mimic real prefixed tool names.
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue