scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-07 08:26:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 36

Merge branch 'main' into LangfuseUsageDetails

Browse source
This commit is contained in:
Fabrício Ceschin 2025-05-23 09:50:05 -04:00 • committed by GitHub
commit d3728e78ff
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
122 changed files with 3028 additions and 449 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
.circleci/config.yml
View file

@ -95,7 +95,7 @@ jobs:
pip install opentelemetry-api==1.25.0
pip install opentelemetry-sdk==1.25.0
pip install opentelemetry-exporter-otlp==1.25.0
pip install openai==1.68.2
pip install openai==1.81.0
pip install prisma==0.11.0
pip install "detect_secrets==1.5.0"
pip install "httpx==0.24.1"
@ -218,7 +218,7 @@ jobs:
pip install opentelemetry-api==1.25.0
pip install opentelemetry-sdk==1.25.0
pip install opentelemetry-exporter-otlp==1.25.0
pip install openai==1.68.2
pip install openai==1.81.0
pip install prisma==0.11.0
pip install "detect_secrets==1.5.0"
pip install "httpx==0.24.1"
@ -325,7 +325,7 @@ jobs:
pip install opentelemetry-api==1.25.0
pip install opentelemetry-sdk==1.25.0
pip install opentelemetry-exporter-otlp==1.25.0
pip install openai==1.68.2
pip install openai==1.81.0
pip install prisma==0.11.0
pip install "detect_secrets==1.5.0"
pip install "httpx==0.24.1"
@ -581,7 +581,7 @@ jobs:
pip install opentelemetry-api==1.25.0
pip install opentelemetry-sdk==1.25.0
pip install opentelemetry-exporter-otlp==1.25.0
pip install openai==1.68.2
pip install openai==1.81.0
pip install prisma==0.11.0
pip install "detect_secrets==1.5.0"
pip install "httpx==0.24.1"
@ -1472,7 +1472,7 @@ jobs:
pip install "aiodynamo==23.10.1"
pip install "asyncio==3.4.3"
pip install "PyGithub==1.59.1"
pip install "openai==1.68.2"
pip install "openai==1.81.0"
- run:
name: Install Grype
command: |
@ -1610,7 +1610,7 @@ jobs:
pip install "aiodynamo==23.10.1"
pip install "asyncio==3.4.3"
pip install "PyGithub==1.59.1"
pip install "openai==1.68.2"
pip install "openai==1.81.0"
# Run pytest and generate JUnit XML report
- run:
name: Build Docker image
@ -1733,7 +1733,7 @@ jobs:
pip install "aiodynamo==23.10.1"
pip install "asyncio==3.4.3"
pip install "PyGithub==1.59.1"
pip install "openai==1.68.2"
pip install "openai==1.81.0"
- run:
name: Build Docker image
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
@ -2256,7 +2256,7 @@ jobs:
pip install "pytest-asyncio==0.21.1"
pip install "google-cloud-aiplatform==1.43.0"
pip install aiohttp
pip install "openai==1.68.2"
pip install "openai==1.81.0"
pip install "assemblyai==0.37.0"
python -m pip install --upgrade pip
pip install "pydantic==2.10.2"
@ -2644,7 +2644,7 @@ jobs:
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
pip install aiohttp
pip install "openai==1.68.2"
pip install "openai==1.81.0"
python -m pip install --upgrade pip
pip install "pydantic==2.10.2"
pip install "pytest==7.3.1"

2
.circleci/requirements.txt
View file

@ -1,5 +1,5 @@
# used by CI/CD testing
openai==1.68.2
openai==1.81.0
python-dotenv
tiktoken
importlib_metadata

6
.github/ISSUE_TEMPLATE/feature_request.yml vendored
View file

@ -23,10 +23,10 @@ body:
validations:
required: true
- type: dropdown
id: ml-ops-team
id: hiring-interest
attributes:
label: Are you a ML Ops Team?
description: This helps us prioritize your requests correctly
label: LiteLLM is hiring a founding backend engineer, are you interested in joining us and shipping to all our users?
description: If yes, apply here - https://www.ycombinator.com/companies/litellm/jobs/6uvoBp3-founding-backend-engineer
options:
- "No"
- "Yes"

4
.github/workflows/test-linting.yml vendored
View file

@ -22,9 +22,9 @@ jobs:
- name: Install dependencies
run: |
pip install openai==1.68.2
pip install openai==1.81.0
poetry install --with dev
pip install openai==1.68.2
pip install openai==1.81.0

4
Dockerfile
View file

@ -51,7 +51,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
USER root
# Install runtime dependencies
RUN apk add --no-cache openssl
RUN apk add --no-cache openssl tzdata
WORKDIR /app
# Copy the current directory contents into the container at /app
@ -74,5 +74,5 @@ EXPOSE 4000/tcp
ENTRYPOINT ["docker/prod_entrypoint.sh"]
# Append "--detailed_debug" to the end of CMD to view detailed debug logs
# Append "--detailed_debug" to the end of CMD to view detailed debug logs
CMD ["--port", "4000"]

2
deploy/charts/litellm-helm/Chart.yaml
View file

@ -18,7 +18,7 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.4.3
version: 0.4.4
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to

1
deploy/charts/litellm-helm/README.md
View file

@ -34,6 +34,7 @@ If `db.useStackgresOperator` is used (not yet implemented):
| `serviceAccount.create` | Whether or not to create a Kubernetes Service Account for this deployment. The default is `false` because LiteLLM has no need to access the Kubernetes API. | `false` |
| `service.type` | Kubernetes Service type (e.g. `LoadBalancer`, `ClusterIP`, etc.) | `ClusterIP` |
| `service.port` | TCP port that the Kubernetes Service will listen on. Also the TCP port within the Pod that the proxy will listen on. | `4000` |
| `service.loadBalancerClass` | Optional LoadBalancer implementation class (only used when `service.type` is `LoadBalancer`) | `""` |
| `ingress.*` | See [values.yaml](./values.yaml) for example settings | N/A |
| `proxy_config.*` | See [values.yaml](./values.yaml) for default settings. See [example_config_yaml](../../../litellm/proxy/example_config_yaml/) for configuration examples. | N/A |
| `extraContainers[]` | An array of additional containers to be deployed as sidecars alongside the LiteLLM Proxy. | `[]` |

3
deploy/charts/litellm-helm/templates/service.yaml
View file

@ -10,6 +10,9 @@ metadata:
{{- include "litellm.labels" . | nindent 4 }}
spec:
type: {{ .Values.service.type }}
{{- if and (eq .Values.service.type "LoadBalancer") .Values.service.loadBalancerClass }}
loadBalancerClass: {{ .Values.service.loadBalancerClass }}
{{- end }}
ports:
- port: {{ .Values.service.port }}
targetPort: http

116
deploy/charts/litellm-helm/tests/service_tests.yaml Normal file
View file

@ -0,0 +1,116 @@
suite: Service Configuration Tests
templates:
- service.yaml
tests:
- it: should create a default ClusterIP service
template: service.yaml
asserts:
- isKind:
of: Service
- equal:
path: spec.type
value: ClusterIP
- equal:
path: spec.ports[0].port
value: 4000
- equal:
path: spec.ports[0].targetPort
value: http
- equal:
path: spec.ports[0].protocol
value: TCP
- equal:
path: spec.ports[0].name
value: http
- isNull:
path: spec.loadBalancerClass
- it: should create a NodePort service when specified
template: service.yaml
set:
service.type: NodePort
asserts:
- isKind:
of: Service
- equal:
path: spec.type
value: NodePort
- isNull:
path: spec.loadBalancerClass
- it: should create a LoadBalancer service when specified
template: service.yaml
set:
service.type: LoadBalancer
asserts:
- isKind:
of: Service
- equal:
path: spec.type
value: LoadBalancer
- isNull:
path: spec.loadBalancerClass
- it: should add loadBalancerClass when specified with LoadBalancer type
template: service.yaml
set:
service.type: LoadBalancer
service.loadBalancerClass: tailscale
asserts:
- isKind:
of: Service
- equal:
path: spec.type
value: LoadBalancer
- equal:
path: spec.loadBalancerClass
value: tailscale
- it: should not add loadBalancerClass when specified with ClusterIP type
template: service.yaml
set:
service.type: ClusterIP
service.loadBalancerClass: tailscale
asserts:
- isKind:
of: Service
- equal:
path: spec.type
value: ClusterIP
- isNull:
path: spec.loadBalancerClass
- it: should use custom port when specified
template: service.yaml
set:
service.port: 8080
asserts:
- equal:
path: spec.ports[0].port
value: 8080
- it: should add service annotations when specified
template: service.yaml
set:
service.annotations:
cloud.google.com/load-balancer-type: "Internal"
service.beta.kubernetes.io/aws-load-balancer-internal: "true"
asserts:
- isKind:
of: Service
- equal:
path: metadata.annotations
value:
cloud.google.com/load-balancer-type: "Internal"
service.beta.kubernetes.io/aws-load-balancer-internal: "true"
- it: should use the correct selector labels
template: service.yaml
asserts:
- isNotNull:
path: spec.selector
- equal:
path: spec.selector
value:
app.kubernetes.io/name: litellm
app.kubernetes.io/instance: RELEASE-NAME

3
deploy/charts/litellm-helm/values.yaml
View file

@ -56,6 +56,9 @@ environmentConfigMaps: []
service:
type: ClusterIP
port: 4000
# If service type is `LoadBalancer` you can
# optionally specify loadBalancerClass
# loadBalancerClass: tailscale
ingress:
enabled: false

2
docs/my-website/docs/pass_through/vertex_ai.md
View file

@ -116,7 +116,7 @@ curl \
<TabItem value="curl" label="curl">
```bash
curl http://localhost:4000/vertex_ai/vertex_ai/v1/projects/${PROJECT_ID}/locations/us-central1/publishers/google/models/${MODEL_ID}:generateContent \
curl http://localhost:4000/vertex_ai/v1/projects/${PROJECT_ID}/locations/us-central1/publishers/google/models/${MODEL_ID}:generateContent \
-H "Content-Type: application/json" \
-H "x-litellm-api-key: Bearer sk-1234" \
-d '{

15
docs/my-website/docs/providers/anthropic.md
View file

@ -4,6 +4,8 @@ import TabItem from '@theme/TabItem';
# Anthropic
LiteLLM supports all anthropic models.
- `claude-4` (`claude-opus-4-20250514`, `claude-sonnet-4-20250514`)
- `claude-3.7` (`claude-3-7-sonnet-20250219`)
- `claude-3.5` (`claude-3-5-sonnet-20240620`)
- `claude-3` (`claude-3-haiku-20240307`, `claude-3-opus-20240229`, `claude-3-sonnet-20240229`)
- `claude-2`
@ -64,7 +66,7 @@ from litellm import completion
os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
messages = [{"role": "user", "content": "Hey! how's it going?"}]
response = completion(model="claude-3-opus-20240229", messages=messages)
response = completion(model="claude-opus-4-20250514", messages=messages)
print(response)
```
@ -80,7 +82,7 @@ from litellm import completion
os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
messages = [{"role": "user", "content": "Hey! how's it going?"}]
response = completion(model="claude-3-opus-20240229", messages=messages, stream=True)
response = completion(model="claude-opus-4-20250514", messages=messages, stream=True)
for chunk in response:
print(chunk["choices"][0]["delta"]["content"]) # same as openai format
```
@ -102,9 +104,9 @@ export ANTHROPIC_API_KEY="your-api-key"
```yaml
model_list:
- model_name: claude-3 ### RECEIVED MODEL NAME ###
- model_name: claude-4 ### RECEIVED MODEL NAME ###
litellm_params: # all params accepted by litellm.completion() - https://docs.litellm.ai/docs/completion/input
model: claude-3-opus-20240229 ### MODEL NAME sent to `litellm.completion()` ###
model: claude-opus-4-20250514 ### MODEL NAME sent to `litellm.completion()` ###
api_key: "os.environ/ANTHROPIC_API_KEY" # does os.getenv("AZURE_API_KEY_EU")
```
@ -156,7 +158,7 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
<TabItem value="cli" label="cli">
```bash
$ litellm --model claude-3-opus-20240229
$ litellm --model claude-opus-4-20250514
# Server running on http://0.0.0.0:4000
```
@ -244,6 +246,9 @@ print(response)
| Model Name | Function Call |
|------------------|--------------------------------------------|
| claude-opus-4 | `completion('claude-opus-4-20250514', messages)` | `os.environ['ANTHROPIC_API_KEY']` |
| claude-sonnet-4 | `completion('claude-sonnet-4-20250514', messages)` | `os.environ['ANTHROPIC_API_KEY']` |
| claude-3.7 | `completion('claude-3-7-sonnet-20250219', messages)` | `os.environ['ANTHROPIC_API_KEY']` |
| claude-3-5-sonnet | `completion('claude-3-5-sonnet-20240620', messages)` | `os.environ['ANTHROPIC_API_KEY']` |
| claude-3-haiku | `completion('claude-3-haiku-20240307', messages)` | `os.environ['ANTHROPIC_API_KEY']` |
| claude-3-opus | `completion('claude-3-opus-20240229', messages)` | `os.environ['ANTHROPIC_API_KEY']` |

130
docs/my-website/docs/providers/openai/responses_api.md
View file

@ -318,3 +318,133 @@ print(response)
</TabItem>
</Tabs>
## MCP Tools
<Tabs>
<TabItem value="sdk" label="LiteLLM Python SDK">
```python showLineNumbers title="MCP Tools with LiteLLM SDK"
import litellm
from typing import Optional
# Configure MCP Tools
MCP_TOOLS = [
{
"type": "mcp",
"server_label": "deepwiki",
"server_url": "https://mcp.deepwiki.com/mcp",
"allowed_tools": ["ask_question"]
}
]
# Step 1: Make initial request - OpenAI will use MCP LIST and return MCP calls for approval
response = litellm.responses(
model="openai/gpt-4.1",
tools=MCP_TOOLS,
input="What transport protocols does the 2025-03-26 version of the MCP spec support?"
)
# Get the MCP approval ID
mcp_approval_id = None
for output in response.output:
if output.type == "mcp_approval_request":
mcp_approval_id = output.id
break
# Step 2: Send followup with approval for the MCP call
response_with_mcp_call = litellm.responses(
model="openai/gpt-4.1",
tools=MCP_TOOLS,
input=[
{
"type": "mcp_approval_response",
"approve": True,
"approval_request_id": mcp_approval_id
}
],
previous_response_id=response.id,
)
print(response_with_mcp_call)
```
</TabItem>
<TabItem value="proxy" label="LiteLLM Proxy">
1. Set up config.yaml
```yaml showLineNumbers title="OpenAI Proxy Configuration"
model_list:
- model_name: openai/gpt-4.1
litellm_params:
model: openai/gpt-4.1
api_key: os.environ/OPENAI_API_KEY
```
2. Start LiteLLM Proxy Server
```bash title="Start LiteLLM Proxy Server"
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
3. Test it!
```python showLineNumbers title="MCP Tools with OpenAI SDK via LiteLLM Proxy"
from openai import OpenAI
from typing import Optional
# Initialize client with your proxy URL
client = OpenAI(
base_url="http://localhost:4000", # Your proxy URL
api_key="your-api-key" # Your proxy API key
)
# Configure MCP Tools
MCP_TOOLS = [
{
"type": "mcp",
"server_label": "deepwiki",
"server_url": "https://mcp.deepwiki.com/mcp",
"allowed_tools": ["ask_question"]
}
]
# Step 1: Make initial request - OpenAI will use MCP LIST and return MCP calls for approval
response = client.responses.create(
model="openai/gpt-4.1",
tools=MCP_TOOLS,
input="What transport protocols does the 2025-03-26 version of the MCP spec support?"
)
# Get the MCP approval ID
mcp_approval_id = None
for output in response.output:
if output.type == "mcp_approval_request":
mcp_approval_id = output.id
break
# Step 2: Send followup with approval for the MCP call
response_with_mcp_call = client.responses.create(
model="openai/gpt-4.1",
tools=MCP_TOOLS,
input=[
{
"type": "mcp_approval_response",
"approve": True,
"approval_request_id": mcp_approval_id
}
],
previous_response_id=response.id,
)
print(response_with_mcp_call)
```
</TabItem>
</Tabs>

1
docs/my-website/docs/proxy/config_settings.md
View file

@ -528,6 +528,7 @@ router_settings:
| MAX_TOKEN_TRIMMING_ATTEMPTS | Maximum number of attempts to trim a token message. Default is 10
| MAXIMUM_TRACEBACK_LINES_TO_LOG | Maximum number of lines to log in traceback in LiteLLM Logs UI. Default is 100
| MAX_RETRY_DELAY | Maximum delay in seconds for retrying requests. Default is 8.0
| MAX_LANGFUSE_INITIALIZED_CLIENTS | Maximum number of Langfuse clients to initialize on proxy. Default is 20. This is set since langfuse initializes 1 thread everytime a client is initialized. We've had an incident in the past where we reached 100% cpu utilization because Langfuse was initialized several times.
| MIN_NON_ZERO_TEMPERATURE | Minimum non-zero temperature value. Default is 0.0001
| MINIMUM_PROMPT_CACHE_TOKEN_COUNT | Minimum token count for caching a prompt. Default is 1024
| MISTRAL_API_BASE | Base URL for Mistral API

2
docs/my-website/docs/proxy/guardrails/pii_masking_v2.md
View file

@ -418,7 +418,7 @@ guardrails:
Send ad-hoc recognizers to presidio `/analyze` by passing a json file to the proxy
[**Example** ad-hoc recognizer](../../../../litellm/proxy/hooks/example_presidio_ad_hoc_recognize)
[**Example** ad-hoc recognizer](https://github.com/BerriAI/litellm/blob/b69b7503db5aa039a49b7ca96ae5b34db0d25a3d/litellm/proxy/hooks/example_presidio_ad_hoc_recognizer.json)
#### Define ad-hoc recognizer on your LiteLLM config.yaml

1
enterprise/enterprise_hooks/__init__.py
View file

@ -1,4 +1,3 @@
import os
from typing import Dict, Literal, Type, Union
from litellm.integrations.custom_logger import CustomLogger

296
enterprise/enterprise_hooks/managed_files.py
View file

@ -1,29 +1,44 @@
# What is this?
## This hook is used to check for LiteLLM managed files in the request body, and replace them with model-specific file id
import asyncio
import base64
import json
import uuid
from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union, cast
from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Tuple, Union, cast
from fastapi import HTTPException
from litellm import Router, verbose_logger
from litellm.caching.caching import DualCache
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
from litellm.llms.base_llm.files.transformation import BaseFileEndpoints
from litellm.proxy._types import CallTypes, LiteLLM_ManagedFileTable, UserAPIKeyAuth
from litellm.proxy._types import (
CallTypes,
LiteLLM_ManagedFileTable,
LiteLLM_ManagedObjectTable,
UserAPIKeyAuth,
)
from litellm.proxy.openai_files_endpoints.common_utils import (
_is_base64_encoded_unified_file_id,
convert_b64_uid_to_unified_uid,
)
from litellm.types.llms.openai import (
AllMessageValues,
AsyncCursorPage,
ChatCompletionFileObject,
CreateFileRequest,
FileObject,
OpenAIFileObject,
OpenAIFilesPurpose,
)
from litellm.types.utils import LiteLLMBatch, LLMResponseTypes, SpecialEnums
from litellm.types.utils import (
LiteLLMBatch,
LiteLLMFineTuningJob,
LLMResponseTypes,
SpecialEnums,
)
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
@ -54,6 +69,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
file_object: OpenAIFileObject,
litellm_parent_otel_span: Optional[Span],
model_mappings: Dict[str, str],
user_api_key_dict: UserAPIKeyAuth,
) -> None:
verbose_logger.info(
f"Storing LiteLLM Managed File object with id={file_id} in cache"
@ -62,6 +78,9 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
unified_file_id=file_id,
file_object=file_object,
model_mappings=model_mappings,
flat_model_file_ids=list(model_mappings.values()),
created_by=user_api_key_dict.user_id,
updated_by=user_api_key_dict.user_id,
)
await self.internal_usage_cache.async_set_cache(
key=file_id,
@ -74,6 +93,44 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
"unified_file_id": file_id,
"file_object": file_object.model_dump_json(),
"model_mappings": json.dumps(model_mappings),
"flat_model_file_ids": list(model_mappings.values()),
"created_by": user_api_key_dict.user_id,
"updated_by": user_api_key_dict.user_id,
}
)
async def store_unified_object_id(
self,
unified_object_id: str,
file_object: Union[LiteLLMBatch, LiteLLMFineTuningJob],
litellm_parent_otel_span: Optional[Span],
model_object_id: str,
file_purpose: Literal["batch", "fine-tune"],
user_api_key_dict: UserAPIKeyAuth,
) -> None:
verbose_logger.info(
f"Storing LiteLLM Managed {file_purpose} object with id={unified_object_id} in cache"
)
litellm_managed_object = LiteLLM_ManagedObjectTable(
unified_object_id=unified_object_id,
model_object_id=model_object_id,
file_purpose=file_purpose,
file_object=file_object,
)
await self.internal_usage_cache.async_set_cache(
key=unified_object_id,
value=litellm_managed_object.model_dump(),
litellm_parent_otel_span=litellm_parent_otel_span,
)
await self.prisma_client.db.litellm_managedobjecttable.create(
data={
"unified_object_id": unified_object_id,
"file_object": file_object.model_dump_json(),
"model_object_id": model_object_id,
"file_purpose": file_purpose,
"created_by": user_api_key_dict.user_id,
"updated_by": user_api_key_dict.user_id,
}
)
@ -121,6 +178,71 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
)
return initial_value.file_object
async def can_user_call_unified_file_id(
self, unified_file_id: str, user_api_key_dict: UserAPIKeyAuth
) -> bool:
## check if the user has access to the unified file id
user_id = user_api_key_dict.user_id
managed_file = await self.prisma_client.db.litellm_managedfiletable.find_first(
where={"unified_file_id": unified_file_id}
)
if managed_file:
return managed_file.created_by == user_id
return False
async def can_user_call_unified_object_id(
self, unified_object_id: str, user_api_key_dict: UserAPIKeyAuth
) -> bool:
## check if the user has access to the unified object id
## check if the user has access to the unified object id
user_id = user_api_key_dict.user_id
managed_object = (
await self.prisma_client.db.litellm_managedobjecttable.find_first(
where={"unified_object_id": unified_object_id}
)
)
if managed_object:
return managed_object.created_by == user_id
return False
async def get_user_created_file_ids(
self, user_api_key_dict: UserAPIKeyAuth, model_object_ids: List[str]
) -> List[OpenAIFileObject]:
"""
Get all file ids created by the user for a list of model object ids
Returns:
- List of OpenAIFileObject's
"""
file_ids = await self.prisma_client.db.litellm_managedfiletable.find_many(
where={
"created_by": user_api_key_dict.user_id,
"flat_model_file_ids": {"hasSome": model_object_ids},
}
)
return [OpenAIFileObject(**file_object.file_object) for file_object in file_ids]
async def check_managed_file_id_access(
self, data: Dict, user_api_key_dict: UserAPIKeyAuth
) -> bool:
retrieve_file_id = cast(Optional[str], data.get("file_id"))
potential_file_id = (
_is_base64_encoded_unified_file_id(retrieve_file_id)
if retrieve_file_id
else False
)
if potential_file_id and retrieve_file_id:
if await self.can_user_call_unified_file_id(
retrieve_file_id, user_api_key_dict
):
return True
else:
raise HTTPException(
status_code=403,
detail=f"User {user_api_key_dict.user_id} does not have access to the file {retrieve_file_id}",
)
return False
async def async_pre_call_hook(
self,
user_api_key_dict: UserAPIKeyAuth,
@ -137,13 +259,28 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
"rerank",
"acreate_batch",
"aretrieve_batch",
"acreate_file",
"afile_list",
"afile_delete",
"afile_content",
"acreate_fine_tuning_job",
"aretrieve_fine_tuning_job",
"alist_fine_tuning_jobs",
"acancel_fine_tuning_job",
],
) -> Union[Exception, str, Dict, None]:
"""
- Detect litellm_proxy/ file_id
- add dictionary of mappings of litellm_proxy/ file_id -> provider_file_id => {litellm_proxy/file_id: {"model_id": id, "file_id": provider_file_id}}
"""
### HANDLE FILE ACCESS ### - ensure user has access to the file
if (
call_type == CallTypes.afile_content.value
or call_type == CallTypes.afile_delete.value
):
await self.check_managed_file_id_access(data, user_api_key_dict)
### HANDLE TRANSFORMATIONS ###
if call_type == CallTypes.completion.value:
messages = data.get("messages")
if messages:
@ -176,25 +313,56 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
)
data["model_file_id_mapping"] = model_file_id_mapping
elif call_type == CallTypes.aretrieve_batch.value:
retrieve_batch_id = cast(Optional[str], data.get("batch_id"))
potential_batch_id = (
_is_base64_encoded_unified_file_id(retrieve_batch_id)
if retrieve_batch_id
elif (
call_type == CallTypes.aretrieve_batch.value
or call_type == CallTypes.acancel_fine_tuning_job.value
or call_type == CallTypes.aretrieve_fine_tuning_job.value
):
accessor_key: Optional[str] = None
retrieve_object_id: Optional[str] = None
if call_type == CallTypes.aretrieve_batch.value:
accessor_key = "batch_id"
elif (
call_type == CallTypes.acancel_fine_tuning_job.value
or call_type == CallTypes.aretrieve_fine_tuning_job.value
):
accessor_key = "fine_tuning_job_id"
if accessor_key:
retrieve_object_id = cast(Optional[str], data.get(accessor_key))
potential_llm_object_id = (
_is_base64_encoded_unified_file_id(retrieve_object_id)
if retrieve_object_id
else False
)
if potential_batch_id:
if potential_llm_object_id and retrieve_object_id:
## VALIDATE USER HAS ACCESS TO THE OBJECT ##
if not await self.can_user_call_unified_object_id(
retrieve_object_id, user_api_key_dict
):
raise HTTPException(
status_code=403,
detail=f"User {user_api_key_dict.user_id} does not have access to the object {retrieve_object_id}",
)
## for managed batch id - get the model id
potential_model_id = self.get_model_id_from_unified_batch_id(
potential_batch_id
potential_llm_object_id
)
if potential_model_id is None:
raise Exception(
f"LiteLLM Managed Batch ID with id={retrieve_batch_id} is invalid - does not contain encoded model_id."
f"LiteLLM Managed {accessor_key} with id={retrieve_object_id} is invalid - does not contain encoded model_id."
)
data["model"] = potential_model_id
data["batch_id"] = self.get_batch_id_from_unified_batch_id(
potential_batch_id
data[accessor_key] = self.get_batch_id_from_unified_batch_id(
potential_llm_object_id
)
elif call_type == CallTypes.acreate_fine_tuning_job.value:
input_file_id = cast(Optional[str], data.get("training_file"))
if input_file_id:
model_file_id_mapping = await self.get_model_file_id_mapping(
[input_file_id], user_api_key_dict.parent_otel_span
)
return data
@ -205,8 +373,16 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
"""
Allow modifying the request just before it's sent to the deployment.
"""
accessor_key: Optional[str] = None
if call_type and call_type == CallTypes.acreate_batch:
input_file_id = cast(Optional[str], kwargs.get("input_file_id"))
accessor_key = "input_file_id"
elif call_type and call_type == CallTypes.acreate_fine_tuning_job:
accessor_key = "training_file"
else:
return kwargs
if accessor_key:
input_file_id = cast(Optional[str], kwargs.get(accessor_key))
model_file_id_mapping = cast(
Optional[Dict[str, Dict[str, str]]], kwargs.get("model_file_id_mapping")
)
@ -217,7 +393,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
model_id, None
)
if mapped_file_id:
kwargs["input_file_id"] = mapped_file_id
kwargs[accessor_key] = mapped_file_id
return kwargs
def get_file_ids_from_messages(self, messages: List[AllMessageValues]) -> List[str]:
@ -305,6 +482,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
llm_router: Router,
target_model_names_list: List[str],
litellm_parent_otel_span: Span,
user_api_key_dict: UserAPIKeyAuth,
) -> OpenAIFileObject:
responses = await self.create_file_for_each_model(
llm_router=llm_router,
@ -337,6 +515,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
file_object=response,
litellm_parent_otel_span=litellm_parent_otel_span,
model_mappings=model_mappings,
user_api_key_dict=user_api_key_dict,
)
return response
@ -383,6 +562,20 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
return response
def get_unified_generic_response_id(
self, model_id: str, generic_response_id: str
) -> str:
unified_generic_response_id = (
SpecialEnums.LITELLM_MANAGED_GENERIC_RESPONSE_COMPLETE_STR.value.format(
model_id, generic_response_id
)
)
return (
base64.urlsafe_b64encode(unified_generic_response_id.encode())
.decode()
.rstrip("=")
)
def get_unified_batch_id(self, batch_id: str, model_id: str) -> str:
unified_batch_id = SpecialEnums.LITELLM_MANAGED_BATCH_COMPLETE_STR.value.format(
model_id, batch_id
@ -427,11 +620,15 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
def get_batch_id_from_unified_batch_id(self, file_id: str) -> str:
## use regex to get the batch_id from the file_id
return file_id.split("llm_batch_id:")[1].split(",")[0]
if "llm_batch_id" in file_id:
return file_id.split("llm_batch_id:")[1].split(",")[0]
else:
return file_id.split("generic_response_id:")[1].split(",")[0]
async def async_post_call_success_hook(
self, data: Dict, user_api_key_dict: UserAPIKeyAuth, response: LLMResponseTypes
) -> Any:
print(f"response: {response}, type: {type(response)}")
if isinstance(response, LiteLLMBatch):
## Check if unified_file_id is in the response
unified_file_id = response._hidden_params.get(
@ -442,6 +639,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
) # managed batch id
model_id = cast(Optional[str], response._hidden_params.get("model_id"))
model_name = cast(Optional[str], response._hidden_params.get("model_name"))
original_response_id = response.id
if (unified_batch_id or unified_file_id) and model_id:
response.id = self.get_unified_batch_id(
batch_id=response.id, model_id=model_id
@ -455,10 +653,67 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
model_id=model_id,
model_name=model_name,
)
return await super().async_post_call_success_hook(
data, user_api_key_dict, response
)
asyncio.create_task(
self.store_unified_object_id(
unified_object_id=response.id,
file_object=response,
litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
model_object_id=original_response_id,
file_purpose="batch",
user_api_key_dict=user_api_key_dict,
)
)
elif isinstance(response, LiteLLMFineTuningJob):
## Check if unified_file_id is in the response
unified_file_id = response._hidden_params.get(
"unified_file_id"
) # managed file id
unified_finetuning_job_id = response._hidden_params.get(
"unified_finetuning_job_id"
) # managed finetuning job id
model_id = cast(Optional[str], response._hidden_params.get("model_id"))
model_name = cast(Optional[str], response._hidden_params.get("model_name"))
original_response_id = response.id
if (unified_file_id or unified_finetuning_job_id) and model_id:
response.id = self.get_unified_generic_response_id(
model_id=model_id, generic_response_id=response.id
)
asyncio.create_task(
self.store_unified_object_id(
unified_object_id=response.id,
file_object=response,
litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
model_object_id=original_response_id,
file_purpose="fine-tune",
user_api_key_dict=user_api_key_dict,
)
)
elif isinstance(response, AsyncCursorPage):
"""
For listing files, filter for the ones created by the user
"""
print("INSIDE ASYNC CURSOR PAGE BLOCK")
## check if file object
if hasattr(response, "data") and isinstance(response.data, list):
if all(
isinstance(file_object, FileObject) for file_object in response.data
):
## Get all file id's
## Check which file id's were created by the user
## Filter the response to only include the files created by the user
## Return the filtered response
file_ids = [
file_object.id
for file_object in cast(List[FileObject], response.data) # type: ignore
]
user_created_file_ids = await self.get_user_created_file_ids(
user_api_key_dict, file_ids
)
## Filter the response to only include the files created by the user
response.data = user_created_file_ids # type: ignore
return response
return response
return response
async def afile_retrieve(
self, file_id: str, litellm_parent_otel_span: Optional[Span]
@ -477,6 +732,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
litellm_parent_otel_span: Optional[Span],
**data: Dict,
) -> List[OpenAIFileObject]:
"""Handled in files_endpoints.py"""
return []
async def afile_delete(

2
enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py
View file

@ -189,7 +189,7 @@ class BaseEmailLogger(CustomLogger):
# get the latest invitation link for the user
invitation_rows = await prisma_client.db.litellm_invitationlink.find_many(
where={"user_id": user_id},
orderBy={"created_at": "desc"},
order={"created_at": "desc"},
)
if len(invitation_rows) > 0:
invitation_row = invitation_rows[0]

30
enterprise/litellm_enterprise/proxy/auth/user_api_key_auth.py Normal file
View file

@ -0,0 +1,30 @@
from typing import Any, Optional
from fastapi import Request
from litellm._logging import verbose_proxy_logger
from litellm.proxy._types import UserAPIKeyAuth
async def enterprise_custom_auth(
request: Request, api_key: str, user_custom_auth: Any
) -> Optional[UserAPIKeyAuth]:
from litellm_enterprise.proxy.proxy_server import custom_auth_settings
if custom_auth_settings is None:
return None
if custom_auth_settings["mode"] == "on":
return await user_custom_auth(request, api_key)
elif custom_auth_settings["mode"] == "off":
return None
elif custom_auth_settings["mode"] == "auto":
try:
return await user_custom_auth(request, api_key)
except Exception as e:
verbose_proxy_logger.debug(
f"Error in custom auth, checking litellm auth: {e}"
)
return None
else:
raise ValueError(f"Invalid mode: {custom_auth_settings['mode']}")

25
enterprise/litellm_enterprise/proxy/proxy_server.py Normal file
View file

@ -0,0 +1,25 @@
from typing import Optional
from litellm_enterprise.types.proxy.proxy_server import CustomAuthSettings
custom_auth_settings: Optional[CustomAuthSettings] = None
class EnterpriseProxyConfig:
async def load_custom_auth_settings(
self, general_settings: dict
) -> CustomAuthSettings:
print(f"General settings: {general_settings}")
custom_auth_settings = general_settings.get("custom_auth_settings", None)
print(f"Custom auth settings: {custom_auth_settings}")
if custom_auth_settings is not None:
custom_auth_settings = CustomAuthSettings(
mode=custom_auth_settings.get("mode"),
)
print(f"Custom auth settings: {custom_auth_settings}")
return custom_auth_settings
async def load_enterprise_config(self, general_settings: dict) -> None:
global custom_auth_settings
custom_auth_settings = await self.load_custom_auth_settings(general_settings)
return None

5
enterprise/litellm_enterprise/types/proxy/proxy_server.py Normal file
View file

@ -0,0 +1,5 @@
from typing import Literal, TypedDict
class CustomAuthSettings(TypedDict):
mode: Literal["on", "off", "auto"]

32
litellm-proxy-extras/litellm_proxy_extras/migrations/20250522223020_managed_object_table/migration.sql Normal file
View file

@ -0,0 +1,32 @@
-- AlterTable
ALTER TABLE "LiteLLM_ManagedFileTable" ADD COLUMN "created_by" TEXT,
ADD COLUMN "flat_model_file_ids" TEXT[] DEFAULT ARRAY[]::TEXT[],
ADD COLUMN "updated_by" TEXT;
-- CreateTable
CREATE TABLE "LiteLLM_ManagedObjectTable" (
"id" TEXT NOT NULL,
"unified_object_id" TEXT NOT NULL,
"model_object_id" TEXT NOT NULL,
"file_object" JSONB NOT NULL,
"file_purpose" TEXT NOT NULL,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"created_by" TEXT,
"updated_at" TIMESTAMP(3) NOT NULL,
"updated_by" TEXT,
CONSTRAINT "LiteLLM_ManagedObjectTable_pkey" PRIMARY KEY ("id")
);
-- CreateIndex
CREATE UNIQUE INDEX "LiteLLM_ManagedObjectTable_unified_object_id_key" ON "LiteLLM_ManagedObjectTable"("unified_object_id");
-- CreateIndex
CREATE UNIQUE INDEX "LiteLLM_ManagedObjectTable_model_object_id_key" ON "LiteLLM_ManagedObjectTable"("model_object_id");
-- CreateIndex
CREATE INDEX "LiteLLM_ManagedObjectTable_unified_object_id_idx" ON "LiteLLM_ManagedObjectTable"("unified_object_id");
-- CreateIndex
CREATE INDEX "LiteLLM_ManagedObjectTable_model_object_id_idx" ON "LiteLLM_ManagedObjectTable"("model_object_id");

19
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -453,13 +453,30 @@ model LiteLLM_ManagedFileTable {
id String @id @default(uuid())
unified_file_id String @unique // The base64 encoded unified file ID
file_object Json // Stores the OpenAIFileObject
model_mappings Json // Stores the mapping of model_id -> provider_file_id
model_mappings Json
flat_model_file_ids String[] @default([]) // Flat list of model file id's - for faster querying of model id -> unified file id
created_at DateTime @default(now())
created_by String?
updated_at DateTime @updatedAt
updated_by String?
@@index([unified_file_id])
}
model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use the
id String @id @default(uuid())
unified_object_id String @unique // The base64 encoded unified file ID
model_object_id String @unique // the id returned by the backend API provider
file_object Json // Stores the OpenAIFileObject
file_purpose String // either 'batch' or 'fine-tune'
created_at DateTime @default(now())
created_by String?
updated_at DateTime @updatedAt
updated_by String?
@@index([unified_object_id])
@@index([model_object_id])
}
model LiteLLM_ManagedVectorStoresTable {
vector_store_id String @id

3
litellm/__init__.py
View file

@ -126,6 +126,7 @@ _known_custom_logger_compatible_callbacks: List = list(
callbacks: List[
Union[Callable, _custom_logger_compatible_callbacks_literal, CustomLogger]
] = []
initialized_langfuse_clients: int = 0
langfuse_default_tags: Optional[List[str]] = None
langsmith_batch_size: Optional[int] = None
prometheus_initialize_budget_metrics: Optional[bool] = False
@ -354,6 +355,8 @@ project = None
config_path = None
vertex_ai_safety_settings: Optional[dict] = None
BEDROCK_CONVERSE_MODELS = [
"anthropic.claude-opus-4-20250514-v1:0",
"anthropic.claude-sonnet-4-20250514-v1:0",
"anthropic.claude-3-7-sonnet-20250219-v1:0",
"anthropic.claude-3-5-haiku-20241022-v1:0",
"anthropic.claude-3-5-sonnet-20241022-v2:0",

4
litellm/constants.py
View file

@ -141,6 +141,7 @@ DEFAULT_MAX_TOKENS_FOR_TRITON = int(os.getenv("DEFAULT_MAX_TOKENS_FOR_TRITON", 2
#### Networking settings ####
request_timeout: float = float(os.getenv("REQUEST_TIMEOUT", 6000)) # time in seconds
STREAM_SSE_DONE_STRING: str = "[DONE]"
STREAM_SSE_DATA_PREFIX: str = "data: "
### SPEND TRACKING ###
DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND = float(
os.getenv("DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND", 0.001400)
@ -152,6 +153,9 @@ FIREWORKS_AI_16_B = int(os.getenv("FIREWORKS_AI_16_B", 16))
FIREWORKS_AI_80_B = int(os.getenv("FIREWORKS_AI_80_B", 80))
#### Logging callback constants ####
REDACTED_BY_LITELM_STRING = "REDACTED_BY_LITELM"
MAX_LANGFUSE_INITIALIZED_CLIENTS = int(
os.getenv("MAX_LANGFUSE_INITIALIZED_CLIENTS", 20)
)
############### LLM Provider Constants ###############
### ANTHROPIC CONSTANTS ###

23
litellm/fine_tuning/main.py
View file

@ -22,12 +22,9 @@ from litellm.llms.azure.fine_tuning.handler import AzureOpenAIFineTuningAPI
from litellm.llms.openai.fine_tuning.handler import OpenAIFineTuningAPI
from litellm.llms.vertex_ai.fine_tuning.handler import VertexFineTuningAPI
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import (
FineTuningJob,
FineTuningJobCreate,
Hyperparameters,
)
from litellm.types.llms.openai import FineTuningJobCreate, Hyperparameters
from litellm.types.router import *
from litellm.types.utils import LiteLLMFineTuningJob
from litellm.utils import client, supports_httpx_timeout
####### ENVIRONMENT VARIABLES ###################
@ -50,7 +47,7 @@ async def acreate_fine_tuning_job(
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
**kwargs,
) -> FineTuningJob:
) -> LiteLLMFineTuningJob:
"""
Async: Creates and executes a batch from an uploaded file of request
@ -104,7 +101,7 @@ def create_fine_tuning_job(
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
**kwargs,
) -> Union[FineTuningJob, Coroutine[Any, Any, FineTuningJob]]:
) -> Union[LiteLLMFineTuningJob, Coroutine[Any, Any, LiteLLMFineTuningJob]]:
"""
Creates a fine-tuning job which begins the process of creating a new model from a given dataset.
@ -288,13 +285,14 @@ def create_fine_tuning_job(
raise e
@client
async def acancel_fine_tuning_job(
fine_tuning_job_id: str,
custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
**kwargs,
) -> FineTuningJob:
) -> LiteLLMFineTuningJob:
"""
Async: Immediately cancel a fine-tune job.
"""
@ -325,13 +323,14 @@ async def acancel_fine_tuning_job(
raise e
@client
def cancel_fine_tuning_job(
fine_tuning_job_id: str,
custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
**kwargs,
) -> Union[FineTuningJob, Coroutine[Any, Any, FineTuningJob]]:
) -> Union[LiteLLMFineTuningJob, Coroutine[Any, Any, LiteLLMFineTuningJob]]:
"""
Immediately cancel a fine-tune job.
@ -609,13 +608,14 @@ def list_fine_tuning_jobs(
raise e
@client
async def aretrieve_fine_tuning_job(
fine_tuning_job_id: str,
custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
**kwargs,
) -> FineTuningJob:
) -> LiteLLMFineTuningJob:
"""
Async: Get info about a fine-tuning job.
"""
@ -646,13 +646,14 @@ async def aretrieve_fine_tuning_job(
raise e
@client
def retrieve_fine_tuning_job(
fine_tuning_job_id: str,
custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
**kwargs,
) -> Union[FineTuningJob, Coroutine[Any, Any, FineTuningJob]]:
) -> Union[LiteLLMFineTuningJob, Coroutine[Any, Any, LiteLLMFineTuningJob]]:
"""
Get info about a fine-tuning job.
"""

17
litellm/integrations/SlackAlerting/utils.py
View file

@ -5,6 +5,7 @@ Utils used for slack alerting
import asyncio
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
import litellm
from litellm.proxy._types import AlertType
from litellm.secret_managers.main import get_secret
@ -69,7 +70,12 @@ async def _add_langfuse_trace_id_to_alert(
-> trace_id
-> litellm_call_id
"""
# do nothing for now
if "langfuse" not in litellm.logging_callback_manager._get_all_callbacks():
return None
#########################################################
# Only run if langfuse is added as a callback
#########################################################
if (
request_data is not None
and request_data.get("litellm_logging_obj", None) is not None
@ -82,11 +88,12 @@ async def _add_langfuse_trace_id_to_alert(
if trace_id is not None:
break
await asyncio.sleep(3) # wait 3s before retrying for trace id
_langfuse_object = litellm_logging_obj._get_callback_object(
#########################################################
langfuse_object = litellm_logging_obj._get_callback_object(
service_name="langfuse"
)
if _langfuse_object is not None:
base_url = _langfuse_object.Langfuse.base_url
if langfuse_object is not None:
base_url = langfuse_object.Langfuse.base_url
return f"{base_url}/trace/{trace_id}"
return None

9
litellm/integrations/anthropic_cache_control_hook.py
View file

@ -28,6 +28,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
prompt_id: Optional[str],
prompt_variables: Optional[dict],
dynamic_callback_params: StandardCallbackDynamicParams,
prompt_label: Optional[str] = None,
) -> Tuple[str, List[AllMessageValues], dict]:
"""
Apply cache control directives based on specified injection points.
@ -79,10 +80,10 @@ class AnthropicCacheControlHook(CustomPromptManagement):
# Case 1: Target by specific index
if targetted_index is not None:
if 0 <= targetted_index < len(messages):
messages[targetted_index] = (
AnthropicCacheControlHook._safe_insert_cache_control_in_message(
messages[targetted_index], control
)
messages[
targetted_index
] = AnthropicCacheControlHook._safe_insert_cache_control_in_message(
messages[targetted_index], control
)
# Case 2: Target by role
elif targetted_role is not None:

2
litellm/integrations/custom_logger.py
View file

@ -87,6 +87,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
dynamic_callback_params: StandardCallbackDynamicParams,
litellm_logging_obj: LiteLLMLoggingObj,
tools: Optional[List[Dict]] = None,
prompt_label: Optional[str] = None,
) -> Tuple[str, List[AllMessageValues], dict]:
"""
Returns:
@ -104,6 +105,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
prompt_id: Optional[str],
prompt_variables: Optional[dict],
dynamic_callback_params: StandardCallbackDynamicParams,
prompt_label: Optional[str] = None,
) -> Tuple[str, List[AllMessageValues], dict]:
"""
Returns:

2
litellm/integrations/custom_prompt_management.py
View file

@ -18,6 +18,7 @@ class CustomPromptManagement(CustomLogger, PromptManagementBase):
prompt_id: Optional[str],
prompt_variables: Optional[dict],
dynamic_callback_params: StandardCallbackDynamicParams,
prompt_label: Optional[str] = None,
) -> Tuple[str, List[AllMessageValues], dict]:
"""
Returns:
@ -43,6 +44,7 @@ class CustomPromptManagement(CustomLogger, PromptManagementBase):
prompt_id: str,
prompt_variables: Optional[dict],
dynamic_callback_params: StandardCallbackDynamicParams,
prompt_label: Optional[str] = None,
) -> PromptManagementClient:
raise NotImplementedError(
"Custom prompt management does not support compile prompt helper"

7
litellm/integrations/humanloop.py
View file

@ -155,11 +155,8 @@ class HumanloopLogger(CustomLogger):
prompt_id: Optional[str],
prompt_variables: Optional[dict],
dynamic_callback_params: StandardCallbackDynamicParams,
) -> Tuple[
str,
List[AllMessageValues],
dict,
]:
prompt_label: Optional[str] = None,
) -> Tuple[str, List[AllMessageValues], dict,]:
humanloop_api_key = dynamic_callback_params.get(
"humanloop_api_key"
) or get_secret_str("HUMANLOOP_API_KEY")

25
litellm/integrations/langfuse/langfuse.py
View file

@ -10,6 +10,7 @@ from packaging.version import Version
import litellm
from litellm._logging import verbose_logger
from litellm.constants import MAX_LANGFUSE_INITIALIZED_CLIENTS
from litellm.litellm_core_utils.redact_messages import redact_user_api_key_info
from litellm.llms.custom_httpx.http_handler import _get_httpx_client
from litellm.secret_managers.main import str_to_bool
@ -27,12 +28,13 @@ from litellm.types.utils import (
)
if TYPE_CHECKING:
from langfuse.client import StatefulTraceClient
from langfuse.client import Langfuse, StatefulTraceClient
from litellm.litellm_core_utils.litellm_logging import DynamicLoggingCache
else:
DynamicLoggingCache = Any
StatefulTraceClient = Any
Langfuse = Any
class LangFuseLogger:
@ -84,8 +86,7 @@ class LangFuseLogger:
if Version(self.langfuse_sdk_version) >= Version("2.6.0"):
parameters["sdk_integration"] = "litellm"
self.Langfuse = Langfuse(**parameters)
self.Langfuse: Langfuse = self.safe_init_langfuse_client(parameters)
# set the current langfuse project id in the environ
# this is used by Alerting to link to the correct project
@ -124,6 +125,24 @@ class LangFuseLogger:
else:
self.upstream_langfuse = None
def safe_init_langfuse_client(self, parameters: dict) -> Langfuse:
"""
Safely init a langfuse client if the number of initialized clients is less than the max
Note:
- Langfuse initializes 1 thread everytime a client is initialized.
- We've had an incident in the past where we reached 100% cpu utilization because Langfuse was initialized several times.
"""
from langfuse import Langfuse
if litellm.initialized_langfuse_clients >= MAX_LANGFUSE_INITIALIZED_CLIENTS:
raise Exception(
f"Max langfuse clients reached: {litellm.initialized_langfuse_clients} is greater than {MAX_LANGFUSE_INITIALIZED_CLIENTS}"
)
langfuse_client = Langfuse(**parameters)
litellm.initialized_langfuse_clients += 1
return langfuse_client
@staticmethod
def add_metadata_from_header(litellm_params: dict, metadata: dict) -> dict:
"""

20
litellm/integrations/langfuse/langfuse_prompt_management.py
View file

@ -130,9 +130,12 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge
return "langfuse"
def _get_prompt_from_id(
self, langfuse_prompt_id: str, langfuse_client: LangfuseClass
self,
langfuse_prompt_id: str,
langfuse_client: LangfuseClass,
prompt_label: Optional[str] = None,
) -> PROMPT_CLIENT:
return langfuse_client.get_prompt(langfuse_prompt_id)
return langfuse_client.get_prompt(langfuse_prompt_id, label=prompt_label)
def _compile_prompt(
self,
@ -176,11 +179,8 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge
dynamic_callback_params: StandardCallbackDynamicParams,
litellm_logging_obj: LiteLLMLoggingObj,
tools: Optional[List[Dict]] = None,
) -> Tuple[
str,
List[AllMessageValues],
dict,
]:
prompt_label: Optional[str] = None,
) -> Tuple[str, List[AllMessageValues], dict,]:
return self.get_chat_completion_prompt(
model,
messages,
@ -188,6 +188,7 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge
prompt_id,
prompt_variables,
dynamic_callback_params,
prompt_label=prompt_label,
)
def should_run_prompt_management(
@ -211,6 +212,7 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge
prompt_id: str,
prompt_variables: Optional[dict],
dynamic_callback_params: StandardCallbackDynamicParams,
prompt_label: Optional[str] = None,
) -> PromptManagementClient:
langfuse_client = langfuse_client_init(
langfuse_public_key=dynamic_callback_params.get("langfuse_public_key"),
@ -219,7 +221,9 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge
langfuse_host=dynamic_callback_params.get("langfuse_host"),
)
langfuse_prompt_client = self._get_prompt_from_id(
langfuse_prompt_id=prompt_id, langfuse_client=langfuse_client
langfuse_prompt_id=prompt_id,
langfuse_client=langfuse_client,
prompt_label=prompt_label,
)
## SET PROMPT

5
litellm/integrations/prompt_management_base.py
View file

@ -33,6 +33,7 @@ class PromptManagementBase(ABC):
prompt_id: str,
prompt_variables: Optional[dict],
dynamic_callback_params: StandardCallbackDynamicParams,
prompt_label: Optional[str] = None,
) -> PromptManagementClient:
pass
@ -49,11 +50,13 @@ class PromptManagementBase(ABC):
prompt_variables: Optional[dict],
client_messages: List[AllMessageValues],
dynamic_callback_params: StandardCallbackDynamicParams,
prompt_label: Optional[str] = None,
) -> PromptManagementClient:
compiled_prompt_client = self._compile_prompt_helper(
prompt_id=prompt_id,
prompt_variables=prompt_variables,
dynamic_callback_params=dynamic_callback_params,
prompt_label=prompt_label,
)
try:
@ -82,6 +85,7 @@ class PromptManagementBase(ABC):
prompt_id: Optional[str],
prompt_variables: Optional[dict],
dynamic_callback_params: StandardCallbackDynamicParams,
prompt_label: Optional[str] = None,
) -> Tuple[str, List[AllMessageValues], dict]:
if prompt_id is None:
raise ValueError("prompt_id is required for Prompt Management Base class")
@ -95,6 +99,7 @@ class PromptManagementBase(ABC):
prompt_variables=prompt_variables,
client_messages=messages,
dynamic_callback_params=dynamic_callback_params,
prompt_label=prompt_label,
)
completed_messages = prompt_template["completed_messages"] or messages

22
litellm/integrations/vector_stores/bedrock_vector_store.py
View file

@ -75,6 +75,7 @@ class BedrockVectorStore(BaseVectorStore, BaseAWSLLM):
dynamic_callback_params: StandardCallbackDynamicParams,
litellm_logging_obj: LiteLLMLoggingObj,
tools: Optional[List[Dict]] = None,
prompt_label: Optional[str] = None,
) -> Tuple[str, List[AllMessageValues], dict]:
"""
Retrieves the context from the Bedrock Knowledge Base and appends it to the messages.
@ -99,10 +100,11 @@ class BedrockVectorStore(BaseVectorStore, BaseAWSLLM):
f"Bedrock Knowledge Base Response: {bedrock_kb_response}"
)
context_message, context_string = (
self.get_chat_completion_message_from_bedrock_kb_response(
bedrock_kb_response
)
(
context_message,
context_string,
) = self.get_chat_completion_message_from_bedrock_kb_response(
bedrock_kb_response
)
if context_message is not None:
messages.append(context_message)
@ -126,9 +128,9 @@ class BedrockVectorStore(BaseVectorStore, BaseAWSLLM):
)
)
litellm_logging_obj.model_call_details["vector_store_request_metadata"] = (
vector_store_request_metadata
)
litellm_logging_obj.model_call_details[
"vector_store_request_metadata"
] = vector_store_request_metadata
return model, messages, non_default_params
@ -140,9 +142,9 @@ class BedrockVectorStore(BaseVectorStore, BaseAWSLLM):
"""
Transform a BedrockKBResponse to a VectorStoreSearchResponse
"""
retrieval_results: Optional[List[BedrockKBRetrievalResult]] = (
bedrock_kb_response.get("retrievalResults", None)
)
retrieval_results: Optional[
List[BedrockKBRetrievalResult]
] = bedrock_kb_response.get("retrievalResults", None)
vector_store_search_response: VectorStoreSearchResponse = (
VectorStoreSearchResponse(search_query=query, data=[])
)

4
litellm/litellm_core_utils/litellm_logging.py
View file

@ -539,6 +539,7 @@ class Logging(LiteLLMLoggingBaseClass):
prompt_id: Optional[str],
prompt_variables: Optional[dict],
prompt_management_logger: Optional[CustomLogger] = None,
prompt_label: Optional[str] = None,
) -> Tuple[str, List[AllMessageValues], dict]:
custom_logger = (
prompt_management_logger
@ -559,6 +560,7 @@ class Logging(LiteLLMLoggingBaseClass):
prompt_id=prompt_id,
prompt_variables=prompt_variables,
dynamic_callback_params=self.standard_callback_dynamic_params,
prompt_label=prompt_label,
)
self.messages = messages
return model, messages, non_default_params
@ -572,6 +574,7 @@ class Logging(LiteLLMLoggingBaseClass):
prompt_variables: Optional[dict],
prompt_management_logger: Optional[CustomLogger] = None,
tools: Optional[List[Dict]] = None,
prompt_label: Optional[str] = None,
) -> Tuple[str, List[AllMessageValues], dict]:
custom_logger = (
prompt_management_logger
@ -594,6 +597,7 @@ class Logging(LiteLLMLoggingBaseClass):
dynamic_callback_params=self.standard_callback_dynamic_params,
litellm_logging_obj=self,
tools=tools,
prompt_label=prompt_label,
)
self.messages = messages
return model, messages, non_default_params

9
litellm/litellm_core_utils/logging_callback_manager.py
View file

@ -266,3 +266,12 @@ class LoggingCallbackManager:
if isinstance(callback, callback_type) and callback not in all_callbacks:
all_callbacks.append(callback)
return all_callbacks
def callback_is_active(self, callback_type: Type[CustomLogger]) -> bool:
"""
Returns True if any of the active callbacks are of the given type
"""
return any(
isinstance(callback, callback_type)
for callback in self._get_all_callbacks()
)

9
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -573,3 +573,12 @@ def get_tool_call_names(tools: List[ChatCompletionToolParam]) -> List[str]:
if tool_call_name:
tool_call_names.append(tool_call_name)
return tool_call_names
def is_function_call(optional_params: dict) -> bool:
"""
Checks if the optional params contain the function call
"""
if "functions" in optional_params and optional_params.get("functions"):
return True
return False

10
litellm/litellm_core_utils/streaming_handler.py
View file

@ -149,14 +149,14 @@ class CustomStreamWrapper:
)
def check_is_function_call(self, logging_obj) -> bool:
from litellm.litellm_core_utils.prompt_templates.common_utils import (
is_function_call,
)
if hasattr(logging_obj, "optional_params") and isinstance(
logging_obj.optional_params, dict
):
if (
"litellm_param_is_function_call" in logging_obj.optional_params
and logging_obj.optional_params["litellm_param_is_function_call"]
is True
):
if is_function_call(logging_obj.optional_params):
return True
return False

6
litellm/llms/anthropic/chat/transformation.py
View file

@ -49,6 +49,7 @@ from litellm.utils import (
Usage,
add_dummy_tool,
has_tool_call_blocks,
supports_reasoning,
token_counter,
)
@ -121,7 +122,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
"web_search_options",
]
if "claude-3-7-sonnet" in model:
if "claude-3-7-sonnet" in model or supports_reasoning(
model=model,
custom_llm_provider=self.custom_llm_provider,
):
params.append("thinking")
return params

2
litellm/llms/base_llm/files/transformation.py
View file

@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
import httpx
from litellm.proxy._types import UserAPIKeyAuth
from litellm.types.llms.openai import (
AllMessageValues,
CreateFileRequest,
@ -115,6 +116,7 @@ class BaseFileEndpoints(ABC):
llm_router: Router,
target_model_names_list: List[str],
litellm_parent_otel_span: Span,
user_api_key_dict: UserAPIKeyAuth,
) -> OpenAIFileObject:
pass

9
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -45,7 +45,7 @@ from litellm.types.utils import (
PromptTokensDetailsWrapper,
Usage,
)
from litellm.utils import add_dummy_tool, has_tool_call_blocks
from litellm.utils import add_dummy_tool, has_tool_call_blocks, supports_reasoning
from ..common_utils import BedrockError, BedrockModelInfo, get_bedrock_tool_name
@ -146,9 +146,10 @@ class AmazonConverseConfig(BaseConfig):
# only anthropic and mistral support tool choice config. otherwise (E.g. cohere) will fail the call - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ToolChoice.html
supported_params.append("tool_choice")
if (
"claude-3-7" in model
): # [TODO]: move to a 'supports_reasoning_content' param from model cost map
if "claude-3-7" in model or supports_reasoning(
model=model,
custom_llm_provider=self.custom_llm_provider,
):
supported_params.append("thinking")
supported_params.append("reasoning_effort")
return supported_params

4
litellm/llms/bedrock/image/cost_calculator.py
View file

@ -37,5 +37,7 @@ def cost_calculator(
)
output_cost_per_image: float = _model_info.get("output_cost_per_image") or 0.0
num_images: int = len(image_response.data)
num_images: int = 0
if image_response.data:
num_images = len(image_response.data)
return output_cost_per_image * num_images

26
litellm/llms/custom_llm.py
View file

@ -14,7 +14,7 @@ import httpx
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.types.utils import GenericStreamingChunk
from litellm.utils import ImageResponse, ModelResponse
from litellm.utils import ImageResponse, ModelResponse, EmbeddingResponse
from .base import BaseLLM
@ -152,6 +152,30 @@ class CustomLLM(BaseLLM):
) -> ImageResponse:
raise CustomLLMError(status_code=500, message="Not implemented yet!")
def embedding(
self,
model: str,
input: list,
model_response: EmbeddingResponse,
print_verbose: Callable,
logging_obj: Any,
optional_params: dict,
litellm_params=None,
) -> EmbeddingResponse:
raise CustomLLMError(status_code=500, message="Not implemented yet!")
async def aembedding(
self,
model: str,
input: list,
model_response: EmbeddingResponse,
print_verbose: Callable,
logging_obj: Any,
optional_params: dict,
litellm_params=None,
) -> EmbeddingResponse:
raise CustomLLMError(status_code=500, message="Not implemented yet!")
def custom_chat_llm_router(
async_fn: bool, stream: Optional[bool], custom_llm: CustomLLM

4
litellm/llms/ollama/common_utils.py
View file

@ -1,4 +1,4 @@
from typing import Optional, Union
from typing import List, Optional, Union
import httpx
@ -67,7 +67,7 @@ class OllamaModelInfo(BaseLLMModelInfo):
# env var OLLAMA_API_BASE or default
return api_base or get_secret_str("OLLAMA_API_BASE") or "http://localhost:11434"
def get_models(self, api_key=None, api_base: Optional[str] = None) -> list[str]:
def get_models(self, api_key=None, api_base: Optional[str] = None) -> List[str]:
"""
List all models available on the Ollama server via /api/tags endpoint.
"""

37
litellm/llms/openai/fine_tuning/handler.py
View file

@ -1,10 +1,10 @@
from typing import Any, Coroutine, Optional, Union
from typing import Any, Coroutine, Optional, Union, cast
import httpx
from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI
from openai.types.fine_tuning import FineTuningJob
from litellm._logging import verbose_logger
from litellm.types.utils import LiteLLMFineTuningJob
class OpenAIFineTuningAPI:
@ -55,11 +55,12 @@ class OpenAIFineTuningAPI:
self,
create_fine_tuning_job_data: dict,
openai_client: Union[AsyncOpenAI, AsyncAzureOpenAI],
) -> FineTuningJob:
) -> LiteLLMFineTuningJob:
response = await openai_client.fine_tuning.jobs.create(
**create_fine_tuning_job_data
)
return response
return LiteLLMFineTuningJob(**response.model_dump())
def create_fine_tuning_job(
self,
@ -74,7 +75,7 @@ class OpenAIFineTuningAPI:
client: Optional[
Union[OpenAI, AsyncOpenAI, AzureOpenAI, AsyncAzureOpenAI]
] = None,
) -> Union[FineTuningJob, Coroutine[Any, Any, FineTuningJob]]:
) -> Union[LiteLLMFineTuningJob, Coroutine[Any, Any, LiteLLMFineTuningJob]]:
openai_client: Optional[
Union[OpenAI, AsyncOpenAI, AzureOpenAI, AsyncAzureOpenAI]
] = self.get_openai_client(
@ -104,18 +105,20 @@ class OpenAIFineTuningAPI:
verbose_logger.debug(
"creating fine tuning job, args= %s", create_fine_tuning_job_data
)
response = openai_client.fine_tuning.jobs.create(**create_fine_tuning_job_data)
return response
response = cast(OpenAI, openai_client).fine_tuning.jobs.create(
**create_fine_tuning_job_data
)
return LiteLLMFineTuningJob(**response.model_dump())
async def acancel_fine_tuning_job(
self,
fine_tuning_job_id: str,
openai_client: Union[AsyncOpenAI, AsyncAzureOpenAI],
) -> FineTuningJob:
) -> LiteLLMFineTuningJob:
response = await openai_client.fine_tuning.jobs.cancel(
fine_tuning_job_id=fine_tuning_job_id
)
return response
return LiteLLMFineTuningJob(**response.model_dump())
def cancel_fine_tuning_job(
self,
@ -130,7 +133,7 @@ class OpenAIFineTuningAPI:
client: Optional[
Union[OpenAI, AsyncOpenAI, AzureOpenAI, AsyncAzureOpenAI]
] = None,
):
) -> Union[LiteLLMFineTuningJob, Coroutine[Any, Any, LiteLLMFineTuningJob]]:
openai_client: Optional[
Union[OpenAI, AsyncOpenAI, AzureOpenAI, AsyncAzureOpenAI]
] = self.get_openai_client(
@ -158,10 +161,10 @@ class OpenAIFineTuningAPI:
openai_client=openai_client,
)
verbose_logger.debug("canceling fine tuning job, args= %s", fine_tuning_job_id)
response = openai_client.fine_tuning.jobs.cancel(
response = cast(OpenAI, openai_client).fine_tuning.jobs.cancel(
fine_tuning_job_id=fine_tuning_job_id
)
return response
return LiteLLMFineTuningJob(**response.model_dump())
async def alist_fine_tuning_jobs(
self,
@ -222,11 +225,11 @@ class OpenAIFineTuningAPI:
self,
fine_tuning_job_id: str,
openai_client: Union[AsyncOpenAI, AsyncAzureOpenAI],
) -> FineTuningJob:
) -> LiteLLMFineTuningJob:
response = await openai_client.fine_tuning.jobs.retrieve(
fine_tuning_job_id=fine_tuning_job_id
)
return response
return LiteLLMFineTuningJob(**response.model_dump())
def retrieve_fine_tuning_job(
self,
@ -241,7 +244,7 @@ class OpenAIFineTuningAPI:
client: Optional[
Union[OpenAI, AsyncOpenAI, AzureOpenAI, AsyncAzureOpenAI]
] = None,
):
) -> Union[LiteLLMFineTuningJob, Coroutine[Any, Any, LiteLLMFineTuningJob]]:
openai_client: Optional[
Union[OpenAI, AsyncOpenAI, AzureOpenAI, AsyncAzureOpenAI]
] = self.get_openai_client(
@ -269,7 +272,7 @@ class OpenAIFineTuningAPI:
openai_client=openai_client,
)
verbose_logger.debug("retrieving fine tuning job, id= %s", fine_tuning_job_id)
response = openai_client.fine_tuning.jobs.retrieve(
response = cast(OpenAI, openai_client).fine_tuning.jobs.retrieve(
fine_tuning_job_id=fine_tuning_job_id
)
return response
return LiteLLMFineTuningJob(**response.model_dump())

1
litellm/llms/openrouter/chat/transformation.py
View file

@ -120,6 +120,7 @@ class OpenRouterChatCompletionStreamingHandler(BaseModelResponseIterator):
id=chunk["id"],
object="chat.completion.chunk",
created=chunk["created"],
usage=chunk.get("usage"),
model=chunk["model"],
choices=new_choices,
)

10
litellm/llms/vertex_ai/fine_tuning/handler.py
View file

@ -1,10 +1,9 @@
import json
import traceback
from datetime import datetime
from typing import Literal, Optional, Union
from typing import Any, Coroutine, Literal, Optional, Union
import httpx
from openai.types.fine_tuning.fine_tuning_job import FineTuningJob
import litellm
from litellm._logging import verbose_logger
@ -20,6 +19,7 @@ from litellm.types.llms.vertex_ai import (
ResponseSupervisedTuningSpec,
ResponseTuningJob,
)
from litellm.types.utils import LiteLLMFineTuningJob
class VertexFineTuningAPI(VertexLLM):
@ -113,7 +113,7 @@ class VertexFineTuningAPI(VertexLLM):
def convert_vertex_response_to_open_ai_response(
self, response: ResponseTuningJob
) -> FineTuningJob:
) -> LiteLLMFineTuningJob:
status: Literal[
"validating_files", "queued", "running", "succeeded", "failed", "cancelled"
] = "queued"
@ -134,7 +134,7 @@ class VertexFineTuningAPI(VertexLLM):
response.get("supervisedTuningSpec", None) or {}
)
training_uri: str = _supervisedTuningSpec.get("trainingDatasetUri", "") or ""
return FineTuningJob(
return LiteLLMFineTuningJob(
id=response.get("name", "") or "",
created_at=created_at,
fine_tuned_model=response.get("tunedModelDisplayName", ""),
@ -226,7 +226,7 @@ class VertexFineTuningAPI(VertexLLM):
timeout: Union[float, httpx.Timeout],
kwargs: Optional[dict] = None,
original_hyperparameters: Optional[dict] = {},
):
) -> Union[LiteLLMFineTuningJob, Coroutine[Any, Any, LiteLLMFineTuningJob]]:
verbose_logger.debug(
"creating fine tuning job, args= %s", create_fine_tuning_job_data
)

17
litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
View file

@ -455,9 +455,6 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
and value
):
optional_params["tools"] = self._map_function(value=value)
optional_params["litellm_param_is_function_call"] = (
True if param == "functions" else False
)
elif param == "tool_choice" and (
isinstance(value, str) or isinstance(value, dict)
):
@ -880,8 +877,14 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
else:
return "stop"
def _process_candidates(self, _candidates, model_response, litellm_params):
def _process_candidates(
self, _candidates, model_response, standard_optional_params: dict
):
"""Helper method to process candidates and extract metadata"""
from litellm.litellm_core_utils.prompt_templates.common_utils import (
is_function_call,
)
grounding_metadata: List[dict] = []
safety_ratings: List = []
citation_metadata: List = []
@ -918,9 +921,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
functions, tools = self._transform_parts(
parts=candidate["content"]["parts"],
index=candidate.get("index", idx),
is_function_call=litellm_params.get(
"litellm_param_is_function_call"
),
is_function_call=is_function_call(standard_optional_params),
)
if "logprobsResult" in candidate:
@ -1019,7 +1020,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
safety_ratings,
citation_metadata,
) = self._process_candidates(
_candidates, model_response, litellm_params
_candidates, model_response, logging_obj.optional_params
)
usage = self._calculate_usage(completion_response=completion_response)

4
litellm/llms/vertex_ai/image_generation/cost_calculator.py
View file

@ -19,5 +19,7 @@ def cost_calculator(
)
output_cost_per_image: float = _model_info.get("output_cost_per_image") or 0.0
num_images: int = len(image_response.data)
num_images: int = 0
if image_response.data:
num_images = len(image_response.data)
return output_cost_per_image * num_images

35
litellm/main.py
View file

@ -97,6 +97,7 @@ from litellm.utils import (
get_optional_params_image_gen,
get_optional_params_transcription,
get_secret,
get_standard_openai_params,
mock_completion_streaming_obj,
read_config_args,
supports_httpx_timeout,
@ -428,6 +429,7 @@ async def acompletion(
prompt_id=kwargs.get("prompt_id", None),
prompt_variables=kwargs.get("prompt_variables", None),
tools=tools,
prompt_label=kwargs.get("prompt_label", None),
)
#########################################################
@ -983,6 +985,7 @@ def completion( # type: ignore # noqa: PLR0915
assistant_continue_message=assistant_continue_message,
)
######## end of unpacking kwargs ###########
standard_openai_params = get_standard_openai_params(params=args)
non_default_params = get_non_default_completion_params(kwargs=kwargs)
litellm_params = {} # used to prevent unbound var errors
## PROMPT MANAGEMENT HOOKS ##
@ -1001,6 +1004,7 @@ def completion( # type: ignore # noqa: PLR0915
non_default_params=non_default_params,
prompt_id=prompt_id,
prompt_variables=prompt_variables,
prompt_label=kwargs.get("prompt_label", None),
)
try:
@ -1234,10 +1238,13 @@ def completion( # type: ignore # noqa: PLR0915
max_retries=max_retries,
timeout=timeout,
)
logging.update_environment_variables(
cast(LiteLLMLoggingObj, logging).update_environment_variables(
model=model,
user=user,
optional_params=optional_params,
optional_params={
**standard_openai_params,
**non_default_params,
}, # [IMPORTANT] - using standard_openai_params ensures consistent params logged to langfuse for finetuning / eval datasets.
litellm_params=litellm_params,
custom_llm_provider=custom_llm_provider,
)
@ -4020,6 +4027,30 @@ def embedding( # noqa: PLR0915
client=client,
aembedding=aembedding,
)
elif (
custom_llm_provider in litellm._custom_providers
):
custom_handler: Optional[CustomLLM] = None
for item in litellm.custom_provider_map:
if item["provider"] == custom_llm_provider:
custom_handler = item["custom_handler"]
if custom_handler is None:
raise LiteLLMUnknownProvider(
model=model, custom_llm_provider=custom_llm_provider
)
handler_fn = custom_handler.embedding if not aembedding else custom_handler.aembedding
response = handler_fn(
model=model,
input=input,
logging_obj=logging,
optional_params=optional_params,
model_response=EmbeddingResponse(),
print_verbose=print_verbose,
litellm_params=litellm_params
)
else:
raise LiteLLMUnknownProvider(
model=model, custom_llm_provider=custom_llm_provider

278
litellm/model_prices_and_context_window_backup.json
View file

@ -4684,6 +4684,58 @@
"deprecation_date": "2025-06-01",
"supports_tool_choice": true
},
"claude-opus-4-20250514": {
"max_tokens": 32000,
"max_input_tokens": 200000,
"max_output_tokens": 32000,
"input_cost_per_token": 15e-6,
"output_cost_per_token": 75e-6,
"search_context_cost_per_query": {
"search_context_size_low": 1e-2,
"search_context_size_medium": 1e-2,
"search_context_size_high": 1e-2
},
"cache_creation_input_token_cost": 18.75e-6,
"cache_read_input_token_cost": 1.5e-6,
"litellm_provider": "anthropic",
"mode": "chat",
"supports_function_calling": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 159,
"supports_assistant_prefill": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_computer_use": true
},
"claude-sonnet-4-20250514": {
"max_tokens": 64000,
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"input_cost_per_token": 3e-6,
"output_cost_per_token": 15e-6,
"search_context_cost_per_query": {
"search_context_size_low": 1e-2,
"search_context_size_medium": 1e-2,
"search_context_size_high": 1e-2
},
"cache_creation_input_token_cost": 3.75e-6,
"cache_read_input_token_cost": 0.3e-6,
"litellm_provider": "anthropic",
"mode": "chat",
"supports_function_calling": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 159,
"supports_assistant_prefill": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_computer_use": true
},
"claude-3-7-sonnet-latest": {
"supports_computer_use": true,
"max_tokens": 128000,
@ -6753,6 +6805,58 @@
"supports_reasoning": true,
"supports_tool_choice": true
},
"vertex_ai/claude-opus-4@20250514": {
"max_tokens": 32000,
"max_input_tokens": 200000,
"max_output_tokens": 32000,
"input_cost_per_token": 15e-6,
"output_cost_per_token": 75e-6,
"search_context_cost_per_query": {
"search_context_size_low": 1e-2,
"search_context_size_medium": 1e-2,
"search_context_size_high": 1e-2
},
"cache_creation_input_token_cost": 18.75e-6,
"cache_read_input_token_cost": 1.5e-6,
"litellm_provider": "vertex_ai-anthropic_models",
"mode": "chat",
"supports_function_calling": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 159,
"supports_assistant_prefill": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_computer_use": true
},
"vertex_ai/claude-sonnet-4@20250514": {
"max_tokens": 64000,
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"input_cost_per_token": 3e-6,
"output_cost_per_token": 15e-6,
"search_context_cost_per_query": {
"search_context_size_low": 1e-2,
"search_context_size_medium": 1e-2,
"search_context_size_high": 1e-2
},
"cache_creation_input_token_cost": 3.75e-6,
"cache_read_input_token_cost": 0.3e-6,
"litellm_provider": "vertex_ai-anthropic_models",
"mode": "chat",
"supports_function_calling": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 159,
"supports_assistant_prefill": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_computer_use": true
},
"vertex_ai/claude-3-haiku": {
"max_tokens": 4096,
"max_input_tokens": 200000,
@ -9332,6 +9436,58 @@
"supports_pdf_input": true,
"supports_tool_choice": true
},
"anthropic.claude-opus-4-20250514-v1:0": {
"max_tokens": 32000,
"max_input_tokens": 200000,
"max_output_tokens": 32000,
"input_cost_per_token": 15e-6,
"output_cost_per_token": 75e-6,
"search_context_cost_per_query": {
"search_context_size_low": 1e-2,
"search_context_size_medium": 1e-2,
"search_context_size_high": 1e-2
},
"cache_creation_input_token_cost": 18.75e-6,
"cache_read_input_token_cost": 1.5e-6,
"litellm_provider": "bedrock_converse",
"mode": "chat",
"supports_function_calling": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 159,
"supports_assistant_prefill": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_computer_use": true
},
"anthropic.claude-sonnet-4-20250514-v1:0": {
"max_tokens": 64000,
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"input_cost_per_token": 3e-6,
"output_cost_per_token": 15e-6,
"search_context_cost_per_query": {
"search_context_size_low": 1e-2,
"search_context_size_medium": 1e-2,
"search_context_size_high": 1e-2
},
"cache_creation_input_token_cost": 3.75e-6,
"cache_read_input_token_cost": 0.3e-6,
"litellm_provider": "bedrock_converse",
"mode": "chat",
"supports_function_calling": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 159,
"supports_assistant_prefill": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_computer_use": true
},
"anthropic.claude-3-7-sonnet-20250219-v1:0": {
"supports_computer_use": true,
"max_tokens": 8192,
@ -9482,6 +9638,58 @@
"supports_tool_choice": true,
"supports_reasoning": true
},
"us.anthropic.claude-opus-4-20250514-v1:0": {
"max_tokens": 32000,
"max_input_tokens": 200000,
"max_output_tokens": 32000,
"input_cost_per_token": 15e-6,
"output_cost_per_token": 75e-6,
"search_context_cost_per_query": {
"search_context_size_low": 1e-2,
"search_context_size_medium": 1e-2,
"search_context_size_high": 1e-2
},
"cache_creation_input_token_cost": 18.75e-6,
"cache_read_input_token_cost": 1.5e-6,
"litellm_provider": "bedrock_converse",
"mode": "chat",
"supports_function_calling": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 159,
"supports_assistant_prefill": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_computer_use": true
},
"us.anthropic.claude-sonnet-4-20250514-v1:0": {
"max_tokens": 64000,
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"input_cost_per_token": 3e-6,
"output_cost_per_token": 15e-6,
"search_context_cost_per_query": {
"search_context_size_low": 1e-2,
"search_context_size_medium": 1e-2,
"search_context_size_high": 1e-2
},
"cache_creation_input_token_cost": 3.75e-6,
"cache_read_input_token_cost": 0.3e-6,
"litellm_provider": "bedrock_converse",
"mode": "chat",
"supports_function_calling": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 159,
"supports_assistant_prefill": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_computer_use": true
},
"us.anthropic.claude-3-haiku-20240307-v1:0": {
"max_tokens": 4096,
"max_input_tokens": 200000,
@ -9603,6 +9811,58 @@
"supports_pdf_input": true,
"supports_tool_choice": true
},
"eu.anthropic.claude-opus-4-20250514-v1:0": {
"max_tokens": 32000,
"max_input_tokens": 200000,
"max_output_tokens": 32000,
"input_cost_per_token": 15e-6,
"output_cost_per_token": 75e-6,
"search_context_cost_per_query": {
"search_context_size_low": 1e-2,
"search_context_size_medium": 1e-2,
"search_context_size_high": 1e-2
},
"cache_creation_input_token_cost": 18.75e-6,
"cache_read_input_token_cost": 1.5e-6,
"litellm_provider": "bedrock_converse",
"mode": "chat",
"supports_function_calling": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 159,
"supports_assistant_prefill": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_computer_use": true
},
"eu.anthropic.claude-sonnet-4-20250514-v1:0": {
"max_tokens": 64000,
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"input_cost_per_token": 3e-6,
"output_cost_per_token": 15e-6,
"search_context_cost_per_query": {
"search_context_size_low": 1e-2,
"search_context_size_medium": 1e-2,
"search_context_size_high": 1e-2
},
"cache_creation_input_token_cost": 3.75e-6,
"cache_read_input_token_cost": 0.3e-6,
"litellm_provider": "bedrock_converse",
"mode": "chat",
"supports_function_calling": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 159,
"supports_assistant_prefill": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_computer_use": true
},
"eu.anthropic.claude-3-5-haiku-20241022-v1:0": {
"max_tokens": 8192,
"max_input_tokens": 200000,
@ -12297,7 +12557,7 @@
"max_output_tokens": 128000,
"input_cost_per_token": 0.0000025,
"input_dbu_cost_per_token": 0.00003571,
"output_cost_per_token": 0.00017857,
"output_cost_per_token": 0.000017857,
"output_db_cost_per_token": 0.000214286,
"litellm_provider": "databricks",
"mode": "chat",
@ -12350,6 +12610,20 @@
"metadata": {"notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."},
"supports_tool_choice": true
},
"databricks/databricks-llama-4-maverick": {
"max_tokens": 128000,
"max_input_tokens": 128000,
"max_output_tokens": 128000,
"input_cost_per_token": 0.000005,
"input_dbu_cost_per_token": 0.00007143,
"output_cost_per_token": 0.000015,
"output_dbu_cost_per_token": 0.00021429,
"litellm_provider": "databricks",
"mode": "chat",
"source": "https://www.databricks.com/product/pricing/foundation-model-serving",
"metadata": {"notes": "Databricks documentation now provides both DBU costs (_dbu_cost_per_token) and dollar costs(_cost_per_token)."},
"supports_tool_choice": true
},
"databricks/databricks-dbrx-instruct": {
"max_tokens": 32768,
"max_input_tokens": 32768,
@ -12976,4 +13250,4 @@
"litellm_provider": "featherless_ai",
"mode": "chat"
}
}
}

24
litellm/proxy/_new_secret_config.yaml
View file

@ -1,17 +1,17 @@
model_list:
- model_name: "gemini-2.0-flash"
- model_name: "gemini-2.0-flash-gemini"
litellm_params:
model: gemini/gemini-2.0-flash-live-001
- model_name: "gpt-4o-mini-openai"
model: gemini/gemini-2.0-flash
- model_name: "gpt-4.1-openai"
litellm_params:
model: gpt-4o-mini
model: gpt-4.1
api_key: os.environ/OPENAI_API_KEY
model_info:
access_groups: ["default-openai-models"]
- model_name: "gpt-4o-realtime-preview"
litellm_params:
model: gpt-4o-realtime-preview-2024-10-01
api_key: os.environ/OPENAI_API_KEY
api_key: os.environ/OPENAI_API_KEY_2
- model_name: "bedrock-nova"
litellm_params:
model: us.amazon.nova-pro-v1:0
@ -71,6 +71,16 @@ model_list:
model: mistral/*
api_key: os.environ/MISTRAL_API_KEY
access_groups: ["beta-models"]
- model_name: my-langfuse-model
litellm_params:
model: langfuse/gpt-3.5-turbo
prompt_id: "jokes"
prompt_label: "latest"
api_key: os.environ/OPENAI_API_KEY
litellm_settings:
cache: true
general_settings:
store_model_in_db: true
store_prompts_in_spend_logs: true
custom_auth: custom_auth_auto.user_api_key_auth
custom_auth_settings:
mode: "auto"

14
litellm/proxy/_types.py
View file

@ -23,6 +23,8 @@ from litellm.types.utils import (
EmbeddingResponse,
GenericBudgetConfigType,
ImageResponse,
LiteLLMBatch,
LiteLLMFineTuningJob,
LiteLLMPydanticObjectBase,
ModelResponse,
ProviderField,
@ -938,6 +940,7 @@ class NewCustomerRequest(BudgetNewRequest):
alias: Optional[str] = None # human-friendly alias
blocked: bool = False # allow/disallow requests for this end-user
budget_id: Optional[str] = None # give either a budget_id or max_budget
spend: Optional[float] = None
allowed_model_region: Optional[
AllowedModelRegion
] = None # require all user requests to use models in this specific region
@ -1222,6 +1225,7 @@ class TeamRequest(LiteLLMPydanticObjectBase):
class LiteLLM_BudgetTable(LiteLLMPydanticObjectBase):
"""Represents user-controllable params for a LiteLLM_BudgetTable record"""
budget_id: Optional[str] = None
soft_budget: Optional[float] = None
max_budget: Optional[float] = None
max_parallel_requests: Optional[int] = None
@ -2879,3 +2883,13 @@ class LiteLLM_ManagedFileTable(LiteLLMPydanticObjectBase):
unified_file_id: str
file_object: OpenAIFileObject
model_mappings: Dict[str, str]
flat_model_file_ids: List[str]
created_by: Optional[str]
updated_by: Optional[str]
class LiteLLM_ManagedObjectTable(LiteLLMPydanticObjectBase):
unified_object_id: str
model_object_id: str
file_purpose: Literal["batch", "fine-tune"]
file_object: Union[LiteLLMBatch, LiteLLMFineTuningJob]

25
litellm/proxy/anthropic_endpoints/endpoints.py
View file

@ -12,6 +12,8 @@ from fastapi.responses import StreamingResponse
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import STREAM_SSE_DATA_PREFIX
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.proxy._types import *
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
@ -22,6 +24,24 @@ from litellm.proxy.utils import ProxyLogging
router = APIRouter()
def return_anthropic_chunk(chunk: Any) -> str:
"""
Helper function to format streaming chunks for Anthropic API format
Args:
chunk: A string or dictionary to be returned in SSE format
Returns:
str: A properly formatted SSE chunk string
"""
if isinstance(chunk, dict):
# Use safe_dumps for proper JSON serialization with circular reference detection
chunk_str = safe_dumps(chunk)
return f"{STREAM_SSE_DATA_PREFIX}{chunk_str}\n\n"
else:
return chunk
async def async_data_generator_anthropic(
response,
user_api_key_dict: UserAPIKeyAuth,
@ -40,7 +60,8 @@ async def async_data_generator_anthropic(
user_api_key_dict=user_api_key_dict, response=chunk
)
yield chunk
# Format chunk using helper function
yield return_anthropic_chunk(chunk)
except Exception as e:
verbose_proxy_logger.exception(
"litellm.proxy.proxy_server.async_data_generator(): Exception occured - {}".format(
@ -69,7 +90,7 @@ async def async_data_generator_anthropic(
code=getattr(e, "status_code", 500),
)
error_returned = json.dumps({"error": proxy_exception.to_dict()})
yield f"data: {error_returned}\n\n"
yield f"{STREAM_SSE_DATA_PREFIX}{error_returned}\n\n"
@router.post(

17
litellm/proxy/auth/auth_checks.py
View file

@ -670,15 +670,20 @@ class UserObjectCache:
- update user object in cache
"""
if isinstance(user_object, LiteLLM_UserTable):
user_object = user_object.model_dump()
for k, v in user_object.items():
if isinstance(v, datetime):
user_object[k] = v.isoformat()
await self.user_api_key_cache.async_set_cache(key=user_id, value=user_object)
user_object_dict = user_object.model_dump()
else:
user_object_dict = user_object
for k, v in user_object_dict.items():
if isinstance(v, datetime):
user_object_dict[k] = v.isoformat()
await self.user_api_key_cache.async_set_cache(
key=user_id, value=user_object_dict
)
if self.internal_usage_cache is not None:
await self.internal_usage_cache.async_set_cache(
key=user_id,
value=user_object,
value=user_object_dict,
litellm_parent_otel_span=litellm_parent_otel_span,
)

18
litellm/proxy/auth/user_api_key_auth.py
View file

@ -55,6 +55,16 @@ from litellm.proxy.utils import PrismaClient, ProxyLogging
from litellm.secret_managers.main import get_secret_bool
from litellm.types.services import ServiceTypes
try:
from litellm_enterprise.proxy.auth.user_api_key_auth import (
enterprise_custom_auth as _enterprise_custom_auth,
)
enterprise_custom_auth: Optional[Callable] = _enterprise_custom_auth
except ImportError as e:
verbose_proxy_logger.debug(f"Error in enterprise custom auth: {e}")
enterprise_custom_auth = None
user_api_key_service_logger_obj = ServiceLogging() # used for tracking latency on OTEL
custom_litellm_key_header = APIKeyHeader(
@ -346,7 +356,13 @@ async def _user_api_key_auth_builder( # noqa: PLR0915
)
### USER-DEFINED AUTH FUNCTION ###
if user_custom_auth is not None:
if enterprise_custom_auth is not None:
response = await enterprise_custom_auth(
request=request, api_key=api_key, user_custom_auth=user_custom_auth
)
if response is not None:
return UserAPIKeyAuth.model_validate(response)
elif user_custom_auth is not None:
response = await user_custom_auth(request=request, api_key=api_key) # type: ignore
return UserAPIKeyAuth.model_validate(response)

4
litellm/proxy/common_request_processing.py
View file

@ -117,6 +117,10 @@ class ProxyBaseLLMRequestProcessing:
"acreate_batch",
"aretrieve_batch",
"afile_content",
"acreate_fine_tuning_job",
"acancel_fine_tuning_job",
"alist_fine_tuning_jobs",
"aretrieve_fine_tuning_job",
],
version: Optional[str] = None,
user_model: Optional[str] = None,

18
litellm/proxy/custom_auth_auto.py Normal file
View file

@ -0,0 +1,18 @@
"""
Example custom auth function.
This will allow all keys starting with "my-custom-key" to pass through.
"""
from fastapi import Request
from litellm.proxy._types import UserAPIKeyAuth
async def user_api_key_auth(request: Request, api_key: str) -> UserAPIKeyAuth:
try:
if api_key.startswith("my-custom-key"):
return UserAPIKeyAuth(api_key=api_key)
else:
raise Exception("Invalid API key")
except Exception:
raise Exception("Invalid API key")

1
litellm/proxy/custom_prompt_management.py
View file

@ -15,6 +15,7 @@ class X42PromptManagement(CustomPromptManagement):
prompt_id: Optional[str],
prompt_variables: Optional[dict],
dynamic_callback_params: StandardCallbackDynamicParams,
prompt_label: Optional[str] = None,
) -> Tuple[str, List[AllMessageValues], dict]:
"""
Returns:

325
litellm/proxy/fine_tuning_endpoints/endpoints.py
View file

@ -6,17 +6,20 @@
##########################################################################
import asyncio
import traceback
from typing import Optional
from typing import Optional, cast
from fastapi import APIRouter, Depends, Request, Response
from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.proxy._types import *
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
from litellm.proxy.openai_files_endpoints.common_utils import (
_is_base64_encoded_unified_file_id,
)
from litellm.proxy.utils import handle_exception_on_proxy
from litellm.types.utils import LiteLLMFineTuningJob
router = APIRouter()
@ -96,8 +99,8 @@ async def create_fine_tuning_job(
```
"""
from litellm.proxy.proxy_server import (
add_litellm_data_to_request,
general_settings,
llm_router,
premium_user,
proxy_config,
proxy_logging_obj,
@ -117,25 +120,67 @@ async def create_fine_tuning_job(
)
# Include original request and headers in the data
data = await add_litellm_data_to_request(
data=data,
base_llm_response_processor = ProxyBaseLLMRequestProcessing(data=data)
(
data,
litellm_logging_obj,
) = await base_llm_response_processor.common_processing_pre_call_logic(
request=request,
general_settings=general_settings,
user_api_key_dict=user_api_key_dict,
version=version,
proxy_logging_obj=proxy_logging_obj,
proxy_config=proxy_config,
route_type="acreate_fine_tuning_job",
)
# get configs for custom_llm_provider
llm_provider_config = get_fine_tuning_provider_config(
custom_llm_provider=fine_tuning_request.custom_llm_provider,
## CHECK IF MANAGED FILE ID
unified_file_id: Union[str, Literal[False]] = False
training_file = fine_tuning_request.training_file
response: Optional[LiteLLMFineTuningJob] = None
if training_file:
unified_file_id = _is_base64_encoded_unified_file_id(training_file)
## IF SO, Route based on that
if unified_file_id:
""" """
if llm_router is None:
raise HTTPException(
status_code=500,
detail={
"error": "LLM Router not initialized. Ensure models added to proxy."
},
)
response = cast(
LiteLLMFineTuningJob, await llm_router.acreate_fine_tuning_job(**data)
)
response.training_file = unified_file_id
response._hidden_params["unified_file_id"] = unified_file_id
## ELSE, Route based on custom_llm_provider
elif fine_tuning_request.custom_llm_provider:
# get configs for custom_llm_provider
llm_provider_config = get_fine_tuning_provider_config(
custom_llm_provider=fine_tuning_request.custom_llm_provider,
)
# add llm_provider_config to data
if llm_provider_config is not None:
data.update(llm_provider_config)
response = await litellm.acreate_fine_tuning_job(**data)
if response is None:
raise ValueError(
"Invalid request, No litellm managed file id or custom_llm_provider provided."
)
### CALL HOOKS ### - modify outgoing data
_response = await proxy_logging_obj.post_call_success_hook(
data=data,
user_api_key_dict=user_api_key_dict,
response=response,
)
# add llm_provider_config to data
if llm_provider_config is not None:
data.update(llm_provider_config)
response = await litellm.acreate_fine_tuning_job(**data)
if _response is not None and isinstance(_response, LiteLLMFineTuningJob):
response = _response
### ALERTING ###
asyncio.create_task(
@ -166,12 +211,11 @@ async def create_fine_tuning_job(
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
)
verbose_proxy_logger.error(
verbose_proxy_logger.exception(
"litellm.proxy.proxy_server.create_fine_tuning_job(): Exception occurred - {}".format(
str(e)
)
)
verbose_proxy_logger.debug(traceback.format_exc())
raise handle_exception_on_proxy(e)
@ -191,7 +235,7 @@ async def retrieve_fine_tuning_job(
request: Request,
fastapi_response: Response,
fine_tuning_job_id: str,
custom_llm_provider: Literal["openai", "azure"],
custom_llm_provider: Optional[Literal["openai", "azure"]] = None,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
):
"""
@ -203,41 +247,99 @@ async def retrieve_fine_tuning_job(
- `fine_tuning_job_id`: The ID of the fine-tuning job to retrieve.
"""
from litellm.proxy.proxy_server import (
add_litellm_data_to_request,
general_settings,
llm_router,
premium_user,
proxy_config,
proxy_logging_obj,
version,
)
data: dict = {}
data: dict = {"fine_tuning_job_id": fine_tuning_job_id}
try:
if premium_user is not True:
raise ValueError(
f"Only premium users can use this endpoint + {CommonProxyErrors.not_premium_user.value}"
)
# Include original request and headers in the data
data = await add_litellm_data_to_request(
data=data,
base_llm_response_processor = ProxyBaseLLMRequestProcessing(data=data)
(
data,
litellm_logging_obj,
) = await base_llm_response_processor.common_processing_pre_call_logic(
request=request,
general_settings=general_settings,
user_api_key_dict=user_api_key_dict,
version=version,
proxy_logging_obj=proxy_logging_obj,
proxy_config=proxy_config,
route_type=CallTypes.aretrieve_fine_tuning_job.value,
)
# get configs for custom_llm_provider
llm_provider_config = get_fine_tuning_provider_config(
custom_llm_provider=custom_llm_provider
try:
request_body = await request.json()
except Exception:
request_body = {}
custom_llm_provider = request_body.get("custom_llm_provider", None)
## CHECK IF MANAGED FILE ID
unified_finetuning_job_id: Union[str, Literal[False]] = False
response: Optional[LiteLLMFineTuningJob] = None
if fine_tuning_job_id:
unified_finetuning_job_id = _is_base64_encoded_unified_file_id(
fine_tuning_job_id
)
if unified_finetuning_job_id:
if llm_router is None:
raise HTTPException(
status_code=500,
detail={
"error": "LLM Router not initialized. Ensure models added to proxy."
},
)
response = cast(
LiteLLMFineTuningJob,
await llm_router.aretrieve_fine_tuning_job(
**data,
),
)
response._hidden_params[
"unified_finetuning_job_id"
] = unified_finetuning_job_id
elif custom_llm_provider:
# get configs for custom_llm_provider
llm_provider_config = get_fine_tuning_provider_config(
custom_llm_provider=custom_llm_provider
)
if llm_provider_config is not None:
data.update(llm_provider_config)
response = await litellm.aretrieve_fine_tuning_job(
**data,
)
if response is None:
raise HTTPException(
status_code=400,
detail="Invalid request, No litellm managed file id or custom_llm_provider provided.",
)
### CALL HOOKS ### - modify outgoing data
_response = await proxy_logging_obj.post_call_success_hook(
data=data,
user_api_key_dict=user_api_key_dict,
response=response,
)
if _response is not None and isinstance(_response, LiteLLMFineTuningJob):
response = _response
if llm_provider_config is not None:
data.update(llm_provider_config)
response = await litellm.aretrieve_fine_tuning_job(
**data,
fine_tuning_job_id=fine_tuning_job_id,
### ALERTING ###
asyncio.create_task(
proxy_logging_obj.update_request_status(
litellm_call_id=data.get("litellm_call_id", ""), status="success"
)
)
### RESPONSE HEADERS ###
@ -263,12 +365,11 @@ async def retrieve_fine_tuning_job(
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
)
verbose_proxy_logger.error(
"litellm.proxy.proxy_server.list_fine_tuning_jobs(): Exception occurred - {}".format(
verbose_proxy_logger.exception(
"litellm.proxy.proxy_server.retrieve_fine_tuning_job(): Exception occurred - {}".format(
str(e)
)
)
verbose_proxy_logger.debug(traceback.format_exc())
raise handle_exception_on_proxy(e)
@ -287,7 +388,11 @@ async def retrieve_fine_tuning_job(
async def list_fine_tuning_jobs(
request: Request,
fastapi_response: Response,
custom_llm_provider: Literal["openai", "azure"],
custom_llm_provider: Optional[Literal["openai", "azure"]] = None,
target_model_names: Optional[str] = Query(
default=None,
description="Comma separated list of model names to filter by. Example: 'gpt-4o,gpt-4o-mini'",
),
after: Optional[str] = None,
limit: Optional[int] = None,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
@ -302,8 +407,8 @@ async def list_fine_tuning_jobs(
- `limit`: Number of fine-tuning jobs to retrieve (default is 20).
"""
from litellm.proxy.proxy_server import (
add_litellm_data_to_request,
general_settings,
llm_router,
premium_user,
proxy_config,
proxy_logging_obj,
@ -317,28 +422,60 @@ async def list_fine_tuning_jobs(
f"Only premium users can use this endpoint + {CommonProxyErrors.not_premium_user.value}"
)
# Include original request and headers in the data
data = await add_litellm_data_to_request(
data=data,
base_llm_response_processor = ProxyBaseLLMRequestProcessing(data=data)
(
data,
litellm_logging_obj,
) = await base_llm_response_processor.common_processing_pre_call_logic(
request=request,
general_settings=general_settings,
user_api_key_dict=user_api_key_dict,
version=version,
proxy_logging_obj=proxy_logging_obj,
proxy_config=proxy_config,
route_type=CallTypes.alist_fine_tuning_jobs.value,
)
# get configs for custom_llm_provider
llm_provider_config = get_fine_tuning_provider_config(
custom_llm_provider=custom_llm_provider
)
response: Optional[Any] = None
if target_model_names and isinstance(target_model_names, str):
target_model_names_list = target_model_names.split(",")
if len(target_model_names_list) != 1:
raise HTTPException(
status_code=400,
detail="target_model_names on list fine-tuning jobs must be a list of one model name. Example: ['gpt-4o']",
)
## Use router to list fine-tuning jobs for that model
if llm_router is None:
raise HTTPException(
status_code=500,
detail="LLM Router not initialized. Ensure models added to proxy.",
)
data["model"] = target_model_names_list[0]
response = await llm_router.alist_fine_tuning_jobs(
**data,
after=after,
limit=limit,
)
return response
elif custom_llm_provider:
# get configs for custom_llm_provider
llm_provider_config = get_fine_tuning_provider_config(
custom_llm_provider=custom_llm_provider
)
if llm_provider_config is not None:
data.update(llm_provider_config)
if llm_provider_config is not None:
data.update(llm_provider_config)
response = await litellm.alist_fine_tuning_jobs(
**data,
after=after,
limit=limit,
)
response = await litellm.alist_fine_tuning_jobs(
**data,
after=after,
limit=limit,
)
if response is None:
raise HTTPException(
status_code=400,
detail="Invalid request, No litellm managed file id or custom_llm_provider provided.",
)
### RESPONSE HEADERS ###
hidden_params = getattr(response, "_hidden_params", {}) or {}
@ -363,12 +500,11 @@ async def list_fine_tuning_jobs(
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
)
verbose_proxy_logger.error(
verbose_proxy_logger.exception(
"litellm.proxy.proxy_server.list_fine_tuning_jobs(): Exception occurred - {}".format(
str(e)
)
)
verbose_proxy_logger.debug(traceback.format_exc())
raise handle_exception_on_proxy(e)
@ -400,45 +536,99 @@ async def cancel_fine_tuning_job(
- `fine_tuning_job_id`: The ID of the fine-tuning job to cancel.
"""
from litellm.proxy.proxy_server import (
add_litellm_data_to_request,
general_settings,
llm_router,
premium_user,
proxy_config,
proxy_logging_obj,
version,
)
data: dict = {}
data: dict = {"fine_tuning_job_id": fine_tuning_job_id}
try:
if premium_user is not True:
raise ValueError(
f"Only premium users can use this endpoint + {CommonProxyErrors.not_premium_user.value}"
)
# Include original request and headers in the data
data = await add_litellm_data_to_request(
data=data,
base_llm_response_processor = ProxyBaseLLMRequestProcessing(data=data)
(
data,
litellm_logging_obj,
) = await base_llm_response_processor.common_processing_pre_call_logic(
request=request,
general_settings=general_settings,
user_api_key_dict=user_api_key_dict,
version=version,
proxy_logging_obj=proxy_logging_obj,
proxy_config=proxy_config,
route_type=CallTypes.acancel_fine_tuning_job.value,
)
request_body = await request.json()
try:
request_body = await request.json()
except Exception:
request_body = {}
custom_llm_provider = request_body.get("custom_llm_provider", None)
# get configs for custom_llm_provider
llm_provider_config = get_fine_tuning_provider_config(
custom_llm_provider=custom_llm_provider
## CHECK IF MANAGED FILE ID
unified_finetuning_job_id: Union[str, Literal[False]] = False
response: Optional[LiteLLMFineTuningJob] = None
if fine_tuning_job_id:
unified_finetuning_job_id = _is_base64_encoded_unified_file_id(
fine_tuning_job_id
)
if unified_finetuning_job_id:
if llm_router is None:
raise HTTPException(
status_code=500,
detail={
"error": "LLM Router not initialized. Ensure models added to proxy."
},
)
response = cast(
LiteLLMFineTuningJob,
await llm_router.acancel_fine_tuning_job(
**data,
),
)
response._hidden_params[
"unified_finetuning_job_id"
] = unified_finetuning_job_id
else:
# get configs for custom_llm_provider
llm_provider_config = get_fine_tuning_provider_config(
custom_llm_provider=custom_llm_provider
)
if llm_provider_config is not None:
data.update(llm_provider_config)
response = await litellm.acancel_fine_tuning_job(
**data,
)
if response is None:
raise HTTPException(
status_code=400,
detail="Invalid request, No litellm managed file id or custom_llm_provider provided.",
)
### CALL HOOKS ### - modify outgoing data
_response = await proxy_logging_obj.post_call_success_hook(
data=data,
user_api_key_dict=user_api_key_dict,
response=response,
)
if _response is not None and isinstance(_response, LiteLLMFineTuningJob):
response = _response
if llm_provider_config is not None:
data.update(llm_provider_config)
response = await litellm.acancel_fine_tuning_job(
**data,
fine_tuning_job_id=fine_tuning_job_id,
### ALERTING ###
asyncio.create_task(
proxy_logging_obj.update_request_status(
litellm_call_id=data.get("litellm_call_id", ""), status="success"
)
)
### RESPONSE HEADERS ###
@ -464,10 +654,9 @@ async def cancel_fine_tuning_job(
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
)
verbose_proxy_logger.error(
"litellm.proxy.proxy_server.list_fine_tuning_jobs(): Exception occurred - {}".format(
verbose_proxy_logger.exception(
"litellm.proxy.proxy_server.cancel_fine_tuning_job(): Exception occurred - {}".format(
str(e)
)
)
verbose_proxy_logger.debug(traceback.format_exc())
raise handle_exception_on_proxy(e)

63
litellm/proxy/management_endpoints/customer_endpoints.py
View file

@ -185,6 +185,7 @@ async def new_end_user(
- model_max_budget: Optional[dict] - [Not Implemented Yet] Specify max budget for a given model. Example: {"openai/gpt-4o-mini": {"max_budget": 100.0, "budget_duration": "1d"}}
- max_parallel_requests: Optional[int] - [Not Implemented Yet] Specify max parallel requests for a given customer.
- soft_budget: Optional[float] - [Not Implemented Yet] Get alerts when customer crosses given budget, doesn't block requests.
- spend: Optional[float] - Specify initial spend for a given customer.
- Allow specifying allowed regions
@ -424,13 +425,65 @@ async def update_end_user(
): # models default to [], spend defaults to 0, we should not reset these values
non_default_values[k] = v
## ADD USER, IF NEW ##
## Get end user table data ##
end_user_table_data = await prisma_client.db.litellm_endusertable.find_first(
where={"user_id": data.user_id}, include={"litellm_budget_table": True}
)
if end_user_table_data is None:
raise HTTPException(
status_code=400,
detail={
"error": "End User Id={} does not exist in db".format(data.user_id)
},
)
end_user_table_data_typed = LiteLLM_EndUserTable(
**end_user_table_data.model_dump()
)
## Get budget table data ##
end_user_budget_table = end_user_table_data_typed.litellm_budget_table
## Get all params for budget table ##
budget_table_data = {}
update_end_user_table_data = {}
for k, v in non_default_values.items():
if k in LiteLLM_BudgetTable.model_fields.keys():
budget_table_data[k] = v
if k in LiteLLM_EndUserTable.model_fields.keys():
update_end_user_table_data[k] = v
## Check if budget id is set ##
if budget_table_data:
if end_user_budget_table is None:
## Create new budget ##
budget_table_data_record = (
await prisma_client.db.litellm_budgettable.create(
data=budget_table_data, include={"litellm_endusertable": True}
)
)
update_end_user_table_data[
"budget_id"
] = budget_table_data_record.budget_id
else:
## Update existing budget ##
budget_table_data_record = (
await prisma_client.db.litellm_budgettable.update(
where={"budget_id": end_user_budget_table.budget_id},
data=budget_table_data,
)
)
## Update user table, with update params + new budget id (if set) ##
verbose_proxy_logger.debug("/customer/update: Received data = %s", data)
if data.user_id is not None and len(data.user_id) > 0:
non_default_values["user_id"] = data.user_id # type: ignore
update_end_user_table_data["user_id"] = data.user_id # type: ignore
verbose_proxy_logger.debug("In update customer, user_id condition block.")
response = await prisma_client.db.litellm_endusertable.update(
where={"user_id": data.user_id}, data=non_default_values # type: ignore
where={"user_id": data.user_id}, data=update_end_user_table_data, include={"litellm_budget_table": True} # type: ignore
)
if response is None:
raise ValueError(
@ -444,13 +497,13 @@ async def update_end_user(
raise ValueError(f"user_id is required, passed user_id = {data.user_id}")
# update based on remaining passed in values
except Exception as e:
verbose_proxy_logger.error(
verbose_proxy_logger.exception(
"litellm.proxy.proxy_server.update_end_user(): Exception occured - {}".format(
str(e)
)
)
verbose_proxy_logger.debug(traceback.format_exc())
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "detail", f"Internal Server Error({str(e)})"),

9
litellm/proxy/management_endpoints/team_endpoints.py
View file

@ -1110,7 +1110,7 @@ async def team_member_update(
Update team member budgets and team member role
"""
from litellm.proxy.proxy_server import prisma_client
from litellm.proxy.proxy_server import prisma_client, premium_user
if prisma_client is None:
raise HTTPException(status_code=500, detail={"error": "No db connected"})
@ -1118,6 +1118,13 @@ async def team_member_update(
if data.team_id is None:
raise HTTPException(status_code=400, detail={"error": "No team id passed in"})
if data.role == "admin" and not premium_user:
# exactly the same text your proxy throws for add:
raise HTTPException(
status_code=400,
detail="Assigning team admins is a premium feature. You must be a LiteLLM Enterprise user to use this feature. If you have a license please set `LITELLM_LICENSE` in your env. Get a 7 day trial key here: https://www.litellm.ai/#trial. Pricing: https://www.litellm.ai/#pricing"
)
if data.user_id is None and data.user_email is None:
raise HTTPException(
status_code=400,

60
litellm/proxy/openai_files_endpoints/files_endpoints.py
View file

@ -179,6 +179,7 @@ async def route_create_file(
create_file_request=_create_file_request,
target_model_names_list=target_model_names_list,
litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
user_api_key_dict=user_api_key_dict,
)
else:
# get configs for custom_llm_provider
@ -869,6 +870,7 @@ async def list_files(
fastapi_response: Response,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
provider: Optional[str] = None,
target_model_names: Optional[str] = None,
purpose: Optional[str] = None,
):
"""
@ -885,8 +887,8 @@ async def list_files(
```
"""
from litellm.proxy.proxy_server import (
add_litellm_data_to_request,
general_settings,
llm_router,
proxy_config,
proxy_logging_obj,
version,
@ -894,24 +896,62 @@ async def list_files(
data: Dict = {}
try:
custom_llm_provider = (
provider
or await get_custom_llm_provider_from_request_body(request=request)
or "openai"
)
# Include original request and headers in the data
data = await add_litellm_data_to_request(
data=data,
base_llm_response_processor = ProxyBaseLLMRequestProcessing(data=data)
(
data,
litellm_logging_obj,
) = await base_llm_response_processor.common_processing_pre_call_logic(
request=request,
general_settings=general_settings,
user_api_key_dict=user_api_key_dict,
version=version,
proxy_logging_obj=proxy_logging_obj,
proxy_config=proxy_config,
route_type=CallTypes.alist_fine_tuning_jobs.value,
)
response = await litellm.afile_list(
custom_llm_provider=custom_llm_provider, purpose=purpose, **data # type: ignore
response: Optional[Any] = None
if target_model_names and isinstance(target_model_names, str):
target_model_names_list = target_model_names.split(",")
if len(target_model_names_list) != 1:
raise HTTPException(
status_code=400,
detail="target_model_names on list files must be a list of one model name. Example: ['gpt-4o']",
)
## Use router to list fine-tuning jobs for that model
if llm_router is None:
raise HTTPException(
status_code=500,
detail="LLM Router not initialized. Ensure models added to proxy.",
)
data["model"] = target_model_names_list[0]
response = await llm_router.afile_list(
**data,
)
else:
custom_llm_provider = (
provider
or await get_custom_llm_provider_from_request_body(request=request)
or "openai"
)
response = await litellm.afile_list(
custom_llm_provider=custom_llm_provider, purpose=purpose, **data # type: ignore
)
if response is None:
raise HTTPException(
status_code=500,
detail="Either 'provider' or 'target_model_names' must be provided e.g. `?target_model_names=gpt-4o`",
)
## POST CALL HOOKS ###
_response = await proxy_logging_obj.post_call_success_hook(
data=data, user_api_key_dict=user_api_key_dict, response=response
)
if _response is not None and isinstance(_response, OpenAIFileObject):
response = _response
### ALERTING ###
asyncio.create_task(

2
litellm/proxy/proxy_cli.py
View file

@ -659,7 +659,7 @@ def run_server( # noqa: PLR0915
**key_management_settings
)
database_url = general_settings.get("database_url", None)
if database_url is None:
if database_url is None and os.getenv("DATABASE_URL") is None:
# Check if all required variables are provided
database_host = os.getenv("DATABASE_HOST")
database_username = os.getenv("DATABASE_USERNAME")

21
litellm/proxy/proxy_config.yaml
View file

@ -2,25 +2,10 @@ model_list:
- model_name: openai/gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: any_key
api_base: https://exampleopenaiendpoint-production.up.railway.app/
api_key: a
api_base: hi
general_settings:
store_prompts_in_spend_logs: true
guardrails:
- guardrail_name: "custom-pre-guard"
litellm_params:
guardrail: custom_guardrail.myCustomGuardrail # 👈 Key change
mode: "pre_call" # runs async_pre_call_hook
- guardrail_name: "custom-during-guard"
litellm_params:
guardrail: custom_guardrail.myCustomGuardrail
mode: "during_call" # runs async_moderation_hook
- guardrail_name: "custom-post-guard"
litellm_params:
guardrail: custom_guardrail.myCustomGuardrail
mode: "post_call" # runs async_post_call_success_hook
alerting: ["slack"]

7
litellm/proxy/proxy_server.py
View file

@ -395,10 +395,12 @@ except Exception:
# Import enterprise routes
try:
from litellm_enterprise.proxy.enterprise_routes import router as _enterprise_router
from litellm_enterprise.proxy.proxy_server import EnterpriseProxyConfig
enterprise_router = _enterprise_router
enterprise_proxy_config: Optional[EnterpriseProxyConfig] = EnterpriseProxyConfig()
except ImportError:
pass
enterprise_proxy_config = None
###################
server_root_path = os.getenv("SERVER_ROOT_PATH", "")
@ -1863,6 +1865,9 @@ class ProxyConfig:
value=custom_sso, config_file_path=config_file_path
)
if enterprise_proxy_config is not None:
await enterprise_proxy_config.load_enterprise_config(general_settings)
## pass through endpoints
if general_settings.get("pass_through_endpoints", None) is not None:
await initialize_pass_through_endpoints(

19
litellm/proxy/schema.prisma
View file

@ -453,13 +453,30 @@ model LiteLLM_ManagedFileTable {
id String @id @default(uuid())
unified_file_id String @unique // The base64 encoded unified file ID
file_object Json // Stores the OpenAIFileObject
model_mappings Json // Stores the mapping of model_id -> provider_file_id
model_mappings Json
flat_model_file_ids String[] @default([]) // Flat list of model file id's - for faster querying of model id -> unified file id
created_at DateTime @default(now())
created_by String?
updated_at DateTime @updatedAt
updated_by String?
@@index([unified_file_id])
}
model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use the
id String @id @default(uuid())
unified_object_id String @unique // The base64 encoded unified file ID
model_object_id String @unique // the id returned by the backend API provider
file_object Json // Stores the OpenAIFileObject
file_purpose String // either 'batch' or 'fine-tune'
created_at DateTime @default(now())
created_by String?
updated_at DateTime @updatedAt
updated_by String?
@@index([unified_object_id])
@@index([model_object_id])
}
model LiteLLM_ManagedVectorStoresTable {
vector_store_id String @id

4
litellm/responses/litellm_completion_transformation/transformation.py
View file

@ -457,8 +457,8 @@ class LiteLLMCompletionResponsesConfig:
function=ChatCompletionToolParamFunctionChunk(
name=tool["name"],
description=tool.get("description") or "",
parameters=tool.get("parameters", {}),
strict=tool.get("strict", False),
parameters=dict(tool.get("parameters", {}) or {}),
strict=tool.get("strict", False) or False,
),
)
)

5
litellm/responses/streaming_iterator.py
View file

@ -44,7 +44,7 @@ class BaseResponsesAPIStreamingIterator:
self.responses_api_provider_config = responses_api_provider_config
self.completed_response: Optional[ResponsesAPIStreamingResponse] = None
self.start_time = datetime.now()
# set request kwargs
self.litellm_metadata = litellm_metadata
self.custom_llm_provider = custom_llm_provider
@ -330,7 +330,8 @@ class MockResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator):
def _collect_text(self, resp: ResponsesAPIResponse) -> str:
out = ""
for out_item in resp.output:
if out_item.type == "message":
item_type = getattr(out_item, "type", None)
if item_type == "message":
for c in getattr(out_item, "content", []):
out += c.text
return out

41
litellm/router.py
View file

@ -752,6 +752,21 @@ class Router:
self._arealtime = self.factory_function(
litellm._arealtime, call_type="_arealtime"
)
self.acreate_fine_tuning_job = self.factory_function(
litellm.acreate_fine_tuning_job, call_type="acreate_fine_tuning_job"
)
self.acancel_fine_tuning_job = self.factory_function(
litellm.acancel_fine_tuning_job, call_type="acancel_fine_tuning_job"
)
self.alist_fine_tuning_jobs = self.factory_function(
litellm.alist_fine_tuning_jobs, call_type="alist_fine_tuning_jobs"
)
self.aretrieve_fine_tuning_job = self.factory_function(
litellm.aretrieve_fine_tuning_job, call_type="aretrieve_fine_tuning_job"
)
self.afile_list = self.factory_function(
litellm.afile_list, call_type="alist_files"
)
def validate_fallbacks(self, fallback_param: Optional[List]):
"""
@ -1697,9 +1712,13 @@ class Router:
specific_deployment=kwargs.pop("specific_deployment", None),
)
litellm_model = prompt_management_deployment["litellm_params"].get(
"model", None
self._update_kwargs_with_deployment(
deployment=prompt_management_deployment, kwargs=kwargs
)
data = prompt_management_deployment["litellm_params"].copy()
litellm_model = data.get("model", None)
prompt_id = kwargs.get("prompt_id") or prompt_management_deployment[
"litellm_params"
].get("prompt_id", None)
@ -1708,6 +1727,9 @@ class Router:
) or prompt_management_deployment["litellm_params"].get(
"prompt_variables", None
)
prompt_label = kwargs.get("prompt_label", None) or prompt_management_deployment[
"litellm_params"
].get("prompt_label", None)
if prompt_id is None or not isinstance(prompt_id, str):
raise ValueError(
@ -1728,14 +1750,16 @@ class Router:
non_default_params=get_non_default_completion_params(kwargs=kwargs),
prompt_id=prompt_id,
prompt_variables=prompt_variables,
prompt_label=prompt_label,
)
kwargs = {**kwargs, **optional_params}
kwargs = {**data, **kwargs, **optional_params}
kwargs["model"] = model
kwargs["messages"] = messages
kwargs["litellm_logging_obj"] = litellm_logging_object
kwargs["prompt_id"] = prompt_id
kwargs["prompt_variables"] = prompt_variables
kwargs["prompt_label"] = prompt_label
_model_list = self.get_model_list(model_name=model)
if _model_list is None or len(_model_list) == 0: # if direct call to model
@ -2427,6 +2451,7 @@ class Router:
messages=kwargs.get("messages", None),
specific_deployment=kwargs.pop("specific_deployment", None),
)
self._update_kwargs_with_deployment(
deployment=deployment, kwargs=kwargs, function_name="generic_api_call"
)
@ -3159,6 +3184,11 @@ class Router:
"afile_delete",
"afile_content",
"_arealtime",
"acreate_fine_tuning_job",
"acancel_fine_tuning_job",
"alist_fine_tuning_jobs",
"aretrieve_fine_tuning_job",
"alist_files",
] = "assistants",
):
"""
@ -3207,6 +3237,11 @@ class Router:
"anthropic_messages",
"aresponses",
"_arealtime",
"acreate_fine_tuning_job",
"acancel_fine_tuning_job",
"alist_fine_tuning_jobs",
"aretrieve_fine_tuning_job",
"alist_files",
):
return await self._ageneric_api_call_with_fallbacks(
original_function=original_function,

2
litellm/router_strategy/lowest_tpm_rpm.py
View file

@ -96,6 +96,8 @@ class LowestTPMLoggingHandler(CustomLogger):
if kwargs["litellm_params"].get("metadata") is None:
pass
else:
if "litellm_params" not in kwargs:
return
model_group = kwargs["litellm_params"]["metadata"].get(
"model_group", None
)

45
litellm/secret_managers/get_azure_ad_token_provider.py
View file

@ -1,5 +1,9 @@
import os
from typing import Callable
from typing import Any, Callable, Optional, Union
from litellm.types.secret_managers.get_azure_ad_token_provider import (
AzureCredentialType,
)
def get_azure_ad_token_provider() -> Callable[[], str]:
@ -15,24 +19,45 @@ def get_azure_ad_token_provider() -> Callable[[], str]:
Callable that returns a temporary authentication token.
"""
import azure.identity as identity
from azure.identity import get_bearer_token_provider
from azure.identity import (
CertificateCredential,
ClientSecretCredential,
ManagedIdentityCredential,
get_bearer_token_provider,
)
azure_scope = os.environ.get(
"AZURE_SCOPE", "https://cognitiveservices.azure.com/.default"
)
cred = os.environ.get("AZURE_CREDENTIAL", "ClientSecretCredential")
cred_cls = getattr(identity, cred)
# ClientSecretCredential, DefaultAzureCredential, AzureCliCredential
if cred == "ClientSecretCredential":
credential = cred_cls(
cred: Union[AzureCredentialType, str] = AzureCredentialType(
os.environ.get("AZURE_CREDENTIAL", AzureCredentialType.ClientSecretCredential)
)
credential: Optional[
Union[
ClientSecretCredential,
ManagedIdentityCredential,
CertificateCredential,
Any,
]
] = None
if cred == AzureCredentialType.ClientSecretCredential:
credential = ClientSecretCredential(
client_id=os.environ["AZURE_CLIENT_ID"],
client_secret=os.environ["AZURE_CLIENT_SECRET"],
tenant_id=os.environ["AZURE_TENANT_ID"],
)
elif cred == "ManagedIdentityCredential":
credential = cred_cls(client_id=os.environ["AZURE_CLIENT_ID"])
elif cred == AzureCredentialType.ManagedIdentityCredential:
credential = ManagedIdentityCredential(client_id=os.environ["AZURE_CLIENT_ID"])
elif cred == AzureCredentialType.CertificateCredential:
credential = CertificateCredential(
client_id=os.environ["AZURE_CLIENT_ID"],
tenant_id=os.environ["AZURE_TENANT_ID"],
certificate_path=os.environ["AZURE_CERTIFICATE_PATH"],
)
else:
cred_cls = getattr(identity, cred)
credential = cred_cls()
if credential is None:
raise ValueError("No credential provided")
return get_bearer_token_provider(credential, azure_scope)

4
litellm/types/llms/openai.py
View file

@ -878,7 +878,7 @@ class FineTuningJobCreate(BaseModel):
class LiteLLMFineTuningJobCreate(FineTuningJobCreate):
custom_llm_provider: Literal["openai", "azure", "vertex_ai"]
custom_llm_provider: Optional[Literal["openai", "azure", "vertex_ai"]] = None
model_config = {
"extra": "allow"
@ -1000,7 +1000,7 @@ class ResponsesAPIResponse(BaseLiteLLMOpenAIResponseObject):
model: Optional[str]
object: Optional[str]
output: Union[
List[ResponseOutputItem],
List[Union[ResponseOutputItem, Dict]],
List[Union[GenericResponseOutputItem, OutputFunctionToolCall]],
]
parallel_tool_calls: bool

7
litellm/types/secret_managers/get_azure_ad_token_provider.py Normal file
View file

@ -0,0 +1,7 @@
from enum import Enum
class AzureCredentialType(str, Enum):
ClientSecretCredential = "ClientSecretCredential"
ManagedIdentityCredential = "ManagedIdentityCredential"
CertificateCredential = "CertificateCredential"

69
litellm/types/utils.py
View file

@ -33,6 +33,7 @@ from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, model_validator
from typing_extensions import Callable, Dict, Required, TypedDict, override
import litellm
from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject
from ..litellm_core_utils.core_helpers import map_finish_reason
from .guardrails import GuardrailEventHooks
@ -44,6 +45,7 @@ from .llms.openai import (
ChatCompletionToolCallChunk,
ChatCompletionUsageBlock,
FileSearchTool,
FineTuningJob,
OpenAIChatCompletionChunk,
OpenAIFileObject,
OpenAIRealtimeStreamList,
@ -1541,19 +1543,46 @@ class ImageObject(OpenAIImage):
return self.dict()
class ImageUsageInputTokensDetails(BaseLiteLLMOpenAIResponseObject):
image_tokens: int
"""The number of image tokens in the input prompt."""
text_tokens: int
"""The number of text tokens in the input prompt."""
class ImageUsage(BaseLiteLLMOpenAIResponseObject):
input_tokens: int
"""The number of tokens (images and text) in the input prompt."""
input_tokens_details: ImageUsageInputTokensDetails
"""The input tokens detailed information for the image generation."""
output_tokens: int
"""The number of image tokens in the output image."""
total_tokens: int
"""The total number of tokens (images and text) used for the image generation."""
from openai.types.images_response import ImagesResponse as OpenAIImageResponse
class ImageResponse(OpenAIImageResponse):
class ImageResponse(OpenAIImageResponse, BaseLiteLLMOpenAIResponseObject):
_hidden_params: dict = {}
usage: Usage
usage: Optional[ImageUsage] = None # type: ignore
"""
Users might use litellm with older python versions, we don't want this to break for them.
Happens when their OpenAIImageResponse has the old OpenAI usage class.
"""
def __init__(
self,
created: Optional[int] = None,
data: Optional[List[ImageObject]] = None,
response_ms=None,
usage: Optional[Usage] = None,
usage: Optional[ImageUsage] = None,
hidden_params: Optional[dict] = None,
):
if response_ms:
@ -1576,9 +1605,14 @@ class ImageResponse(OpenAIImageResponse):
_data.append(ImageObject(**d))
elif isinstance(d, BaseModel):
_data.append(ImageObject(**d.model_dump()))
_usage = usage or Usage(
prompt_tokens=0,
completion_tokens=0,
_usage = usage or ImageUsage(
input_tokens=0,
input_tokens_details=ImageUsageInputTokensDetails(
image_tokens=0,
text_tokens=0,
),
output_tokens=0,
total_tokens=0,
)
super().__init__(created=created, data=_data, usage=_usage) # type: ignore
@ -2083,6 +2117,7 @@ all_litellm_params = [
"allowed_openai_params",
"litellm_session_id",
"use_litellm_proxy",
"prompt_label",
] + list(StandardCallbackDynamicParams.__annotations__.keys())
@ -2256,6 +2291,19 @@ class SelectTokenizerResponse(TypedDict):
tokenizer: Any
class LiteLLMFineTuningJob(FineTuningJob):
_hidden_params: dict = {}
seed: Optional[int] = None # type: ignore
def __init__(self, **kwargs):
if "error" in kwargs and kwargs["error"] is not None:
# check if error is all None - if so, set error to None
if all(value is None for value in kwargs["error"].values()):
kwargs["error"] = None
super().__init__(**kwargs)
self._hidden_params = kwargs.get("_hidden_params", {})
class LiteLLMBatch(Batch):
_hidden_params: dict = {}
usage: Optional[Usage] = None
@ -2360,9 +2408,16 @@ class SpecialEnums(Enum):
LITELLM_MANAGED_BATCH_COMPLETE_STR = "litellm_proxy;model_id:{};llm_batch_id:{}"
LITELLM_MANAGED_GENERIC_RESPONSE_COMPLETE_STR = "litellm_proxy;model_id:{};generic_response_id:{}" # generic implementation of 'managed batches' - used for finetuning and any future work.
LLMResponseTypes = Union[
ModelResponse, EmbeddingResponse, ImageResponse, OpenAIFileObject, LiteLLMBatch
ModelResponse,
EmbeddingResponse,
ImageResponse,
OpenAIFileObject,
LiteLLMBatch,
LiteLLMFineTuningJob,
]

8
litellm/utils.py
View file

@ -6835,6 +6835,14 @@ def _add_path_to_api_base(api_base: str, ending_path: str) -> str:
return str(modified_url.copy_with(params=original_url.params))
def get_standard_openai_params(params: dict) -> dict:
return {
k: v
for k, v in params.items()
if k in litellm.OPENAI_CHAT_COMPLETION_PARAMS and v is not None
}
def get_non_default_completion_params(kwargs: dict) -> dict:
openai_params = litellm.OPENAI_CHAT_COMPLETION_PARAMS
default_params = openai_params + all_litellm_params

290
model_prices_and_context_window.json
View file

@ -3117,6 +3117,18 @@
"supports_function_calling": true,
"source": "https://azuremarketplace.microsoft.com/en/marketplace/apps/000-000.mistral-nemo-12b-2407?tab=PlansAndPrice"
},
"azure_ai/mistral-medium-2505": {
"max_tokens": 8191,
"max_input_tokens": 131072,
"max_output_tokens": 8191,
"input_cost_per_token": 0.0000004,
"output_cost_per_token": 0.000002,
"litellm_provider": "azure_ai",
"mode": "chat",
"supports_function_calling": true,
"supports_assistant_prefill": true,
"supports_tool_choice": true
},
"azure_ai/mistral-large": {
"max_tokens": 8191,
"max_input_tokens": 32000,
@ -4684,6 +4696,58 @@
"deprecation_date": "2025-06-01",
"supports_tool_choice": true
},
"claude-opus-4-20250514": {
"max_tokens": 32000,
"max_input_tokens": 200000,
"max_output_tokens": 32000,
"input_cost_per_token": 15e-6,
"output_cost_per_token": 75e-6,
"search_context_cost_per_query": {
"search_context_size_low": 1e-2,
"search_context_size_medium": 1e-2,
"search_context_size_high": 1e-2
},
"cache_creation_input_token_cost": 18.75e-6,
"cache_read_input_token_cost": 1.5e-6,
"litellm_provider": "anthropic",
"mode": "chat",
"supports_function_calling": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 159,
"supports_assistant_prefill": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_computer_use": true
},
"claude-sonnet-4-20250514": {
"max_tokens": 64000,
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"input_cost_per_token": 3e-6,
"output_cost_per_token": 15e-6,
"search_context_cost_per_query": {
"search_context_size_low": 1e-2,
"search_context_size_medium": 1e-2,
"search_context_size_high": 1e-2
},
"cache_creation_input_token_cost": 3.75e-6,
"cache_read_input_token_cost": 0.3e-6,
"litellm_provider": "anthropic",
"mode": "chat",
"supports_function_calling": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 159,
"supports_assistant_prefill": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_computer_use": true
},
"claude-3-7-sonnet-latest": {
"supports_computer_use": true,
"max_tokens": 128000,
@ -6753,6 +6817,58 @@
"supports_reasoning": true,
"supports_tool_choice": true
},
"vertex_ai/claude-opus-4@20250514": {
"max_tokens": 32000,
"max_input_tokens": 200000,
"max_output_tokens": 32000,
"input_cost_per_token": 15e-6,
"output_cost_per_token": 75e-6,
"search_context_cost_per_query": {
"search_context_size_low": 1e-2,
"search_context_size_medium": 1e-2,
"search_context_size_high": 1e-2
},
"cache_creation_input_token_cost": 18.75e-6,
"cache_read_input_token_cost": 1.5e-6,
"litellm_provider": "vertex_ai-anthropic_models",
"mode": "chat",
"supports_function_calling": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 159,
"supports_assistant_prefill": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_computer_use": true
},
"vertex_ai/claude-sonnet-4@20250514": {
"max_tokens": 64000,
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"input_cost_per_token": 3e-6,
"output_cost_per_token": 15e-6,
"search_context_cost_per_query": {
"search_context_size_low": 1e-2,
"search_context_size_medium": 1e-2,
"search_context_size_high": 1e-2
},
"cache_creation_input_token_cost": 3.75e-6,
"cache_read_input_token_cost": 0.3e-6,
"litellm_provider": "vertex_ai-anthropic_models",
"mode": "chat",
"supports_function_calling": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 159,
"supports_assistant_prefill": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_computer_use": true
},
"vertex_ai/claude-3-haiku": {
"max_tokens": 4096,
"max_input_tokens": 200000,
@ -9332,6 +9448,58 @@
"supports_pdf_input": true,
"supports_tool_choice": true
},
"anthropic.claude-opus-4-20250514-v1:0": {
"max_tokens": 32000,
"max_input_tokens": 200000,
"max_output_tokens": 32000,
"input_cost_per_token": 15e-6,
"output_cost_per_token": 75e-6,
"search_context_cost_per_query": {
"search_context_size_low": 1e-2,
"search_context_size_medium": 1e-2,
"search_context_size_high": 1e-2
},
"cache_creation_input_token_cost": 18.75e-6,
"cache_read_input_token_cost": 1.5e-6,
"litellm_provider": "bedrock_converse",
"mode": "chat",
"supports_function_calling": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 159,
"supports_assistant_prefill": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_computer_use": true
},
"anthropic.claude-sonnet-4-20250514-v1:0": {
"max_tokens": 64000,
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"input_cost_per_token": 3e-6,
"output_cost_per_token": 15e-6,
"search_context_cost_per_query": {
"search_context_size_low": 1e-2,
"search_context_size_medium": 1e-2,
"search_context_size_high": 1e-2
},
"cache_creation_input_token_cost": 3.75e-6,
"cache_read_input_token_cost": 0.3e-6,
"litellm_provider": "bedrock_converse",
"mode": "chat",
"supports_function_calling": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 159,
"supports_assistant_prefill": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_computer_use": true
},
"anthropic.claude-3-7-sonnet-20250219-v1:0": {
"supports_computer_use": true,
"max_tokens": 8192,
@ -9482,6 +9650,58 @@
"supports_tool_choice": true,
"supports_reasoning": true
},
"us.anthropic.claude-opus-4-20250514-v1:0": {
"max_tokens": 32000,
"max_input_tokens": 200000,
"max_output_tokens": 32000,
"input_cost_per_token": 15e-6,
"output_cost_per_token": 75e-6,
"search_context_cost_per_query": {
"search_context_size_low": 1e-2,
"search_context_size_medium": 1e-2,
"search_context_size_high": 1e-2
},
"cache_creation_input_token_cost": 18.75e-6,
"cache_read_input_token_cost": 1.5e-6,
"litellm_provider": "bedrock_converse",
"mode": "chat",
"supports_function_calling": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 159,
"supports_assistant_prefill": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_computer_use": true
},
"us.anthropic.claude-sonnet-4-20250514-v1:0": {
"max_tokens": 64000,
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"input_cost_per_token": 3e-6,
"output_cost_per_token": 15e-6,
"search_context_cost_per_query": {
"search_context_size_low": 1e-2,
"search_context_size_medium": 1e-2,
"search_context_size_high": 1e-2
},
"cache_creation_input_token_cost": 3.75e-6,
"cache_read_input_token_cost": 0.3e-6,
"litellm_provider": "bedrock_converse",
"mode": "chat",
"supports_function_calling": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 159,
"supports_assistant_prefill": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_computer_use": true
},
"us.anthropic.claude-3-haiku-20240307-v1:0": {
"max_tokens": 4096,
"max_input_tokens": 200000,
@ -9603,6 +9823,58 @@
"supports_pdf_input": true,
"supports_tool_choice": true
},
"eu.anthropic.claude-opus-4-20250514-v1:0": {
"max_tokens": 32000,
"max_input_tokens": 200000,
"max_output_tokens": 32000,
"input_cost_per_token": 15e-6,
"output_cost_per_token": 75e-6,
"search_context_cost_per_query": {
"search_context_size_low": 1e-2,
"search_context_size_medium": 1e-2,
"search_context_size_high": 1e-2
},
"cache_creation_input_token_cost": 18.75e-6,
"cache_read_input_token_cost": 1.5e-6,
"litellm_provider": "bedrock_converse",
"mode": "chat",
"supports_function_calling": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 159,
"supports_assistant_prefill": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_computer_use": true
},
"eu.anthropic.claude-sonnet-4-20250514-v1:0": {
"max_tokens": 64000,
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"input_cost_per_token": 3e-6,
"output_cost_per_token": 15e-6,
"search_context_cost_per_query": {
"search_context_size_low": 1e-2,
"search_context_size_medium": 1e-2,
"search_context_size_high": 1e-2
},
"cache_creation_input_token_cost": 3.75e-6,
"cache_read_input_token_cost": 0.3e-6,
"litellm_provider": "bedrock_converse",
"mode": "chat",
"supports_function_calling": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 159,
"supports_assistant_prefill": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_computer_use": true
},
"eu.anthropic.claude-3-5-haiku-20241022-v1:0": {
"max_tokens": 8192,
"max_input_tokens": 200000,
@ -12297,7 +12569,7 @@
"max_output_tokens": 128000,
"input_cost_per_token": 0.0000025,
"input_dbu_cost_per_token": 0.00003571,
"output_cost_per_token": 0.00017857,
"output_cost_per_token": 0.000017857,
"output_db_cost_per_token": 0.000214286,
"litellm_provider": "databricks",
"mode": "chat",
@ -12350,6 +12622,20 @@
"metadata": {"notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."},
"supports_tool_choice": true
},
"databricks/databricks-llama-4-maverick": {
"max_tokens": 128000,
"max_input_tokens": 128000,
"max_output_tokens": 128000,
"input_cost_per_token": 0.000005,
"input_dbu_cost_per_token": 0.00007143,
"output_cost_per_token": 0.000015,
"output_dbu_cost_per_token": 0.00021429,
"litellm_provider": "databricks",
"mode": "chat",
"source": "https://www.databricks.com/product/pricing/foundation-model-serving",
"metadata": {"notes": "Databricks documentation now provides both DBU costs (_dbu_cost_per_token) and dollar costs(_cost_per_token)."},
"supports_tool_choice": true
},
"databricks/databricks-dbrx-instruct": {
"max_tokens": 32768,
"max_input_tokens": 32768,
@ -12976,4 +13262,4 @@
"litellm_provider": "featherless_ai",
"mode": "chat"
}
}
}

43
poetry.lock generated
View file

@ -258,10 +258,9 @@ tests-mypy = ["mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" a
name = "azure-core"
version = "1.33.0"
description = "Microsoft Azure Core Library for Python"
optional = true
optional = false
python-versions = ">=3.8"
groups = ["main"]
markers = "extra == \"extra-proxy\""
groups = ["main", "proxy-dev"]
files = [
{file = "azure_core-1.33.0-py3-none-any.whl", hash = "sha256:9b5b6d0223a1d38c37500e6971118c1e0f13f54951e6893968b38910bc9cda8f"},
{file = "azure_core-1.33.0.tar.gz", hash = "sha256:f367aa07b5e3005fec2c1e184b882b0b039910733907d001c20fb08ebb8c0eb9"},
@ -280,10 +279,9 @@ tracing = ["opentelemetry-api (>=1.26,<2.0)"]
name = "azure-identity"
version = "1.21.0"
description = "Microsoft Azure Identity Library for Python"
optional = true
optional = false
python-versions = ">=3.8"
groups = ["main"]
markers = "extra == \"extra-proxy\""
groups = ["main", "proxy-dev"]
files = [
{file = "azure_identity-1.21.0-py3-none-any.whl", hash = "sha256:258ea6325537352440f71b35c3dffe9d240eae4a5126c1b7ce5efd5766bd9fd9"},
{file = "azure_identity-1.21.0.tar.gz", hash = "sha256:ea22ce6e6b0f429bc1b8d9212d5b9f9877bd4c82f1724bfa910760612c07a9a6"},
@ -499,7 +497,7 @@ version = "1.17.1"
description = "Foreign Function Interface for Python calling C code."
optional = false
python-versions = ">=3.8"
groups = ["main", "dev"]
groups = ["main", "dev", "proxy-dev"]
files = [
{file = "cffi-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14"},
{file = "cffi-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67"},
@ -569,7 +567,7 @@ files = [
{file = "cffi-1.17.1-cp39-cp39-win_amd64.whl", hash = "sha256:d016c76bdd850f3c626af19b0542c9677ba156e4ee4fccfdd7848803533ef662"},
{file = "cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824"},
]
markers = {main = "(extra == \"proxy\" or extra == \"extra-proxy\") and (platform_python_implementation != \"PyPy\" or extra == \"proxy\")", dev = "platform_python_implementation != \"PyPy\""}
markers = {main = "platform_python_implementation != \"PyPy\" or extra == \"proxy\"", dev = "platform_python_implementation != \"PyPy\"", proxy-dev = "platform_python_implementation != \"PyPy\""}
[package.dependencies]
pycparser = "*"
@ -729,7 +727,7 @@ version = "43.0.3"
description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers."
optional = false
python-versions = ">=3.7"
groups = ["main", "dev"]
groups = ["main", "dev", "proxy-dev"]
files = [
{file = "cryptography-43.0.3-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:bf7a1932ac4176486eab36a19ed4c0492da5d97123f1406cf15e41b05e787d2e"},
{file = "cryptography-43.0.3-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63efa177ff54aec6e1c0aefaa1a241232dcd37413835a9b674b6e3f0ae2bfd3e"},
@ -759,7 +757,6 @@ files = [
{file = "cryptography-43.0.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:2ce6fae5bdad59577b44e4dfed356944fbf1d925269114c28be377692643b4ff"},
{file = "cryptography-43.0.3.tar.gz", hash = "sha256:315b9001266a492a6ff443b61238f956b214dbec9910a081ba5b6646a055a805"},
]
markers = {main = "extra == \"proxy\" or extra == \"extra-proxy\""}
[package.dependencies]
cffi = {version = ">=1.12", markers = "platform_python_implementation != \"PyPy\""}
@ -1986,10 +1983,9 @@ dev = ["absl-py", "pyink", "pylint (>=2.6.0)", "pytest", "pytest-xdist"]
name = "msal"
version = "1.32.3"
description = "The Microsoft Authentication Library (MSAL) for Python library enables your app to access the Microsoft Cloud by supporting authentication of users with Microsoft Azure Active Directory accounts (AAD) and Microsoft Accounts (MSA) using industry standard OAuth2 and OpenID Connect."
optional = true
optional = false
python-versions = ">=3.7"
groups = ["main"]
markers = "extra == \"extra-proxy\""
groups = ["main", "proxy-dev"]
files = [
{file = "msal-1.32.3-py3-none-any.whl", hash = "sha256:b2798db57760b1961b142f027ffb7c8169536bf77316e99a0df5c4aaebb11569"},
{file = "msal-1.32.3.tar.gz", hash = "sha256:5eea038689c78a5a70ca8ecbe1245458b55a857bd096efb6989c69ba15985d35"},
@ -2007,10 +2003,9 @@ broker = ["pymsalruntime (>=0.14,<0.18) ; python_version >= \"3.6\" and platform
name = "msal-extensions"
version = "1.3.0"
description = "Microsoft Authentication Library extensions (MSAL EX) provides a persistence API that can save your data on disk, encrypted on Windows, macOS and Linux. Concurrent data access will be coordinated by a file lock mechanism."
optional = true
optional = false
python-versions = ">=3.7"
groups = ["main"]
markers = "extra == \"extra-proxy\""
groups = ["main", "proxy-dev"]
files = [
{file = "msal_extensions-1.3.0-py3-none-any.whl", hash = "sha256:105328ddcbdd342016c9949d8f89e3917554740c8ab26669c0fa0e069e730a0e"},
{file = "msal_extensions-1.3.0.tar.gz", hash = "sha256:96918996642b38c78cd59b55efa0f06fd1373c90e0949be8615697c048fba62c"},
@ -2937,12 +2932,12 @@ version = "2.22"
description = "C parser in Python"
optional = false
python-versions = ">=3.8"
groups = ["main", "dev"]
groups = ["main", "dev", "proxy-dev"]
files = [
{file = "pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc"},
{file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"},
]
markers = {main = "(extra == \"proxy\" or extra == \"extra-proxy\") and (platform_python_implementation != \"PyPy\" or extra == \"proxy\")", dev = "platform_python_implementation != \"PyPy\""}
markers = {main = "platform_python_implementation != \"PyPy\" or extra == \"proxy\"", dev = "platform_python_implementation != \"PyPy\"", proxy-dev = "platform_python_implementation != \"PyPy\""}
[[package]]
name = "pydantic"
@ -3136,10 +3131,9 @@ windows-terminal = ["colorama (>=0.4.6)"]
name = "pyjwt"
version = "2.9.0"
description = "JSON Web Token implementation in Python"
optional = true
optional = false
python-versions = ">=3.8"
groups = ["main"]
markers = "extra == \"extra-proxy\" or extra == \"proxy\""
groups = ["main", "proxy-dev"]
files = [
{file = "PyJWT-2.9.0-py3-none-any.whl", hash = "sha256:3b02fb0f44517787776cf48f2ae25d8e14f300e6d7545a4315cee571a415e850"},
{file = "pyjwt-2.9.0.tar.gz", hash = "sha256:7e1e5b56cc735432a7369cbfa0efe50fa113ebecdc04ae6922deba8b84582d0c"},
@ -3879,10 +3873,9 @@ crt = ["botocore[crt] (>=1.33.2,<2.0a.0)"]
name = "six"
version = "1.17.0"
description = "Python 2 and 3 compatibility utilities"
optional = true
optional = false
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7"
groups = ["main"]
markers = "extra == \"extra-proxy\" or extra == \"proxy\""
groups = ["main", "proxy-dev"]
files = [
{file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"},
{file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"},
@ -4935,4 +4928,4 @@ utils = ["numpydoc"]
[metadata]
lock-version = "2.1"
python-versions = ">=3.8.1,<4.0, !=3.9.7"
content-hash = "4c385d4e27013d9cacf9573f5532c5e13d22fc27af6344e87ba2417340c9be93"
content-hash = "fe1bc122aaeae89043f8099bfe7fda082110b06ddee5147cbfcdd085f072ef72"

7
pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm"
version = "1.70.3"
version = "1.70.4"
description = "Library to easily interface with LLM API providers"
authors = ["BerriAI"]
license = "MIT"
@ -21,7 +21,7 @@ Documentation = "https://docs.litellm.ai"
[tool.poetry.dependencies]
python = ">=3.8.1,<4.0, !=3.9.7"
httpx = ">=0.23.0"
openai = ">=1.68.2, <1.76.0"
openai = ">=1.68.2"
python-dotenv = ">=0.2.0"
tiktoken = ">=0.7.0"
importlib-metadata = ">=6.8.0"
@ -130,13 +130,14 @@ prometheus-client = "0.20.0"
opentelemetry-api = "1.25.0"
opentelemetry-sdk = "1.25.0"
opentelemetry-exporter-otlp = "1.25.0"
azure-identity = "^1.15.0"
[build-system]
requires = ["poetry-core", "wheel"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "1.70.3"
version = "1.70.4"
version_files = [
"pyproject.toml:^version"
]

2
requirements.txt
View file

@ -1,7 +1,7 @@
# LITELLM PROXY DEPENDENCIES #
anyio==4.5.0 # openai + http req.
httpx==0.27.0 # Pin Httpx dependency
openai==1.68.2 # openai req.
openai==1.81.0 # openai req.
fastapi==0.115.5 # server dep
backoff==2.2.1 # server dep
pyyaml==6.0.2 # server dep

19
schema.prisma
View file

@ -453,13 +453,30 @@ model LiteLLM_ManagedFileTable {
id String @id @default(uuid())
unified_file_id String @unique // The base64 encoded unified file ID
file_object Json // Stores the OpenAIFileObject
model_mappings Json // Stores the mapping of model_id -> provider_file_id
model_mappings Json
flat_model_file_ids String[] @default([]) // Flat list of model file id's - for faster querying of model id -> unified file id
created_at DateTime @default(now())
created_by String?
updated_at DateTime @updatedAt
updated_by String?
@@index([unified_file_id])
}
model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use managed files
id String @id @default(uuid())
unified_object_id String @unique // The base64 encoded unified object ID
model_object_id String @unique // the id returned by the backend API provider
file_object Json // Stores the OpenAIFileObject
file_purpose String // either 'batch' or 'fine-tune'
created_at DateTime @default(now())
created_by String?
updated_at DateTime @updatedAt
updated_by String?
@@index([unified_object_id])
@@index([model_object_id])
}
model LiteLLM_ManagedVectorStoresTable {
vector_store_id String @id

18
tests/batches_tests/test_fine_tuning_api.py
View file

@ -525,9 +525,12 @@ async def test_mock_openai_cancel_fine_tune_job():
client = AsyncOpenAI(api_key="fake-api-key")
with patch.object(client.fine_tuning.jobs, "cancel") as mock_cancel:
await litellm.acancel_fine_tuning_job(
fine_tuning_job_id="ft-123", client=client
)
try:
await litellm.acancel_fine_tuning_job(
fine_tuning_job_id="ft-123", client=client
)
except Exception as e:
print("error=", e)
# Only verify that the client was called with correct parameters
mock_cancel.assert_called_once_with(fine_tuning_job_id="ft-123")
@ -541,10 +544,13 @@ async def test_mock_openai_retrieve_fine_tune_job():
client = AsyncOpenAI(api_key="fake-api-key")
with patch.object(client.fine_tuning.jobs, "retrieve") as mock_retrieve:
try:
response = await litellm.aretrieve_fine_tuning_job(
fine_tuning_job_id="ft-123", client=client
)
except Exception as e:
print("error=", e)
response = await litellm.aretrieve_fine_tuning_job(
fine_tuning_job_id="ft-123", client=client
)
# Verify the request
mock_retrieve.assert_called_once_with(fine_tuning_job_id="ft-123")

114
tests/enterprise/enterprise_hooks/test_managed_files.py
View file

@ -3,16 +3,20 @@ import os
import sys
import pytest
from fastapi import HTTPException
from fastapi.testclient import TestClient
sys.path.insert(
0, os.path.abspath("../../..")
) # Adds the parent directory to the system path
from unittest.mock import MagicMock
from unittest.mock import AsyncMock, MagicMock, patch
from enterprise.enterprise_hooks.managed_files import _PROXY_LiteLLMManagedFiles
from litellm.caching import DualCache
from litellm.proxy.openai_files_endpoints.common_utils import (
_is_base64_encoded_unified_file_id,
)
from litellm.types.utils import SpecialEnums
@ -42,11 +46,19 @@ def test_get_file_ids_from_messages():
@pytest.mark.asyncio
async def test_async_pre_call_hook_batch_retrieve():
from litellm.proxy._types import UserAPIKeyAuth
prisma_client = AsyncMock()
return_value = MagicMock()
return_value.created_by = "123"
prisma_client.db.litellm_managedobjecttable.find_first.return_value = return_value
proxy_managed_files = _PROXY_LiteLLMManagedFiles(
DualCache(), prisma_client=MagicMock()
DualCache(), prisma_client=prisma_client
)
data = {
"user_api_key_dict": {"parent_otel_span": MagicMock()},
"user_api_key_dict": UserAPIKeyAuth(
user_id="123", parent_otel_span=MagicMock()
),
"data": {
"batch_id": "bGl0ZWxsbV9wcm94eTttb2RlbF9pZDpteS1nZW5lcmFsLWF6dXJlLWRlcGxveW1lbnQ7bGxtX2JhdGNoX2lkOmJhdGNoX2EzMjJiNmJhLWFjN2UtNDg4OC05MjljLTFhZDM0NDJmMDZlZA",
},
@ -161,3 +173,99 @@ async def test_async_pre_call_hook_batch_retrieve():
# assert len(batch_files) == 1
# assert assistant_files[0].id == file1.id
# assert batch_files[0].id == file2.id
@pytest.mark.asyncio
async def test_async_post_call_success_hook_for_unified_finetuning_job():
from litellm.types.utils import LiteLLMFineTuningJob
unified_file_id = "bGl0ZWxsbV9wcm94eTphcHBsaWNhdGlvbi9vY3RldC1zdHJlYW07dW5pZmllZF9pZCxiZTQ0ZDVlYi1mNDU3LTRiNzktOWM4My01N2QxMTMxYWM0YzY7dGFyZ2V0X21vZGVsX25hbWVzLGdwdC00LjEtb3BlbmFpO2xsbV9vdXRwdXRfZmlsZV9pZCxmaWxlLURKMnQ0OWZlQ2NTQk5vNG9oekZ6NGc7bGxtX291dHB1dF9maWxlX21vZGVsX2lkLGRiNjY5ODcwNzdkZTdmYzZjNzAzY2Y1MDczMGU2MmNkOWQ3YTU1N2NlNjVmMDUzNTFkYTM4YTA3ZjBlZDEyNzQ"
provider_ft_job = LiteLLMFineTuningJob(
object="fine_tuning.job",
id="ftjob-0kEBV5b4sPrFcMnuzmYSzU1G",
model="gpt-3.5-turbo-0613",
created_at=1692779769,
finished_at=None,
fine_tuned_model=None,
organization_id="org-dUVLhaAQ37YCGwVC2QVY8sdB",
result_files=[],
status="validating_files",
validation_file=None,
training_file="file-azQuKMLAmiFdEjxpCcbI11zF",
hyperparameters={"n_epochs": 8},
trained_tokens=None,
seed=0,
)
provider_ft_job._hidden_params = {
"unified_file_id": unified_file_id,
"model_id": "gpt-3.5-turbo-0613",
}
proxy_managed_files = _PROXY_LiteLLMManagedFiles(
DualCache(), prisma_client=MagicMock()
)
data = {
"user_api_key_dict": {"parent_otel_span": MagicMock()},
}
response = await proxy_managed_files.async_post_call_success_hook(
data=data,
user_api_key_dict=MagicMock(),
response=provider_ft_job,
)
assert isinstance(response, LiteLLMFineTuningJob)
assert _is_base64_encoded_unified_file_id(response.id)
@pytest.mark.asyncio
async def test_async_pre_call_hook_for_unified_finetuning_job():
from litellm.proxy._types import UserAPIKeyAuth
prisma_client = AsyncMock()
return_value = MagicMock()
return_value.created_by = "123"
prisma_client.db.litellm_managedobjecttable.find_first.return_value = return_value
proxy_managed_files = _PROXY_LiteLLMManagedFiles(
DualCache(), prisma_client=prisma_client
)
data = {
"user_api_key_dict": UserAPIKeyAuth(
user_id="123", parent_otel_span=MagicMock()
),
"data": {
"fine_tuning_job_id": "bGl0ZWxsbV9wcm94eTttb2RlbF9pZDo0OTIxODU4MWY3OGViZTllZjE4NDE0ZmE0ZjdmYjlmYTc0YzA5NWVkMTEyY2E4NDBkZDU2ZGZmZTliZDMwZGQxO2dlbmVyaWNfcmVzcG9uc2VfaWQ6ZnRqb2ItalRCeXM3YlZzYnlaRE93TDlHbHBZcVhS",
},
"call_type": "acancel_fine_tuning_job",
"cache": MagicMock(),
}
response = await proxy_managed_files.async_pre_call_hook(**data)
assert response["fine_tuning_job_id"] == "ftjob-jTBys7bVsbyZDOwL9GlpYqXR"
@pytest.mark.asyncio
@pytest.mark.parametrize("call_type", ["afile_content", "afile_delete"])
async def test_can_user_call_unified_file_id(call_type):
"""
Test that on file retrieve, delete we check if the user has access to the file
"""
from litellm.proxy._types import UserAPIKeyAuth
prisma_client = AsyncMock()
return_value = MagicMock()
return_value.created_by = "123"
prisma_client.db.litellm_managedfiletable.find_first.return_value = return_value
proxy_managed_files = _PROXY_LiteLLMManagedFiles(
MagicMock(), prisma_client=prisma_client
)
unified_file_id = "bGl0ZWxsbV9wcm94eTphcHBsaWNhdGlvbi9vY3RldC1zdHJlYW07dW5pZmllZF9pZCxmMTNlNDAzZS01YWM3LTRhZjktOGQzNS0wNDgwZDMxOTgyYTg7dGFyZ2V0X21vZGVsX25hbWVzLGdwdC00by1taW5pLW9wZW5haTtsbG1fb3V0cHV0X2ZpbGVfaWQsZmlsZS1Ib3UxZDFXc3c1SDNKcjFMYllpZDJiO2xsbV9vdXRwdXRfZmlsZV9tb2RlbF9pZCxmODBiNWU2NzQ1NzdkNjkyMjM4YmVhNTIxZDdiMGI5ZGYyY2FmMTEwMTU2YmU5YzBjM2NjMmNkNTBjOTM1ZDI0"
with pytest.raises(HTTPException) as e:
await proxy_managed_files.async_pre_call_hook(
user_api_key_dict=UserAPIKeyAuth(
user_id="456", parent_otel_span=MagicMock()
),
cache=MagicMock(),
data={"file_id": unified_file_id},
call_type=call_type,
)

11
tests/image_gen_tests/base_image_generation_test.py
View file

@ -68,10 +68,17 @@ class BaseImageGenTest(ABC):
assert logged_standard_logging_payload is not None
assert logged_standard_logging_payload["response_cost"] is not None
assert logged_standard_logging_payload["response_cost"] > 0
import openai
from openai.types.images_response import ImagesResponse
ImagesResponse.model_validate(response.model_dump())
# print openai version
print("openai version=", openai.__version__)
response_dict = dict(response)
if "usage" in response_dict:
response_dict["usage"] = dict(response_dict["usage"])
print("response usage=", response_dict.get("usage"))
ImagesResponse.model_validate(response_dict)
for d in response.data:
assert isinstance(d, Image)

39
tests/litellm/integrations/SlackAlerting/test_slack_alerting_utils.py Normal file
View file

@ -0,0 +1,39 @@
import json
import os
import sys
from typing import Optional
from unittest.mock import MagicMock
import pytest
# Adds the grandparent directory to sys.path to allow importing project modules
sys.path.insert(0, os.path.abspath("../.."))
import litellm
from litellm.integrations.langfuse.langfuse_prompt_management import (
LangfusePromptManagement,
)
from litellm.integrations.SlackAlerting.utils import _add_langfuse_trace_id_to_alert
from litellm.litellm_core_utils.logging_callback_manager import LoggingCallbackManager
@pytest.mark.asyncio
async def test_langfuse_not_initialized_returns_none_early():
"""
Test that when no LangfusePromptManagement is initialized,
the function returns None immediately without executing further logic
"""
# Ensure no Langfuse logger is in the callback manager
litellm.logging_callback_manager = LoggingCallbackManager()
# Create request data that would normally trigger processing
request_data = {"litellm_logging_obj": MagicMock(), "trace_id": "test-trace-id"}
# Call the function
result = await _add_langfuse_trace_id_to_alert(request_data)
# Should return None early without processing request_data
assert result is None
# Verify the litellm_logging_obj was never accessed (early return)
request_data["litellm_logging_obj"].assert_not_called()

1
tests/litellm/integrations/test_custom_prompt_management.py
View file

@ -33,6 +33,7 @@ class TestCustomPromptManagement(CustomPromptManagement):
prompt_id: Optional[str],
prompt_variables: Optional[dict],
dynamic_callback_params: StandardCallbackDynamicParams,
prompt_label: Optional[str],
) -> Tuple[str, List[AllMessageValues], dict]:
print(
"TestCustomPromptManagement: running get_chat_completion_prompt for prompt_id: ",

46
tests/litellm/integrations/test_langfuse.py
View file

@ -1,8 +1,13 @@
import unittest
import asyncio
from unittest.mock import patch, MagicMock
from typing import Optional
import sys
import os
import datetime
import json
import pytest
import litellm
sys.path.insert(0, os.path.abspath("../.."))
from litellm.integrations.langfuse.langfuse import LangFuseLogger
@ -210,3 +215,44 @@ class TestLangfuseUsageDetails(unittest.TestCase):
self.assertEqual(usage_details["output"], 25)
self.assertEqual(usage_details["cache_creation_input_tokens"], 7)
self.assertEqual(usage_details["cache_read_input_tokens"], 4)
def test_max_langfuse_clients_limit():
"""
Test that the max langfuse clients limit is respected when initializing multiple clients
"""
# Set max clients to 2 for testing
with patch(
"litellm.integrations.langfuse.langfuse.MAX_LANGFUSE_INITIALIZED_CLIENTS", 2
):
# Reset the counter
litellm.initialized_langfuse_clients = 0
# First client should succeed
logger1 = LangFuseLogger(
langfuse_public_key="test_key_1",
langfuse_secret="test_secret_1",
langfuse_host="https://test1.langfuse.com",
)
assert litellm.initialized_langfuse_clients == 1
# Second client should succeed
logger2 = LangFuseLogger(
langfuse_public_key="test_key_2",
langfuse_secret="test_secret_2",
langfuse_host="https://test2.langfuse.com",
)
assert litellm.initialized_langfuse_clients == 2
# Third client should fail with exception
with pytest.raises(Exception) as exc_info:
logger3 = LangFuseLogger(
langfuse_public_key="test_key_3",
langfuse_secret="test_secret_3",
langfuse_host="https://test3.langfuse.com",
)
# Verify the error message contains the expected text
assert "Max langfuse clients reached" in str(exc_info.value)
# Counter should still be 2 (third client failed to initialize)
assert litellm.initialized_langfuse_clients == 2

5
tests/litellm/llms/azure/test_azure_common_utils.py
View file

@ -391,6 +391,11 @@ def test_select_azure_base_url_called(setup_mocks):
"add_message",
"arun_thread_stream",
"aresponses",
"acreate_fine_tuning_job",
"acancel_fine_tuning_job",
"alist_fine_tuning_jobs",
"aretrieve_fine_tuning_job",
"afile_list",
]
],
)

8
tests/litellm/llms/openrouter/chat/test_openrouter_chat_transformation.py
View file

@ -26,6 +26,11 @@ class TestOpenRouterChatCompletionStreamingHandler:
"id": "test_id",
"created": 1234567890,
"model": "test_model",
"usage": {
"prompt_tokens": 10,
"completion_tokens": 20,
"total_tokens": 30
},
"choices": [
{"delta": {"content": "test content", "reasoning": "test reasoning"}}
],
@ -39,6 +44,9 @@ class TestOpenRouterChatCompletionStreamingHandler:
assert result.object == "chat.completion.chunk"
assert result.created == 1234567890
assert result.model == "test_model"
assert result.usage.prompt_tokens == chunk["usage"]["prompt_tokens"]
assert result.usage.completion_tokens == chunk["usage"]["completion_tokens"]
assert result.usage.total_tokens == chunk["usage"]["total_tokens"]
assert len(result.choices) == 1
assert result.choices[0]["delta"]["reasoning_content"] == "test reasoning"

0
tests/litellm/proxy/anthropic_endpoints/__init__.py Normal file
View file

61
tests/litellm/proxy/anthropic_endpoints/test_endpoints.py Normal file
View file

@ -0,0 +1,61 @@
"""
Test for anthropic_endpoints/endpoints.py, focusing on handling dictionary objects in streaming responses
"""
import json
import unittest
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from litellm.proxy.anthropic_endpoints.endpoints import async_data_generator_anthropic
class TestAnthropicEndpoints(unittest.TestCase):
@patch("litellm.litellm_core_utils.safe_json_dumps.safe_dumps")
@pytest.mark.asyncio
async def test_async_data_generator_anthropic_dict_handling(self, mock_safe_dumps):
"""Test async_data_generator_anthropic handles dictionary chunks properly"""
# Setup
mock_response = AsyncMock()
mock_response.__aiter__.return_value = [
{"type": "message_start", "message": {"id": "msg_123"}},
"text chunk data",
{"type": "content_block_delta", "delta": {"text": "more data"}},
"text chunk data again",
]
mock_user_api_key_dict = MagicMock()
mock_request_data = {}
mock_proxy_logging_obj = MagicMock()
mock_proxy_logging_obj.async_post_call_streaming_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["response"])
# Configure safe_dumps to return a properly formatted JSON string
mock_safe_dumps.side_effect = lambda chunk: json.dumps(chunk)
# Execute
result = [chunk async for chunk in async_data_generator_anthropic(
response=mock_response,
user_api_key_dict=mock_user_api_key_dict,
request_data=mock_request_data,
proxy_logging_obj=mock_proxy_logging_obj,
)]
# Verify
expected_result = [
'data: {"type": "message_start", "message": {"id": "msg_123"}}\n\n',
'text chunk data',
'data: {"type": "content_block_delta", "delta": {"text": "more data"}}\n\n',
'text chunk data again',
]
self.assertEqual(result, expected_result)
# Assert safe_dumps was called for dictionary objects
mock_safe_dumps.assert_any_call({"type": "message_start", "message": {"id": "msg_123"}})
mock_safe_dumps.assert_any_call({"type": "content_block_delta", "delta": {"text": "more data"}})
assert mock_safe_dumps.call_count == 2 # Called twice, once for each dict object
if __name__ == "__main__":
unittest.main()

2
tests/litellm/proxy/hooks/test_parallel_request_limiter_v2.py
View file

@ -19,6 +19,7 @@ from litellm.proxy.hooks.parallel_request_limiter_v2 import (
from litellm.proxy.utils import InternalUsageCache, ProxyLogging, hash_token
@pytest.mark.flaky(reruns=3)
@pytest.mark.asyncio
async def test_normal_router_call_v2(monkeypatch):
"""
@ -340,6 +341,7 @@ async def test_normal_router_call_rpm(monkeypatch, rate_limit_object):
)
@pytest.mark.flaky(reruns=3)
@pytest.mark.asyncio
async def test_streaming_router_call_v2(monkeypatch):
"""

87
tests/litellm/proxy/management_endpoints/test_customer_endpoints.py Normal file
View file

@ -0,0 +1,87 @@
from unittest.mock import AsyncMock, patch
import pytest
from fastapi import FastAPI, HTTPException
from fastapi.testclient import TestClient
from litellm.proxy._types import (
LiteLLM_BudgetTable,
LiteLLM_EndUserTable,
LitellmUserRoles,
)
from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth
from litellm.proxy.management_endpoints.customer_endpoints import router
from litellm.proxy.proxy_server import ProxyException
app = FastAPI()
app.include_router(router)
client = TestClient(app)
@pytest.fixture
def mock_prisma_client():
with patch("litellm.proxy.proxy_server.prisma_client") as mock:
yield mock
@pytest.fixture
def mock_user_api_key_auth():
with patch("litellm.proxy.proxy_server.user_api_key_auth") as mock:
mock.return_value = UserAPIKeyAuth(
user_id="test-user", user_role=LitellmUserRoles.PROXY_ADMIN
)
yield mock
def test_update_customer_success(mock_prisma_client, mock_user_api_key_auth):
# Mock the database responses
mock_end_user = LiteLLM_EndUserTable(
user_id="test-user-1", alias="Test User", blocked=False
)
updated_mock_end_user = LiteLLM_EndUserTable(
user_id="test-user-1", alias="Updated Test User", blocked=False
)
# Mock the find_first response
mock_prisma_client.db.litellm_endusertable.find_first = AsyncMock(
return_value=mock_end_user
)
# Mock the update response
mock_prisma_client.db.litellm_endusertable.update = AsyncMock(
return_value=updated_mock_end_user
)
# Test data
test_data = {"user_id": "test-user-1", "alias": "Updated Test User"}
# Make the request
response = client.post(
"/customer/update", json=test_data, headers={"Authorization": "Bearer test-key"}
)
# Assert response
assert response.status_code == 200
assert response.json()["user_id"] == "test-user-1"
assert response.json()["alias"] == "Updated Test User"
def test_update_customer_not_found(mock_prisma_client, mock_user_api_key_auth):
# Mock the database response to return None (user not found)
mock_prisma_client.db.litellm_endusertable.find_first = AsyncMock(return_value=None)
# Test data
test_data = {"user_id": "non-existent-user", "alias": "Test User"}
# Make the request
try:
response = client.post(
"/customer/update",
json=test_data,
headers={"Authorization": "Bearer test-key"},
)
except Exception as e:
print(e, type(e))
assert isinstance(e, ProxyException)
assert int(e.code) == 400
assert "End User Id=non-existent-user does not exist in db" in e.message

39
tests/litellm/proxy/test_team_member_update.py Normal file
View file

@ -0,0 +1,39 @@
import pytest
from fastapi import HTTPException
from starlette.requests import Request
from litellm.proxy.management_endpoints.team_endpoints import team_member_update
from litellm.proxy._types import TeamMemberUpdateRequest
import litellm.proxy.proxy_server as proxy_server
@pytest.mark.asyncio
async def test_team_member_update_admin_requires_premium(monkeypatch):
# Arrange: patch prisma_client and premium_user
monkeypatch.setattr(proxy_server, 'prisma_client', object())
monkeypatch.setattr(proxy_server, 'premium_user', False)
# Create a request body that tries to set role=admin
data = TeamMemberUpdateRequest(
team_id="team-1234",
user_id="user-1",
user_email=None,
role="admin",
max_budget_in_team=None,
)
scope = {"type": "http", "method": "POST", "path": "/team/member_update"}
request = Request(scope)
# We don't need a full auth object since premium check happens before auth is used
auth = object()
# Act & Assert: expect HTTPException 400 with the exact premium feature message
with pytest.raises(HTTPException) as exc_info:
await team_member_update(data, request, auth)
assert exc_info.value.status_code == 400
expected_msg = (
"Assigning team admins is a premium feature. You must be a LiteLLM Enterprise user to use this feature. "
"If you have a license please set `LITELLM_LICENSE` in your env. Get a 7 day trial key here: https://www.litellm.ai/#trial. "
"Pricing: https://www.litellm.ai/#pricing"
)
assert exc_info.value.detail == expected_msg

138
tests/litellm/secret_managers/test_get_azure_ad_token_provider.py Normal file
View file

@ -0,0 +1,138 @@
import json
import os
import sys
from typing import Optional
from unittest.mock import MagicMock, patch
# Adds the grandparent directory to sys.path to allow importing project modules
sys.path.insert(0, os.path.abspath("../.."))
import pytest
from litellm.secret_managers.get_azure_ad_token_provider import (
get_azure_ad_token_provider,
)
class TestGetAzureAdTokenProvider:
@patch.dict(
os.environ,
{
"AZURE_CLIENT_ID": "test-client-id",
"AZURE_CLIENT_SECRET": "test-client-secret",
"AZURE_TENANT_ID": "test-tenant-id",
"AZURE_SCOPE": "https://cognitiveservices.azure.com/.default",
"AZURE_CREDENTIAL": "ClientSecretCredential",
},
)
@patch("azure.identity.get_bearer_token_provider")
@patch("azure.identity.ClientSecretCredential")
def test_get_azure_ad_token_provider_client_secret_credential(
self, mock_client_secret_credential, mock_get_bearer_token_provider
):
"""Test get_azure_ad_token_provider with ClientSecretCredential."""
# Mock the Azure identity credential instance
mock_credential_instance = MagicMock()
mock_client_secret_credential.return_value = mock_credential_instance
# Mock the bearer token provider
mock_token_provider = MagicMock(return_value="mock-token")
mock_get_bearer_token_provider.return_value = mock_token_provider
# Call the function
result = get_azure_ad_token_provider()
# Assertions
assert callable(result)
mock_client_secret_credential.assert_called_once_with(
client_id="test-client-id",
client_secret="test-client-secret",
tenant_id="test-tenant-id",
)
mock_get_bearer_token_provider.assert_called_once_with(
mock_credential_instance, "https://cognitiveservices.azure.com/.default"
)
# Test that the returned callable works
token = result()
assert token == "mock-token"
@patch.dict(
os.environ,
{
"AZURE_CLIENT_ID": "test-client-id",
"AZURE_SCOPE": "https://cognitiveservices.azure.com/.default",
"AZURE_CREDENTIAL": "ManagedIdentityCredential",
},
)
@patch("azure.identity.get_bearer_token_provider")
@patch("azure.identity.ManagedIdentityCredential")
def test_get_azure_ad_token_provider_managed_identity_credential(
self, mock_managed_identity_credential, mock_get_bearer_token_provider
):
"""Test get_azure_ad_token_provider with ManagedIdentityCredential."""
# Mock the Azure identity credential instance
mock_credential_instance = MagicMock()
mock_managed_identity_credential.return_value = mock_credential_instance
# Mock the bearer token provider
mock_token_provider = MagicMock(return_value="mock-managed-identity-token")
mock_get_bearer_token_provider.return_value = mock_token_provider
# Call the function
result = get_azure_ad_token_provider()
# Assertions
assert callable(result)
mock_managed_identity_credential.assert_called_once_with(
client_id="test-client-id"
)
mock_get_bearer_token_provider.assert_called_once_with(
mock_credential_instance, "https://cognitiveservices.azure.com/.default"
)
# Test that the returned callable works
token = result()
assert token == "mock-managed-identity-token"
@patch.dict(
os.environ,
{
"AZURE_CLIENT_ID": "test-client-id",
"AZURE_TENANT_ID": "test-tenant-id",
"AZURE_CERTIFICATE_PATH": "/path/to/cert.pem",
"AZURE_SCOPE": "https://cognitiveservices.azure.com/.default",
"AZURE_CREDENTIAL": "CertificateCredential",
},
)
@patch("azure.identity.get_bearer_token_provider")
@patch("azure.identity.CertificateCredential")
def test_get_azure_ad_token_provider_certificate_credential(
self, mock_certificate_credential, mock_get_bearer_token_provider
):
"""Test get_azure_ad_token_provider with CertificateCredential."""
# Mock the Azure identity credential instance
mock_credential_instance = MagicMock()
mock_certificate_credential.return_value = mock_credential_instance
# Mock the bearer token provider
mock_token_provider = MagicMock(return_value="mock-certificate-token")
mock_get_bearer_token_provider.return_value = mock_token_provider
# Call the function
result = get_azure_ad_token_provider()
# Assertions
assert callable(result)
mock_certificate_credential.assert_called_once_with(
client_id="test-client-id",
tenant_id="test-tenant-id",
certificate_path="/path/to/cert.pem",
)
mock_get_bearer_token_provider.assert_called_once_with(
mock_credential_instance, "https://cognitiveservices.azure.com/.default"
)
# Test that the returned callable works
token = result()
assert token == "mock-certificate-token"

51
tests/llm_responses_api_testing/test_openai_responses_api.py
View file

@ -2,7 +2,7 @@ import os
import sys
import pytest
import asyncio
from typing import Optional
from typing import Optional, cast
from unittest.mock import patch, AsyncMock
sys.path.insert(0, os.path.abspath("../.."))
@ -1032,4 +1032,51 @@ def test_basic_computer_use_preview_tool_call():
# Validate the input format
assert isinstance(request_body["input"], str)
assert request_body["input"] == "Check the latest OpenAI news on bing.com."
def test_mcp_tools_with_responses_api():
litellm._turn_on_debug()
MCP_TOOLS = [
{
"type": "mcp",
"server_label": "deepwiki",
"server_url": "https://mcp.deepwiki.com/mcp",
"allowed_tools": ["ask_question"]
}
]
MODEL = "openai/gpt-4.1"
USER_QUERY = "What transport protocols does the 2025-03-26 version of the MCP spec (modelcontextprotocol/modelcontextprotocol) support?"
#########################################################
# Step 1: OpenAI will use MCP LIST, and return a list of MCP calls for our approval
response = litellm.responses(
model=MODEL,
tools=MCP_TOOLS,
input=USER_QUERY
)
print(response)
response = cast(ResponsesAPIResponse, response)
mcp_approval_id: Optional[str]
for output in response.output:
if output.type == "mcp_approval_request":
mcp_approval_id = output.id
break
# Step 2: Send followup with approval for the MCP call
response_with_mcp_call = litellm.responses(
model=MODEL,
tools=MCP_TOOLS,
input=[
{
"type": "mcp_approval_response",
"approve": True,
"approval_request_id": mcp_approval_id
}
],
previous_response_id=response.id,
)
print(response_with_mcp_call)

82
tests/local_testing/test_custom_llm.py
View file

@ -44,7 +44,7 @@ from litellm import (
image_generation,
)
from litellm.utils import ModelResponseIterator
from litellm.types.utils import ImageResponse, ImageObject
from litellm.types.utils import ImageResponse, ImageObject, EmbeddingResponse
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
@ -257,6 +257,53 @@ class MyCustomLLM(CustomLLM):
response_ms=1000,
)
def embedding(
self,
model: str,
input: list,
model_response: EmbeddingResponse,
print_verbose: Callable,
logging_obj: Any,
optional_params: dict,
litellm_params=None,
aembedding=None,
) -> EmbeddingResponse:
model_response.model = model
model_response.data = [
{
"object": "embedding",
"embedding": [0.1, 0.2, 0.3],
"index": i,
}
for i, _ in enumerate(input)
]
return model_response
async def aembedding(
self,
model: str,
input: list,
model_response: EmbeddingResponse,
print_verbose: Callable,
logging_obj: Any,
optional_params: dict,
litellm_params=None,
) -> EmbeddingResponse:
model_response.model = model
model_response.data = [
{
"object": "embedding",
"embedding": [0.1, 0.2, 0.3],
"index": i,
}
for i, _ in enumerate(input)
]
return model_response
def test_get_llm_provider():
""""""
@ -452,3 +499,36 @@ def test_get_supported_openai_params():
response = get_supported_openai_params(model="my-custom-llm/my-fake-model")
assert response is not None
def test_simple_embedding():
my_custom_llm = MyCustomLLM()
litellm.custom_provider_map = [
{"provider": "custom_llm", "custom_handler": my_custom_llm}
]
resp = litellm.embedding(
model="custom_llm/my-fake-model",
input=["good morning from litellm", "good night from litellm"]
)
assert resp.data[1] == {
"object": "embedding",
"embedding": [0.1, 0.2, 0.3],
"index": 1,
}
@pytest.mark.asyncio
async def test_simple_aembedding():
my_custom_llm = MyCustomLLM()
litellm.custom_provider_map = [
{"provider": "custom_llm", "custom_handler": my_custom_llm}
]
resp = await litellm.aembedding(
model="custom_llm/my-fake-model",
input=["good morning from litellm", "good night from litellm"]
)
assert resp.data[1] == {
"object": "embedding",
"embedding": [0.1, 0.2, 0.3],
"index": 1,
}

3
tests/local_testing/test_stream_chunk_builder.py
View file

@ -931,9 +931,6 @@ def execute_completion(opts: dict):
print("\n\n")
assembly = litellm.stream_chunk_builder(partial_streaming_chunks)
print(f"assembly.choices[0].message.tool_calls: {assembly.choices[0].message.tool_calls}")
assert len(assembly.choices[0].message.tool_calls) == 3, (
assembly.choices[0].message.tool_calls[0].function.arguments[0]
)
print(assembly.choices[0].message.tool_calls)
for tool_call in assembly.choices[0].message.tool_calls:
json.loads(tool_call.function.arguments) # assert valid json - https://github.com/BerriAI/litellm/issues/10034

4
tests/logging_callback_tests/langfuse_expected_request_body/completion.json
View file

@ -62,9 +62,7 @@
"endTime": "2025-01-16T11:28:55.124353-08:00",
"completionStartTime": "2025-01-16T11:28:55.124353-08:00",
"model": "gpt-3.5-turbo",
"modelParameters": {
"extra_body": "{}"
},
"modelParameters": {},
"usage": {
"input": 10,
"output": 20,

Some files were not shown because too many files have changed in this diff Show more