scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-13 23:11:40 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 101

Merge branch 'litellm_internal_staging' into fix/copilot-premium-request-billing

Browse source
This commit is contained in:
Jay Stothard 2026-05-05 10:02:51 +01:00 • committed by GitHub
commit 1edbdcef11
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1235 changed files with 102203 additions and 11417 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
.circleci/config.yml
View file

@ -1475,7 +1475,7 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -v tests/otel_tests -x --junitxml=test-results/junit.xml --durations=5
uv run --no-sync python -m pytest -v tests/otel_tests --junitxml=test-results/junit.xml --durations=5
no_output_timeout: 15m
# Clean up first container
- run:
@ -1935,7 +1935,7 @@ jobs:
name: Run Vertex AI, Google AI Studio Node.js tests
command: |
cd tests/pass_through_tests
npx jest . --verbose
NODE_OPTIONS=--experimental-vm-modules npx jest . --verbose
no_output_timeout: 30m
- run:
name: Run tests
@ -2138,17 +2138,23 @@ jobs:
- ~/.cache/uv
- restore_cache:
keys:
- ui-e2e-node-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
- ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
- run:
name: Install Node dependencies and Playwright
# The cimg/python:3.12-browsers image already ships the Chromium system
# libraries Playwright needs (libnss3, libatk-bridge2.0-0, libcups2, etc.).
# `--with-deps` triggers a redundant apt-get update + install that adds
# 5-10 minutes to the job and frequently stalls on flaky Ubuntu mirrors,
# so we install just the browser binary.
command: |
cd ui/litellm-dashboard
npm ci
npx playwright install chromium --with-deps
npx playwright install chromium
- save_cache:
key: ui-e2e-node-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
key: ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
paths:
- ui/litellm-dashboard/node_modules
- ~/.cache/ms-playwright
- run:
name: Build UI from source
# Prior version used `cp -r out/ ../../litellm/proxy/_experimental/out/`.

8
.github/workflows/create-release-branch.yml vendored
View file

@ -4,7 +4,7 @@ on:
workflow_dispatch:
inputs:
tag:
description: "Release tag (e.g. v1.83.0-stable) — branch will be named release/<tag>"
description: "Release tag (e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, 1.84.0.post1; legacy v1.83.10-stable still accepted) — branch will be named release/<tag>"
required: true
type: string
commit_hash:
@ -14,7 +14,7 @@ on:
workflow_call:
inputs:
tag:
description: "Release tag"
description: "Release tag (e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, 1.84.0.post1; legacy v1.83.10-stable still accepted)"
required: true
type: string
commit_hash:
@ -40,8 +40,8 @@ jobs:
echo "::error::commit_hash must be a full 40-character commit SHA"
exit 1
fi
if ! echo "${TAG}" | grep -qE '^v[0-9]+\.[0-9]+\.[0-9]+'; then
echo "::error::tag must start with vX.Y.Z"
if ! echo "${TAG}" | grep -qE '^v?[0-9]+\.[0-9]+\.[0-9]+'; then
echo "::error::tag must start with X.Y.Z (optional leading v), e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, or v1.83.10-stable"
exit 1
fi

15
.github/workflows/create-release.yml vendored
View file

@ -4,7 +4,7 @@ on:
workflow_dispatch:
inputs:
tag:
description: "Release tag (e.g. v1.83.0-stable)"
description: "Release tag (e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, 1.84.0-dev.2, 1.84.0.post1; legacy v1.83.10-stable still accepted)"
required: true
type: string
commit_hash:
@ -30,8 +30,8 @@ jobs:
echo "::error::commit_hash must be a full 40-character commit SHA"
exit 1
fi
if ! echo "${TAG}" | grep -qE '^v[0-9]+\.[0-9]+\.[0-9]+'; then
echo "::error::tag must start with vX.Y.Z"
if ! echo "${TAG}" | grep -qE '^v?[0-9]+\.[0-9]+\.[0-9]+'; then
echo "::error::tag must start with X.Y.Z (optional leading v), e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, or v1.83.10-stable"
exit 1
fi
@ -45,6 +45,13 @@ jobs:
const tag = process.env.TAG;
const commitHash = process.env.COMMIT_HASH;
// Mark RC / dev / nightly / alpha / beta tags as GitHub pre-releases.
// Accept both PEP 440 (`.dev`) and SemVer (`-dev`) separators so tags
// like `1.84.0.dev2` and `1.84.0-dev.2` are both detected.
// PEP 440 post-releases (e.g. `1.84.0.post1`) and legacy `-stable[.patch.N]`
// are stable maintenance releases, not pre-releases.
const isPrerelease = /(?:rc|nightly|alpha|beta|[-.]dev)/i.test(tag);
const cosignSection = [
`## Verify Docker Image Signature`,
``,
@ -89,7 +96,7 @@ jobs:
target_commitish: commitHash,
name: tag,
owner: context.repo.owner,
prerelease: false,
prerelease: isPrerelease,
repo: context.repo.repo,
tag_name: tag,
});

4
.gitignore vendored
View file

@ -91,7 +91,6 @@ test.py
litellm_config.yaml
!.github/observatory/litellm_config.yaml
.cursor
.vscode/launch.json
litellm/proxy/to_delete_loadtest_work/*
update_model_cost_map.py
tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
@ -102,6 +101,9 @@ STABILIZATION_TODO.md
**/playwright-report
**/*.storageState.json
**/coverage
# GSD agent
.gsd/
.bg-shell/
test-config

2
.npmrc
View file

@ -2,4 +2,4 @@
# Packages needing lifecycle scripts: npm rebuild <pkg>
ignore-scripts=true
# Protects local npm install only — npm ci (used in CI) ignores this
min-release-age=3d
min-release-age=3

6
Dockerfile
View file

@ -1,9 +1,9 @@
# Base image for building
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:f26d42a15d09d9a643b231df929fa3cf609bedc58a728eb445be89a9d8d1da9f
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:3258be472764337fd13095bcbb3182da170243b5819fd67ad4c0754590588b31
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:f26d42a15d09d9a643b231df929fa3cf609bedc58a728eb445be89a9d8d1da9f
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:733b4042187702f832f7fdecb3aff14a61b288c4ca37af188bb5715c1caebaf8
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:3258be472764337fd13095bcbb3182da170243b5819fd67ad4c0754590588b31
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin

3
Makefile
View file

@ -185,3 +185,6 @@ test-llm-translation-single: install-test-deps
$(UV_RUN) pytest tests/llm_translation/$(FILE) \
--junitxml=test-results/junit.xml \
-v --tb=short --maxfail=100 --timeout=300
test-llm-translation-flush-vcr-cache:
$(UV_RUN) python tests/_flush_vcr_cache.py

2
README.md
View file

@ -68,7 +68,7 @@ Managing LLM calls across providers gets complicated fast — different SDKs, au
<td><img height="60" alt="Stripe" src="https://github.com/user-attachments/assets/f7296d4f-9fbd-460d-9d05-e4df31697c4b" /></td>
<td><img height="60" alt="image" src="https://github.com/user-attachments/assets/436fca71-988b-40bb-b5fe-8450c80fdbd0" /></td>
<td><img height="60" alt="Google ADK" src="https://github.com/user-attachments/assets/caf270a2-5aee-45c4-8222-41a2070c4f19" /></td>
<td><img height="60" alt="Greptile" src="https://github.com/user-attachments/assets/0be4bd8a-7cfa-48d3-9090-f415fe948280" /></td>
<td><img height="60" alt="Greptile" src="https://github.com/user-attachments/assets/3db0ae72-0843-4005-a56d-bba1dde2193d" /></td>
<td><img height="60" alt="OpenHands" src="https://github.com/user-attachments/assets/a6150c4c-149e-4cae-888b-8b92be6e003f" /></td>
<td><h2>Netflix</h2></td>
<td><img height="60" alt="OpenAI Agents SDK" src="https://github.com/user-attachments/assets/c02f7be0-8c2e-4d27-aea7-7c024bfaebc0" /></td>

2
cookbook/litellm-ollama-docker-image/requirements.txt
View file

@ -1 +1 @@
litellm==1.83.5
litellm==1.83.14

2
docker/Dockerfile.alpine
View file

@ -3,7 +3,7 @@ ARG LITELLM_BUILD_IMAGE=python:3.11-alpine@sha256:f07e2ace46f560f09a6eeec7b4913b
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=python:3.11-alpine@sha256:f07e2ace46f560f09a6eeec7b4913b80ee99546e749ef82342a419a326620856
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:733b4042187702f832f7fdecb3aff14a61b288c4ca37af188bb5715c1caebaf8
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin

6
docker/Dockerfile.database
View file

@ -1,9 +1,9 @@
# Base image for building
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:f26d42a15d09d9a643b231df929fa3cf609bedc58a728eb445be89a9d8d1da9f
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:3258be472764337fd13095bcbb3182da170243b5819fd67ad4c0754590588b31
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:f26d42a15d09d9a643b231df929fa3cf609bedc58a728eb445be89a9d8d1da9f
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:733b4042187702f832f7fdecb3aff14a61b288c4ca37af188bb5715c1caebaf8
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:3258be472764337fd13095bcbb3182da170243b5819fd67ad4c0754590588b31
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin

2
docker/Dockerfile.dev
View file

@ -3,7 +3,7 @@ ARG LITELLM_BUILD_IMAGE=python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973a
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:733b4042187702f832f7fdecb3aff14a61b288c4ca37af188bb5715c1caebaf8
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin

2
docker/Dockerfile.health_check
View file

@ -1,4 +1,4 @@
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:733b4042187702f832f7fdecb3aff14a61b288c4ca37af188bb5715c1caebaf8
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin
FROM python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d

8
docker/Dockerfile.non_root
View file

@ -1,8 +1,8 @@
# Base images
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:f26d42a15d09d9a643b231df929fa3cf609bedc58a728eb445be89a9d8d1da9f
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:f26d42a15d09d9a643b231df929fa3cf609bedc58a728eb445be89a9d8d1da9f
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:3258be472764337fd13095bcbb3182da170243b5819fd67ad4c0754590588b31
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:3258be472764337fd13095bcbb3182da170243b5819fd67ad4c0754590588b31
ARG PROXY_EXTRAS_SOURCE=published
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:733b4042187702f832f7fdecb3aff14a61b288c4ca37af188bb5715c1caebaf8
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin
@ -32,7 +32,6 @@ ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
PATH="/app/.venv/bin:${PATH}" \
LITELLM_NON_ROOT=true \
PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
PRISMA_CLI_BINARY_TARGETS="debian-openssl-3.0.x" \
XDG_CACHE_HOME=/app/.cache
# Copy dependency metadata first for layer caching
@ -114,7 +113,6 @@ COPY --from=builder /app/docker/supervisord.conf /etc/supervisord.conf
ENV PATH="/app/.venv/bin:${PATH}" \
PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
PRISMA_CLI_BINARY_TARGETS="debian-openssl-3.0.x" \
HOME=/app \
LITELLM_NON_ROOT=true \
XDG_CACHE_HOME=/app/.cache \

196
docs/my-website/docs/providers/crusoe.md Normal file
View file

@ -0,0 +1,196 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Crusoe
## Overview
| Property | Details |
|-------|-------|
| Description | Crusoe Cloud provides GPU-accelerated inference for open-source large language models, optimized for performance and cost efficiency. |
| Provider Route on LiteLLM | `crusoe/` |
| Link to Provider Doc | [Crusoe Managed Inference Documentation ↗](https://docs.crusoecloud.com/managed-inference/overview/index.html) |
| Base URL | `https://managed-inference-api-proxy.crusoecloud.com/v1` |
| Supported Operations | [`/chat/completions`](#sample-usage) |
<br />
<br />
**We support ALL Crusoe models, just set `crusoe/` as a prefix when sending completion requests**
## Available Models
| Model | Description | Context Window |
|-------|-------------|----------------|
| `crusoe/deepseek-ai/DeepSeek-R1-0528` | DeepSeek R1 reasoning model (May 2025) | 163,840 tokens |
| `crusoe/deepseek-ai/DeepSeek-V3-0324` | DeepSeek V3 chat model (March 2025) | 163,840 tokens |
| `crusoe/google/gemma-3-12b-it` | Google Gemma 3 12B instruction-tuned | 131,072 tokens |
| `crusoe/meta-llama/Llama-3.3-70B-Instruct` | Llama 3.3 70B instruction-tuned | 131,072 tokens |
| `crusoe/moonshotai/Kimi-K2-Thinking` | Kimi K2 extended thinking model | 262,144 tokens |
| `crusoe/openai/gpt-oss-120b` | OpenAI 120B open-source model | 131,072 tokens |
| `crusoe/Qwen/Qwen3-235B-A22B-Instruct-2507` | Qwen3 235B MoE instruction-tuned | 262,144 tokens |
## Required Variables
```python showLineNumbers title="Environment Variables"
os.environ["CRUSOE_API_KEY"] = "" # your Crusoe API key
```
## Usage - LiteLLM Python SDK
### Non-streaming
```python showLineNumbers title="Crusoe Non-streaming Completion"
import os
import litellm
from litellm import completion
os.environ["CRUSOE_API_KEY"] = "" # your Crusoe API key
messages = [{"content": "Hello, how are you?", "role": "user"}]
# Crusoe call
response = completion(
model="crusoe/meta-llama/Llama-3.3-70B-Instruct",
messages=messages
)
print(response)
```
### Streaming
```python showLineNumbers title="Crusoe Streaming Completion"
import os
import litellm
from litellm import completion
os.environ["CRUSOE_API_KEY"] = "" # your Crusoe API key
messages = [{"content": "Write a short story about AI", "role": "user"}]
# Crusoe call with streaming
response = completion(
model="crusoe/meta-llama/Llama-3.3-70B-Instruct",
messages=messages,
stream=True
)
for chunk in response:
print(chunk)
```
### Function Calling
```python showLineNumbers title="Crusoe Function Calling"
import os
import litellm
from litellm import completion
os.environ["CRUSOE_API_KEY"] = "" # your Crusoe API key
tools = [{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the current weather in a location",
"parameters": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The city and state, e.g. San Francisco, CA"
}
},
"required": ["location"]
}
}
}]
messages = [{"role": "user", "content": "What's the weather in Boston?"}]
response = completion(
model="crusoe/meta-llama/Llama-3.3-70B-Instruct",
messages=messages,
tools=tools,
tool_choice="auto"
)
print(response)
```
## Usage - LiteLLM Proxy Server
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: llama-3.3-70b
litellm_params:
model: crusoe/meta-llama/Llama-3.3-70B-Instruct
api_key: os.environ/CRUSOE_API_KEY
- model_name: deepseek-r1
litellm_params:
model: crusoe/deepseek-ai/DeepSeek-R1-0528
api_key: os.environ/CRUSOE_API_KEY
- model_name: deepseek-v3
litellm_params:
model: crusoe/deepseek-ai/DeepSeek-V3-0324
api_key: os.environ/CRUSOE_API_KEY
- model_name: qwen3-235b
litellm_params:
model: crusoe/Qwen/Qwen3-235B-A22B-Instruct-2507
api_key: os.environ/CRUSOE_API_KEY
- model_name: kimi-k2
litellm_params:
model: crusoe/moonshotai/Kimi-K2-Thinking
api_key: os.environ/CRUSOE_API_KEY
```
## Custom API Base
**Option 1: Environment variable**
```python showLineNumbers title="Custom API Base via env var"
import os
from litellm import completion
os.environ["CRUSOE_API_BASE"] = "https://custom.crusoecloud.com/v1"
os.environ["CRUSOE_API_KEY"] = "" # your API key
response = completion(
model="crusoe/meta-llama/Llama-3.3-70B-Instruct",
messages=[{"content": "Hello!", "role": "user"}],
)
```
**Option 2: Pass directly**
```python showLineNumbers title="Custom API Base via parameter"
from litellm import completion
response = completion(
model="crusoe/meta-llama/Llama-3.3-70B-Instruct",
messages=[{"content": "Hello!", "role": "user"}],
api_base="https://custom.crusoecloud.com/v1",
api_key="your-api-key",
)
```
## Supported OpenAI Parameters
- `temperature`
- `max_tokens`
- `max_completion_tokens`
- `top_p`
- `frequency_penalty`
- `presence_penalty`
- `stop`
- `n`
- `stream`
- `tools`
- `tool_choice`
- `response_format`
- `seed`
- `user`
- `logit_bias`
- `logprobs`
- `top_logprobs`

26
enterprise/enterprise_hooks/banned_keywords.py
View file

@ -11,6 +11,10 @@ from typing import Literal
import litellm
from litellm.caching.caching import DualCache
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.guardrails._content_utils import (
is_text_content_call_type,
iter_message_text,
)
from litellm.integrations.custom_logger import CustomLogger
from litellm._logging import verbose_proxy_logger
from fastapi import HTTPException
@ -73,10 +77,9 @@ class _ENTERPRISE_BannedKeywords(CustomLogger):
- check if user id part of blocked list
"""
self.print_verbose("Inside Banned Keyword List Pre-Call Hook")
if call_type == "completion" and "messages" in data:
for m in data["messages"]:
if "content" in m and isinstance(m["content"], str):
self.test_violation(test_str=m["content"])
if is_text_content_call_type(call_type):
for text in iter_message_text(data):
self.test_violation(test_str=text)
except HTTPException as e:
raise e
@ -93,11 +96,16 @@ class _ENTERPRISE_BannedKeywords(CustomLogger):
user_api_key_dict: UserAPIKeyAuth,
response,
):
if isinstance(response, litellm.ModelResponse) and isinstance(
response.choices[0], litellm.utils.Choices
):
for word in self.banned_keywords_list:
self.test_violation(test_str=response.choices[0].message.content or "")
if not isinstance(response, litellm.ModelResponse):
return
for choice in response.choices:
if not isinstance(choice, litellm.utils.Choices):
continue
message = getattr(choice, "message", None)
content = getattr(message, "content", None)
if isinstance(content, str):
self.test_violation(test_str=content)
async def async_post_call_streaming_hook(
self,

9
enterprise/enterprise_hooks/google_text_moderation.py
View file

@ -12,6 +12,7 @@ import litellm
from litellm._logging import verbose_proxy_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.guardrails._content_utils import iter_message_text
from litellm.types.utils import CallTypesLiteral
@ -94,11 +95,9 @@ class _ENTERPRISE_GoogleTextModeration(CustomLogger):
- Calls Google's Text Moderation API
- Rejects request if it fails safety check
"""
if "messages" in data and isinstance(data["messages"], list):
text = ""
for m in data["messages"]: # assume messages is a list
if "content" in m and isinstance(m["content"], str):
text += m["content"]
# Covers multimodal list content + Responses-API input.
text = "".join(iter_message_text(data))
if text:
document = self.language_document(content=text, type_=self.document_type)
request = self.moderate_text_request(

8
enterprise/enterprise_hooks/openai_moderation.py
View file

@ -19,6 +19,7 @@ import litellm
from litellm._logging import verbose_proxy_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.guardrails._content_utils import iter_message_text
from litellm.types.utils import CallTypesLiteral
@ -37,11 +38,8 @@ class _ENTERPRISE_OpenAI_Moderation(CustomLogger):
user_api_key_dict: UserAPIKeyAuth,
call_type: CallTypesLiteral,
):
text = ""
if "messages" in data and isinstance(data["messages"], list):
for m in data["messages"]: # assume messages is a list
if "content" in m and isinstance(m["content"], str):
text += m["content"]
# Covers multimodal list content + Responses-API input.
text = "".join(iter_message_text(data))
from litellm.proxy.proxy_server import llm_router

64
enterprise/litellm_enterprise/enterprise_callbacks/secret_detection.py
View file

@ -18,6 +18,7 @@ from litellm._logging import verbose_proxy_logger
from litellm.caching.caching import DualCache
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.guardrails._content_utils import walk_user_text
GUARDRAIL_NAME = "hide_secrets"
@ -473,23 +474,19 @@ class _ENTERPRISE_SecretDetection(CustomGuardrail):
if await self.should_run_check(user_api_key_dict) is False:
return
if "messages" in data and isinstance(data["messages"], list):
for message in data["messages"]:
if "content" in message and isinstance(message["content"], str):
detected_secrets = self.scan_message_for_secrets(message["content"])
# Covers multimodal list content + Responses-API input.
def _redact_message_text(text: str) -> str:
detected_secrets = self.scan_message_for_secrets(text)
for secret in detected_secrets:
text = text.replace(secret["value"], "[REDACTED]")
if detected_secrets:
secret_types = [secret["type"] for secret in detected_secrets]
verbose_proxy_logger.warning(
f"Detected and redacted secrets in message: {secret_types}"
)
return text
for secret in detected_secrets:
message["content"] = message["content"].replace(
secret["value"], "[REDACTED]"
)
if len(detected_secrets) > 0:
secret_types = [secret["type"] for secret in detected_secrets]
verbose_proxy_logger.warning(
f"Detected and redacted secrets in message: {secret_types}"
)
else:
verbose_proxy_logger.debug("No secrets detected on input.")
walk_user_text(data, _redact_message_text)
if "prompt" in data:
if isinstance(data["prompt"], str):
@ -504,11 +501,15 @@ class _ENTERPRISE_SecretDetection(CustomGuardrail):
f"Detected and redacted secrets in prompt: {secret_types}"
)
elif isinstance(data["prompt"], list):
for item in data["prompt"]:
# Index back into the list — assigning to ``item`` would only
# rebind the loop variable and leave ``data["prompt"]``
# carrying the unredacted secret.
for idx, item in enumerate(data["prompt"]):
if isinstance(item, str):
detected_secrets = self.scan_message_for_secrets(item)
for secret in detected_secrets:
item = item.replace(secret["value"], "[REDACTED]")
data["prompt"][idx] = item
if len(detected_secrets) > 0:
secret_types = [
secret["type"] for secret in detected_secrets
@ -517,31 +518,6 @@ class _ENTERPRISE_SecretDetection(CustomGuardrail):
f"Detected and redacted secrets in prompt: {secret_types}"
)
if "input" in data:
if isinstance(data["input"], str):
detected_secrets = self.scan_message_for_secrets(data["input"])
for secret in detected_secrets:
data["input"] = data["input"].replace(secret["value"], "[REDACTED]")
if len(detected_secrets) > 0:
secret_types = [secret["type"] for secret in detected_secrets]
verbose_proxy_logger.warning(
f"Detected and redacted secrets in input: {secret_types}"
)
elif isinstance(data["input"], list):
_input_in_request = data["input"]
for idx, item in enumerate(_input_in_request):
if isinstance(item, str):
detected_secrets = self.scan_message_for_secrets(item)
for secret in detected_secrets:
_input_in_request[idx] = item.replace(
secret["value"], "[REDACTED]"
)
if len(detected_secrets) > 0:
secret_types = [
secret["type"] for secret in detected_secrets
]
verbose_proxy_logger.warning(
f"Detected and redacted secrets in input: {secret_types}"
)
verbose_proxy_logger.debug("Data after redacting input %s", data)
# ``data["input"]`` (Responses API and embeddings/moderation) is
# already covered by ``walk_user_text`` above.
return

54
enterprise/litellm_enterprise/proxy/auth/custom_sso_handler.py
View file

@ -10,28 +10,21 @@ has already authenticated the user) and you need to extract user information fro
custom headers or other request attributes.
"""
from typing import TYPE_CHECKING, Dict, Optional, Union, cast
from typing import cast
from fastapi import Request
from fastapi.responses import RedirectResponse
if TYPE_CHECKING:
from fastapi_sso.sso.base import OpenID
else:
from typing import Any as OpenID
from litellm.proxy.management_endpoints.types import CustomOpenID
class EnterpriseCustomSSOHandler:
"""
Enterprise Custom SSO Handler for LiteLLM Proxy
This class provides methods for handling custom SSO authentication flows
where users can implement their own authentication logic by processing
request headers and returning user information in OpenID format.
"""
@staticmethod
async def handle_custom_ui_sso_sign_in(
request: Request,
@ -40,16 +33,16 @@ class EnterpriseCustomSSOHandler:
Allow a user to execute their custom code to parse incoming request headers and return a OpenID object
Use this when you have an OAuth proxy in front of LiteLLM (where the OAuth proxy has already authenticated the user)
Args:
request: The FastAPI request object containing headers and other request data
Returns:
RedirectResponse: Redirect response that sends the user to the LiteLLM UI with authentication token
Raises:
ValueError: If custom_ui_sso_sign_in_handler is not configured
Example:
This method is typically called when a user has already been authenticated by an
external OAuth proxy and the proxy has added custom headers containing user information.
@ -60,27 +53,44 @@ class EnterpriseCustomSSOHandler:
from litellm.integrations.custom_sso_handler import CustomSSOLoginHandler
from litellm.proxy.proxy_server import (
CommonProxyErrors,
general_settings,
premium_user,
user_custom_ui_sso_sign_in_handler,
)
from litellm.proxy.auth.trusted_proxy_utils import (
require_trusted_proxy_request,
)
if premium_user is not True:
raise ValueError(CommonProxyErrors.not_premium_user.value)
if user_custom_ui_sso_sign_in_handler is None:
raise ValueError("custom_ui_sso_sign_in_handler is not configured. Please set it in general_settings.")
custom_sso_login_handler = cast(CustomSSOLoginHandler, user_custom_ui_sso_sign_in_handler)
openid_response: OpenID = await custom_sso_login_handler.handle_custom_ui_sso_sign_in(
raise ValueError(
"custom_ui_sso_sign_in_handler is not configured. Please set it in general_settings."
)
require_trusted_proxy_request(
request=request,
general_settings=general_settings,
feature_name="Custom UI SSO",
)
custom_sso_login_handler = cast(
CustomSSOLoginHandler, user_custom_ui_sso_sign_in_handler
)
openid_response: OpenID = (
await custom_sso_login_handler.handle_custom_ui_sso_sign_in(
request=request,
)
)
# Import here to avoid circular imports
from litellm.proxy.management_endpoints.ui_sso import SSOAuthenticationHandler
return await SSOAuthenticationHandler.get_redirect_response_from_openid(
result=openid_response,
request=request,
received_response=None,
generic_client_id=None,
ui_access_mode=None,
)
)

318
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -15,6 +15,11 @@ from litellm.caching.caching import DualCache
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
from litellm.llms.base_llm.files.transformation import BaseFileEndpoints
from litellm.llms.base_llm.managed_resources.isolation import (
build_list_page,
build_owner_filter,
can_access_resource,
)
from litellm.proxy._types import (
CallTypes,
LiteLLM_ManagedFileTable,
@ -99,6 +104,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
model_mappings=model_mappings,
flat_model_file_ids=list(model_mappings.values()),
created_by=user_api_key_dict.user_id,
team_id=user_api_key_dict.team_id,
updated_by=user_api_key_dict.user_id,
)
await self.internal_usage_cache.async_set_cache(
@ -114,6 +120,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
"model_mappings": json.dumps(model_mappings),
"flat_model_file_ids": list(model_mappings.values()),
"created_by": user_api_key_dict.user_id,
"team_id": user_api_key_dict.team_id,
"updated_by": user_api_key_dict.user_id,
}
@ -125,7 +132,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
db_data["storage_backend"] = hidden_params["storage_backend"]
if "storage_url" in hidden_params:
db_data["storage_url"] = hidden_params["storage_url"]
verbose_logger.debug(
f"Storage metadata: storage_backend={db_data.get('storage_backend')}, "
f"storage_url={db_data.get('storage_url')}"
@ -171,6 +178,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
"model_object_id": model_object_id,
"file_purpose": file_purpose,
"created_by": user_api_key_dict.user_id,
"team_id": user_api_key_dict.team_id,
"updated_by": user_api_key_dict.user_id,
"status": file_object.status,
},
@ -229,15 +237,16 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
async def can_user_call_unified_file_id(
self, unified_file_id: str, user_api_key_dict: UserAPIKeyAuth
) -> bool:
## check if the user has access to the unified file id
user_id = user_api_key_dict.user_id
managed_file = await self.prisma_client.db.litellm_managedfiletable.find_first(
where={"unified_file_id": unified_file_id}
)
if managed_file:
return managed_file.created_by == user_id
return can_access_resource(
user_api_key_dict=user_api_key_dict,
created_by=managed_file.created_by,
resource_team_id=managed_file.team_id,
)
raise HTTPException(
status_code=404,
detail=f"File not found: {unified_file_id}",
@ -246,8 +255,6 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
async def can_user_call_unified_object_id(
self, unified_object_id: str, user_api_key_dict: UserAPIKeyAuth
) -> bool:
## check if the user has access to the unified object id
user_id = user_api_key_dict.user_id
managed_object = (
await self.prisma_client.db.litellm_managedobjecttable.find_first(
where={"unified_object_id": unified_object_id}
@ -255,7 +262,11 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
)
if managed_object:
return managed_object.created_by == user_id
return can_access_resource(
user_api_key_dict=user_api_key_dict,
created_by=managed_object.created_by,
resource_team_id=managed_object.team_id,
)
raise HTTPException(
status_code=404,
detail=f"Object not found: {unified_object_id}",
@ -285,28 +296,27 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
raise Exception(
"Filtering by 'target_model_names' is not supported when using managed batches."
)
where_clause: Dict[str, Any] = {"file_purpose": "batch"}
# Filter by user who created the batch
if user_api_key_dict.user_id:
where_clause["created_by"] = user_api_key_dict.user_id
owner_filter = build_owner_filter(user_api_key_dict)
if owner_filter is None:
return build_list_page([])
where_clause: Dict[str, Any] = {"file_purpose": "batch", **owner_filter}
if after:
where_clause["id"] = {"gt": after}
# Fetch more than needed to allow for post-fetch filtering
fetch_limit = limit or 20
if target_model_names:
# Fetch extra to account for filtering
# Oversample so post-fetch model-name filtering still has enough rows.
fetch_limit = max(fetch_limit * 3, 100)
batches = await self.prisma_client.db.litellm_managedobjecttable.find_many(
where=where_clause,
take=fetch_limit,
order={"created_at": "desc"},
)
batch_objects: List[LiteLLMBatch] = []
for batch in batches:
try:
@ -314,7 +324,11 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
if len(batch_objects) >= (limit or 20):
break
batch_data = json.loads(batch.file_object) if isinstance(batch.file_object, str) else batch.file_object
batch_data = (
json.loads(batch.file_object)
if isinstance(batch.file_object, str)
else batch.file_object
)
batch_obj = LiteLLMBatch(**batch_data)
batch_obj.id = batch.unified_object_id
batch_objects.append(batch_obj)
@ -324,27 +338,29 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
f"Failed to parse batch object {batch.unified_object_id}: {e}"
)
continue
return {
"object": "list",
"data": batch_objects,
"first_id": batch_objects[0].id if batch_objects else None,
"last_id": batch_objects[-1].id if batch_objects else None,
"has_more": len(batch_objects) == (limit or 20),
}
return build_list_page(
batch_objects, has_more=len(batch_objects) == (limit or 20)
)
async def get_user_created_file_ids(
self, user_api_key_dict: UserAPIKeyAuth, model_object_ids: List[str]
) -> List[OpenAIFileObject]:
"""
Get all file ids created by the user for a list of model object ids
Get all file ids the caller is allowed to see for a list of model
object ids. Service-account keys (no user_id) are scoped to their
team via ``team_id``; admins see all matches.
Returns:
- List of OpenAIFileObject's
"""
owner_filter = build_owner_filter(user_api_key_dict)
if owner_filter is None:
return []
file_ids = await self.prisma_client.db.litellm_managedfiletable.find_many(
where={
"created_by": user_api_key_dict.user_id,
**owner_filter,
"flat_model_file_ids": {"hasSome": model_object_ids},
}
)
@ -377,11 +393,11 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
"""
Check if the user has access to a list of file IDs.
Only checks managed (unified) file IDs.
Args:
file_ids: List of file IDs to check access for
user_api_key_dict: User API key authentication details
Raises:
HTTPException: If user doesn't have access to any of the files
"""
@ -419,10 +435,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
### HANDLE TRANSFORMATIONS ###
# Check both completion and acompletion call types
is_completion_call = (
call_type == CallTypes.completion.value
call_type == CallTypes.completion.value
or call_type == CallTypes.acompletion.value
)
if is_completion_call:
messages = data.get("messages")
model = data.get("model", "")
@ -431,22 +447,27 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
if file_ids:
# Check user has access to all managed files
await self.check_file_ids_access(file_ids, user_api_key_dict)
# Check if any files are stored in storage backends and need base64 conversion
# This is needed for Vertex AI/Gemini which requires base64 content
is_vertex_ai = model and ("vertex_ai" in model or "gemini" in model.lower())
is_vertex_ai = model and (
"vertex_ai" in model or "gemini" in model.lower()
)
if is_vertex_ai:
await self._convert_storage_files_to_base64(
messages=messages,
file_ids=file_ids,
litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
)
model_file_id_mapping = await self.get_model_file_id_mapping(
file_ids, user_api_key_dict.parent_otel_span
)
data["model_file_id_mapping"] = model_file_id_mapping
elif call_type == CallTypes.aresponses.value or call_type == CallTypes.responses.value:
elif (
call_type == CallTypes.aresponses.value
or call_type == CallTypes.responses.value
):
# Handle managed files in responses API input and tools
file_ids = []
@ -611,7 +632,9 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
if model_id is None:
model_id = cast(
Optional[str],
kwargs.get("litellm_metadata", {}).get("model_info", {}).get("id", None),
kwargs.get("litellm_metadata", {})
.get("model_info", {})
.get("id", None),
)
mapped_file_id: Optional[str] = None
if input_file_id and model_file_id_mapping and model_id:
@ -648,7 +671,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
) -> List[str]:
"""
Gets file ids from responses API input.
The input can be:
- A string (no files)
- A list of input items, where each item can have:
@ -656,32 +679,35 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
- content: a list that can contain items with type: "input_file" and file_id
"""
file_ids: List[str] = []
if isinstance(input, str):
return file_ids
if not isinstance(input, list):
return file_ids
for item in input:
if not isinstance(item, dict):
continue
# Check for direct input_file type
if item.get("type") == "input_file":
file_id = item.get("file_id")
if file_id:
file_ids.append(file_id)
# Check for input_file in content array
content = item.get("content")
if isinstance(content, list):
for content_item in content:
if isinstance(content_item, dict) and content_item.get("type") == "input_file":
if (
isinstance(content_item, dict)
and content_item.get("type") == "input_file"
):
file_id = content_item.get("file_id")
if file_id:
file_ids.append(file_id)
return file_ids
def get_file_ids_from_responses_tools(
@ -689,7 +715,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
) -> List[str]:
"""
Gets file ids from responses API tools parameter.
The tools can contain code_interpreter with container.file_ids:
[
{
@ -699,14 +725,14 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
]
"""
file_ids: List[str] = []
if not isinstance(tools, list):
return file_ids
for tool in tools:
if not isinstance(tool, dict):
continue
# Check for code_interpreter with container file_ids
if tool.get("type") == "code_interpreter":
container = tool.get("container")
@ -716,7 +742,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
for file_id in container_file_ids:
if isinstance(file_id, str):
file_ids.append(file_id)
return file_ids
def get_vector_store_ids_from_file_search_tools(
@ -916,10 +942,17 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
# Emit Prometheus metrics for managed file creation
prom_logger = self._get_prometheus_logger()
if prom_logger:
first_model = target_model_names_list[0] if target_model_names_list else None
first_model = (
target_model_names_list[0] if target_model_names_list else None
)
first_provider = ""
if responses:
first_provider = getattr(responses[0], "_hidden_params", {}).get("custom_llm_provider") or ""
first_provider = (
getattr(responses[0], "_hidden_params", {}).get(
"custom_llm_provider"
)
or ""
)
prom_logger.record_managed_file_created(
model=first_model or "",
api_provider=first_provider,
@ -1073,16 +1106,24 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
model_name=resolved_model_name,
)
setattr(response, file_attr, unified_file_id)
# Use llm_router credentials when available. Without credentials,
# Azure and other auth-required providers return 500/401.
file_object = None
try:
# Import module and use getattr for better testability with mocks
import litellm.proxy.proxy_server as proxy_server_module
_llm_router = getattr(proxy_server_module, 'llm_router', None)
_llm_router = getattr(
proxy_server_module, "llm_router", None
)
if _llm_router is not None and model_id:
_creds = _llm_router.get_deployment_credentials_with_provider(model_id) or {}
_creds = (
_llm_router.get_deployment_credentials_with_provider(
model_id
)
or {}
)
file_object = await litellm.afile_retrieve(
file_id=original_file_id,
**_creds,
@ -1099,7 +1140,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
verbose_logger.warning(
f"Failed to retrieve file object for {file_attr}={original_file_id}: {str(e)}. Storing with None and will fetch on-demand."
)
await self.store_unified_file_id(
file_id=unified_file_id,
file_object=file_object,
@ -1128,6 +1169,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
from litellm.litellm_core_utils.get_llm_provider_logic import (
get_llm_provider,
)
_, batch_provider, _, _ = get_llm_provider(model=model_name)
except Exception:
if "/" in model_name:
@ -1199,7 +1241,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
# Case 1 : This is not a managed file
if not stored_file_object:
raise Exception(f"LiteLLM Managed File object with id={file_id} not found")
# Case 2: Managed file and the file object exists in the database
# The stored file_object has the raw provider ID. Replace with the unified ID
# so callers see a consistent ID (matching Case 3 which does response.id = file_id).
@ -1217,13 +1259,21 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
)
try:
model_id, model_file_id = next(iter(stored_file_object.model_mappings.items()))
credentials = llm_router.get_deployment_credentials_with_provider(model_id) or {}
response = await litellm.afile_retrieve(file_id=model_file_id, **credentials)
model_id, model_file_id = next(
iter(stored_file_object.model_mappings.items())
)
credentials = (
llm_router.get_deployment_credentials_with_provider(model_id) or {}
)
response = await litellm.afile_retrieve(
file_id=model_file_id, **credentials
)
response.id = file_id # Replace with unified ID
return response
except Exception as e:
raise Exception(f"Failed to retrieve file {file_id} from provider: {str(e)}") from e
raise Exception(
f"Failed to retrieve file {file_id} from provider: {str(e)}"
) from e
async def afile_list(
self,
@ -1245,19 +1295,19 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
import litellm.proxy.proxy_server as proxy_server_module
# Check if the scheduler has the batch cost checking job registered
scheduler = getattr(proxy_server_module, 'scheduler', None)
scheduler = getattr(proxy_server_module, "scheduler", None)
if scheduler is None:
return False
# Check if the check_batch_cost_job exists in the scheduler
try:
job = scheduler.get_job('check_batch_cost_job')
job = scheduler.get_job("check_batch_cost_job")
if job is not None:
return True
except Exception:
# Job not found or scheduler doesn't support get_job
pass
return False
except Exception as e:
verbose_logger.warning(
@ -1265,28 +1315,26 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
)
return False
async def _get_batches_referencing_file(
self, file_id: str
) -> List[Dict[str, Any]]:
async def _get_batches_referencing_file(self, file_id: str) -> List[Dict[str, Any]]:
"""
Find batches that reference this file and still need cost tracking.
Find batches that are in non-terminal state and have not yet been processed by CheckBatchCost.
Args:
file_id: The unified file ID to check
Returns:
List of batch objects referencing this file in non-terminal state
(max 10 for error message display)
"""
# Prepare list of file IDs to check (both unified and provider IDs)
file_ids_to_check = [file_id]
# Get model-specific file IDs for this unified file ID if it's a managed file
try:
model_file_id_mapping = await self.get_model_file_id_mapping(
[file_id], litellm_parent_otel_span=None
)
if model_file_id_mapping and file_id in model_file_id_mapping:
# Add all provider file IDs for this unified file
provider_file_ids = list(model_file_id_mapping[file_id].values())
@ -1296,59 +1344,67 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
f"Could not get model file ID mapping for {file_id}: {e}. "
f"Will only check unified file ID."
)
MAX_MATCHES_TO_RETURN = 10
MAX_MATCHES_TO_RETURN = 10
batches = await self.prisma_client.db.litellm_managedobjecttable.find_many(
where={
"file_purpose": "batch",
"batch_processed": False,
"status": {"not_in": ["failed", "expired", "cancelled"]}
"status": {"not_in": ["failed", "expired", "cancelled"]},
},
take=MAX_MATCHES_TO_RETURN,
order={"created_at": "desc"},
)
referencing_batches = []
for batch in batches:
try:
# Parse the batch file_object to check for file references
batch_data = json.loads(batch.file_object) if isinstance(batch.file_object, str) else batch.file_object
batch_data = (
json.loads(batch.file_object)
if isinstance(batch.file_object, str)
else batch.file_object
)
# Extract file IDs from batch
# Batches typically reference the unified file ID in input_file_id
# Output and error files are generated by the provider
input_file_id = batch_data.get("input_file_id")
output_file_id = batch_data.get("output_file_id")
error_file_id = batch_data.get("error_file_id")
referenced_file_ids = [fid for fid in [input_file_id, output_file_id, error_file_id] if fid]
referenced_file_ids = [
fid for fid in [input_file_id, output_file_id, error_file_id] if fid
]
# Check if any referenced file ID matches the file we're trying to delete
if any(ref_id in file_ids_to_check for ref_id in referenced_file_ids):
referencing_batches.append({
"batch_id": batch.unified_object_id,
"status": batch.status,
"created_at": batch.created_at,
})
referencing_batches.append(
{
"batch_id": batch.unified_object_id,
"status": batch.status,
"created_at": batch.created_at,
}
)
except Exception as e:
verbose_logger.warning(
f"Error parsing batch object {batch.unified_object_id}: {e}"
)
continue
return referencing_batches
async def _check_file_deletion_allowed(self, file_id: str) -> None:
"""
Check if file deletion should be blocked due to batch references.
Blocks deletion if:
1. File is referenced by any batch in non-terminal state, AND
2. Batch polling is configured (user wants cost tracking)
Args:
file_id: The unified file ID to check
Raises:
HTTPException: If file deletion should be blocked
"""
@ -1356,39 +1412,45 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
if not self._is_batch_polling_enabled():
# Batch polling not configured, allow deletion
return
# Check if file is referenced by any non-terminal batches
referencing_batches = await self._get_batches_referencing_file(file_id)
if referencing_batches:
# File is referenced by non-terminal batches and polling is enabled
MAX_BATCHES_IN_ERROR = 5 # Limit batches shown in error message for readability
MAX_BATCHES_IN_ERROR = (
5 # Limit batches shown in error message for readability
)
# Show up to MAX_BATCHES_IN_ERROR in the error message
batches_to_show = referencing_batches[:MAX_BATCHES_IN_ERROR]
batch_statuses = [f"{b['batch_id']}: {b['status']}" for b in batches_to_show]
batch_statuses = [
f"{b['batch_id']}: {b['status']}" for b in batches_to_show
]
# Determine the count message
count_message = f"{len(referencing_batches)}"
if len(referencing_batches) >= 10: # MAX_MATCHES_TO_RETURN from _get_batches_referencing_file
if (
len(referencing_batches) >= 10
): # MAX_MATCHES_TO_RETURN from _get_batches_referencing_file
count_message = "10+"
error_message = (
f"Cannot delete file {file_id}. "
f"The file is referenced by {count_message} batch(es) in non-terminal state"
)
# Add specific batch details if not too many
if len(referencing_batches) <= MAX_BATCHES_IN_ERROR:
error_message += f": {', '.join(batch_statuses)}. "
else:
error_message += f" (showing {MAX_BATCHES_IN_ERROR} most recent): {', '.join(batch_statuses)}. "
error_message += (
f"To delete this file before complete cost tracking, please delete or cancel the referencing batch(es) first. "
f"Alternatively, wait for all batches to complete and for cost to be computed (batch_processed=true)."
)
# Record blocked deletion metric
prom_logger = self._get_prometheus_logger()
if prom_logger:
@ -1419,7 +1481,9 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
specific_model_file_id_mapping = model_file_id_mapping.get(file_id)
if specific_model_file_id_mapping:
# Remove conflicting keys from data to avoid duplicate keyword arguments
filtered_data = {k: v for k, v in data.items() if k not in ("model", "file_id")}
filtered_data = {
k: v for k, v in data.items() if k not in ("model", "file_id")
}
for model_id, model_file_id in specific_model_file_id_mapping.items():
delete_response = await llm_router.afile_delete(model=model_id, file_id=model_file_id, **filtered_data) # type: ignore
@ -1480,7 +1544,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
) -> None:
"""
Convert files stored in storage backends to base64 format for Vertex AI/Gemini.
This method checks if any managed files are stored in storage backends,
downloads them, and converts them to base64 format in the messages.
"""
@ -1488,29 +1552,29 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
for file_id in file_ids:
# Check if this is a base64 encoded unified file ID
decoded_unified_file_id = _is_base64_encoded_unified_file_id(file_id)
if not decoded_unified_file_id:
continue
# Check database for storage backend info
# IMPORTANT: The database stores the base64 encoded unified_file_id (not the decoded version)
# So we query with the original file_id (which is base64 encoded)
db_file = await self.prisma_client.db.litellm_managedfiletable.find_first(
where={"unified_file_id": file_id}
)
if not db_file or not db_file.storage_backend or not db_file.storage_url:
continue
# File is stored in a storage backend, download and convert to base64
try:
from litellm.llms.base_llm.files.storage_backend_factory import (
get_storage_backend,
)
storage_backend_name = db_file.storage_backend
storage_url = db_file.storage_url
# Get storage backend (uses same env vars as callback)
try:
storage_backend = get_storage_backend(storage_backend_name)
@ -1519,18 +1583,22 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
f"Storage backend '{storage_backend_name}' error for file {file_id}: {str(e)}"
)
continue
file_content = await storage_backend.download_file(storage_url)
# Determine content type from file object
content_type = self._get_content_type_from_file_object(db_file.file_object)
content_type = self._get_content_type_from_file_object(
db_file.file_object
)
# Convert to base64
base64_data = base64.b64encode(file_content).decode("utf-8")
base64_data_uri = f"data:{content_type};base64,{base64_data}"
# Update messages to use base64 instead of file_id
self._update_messages_with_base64_data(messages, file_id, base64_data_uri, content_type)
self._update_messages_with_base64_data(
messages, file_id, base64_data_uri, content_type
)
except Exception as e:
verbose_logger.exception(
f"Error converting file {file_id} from storage backend to base64: {str(e)}"
@ -1541,21 +1609,21 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
def _get_content_type_from_file_object(self, file_object: Optional[Any]) -> str:
"""
Determine content type from file object.
Uses the MIME type utility for consistent detection and normalization.
Args:
file_object: The file object from the database (can be dict, JSON string, or None)
Returns:
str: MIME type (defaults to "application/octet-stream" if cannot be determined)
"""
# Use utility function for detection
content_type = get_content_type_from_file_object(file_object)
# Normalize for Gemini/Vertex AI (requires image/jpeg, not image/jpg)
content_type = normalize_mime_type_for_provider(content_type, provider="gemini")
return content_type
def _update_messages_with_base64_data(
@ -1567,7 +1635,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
) -> None:
"""
Update messages to replace file_id with base64 data URI.
Args:
messages: List of messages to update
file_id: The file ID to replace
@ -1582,7 +1650,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
if element.get("type") == "file":
file_element = cast(ChatCompletionFileObject, element)
file_element_file = file_element.get("file", {})
if file_element_file.get("file_id") == file_id:
# Replace file_id with base64 data
file_element_file["file_data"] = base64_data_uri
@ -1590,7 +1658,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
file_element_file["format"] = content_type
# Remove file_id to ensure only file_data is used
file_element_file.pop("file_id", None)
verbose_logger.debug(
f"Converted file {file_id} from storage backend to base64 with format {content_type}"
)

83
enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py
View file

@ -588,24 +588,21 @@ async def update_project( # noqa: PLR0915
param="project_id",
)
# Validate team exists and get team object for limit + permission checks
team_id_to_check = data.team_id or existing_project.team_id
team_obj_for_checks = None
if team_id_to_check is not None:
team_obj_for_checks = await _validate_team_exists(
team_id=team_id_to_check, prisma_client=prisma_client
# Permission to *edit* the project must be evaluated against the
# project's CURRENT team. Sourcing the team from `data.team_id`
# would let an admin of any team pass the check by supplying their
# own team_id, hijacking the project (VERIA-55).
target_team_id = data.team_id or existing_project.team_id
target_team_obj = None
if target_team_id is not None:
target_team_obj = await _validate_team_exists(
team_id=target_team_id, prisma_client=prisma_client
)
# Check if user has permission to update this project
has_permission = await _check_user_permission_for_project(
user_api_key_dict=user_api_key_dict,
team_id=existing_project.team_id,
prisma_client=prisma_client,
team_object=(
LiteLLM_TeamTable(**team_obj_for_checks.model_dump())
if team_obj_for_checks
else None
),
)
if not has_permission:
@ -614,10 +611,32 @@ async def update_project( # noqa: PLR0915
detail={"error": "Only admins or team admins can update projects"},
)
# Reassigning to a different team also requires admin rights on the
# destination team — otherwise a team admin could shed projects into
# an unsuspecting team's namespace.
if data.team_id is not None and data.team_id != existing_project.team_id:
can_assign_to_target = await _check_user_permission_for_project(
user_api_key_dict=user_api_key_dict,
team_id=data.team_id,
prisma_client=prisma_client,
team_object=(
LiteLLM_TeamTable(**target_team_obj.model_dump())
if target_team_obj
else None
),
)
if not can_assign_to_target:
raise HTTPException(
status_code=403,
detail={
"error": "Cannot reassign project to a team you are not an admin of"
},
)
# Validate project limits against team limits
if team_obj_for_checks is not None:
if target_team_obj is not None:
_check_team_project_limits(
team_object=LiteLLM_TeamTable(**team_obj_for_checks.model_dump()),
team_object=LiteLLM_TeamTable(**target_team_obj.model_dump()),
data=data,
)
@ -857,10 +876,16 @@ async def project_info(
where={"team_id": project.team_id}
)
if team:
is_team_member = (
user_api_key_dict.user_id in team.admins
or user_api_key_dict.user_id in team.members
)
caller_user_id = user_api_key_dict.user_id
for m in team.members_with_roles or []:
m_user_id = (
m.get("user_id")
if isinstance(m, dict)
else getattr(m, "user_id", None)
)
if m_user_id == caller_user_id:
is_team_member = True
break
if not (is_admin or is_team_member):
raise HTTPException(
@ -911,20 +936,20 @@ async def list_projects(
include={"litellm_budget_table": True, "object_permission": True}
)
else:
# Get projects for teams the user belongs to
user_teams = await prisma_client.db.litellm_teamtable.find_many(
where={
"OR": [
{"members": {"has": user_api_key_dict.user_id}},
{"admins": {"has": user_api_key_dict.user_id}},
]
}
# Look up the user's team memberships via the reverse-index on
# LiteLLM_UserTable.teams (maintained by team_member_add alongside
# members_with_roles). This avoids a full scan of all team rows.
user_record = await prisma_client.db.litellm_usertable.find_unique(
where={"user_id": user_api_key_dict.user_id},
)
user_team_ids = (
user_record.teams
if user_record is not None and user_record.teams
else []
)
team_ids = [team.team_id for team in user_teams]
projects = await prisma_client.db.litellm_projecttable.find_many(
where={"team_id": {"in": team_ids}},
where={"team_id": {"in": user_team_ids}},
include={"litellm_budget_table": True, "object_permission": True},
)

2
enterprise/pyproject.toml
View file

@ -16,7 +16,7 @@ Repository = "https://github.com/BerriAI/litellm"
Documentation = "https://docs.litellm.ai"
[build-system]
requires = ["uv_build==0.10.7"]
requires = ["uv_build==0.11.8"]
build-backend = "uv_build"
[tool.uv]

2
litellm-js/proxy/.npmrc
View file

@ -2,4 +2,4 @@
# Packages needing lifecycle scripts: npm rebuild <pkg>
ignore-scripts=true
# Protects local npm install only — npm ci (used in CI) ignores this
min-release-age=3d
min-release-age=3

2054
litellm-js/proxy/package-lock.json generated Normal file
View file

File diff suppressed because it is too large Load diff

6
litellm-js/proxy/package.json
View file

@ -4,11 +4,11 @@
"deploy": "wrangler deploy --minify src/index.ts"
},
"dependencies": {
"hono": "4.12.12",
"hono": "4.12.16",
"openai": "4.29.2"
},
"devDependencies": {
"@cloudflare/workers-types": "4.20240208.0",
"wrangler": "3.32.0"
"@cloudflare/workers-types": "4.20260501.1",
"wrangler": "4.87.0"
}
}

2
litellm-js/spend-logs/.npmrc
View file

@ -2,4 +2,4 @@
# Packages needing lifecycle scripts: npm rebuild <pkg>
ignore-scripts=true
# Protects local npm install only — npm ci (used in CI) ignores this
min-release-age=3d
min-release-age=3

8
litellm-js/spend-logs/package-lock.json generated
View file

@ -6,7 +6,7 @@
"": {
"dependencies": {
"@hono/node-server": "1.19.13",
"hono": "4.12.12"
"hono": "4.12.16"
},
"devDependencies": {
"@types/node": "20.19.25",
@ -548,9 +548,9 @@
}
},
"node_modules/hono": {
"version": "4.12.12",
"resolved": "https://registry.npmjs.org/hono/-/hono-4.12.12.tgz",
"integrity": "sha512-p1JfQMKaceuCbpJKAPKVqyqviZdS0eUxH9v82oWo1kb9xjQ5wA6iP3FNVAPDFlz5/p7d45lO+BpSk1tuSZMF4Q==",
"version": "4.12.16",
"resolved": "https://registry.npmjs.org/hono/-/hono-4.12.16.tgz",
"integrity": "sha512-jN0ZewiNAWSe5khM3EyCmBb250+b40wWbwNILNfEvq84VREWwOIkuUsFONk/3i3nqkz7Oe1PcpM2mwQEK2L9Kg==",
"license": "MIT",
"engines": {
"node": ">=16.9.0"

2
litellm-js/spend-logs/package.json
View file

@ -4,7 +4,7 @@
},
"dependencies": {
"@hono/node-server": "1.19.13",
"hono": "4.12.12"
"hono": "4.12.16"
},
"devDependencies": {
"@types/node": "20.19.25",

2
litellm-proxy-extras/litellm_proxy_extras/migrations/20260429120000_search_tools_on_object_permission/migration.sql Normal file
View file

@ -0,0 +1,2 @@
-- Search tool allowlists live on LiteLLM_ObjectPermissionTable (with agents, MCP, vector stores).
ALTER TABLE "LiteLLM_ObjectPermissionTable" ADD COLUMN IF NOT EXISTS "search_tools" TEXT[] DEFAULT ARRAY[]::TEXT[];

75
litellm-proxy-extras/litellm_proxy_extras/migrations/20260429161855_workflow_runs_tables/migration.sql Normal file
View file

@ -0,0 +1,75 @@
-- CreateTable
CREATE TABLE "LiteLLM_WorkflowRun" (
"run_id" TEXT NOT NULL,
"session_id" TEXT NOT NULL,
"workflow_type" TEXT NOT NULL,
"status" TEXT NOT NULL DEFAULT 'pending',
"created_by" TEXT,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updated_at" TIMESTAMP(3) NOT NULL,
"input" JSONB,
"output" JSONB,
"metadata" JSONB,
CONSTRAINT "LiteLLM_WorkflowRun_pkey" PRIMARY KEY ("run_id")
);
-- CreateTable
CREATE TABLE "LiteLLM_WorkflowEvent" (
"event_id" TEXT NOT NULL,
"run_id" TEXT NOT NULL,
"event_type" TEXT NOT NULL,
"step_name" TEXT NOT NULL,
"sequence_number" INTEGER NOT NULL,
"data" JSONB,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "LiteLLM_WorkflowEvent_pkey" PRIMARY KEY ("event_id")
);
-- CreateTable
CREATE TABLE "LiteLLM_WorkflowMessage" (
"message_id" TEXT NOT NULL,
"run_id" TEXT NOT NULL,
"role" TEXT NOT NULL,
"content" TEXT NOT NULL,
"sequence_number" INTEGER NOT NULL,
"session_id" TEXT,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "LiteLLM_WorkflowMessage_pkey" PRIMARY KEY ("message_id")
);
-- CreateIndex
CREATE UNIQUE INDEX "LiteLLM_WorkflowRun_session_id_key" ON "LiteLLM_WorkflowRun"("session_id");
-- CreateIndex
CREATE INDEX "LiteLLM_WorkflowRun_workflow_type_status_idx" ON "LiteLLM_WorkflowRun"("workflow_type", "status");
-- CreateIndex
CREATE INDEX "LiteLLM_WorkflowRun_session_id_idx" ON "LiteLLM_WorkflowRun"("session_id");
-- CreateIndex
CREATE INDEX "LiteLLM_WorkflowRun_created_at_idx" ON "LiteLLM_WorkflowRun"("created_at");
-- CreateIndex
CREATE INDEX "LiteLLM_WorkflowRun_created_by_idx" ON "LiteLLM_WorkflowRun"("created_by");
-- CreateIndex
CREATE INDEX "LiteLLM_WorkflowEvent_run_id_idx" ON "LiteLLM_WorkflowEvent"("run_id");
-- CreateIndex
CREATE UNIQUE INDEX "LiteLLM_WorkflowEvent_run_id_sequence_number_key" ON "LiteLLM_WorkflowEvent"("run_id", "sequence_number");
-- CreateIndex
CREATE INDEX "LiteLLM_WorkflowMessage_run_id_idx" ON "LiteLLM_WorkflowMessage"("run_id");
-- CreateIndex
CREATE UNIQUE INDEX "LiteLLM_WorkflowMessage_run_id_sequence_number_key" ON "LiteLLM_WorkflowMessage"("run_id", "sequence_number");
-- AddForeignKey
ALTER TABLE "LiteLLM_WorkflowEvent" ADD CONSTRAINT "LiteLLM_WorkflowEvent_run_id_fkey" FOREIGN KEY ("run_id") REFERENCES "LiteLLM_WorkflowRun"("run_id") ON DELETE RESTRICT ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "LiteLLM_WorkflowMessage" ADD CONSTRAINT "LiteLLM_WorkflowMessage_run_id_fkey" FOREIGN KEY ("run_id") REFERENCES "LiteLLM_WorkflowRun"("run_id") ON DELETE RESTRICT ON UPDATE CASCADE;

20
litellm-proxy-extras/litellm_proxy_extras/migrations/20260501195714_managed_resource_team_owner/migration.sql Normal file
View file

@ -0,0 +1,20 @@
-- Adds `team_id` to managed-resource tables so service-account API
-- keys (no `user_id`) can be scoped by team instead of bypassing the
-- `created_by` filter entirely. Existing rows keep `team_id = NULL`
-- and become invisible to team-only callers — that is the intended isolation
-- outcome; backfill manually if legacy rows must remain visible.
--
-- The composite indexes match the listing query: filter by team owner, sort by
-- created_at DESC. Tables are typically small (resources per tenant, not per
-- request); a future operator with a large table can switch to
-- CREATE INDEX CONCURRENTLY in a follow-up migration.
ALTER TABLE "LiteLLM_ManagedFileTable" ADD COLUMN IF NOT EXISTS "team_id" TEXT;
ALTER TABLE "LiteLLM_ManagedObjectTable" ADD COLUMN IF NOT EXISTS "team_id" TEXT;
ALTER TABLE "LiteLLM_ManagedVectorStoreTable" ADD COLUMN IF NOT EXISTS "team_id" TEXT;
-- Index names follow Prisma's auto-generated convention so `prisma migrate diff`
-- against the schema is clean.
CREATE INDEX IF NOT EXISTS "LiteLLM_ManagedFileTable_team_id_created_at_idx" ON "LiteLLM_ManagedFileTable" ("team_id", "created_at" DESC);
CREATE INDEX IF NOT EXISTS "LiteLLM_ManagedObjectTable_team_id_created_at_idx" ON "LiteLLM_ManagedObjectTable" ("team_id", "created_at" DESC);
CREATE INDEX IF NOT EXISTS "LiteLLM_ManagedVectorStoreTable_team_id_created_at_idx" ON "LiteLLM_ManagedVectorStoreTable" ("team_id", "created_at" DESC);

94
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -277,6 +277,7 @@ model LiteLLM_ObjectPermissionTable {
models String[] @default([])
blocked_tools String[] @default([]) // Tool names blocked for any key/team/user with this permission
mcp_toolsets String[] @default([]) // Toolset IDs granted to this key/team/user
search_tools String[] @default([]) // search_tool_name values this key/team/user may call
teams LiteLLM_TeamTable[]
projects LiteLLM_ProjectTable[]
verification_tokens LiteLLM_VerificationToken[]
@ -883,28 +884,32 @@ model LiteLLM_ManagedFileTable {
storage_backend String? // Storage backend name (e.g., "azure_storage", "gcs", "default")
storage_url String? // The actual storage URL where the file is stored
created_at DateTime @default(now())
created_by String?
created_by String?
team_id String? // Team that owns the resource; populated for service-account keys without a user_id so listings can isolate by team.
updated_at DateTime @updatedAt
updated_by String?
@@index([unified_file_id])
@@index([team_id, created_at(sort: Desc)])
}
model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use the
model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use the
id String @id @default(uuid())
unified_object_id String @unique // The base64 encoded unified file ID
model_object_id String @unique // the id returned by the backend API provider
model_object_id String @unique // the id returned by the backend API provider
file_object Json // Stores the OpenAIFileObject
file_purpose String // either 'batch' or 'fine-tune'
status String? // check if batch cost has been tracked
status String? // check if batch cost has been tracked
batch_processed Boolean @default(false) // set to true by CheckBatchCost after cost is computed
created_at DateTime @default(now())
created_by String?
team_id String?
updated_at DateTime @updatedAt
updated_by String?
updated_by String?
@@index([unified_object_id])
@@index([model_object_id])
@@index([team_id, created_at(sort: Desc)])
}
model LiteLLM_ManagedVectorStoreTable {
@ -917,10 +922,12 @@ model LiteLLM_ManagedVectorStoreTable {
storage_url String? // Storage URL (if applicable)
created_at DateTime @default(now())
created_by String?
team_id String?
updated_at DateTime @updatedAt
updated_by String?
@@index([unified_resource_id])
@@index([team_id, created_at(sort: Desc)])
}
model LiteLLM_ManagedVectorStoresTable {
@ -1290,3 +1297,80 @@ model LiteLLM_AdaptiveRouterSession {
@@id([session_id, router_name, model_name])
@@index([last_activity_at], map: "idx_adaptive_router_session_activity")
}
// ---------------------------------------------------------------------------
// Workflow Run Tracking
//
// Generic durable state tracking for any agent or automated workflow.
// Design: three tables — run (header + materialized status), event (append-only
// source of truth for state transitions), message (conversation inbox/outbox).
//
// Usage:
// - Set `workflow_type` to identify the owning system (e.g. "shin-builder").
// - Store domain-specific fields in `metadata` (worktree_path, pr_url, etc.).
// - `session_id` on WorkflowRun matches `x-litellm-session-id` header sent to
// the proxy — all spend logs for this run are automatically tagged.
// ---------------------------------------------------------------------------
// One instance of work being done. `status` is a materialized cache of the
// latest event; the event log is the authoritative source of truth.
model LiteLLM_WorkflowRun {
run_id String @id @default(uuid())
session_id String @unique @default(uuid())
workflow_type String
status String @default("pending")
created_by String? // user_id of the key that created this run; null = created by master key
created_at DateTime @default(now())
updated_at DateTime @updatedAt
input Json?
output Json?
metadata Json?
events LiteLLM_WorkflowEvent[]
messages LiteLLM_WorkflowMessage[]
@@index([workflow_type, status])
@@index([session_id])
@@index([created_at])
@@index([created_by])
}
// Append-only log of state transitions. Never mutate rows here.
// `step_name` and `event_type` are caller-defined strings — no hardcoded enums.
// Status auto-update rules (applied by the append endpoint):
// step.started → run.status = running
// step.failed → run.status = failed
// hook.waiting → run.status = paused
// hook.received → run.status = running
model LiteLLM_WorkflowEvent {
event_id String @id @default(uuid())
run_id String
event_type String
step_name String
sequence_number Int
data Json?
created_at DateTime @default(now())
run LiteLLM_WorkflowRun @relation(fields: [run_id], references: [run_id])
@@unique([run_id, sequence_number])
@@index([run_id])
}
// Conversation inbox/outbox — full message content, separate from the durable
// event log. Spend logs truncate messages; this table stores them in full.
// `session_id` here is the Claude --resume session ID (or similar).
model LiteLLM_WorkflowMessage {
message_id String @id @default(uuid())
run_id String
role String
content String
sequence_number Int
session_id String?
created_at DateTime @default(now())
run LiteLLM_WorkflowRun @relation(fields: [run_id], references: [run_id])
@@unique([run_id, sequence_number])
@@index([run_id])
}

6
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
version = "0.4.69"
version = "0.4.70"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@ -16,7 +16,7 @@ Repository = "https://github.com/BerriAI/litellm"
Documentation = "https://docs.litellm.ai"
[build-system]
requires = ["uv_build==0.10.7"]
requires = ["uv_build==0.11.8"]
build-backend = "uv_build"
[tool.uv]
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.4.69"
version = "0.4.70"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-proxy-extras==",

7
litellm/__init__.py
View file

@ -166,7 +166,7 @@ langfuse_default_tags: Optional[List[str]] = None
langsmith_batch_size: Optional[int] = None
prometheus_initialize_budget_metrics: Optional[bool] = False
prometheus_latency_buckets: Optional[List[float]] = None
require_auth_for_metrics_endpoint: Optional[bool] = False
require_auth_for_metrics_endpoint: Optional[bool] = True
argilla_batch_size: Optional[int] = None
datadog_use_v1: Optional[bool] = False # if you want to use v1 datadog logged payload.
gcs_pub_sub_use_v1: Optional[bool] = (
@ -280,6 +280,7 @@ ssl_security_level: Optional[str] = None
ssl_certificate: Optional[str] = None
user_url_validation: bool = True
user_url_allowed_hosts: List[str] = []
provider_url_destination_allowed_hosts: List[str] = []
ssl_ecdh_curve: Optional[str] = (
None # Set to 'X25519' to disable PQC and improve performance
)
@ -288,6 +289,7 @@ disable_token_counter: bool = False
disable_add_transform_inline_image_block: bool = False
disable_add_user_agent_to_request_tags: bool = False
disable_anthropic_gemini_context_caching_transform: bool = False
disable_vertex_batch_output_transformation: bool = False
extra_spend_tag_headers: Optional[List[str]] = None
in_memory_llm_clients_cache: "LLMClientCache"
safe_memory_mode: bool = False
@ -330,6 +332,9 @@ enable_model_config_credential_overrides: bool = False
enable_key_alias_format_validation: bool = (
False # opt-in validation of key_alias format on /key/generate and /key/update
)
enable_gemini_default_thinking_level_low: bool = (
False # opt-in: force thinkingLevel low/minimal for Gemini 3 thinking param mapping
)
####################
logging: bool = True
enable_loadbalancing_on_batch_endpoints: Optional[bool] = None

69
litellm/_logging.py
View file

@ -1,12 +1,12 @@
import ast
import logging
import os
import re
import sys
from datetime import datetime
from logging import Formatter
from typing import Any, Dict, List, Optional
from typing import Any, Dict, Optional
from litellm.litellm_core_utils.secret_redaction import redact_string
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
@ -21,74 +21,11 @@ _ENABLE_SECRET_REDACTION = (
os.getenv("LITELLM_DISABLE_REDACT_SECRETS", "").lower() != "true"
)
_REDACTED = "REDACTED"
def _build_secret_patterns() -> re.Pattern:
patterns: List[str] = [
# ── PEM private key / certificate blocks ──
r"-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----",
# ── GCP OAuth2 access tokens (ya29.*) ──
r"\bya29\.[A-Za-z0-9_.~+/-]+",
# ── Credential %s formatting (space separator, no key= prefix) ──
r"(?:client_secret|azure_password|azure_username)\s+[^\s,'\"})\]{}>]+",
# AWS access key IDs
r"(?:AKIA|ASIA)[0-9A-Z]{16}",
# AWS secrets / session tokens / access key IDs (key=value)
r"(?:aws_secret_access_key|aws_session_token|aws_access_key_id)"
r"\s*[:=]\s*[A-Za-z0-9/+=]{20,}",
# Bearer tokens (OAuth, JWT, etc.)
r"Bearer\s+[A-Za-z0-9\-._~+/]{10,}=*",
# Basic auth headers
r"Basic\s+[A-Za-z0-9+/]{10,}={0,2}",
# OpenAI / Anthropic sk- prefixed keys
r"sk-[A-Za-z0-9\-_]{20,}",
# Generic api_key / api-key / apikey (handles 'key': 'value' dict repr)
r"(?:api[_-]?key)['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]{8,}",
# x-api-key / api-key header values (handles 'key': 'value' dict repr)
r"(?:x-api-key|api-key)['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+",
# Anthropic internal header keys
r"x-ak-[A-Za-z0-9\-_]{20,}",
# Google API keys
r"AIza[0-9A-Za-z\-_]{35}",
# Password / secret params (handles key=value and 'key': 'value')
# Word boundary prevents O(n^2) backtracking on long word-char runs.
r"(?:^|(?<=\W))\w*(?:password|passwd|client_secret|secret_key|_secret)"
r"['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+",
# Database connection string credentials (scheme://user:pass@host)
r"(?<=://)[^\s'\"]*:[^\s'\"@]+(?=@)",
# Databricks personal access tokens
r"dapi[0-9a-f]{32}",
# ── Key-name-based redaction ──
# Catches secrets inside dicts/config dumps by matching on the KEY name
# regardless of what the value looks like.
# e.g. 'master_key': 'any-value-here', "database_url": "postgres://..."
# private_key with PEM-aware value capture
r"""private_key['\"]?\s*[:=]\s*['\"]?(?:-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----|[^\s,'\"})\]{}>]+)""",
r"(?:master_key|database_url|db_url|connection_string|"
r"signing_key|encryption_key|"
r"auth_token|access_token|refresh_token|"
r"slack_webhook_url|webhook_url|"
r"database_connection_string|"
r"huggingface_token|jwt_secret)"
r"""['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+""",
# ── Raw JWTs (without Bearer prefix) ──
r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*",
# ── Azure SAS tokens in URLs ──
r"[?&]sig=[A-Za-z0-9%+/=]+",
# ── Full JSON service-account blobs (single-line and multi-line) ──
r'\{[^{}]*"type"\s*:\s*"service_account"[^{}]*(?:\{[^{}]*\}[^{}]*)*\}',
]
return re.compile("|".join(patterns), re.IGNORECASE)
_SECRET_RE = _build_secret_patterns()
def _redact_string(value: str) -> str:
if not _ENABLE_SECRET_REDACTION:
return value
return _SECRET_RE.sub(_REDACTED, value)
return redact_string(value)
def redact_secrets(value: str) -> str:

4
litellm/anthropic_beta_headers_config.json
View file

@ -72,7 +72,7 @@
"computer-use-2025-11-24": "computer-use-2025-11-24",
"context-1m-2025-08-07": "context-1m-2025-08-07",
"context-management-2025-06-27": null,
"effort-2025-11-24": null,
"effort-2025-11-24": "effort-2025-11-24",
"fast-mode-2026-02-01": null,
"files-api-2025-04-14": null,
"fine-grained-tool-streaming-2025-05-14": null,
@ -103,7 +103,7 @@
"computer-use-2025-11-24": "computer-use-2025-11-24",
"context-1m-2025-08-07": "context-1m-2025-08-07",
"context-management-2025-06-27": null,
"effort-2025-11-24": null,
"effort-2025-11-24": "effort-2025-11-24",
"fast-mode-2026-02-01": null,
"files-api-2025-04-14": null,
"fine-grained-tool-streaming-2025-05-14": null,

78
litellm/batches/batch_utils.py
View file

@ -387,6 +387,27 @@ def _get_batch_job_total_usage_from_file_content(
)
def _get_models_from_batch_input_file_content(
file_content_dictionary: List[dict],
) -> List[str]:
"""Extract the distinct ``body.model`` values from a batch *input* file.
Used by the proxy's batch pre-call hook to enforce that the caller is
authorized for every model named inside the JSONL — not just the one
on the outer request — so the proxy's per-key model allowlist isn't
bypassed by smuggling expensive models into the batch file.
"""
models: List[str] = []
seen: set = set()
for _item in file_content_dictionary:
body = _item.get("body") or {}
model = body.get("model")
if model and model not in seen:
seen.add(model)
models.append(model)
return models
def _get_batch_job_input_file_usage(
file_content_dictionary: List[dict],
custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
@ -403,11 +424,25 @@ def _get_batch_job_input_file_usage(
for _item in file_content_dictionary:
body = _item.get("body", {})
model = body.get("model", model_name or "")
messages = body.get("messages", [])
# Chat completion payloads.
messages = body.get("messages")
if messages:
item_tokens = token_counter(model=model, messages=messages)
prompt_tokens += item_tokens
prompt_tokens += token_counter(model=model, messages=messages)
continue
# Text completion payloads (`prompt`).
prompt = body.get("prompt")
if prompt:
prompt_tokens += _count_prompt_or_input_tokens(model=model, value=prompt)
continue
# Embedding payloads (`input`).
input_data = body.get("input")
if input_data:
prompt_tokens += _count_prompt_or_input_tokens(
model=model, value=input_data
)
return Usage(
total_tokens=prompt_tokens + completion_tokens,
@ -416,6 +451,43 @@ def _get_batch_job_input_file_usage(
)
def _count_prompt_or_input_tokens(model: str, value: Any) -> int:
"""Token-count a ``prompt`` / ``input`` field that the OpenAI batch
schema allows in four shapes:
- ``str``: a single text prompt.
- ``list[str]``: multiple text prompts.
- ``list[int]``: a pre-tokenized prompt (each int counts as 1 token).
- ``list[list[int]]``: multiple pre-tokenized prompts.
Pre-fix only the string shapes were counted, so a caller could send
a large ``list[list[int]]`` payload and slip past TPM rate limits
with a recorded cost of zero tokens.
"""
if isinstance(value, str):
return token_counter(model=model, text=value)
if isinstance(value, list):
total = 0
for chunk in value:
if isinstance(chunk, str):
total += token_counter(model=model, text=chunk)
elif isinstance(chunk, int):
# Single pre-tokenized prompt at the top level: each
# int counts as one token.
total += 1
elif isinstance(chunk, list):
# Nested pre-tokenized prompt: every int contributes a
# token. Mixed string/int items still count.
total += sum(1 if isinstance(t, int) else 0 for t in chunk)
total += sum(
token_counter(model=model, text=t)
for t in chunk
if isinstance(t, str)
)
return total
return 0
def _get_batch_job_usage_from_response_body(response_body: dict) -> Usage:
"""
Get the tokens of a batch job from the response body

10
litellm/batches/main.py
View file

@ -543,15 +543,17 @@ def _handle_retrieve_batch_providers_without_provider_config(
)
else:
raise litellm.exceptions.BadRequestError(
message="LiteLLM doesn't support {} for 'create_batch'. Only 'openai' is supported.".format(
custom_llm_provider
),
message=(
"LiteLLM doesn't support custom_llm_provider={} for 'retrieve_batch' without a `model` kwarg. "
"Supported via this path: 'openai', 'azure', 'vertex_ai', 'anthropic'. "
"'bedrock' is supported but requires `model` to be passed so the provider config can be loaded."
).format(custom_llm_provider),
model="n/a",
llm_provider=custom_llm_provider,
response=httpx.Response(
status_code=400,
content="Unsupported provider",
request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore
request=httpx.Request(method="retrieve_batch", url="https://github.com/BerriAI/litellm"), # type: ignore
),
)
return response

64
litellm/caching/caching.py
View file

@ -432,9 +432,10 @@ class Cache:
str: The final hashed cache key with the redis namespace.
"""
dynamic_cache_control: DynamicCacheControl = kwargs.get("cache", {})
metadata = kwargs.get("metadata") or {}
namespace = (
dynamic_cache_control.get("namespace")
or kwargs.get("metadata", {}).get("redis_namespace")
or metadata.get("redis_namespace")
or self.namespace
)
if namespace:
@ -650,7 +651,10 @@ class Cache:
verbose_logger.exception(f"LiteLLM Cache: Excepton add_cache: {str(e)}")
def _convert_to_cached_embedding(
self, embedding_response: Any, model: Optional[str]
self,
embedding_response: Any,
model: Optional[str],
prompt_tokens_details: Optional[dict] = None,
) -> CachedEmbedding:
"""
Convert any embedding response into the standardized CachedEmbedding TypedDict format.
@ -662,6 +666,7 @@ class Cache:
"index": embedding_response.get("index"),
"object": embedding_response.get("object"),
"model": model,
"prompt_tokens_details": prompt_tokens_details,
}
elif hasattr(embedding_response, "model_dump"):
data = embedding_response.model_dump()
@ -670,6 +675,7 @@ class Cache:
"index": data.get("index"),
"object": data.get("object"),
"model": model,
"prompt_tokens_details": prompt_tokens_details,
}
else:
data = vars(embedding_response)
@ -678,10 +684,54 @@ class Cache:
"index": data.get("index"),
"object": data.get("object"),
"model": model,
"prompt_tokens_details": prompt_tokens_details,
}
except KeyError as e:
raise ValueError(f"Missing expected key in embedding response: {e}")
def _get_per_item_prompt_tokens_details(
self,
result: EmbeddingResponse,
idx_in_result_data: int,
) -> Optional[dict]:
"""
Extract per-item prompt_tokens_details from a response for caching.
For single-item responses (common for multimodal providers like Bedrock Titan,
Nova, Vertex AI), returns the full prompt_tokens_details.
For multi-item responses, distributes integer fields evenly across items
so that summing all per-item details reconstructs the original totals.
"""
if result.usage is None or result.usage.prompt_tokens_details is None:
return None
details = result.usage.prompt_tokens_details
if hasattr(details, "model_dump"):
details_dict = details.model_dump(exclude_none=True)
elif isinstance(details, dict):
details_dict = {k: v for k, v in details.items() if v is not None}
else:
return None
if not details_dict:
return None
num_items = len(result.data)
if num_items <= 1:
return details_dict
# Distribute integer/float fields evenly across items
per_item: dict = {}
for key, value in details_dict.items():
if isinstance(value, int):
quotient, remainder = divmod(value, num_items)
per_item[key] = quotient + (1 if idx_in_result_data < remainder else 0)
elif isinstance(value, float):
per_item[key] = value / num_items
else:
per_item[key] = value
return per_item if per_item else None
def add_embedding_response_to_cache(
self,
result: EmbeddingResponse,
@ -693,10 +743,18 @@ class Cache:
kwargs["cache_key"] = preset_cache_key
embedding_response = result.data[idx_in_result_data]
# Extract per-item prompt_tokens_details from response usage
prompt_tokens_details = self._get_per_item_prompt_tokens_details(
result=result,
idx_in_result_data=idx_in_result_data,
)
# Always convert to properly typed CachedEmbedding
model_name = result.model
embedding_dict: CachedEmbedding = self._convert_to_cached_embedding(
embedding_response, model_name
embedding_response,
model_name,
prompt_tokens_details=prompt_tokens_details,
)
cache_key, cached_data, kwargs = self._add_cache_logic(

187
litellm/caching/caching_handler.py
View file

@ -59,6 +59,7 @@ from litellm.types.utils import (
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.types.utils import PromptTokensDetailsWrapper
else:
LiteLLMLoggingObj = Any
@ -86,6 +87,18 @@ class CachingHandlerResponse(BaseModel):
in_memory_cache_obj = InMemoryCache()
def _should_defer_streaming_cache_hit_callbacks(*, kwargs: Dict[str, Any]) -> bool:
"""
When stream=True, do not run success callbacks at cache-hit time.
Cached chat/text completion replay uses CustomStreamWrapper; cached Responses
replay uses CachedResponsesAPIStreamingIterator. Both invoke logging success
handlers when the stream finishes; firing them here too would double-count
spend and callback records.
"""
return kwargs.get("stream", False) is True
class LLMCachingHandler:
def __init__(
self,
@ -98,6 +111,7 @@ class LLMCachingHandler:
self.async_streaming_chunks: List[ModelResponse] = []
self.sync_streaming_chunks: List[ModelResponse] = []
self.request_kwargs = request_kwargs
self.preset_cache_key: Optional[str] = None
self.original_function = original_function
self.start_time = start_time
if litellm.cache is not None and isinstance(litellm.cache.cache, RedisCache):
@ -205,7 +219,7 @@ class LLMCachingHandler:
custom_llm_provider=kwargs.get("custom_llm_provider", None),
args=args,
)
if kwargs.get("stream", False) is False:
if not _should_defer_streaming_cache_hit_callbacks(kwargs=kwargs):
# LOG SUCCESS
self._async_log_cache_hit_on_callbacks(
logging_obj=logging_obj,
@ -214,11 +228,12 @@ class LLMCachingHandler:
end_time=end_time,
cache_hit=cache_hit,
)
cache_key = litellm.cache.get_cache_key(**kwargs)
if (
isinstance(cached_result, BaseModel)
or isinstance(cached_result, CustomStreamWrapper)
) and hasattr(cached_result, "_hidden_params"):
cache_key = (
self.preset_cache_key
or self.request_kwargs.get("cache_key")
or litellm.cache.get_cache_key(**self.request_kwargs)
)
if hasattr(cached_result, "_hidden_params"):
cached_result._hidden_params["cache_key"] = cache_key # type: ignore
return CachingHandlerResponse(cached_result=cached_result)
elif (
@ -264,8 +279,6 @@ class LLMCachingHandler:
kwargs: Dict[str, Any],
args: Optional[Tuple[Any, ...]] = None,
) -> CachingHandlerResponse:
from litellm.utils import CustomStreamWrapper
cached_result: Optional[Any] = None
# Check if caching should be performed BEFORE doing expensive kwargs copy
@ -281,6 +294,11 @@ class LLMCachingHandler:
args,
)
)
if new_kwargs.get("metadata") is None:
new_kwargs.pop("metadata", None)
if new_kwargs.get("stream") is True and "cache_key" not in new_kwargs:
new_kwargs["cache_key"] = litellm.cache.get_cache_key(**new_kwargs)
self.request_kwargs = new_kwargs
print_verbose("Checking Sync Cache")
cached_result = litellm.cache.get_cache(**new_kwargs)
if cached_result is not None:
@ -321,17 +339,19 @@ class LLMCachingHandler:
is_async=False,
)
logging_obj.handle_sync_success_callbacks_for_async_calls(
result=cached_result,
start_time=start_time,
end_time=end_time,
cache_hit=cache_hit,
if not _should_defer_streaming_cache_hit_callbacks(kwargs=kwargs):
logging_obj.handle_sync_success_callbacks_for_async_calls(
result=cached_result,
start_time=start_time,
end_time=end_time,
cache_hit=cache_hit,
)
cache_key = (
self.preset_cache_key
or self.request_kwargs.get("cache_key")
or litellm.cache.get_cache_key(**self.request_kwargs)
)
cache_key = litellm.cache.get_cache_key(**kwargs)
if (
isinstance(cached_result, BaseModel)
or isinstance(cached_result, CustomStreamWrapper)
) and hasattr(cached_result, "_hidden_params"):
if hasattr(cached_result, "_hidden_params"):
cached_result._hidden_params["cache_key"] = cache_key # type: ignore
return CachingHandlerResponse(cached_result=cached_result)
return CachingHandlerResponse(cached_result=cached_result)
@ -415,6 +435,7 @@ class LLMCachingHandler:
final_embedding_cached_response._hidden_params["cache_hit"] = True
prompt_tokens = 0
aggregated_details: Optional[dict] = None
for val in non_null_list:
idx, cr = val # (idx, cr) tuple
if cr is not None:
@ -431,11 +452,35 @@ class LLMCachingHandler:
prompt_tokens += token_counter(
text=kwargs_input_as_list[idx], count_response_tokens=True
)
# Aggregate prompt_tokens_details from cached items
item_details = cr.get("prompt_tokens_details")
if item_details:
if aggregated_details is None:
aggregated_details = {}
for key, value in item_details.items():
if isinstance(value, (int, float)):
aggregated_details[key] = (
aggregated_details.get(key, 0) + value
)
else:
aggregated_details[key] = value
## USAGE
prompt_tokens_details: Optional["PromptTokensDetailsWrapper"] = None
if aggregated_details:
from litellm.types.utils import PromptTokensDetailsWrapper
try:
prompt_tokens_details = PromptTokensDetailsWrapper(
**aggregated_details
)
except Exception:
prompt_tokens_details = None
usage = Usage(
prompt_tokens=prompt_tokens,
completion_tokens=0,
total_tokens=prompt_tokens,
prompt_tokens_details=prompt_tokens_details,
)
final_embedding_cached_response.usage = usage
if len(remaining_list) == 0:
@ -478,8 +523,70 @@ class LLMCachingHandler:
prompt_tokens=usage1.prompt_tokens + usage2.prompt_tokens,
completion_tokens=usage1.completion_tokens + usage2.completion_tokens,
total_tokens=usage1.total_tokens + usage2.total_tokens,
prompt_tokens_details=self._merge_prompt_tokens_details(
usage1.prompt_tokens_details,
usage2.prompt_tokens_details,
),
)
def _merge_prompt_tokens_details(
self,
details1: Optional["PromptTokensDetailsWrapper"],
details2: Optional["PromptTokensDetailsWrapper"],
) -> Optional["PromptTokensDetailsWrapper"]:
"""Merge two PromptTokensDetailsWrapper objects by summing numeric fields."""
if details1 is None and details2 is None:
return None
if details1 is None:
return details2
if details2 is None:
return details1
dict1 = (
details1.model_dump(exclude_none=True)
if hasattr(details1, "model_dump")
else {}
)
dict2 = (
details2.model_dump(exclude_none=True)
if hasattr(details2, "model_dump")
else {}
)
merged: dict = {}
for key in set(dict1.keys()) | set(dict2.keys()):
v1 = dict1.get(key, 0)
v2 = dict2.get(key, 0)
if isinstance(v1, (int, float)) and isinstance(v2, (int, float)):
merged[key] = v1 + v2
elif isinstance(v1, dict) and isinstance(v2, dict):
# Recursively merge nested dicts (e.g. cache_creation_token_details)
nested: dict = {}
for nk in set(v1.keys()) | set(v2.keys()):
nv1 = v1.get(nk, 0)
nv2 = v2.get(nk, 0)
if isinstance(nv1, (int, float)) and isinstance(nv2, (int, float)):
nested[nk] = nv1 + nv2
elif nv1:
nested[nk] = nv1
else:
nested[nk] = nv2
merged[key] = nested
elif v1:
merged[key] = v1
else:
merged[key] = v2
if not merged:
return None
from litellm.types.utils import PromptTokensDetailsWrapper
try:
return PromptTokensDetailsWrapper(**merged)
except Exception:
return None
def _combine_cached_embedding_response_with_api_result(
self,
_caching_handler_response: CachingHandlerResponse,
@ -598,6 +705,11 @@ class LLMCachingHandler:
args,
)
)
if new_kwargs.get("metadata") is None:
new_kwargs.pop("metadata", None)
if new_kwargs.get("stream") is True and "cache_key" not in new_kwargs:
new_kwargs["cache_key"] = litellm.cache.get_cache_key(**new_kwargs)
self.request_kwargs = new_kwargs
cached_result: Optional[Any] = None
if call_type == CallTypes.aembedding.value:
if isinstance(new_kwargs["input"], str):
@ -622,14 +734,26 @@ class LLMCachingHandler:
if all(result is None for result in cached_result):
cached_result = None
else:
request_kwargs = new_kwargs.copy()
request_cache_key = request_kwargs.pop("cache_key", None)
if litellm.cache._supports_async() is True:
## check if dual cache is supported ##
self.preset_cache_key = (
request_cache_key or litellm.cache.get_cache_key(**request_kwargs)
)
cached_result = await litellm.cache.async_get_cache(
dynamic_cache_object=self.dual_cache, **new_kwargs
dynamic_cache_object=self.dual_cache,
cache_key=self.preset_cache_key,
**request_kwargs,
)
else: # fallback for caches that don't support async
self.preset_cache_key = (
request_cache_key or litellm.cache.get_cache_key(**request_kwargs)
)
cached_result = litellm.cache.get_cache(
dynamic_cache_object=self.dual_cache, **new_kwargs
dynamic_cache_object=self.dual_cache,
cache_key=self.preset_cache_key,
**request_kwargs,
)
return cached_result
@ -737,8 +861,27 @@ class LLMCachingHandler:
elif (call_type == "aresponses" or call_type == "responses") and isinstance(
cached_result, dict
):
# Convert cached dict back to ResponsesAPIResponse object
cached_result = ResponsesAPIResponse(**cached_result)
from litellm.responses.streaming_iterator import (
CachedResponsesAPIStreamingIterator,
)
response_obj = ResponsesAPIResponse(**cached_result)
if (
hasattr(response_obj, "_hidden_params")
and response_obj._hidden_params is not None
and isinstance(response_obj._hidden_params, dict)
):
response_obj._hidden_params["cache_hit"] = True
if kwargs.get("stream", False) is True:
cached_result = CachedResponsesAPIStreamingIterator(
response=response_obj,
logging_obj=logging_obj,
request_data=kwargs,
call_type=call_type,
)
else:
cached_result = response_obj
if (
hasattr(cached_result, "_hidden_params")

24
litellm/caching/dual_cache.py
View file

@ -92,6 +92,25 @@ class DualCache(BaseCache):
if default_redis_ttl is not None:
self.default_redis_ttl = default_redis_ttl
def attach_redis_cache(
self,
redis_cache: Optional[RedisCache] = None,
*,
default_redis_ttl: Optional[float] = None,
) -> None:
"""
Attach a Redis backend if this DualCache does not already have one.
No-op when ``redis_cache`` is None or when Redis was already set (constructor
or a prior attach). Use this for lazy wiring after a shared Redis client exists.
Does not backfill in-memory-only keys to Redis.
"""
if redis_cache is None or self.redis_cache is not None:
return
self.redis_cache = redis_cache
if default_redis_ttl is not None:
self.default_redis_ttl = default_redis_ttl
def set_cache(self, key, value, local_only: bool = False, **kwargs):
# Update both Redis and in-memory cache
try:
@ -392,6 +411,7 @@ class DualCache(BaseCache):
value: float,
parent_otel_span: Optional[Span] = None,
local_only: bool = False,
refresh_ttl: bool = False,
**kwargs,
) -> Optional[float]:
"""
@ -399,6 +419,9 @@ class DualCache(BaseCache):
Value - float - the value you want to increment by
Refresh_ttl - bool - if True, resets the Redis TTL on every write.
Default False preserves window-style semantics.
Returns - the incremented value, or None if no cache backend is
available (in_memory_cache is None and Redis failed/is absent).
"""
@ -415,6 +438,7 @@ class DualCache(BaseCache):
value,
parent_otel_span=parent_otel_span,
ttl=kwargs.get("ttl", None),
refresh_ttl=refresh_ttl,
)
return result

194
litellm/caching/qdrant_semantic_cache.py
View file

@ -11,17 +11,23 @@ Has 4 methods:
import ast
import asyncio
import json
from typing import Any, cast
import os
from typing import Any, Dict, cast
import litellm
from litellm._logging import print_verbose
from litellm.constants import QDRANT_SCALAR_QUANTILE, QDRANT_VECTOR_SIZE
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_str_from_messages,
)
from litellm.types.utils import EmbeddingResponse
from .base_cache import BaseCache
class QdrantSemanticCache(BaseCache):
CACHE_KEY_FIELD_NAME = "litellm_cache_key"
def __init__( # noqa: PLR0915
self,
qdrant_api_base=None,
@ -33,8 +39,6 @@ class QdrantSemanticCache(BaseCache):
host_type=None,
vector_size=None,
):
import os
from litellm.llms.custom_httpx.http_handler import (
_get_httpx_client,
get_async_httpx_client,
@ -115,7 +119,9 @@ class QdrantSemanticCache(BaseCache):
print_verbose(
f"Collection already exists.\nCollection details:{self.collection_info}"
)
self._ensure_cache_key_payload_index()
else:
quantization_params: Dict[str, Any]
if quantization_config is None or quantization_config == "binary":
quantization_params = {
"binary": {
@ -156,6 +162,7 @@ class QdrantSemanticCache(BaseCache):
print_verbose(
f"New collection created.\nCollection details:{self.collection_info}"
)
self._ensure_cache_key_payload_index()
else:
raise Exception("Error while creating new collection")
@ -170,15 +177,94 @@ class QdrantSemanticCache(BaseCache):
cached_response = ast.literal_eval(cached_response)
return cached_response
def _get_qdrant_cache_key_filter(self, key: str) -> dict:
return {
"must": [
{
"key": self.CACHE_KEY_FIELD_NAME,
"match": {"value": str(key)},
}
]
}
def _add_cache_key_filter_to_search_data(self, data: dict, key: str) -> None:
data["filter"] = self._get_qdrant_cache_key_filter(key)
def _ensure_cache_key_payload_index(self) -> None:
try:
response = self.sync_client.put(
url=f"{self.qdrant_api_base}/collections/{self.collection_name}/index",
headers=self.headers,
json={
"field_name": self.CACHE_KEY_FIELD_NAME,
"field_schema": "keyword",
},
)
if response.status_code not in (200, 201):
print_verbose(
"Qdrant semantic-cache could not create cache-key payload index: "
f"{response.text}"
)
except Exception as exc:
print_verbose(
"Qdrant semantic-cache could not create cache-key payload index: "
f"{str(exc)}"
)
def _payload_matches_cache_key(self, payload: dict, key: str) -> bool:
# Pre-isolation points stored only prompt + response with no cache-key
# payload field. Reassigning them to a caller's key would risk
# cross-scope hits, so they're treated as misses and re-populated on
# the next set_cache.
cached_key = payload.get(self.CACHE_KEY_FIELD_NAME)
return cached_key is not None and str(cached_key) == str(key)
async def _get_async_embedding(self, prompt: str, **kwargs) -> Any:
llm_model_list = None
llm_router = None
try:
from litellm.proxy.proxy_server import (
llm_model_list as proxy_llm_model_list,
llm_router as proxy_llm_router,
)
llm_model_list = proxy_llm_model_list
llm_router = proxy_llm_router
except ImportError:
pass
router_model_names = (
[m["model_name"] for m in llm_model_list]
if llm_model_list is not None
else []
)
if llm_router is not None and self.embedding_model in router_model_names:
user_api_key = kwargs.get("metadata", {}).get("user_api_key", "")
return await llm_router.aembedding(
model=self.embedding_model,
input=prompt,
cache={"no-store": True, "no-cache": True},
metadata={
"user_api_key": user_api_key,
"semantic-cache-embedding": True,
"trace_id": kwargs.get("metadata", {}).get("trace_id", None),
},
)
return await litellm.aembedding(
model=self.embedding_model,
input=prompt,
cache={"no-store": True, "no-cache": True},
)
def set_cache(self, key, value, **kwargs):
print_verbose(f"qdrant semantic-cache set_cache, kwargs: {kwargs}")
from litellm._uuid import uuid
# get the prompt
messages = kwargs["messages"]
prompt = ""
for message in messages:
prompt += message["content"]
prompt = get_str_from_messages(messages)
# create an embedding for prompt
embedding_response = cast(
@ -202,6 +288,7 @@ class QdrantSemanticCache(BaseCache):
"id": str(uuid.uuid4()),
"vector": embedding,
"payload": {
self.CACHE_KEY_FIELD_NAME: str(key),
"text": prompt,
"response": value,
},
@ -220,9 +307,7 @@ class QdrantSemanticCache(BaseCache):
# get the messages
messages = kwargs["messages"]
prompt = ""
for message in messages:
prompt += message["content"]
prompt = get_str_from_messages(messages)
# convert to embedding
embedding_response = cast(
@ -249,6 +334,7 @@ class QdrantSemanticCache(BaseCache):
"limit": 1,
"with_payload": True,
}
self._add_cache_key_filter_to_search_data(data=data, key=key)
search_response = self.sync_client.post(
url=f"{self.qdrant_api_base}/collections/{self.collection_name}/points/search",
@ -258,21 +344,33 @@ class QdrantSemanticCache(BaseCache):
results = search_response.json()["result"]
if results is None:
kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
return None
if isinstance(results, list):
if len(results) == 0:
kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
return None
similarity = results[0]["score"]
cached_prompt = results[0]["payload"]["text"]
payload = results[0]["payload"]
if not self._payload_matches_cache_key(payload=payload, key=key):
print_verbose("Qdrant semantic-cache hit did not match cache key scope")
kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
return None
cached_prompt = payload["text"]
# check similarity, if more than self.similarity_threshold, return results
print_verbose(
f"semantic cache: similarity threshold: {self.similarity_threshold}, similarity: {similarity}, prompt: {prompt}, closest_cached_prompt: {cached_prompt}"
)
# update kwargs["metadata"] with similarity, don't rewrite the original metadata
kwargs.setdefault("metadata", {})["semantic-similarity"] = similarity
if similarity >= self.similarity_threshold:
# cache hit !
cached_value = results[0]["payload"]["response"]
cached_value = payload["response"]
print_verbose(
f"got a cache hit, similarity: {similarity}, Current prompt: {prompt}, cached_prompt: {cached_prompt}"
)
@ -285,40 +383,12 @@ class QdrantSemanticCache(BaseCache):
async def async_set_cache(self, key, value, **kwargs):
from litellm._uuid import uuid
from litellm.proxy.proxy_server import llm_model_list, llm_router
print_verbose(f"async qdrant semantic-cache set_cache, kwargs: {kwargs}")
# get the prompt
messages = kwargs["messages"]
prompt = ""
for message in messages:
prompt += message["content"]
# create an embedding for prompt
router_model_names = (
[m["model_name"] for m in llm_model_list]
if llm_model_list is not None
else []
)
if llm_router is not None and self.embedding_model in router_model_names:
user_api_key = kwargs.get("metadata", {}).get("user_api_key", "")
embedding_response = await llm_router.aembedding(
model=self.embedding_model,
input=prompt,
cache={"no-store": True, "no-cache": True},
metadata={
"user_api_key": user_api_key,
"semantic-cache-embedding": True,
"trace_id": kwargs.get("metadata", {}).get("trace_id", None),
},
)
else:
# convert to embedding
embedding_response = await litellm.aembedding(
model=self.embedding_model,
input=prompt,
cache={"no-store": True, "no-cache": True},
)
prompt = get_str_from_messages(messages)
embedding_response = await self._get_async_embedding(prompt, **kwargs)
# get the embedding
embedding = embedding_response["data"][0]["embedding"]
@ -332,6 +402,7 @@ class QdrantSemanticCache(BaseCache):
"id": str(uuid.uuid4()),
"vector": embedding,
"payload": {
self.CACHE_KEY_FIELD_NAME: str(key),
"text": prompt,
"response": value,
},
@ -348,38 +419,12 @@ class QdrantSemanticCache(BaseCache):
async def async_get_cache(self, key, **kwargs):
print_verbose(f"async qdrant semantic-cache get_cache, kwargs: {kwargs}")
from litellm.proxy.proxy_server import llm_model_list, llm_router
# get the messages
messages = kwargs["messages"]
prompt = ""
for message in messages:
prompt += message["content"]
prompt = get_str_from_messages(messages)
router_model_names = (
[m["model_name"] for m in llm_model_list]
if llm_model_list is not None
else []
)
if llm_router is not None and self.embedding_model in router_model_names:
user_api_key = kwargs.get("metadata", {}).get("user_api_key", "")
embedding_response = await llm_router.aembedding(
model=self.embedding_model,
input=prompt,
cache={"no-store": True, "no-cache": True},
metadata={
"user_api_key": user_api_key,
"semantic-cache-embedding": True,
"trace_id": kwargs.get("metadata", {}).get("trace_id", None),
},
)
else:
# convert to embedding
embedding_response = await litellm.aembedding(
model=self.embedding_model,
input=prompt,
cache={"no-store": True, "no-cache": True},
)
embedding_response = await self._get_async_embedding(prompt, **kwargs)
# get the embedding
embedding = embedding_response["data"][0]["embedding"]
@ -396,6 +441,7 @@ class QdrantSemanticCache(BaseCache):
"limit": 1,
"with_payload": True,
}
self._add_cache_key_filter_to_search_data(data=data, key=key)
search_response = await self.async_client.post(
url=f"{self.qdrant_api_base}/collections/{self.collection_name}/points/search",
@ -414,7 +460,13 @@ class QdrantSemanticCache(BaseCache):
return None
similarity = results[0]["score"]
cached_prompt = results[0]["payload"]["text"]
payload = results[0]["payload"]
if not self._payload_matches_cache_key(payload=payload, key=key):
print_verbose("Qdrant semantic-cache hit did not match cache key scope")
kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
return None
cached_prompt = payload["text"]
# check similarity, if more than self.similarity_threshold, return results
print_verbose(
@ -426,7 +478,7 @@ class QdrantSemanticCache(BaseCache):
if similarity >= self.similarity_threshold:
# cache hit !
cached_value = results[0]["payload"]["response"]
cached_value = payload["response"]
print_verbose(
f"got a cache hit, similarity: {similarity}, Current prompt: {prompt}, cached_prompt: {cached_prompt}"
)

20
litellm/caching/redis_cache.py
View file

@ -551,6 +551,13 @@ class RedisCache(BaseCache):
async def async_set_cache(self, key, value, **kwargs):
from redis.asyncio import Redis
if key is None:
verbose_logger.debug(
"LiteLLM Redis Caching: async set() skipped — key is None, value=%r",
value,
)
return None
start_time = time.time()
try:
_redis_client: Redis = self.init_async_client() # type: ignore
@ -569,8 +576,9 @@ class RedisCache(BaseCache):
)
)
verbose_logger.error(
"LiteLLM Redis Caching: async set() - Got exception from REDIS %s, Writing value=%s",
"LiteLLM Redis Caching: async set() - Got exception from REDIS %s, key=%r, value=%r",
str(e),
key,
value,
)
raise e
@ -824,6 +832,7 @@ class RedisCache(BaseCache):
value: float,
ttl: Optional[int] = None,
parent_otel_span: Optional[Span] = None,
refresh_ttl: bool = False,
) -> float:
from redis.asyncio import Redis
@ -834,11 +843,12 @@ class RedisCache(BaseCache):
try:
result = await _redis_client.incrbyfloat(name=key, amount=value)
if _used_ttl is not None:
# check if key already has ttl, if not -> set ttl
current_ttl = await _redis_client.ttl(key)
if current_ttl == -1:
# Key has no expiration
if refresh_ttl:
await _redis_client.expire(key, _used_ttl)
else:
current_ttl = await _redis_client.ttl(key)
if current_ttl == -1:
await _redis_client.expire(key, _used_ttl)
## LOGGING ##
end_time = time.time()

172
litellm/caching/redis_semantic_cache.py
View file

@ -35,6 +35,7 @@ class RedisSemanticCache(BaseCache):
"""
DEFAULT_REDIS_INDEX_NAME: str = "litellm_semantic_cache_index"
CACHE_KEY_FIELD_NAME: str = "litellm_cache_key"
def __init__(
self,
@ -66,8 +67,8 @@ class RedisSemanticCache(BaseCache):
Exception: If similarity_threshold is not provided or required Redis
connection information is missing
"""
from redisvl.extensions.llmcache import SemanticCache
from redisvl.utils.vectorize import CustomTextVectorizer
from redisvl.extensions.llmcache import SemanticCache # type: ignore[import-not-found, import-untyped]
from redisvl.utils.vectorize import CustomTextVectorizer # type: ignore[import-not-found, import-untyped]
if index_name is None:
index_name = self.DEFAULT_REDIS_INDEX_NAME
@ -109,14 +110,94 @@ class RedisSemanticCache(BaseCache):
# Initialize the Redis vectorizer and cache
cache_vectorizer = CustomTextVectorizer(self._get_embedding)
self.llmcache = SemanticCache(
name=index_name,
self.llmcache = self._init_semantic_cache(
semantic_cache_cls=SemanticCache,
index_name=index_name,
redis_url=redis_url,
vectorizer=cache_vectorizer,
distance_threshold=self.distance_threshold,
overwrite=False,
cache_vectorizer=cache_vectorizer,
)
@classmethod
def _cache_key_filterable_field(cls) -> Dict[str, str]:
return {
"name": cls.CACHE_KEY_FIELD_NAME,
"type": "tag",
}
def _init_semantic_cache(
self,
semantic_cache_cls: Any,
index_name: str,
redis_url: str,
cache_vectorizer: Any,
) -> Any:
def _is_schema_mismatch(exc: ValueError) -> bool:
error_message = str(exc).lower()
return any(
phrase in error_message
for phrase in ("schema does not match", "index schema")
)
try:
return semantic_cache_cls(
name=index_name,
redis_url=redis_url,
vectorizer=cache_vectorizer,
distance_threshold=self.distance_threshold,
filterable_fields=[self._cache_key_filterable_field()],
overwrite=False,
)
except ValueError as exc:
if not _is_schema_mismatch(exc):
raise
isolated_index_name = f"{index_name}_isolated"
print_verbose(
"Redis semantic-cache existing index schema is not isolated; "
f"using isolated index - {isolated_index_name}"
)
try:
return semantic_cache_cls(
name=isolated_index_name,
redis_url=redis_url,
vectorizer=cache_vectorizer,
distance_threshold=self.distance_threshold,
filterable_fields=[self._cache_key_filterable_field()],
overwrite=False,
)
except ValueError as isolated_exc:
if not _is_schema_mismatch(isolated_exc):
raise
print_verbose(
"Redis semantic-cache isolated index schema is stale; "
f"recreating isolated index - {isolated_index_name}"
)
return semantic_cache_cls(
name=isolated_index_name,
redis_url=redis_url,
vectorizer=cache_vectorizer,
distance_threshold=self.distance_threshold,
filterable_fields=[self._cache_key_filterable_field()],
overwrite=True,
)
def _get_cache_filters(self, key: str) -> Dict[str, str]:
return {self.CACHE_KEY_FIELD_NAME: str(key)}
def _get_cache_key_filter_expression(self, key: str) -> Any:
from redisvl.query.filter import Tag # type: ignore[import-not-found, import-untyped]
return Tag(self.CACHE_KEY_FIELD_NAME) == str(key)
def _cache_hit_matches_key(self, cache_hit: Dict[str, Any], key: str) -> bool:
# Pre-isolation entries with no ``litellm_cache_key`` field cannot be
# safely reassigned to a caller's scope and are treated as misses.
cached_key = cache_hit.get(self.CACHE_KEY_FIELD_NAME)
if isinstance(cached_key, bytes):
cached_key = cached_key.decode("utf-8")
return cached_key is not None and str(cached_key) == str(key)
def _get_ttl(self, **kwargs) -> Optional[int]:
"""
Get the TTL (time-to-live) value for cache entries.
@ -188,7 +269,7 @@ class RedisSemanticCache(BaseCache):
Store a value in the semantic cache.
Args:
key: The cache key (not directly used in semantic caching)
key: The cache key used to isolate semantic cache entries
value: The response value to cache
**kwargs: Additional arguments including 'messages' for the prompt
and optional 'ttl' for time-to-live
@ -206,12 +287,15 @@ class RedisSemanticCache(BaseCache):
prompt = get_str_from_messages(messages)
value_str = str(value)
store_kwargs: Dict[str, Any] = {
"filters": self._get_cache_filters(key),
}
# Get TTL and store in Redis semantic cache
ttl = self._get_ttl(**kwargs)
if ttl is not None:
self.llmcache.store(prompt, value_str, ttl=int(ttl))
else:
self.llmcache.store(prompt, value_str)
store_kwargs["ttl"] = int(ttl)
self.llmcache.store(prompt, value_str, **store_kwargs)
except Exception as e:
print_verbose(
f"Error setting {value_str or value} in the Redis semantic cache: {str(e)}"
@ -222,7 +306,7 @@ class RedisSemanticCache(BaseCache):
Retrieve a semantically similar cached response.
Args:
key: The cache key (not directly used in semantic caching)
key: The cache key used to isolate semantic cache entries
**kwargs: Additional arguments including 'messages' for the prompt
Returns:
@ -235,18 +319,29 @@ class RedisSemanticCache(BaseCache):
messages = kwargs.get("messages", [])
if not messages:
print_verbose("No messages provided for semantic cache lookup")
kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
return None
prompt = get_str_from_messages(messages)
# Check the cache for semantically similar prompts
results = self.llmcache.check(prompt=prompt)
# Check the cache for semantically similar prompts in this exact
# LiteLLM cache-key scope.
check_kwargs: Dict[str, Any] = {
"prompt": prompt,
"filter_expression": self._get_cache_key_filter_expression(key),
}
results = self.llmcache.check(**check_kwargs)
# Return None if no similar prompts found
if not results:
kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
return None
# Process the best matching result
cache_hit = results[0]
if not self._cache_hit_matches_key(cache_hit=cache_hit, key=key):
print_verbose("Redis semantic-cache hit did not match cache key scope")
kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
return None
vector_distance = float(cache_hit["vector_distance"])
# Convert vector distance back to similarity score
@ -257,6 +352,9 @@ class RedisSemanticCache(BaseCache):
cached_prompt = cache_hit["prompt"]
cached_response = cache_hit["response"]
# update kwargs["metadata"] with similarity, don't rewrite the original metadata
kwargs.setdefault("metadata", {})["semantic-similarity"] = similarity
print_verbose(
f"Cache hit: similarity threshold: {self.similarity_threshold}, "
f"actual similarity: {similarity}, "
@ -267,6 +365,7 @@ class RedisSemanticCache(BaseCache):
return self._get_cache_logic(cached_response=cached_response)
except Exception as e:
print_verbose(f"Error retrieving from Redis semantic cache: {str(e)}")
kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
async def _get_async_embedding(self, prompt: str, **kwargs) -> List[float]:
"""
@ -321,7 +420,7 @@ class RedisSemanticCache(BaseCache):
Asynchronously store a value in the semantic cache.
Args:
key: The cache key (not directly used in semantic caching)
key: The cache key used to isolate semantic cache entries
value: The response value to cache
**kwargs: Additional arguments including 'messages' for the prompt
and optional 'ttl' for time-to-live
@ -341,21 +440,20 @@ class RedisSemanticCache(BaseCache):
# Generate embedding for the value (response) to cache
prompt_embedding = await self._get_async_embedding(prompt, **kwargs)
store_kwargs: Dict[str, Any] = {
"vector": prompt_embedding,
"filters": self._get_cache_filters(key),
}
# Get TTL and store in Redis semantic cache
ttl = self._get_ttl(**kwargs)
if ttl is not None:
await self.llmcache.astore(
prompt,
value_str,
vector=prompt_embedding, # Pass through custom embedding
ttl=ttl,
)
else:
await self.llmcache.astore(
prompt,
value_str,
vector=prompt_embedding, # Pass through custom embedding
)
store_kwargs["ttl"] = ttl
await self.llmcache.astore(
prompt,
value_str,
**store_kwargs,
)
except Exception as e:
print_verbose(f"Error in async_set_cache: {str(e)}")
@ -364,7 +462,7 @@ class RedisSemanticCache(BaseCache):
Asynchronously retrieve a semantically similar cached response.
Args:
key: The cache key (not directly used in semantic caching)
key: The cache key used to isolate semantic cache entries
**kwargs: Additional arguments including 'messages' for the prompt
Returns:
@ -385,17 +483,25 @@ class RedisSemanticCache(BaseCache):
# Generate embedding for the prompt
prompt_embedding = await self._get_async_embedding(prompt, **kwargs)
# Check the cache for semantically similar prompts
results = await self.llmcache.acheck(prompt=prompt, vector=prompt_embedding)
# Check the cache for semantically similar prompts in this exact
# LiteLLM cache-key scope.
check_kwargs: Dict[str, Any] = {
"prompt": prompt,
"vector": prompt_embedding,
"filter_expression": self._get_cache_key_filter_expression(key),
}
results = await self.llmcache.acheck(**check_kwargs)
# handle results / cache hit
if not results:
kwargs.setdefault("metadata", {})[
"semantic-similarity"
] = 0.0 # TODO why here but not above??
kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
return None
cache_hit = results[0]
if not self._cache_hit_matches_key(cache_hit=cache_hit, key=key):
print_verbose("Redis semantic-cache hit did not match cache key scope")
kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
return None
vector_distance = float(cache_hit["vector_distance"])
# Convert vector distance back to similarity

26
litellm/constants.py
View file

@ -202,6 +202,12 @@ DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET = int(
DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET = int(
os.getenv("DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET", 4096)
)
DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET = int(
os.getenv("DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET", 8192)
)
DEFAULT_REASONING_EFFORT_MAX_THINKING_BUDGET = int(
os.getenv("DEFAULT_REASONING_EFFORT_MAX_THINKING_BUDGET", 16384)
)
MAX_TOKEN_TRIMMING_ATTEMPTS = int(
os.getenv("MAX_TOKEN_TRIMMING_ATTEMPTS", 10)
) # Maximum number of attempts to trim the message
@ -224,6 +230,16 @@ AIOHTTP_CONNECTOR_LIMIT_PER_HOST = int(
)
AIOHTTP_KEEPALIVE_TIMEOUT = int(os.getenv("AIOHTTP_KEEPALIVE_TIMEOUT", 120))
AIOHTTP_TTL_DNS_CACHE = int(os.getenv("AIOHTTP_TTL_DNS_CACHE", 300))
# TCP keep-alive (SO_KEEPALIVE) — opt-in. Required when running behind NAT/LBs
# whose idle timeout is shorter than provider response timeouts (e.g. AWS NAT
# Gateway: 350s vs OpenAI/Azure: 600s). Without this, the kernel sends nothing
# during a long provider call and the NAT reaps the flow before the response
# arrives. Enabling SO_KEEPALIVE makes the kernel emit TCP probes that reset
# the NAT idle timer.
AIOHTTP_SO_KEEPALIVE = os.getenv("AIOHTTP_SO_KEEPALIVE", "False").lower() == "true"
AIOHTTP_TCP_KEEPIDLE = int(os.getenv("AIOHTTP_TCP_KEEPIDLE", 60))
AIOHTTP_TCP_KEEPINTVL = int(os.getenv("AIOHTTP_TCP_KEEPINTVL", 30))
AIOHTTP_TCP_KEEPCNT = int(os.getenv("AIOHTTP_TCP_KEEPCNT", 5))
# enable_cleanup_closed is only needed for Python versions with the SSL leak bug
# Fixed in Python 3.12.7+ and 3.13.1+ (see https://github.com/python/cpython/pull/118960)
# Reference: https://github.com/aio-libs/aiohttp/blob/master/aiohttp/connector.py#L74-L78
@ -389,6 +405,8 @@ BEDROCK_MAX_POLICY_SIZE = int(os.getenv("BEDROCK_MAX_POLICY_SIZE", 75))
BEDROCK_MIN_THINKING_BUDGET_TOKENS = int(
os.getenv("BEDROCK_MIN_THINKING_BUDGET_TOKENS", 1024)
)
# Anthropic's Messages API rejects thinking.budget_tokens < 1024.
ANTHROPIC_MIN_THINKING_BUDGET_TOKENS = 1024
REPLICATE_POLLING_DELAY_SECONDS = float(
os.getenv("REPLICATE_POLLING_DELAY_SECONDS", 0.5)
)
@ -409,9 +427,6 @@ CACHED_STREAMING_CHUNK_DELAY = float(os.getenv("CACHED_STREAMING_CHUNK_DELAY", 0
AUDIO_SPEECH_CHUNK_SIZE = int(
os.getenv("AUDIO_SPEECH_CHUNK_SIZE", 8192)
) # chunk_size for audio speech streaming. Balance between latency and memory usage
MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB = int(
os.getenv("MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB", 512)
)
DEFAULT_MAX_TOKENS_FOR_TRITON = int(os.getenv("DEFAULT_MAX_TOKENS_FOR_TRITON", 2000))
#### Networking settings ####
# Sentinel used when `REQUEST_TIMEOUT` is unset: `litellm.request_timeout` keeps this
@ -1383,6 +1398,10 @@ except (ValueError, TypeError):
LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME = "litellm-internal-health-check"
LITTELM_CLI_SERVICE_ACCOUNT_NAME = "litellm-cli"
LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME = "litellm_internal_jobs"
# Stable identifier substituted in place of the master key on UserAPIKeyAuth
# objects so the master key (or its hash) never propagates to spend logs,
# Prometheus metrics, audit trails, or any other downstream consumer.
LITELLM_PROXY_MASTER_KEY_ALIAS = "litellm_proxy_master_key"
# Key Rotation Constants
LITELLM_KEY_ROTATION_ENABLED = os.getenv("LITELLM_KEY_ROTATION_ENABLED", "false")
@ -1411,6 +1430,7 @@ LITELLM_PROXY_ADMIN_NAME = "default_user_id"
LITELLM_CLI_SOURCE_IDENTIFIER = "litellm-cli"
LITELLM_CLI_SESSION_TOKEN_PREFIX = "litellm-session-token"
CLI_SSO_SESSION_CACHE_KEY_PREFIX = "cli_sso_session"
CLI_SSO_SESSION_TTL_SECONDS = 600
CLI_JWT_TOKEN_NAME = "cli-jwt-token"
# Support both CLI_JWT_EXPIRATION_HOURS and LITELLM_CLI_JWT_EXPIRATION_HOURS for backwards compatibility
CLI_JWT_EXPIRATION_HOURS = int(

6
litellm/cost_calculator.py
View file

@ -513,7 +513,10 @@ def cost_per_token( # noqa: PLR0915
return fireworks_ai_cost_per_token(model=model, usage=usage_block)
elif custom_llm_provider == "azure":
return azure_openai_cost_per_token(
model=model, usage=usage_block, response_time_ms=response_time_ms
model=model,
usage=usage_block,
response_time_ms=response_time_ms,
service_tier=service_tier,
)
elif custom_llm_provider == "gemini":
return gemini_cost_per_token(
@ -539,6 +542,7 @@ def cost_per_token( # noqa: PLR0915
usage=usage_block,
response_time_ms=response_time_ms,
request_model=request_model,
service_tier=service_tier,
)
else:
model_info = _cached_get_model_info_helper(

24
litellm/files/main.py
View file

@ -10,6 +10,7 @@ import contextvars
import time
import uuid as uuid_module
from functools import partial
from types import MappingProxyType
from typing import Any, Coroutine, Dict, Literal, Optional, Union, cast
import httpx
@ -85,6 +86,16 @@ bedrock_files_instance = BedrockFilesHandler()
#################################################
def _add_trusted_model_credentials_to_litellm_params(
litellm_params_dict: Dict[str, Any], kwargs: Dict[str, Any]
) -> None:
trusted_model_credentials = kwargs.get("_litellm_internal_model_credentials")
if isinstance(trusted_model_credentials, type(MappingProxyType({}))):
litellm_params_dict["_litellm_internal_model_credentials"] = (
trusted_model_credentials
)
@client
async def acreate_file(
file: FileTypes,
@ -373,6 +384,10 @@ def file_retrieve(
)
if provider_config is not None:
litellm_params_dict = get_litellm_params(**kwargs)
_add_trusted_model_credentials_to_litellm_params(
litellm_params_dict=litellm_params_dict,
kwargs=kwargs,
)
litellm_params_dict["api_key"] = optional_params.api_key
litellm_params_dict["api_base"] = optional_params.api_base
@ -497,6 +512,10 @@ def file_delete(
pass
optional_params = GenericLiteLLMParams(**kwargs)
litellm_params_dict = get_litellm_params(**kwargs)
_add_trusted_model_credentials_to_litellm_params(
litellm_params_dict=litellm_params_dict,
kwargs=kwargs,
)
### TIMEOUT LOGIC ###
timeout = optional_params.timeout or kwargs.get("request_timeout", 600) or 600
# set timeout for 10 minutes by default
@ -846,6 +865,10 @@ def file_content(
try:
optional_params = GenericLiteLLMParams(**kwargs)
litellm_params_dict = get_litellm_params(**kwargs)
_add_trusted_model_credentials_to_litellm_params(
litellm_params_dict=litellm_params_dict,
kwargs=kwargs,
)
### TIMEOUT LOGIC ###
timeout = optional_params.timeout or kwargs.get("request_timeout", 600) or 600
client = kwargs.get("client")
@ -993,6 +1016,7 @@ def file_content(
vertex_location=vertex_ai_location,
timeout=timeout,
max_retries=optional_params.max_retries,
litellm_params=litellm_params_dict,
)
elif custom_llm_provider == "bedrock":
response = bedrock_files_instance.file_content(

8
litellm/google_genai/streaming_iterator.py
View file

@ -1,6 +1,6 @@
import asyncio
from datetime import datetime
from typing import TYPE_CHECKING, Any, List, Optional
from typing import TYPE_CHECKING, Any, Dict, List, Optional
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.proxy.pass_through_endpoints.success_handler import (
@ -29,12 +29,14 @@ class BaseGoogleGenAIGenerateContentStreamingIterator:
litellm_logging_obj: LiteLLMLoggingObj,
request_body: dict,
model: str,
hidden_params: Optional[Dict[str, Any]] = None,
):
self.litellm_logging_obj = litellm_logging_obj
self.request_body = request_body
self.start_time = datetime.now()
self.collected_chunks: List[bytes] = []
self.model = model
self._hidden_params: Dict[str, Any] = hidden_params or {}
async def _handle_async_streaming_logging(
self,
@ -76,11 +78,13 @@ class GoogleGenAIGenerateContentStreamingIterator(
litellm_metadata: dict,
custom_llm_provider: str,
request_body: Optional[dict] = None,
hidden_params: Optional[Dict[str, Any]] = None,
):
super().__init__(
litellm_logging_obj=logging_obj,
request_body=request_body or {},
model=model,
hidden_params=hidden_params,
)
self.response = response
self.model = model
@ -130,11 +134,13 @@ class AsyncGoogleGenAIGenerateContentStreamingIterator(
litellm_metadata: dict,
custom_llm_provider: str,
request_body: Optional[dict] = None,
hidden_params: Optional[Dict[str, Any]] = None,
):
super().__init__(
litellm_logging_obj=logging_obj,
request_body=request_body or {},
model=model,
hidden_params=hidden_params,
)
self.response = response
self.model = model

42
litellm/integrations/arize/_utils.py
View file

@ -220,23 +220,57 @@ def _set_structured_outputs(span: "Span", response_obj, msg_attrs, span_attrs):
safe_set_attribute(span, f"{prefix}.{msg_attrs.MESSAGE_ROLE}", message_role)
def _safe_get(obj, key, default=None):
"""Read ``key`` from a dict-like or Pydantic-model-like object.
The arize/langfuse_otel logger receives ``usage`` objects from many sources:
plain dicts, litellm ``Usage`` (which exposes ``.get``), and raw OpenAI
Pydantic models (e.g. ``openai.types.completion_usage.CompletionUsage`` and
nested ``CompletionTokensDetails`` / ``OutputTokensDetails``) which do NOT
expose ``.get``. Calling ``.get`` on the latter raised ``AttributeError`` —
see https://github.com/BerriAI/litellm/issues/13672.
"""
if obj is None:
return default
getter = getattr(obj, "get", None)
if callable(getter):
try:
return getter(key, default)
except TypeError:
# Some objects expose `.get` with a different signature
pass
return getattr(obj, key, default)
def _set_usage_outputs(span: "Span", response_obj, span_attrs):
usage = response_obj and response_obj.get("usage")
if not usage:
return
safe_set_attribute(
span, span_attrs.LLM_TOKEN_COUNT_TOTAL, usage.get("total_tokens")
span, span_attrs.LLM_TOKEN_COUNT_TOTAL, _safe_get(usage, "total_tokens")
)
completion_tokens = _safe_get(usage, "completion_tokens") or _safe_get(
usage, "output_tokens"
)
completion_tokens = usage.get("completion_tokens") or usage.get("output_tokens")
if completion_tokens:
safe_set_attribute(
span, span_attrs.LLM_TOKEN_COUNT_COMPLETION, completion_tokens
)
prompt_tokens = usage.get("prompt_tokens") or usage.get("input_tokens")
prompt_tokens = _safe_get(usage, "prompt_tokens") or _safe_get(
usage, "input_tokens"
)
if prompt_tokens:
safe_set_attribute(span, span_attrs.LLM_TOKEN_COUNT_PROMPT, prompt_tokens)
reasoning_tokens = usage.get("output_tokens_details", {}).get("reasoning_tokens")
# Reasoning tokens live in `completion_tokens_details` for Chat Completions
# API (Usage) and in `output_tokens_details` for Responses API
# (ResponseAPIUsage). Both nested objects may be plain Pydantic models
# without `.get`.
token_details = _safe_get(usage, "completion_tokens_details") or _safe_get(
usage, "output_tokens_details"
)
reasoning_tokens = _safe_get(token_details, "reasoning_tokens")
if reasoning_tokens:
safe_set_attribute(
span,

15
litellm/integrations/arize/arize_phoenix_client.py
View file

@ -2,11 +2,23 @@
Arize Phoenix API client for fetching prompt versions from Arize Phoenix.
"""
import urllib.parse
from typing import Any, Dict, Optional
from litellm.llms.custom_httpx.http_handler import HTTPHandler
def _sanitize_id(identifier: str) -> str:
"""Reject path traversal characters and URL-encode the identifier."""
if any(c in identifier for c in ("/", "\\", "#", "?")):
raise ValueError(
f"Invalid identifier {identifier!r}: contains disallowed characters"
)
if ".." in identifier:
raise ValueError(f"Invalid identifier {identifier!r}: path traversal detected")
return urllib.parse.quote(identifier, safe="")
class ArizePhoenixClient:
"""
Client for interacting with Arize Phoenix API to fetch prompt versions.
@ -53,7 +65,8 @@ class ArizePhoenixClient:
Returns:
Dictionary containing prompt version data, or None if not found
"""
url = f"{self.api_base}/v1/prompt_versions/{prompt_version_id}"
safe_id = _sanitize_id(prompt_version_id)
url = f"{self.api_base}/v1/prompt_versions/{safe_id}"
try:
# Use the underlying httpx client directly to avoid query param extraction

11
litellm/integrations/arize/arize_phoenix_prompt_manager.py
View file

@ -5,7 +5,8 @@ Fetches prompt versions from Arize Phoenix and provides workspace-based access c
from typing import Any, Dict, List, Optional, Tuple, Union
from jinja2 import DictLoader, Environment, select_autoescape
from jinja2 import DictLoader, select_autoescape
from jinja2.sandbox import ImmutableSandboxedEnvironment
from litellm.integrations.custom_prompt_management import CustomPromptManagement
from litellm.integrations.prompt_management_base import (
@ -74,7 +75,13 @@ class ArizePhoenixTemplateManager:
api_key=self.api_key, api_base=self.api_base
)
self.jinja_env = Environment(
# Templates fetched from Arize Phoenix come from external workspace
# users; in a plain `Environment()` a malicious template could reach
# `__class__.__init__.__globals__` and execute arbitrary code on the
# proxy host. The sandbox blocks that attribute traversal while
# leaving normal `{{ var }}` substitution intact. Matches the
# dotprompt manager's hardening.
self.jinja_env = ImmutableSandboxedEnvironment(
loader=DictLoader({}),
autoescape=select_autoescape(["html", "xml"]),
# Use Mustache/Handlebars-style delimiters

25
litellm/integrations/bitbucket/bitbucket_client.py
View file

@ -3,11 +3,27 @@ BitBucket API client for fetching .prompt files from BitBucket repositories.
"""
import base64
import urllib.parse
from typing import Any, Dict, List, Optional
from litellm.llms.custom_httpx.http_handler import HTTPHandler
def _sanitize_file_path(file_path: str) -> str:
"""Reject path traversal and URL-encode each path segment."""
if "#" in file_path or "?" in file_path:
raise ValueError(
f"Invalid file path {file_path!r}: contains URL special characters"
)
parts = file_path.split("/")
for part in parts:
if part == "..":
raise ValueError(
f"Invalid file path {file_path!r}: path traversal detected"
)
return "/".join(urllib.parse.quote(part, safe="") for part in parts)
class BitBucketClient:
"""
Client for interacting with BitBucket API to fetch .prompt files.
@ -72,7 +88,8 @@ class BitBucketClient:
Returns:
File content as string, or None if file not found
"""
url = f"{self.base_url}/repositories/{self.workspace}/{self.repository}/src/{self.branch}/{file_path}"
safe_path = _sanitize_file_path(file_path)
url = f"{self.base_url}/repositories/{self.workspace}/{self.repository}/src/{self.branch}/{safe_path}"
try:
response = self.http_handler.get(url, headers=self.headers)
@ -119,7 +136,8 @@ class BitBucketClient:
Returns:
List of file paths
"""
url = f"{self.base_url}/repositories/{self.workspace}/{self.repository}/src/{self.branch}/{directory_path}"
safe_dir = _sanitize_file_path(directory_path) if directory_path else ""
url = f"{self.base_url}/repositories/{self.workspace}/{self.repository}/src/{self.branch}/{safe_dir}"
try:
response = self.http_handler.get(url, headers=self.headers)
@ -211,7 +229,8 @@ class BitBucketClient:
Returns:
Dictionary containing file metadata, or None if file not found
"""
url = f"{self.base_url}/repositories/{self.workspace}/{self.repository}/src/{self.branch}/{file_path}"
safe_path = _sanitize_file_path(file_path)
url = f"{self.base_url}/repositories/{self.workspace}/{self.repository}/src/{self.branch}/{safe_path}"
try:
# Use GET with Range header to get just the headers (HEAD equivalent)

11
litellm/integrations/bitbucket/bitbucket_prompt_manager.py
View file

@ -5,7 +5,8 @@ Fetches .prompt files from BitBucket repositories and provides team-based access
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
from jinja2 import DictLoader, Environment, select_autoescape
from jinja2 import DictLoader, select_autoescape
from jinja2.sandbox import ImmutableSandboxedEnvironment
from litellm.integrations.custom_prompt_management import CustomPromptManagement
@ -74,7 +75,13 @@ class BitBucketTemplateManager:
self.prompts: Dict[str, BitBucketPromptTemplate] = {}
self.bitbucket_client = BitBucketClient(bitbucket_config)
self.jinja_env = Environment(
# Templates fetched from a BitBucket repo are not trustworthy:
# anyone with repo write access can ship Jinja syntax that, in a
# plain `Environment()`, would reach `__class__.__init__.__globals__`
# and pivot into RCE on the proxy host. The sandbox blocks that
# attribute traversal while leaving normal `{{ var }}` substitution
# intact. Matches the dotprompt manager's hardening.
self.jinja_env = ImmutableSandboxedEnvironment(
loader=DictLoader({}),
autoescape=select_autoescape(["html", "xml"]),
# Use Handlebars-style delimiters to match Dotprompt spec

11
litellm/integrations/custom_sso_handler.py
View file

@ -18,6 +18,17 @@ class CustomSSOLoginHandler(CustomLogger):
self,
request: Request,
) -> OpenID:
from litellm.proxy.auth.trusted_proxy_utils import (
require_trusted_proxy_request,
)
from litellm.proxy.proxy_server import general_settings
require_trusted_proxy_request(
request=request,
general_settings=general_settings,
feature_name="Custom UI SSO",
)
request_headers_dict = dict(request.headers)
return OpenID(
id=request_headers_dict.get("x-litellm-user-id"),

13
litellm/integrations/gcs_bucket/gcs_bucket.py
View file

@ -6,12 +6,14 @@ import time
from litellm._uuid import uuid
from datetime import datetime, timedelta, timezone
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
from urllib.parse import quote
from litellm._logging import verbose_logger
from litellm.constants import LITELLM_ASYNCIO_QUEUE_MAXSIZE
from litellm.integrations.additional_logging_utils import AdditionalLoggingUtils
from litellm.integrations.gcs_bucket.gcs_bucket_base import GCSBucketBase
from litellm.litellm_core_utils.cloud_storage_security import (
sanitize_cloud_object_component,
)
from litellm.proxy._types import CommonProxyErrors
from litellm.types.integrations.base_health_check import IntegrationHealthCheckStatus
from litellm.types.integrations.gcs_bucket import *
@ -335,7 +337,11 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
_litellm_params = kwargs.get("litellm_params", None) or {}
_metadata = _litellm_params.get("metadata", None) or {}
if "gcs_log_id" in _metadata:
object_name = _metadata["gcs_log_id"]
safe_log_id = sanitize_cloud_object_component(
_metadata.get("gcs_log_id"), fallback=""
)
if safe_log_id:
object_name = f"{current_date}/custom-{uuid.uuid4().hex}-{safe_log_id}"
return object_name
@ -367,8 +373,7 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
request_date_str=date_str,
response_id=request_id,
)
encoded_object_name = quote(object_name, safe="")
response = await self.download_gcs_object(encoded_object_name)
response = await self.download_gcs_object(object_name)
if response is not None:
loaded_response = json.loads(response)

13
litellm/integrations/gcs_bucket/gcs_bucket_base.py
View file

@ -11,6 +11,10 @@ from litellm.integrations.gcs_bucket.gcs_bucket_mock_client import (
from litellm._logging import verbose_logger
from litellm.integrations.custom_batch_logger import CustomBatchLogger
from litellm.litellm_core_utils.cloud_storage_security import (
encode_gcs_object_name_for_url,
split_configured_cloud_bucket_name,
)
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
@ -133,8 +137,8 @@ class GCSBucketBase(CustomBatchLogger):
- Returns: bucket_name="my-bucket", object_name="my-folder/dev/my-object"
"""
if "/" in bucket_name:
bucket_name, prefix = bucket_name.split("/", 1)
bucket_name, prefix = split_configured_cloud_bucket_name(bucket_name)
if prefix:
object_name = f"{prefix}/{object_name}"
return bucket_name, object_name
return bucket_name, object_name
@ -248,6 +252,7 @@ class GCSBucketBase(CustomBatchLogger):
bucket_name=bucket_name,
object_name=object_name,
)
object_name = encode_gcs_object_name_for_url(object_name)
url = f"https://storage.googleapis.com/storage/v1/b/{bucket_name}/o/{object_name}?alt=media"
@ -288,6 +293,7 @@ class GCSBucketBase(CustomBatchLogger):
bucket_name=bucket_name,
object_name=object_name,
)
object_name = encode_gcs_object_name_for_url(object_name)
url = f"https://storage.googleapis.com/storage/v1/b/{bucket_name}/o/{object_name}"
@ -334,10 +340,11 @@ class GCSBucketBase(CustomBatchLogger):
bucket_name=bucket_name,
object_name=object_name,
)
encoded_object_name = encode_gcs_object_name_for_url(object_name)
response = await self.async_httpx_client.post(
headers=headers,
url=f"https://storage.googleapis.com/upload/storage/v1/b/{bucket_name}/o?uploadType=media&name={object_name}",
url=f"https://storage.googleapis.com/upload/storage/v1/b/{bucket_name}/o?uploadType=media&name={encoded_object_name}",
data=json_logged_payload,
)

72
litellm/integrations/generic_api/generic_api_callback.py
View file

@ -11,8 +11,9 @@ import json
import os
import re
import traceback
from typing import Dict, List, Literal, Optional, Union
from typing import Any, Dict, List, Literal, Optional, Union
import httpx
import litellm
from litellm._logging import verbose_logger
from litellm._uuid import uuid
@ -103,6 +104,9 @@ class GenericAPILogger(CustomBatchLogger):
event_types: Optional[List[API_EVENT_TYPES]] = None,
callback_name: Optional[str] = None,
log_format: Optional[LOG_FORMAT_TYPES] = None,
max_retries: int = 0,
retry_delay: float = 1.0,
timeout: Optional[Union[float, httpx.Timeout]] = None,
**kwargs,
):
"""
@ -114,6 +118,9 @@ class GenericAPILogger(CustomBatchLogger):
event_types: Optional[List[API_EVENT_TYPES]] = None,
callback_name: Optional[str] = None - If provided, loads config from generic_api_compatible_callbacks.json
log_format: Optional[LOG_FORMAT_TYPES] = None - Format for log output: "json_array" (default), "ndjson", or "single"
max_retries: Number of retry attempts after the initial request fails. Defaults to 0.
retry_delay: Initial retry delay in seconds. Retries use exponential backoff.
timeout: Optional timeout to use for Generic API callback requests.
"""
#########################################################
# Check if callback_name is provided and load config
@ -162,6 +169,10 @@ class GenericAPILogger(CustomBatchLogger):
self.endpoint: str = endpoint
self.event_types: Optional[List[API_EVENT_TYPES]] = event_types
self.callback_name: Optional[str] = callback_name
self.max_retries = max(0, int(max_retries or 0))
retry_delay_value = 0.0 if retry_delay is None else retry_delay
self.retry_delay = max(0.0, float(retry_delay_value))
self.timeout = timeout
# Validate and store log_format
if log_format is not None and log_format not in [
@ -226,6 +237,53 @@ class GenericAPILogger(CustomBatchLogger):
return headers_dict
def _should_retry_exception(self, exception: Exception) -> bool:
if isinstance(exception, (litellm.Timeout, httpx.TransportError)):
return True
if isinstance(exception, httpx.HTTPStatusError):
return exception.response.status_code >= 500
return False
async def _sleep_before_retry(self, attempt: int) -> None:
if self.retry_delay <= 0:
return
delay = self.retry_delay * (2**attempt)
await asyncio.sleep(delay)
async def _post_with_retries(self, data: str) -> httpx.Response:
post_kwargs: Dict[str, Any] = {
"url": self.endpoint,
"headers": self.headers,
"data": data,
}
if self.timeout is not None:
post_kwargs["timeout"] = self.timeout
total_attempts = self.max_retries + 1
for attempt in range(total_attempts):
try:
return await self.async_httpx_client.post(**post_kwargs)
except Exception as e:
is_last_attempt = attempt == self.max_retries
should_retry = self._should_retry_exception(e)
if is_last_attempt or not should_retry:
raise
verbose_logger.warning(
"Generic API Logger - retrying request to %s after error: %s "
"(attempt %s/%s)",
self.endpoint,
str(e),
attempt + 1,
total_attempts,
)
await self._sleep_before_retry(attempt)
raise RuntimeError("Generic API Logger retry loop exited unexpectedly")
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
"""
Async Log success events to Generic API Endpoint
@ -325,11 +383,7 @@ class GenericAPILogger(CustomBatchLogger):
# Send each log as individual HTTP request in parallel
tasks = []
for log_entry in self.log_queue:
task = self.async_httpx_client.post(
url=self.endpoint,
headers=self.headers,
data=safe_dumps(log_entry),
)
task = self._post_with_retries(data=safe_dumps(log_entry))
tasks.append(task)
# Execute all requests in parallel
@ -356,11 +410,7 @@ class GenericAPILogger(CustomBatchLogger):
raise ValueError(f"Unknown log_format: {self.log_format}")
# Make POST request
response = await self.async_httpx_client.post(
url=self.endpoint,
headers=self.headers,
data=data,
)
response = await self._post_with_retries(data=data)
verbose_logger.debug(
f"Generic API Logger - sent batch to {self.endpoint}, "

11
litellm/integrations/gitlab/gitlab_prompt_manager.py
View file

@ -4,7 +4,8 @@ GitLab prompt manager with configurable prompts folder.
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
from jinja2 import DictLoader, Environment, select_autoescape
from jinja2 import DictLoader, select_autoescape
from jinja2.sandbox import ImmutableSandboxedEnvironment
from litellm.integrations.custom_prompt_management import CustomPromptManagement
@ -90,7 +91,13 @@ class GitLabTemplateManager:
or ""
).strip("/")
self.jinja_env = Environment(
# Templates fetched from a GitLab repo are not trustworthy:
# anyone with repo write access can ship Jinja syntax that, in a
# plain `Environment()`, would reach `__class__.__init__.__globals__`
# and pivot into RCE on the proxy host. The sandbox blocks that
# attribute traversal while leaving normal `{{ var }}` substitution
# intact. Matches the dotprompt manager's hardening.
self.jinja_env = ImmutableSandboxedEnvironment(
loader=DictLoader({}),
autoescape=select_autoescape(["html", "xml"]),
variable_start_string="{{",

43
litellm/integrations/langfuse/langfuse.py
View file

@ -90,6 +90,29 @@ def _extract_cache_read_input_tokens(usage_obj) -> int:
return cache_read_input_tokens
def resolve_langfuse_credentials(
langfuse_public_key=None,
langfuse_secret=None,
langfuse_secret_key=None,
langfuse_host=None,
allow_env_credentials: bool = True,
):
if allow_env_credentials is False and langfuse_host is not None:
secret_key = langfuse_secret or langfuse_secret_key
public_key = langfuse_public_key
else:
secret_key = (
langfuse_secret or langfuse_secret_key or os.getenv("LANGFUSE_SECRET_KEY")
)
public_key = langfuse_public_key or os.getenv("LANGFUSE_PUBLIC_KEY")
resolved_host = langfuse_host or os.getenv(
"LANGFUSE_HOST", "https://cloud.langfuse.com"
)
return public_key, secret_key, resolved_host
class LangFuseLogger:
# Class variables or attributes
def __init__(
@ -98,6 +121,7 @@ class LangFuseLogger:
langfuse_secret=None,
langfuse_host=None,
flush_interval=1,
allow_env_credentials: bool = True,
):
try:
import langfuse
@ -106,11 +130,13 @@ class LangFuseLogger:
raise Exception(
f"\033[91mLangfuse not installed, try running 'pip install langfuse' to fix this error: {e}\n{traceback.format_exc()}\033[0m"
)
# Instance variables
self.secret_key = langfuse_secret or os.getenv("LANGFUSE_SECRET_KEY")
self.public_key = langfuse_public_key or os.getenv("LANGFUSE_PUBLIC_KEY")
self.langfuse_host = langfuse_host or os.getenv(
"LANGFUSE_HOST", "https://cloud.langfuse.com"
self.public_key, self.secret_key, self.langfuse_host = (
resolve_langfuse_credentials(
langfuse_public_key=langfuse_public_key,
langfuse_secret=langfuse_secret,
langfuse_host=langfuse_host,
allow_env_credentials=allow_env_credentials,
)
)
if not (
self.langfuse_host.startswith("http://")
@ -160,9 +186,10 @@ class LangFuseLogger:
project_id = None
if os.getenv("UPSTREAM_LANGFUSE_SECRET_KEY") is not None:
upstream_langfuse_debug_env = os.getenv("UPSTREAM_LANGFUSE_DEBUG")
upstream_langfuse_debug = (
str_to_bool(self.upstream_langfuse_debug)
if self.upstream_langfuse_debug is not None
str_to_bool(upstream_langfuse_debug_env)
if upstream_langfuse_debug_env is not None
else None
)
self.upstream_langfuse_secret_key = os.getenv(
@ -173,7 +200,7 @@ class LangFuseLogger:
)
self.upstream_langfuse_host = os.getenv("UPSTREAM_LANGFUSE_HOST")
self.upstream_langfuse_release = os.getenv("UPSTREAM_LANGFUSE_RELEASE")
self.upstream_langfuse_debug = os.getenv("UPSTREAM_LANGFUSE_DEBUG")
self.upstream_langfuse_debug = upstream_langfuse_debug_env
self.upstream_langfuse = Langfuse(
public_key=self.upstream_langfuse_public_key,
secret_key=self.upstream_langfuse_secret_key,

4
litellm/integrations/langfuse/langfuse_handler.py
View file

@ -115,8 +115,10 @@ class LangFuseHandler:
langfuse_logger = LangFuseLogger(
langfuse_public_key=credentials.get("langfuse_public_key"),
langfuse_secret=credentials.get("langfuse_secret"),
langfuse_secret=credentials.get("langfuse_secret")
or credentials.get("langfuse_secret_key"),
langfuse_host=credentials.get("langfuse_host"),
allow_env_credentials=credentials.get("langfuse_host") is None,
)
in_memory_dynamic_logger_cache.set_cache(
credentials=credentials,

19
litellm/integrations/langfuse/langfuse_prompt_management.py
View file

@ -20,7 +20,7 @@ from ...litellm_core_utils.specialty_caches.dynamic_logging_cache import (
DynamicLoggingCache,
)
from ..prompt_management_base import PromptManagementBase
from .langfuse import LangFuseLogger
from .langfuse import LangFuseLogger, resolve_langfuse_credentials
from .langfuse_handler import LangFuseHandler
if TYPE_CHECKING:
@ -46,6 +46,7 @@ def langfuse_client_init(
langfuse_secret_key=None,
langfuse_host=None,
flush_interval=1,
allow_env_credentials: bool = True,
) -> LangfuseClass:
"""
Initialize Langfuse client with caching to prevent multiple initializations.
@ -70,14 +71,12 @@ def langfuse_client_init(
f"\033[91mLangfuse not installed, try running 'pip install langfuse' to fix this error: {e}\n\033[0m"
)
# Instance variables
secret_key = (
langfuse_secret or langfuse_secret_key or os.getenv("LANGFUSE_SECRET_KEY")
)
public_key = langfuse_public_key or os.getenv("LANGFUSE_PUBLIC_KEY")
langfuse_host = langfuse_host or os.getenv(
"LANGFUSE_HOST", "https://cloud.langfuse.com"
public_key, secret_key, langfuse_host = resolve_langfuse_credentials(
langfuse_public_key=langfuse_public_key,
langfuse_secret=langfuse_secret,
langfuse_secret_key=langfuse_secret_key,
langfuse_host=langfuse_host,
allow_env_credentials=allow_env_credentials,
)
if not (
@ -222,6 +221,7 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge
langfuse_secret=dynamic_callback_params.get("langfuse_secret"),
langfuse_secret_key=dynamic_callback_params.get("langfuse_secret_key"),
langfuse_host=dynamic_callback_params.get("langfuse_host"),
allow_env_credentials=dynamic_callback_params.get("langfuse_host") is None,
)
langfuse_prompt_client = self._get_prompt_from_id(
langfuse_prompt_id=prompt_id,
@ -246,6 +246,7 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge
langfuse_secret=dynamic_callback_params.get("langfuse_secret"),
langfuse_secret_key=dynamic_callback_params.get("langfuse_secret_key"),
langfuse_host=dynamic_callback_params.get("langfuse_host"),
allow_env_credentials=dynamic_callback_params.get("langfuse_host") is None,
)
langfuse_prompt_client = self._get_prompt_from_id(
langfuse_prompt_id=prompt_id,

45
litellm/integrations/langsmith.py
View file

@ -19,6 +19,7 @@ from litellm.integrations.langsmith_mock_client import (
create_mock_langsmith_client,
should_use_langsmith_mock,
)
from litellm.litellm_core_utils.redact_messages import redact_user_api_key_info
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
@ -112,17 +113,28 @@ class LangsmithLogger(CustomBatchLogger):
langsmith_project: Optional[str] = None,
langsmith_base_url: Optional[str] = None,
langsmith_tenant_id: Optional[str] = None,
allow_env_credentials: bool = True,
) -> LangsmithCredentialsObject:
_credentials_api_key = langsmith_api_key or os.getenv("LANGSMITH_API_KEY")
_credentials_project = (
langsmith_project or os.getenv("LANGSMITH_PROJECT") or "litellm-completion"
)
_credentials_base_url = (
langsmith_base_url
or os.getenv("LANGSMITH_BASE_URL")
or "https://api.smith.langchain.com"
)
_credentials_tenant_id = langsmith_tenant_id or os.getenv("LANGSMITH_TENANT_ID")
if allow_env_credentials is False and langsmith_base_url is not None:
_credentials_api_key = langsmith_api_key
_credentials_project = langsmith_project or "litellm-completion"
_credentials_base_url = langsmith_base_url
_credentials_tenant_id = langsmith_tenant_id
else:
_credentials_api_key = langsmith_api_key or os.getenv("LANGSMITH_API_KEY")
_credentials_project = (
langsmith_project
or os.getenv("LANGSMITH_PROJECT")
or "litellm-completion"
)
_credentials_base_url = (
langsmith_base_url
or os.getenv("LANGSMITH_BASE_URL")
or "https://api.smith.langchain.com"
)
_credentials_tenant_id = langsmith_tenant_id or os.getenv(
"LANGSMITH_TENANT_ID"
)
return LangsmithCredentialsObject(
LANGSMITH_API_KEY=_credentials_api_key,
@ -153,6 +165,15 @@ class LangsmithLogger(CustomBatchLogger):
for key in ("session_id", "thread_id", "conversation_id"):
if key in requester_metadata and key not in extra_metadata:
extra_metadata[key] = requester_metadata[key]
# helper is shallow; also scrub nested requester_metadata since
# LangSmith forwards the whole dict into `extra`
extra_metadata = redact_user_api_key_info(metadata=extra_metadata)
nested = extra_metadata.get("requester_metadata")
if isinstance(nested, dict):
extra_metadata["requester_metadata"] = redact_user_api_key_info(
metadata=nested
)
return extra_metadata
def _build_outputs_with_usage(
@ -540,6 +561,10 @@ class LangsmithLogger(CustomBatchLogger):
langsmith_tenant_id=standard_callback_dynamic_params.get(
"langsmith_tenant_id", None
),
allow_env_credentials=standard_callback_dynamic_params.get(
"langsmith_base_url", None
)
is None,
)
else:
credentials = self.default_credentials

53
litellm/integrations/opentelemetry.py
View file

@ -69,6 +69,8 @@ class OpenTelemetryConfig:
deployment_environment: Optional[str] = None
model_id: Optional[str] = None
ignore_context_propagation: Optional[bool] = None
# When True, create a private TracerProvider instead of reusing or setting the global one.
skip_set_global: bool = False
def __post_init__(self) -> None:
# If endpoint is specified but exporter is still the default "console",
@ -259,16 +261,21 @@ class OpenTelemetry(CustomLogger):
try:
existing_provider = get_existing_provider_fn()
# If a real SDK provider exists (set by another SDK like Langfuse), use it
# This uses a positive check for SDK providers instead of a negative check for proxy providers
if isinstance(existing_provider, sdk_provider_class):
verbose_logger.debug(
"OpenTelemetry: Using existing %s: %s",
provider_name,
type(existing_provider).__name__,
)
provider = existing_provider
# Don't call set_provider to preserve existing context
if skip_set_global:
verbose_logger.debug(
"OpenTelemetry: existing %s found but skip_set_global=True; creating private %s for isolation",
provider_name,
provider_name,
)
provider = create_new_provider_fn()
else:
verbose_logger.debug(
"OpenTelemetry: Using existing %s: %s",
provider_name,
type(existing_provider).__name__,
)
provider = existing_provider
else:
# Default proxy provider or unknown type, create our own
verbose_logger.debug("OpenTelemetry: Creating new %s", provider_name)
@ -293,6 +300,12 @@ class OpenTelemetry(CustomLogger):
return provider
def _skip_set_global(self) -> bool:
# langfuse_otel relies on the Langfuse SDK's providers; don't overwrite them.
return self.config.skip_set_global or (
hasattr(self, "callback_name") and self.callback_name == "langfuse_otel"
)
def _init_tracing(self, tracer_provider):
from opentelemetry import trace
from opentelemetry.sdk.trace import TracerProvider
@ -303,11 +316,6 @@ class OpenTelemetry(CustomLogger):
provider.add_span_processor(self._get_span_processor())
return provider
# CRITICAL FIX: For Langfuse OTEL, skip setting global provider to prevent interference
skip_global = (
hasattr(self, "callback_name") and self.callback_name == "langfuse_otel"
)
tracer_provider = self._get_or_create_provider(
provider=tracer_provider,
provider_name="TracerProvider",
@ -315,16 +323,18 @@ class OpenTelemetry(CustomLogger):
sdk_provider_class=TracerProvider,
create_new_provider_fn=create_tracer_provider,
set_provider_fn=trace.set_tracer_provider,
skip_set_global=skip_global,
skip_set_global=self._skip_set_global(),
)
# Grab our tracer from the TracerProvider (not from global context)
# This ensures we use the provided TracerProvider (e.g., for testing)
self.tracer = tracer_provider.get_tracer(LITELLM_TRACER_NAME)
self._tracer_provider = tracer_provider
self.span_kind = SpanKind
def _init_metrics(self, meter_provider):
if not self.config.enable_metrics:
self._meter_provider = None
self._operation_duration_histogram = None
self._token_usage_histogram = None
self._cost_histogram = None
@ -350,7 +360,9 @@ class OpenTelemetry(CustomLogger):
sdk_provider_class=MeterProvider,
create_new_provider_fn=create_meter_provider,
set_provider_fn=metrics.set_meter_provider,
skip_set_global=self._skip_set_global(),
)
self._meter_provider = meter_provider
meter = meter_provider.get_meter(__name__)
@ -388,6 +400,7 @@ class OpenTelemetry(CustomLogger):
def _init_logs(self, logger_provider):
# nothing to do if events disabled
if not self.config.enable_events:
self._logger_provider = None
return
from opentelemetry._logs import get_logger_provider, set_logger_provider
@ -404,13 +417,14 @@ class OpenTelemetry(CustomLogger):
)
return provider
self._get_or_create_provider(
self._logger_provider = self._get_or_create_provider(
provider=logger_provider,
provider_name="LoggerProvider",
get_existing_provider_fn=get_logger_provider,
sdk_provider_class=OTLoggerProvider,
create_new_provider_fn=create_logger_provider,
set_provider_fn=set_logger_provider,
skip_set_global=self._skip_set_global(),
)
def log_success_event(self, kwargs, response_obj, start_time, end_time):
@ -1073,7 +1087,7 @@ class OpenTelemetry(CustomLogger):
# See: https://github.com/open-telemetry/opentelemetry-python/pull/4676
# TODO: Refactor to use the proper OTEL Logs API instead of directly creating SDK LogRecords
from opentelemetry._logs import SeverityNumber, get_logger
from opentelemetry._logs import SeverityNumber
try:
from opentelemetry.sdk._logs import ( # type: ignore[attr-defined] # OTEL < 1.39.0
@ -1084,7 +1098,10 @@ class OpenTelemetry(CustomLogger):
LogRecord as SdkLogRecord, # type: ignore[attr-defined] # OTEL >= 1.39.0
)
otel_logger = get_logger(LITELLM_LOGGER_NAME)
# Resolve through the handler's own LoggerProvider (which may be a
# private one when skip_set_global=True) rather than the module-level
# get_logger() which always goes through the global provider.
otel_logger = self._logger_provider.get_logger(LITELLM_LOGGER_NAME)
parent_ctx = span.get_span_context()
provider = (kwargs.get("litellm_params") or {}).get(

44
litellm/integrations/prometheus.py
View file

@ -265,6 +265,7 @@ class PrometheusLogger(CustomLogger):
########################################
# LiteLLM Virtual API KEY metrics
########################################
# Remaining MODEL RPM limit for API Key
self.litellm_remaining_api_key_requests_for_model = self._gauge_factory(
"litellm_remaining_api_key_requests_for_model",
@ -1928,7 +1929,7 @@ class PrometheusLogger(CustomLogger):
or _litellm_params_metadata.get("user_agent"),
}
def set_llm_deployment_failure_metrics(self, request_kwargs: dict):
def set_llm_deployment_failure_metrics(self, request_kwargs: dict): # noqa: PLR0915
"""
Sets Failure metrics when an LLM API call fails
@ -2006,17 +2007,32 @@ class PrometheusLogger(CustomLogger):
if code is not None:
exception_status = str(code)
# Create enum_values for the label factory (always create for use in different metrics)
# On LiteLLM-side rejects (no deployment picked), route request_kwargs["model"]
# into requested_model and leave deployment-scoped labels empty.
deployment_selected = bool(model_id)
if deployment_selected:
label_litellm_model_name = litellm_model_name
label_model_id = model_id
label_api_base = api_base
label_api_provider = llm_provider
label_requested_model = model_group or litellm_model_name
else:
label_litellm_model_name = ""
label_model_id = ""
label_api_base = ""
label_api_provider = ""
label_requested_model = litellm_model_name or model_group or ""
enum_values = UserAPIKeyLabelValues(
litellm_model_name=litellm_model_name,
model_id=model_id,
api_base=api_base,
api_provider=llm_provider,
litellm_model_name=label_litellm_model_name,
model_id=label_model_id,
api_base=label_api_base,
api_provider=label_api_provider,
exception_status=exception_status,
exception_class=(
self._get_exception_class_name(exception) if exception else None
),
requested_model=model_group or litellm_model_name,
requested_model=label_requested_model,
hashed_api_key=hashed_api_key,
api_key_alias=api_key_alias,
team=team,
@ -2030,12 +2046,14 @@ class PrometheusLogger(CustomLogger):
log these labels
["litellm_model_name", "model_id", "api_base", "api_provider"]
"""
self.set_deployment_partial_outage(
litellm_model_name=litellm_model_name or "",
model_id=model_id,
api_base=api_base,
api_provider=llm_provider or "",
)
# Only mark a deployment outage when one was actually picked.
if deployment_selected:
self.set_deployment_partial_outage(
litellm_model_name=litellm_model_name or "",
model_id=model_id,
api_base=api_base,
api_provider=llm_provider or "",
)
_deployment_label_ctx = PrometheusLabelFactoryContext(enum_values)
if exception is not None:
PrometheusLogger._inc_labeled_counter(

24
litellm/integrations/prometheus_helpers/prometheus_api.py
View file

@ -2,6 +2,7 @@
Helper functions to query prometheus API
"""
import json
import time
from datetime import datetime, timedelta
from typing import Optional
@ -81,6 +82,24 @@ def is_prometheus_connected() -> bool:
return False
def _quote_promql_string_literal(value: str) -> str:
"""Render ``value`` as a PromQL double-quoted string literal.
PromQL string literals follow Go's escape rules
(https://prometheus.io/docs/prometheus/latest/querying/basics/): a
backslash begins an escape sequence and a bare ``"`` ends the literal.
Without escaping, callers that accept arbitrary user-supplied values
(like the ``api_key`` filter on ``/global/spend/logs``) can inject extra
label matchers or selectors and read cross-tenant metrics.
JSON's quoting rules are a strict subset of Go's, so ``json.dumps`` of
a Python string produces a literal Prometheus accepts: ``\\``, ``\\"``,
and the standard ``\\n`` / ``\\t`` / ``\\uNNNN`` control-character
escapes. The returned value already includes the surrounding quotes.
"""
return json.dumps(value, ensure_ascii=False)
async def get_daily_spend_from_prometheus(api_key: Optional[str]):
"""
Expected Response Format:
@ -109,8 +128,11 @@ async def get_daily_spend_from_prometheus(api_key: Optional[str]):
if api_key is None:
query = "sum(delta(litellm_spend_metric_total[1d]))"
else:
quoted_api_key = _quote_promql_string_literal(api_key)
query = (
f'sum(delta(litellm_spend_metric_total{{hashed_api_key="{api_key}"}}[1d]))'
"sum(delta(litellm_spend_metric_total{"
f"hashed_api_key={quoted_api_key}"
"}[1d]))"
)
params = {

8
litellm/integrations/prompt_management_base.py
View file

@ -87,9 +87,7 @@ class PromptManagementBase(ABC):
try:
messages = compiled_prompt_client["prompt_template"] + client_messages
except Exception as e:
raise ValueError(
f"Error compiling prompt: {e}. Prompt id={prompt_id}, prompt_variables={prompt_variables}, client_messages={client_messages}, dynamic_callback_params={dynamic_callback_params}"
)
raise ValueError(f"Error compiling prompt: {e}. Prompt id={prompt_id}")
compiled_prompt_client["completed_messages"] = messages
return compiled_prompt_client
@ -116,9 +114,7 @@ class PromptManagementBase(ABC):
try:
messages = compiled_prompt_client["prompt_template"] + client_messages
except Exception as e:
raise ValueError(
f"Error compiling prompt: {e}. Prompt id={prompt_id}, prompt_variables={prompt_variables}, client_messages={client_messages}, dynamic_callback_params={dynamic_callback_params}"
)
raise ValueError(f"Error compiling prompt: {e}. Prompt id={prompt_id}")
compiled_prompt_client["completed_messages"] = messages
return compiled_prompt_client

22
litellm/litellm_core_utils/cli_token_utils.py
View file

@ -31,15 +31,23 @@ def load_cli_token() -> Optional[dict]:
return None
def get_litellm_gateway_api_key() -> Optional[str]:
def get_litellm_gateway_api_key(
expected_base_url: Optional[str] = None,
) -> Optional[str]:
"""
Get the stored CLI API key for use with LiteLLM SDK.
This function reads the token file created by `litellm-proxy login`
and returns the API key for use in Python scripts.
Args:
expected_base_url: When provided, the key is only returned if it was
originally issued for this URL. Pass the target server URL to
prevent credential leakage when the client is pointed at a
different (possibly malicious) server.
Returns:
str: The API key if found, None otherwise
str: The API key if found (and origin matches), None otherwise
Example:
>>> import litellm
@ -53,6 +61,10 @@ def get_litellm_gateway_api_key() -> Optional[str]:
>>> )
"""
token_data = load_cli_token()
if token_data and "key" in token_data:
return token_data["key"]
return None
if not token_data or "key" not in token_data:
return None
if expected_base_url is not None:
stored_url = token_data.get("base_url")
if stored_url != expected_base_url.rstrip("/"):
return None
return token_data["key"]

175
litellm/litellm_core_utils/cloud_storage_security.py Normal file
View file

@ -0,0 +1,175 @@
import posixpath
import re
from types import MappingProxyType
from typing import Any, Mapping, Optional, Sequence, Tuple, cast
from urllib.parse import quote, unquote
from litellm._uuid import uuid
VERTEX_AI_MANAGED_GCS_PREFIX = "litellm-vertex-files/"
BEDROCK_MANAGED_S3_BATCH_PREFIX = "litellm-bedrock-files-"
BEDROCK_MANAGED_S3_UPLOAD_PREFIX = "litellm-bedrock-files/"
BEDROCK_MANAGED_S3_OUTPUT_PREFIX = "litellm-batch-outputs/"
BEDROCK_MANAGED_S3_PREFIXES = (
BEDROCK_MANAGED_S3_BATCH_PREFIX,
BEDROCK_MANAGED_S3_UPLOAD_PREFIX,
BEDROCK_MANAGED_S3_OUTPUT_PREFIX,
)
_MAPPING_PROXY_TYPE: type = type(MappingProxyType({}))
_SAFE_OBJECT_COMPONENT_PATTERN = re.compile(r"[^A-Za-z0-9._-]+")
def sanitize_cloud_object_component(
value: Optional[str], fallback: str = "file"
) -> str:
if not isinstance(value, str):
return fallback
component = posixpath.basename(value.replace("\\", "/")).strip()
if component in {"", ".", ".."}:
return fallback
component = "".join(
"_" if ord(char) < 32 or ord(char) == 127 else char for char in component
)
component = _SAFE_OBJECT_COMPONENT_PATTERN.sub("_", component)
component = component.strip("._")
if not component:
return fallback
return component[:255]
def sanitize_cloud_object_path(value: Optional[str], fallback: str = "file") -> str:
if not isinstance(value, str):
return fallback
segments = []
for segment in value.replace("\\", "/").split("/"):
sanitized_segment = sanitize_cloud_object_component(segment, fallback="")
if sanitized_segment:
segments.append(sanitized_segment)
if not segments:
return fallback
return "/".join(segments)
def build_managed_cloud_object_name(
prefix: str, filename: Optional[str], fallback_filename: str = "file"
) -> str:
safe_filename = sanitize_cloud_object_component(
filename, fallback=fallback_filename
)
return f"{prefix}{uuid.uuid4().hex}-{safe_filename}"
def _validate_cloud_object_path(object_name: str) -> None:
if not object_name:
raise ValueError("Cloud storage object name is required")
if object_name.startswith("/"):
raise ValueError("Cloud storage object name must be relative")
if any(ord(char) < 32 or ord(char) == 127 for char in object_name):
raise ValueError("Cloud storage object name contains control characters")
segments = object_name.split("/")
if any(segment in {".", ".."} for segment in segments):
raise ValueError("Cloud storage object name contains an invalid path segment")
if "" in segments[:-1]:
raise ValueError("Cloud storage object name contains an invalid path segment")
def split_configured_cloud_bucket_name(bucket_name: str) -> Tuple[str, str]:
if not isinstance(bucket_name, str) or not bucket_name.strip():
raise ValueError("Cloud storage bucket name is required")
bucket_name = bucket_name.strip()
if "://" in bucket_name or "?" in bucket_name or "#" in bucket_name:
raise ValueError(
"Cloud storage bucket name must not include a URI scheme or query"
)
if any(ord(char) < 32 or ord(char) == 127 for char in bucket_name):
raise ValueError("Cloud storage bucket name contains control characters")
bucket, _, prefix = bucket_name.partition("/")
if not bucket:
raise ValueError("Cloud storage bucket name is required")
if "\\" in bucket:
raise ValueError("Cloud storage bucket name contains an invalid separator")
prefix = prefix.strip("/")
if prefix:
_validate_cloud_object_path(prefix)
return bucket, prefix
def encode_gcs_object_name_for_url(object_name: str) -> str:
return quote(unquote(object_name), safe="")
def encode_s3_object_key_for_url(object_key: str) -> str:
return quote(unquote(object_key), safe="/")
def should_allow_legacy_cloud_file_ids(
litellm_params: Optional[Mapping[str, Any]] = None,
) -> bool:
value = None
if isinstance(litellm_params, Mapping):
trusted_model_credentials = litellm_params.get(
"_litellm_internal_model_credentials"
)
if isinstance(trusted_model_credentials, _MAPPING_PROXY_TYPE):
value = cast(Mapping[str, Any], trusted_model_credentials).get(
"allow_legacy_cloud_file_ids"
)
if isinstance(value, bool):
return value
if isinstance(value, str):
return value.strip().lower() in {"1", "true", "yes", "on"}
return False
def validate_managed_cloud_file_id(
file_id: str,
scheme: str,
configured_bucket_name: str,
allowed_object_prefixes: Sequence[str],
allow_legacy_cloud_file_ids: bool = False,
) -> Tuple[str, str]:
decoded_file_id = unquote(file_id)
if not decoded_file_id.startswith(scheme):
raise ValueError(f"file_id must be a {scheme} URI")
full_path = decoded_file_id[len(scheme) :]
if "/" not in full_path:
raise ValueError("file_id must include a cloud storage object name")
bucket_name, object_name = full_path.split("/", 1)
configured_bucket, configured_prefix = split_configured_cloud_bucket_name(
configured_bucket_name
)
if bucket_name != configured_bucket:
raise ValueError("file_id bucket does not match the configured storage bucket")
_validate_cloud_object_path(object_name)
allowed_prefixes = tuple(allowed_object_prefixes)
if configured_prefix:
allowed_prefixes = tuple(
f"{configured_prefix.rstrip('/')}/{prefix}" for prefix in allowed_prefixes
)
if object_name.startswith(allowed_prefixes):
return bucket_name, object_name
if allow_legacy_cloud_file_ids:
if configured_prefix and not object_name.startswith(
f"{configured_prefix.rstrip('/')}/"
):
raise ValueError(
"file_id object does not match the configured storage prefix"
)
return bucket_name, object_name
raise ValueError("file_id must reference a LiteLLM-managed storage object")

21
litellm/litellm_core_utils/exception_mapping_utils.py
View file

@ -6,7 +6,8 @@ from typing import Any, Optional
import httpx
import litellm
from litellm._logging import _redact_string, verbose_logger
from litellm._logging import _ENABLE_SECRET_REDACTION, _redact_string, verbose_logger
from litellm.litellm_core_utils.secret_redaction import redact_string
from litellm.types.utils import LlmProviders
from ..exceptions import (
@ -261,10 +262,18 @@ def exception_type( # type: ignore # noqa: PLR0915
original_exception=original_exception
)
try:
error_str = str(original_exception)
error_str = (
redact_string(str(original_exception))
if _ENABLE_SECRET_REDACTION
else str(original_exception)
)
if model:
if hasattr(original_exception, "message"):
error_str = str(original_exception.message)
error_str = (
redact_string(str(original_exception.message))
if _ENABLE_SECRET_REDACTION
else str(original_exception.message)
)
if isinstance(original_exception, BaseException):
exception_type = type(original_exception).__name__
else:
@ -2431,7 +2440,8 @@ def exception_type( # type: ignore # noqa: PLR0915
else:
raise APIConnectionError(
message="{}\n{}".format(
str(original_exception), _redact_string(traceback.format_exc())
str(original_exception),
_redact_string(traceback.format_exc()),
),
llm_provider=custom_llm_provider,
model=model,
@ -2461,7 +2471,8 @@ def exception_type( # type: ignore # noqa: PLR0915
raise e # it's already mapped
raised_exc = APIConnectionError(
message="{}\n{}".format(
original_exception, _redact_string(traceback.format_exc())
original_exception,
_redact_string(traceback.format_exc()),
),
llm_provider="",
model="",

41
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -1,4 +1,5 @@
from typing import Optional, Tuple
from urllib.parse import urlparse
import litellm
from litellm.constants import REPLICATE_MODEL_NAME_WITH_ID_LENGTH
@ -8,6 +9,43 @@ from litellm.secret_managers.main import get_secret, get_secret_str
from ..types.router import LiteLLM_Params
def _endpoint_matches_api_base(endpoint: str, api_base: str) -> bool:
"""
Match a registered openai-compatible endpoint against a caller-supplied
``api_base`` using parsed-URL semantics, not unanchored substring search.
Both inputs may be a bare hostname (``api.perplexity.ai``), host+path
(``api.deepinfra.com/v1/openai``), or a full URL
(``https://api.cerebras.ai/v1``). Hostnames must match exactly
(case-insensitive); if the registered endpoint has a non-trivial path,
the api_base path must start with it on a segment boundary.
The naive ``endpoint in api_base`` shape lets a caller pass
``https://attacker.com/api.groq.com/openai/v1`` to coerce the proxy
into reading the server's GROQ_API_KEY from the environment and
forwarding it to the attacker's host as a Bearer credential.
"""
def _parse(value: str):
# Ensure urlparse sees a scheme so it populates hostname / path.
normalized = value if "://" in value else f"https://{value}"
return urlparse(normalized)
parsed_endpoint = _parse(endpoint)
parsed_url = _parse(api_base)
endpoint_host = (parsed_endpoint.hostname or "").lower()
url_host = (parsed_url.hostname or "").lower()
if not endpoint_host or endpoint_host != url_host:
return False
endpoint_path = parsed_endpoint.path.rstrip("/")
if not endpoint_path:
return True
url_path = parsed_url.path.rstrip("/")
return url_path == endpoint_path or url_path.startswith(endpoint_path + "/")
def _is_non_openai_azure_model(model: str) -> bool:
try:
model_name = model.split("/", 1)[1]
@ -210,7 +248,7 @@ def get_llm_provider( # noqa: PLR0915
# check if api base is a known openai compatible endpoint
if api_base:
for endpoint in litellm.openai_compatible_endpoints:
if endpoint in api_base:
if _endpoint_matches_api_base(endpoint, api_base):
if endpoint == "api.perplexity.ai":
custom_llm_provider = "perplexity"
dynamic_api_key = get_secret_str("PERPLEXITYAI_API_KEY")
@ -348,6 +386,7 @@ def get_llm_provider( # noqa: PLR0915
or "ft:gpt-3.5-turbo" in model
or "ft:gpt-4" in model # catches ft:gpt-4-0613, ft:gpt-4o
or model in litellm.openai_image_generation_models
or model.startswith("gpt-image")
or model in litellm.openai_video_generation_models
):
custom_llm_provider = "openai"

30
litellm/litellm_core_utils/initialize_dynamic_callback_params.py
View file

@ -23,6 +23,13 @@ def _raise_env_reference_error(param: str, *, source: str) -> None:
)
def validate_no_callback_env_reference(
param: str, value: object, *, source: str
) -> None:
if _is_env_reference(value):
_raise_env_reference_error(param, source=source)
# Hardcoded list of supported callback params to avoid runtime inspection issues with TypedDict
_supported_callback_params = [
"langfuse_public_key",
@ -30,8 +37,6 @@ _supported_callback_params = [
"langfuse_secret_key",
"langfuse_host",
"langfuse_prompt_version",
"gcs_bucket_name",
"gcs_path_service_account",
"langsmith_api_key",
"langsmith_project",
"langsmith_base_url",
@ -50,6 +55,11 @@ _supported_callback_params = [
"lunary_public_key",
]
_request_blocked_callback_params = {
"gcs_bucket_name",
"gcs_path_service_account",
}
def initialize_standard_callback_dynamic_params(
kwargs: Optional[Dict] = None,
@ -57,17 +67,20 @@ def initialize_standard_callback_dynamic_params(
"""
Initialize the standard callback dynamic params from the kwargs
checks if langfuse_secret_key, gcs_bucket_name in kwargs and sets the corresponding attributes in StandardCallbackDynamicParams
checks supported request callback params in kwargs and sets the corresponding attributes in StandardCallbackDynamicParams
"""
standard_callback_dynamic_params = StandardCallbackDynamicParams()
if kwargs:
# 1. Check top-level kwargs
for param in _supported_callback_params:
if param in _request_blocked_callback_params:
continue
if param in kwargs:
_param_value = kwargs.get(param)
if _is_env_reference(_param_value):
_raise_env_reference_error(param, source="request body")
validate_no_callback_env_reference(
param, _param_value, source="request body"
)
standard_callback_dynamic_params[param] = _param_value # type: ignore
# 2. Fallback: check "metadata" or "litellm_params" -> "metadata"
@ -78,10 +91,13 @@ def initialize_standard_callback_dynamic_params(
if isinstance(metadata, dict):
for param in _supported_callback_params:
if param in _request_blocked_callback_params:
continue
if param not in standard_callback_dynamic_params and param in metadata:
_param_value = metadata.get(param)
if _is_env_reference(_param_value):
_raise_env_reference_error(param, source="metadata")
validate_no_callback_env_reference(
param, _param_value, source="metadata"
)
standard_callback_dynamic_params[param] = _param_value # type: ignore
return standard_callback_dynamic_params

62
litellm/litellm_core_utils/litellm_logging.py
View file

@ -1467,6 +1467,8 @@ class Logging(LiteLLMLoggingBaseClass):
LiteLLMRealtimeStreamLoggingObject,
OpenAIModerationResponse,
"SearchResponse",
dict,
list,
],
cache_hit: Optional[bool] = None,
litellm_model_name: Optional[str] = None,
@ -1725,12 +1727,18 @@ class Logging(LiteLLMLoggingBaseClass):
return
if self.model_call_details.get("litellm_params") is None:
return
self.model_call_details["litellm_params"].setdefault("metadata", {})
if self.model_call_details["litellm_params"]["metadata"] is None:
self.model_call_details["litellm_params"]["metadata"] = {}
self.model_call_details["litellm_params"]["metadata"]["hidden_params"] = (
getattr(logging_result, "_hidden_params", {})
)
metadata_hidden_params = hidden_params.copy()
response_cost = self.model_call_details.get("response_cost")
if (
metadata_hidden_params.get("response_cost") is None
and response_cost is not None
):
metadata_hidden_params["response_cost"] = response_cost
litellm_params = self.model_call_details["litellm_params"]
metadata = litellm_params.get("metadata") or {}
litellm_params["metadata"] = metadata
metadata["hidden_params"] = metadata_hidden_params
def _process_hidden_params_and_response_cost(
self,
@ -1738,6 +1746,7 @@ class Logging(LiteLLMLoggingBaseClass):
start_time,
end_time,
):
"""Resolve hidden params, compute response cost, and emit the standard logging payload."""
hidden_params = getattr(logging_result, "_hidden_params", {})
if hidden_params:
if self.model_call_details.get("litellm_params") is not None:
@ -1871,24 +1880,12 @@ class Logging(LiteLLMLoggingBaseClass):
):
if self._is_recognized_call_type_for_logging(
logging_result=logging_result
):
) or isinstance(logging_result, (dict, list)):
self._process_hidden_params_and_response_cost(
logging_result=logging_result,
start_time=start_time,
end_time=end_time,
)
elif isinstance(result, dict) or isinstance(result, list):
self.model_call_details["standard_logging_object"] = (
self._build_standard_logging_payload(
result, start_time, end_time
)
)
if (
standard_logging_payload := self.model_call_details.get(
"standard_logging_object"
)
) is not None:
emit_standard_logging_payload(standard_logging_payload)
elif standard_logging_object is not None:
self.model_call_details["standard_logging_object"] = (
standard_logging_object
@ -3245,10 +3242,15 @@ class Logging(LiteLLMLoggingBaseClass):
),
langfuse_secret=self.standard_callback_dynamic_params.get(
"langfuse_secret"
),
)
or self.standard_callback_dynamic_params.get("langfuse_secret_key"),
langfuse_host=self.standard_callback_dynamic_params.get(
"langfuse_host"
),
allow_env_credentials=self.standard_callback_dynamic_params.get(
"langfuse_host"
)
is None,
)
return langFuseLogger
@ -4723,7 +4725,7 @@ class StandardLoggingPayloadSetup:
):
for key, value in litellm_params["metadata"].items():
# Skip non-serializable objects like UserAPIKeyAuth
if key == "user_api_key_auth":
if key in {"user_api_key_auth", "user_api_key_budget_reservation"}:
continue
merged_metadata[key] = value
@ -5438,11 +5440,6 @@ def get_standard_logging_object_payload(
completion_start_time_float=completion_start_time_float,
stream=kwargs.get("stream", False),
)
# clean up litellm hidden params
clean_hidden_params = StandardLoggingPayloadSetup.get_hidden_params(
hidden_params
)
# clean up litellm metadata
clean_metadata = StandardLoggingPayloadSetup.get_standard_logging_metadata(
metadata=metadata,
@ -5476,6 +5473,18 @@ def get_standard_logging_object_payload(
## Get model cost information ##
base_model = _get_base_model_from_metadata(model_call_details=kwargs)
custom_pricing = use_custom_pricing_for_model(litellm_params=litellm_params)
raw_response_cost = kwargs.get("response_cost")
response_cost: float = raw_response_cost or 0.0
# clean up litellm hidden params
clean_hidden_params = StandardLoggingPayloadSetup.get_hidden_params(
hidden_params
)
if (
clean_hidden_params["response_cost"] is None
and raw_response_cost is not None
):
clean_hidden_params["response_cost"] = response_cost
model_cost_information = StandardLoggingPayloadSetup.get_model_cost_information(
base_model=base_model,
@ -5484,7 +5493,6 @@ def get_standard_logging_object_payload(
init_response_obj=init_response_obj,
api_base=litellm_params.get("api_base"),
)
response_cost: float = kwargs.get("response_cost", 0) or 0.0
error_information = StandardLoggingPayloadSetup.get_error_information(
original_exception=original_exception,

8
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -982,9 +982,9 @@ class CostCalculatorUtils:
image_response=completion_response,
)
elif custom_llm_provider == litellm.LlmProviders.OPENAI.value:
# Check if this is a gpt-image model (token-based pricing)
# gpt-image models use token-based pricing.
model_lower = model.lower()
if "gpt-image-1" in model_lower:
if "gpt-image" in model_lower:
from litellm.llms.openai.image_generation.cost_calculator import (
cost_calculator as openai_gpt_image_cost_calculator,
)
@ -1004,9 +1004,9 @@ class CostCalculatorUtils:
optional_params=optional_params,
)
elif custom_llm_provider == litellm.LlmProviders.AZURE.value:
# Check if this is a gpt-image model (token-based pricing)
# gpt-image models use token-based pricing.
model_lower = model.lower()
if "gpt-image-1" in model_lower:
if "gpt-image" in model_lower:
from litellm.llms.openai.image_generation.cost_calculator import (
cost_calculator as openai_gpt_image_cost_calculator,
)

6
litellm/litellm_core_utils/llm_request_utils.py
View file

@ -77,8 +77,8 @@ def get_proxy_server_request_headers(litellm_params: Optional[dict]) -> dict:
if litellm_params is None:
return {}
proxy_request_headers = (
litellm_params.get("proxy_server_request", {}).get("headers", {}) or {}
)
proxy_request_headers = (litellm_params.get("proxy_server_request") or {}).get(
"headers"
) or {}
return proxy_request_headers

2
litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py
View file

@ -824,8 +824,6 @@ def convert_to_model_response_object( # noqa: PLR0915
stream=stream,
start_time=start_time,
end_time=end_time,
hidden_params=hidden_params,
_response_headers=_response_headers,
convert_tool_call_to_json_mode=convert_tool_call_to_json_mode,
)
raise Exception(

13
litellm/litellm_core_utils/logging_callback_manager.py
View file

@ -221,6 +221,13 @@ class LoggingCallbackManager:
headers = callback_config.get("headers")
event_types = callback_config.get("event_types")
log_format = callback_config.get("log_format")
max_retries = max(0, int(callback_config.get("max_retries", 0) or 0))
retry_delay_value = callback_config.get("retry_delay")
retry_delay = max(
0.0,
float(0.0 if retry_delay_value is None else retry_delay_value),
)
timeout = callback_config.get("timeout")
if endpoint is None or headers is None:
verbose_logger.warning(
@ -236,6 +243,9 @@ class LoggingCallbackManager:
and cached_logger.headers == headers
and cached_logger.event_types == event_types
and cached_logger.log_format == log_format
and cached_logger.max_retries == max_retries
and cached_logger.retry_delay == retry_delay
and cached_logger.timeout == timeout
):
return cached_logger
@ -244,6 +254,9 @@ class LoggingCallbackManager:
headers=headers,
event_types=event_types,
log_format=log_format,
max_retries=max_retries,
retry_delay=retry_delay,
timeout=timeout,
)
_generic_api_logger_cache[callback] = new_logger
return new_logger

184
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -1661,6 +1661,20 @@ def _sanitize_anthropic_tool_use_id(tool_use_id: str) -> str:
return sanitized
_ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES = {"application/pdf", "text/plain"}
def _is_anthropic_document_data_uri(url: str) -> bool:
# Anthropic's base64 document source accepts only application/pdf and
# text/plain (see select_anthropic_content_block_type_for_file). Routing
# other mimes here would produce a document block the API rejects, so we
# leave them on the image code path.
match = re.match(r"data:([^;,]+)", url)
if not match:
return False
return match.group(1) in _ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES
def convert_to_anthropic_tool_result(
message: Union[ChatCompletionToolMessage, ChatCompletionFunctionMessage],
force_base64: bool = False,
@ -1698,14 +1712,24 @@ def convert_to_anthropic_tool_result(
"""
anthropic_content: Union[
str,
List[Union[AnthropicMessagesToolResultContent, AnthropicMessagesImageParam]],
List[
Union[
AnthropicMessagesToolResultContent,
AnthropicMessagesImageParam,
AnthropicMessagesDocumentParam,
]
],
] = ""
if isinstance(message["content"], str):
anthropic_content = message["content"]
elif isinstance(message["content"], List):
content_list = message["content"]
anthropic_content_list: List[
Union[AnthropicMessagesToolResultContent, AnthropicMessagesImageParam]
Union[
AnthropicMessagesToolResultContent,
AnthropicMessagesImageParam,
AnthropicMessagesDocumentParam,
]
] = []
for content in content_list:
if content["type"] == "text":
@ -1720,21 +1744,62 @@ def convert_to_anthropic_tool_result(
text_content["cache_control"] = cache_control_value
anthropic_content_list.append(text_content)
elif content["type"] == "image_url":
image_url_value = content["image_url"]
format = (
content["image_url"].get("format")
if isinstance(content["image_url"], dict)
image_url_value.get("format")
if isinstance(image_url_value, dict)
else None
)
_anthropic_image_param = create_anthropic_image_param(
content["image_url"], format=format, is_bedrock_invoke=force_base64
url_str = (
image_url_value.get("url")
if isinstance(image_url_value, dict)
else image_url_value
)
_anthropic_image_param = add_cache_control_to_content(
anthropic_content_element=_anthropic_image_param,
# Data URIs with non-image mime types (e.g. application/pdf) must
# translate to Anthropic document blocks, not image blocks —
# wrapping a PDF in `type: "image"` is rejected by the API.
if isinstance(url_str, str) and _is_anthropic_document_data_uri(
url_str
):
synth_file_message: ChatCompletionFileObject = {
"type": "file",
"file": {"file_data": url_str},
}
_document_block = anthropic_process_openai_file_message(
synth_file_message
)
_document_block = add_cache_control_to_content(
anthropic_content_element=cast(
AnthropicMessagesDocumentParam, _document_block
),
original_content_element=content,
)
anthropic_content_list.append(
cast(AnthropicMessagesDocumentParam, _document_block)
)
else:
_anthropic_image_param = create_anthropic_image_param(
image_url_value,
format=format,
is_bedrock_invoke=force_base64,
)
_anthropic_image_param = add_cache_control_to_content(
anthropic_content_element=_anthropic_image_param,
original_content_element=content,
)
anthropic_content_list.append(
cast(AnthropicMessagesImageParam, _anthropic_image_param)
)
elif content["type"] == "file":
file_content = cast(ChatCompletionFileObject, content)
_file_block = anthropic_process_openai_file_message(file_content)
_file_block = add_cache_control_to_content(
anthropic_content_element=cast(
AnthropicMessagesDocumentParam, _file_block
),
original_content_element=content,
)
anthropic_content_list.append(
cast(AnthropicMessagesImageParam, _anthropic_image_param)
)
anthropic_content_list.append(_file_block)
anthropic_content = anthropic_content_list
anthropic_tool_result: Optional[AnthropicMessagesToolResultParam] = None
@ -3977,6 +4042,55 @@ def _convert_to_bedrock_tool_call_result(
tool_result_content_blocks.append(
BedrockToolResultContentBlock(image=_block["image"])
)
elif "document" in _block:
tool_result_content_blocks.append(
BedrockToolResultContentBlock(document=_block["document"])
)
else:
verbose_logger.warning(
"Bedrock Converse: unrecognized BedrockContentBlock keys "
"%s for image_url tool-result block %s; dropping.",
list(_block.keys()),
content,
)
elif content["type"] == "file":
# Match the user-message path (_process_file_message): accept
# either file_data (base64 data URI) or file_id (server-side
# reference / URL) and hand off to BedrockImageProcessor. Raise
# BadRequestError on both-None rather than silently dropping.
file_obj = content.get("file") or {}
file_data = file_obj.get("file_data")
file_id = file_obj.get("file_id")
if file_data is None and file_id is None:
raise litellm.BadRequestError(
message="file_data and file_id cannot both be None. Got={}".format(
content
),
model="",
llm_provider="bedrock",
)
file_format = file_obj.get("format")
_file_block: BedrockContentBlock = (
BedrockImageProcessor.process_image_sync(
image_url=cast(str, file_id or file_data),
format=file_format,
)
)
if "document" in _file_block:
tool_result_content_blocks.append(
BedrockToolResultContentBlock(document=_file_block["document"])
)
elif "image" in _file_block:
tool_result_content_blocks.append(
BedrockToolResultContentBlock(image=_file_block["image"])
)
else:
verbose_logger.warning(
"Bedrock Converse: unrecognized BedrockContentBlock keys "
"%s for file tool-result block %s; dropping.",
list(_file_block.keys()),
content,
)
message.get("name", "")
id = str(message.get("tool_call_id", str(uuid.uuid4())))
@ -4468,6 +4582,11 @@ class BedrockConverseMessagesProcessor:
message=cast(ChatCompletionFileObject, element)
)
_parts.append(_part)
elif element["type"] == "document":
_part = BedrockConverseMessagesProcessor._process_document_message(
element
)
_parts.append(_part)
_cache_point_block = (
litellm.AmazonConverseConfig()._get_cache_point_block(
message_block=cast(
@ -4750,6 +4869,44 @@ class BedrockConverseMessagesProcessor:
image_url=cast(str, file_id or file_data), format=format
)
@staticmethod
def _process_document_message(element: dict) -> BedrockContentBlock:
"""Convert a document content block to a Bedrock DocumentBlock.
Handles the Anthropic-style document format:
{"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": "..."}}
"""
source = element["source"]
source_type = source.get("type")
if source_type != "base64":
raise ValueError(
f"Bedrock Converse only supports base64-encoded document sources, got '{source_type}'. "
"Please convert the document to base64 before sending to Bedrock."
)
media_type: str = source["media_type"]
data: str = source["data"]
doc_format = BedrockImageProcessor._validate_format(
mime_type=media_type, image_format=media_type.split("/")[1]
)
# Deterministic name using the same hashing pattern as _create_bedrock_block
HASH_SAMPLE_BYTES = 64 * 1024
normalized = "".join(data.split()).encode("utf-8")
sample = normalized[:HASH_SAMPLE_BYTES]
hasher = hashlib.sha256()
hasher.update(sample)
hasher.update(str(len(normalized)).encode("utf-8"))
content_hash = hasher.hexdigest()[:16]
document_name = f"Document_{content_hash}_{doc_format}"
return BedrockContentBlock(
document=BedrockDocumentBlock(
source=BedrockSourceBlock(bytes=data),
format=doc_format,
name=document_name,
)
)
@staticmethod
def add_thinking_blocks_to_assistant_content(
thinking_blocks: List[BedrockContentBlock],
@ -4847,6 +5004,11 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915
)
)
_parts.append(_part)
elif element["type"] == "document":
_part = BedrockConverseMessagesProcessor._process_document_message(
element
)
_parts.append(_part)
_cache_point_block = (
litellm.AmazonConverseConfig()._get_cache_point_block(
message_block=cast(

102
litellm/litellm_core_utils/redact_messages.py
View file

@ -60,6 +60,9 @@ def _redact_choice_content(choice):
def _redact_responses_api_output(output_items):
"""Helper to redact ResponsesAPIResponse output items."""
for output_item in output_items:
if hasattr(output_item, "text"):
output_item.text = "redacted-by-litellm"
if hasattr(output_item, "content") and isinstance(output_item.content, list):
for content_part in output_item.content:
if hasattr(content_part, "text"):
@ -75,6 +78,28 @@ def _redact_responses_api_output(output_items):
summary_item.text = "redacted-by-litellm"
def _redact_responses_api_output_dict(output_items, redacted_str: str):
"""Helper to redact ResponsesAPIResponse output items in dict form."""
for output_item in output_items:
if not isinstance(output_item, dict):
continue
if "text" in output_item:
output_item["text"] = redacted_str
if isinstance(output_item.get("content"), list):
for content_item in output_item["content"]:
if isinstance(content_item, dict) and "text" in content_item:
content_item["text"] = redacted_str
if output_item.get("type") == "reasoning" and isinstance(
output_item.get("summary"), list
):
for summary_item in output_item["summary"]:
if isinstance(summary_item, dict) and "text" in summary_item:
summary_item["text"] = redacted_str
def _redact_standard_logging_object(model_call_details: dict):
"""Redact messages and response inside standard_logging_object if present."""
standard_logging_object = model_call_details.get("standard_logging_object")
@ -93,28 +118,11 @@ def _redact_standard_logging_object(model_call_details: dict):
if isinstance(response, dict) and "output" in response:
# ResponsesAPIResponse format - redact content in output items
if isinstance(response.get("output"), list):
for output_item in response["output"]:
if isinstance(output_item, dict) and "content" in output_item:
if isinstance(output_item["content"], list):
for content_item in output_item["content"]:
if (
isinstance(content_item, dict)
and "text" in content_item
):
content_item["text"] = redacted_str
_redact_responses_api_output_dict(response["output"], redacted_str)
elif isinstance(response, dict) and "choices" in response:
# ModelResponse dict format - redact content in choices
if isinstance(response.get("choices"), list):
for choice in response["choices"]:
if isinstance(choice, dict):
if "message" in choice and isinstance(choice["message"], dict):
choice["message"]["content"] = redacted_str
if "audio" in choice["message"]:
choice["message"]["audio"] = None
elif "delta" in choice and isinstance(choice["delta"], dict):
choice["delta"]["content"] = redacted_str
if "audio" in choice["delta"]:
choice["delta"]["audio"] = None
_redact_model_response_dict_choices(response["choices"], redacted_str)
elif isinstance(response, str):
standard_logging_object["response"] = redacted_str
else:
@ -122,6 +130,29 @@ def _redact_standard_logging_object(model_call_details: dict):
standard_logging_object["response"] = {"text": redacted_str}
def _redact_model_response_dict_choices(choices, redacted_str: str):
for choice in choices:
if isinstance(choice, dict):
if "message" in choice and isinstance(choice["message"], dict):
choice["message"]["content"] = redacted_str
if "reasoning_content" in choice["message"]:
choice["message"]["reasoning_content"] = redacted_str
if "thinking_blocks" in choice["message"]:
choice["message"]["thinking_blocks"] = None
if "audio" in choice["message"]:
choice["message"]["audio"] = None
elif "delta" in choice and isinstance(choice["delta"], dict):
choice["delta"]["content"] = redacted_str
if "reasoning_content" in choice["delta"]:
choice["delta"]["reasoning_content"] = redacted_str
if "thinking_blocks" in choice["delta"]:
choice["delta"]["thinking_blocks"] = None
if "audio" in choice["delta"]:
choice["delta"]["audio"] = None
else:
_redact_choice_content(choice)
def perform_redaction(model_call_details: dict, result):
"""
Performs the actual redaction on the logging object and result.
@ -132,6 +163,7 @@ def perform_redaction(model_call_details: dict, result):
]
model_call_details["prompt"] = ""
model_call_details["input"] = ""
_redact_standard_logging_object(model_call_details)
# Redact streaming response
if (
@ -171,30 +203,14 @@ def perform_redaction(model_call_details: dict, result):
elif isinstance(_result, dict) and "choices" in _result:
# Handle dict representation of ModelResponse (e.g., from model_dump())
if _result.get("choices") is not None:
for choice in _result["choices"]:
if isinstance(choice, dict):
if "message" in choice and isinstance(choice["message"], dict):
choice["message"]["content"] = "redacted-by-litellm"
if "reasoning_content" in choice["message"]:
choice["message"][
"reasoning_content"
] = "redacted-by-litellm"
if "thinking_blocks" in choice["message"]:
choice["message"]["thinking_blocks"] = None
if "audio" in choice["message"]:
choice["message"]["audio"] = None
elif "delta" in choice and isinstance(choice["delta"], dict):
choice["delta"]["content"] = "redacted-by-litellm"
if "reasoning_content" in choice["delta"]:
choice["delta"][
"reasoning_content"
] = "redacted-by-litellm"
if "thinking_blocks" in choice["delta"]:
choice["delta"]["thinking_blocks"] = None
if "audio" in choice["delta"]:
choice["delta"]["audio"] = None
else:
_redact_choice_content(choice)
_redact_model_response_dict_choices(
_result["choices"], "redacted-by-litellm"
)
elif isinstance(_result, dict) and "output" in _result:
if isinstance(_result.get("output"), list):
_redact_responses_api_output_dict(
_result["output"], "redacted-by-litellm"
)
elif isinstance(_result, litellm.ResponsesAPIResponse):
if hasattr(_result, "output"):
_redact_responses_api_output(_result.output)

81
litellm/litellm_core_utils/secret_redaction.py Normal file
View file

@ -0,0 +1,81 @@
"""
Credential/secret redaction utilities.
This module owns the compiled regex and the public `redact_string` helper so
that any part of the codebase (logging, exception mapping, etc.) can scrub
secrets from strings without depending on the logging-configuration module.
"""
import re
from typing import List
_REDACTED = "REDACTED"
def _build_secret_patterns() -> "re.Pattern[str]":
patterns: List[str] = [
# PEM private key / certificate blocks
r"-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----",
# GCP OAuth2 access tokens (ya29.*)
r"\bya29\.[A-Za-z0-9_.~+/-]+",
# Credential %s formatting (space separator, no key= prefix)
r"(?:client_secret|azure_password|azure_username)\s+[^\s,'\"})\]{}>]+",
# AWS access key IDs
r"(?:AKIA|ASIA)[0-9A-Z]{16}",
# AWS secrets / session tokens / access key IDs (key=value)
r"(?:aws_secret_access_key|aws_session_token|aws_access_key_id)"
r"\s*[:=]\s*[A-Za-z0-9/+=]{20,}",
# Bearer tokens (OAuth, JWT, etc.)
r"Bearer\s+[A-Za-z0-9\-._~+/]{10,}=*",
# Basic auth headers
r"Basic\s+[A-Za-z0-9+/]{10,}={0,2}",
# OpenAI / Anthropic sk- prefixed keys
r"sk-[A-Za-z0-9\-_]{20,}",
# Generic api_key / api-key / apikey (handles 'key': 'value' dict repr)
r"(?:api[_-]?key)['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]{8,}",
# x-api-key / api-key header values (handles 'key': 'value' dict repr)
r"(?:x-api-key|api-key)['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+",
# Anthropic internal header keys
r"x-ak-[A-Za-z0-9\-_]{20,}",
# Google API keys (bare key value)
r"AIza[0-9A-Za-z\-_]{35}",
# URL query-param key=VALUE (e.g. ?key=AIza... or &key=...) — catches the
# full "key=<secret>" fragment so the value is redacted regardless of format.
r"(?<=[?&])key=[^\s&'\"]{8,}",
# Password / secret params (handles key=value and 'key': 'value')
# Word boundary prevents O(n^2) backtracking on long word-char runs.
r"(?:^|(?<=\W))\w*(?:password|passwd|client_secret|secret_key|_secret)"
r"['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+",
# Database connection string credentials (scheme://user:pass@host)
r"(?<=://)[^\s'\"]*:[^\s'\"@]+(?=@)",
# Databricks personal access tokens
r"dapi[0-9a-f]{32}",
# ── Key-name-based redaction ──
# Catches secrets inside dicts/config dumps by matching on the KEY name
# regardless of what the value looks like.
# e.g. 'master_key': 'any-value-here', "database_url": "postgres://..."
# private_key with PEM-aware value capture
r"""private_key['\"]?\s*[:=]\s*['\"]?(?:-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----|[^\s,'\"})\]{}>]+)""",
r"(?:master_key|database_url|db_url|connection_string|"
r"signing_key|encryption_key|"
r"auth_token|access_token|refresh_token|"
r"slack_webhook_url|webhook_url|"
r"database_connection_string|"
r"huggingface_token|jwt_secret)"
r"""['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+""",
# Raw JWTs (without Bearer prefix)
r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*",
# Azure SAS tokens in URLs
r"[?&]sig=[A-Za-z0-9%+/=]+",
# Full JSON service-account blobs (single-line and multi-line)
r'\{[^{}]*"type"\s*:\s*"service_account"[^{}]*(?:\{[^{}]*\}[^{}]*)*\}',
]
return re.compile("|".join(patterns), re.IGNORECASE)
_SECRET_RE = _build_secret_patterns()
def redact_string(value: str) -> str:
"""Scrub known secret/credential patterns from *value* and return the result."""
return _SECRET_RE.sub(_REDACTED, value)

2
litellm/litellm_core_utils/sensitive_data_masker.py
View file

@ -21,6 +21,8 @@ class SensitiveDataMasker:
"auth",
"authorization",
"credential",
# Plural form: Vertex uses ``vertex_credentials``; segment-exact
# matching otherwise misses it because "credential" != "credentials".
"credentials",
"access",
"private",

2
litellm/litellm_core_utils/streaming_handler.py
View file

@ -2244,7 +2244,7 @@ class CustomStreamWrapper:
asyncio.create_task(
self.logging_obj.async_failure_handler(e, traceback_exception)
)
raise e
self._handle_stream_fallback_error(e)
except Exception as e:
traceback_exception = traceback.format_exc()
if self.logging_obj is not None:

168
litellm/litellm_core_utils/url_utils.py
View file

@ -21,8 +21,8 @@ Admins can opt out via two ``litellm`` globals (wired from proxy config):
import socket
from ipaddress import ip_address, ip_network
from typing import Any, List, Set, Tuple
from urllib.parse import urlparse, urlunparse
from typing import Any, List, Optional, Set, Tuple
from urllib.parse import quote, urlparse, urlunparse
import httpx
@ -46,6 +46,46 @@ class SSRFError(ValueError):
pass
def encode_url_path_segment(value: Any, *, field_name: str = "path parameter") -> str:
"""Percent-encode one user-controlled URL path segment.
``urllib.parse.quote(..., safe="")`` intentionally leaves RFC 3986
unreserved characters such as ``.`` unescaped, so reject standalone dot
segments before they can be appended to an upstream URL and normalized by
the HTTP client.
"""
if value is None:
raise ValueError(f"{field_name} is required")
value_str = str(value)
if value_str == "":
raise ValueError(f"{field_name} is required")
if value_str in {".", ".."}:
raise ValueError(f"{field_name} cannot be a dot path segment")
return quote(value_str, safe="")
def encode_url_path_segments(value: Any, *, field_name: str = "path") -> str:
"""Percent-encode a user-controlled URL path made of multiple segments.
Empty segments are rejected, so leading, trailing, or consecutive slashes
fail closed instead of being normalized by the HTTP client.
"""
if value is None:
raise ValueError(f"{field_name} is required")
value_str = str(value)
if value_str == "":
raise ValueError(f"{field_name} is required")
encoded_segments = []
for segment in value_str.split("/"):
encoded_segments.append(encode_url_path_segment(segment, field_name=field_name))
return "/".join(encoded_segments)
def _is_blocked_ip(addr: str) -> bool:
"""Return True for any IP not safe to reach from a user-supplied URL.
@ -70,6 +110,85 @@ def _normalize_host(host: str) -> str:
return host.lower().rstrip(".")
def _default_port_for_scheme(scheme: str) -> int:
return 443 if scheme == "https" else 80
def _parse_url_destination_allowlist_entry(
entry: str,
) -> Optional[Tuple[str, Optional[str], Optional[int]]]:
"""Parse an admin allowlist entry into host, optional scheme, optional port.
Entries may be bare hosts (``api.example.com``), host+port
(``api.example.com:8443``), or origins (``https://api.example.com``).
URL paths are intentionally ignored so admins can paste an api_base value.
"""
entry = entry.strip()
if not entry:
return None
has_scheme = "://" in entry
parsed = urlparse(entry if has_scheme else f"//{entry}")
if has_scheme and parsed.scheme not in _ALLOWED_SCHEMES:
return None
if parsed.username is not None or parsed.password is not None:
return None
if not parsed.hostname:
return None
try:
port = parsed.port
except ValueError:
return None
scheme: Optional[str] = parsed.scheme if has_scheme else None
if scheme is not None and port is None:
port = _default_port_for_scheme(scheme)
return _normalize_host(parsed.hostname), scheme, port
def is_url_destination_allowed_by_host(url: str, allowed_hosts: List[str]) -> bool:
"""Return True when a credential-bearing provider URL is admin-allowlisted.
This does not fetch, resolve, or rewrite URLs. It only answers whether the
destination origin is explicitly trusted by configuration. Use ``safe_get``
for user-controlled content fetches that require SSRF protection.
"""
parsed = urlparse(url)
if parsed.scheme not in _ALLOWED_SCHEMES:
return False
if parsed.username is not None or parsed.password is not None:
return False
if not parsed.hostname:
return False
try:
effective_port = parsed.port or _default_port_for_scheme(parsed.scheme)
except ValueError:
return False
normalized_host = _normalize_host(parsed.hostname)
configured_entries = (
[allowed_hosts] if isinstance(allowed_hosts, str) else allowed_hosts
)
for entry in configured_entries or []:
if not isinstance(entry, str):
continue
parsed_entry = _parse_url_destination_allowlist_entry(entry)
if parsed_entry is None:
continue
allowed_host, allowed_scheme, allowed_port = parsed_entry
if allowed_host != normalized_host:
continue
if allowed_scheme is not None and allowed_scheme != parsed.scheme:
continue
if allowed_port is not None and allowed_port != effective_port:
continue
return True
return False
def _format_host_header(hostname: str, port: int, default_port: int) -> str:
"""Build an RFC 7230 Host header value, bracketing IPv6 literals."""
bracketed = f"[{hostname}]" if ":" in hostname else hostname
@ -145,7 +264,7 @@ def validate_url(url: str) -> Tuple[str, str]:
raise SSRFError("URL has no hostname")
port = parsed.port
default_port = 443 if parsed.scheme == "https" else 80
default_port = _default_port_for_scheme(parsed.scheme)
effective_port = port if port is not None else default_port
host_header = _format_host_header(hostname, effective_port, default_port)
@ -199,13 +318,54 @@ def validate_url(url: str) -> Tuple[str, str]:
return rewritten, host_header
def assert_same_origin(candidate_url: str, expected_url: str) -> None:
"""Verify ``candidate_url`` shares scheme, host, and port with ``expected_url``.
Use when an upstream API returns a URL meant for follow-up requests
(e.g. an async-job polling URL that will be hit with the operator's
API key in the headers). The upstream is trusted because the operator
configured ``api_base``, but the URL it hands back must actually point
back at the same origin or we'd be blindly forwarding credentials
wherever the upstream told us to.
Hostnames are compared case-insensitively. Default ports are made
explicit (HTTP→80, HTTPS→443) so ``https://api.example.com:443/...``
and ``https://api.example.com/...`` are treated as the same origin.
Error messages identify *which* component mismatched but never echo
the operator's ``expected`` host or the candidate's hostname back to
the caller — in the SSRF threat model the caller is the attacker,
and reflecting host info would be a secondary leak of operator
infrastructure details.
"""
candidate = urlparse(candidate_url)
expected = urlparse(expected_url)
if candidate.scheme not in _ALLOWED_SCHEMES:
raise SSRFError("URL scheme is not allowed")
if candidate.scheme != expected.scheme:
raise SSRFError("Origin mismatch on scheme")
candidate_host = _normalize_host(candidate.hostname or "")
expected_host = _normalize_host(expected.hostname or "")
if not candidate_host or candidate_host != expected_host:
raise SSRFError("Origin mismatch on host")
default_port = 443 if candidate.scheme == "https" else 80
candidate_port = candidate.port if candidate.port is not None else default_port
expected_port = expected.port if expected.port is not None else default_port
if candidate_port != expected_port:
raise SSRFError("Origin mismatch on port")
_MAX_REDIRECTS = 10
def _extract_redirect_url(response: Any, request_url: str) -> str:
"""Extract and resolve the redirect target from a response's Location header."""
location = response.headers.get("location")
if not location:
if not isinstance(location, str) or not location:
raise SSRFError("Redirect response has no Location header")
# Resolve relative URLs against the request URL
return str(httpx.URL(request_url).join(location))

4
litellm/llms/anthropic/batches/transformation.py
View file

@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union, cas
import httpx
from httpx import Headers, Response
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.llms.openai import AllMessageValues, CreateBatchRequest
@ -122,7 +123,8 @@ class AnthropicBatchesConfig(BaseBatchesConfig):
Complete URL for Anthropic batch retrieval: {api_base}/v1/messages/batches/{batch_id}
"""
api_base = api_base or self.anthropic_model_info.get_api_base(api_base)
return f"{api_base.rstrip('/')}/v1/messages/batches/{batch_id}"
encoded_batch_id = encode_url_path_segment(batch_id, field_name="batch_id")
return f"{api_base.rstrip('/')}/v1/messages/batches/{encoded_batch_id}"
def transform_retrieve_batch_request(
self,

301
litellm/llms/anthropic/chat/transformation.py
View file

@ -1,18 +1,31 @@
import json
import re
import time
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast
from typing import (
TYPE_CHECKING,
Any,
Dict,
List,
NoReturn,
Optional,
Tuple,
Union,
cast,
)
import httpx
import litellm
from litellm.constants import (
ANTHROPIC_MIN_THINKING_BUDGET_TOKENS,
ANTHROPIC_WEB_SEARCH_TOOL_MAX_USES,
DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS,
DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET,
DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET,
DEFAULT_REASONING_EFFORT_MAX_THINKING_BUDGET,
DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET,
DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET,
DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET,
RESPONSE_FORMAT_TOOL_NAME,
)
from litellm.litellm_core_utils.core_helpers import map_finish_reason
@ -92,6 +105,22 @@ else:
LoggingClass = Any
REASONING_EFFORT_TO_OUTPUT_CONFIG_EFFORT: Dict[str, str] = {
"low": "low",
"minimal": "low",
"medium": "medium",
"high": "high",
"xhigh": "xhigh",
"max": "max",
}
DROP_UNSUPPORTED_OUTPUT_CONFIG_WARNING = (
"Dropping unsupported `output_config` for model=%s "
"(drop_params=True). Effort is only supported on Opus 4.5+, "
"Sonnet 4.6+, and Mythos Preview."
)
class AnthropicConfig(AnthropicModelInfo, BaseConfig):
"""
Reference: https://docs.anthropic.com/claude/reference/messages_post
@ -202,17 +231,96 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
def _supports_effort_level(model: str, level: str) -> bool:
"""Check ``supports_{level}_reasoning_effort`` in the model map.
Mirrors the pattern used in ``openai/chat/gpt_5_transformation.py`` so
that adding support for a new effort level is a pure model-map change.
Strips bedrock/vertex prefixes so a provider-routed Claude still
resolves to the Anthropic model-map entry.
"""
key = f"supports_{level}_reasoning_effort"
try:
return _supports_factory(
if _supports_factory(
model=model,
custom_llm_provider="anthropic",
key=f"supports_{level}_reasoning_effort",
)
key=key,
):
return True
except Exception:
return False
pass
candidates = [model]
for prefix in (
"bedrock/converse/",
"bedrock/invoke/",
"bedrock/",
"vertex_ai/",
):
if model.startswith(prefix):
candidates.append(model[len(prefix) :])
try:
from litellm.llms.bedrock.common_utils import BedrockModelInfo
base = BedrockModelInfo.get_base_model(model)
if base:
candidates.append(base)
candidates.append(f"bedrock/{base}")
except Exception:
pass
try:
import litellm
for cand in candidates:
if cand in litellm.model_cost and (
litellm.model_cost[cand].get(key) is True
):
return True
except Exception:
pass
return False
@staticmethod
def _validate_effort_for_model(model: str, effort: Optional[str]) -> Optional[str]:
"""Return ``None`` if ``effort`` is allowed on ``model``, else an error message."""
if effort == "max" and not (
AnthropicConfig._is_claude_4_6_model(model)
or AnthropicConfig._is_claude_4_7_model(model)
or AnthropicConfig._supports_effort_level(model, "max")
):
return f"effort='max' is not supported by this model. Got model: {model}"
if effort == "xhigh" and not AnthropicConfig._supports_effort_level(
model, "xhigh"
):
return f"effort='xhigh' is not supported by this model. Got model: {model}"
return None
@staticmethod
def _model_supports_effort_param(model: str) -> bool:
"""Whether the model accepts ``output_config.effort`` at all."""
return any(
AnthropicConfig._supports_effort_level(model, level)
for level in ("low", "minimal", "medium", "high", "xhigh", "max")
)
@staticmethod
def _raise_invalid_reasoning_effort(
model: str, value: Any, llm_provider: str
) -> NoReturn:
"""Raise a ``BadRequestError`` for an unrecognised ``reasoning_effort``.
Args:
model: The model id the request was routed to (surfaced in the error).
value: The offending ``reasoning_effort`` value supplied by the caller.
llm_provider: Provider tag for the raised exception (``"anthropic"``,
``"bedrock_converse"``, ``"databricks"``, ...).
Raises:
litellm.exceptions.BadRequestError: Always.
"""
raise litellm.exceptions.BadRequestError(
message=(
f"Invalid reasoning_effort: {value!r}. "
f"Must be one of: 'minimal', 'low', 'medium', "
f"'high', 'xhigh', 'max', 'none'"
),
model=model,
llm_provider=llm_provider,
)
def get_supported_openai_params(self, model: str):
params = [
@ -794,12 +902,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
def _map_reasoning_effort(
reasoning_effort: Optional[Union[REASONING_EFFORT, str]],
model: str,
llm_provider: str = "anthropic",
) -> Optional[AnthropicThinkingParam]:
if reasoning_effort is None or reasoning_effort == "none":
return None
if AnthropicConfig._is_claude_4_6_model(
model
) or AnthropicConfig._is_claude_4_7_model(model):
if AnthropicConfig._is_adaptive_thinking_model(model):
return AnthropicThinkingParam(
type="adaptive",
)
@ -818,13 +925,34 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
type="enabled",
budget_tokens=DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET,
)
elif reasoning_effort == "xhigh":
return AnthropicThinkingParam(
type="enabled",
budget_tokens=DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET,
)
elif reasoning_effort == "max":
return AnthropicThinkingParam(
type="enabled",
budget_tokens=DEFAULT_REASONING_EFFORT_MAX_THINKING_BUDGET,
)
elif reasoning_effort == "minimal":
return AnthropicThinkingParam(
type="enabled",
budget_tokens=DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET,
budget_tokens=max(
DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET,
ANTHROPIC_MIN_THINKING_BUDGET_TOKENS,
),
)
else:
raise ValueError(f"Unmapped reasoning effort: {reasoning_effort}")
raise litellm.exceptions.BadRequestError(
message=(
f"Unmapped reasoning effort: {reasoning_effort!r}. "
f"Must be one of: 'minimal', 'low', 'medium', 'high', "
f"'xhigh', 'max', 'none'."
),
model=model,
llm_provider=llm_provider,
)
def _extract_json_schema_from_response_format(
self, value: Optional[dict]
@ -1088,24 +1216,27 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
elif param == "thinking":
optional_params["thinking"] = value
elif param == "reasoning_effort" and isinstance(value, str):
optional_params["thinking"] = AnthropicConfig._map_reasoning_effort(
reasoning_effort=value, model=model
mapped_thinking = AnthropicConfig._map_reasoning_effort(
reasoning_effort=value,
model=model,
llm_provider=self.custom_llm_provider or "anthropic",
)
# For Claude 4.6+ models, effort is controlled via output_config,
# not thinking budget_tokens. Map reasoning_effort to output_config.
if AnthropicConfig._is_claude_4_6_model(
model
) or AnthropicConfig._is_claude_4_7_model(model):
effort_map = {
"low": "low",
"minimal": "low",
"medium": "medium",
"high": "high",
"xhigh": "xhigh",
"max": "max",
}
mapped_effort = effort_map.get(value, value)
optional_params["output_config"] = {"effort": mapped_effort}
if mapped_thinking is None:
optional_params.pop("thinking", None)
optional_params.pop("output_config", None)
else:
optional_params["thinking"] = mapped_thinking
if AnthropicConfig._is_adaptive_thinking_model(model):
mapped_effort = REASONING_EFFORT_TO_OUTPUT_CONFIG_EFFORT.get(
value
)
if mapped_effort is None:
AnthropicConfig._raise_invalid_reasoning_effort(
model=model,
value=value,
llm_provider=self.custom_llm_provider or "anthropic",
)
optional_params["output_config"] = {"effort": mapped_effort}
elif param == "web_search_options" and isinstance(value, dict):
hosted_web_search_tool = self.map_web_search_tool(
cast(OpenAIWebSearchOptions, value)
@ -1527,51 +1658,71 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
output_config = optional_params.get("output_config")
if not output_config or not isinstance(output_config, dict):
return
if litellm.drop_params is True and not self._model_supports_effort_param(model):
litellm.verbose_logger.warning(
DROP_UNSUPPORTED_OUTPUT_CONFIG_WARNING,
model,
)
optional_params.pop("output_config", None)
data.pop("output_config", None)
return
effort = output_config.get("effort")
valid_efforts = ["high", "medium", "low", "xhigh", "max"]
if effort and effort not in valid_efforts:
raise ValueError(
f"Invalid effort value: {effort}. Must be one of: "
f"'high', 'medium', 'low', 'xhigh', 'max'"
if effort is not None and effort not in valid_efforts:
raise litellm.exceptions.BadRequestError(
message=(
f"Invalid effort value: {effort!r}. Must be one of: "
f"'high', 'medium', 'low', 'xhigh', 'max'"
),
model=model,
llm_provider=self.custom_llm_provider or "anthropic",
)
# ``max`` is for Opus 4.6+ output effort (not Sonnet 4.6, not Opus 4.5).
# Accept known Opus 4.6/4.7 id patterns and/or ``supports_max_reasoning_effort``
# in the model map (same pattern as ``xhigh`` below).
if effort == "max" and not (
self._is_opus_4_6_model(model)
or self._is_opus_4_7_model(model)
or self._supports_effort_level(model, "max")
):
raise ValueError(
f"effort='max' is not supported by this model. Got model: {model}"
)
# ``xhigh`` is data-driven via ``supports_xhigh_reasoning_effort`` so
# enabling it for a new model is a pure model-map change.
if effort == "xhigh" and not self._supports_effort_level(model, "xhigh"):
raise ValueError(
f"effort='xhigh' is not supported by this model. Got model: {model}"
gate_error = self._validate_effort_for_model(model, effort)
if gate_error is not None:
raise litellm.exceptions.BadRequestError(
message=gate_error,
model=model,
llm_provider=self.custom_llm_provider or "anthropic",
)
data["output_config"] = output_config
def _transform_response_for_json_mode(
def _resolve_json_mode_non_streaming(
self,
json_mode: Optional[bool],
tool_calls: List[ChatCompletionToolCallChunk],
) -> Optional[LitellmMessage]:
_message: Optional[LitellmMessage] = None
if json_mode is True and len(tool_calls) == 1:
# check if tool name is the default tool name
json_mode_content_str: Optional[str] = None
if (
"name" in tool_calls[0]["function"]
and tool_calls[0]["function"]["name"] == RESPONSE_FORMAT_TOOL_NAME
):
json_mode_content_str = tool_calls[0]["function"].get("arguments")
if json_mode_content_str is not None:
_message = AnthropicConfig._convert_tool_response_to_message(
tool_calls=tool_calls,
)
return _message
) -> Tuple[
Optional[LitellmMessage],
List[ChatCompletionToolCallChunk],
Optional[str],
]:
"""Strip internal response_format tool calls; merge payload into content when mixed with user tools."""
if json_mode is not True or not tool_calls:
return None, tool_calls, None
json_indices = [
i
for i, t in enumerate(tool_calls)
if t.get("function", {}).get("name") == RESPONSE_FORMAT_TOOL_NAME
]
if not json_indices:
return None, tool_calls, None
if len(json_indices) == len(tool_calls):
json_tool = tool_calls[json_indices[0]]
if json_tool.get("function", {}).get("arguments") is None:
return None, tool_calls, None
_message = AnthropicConfig._convert_tool_response_to_message(
tool_calls=[json_tool]
)
return _message, [], None
first_json = tool_calls[json_indices[0]]
json_msg = AnthropicConfig._convert_tool_response_to_message([first_json])
extra_content: Optional[str] = (
json_msg.content if json_msg is not None else None
)
filtered_tools = [t for i, t in enumerate(tool_calls) if i not in json_indices]
return None, filtered_tools, extra_content
def extract_response_content(self, completion_response: dict) -> Tuple[
str,
@ -1931,19 +2082,27 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
tool_calls,
)
json_mode_message, tool_calls_for_message, json_extra_content = (
self._resolve_json_mode_non_streaming(
json_mode=json_mode,
tool_calls=tool_calls,
)
)
merged_text = text_content or ""
if json_extra_content:
merged_text = (
merged_text + json_extra_content if merged_text else json_extra_content
)
_message = litellm.Message(
tool_calls=tool_calls,
content=text_content or None,
tool_calls=tool_calls_for_message,
content=merged_text or None,
provider_specific_fields=provider_specific_fields,
thinking_blocks=thinking_blocks,
reasoning_content=reasoning_content,
)
_message.provider_specific_fields = provider_specific_fields
json_mode_message = self._transform_response_for_json_mode(
json_mode=json_mode,
tool_calls=tool_calls,
)
if json_mode_message is not None:
completion_response["stop_reason"] = "stop"
_message = json_mode_message

13
litellm/llms/anthropic/common_utils.py
View file

@ -273,7 +273,18 @@ class AnthropicModelInfo(BaseLLMModelInfo):
@staticmethod
def _is_adaptive_thinking_model(model: str) -> bool:
"""Claude 4.6+ models use adaptive thinking with output_config effort."""
"""Claude 4.6+ models use adaptive thinking with ``output_config.effort``."""
from litellm.utils import _supports_factory
try:
if _supports_factory(
model=model,
custom_llm_provider=None,
key="supports_adaptive_thinking",
):
return True
except Exception:
pass
return AnthropicModelInfo._is_claude_4_6_model(
model
) or AnthropicModelInfo._is_claude_4_7_model(model)

37
litellm/llms/anthropic/experimental_pass_through/adapters/handler.py
View file

@ -27,6 +27,16 @@ from litellm.utils import get_model_info
if TYPE_CHECKING:
pass
# Anthropic-only fields that the translator above already maps into the
# OpenAI-format completion_kwargs (output_config → reasoning_effort /
# response_format, etc.). They must be filtered out of the raw
# extra_kwargs re-merge below or non-Anthropic backends reject the call
# with 400 "Extra inputs are not permitted". Add new entries here when
# extending AnthropicMessagesRequestOptionalParams with another Anthropic-
# specific key.
ANTHROPIC_ONLY_REQUEST_KEYS: frozenset[str] = frozenset({"output_config"})
########################################################
# init adapter
ANTHROPIC_ADAPTER = AnthropicAdapter()
@ -202,8 +212,12 @@ class LiteLLMMessagesToCompletionTransformationHandler:
request_data["output_format"] = output_format
# Extract output_config from extra_kwargs so the translator can use it
# (e.g. output_config.effort for adaptive thinking → reasoning_effort)
extra_kwargs = extra_kwargs or {}
# (e.g. output_config.effort for adaptive thinking → reasoning_effort,
# output_config.format → response_format for structured outputs).
# Use explicit None check rather than `or {}` so an explicit empty dict
# caller-passed argument is preserved (matters for tests that drive
# the fallback inference path).
extra_kwargs = extra_kwargs if extra_kwargs is not None else {}
if "output_config" in extra_kwargs:
request_data["output_config"] = extra_kwargs["output_config"]
@ -225,8 +239,23 @@ class LiteLLMMessagesToCompletionTransformationHandler:
"include_usage": True,
}
excluded_keys = {"anthropic_messages"}
extra_kwargs = extra_kwargs or {}
# Keys that must NOT be forwarded as raw extras into the OpenAI-format
# ``completion_kwargs`` after translation. The translator above has
# already consumed the meaningful parts of these inputs (e.g.
# ``output_config.format`` → ``response_format``, ``output_config.effort``
# → ``reasoning_effort`` for non-Claude targets). Re-adding the raw
# Anthropic-shaped key here causes 400 "Extra inputs are not permitted"
# on non-Anthropic backends (Azure OpenAI, Fireworks, Bedrock Nova,
# etc.) and is silently lossy on Anthropic-family targets, which would
# see the translated key ``response_format`` AND a duplicate, conflicting
# ``output_config``.
#
# Maintainability: when adding a new Anthropic-only request param to
# ``AnthropicMessagesRequestOptionalParams``, also extend
# ``ANTHROPIC_ONLY_REQUEST_KEYS`` here so it doesn't silently leak.
excluded_keys = ANTHROPIC_ONLY_REQUEST_KEYS | {"anthropic_messages"}
# NOTE: extra_kwargs was already coerced from None to {} at the top of
# this method (line ~220). It is guaranteed to be a dict here.
for key, value in extra_kwargs.items():
if (
key == "litellm_logging_obj"

23
litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py
View file

@ -667,7 +667,7 @@ class LiteLLMAnthropicMessagesAdapter:
@staticmethod
def translate_anthropic_thinking_to_reasoning_effort(
thinking: Dict[str, Any]
thinking: Dict[str, Any],
) -> Optional[str]:
"""
Translate Anthropic's thinking parameter to OpenAI's reasoning_effort.
@ -1084,10 +1084,23 @@ class LiteLLMAnthropicMessagesAdapter:
anthropic_message_request: AnthropicMessagesRequest,
new_kwargs: ChatCompletionRequest,
) -> None:
"""Translate output_format to response_format when applicable."""
if "output_format" not in anthropic_message_request:
return
output_format = anthropic_message_request["output_format"]
"""Translate Anthropic structured-output config to OpenAI ``response_format``.
Accepts either the legacy top-level ``output_format`` field OR the
newer ``output_config.format`` (sub-key on ``output_config``) so that
both shapes flow through to non-Anthropic backends as
``response_format``. Without the ``output_config.format`` branch,
callers using the new Anthropic Structured Outputs API would have
their schema silently dropped on the adapter path — only the legacy
top-level ``output_format`` was being mapped.
``output_format`` takes precedence when both are provided.
"""
output_format: Any = anthropic_message_request.get("output_format")
if not output_format:
output_config = anthropic_message_request.get("output_config")
if isinstance(output_config, dict):
output_format = output_config.get("format")
if not output_format:
return
response_format = self.translate_anthropic_output_format_to_openai(

62
litellm/llms/anthropic/experimental_pass_through/messages/transformation.py
View file

@ -47,6 +47,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
"inference_geo",
"speed",
"output_config",
"reasoning_effort",
# TODO: Add Anthropic `metadata` support
# "metadata",
]
@ -166,6 +167,62 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
return headers, api_base
@staticmethod
def _translate_reasoning_effort_to_anthropic(
model: str, optional_params: Dict
) -> None:
"""Map OpenAI-style ``reasoning_effort`` to native Anthropic params.
Caller-supplied ``thinking`` / ``output_config`` win over the alias.
``effort='none'`` clears both. Invalid efforts raise a 400.
"""
from litellm.exceptions import BadRequestError as _BadRequestError
from litellm.llms.anthropic.chat.transformation import (
REASONING_EFFORT_TO_OUTPUT_CONFIG_EFFORT,
AnthropicConfig,
)
reasoning_effort = optional_params.pop("reasoning_effort", None)
if not isinstance(reasoning_effort, str):
return
try:
mapped_thinking = AnthropicConfig._map_reasoning_effort(
reasoning_effort=reasoning_effort, model=model
)
except _BadRequestError as e:
raise AnthropicError(message=str(e.message), status_code=400)
if mapped_thinking is None:
optional_params.pop("thinking", None)
optional_params.pop("output_config", None)
return
optional_params.setdefault("thinking", mapped_thinking)
if AnthropicModelInfo._is_adaptive_thinking_model(model):
mapped_effort = REASONING_EFFORT_TO_OUTPUT_CONFIG_EFFORT.get(
reasoning_effort
)
if mapped_effort is None:
raise AnthropicError(
message=(
f"Invalid reasoning_effort: {reasoning_effort!r}. "
f"Must be one of: 'minimal', 'low', 'medium', 'high', "
f"'xhigh', 'max', 'none'"
),
status_code=400,
)
gate_error = AnthropicConfig._validate_effort_for_model(
model, mapped_effort
)
if gate_error is not None:
raise AnthropicError(message=gate_error, status_code=400)
existing_output_config = optional_params.get("output_config")
if not isinstance(existing_output_config, dict):
existing_output_config = {}
existing_output_config.setdefault("effort", mapped_effort)
optional_params["output_config"] = existing_output_config
@staticmethod
def _translate_legacy_thinking_for_adaptive_model(
model: str, optional_params: Dict
@ -217,6 +274,11 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
status_code=400,
)
self._translate_reasoning_effort_to_anthropic(
model=model,
optional_params=anthropic_messages_optional_request_params,
)
self._translate_legacy_thinking_for_adaptive_model(
model=model,
optional_params=anthropic_messages_optional_request_params,

6
litellm/llms/anthropic/files/handler.py
View file

@ -9,6 +9,7 @@ import litellm
from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.types.llms.openai import (
FileContentRequest,
@ -89,7 +90,10 @@ class AnthropicFilesHandler:
raise ValueError("Missing Anthropic API Key")
# Construct the Anthropic batch results URL
results_url = f"{api_base.rstrip('/')}/v1/messages/batches/{batch_id}/results"
encoded_batch_id = encode_url_path_segment(batch_id, field_name="batch_id")
results_url = (
f"{api_base.rstrip('/')}/v1/messages/batches/{encoded_batch_id}/results"
)
# Prepare headers
headers = {

10
litellm/llms/anthropic/files/transformation.py
View file

@ -19,6 +19,7 @@ from typing import Any, Dict, List, Optional, Union, cast
import httpx
from openai.types.file_deleted import FileDeleted
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.files.transformation import (
@ -185,7 +186,8 @@ class AnthropicFilesConfig(BaseFilesConfig):
AnthropicModelInfo.get_api_base(litellm_params.get("api_base"))
or ANTHROPIC_FILES_API_BASE
)
return f"{api_base.rstrip('/')}/v1/files/{file_id}", {}
encoded_file_id = encode_url_path_segment(file_id, field_name="file_id")
return f"{api_base.rstrip('/')}/v1/files/{encoded_file_id}", {}
def transform_retrieve_file_response(
self,
@ -206,7 +208,8 @@ class AnthropicFilesConfig(BaseFilesConfig):
AnthropicModelInfo.get_api_base(litellm_params.get("api_base"))
or ANTHROPIC_FILES_API_BASE
)
return f"{api_base.rstrip('/')}/v1/files/{file_id}", {}
encoded_file_id = encode_url_path_segment(file_id, field_name="file_id")
return f"{api_base.rstrip('/')}/v1/files/{encoded_file_id}", {}
def transform_delete_file_response(
self,
@ -268,7 +271,8 @@ class AnthropicFilesConfig(BaseFilesConfig):
AnthropicModelInfo.get_api_base(litellm_params.get("api_base"))
or ANTHROPIC_FILES_API_BASE
)
return f"{api_base.rstrip('/')}/v1/files/{file_id}/content", {}
encoded_file_id = encode_url_path_segment(file_id, field_name="file_id")
return f"{api_base.rstrip('/')}/v1/files/{encoded_file_id}/content", {}
def transform_file_content_response(
self,

4
litellm/llms/anthropic/skills/transformation.py
View file

@ -7,6 +7,7 @@ from typing import Any, Dict, Optional, Tuple
import httpx
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.base_llm.skills.transformation import (
BaseSkillsAPIConfig,
LiteLLMLoggingObj,
@ -81,7 +82,8 @@ class AnthropicSkillsConfig(BaseSkillsAPIConfig):
api_base = AnthropicModelInfo.get_api_base()
if skill_id:
return f"{api_base}/v1/skills/{skill_id}"
encoded_skill_id = encode_url_path_segment(skill_id, field_name="skill_id")
return f"{api_base}/v1/skills/{encoded_skill_id}"
return f"{api_base}/v1/{endpoint}"
def transform_create_skill_request(

41
litellm/llms/azure/azure.py
View file

@ -16,6 +16,7 @@ import litellm
from litellm.constants import AZURE_OPERATION_POLLING_TIMEOUT, DEFAULT_MAX_RETRIES
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
from litellm.litellm_core_utils.url_utils import SSRFError, assert_same_origin
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
@ -43,6 +44,7 @@ from .common_utils import (
select_azure_base_url_or_endpoint,
)
from .image_generation import get_azure_image_generation_config
from .image_generation.http_utils import azure_deployment_image_generation_json_body
class AzureOpenAIAssistantsAPIConfig:
@ -792,6 +794,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
client=client,
litellm_params=litellm_params,
api_base=api_base,
api_version=api_version,
)
azure_client = self.get_azure_openai_client(
api_version=api_version,
@ -898,6 +901,17 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
operation_location_url = response.headers["operation-location"]
else:
raise AzureOpenAIError(status_code=500, message=response.text)
# Reject polling URLs that don't share an origin with ``api_base``.
# Without this an upstream-controlled or attacker-controlled
# value would receive the operator's Azure API key in the
# request headers below. VERIA-51.
try:
assert_same_origin(operation_location_url, api_base)
except SSRFError as ssrf_err:
raise AzureOpenAIError(
status_code=502,
message=f"Rejected polling URL: {ssrf_err}",
)
response = await async_handler.get(
url=operation_location_url,
headers=headers,
@ -908,8 +922,13 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
timeout_secs: int = AZURE_OPERATION_POLLING_TIMEOUT
start_time = time.time()
if "status" not in response.json():
raise Exception(
"Expected 'status' in response. Got={}".format(response.json())
# Don't reflect the raw response body — when the polling
# URL points at an internal JSON API (cloud metadata
# service etc.) reflecting it here turns Blind SSRF into
# Full-Read SSRF. VERIA-51.
raise AzureOpenAIError(
status_code=502,
message="Polling response missing 'status' field",
)
while response.json()["status"] not in ["succeeded", "failed"]:
if time.time() - start_time > timeout_secs:
@ -948,9 +967,10 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
content=json.dumps(result).encode("utf-8"),
request=httpx.Request(method="POST", url="https://api.openai.com/v1"),
)
request_json = azure_deployment_image_generation_json_body(api_base, data)
return await async_handler.post(
url=api_base,
json=data,
json=request_json,
headers=headers,
)
@ -1009,6 +1029,13 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
operation_location_url = response.headers["operation-location"]
else:
raise AzureOpenAIError(status_code=500, message=response.text)
try:
assert_same_origin(operation_location_url, api_base)
except SSRFError as ssrf_err:
raise AzureOpenAIError(
status_code=502,
message=f"Rejected polling URL: {ssrf_err}",
)
response = sync_handler.get(
url=operation_location_url,
headers=headers,
@ -1019,8 +1046,9 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
timeout_secs: int = AZURE_OPERATION_POLLING_TIMEOUT
start_time = time.time()
if "status" not in response.json():
raise Exception(
"Expected 'status' in response. Got={}".format(response.json())
raise AzureOpenAIError(
status_code=502,
message="Polling response missing 'status' field",
)
while response.json()["status"] not in ["succeeded", "failed"]:
if time.time() - start_time > timeout_secs:
@ -1059,9 +1087,10 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
content=json.dumps(result).encode("utf-8"),
request=httpx.Request(method="POST", url="https://api.openai.com/v1"),
)
request_json = azure_deployment_image_generation_json_body(api_base, data)
return sync_handler.post(
url=api_base,
json=data,
json=request_json,
headers=headers,
)

6
litellm/llms/azure/cost_calculation.py
View file

@ -12,7 +12,10 @@ from litellm.utils import get_model_info
def cost_per_token(
model: str, usage: Usage, response_time_ms: Optional[float] = 0.0
model: str,
usage: Usage,
response_time_ms: Optional[float] = 0.0,
service_tier: Optional[str] = None,
) -> Tuple[float, float]:
"""
Calculates the cost per token for a given model, prompt tokens, and completion tokens.
@ -47,4 +50,5 @@ def cost_per_token(
model=model,
usage=usage,
custom_llm_provider="azure",
service_tier=service_tier,
)

18
litellm/llms/azure/image_edit/transformation.py
View file

@ -9,6 +9,19 @@ from litellm.utils import _add_path_to_api_base
class AzureImageEditConfig(OpenAIImageEditConfig):
@staticmethod
def azure_deployment_image_edit_form_data(data: dict, request_url: str) -> dict:
"""
Azure OpenAI ``.../openai/deployments/{deployment}/images/edits`` routes by
deployment in the URL; including ``model`` in multipart fields can break
the same way as image generations (LiteLLM #26316).
Non-deployment edit URLs keep ``model`` when present.
"""
if "images/edits" in request_url and "/openai/deployments/" in request_url:
return {k: v for k, v in data.items() if k != "model"}
return data
def validate_environment(
self,
headers: dict,
@ -83,3 +96,8 @@ class AzureImageEditConfig(OpenAIImageEditConfig):
final_url = httpx.URL(new_url).copy_with(params=query_params)
return str(final_url)
def finalize_image_edit_request_data(
self, data: dict, resolved_request_url: str
) -> dict:
return self.azure_deployment_image_edit_form_data(data, resolved_request_url)

4
litellm/llms/azure/image_generation/__init__.py
View file

@ -6,11 +6,13 @@ from litellm.llms.base_llm.image_generation.transformation import (
from .dall_e_2_transformation import AzureDallE2ImageGenerationConfig
from .dall_e_3_transformation import AzureDallE3ImageGenerationConfig
from .gpt_transformation import AzureGPTImageGenerationConfig
from .http_utils import azure_deployment_image_generation_json_body
__all__ = [
"AzureDallE2ImageGenerationConfig",
"AzureDallE3ImageGenerationConfig",
"AzureGPTImageGenerationConfig",
"azure_deployment_image_generation_json_body",
]
@ -24,6 +26,6 @@ def get_azure_image_generation_config(model: str) -> BaseImageGenerationConfig:
return AzureDallE3ImageGenerationConfig()
else:
verbose_logger.debug(
f"Using AzureGPTImageGenerationConfig for model: {model}. This follows the gpt-image-1 model format."
f"Using AzureGPTImageGenerationConfig for model: {model}. This follows the gpt-image model format."
)
return AzureGPTImageGenerationConfig()

2
litellm/llms/azure/image_generation/gpt_transformation.py
View file

@ -3,7 +3,7 @@ from litellm.llms.openai.image_generation import GPTImageGenerationConfig
class AzureGPTImageGenerationConfig(GPTImageGenerationConfig):
"""
Azure gpt-image-1 image generation config
Azure gpt-image image generation config
"""
pass

17
litellm/llms/azure/image_generation/http_utils.py Normal file
View file

@ -0,0 +1,17 @@
"""HTTP helpers for Azure OpenAI image generation (REST, not SDK)."""
def azure_deployment_image_generation_json_body(api_base: str, data: dict) -> dict:
"""
Build the JSON body for Azure OpenAI image generation POSTs.
For ``.../openai/deployments/{deployment}/images/generations``, routing uses the
deployment in the URL only; sending ``model`` in the body (especially the deployment
name) breaks some models (e.g. gpt-image-2). See LiteLLM #26316.
Provider-style URLs (e.g. ``/providers/...`` for FLUX on Azure AI) keep all keys
so non–OpenAI-deployment payloads still work.
"""
if "images/generations" in api_base and "/openai/deployments/" in api_base:
return {k: v for k, v in data.items() if k != "model"}
return data

11
litellm/llms/azure/responses/transformation.py
View file

@ -5,6 +5,7 @@ import httpx
from openai.types.responses import ResponseReasoningItem
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.azure.common_utils import BaseAzureLLM
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.types.llms.openai import *
@ -201,7 +202,10 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
# Insert the response_id at the end of the path component
# Remove trailing slash if present to avoid double slashes
path = parsed_url.path.rstrip("/")
new_path = f"{path}/{response_id}"
encoded_response_id = encode_url_path_segment(
response_id, field_name="response_id"
)
new_path = f"{path}/{encoded_response_id}"
# Reconstruct the URL with all original components but with the modified path
constructed_url = urlunparse(
@ -322,7 +326,10 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
# Insert the response_id and /cancel at the end of the path component
# Remove trailing slash if present to avoid double slashes
path = parsed_url.path.rstrip("/")
new_path = f"{path}/{response_id}/cancel"
encoded_response_id = encode_url_path_segment(
response_id, field_name="response_id"
)
new_path = f"{path}/{encoded_response_id}/cancel"
# Reconstruct the URL with all original components but with the modified path
cancel_url = urlunparse(

18
litellm/llms/azure_ai/agents/handler.py
View file

@ -36,6 +36,7 @@ from typing import (
import httpx
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.azure_ai.agents.transformation import (
AzureAIAgentsConfig,
AzureAIAgentsError,
@ -75,20 +76,29 @@ class AzureAIAgentsHandler:
def _build_messages_url(
self, api_base: str, thread_id: str, api_version: str
) -> str:
return f"{api_base}/threads/{thread_id}/messages?api-version={api_version}"
encoded_thread_id = encode_url_path_segment(thread_id, field_name="thread_id")
return (
f"{api_base}/threads/{encoded_thread_id}/messages?api-version={api_version}"
)
def _build_runs_url(self, api_base: str, thread_id: str, api_version: str) -> str:
return f"{api_base}/threads/{thread_id}/runs?api-version={api_version}"
encoded_thread_id = encode_url_path_segment(thread_id, field_name="thread_id")
return f"{api_base}/threads/{encoded_thread_id}/runs?api-version={api_version}"
def _build_run_status_url(
self, api_base: str, thread_id: str, run_id: str, api_version: str
) -> str:
return f"{api_base}/threads/{thread_id}/runs/{run_id}?api-version={api_version}"
encoded_thread_id = encode_url_path_segment(thread_id, field_name="thread_id")
encoded_run_id = encode_url_path_segment(run_id, field_name="run_id")
return f"{api_base}/threads/{encoded_thread_id}/runs/{encoded_run_id}?api-version={api_version}"
def _build_list_messages_url(
self, api_base: str, thread_id: str, api_version: str
) -> str:
return f"{api_base}/threads/{thread_id}/messages?api-version={api_version}"
encoded_thread_id = encode_url_path_segment(thread_id, field_name="thread_id")
return (
f"{api_base}/threads/{encoded_thread_id}/messages?api-version={api_version}"
)
def _build_create_thread_and_run_url(self, api_base: str, api_version: str) -> str:
"""URL for the create-thread-and-run endpoint (supports streaming)."""

Some files were not shown because too many files have changed in this diff Show more