Merge origin/main into litellm_messages_non_string_text_400

Main's context_management polyfill on the Responses API path called the tool_result-only helpers this branch generalized, so it now uses reject_malformed_text and drop_non_string_texts, and its drop test also covers user text
This commit is contained in:
mateo-berri 2026-10-29 18:34:47 -07:00
commit 0830e27dda
448 changed files with 20338 additions and 4595 deletions

View file

@ -1,14 +1,14 @@
#!/usr/bin/env bash
set -euo pipefail
if [ "${GITHUB_ACTIONS:-}" = true ]; then
echo "Integration contracts are owned by CircleCI" >&2
exit 1
fi
suite="${1:?integration suite required}"
mode="${2:-standard}"
side="${3:-}"
requested_workers="${INTEGRATION_WORKERS:-}"
license_env=()
if [ -n "${LITELLM_LICENSE:-}" ]; then
license_env=("LITELLM_LICENSE=$LITELLM_LICENSE")
fi
if [ "$mode" = replica ]; then
results="test-results/integration-${suite}-replica"
elif [ "$mode" = parity ]; then
@ -84,7 +84,7 @@ if [ "$suite" = browser ]; then
export LITELLM_UI_PATH="$PWD/ui/litellm-dashboard/out"
test -f "$LITELLM_UI_PATH/index.html"
fi
export INTEGRATION_SEED="$(.venv/bin/python -c 'import hashlib,os; print(int(hashlib.sha256((os.environ.get("CIRCLE_SHA1", "local") + os.environ.get("CIRCLE_WORKFLOW_ID", "local")).encode()).hexdigest()[:8],16))')"
export INTEGRATION_SEED="$(.venv/bin/python -c 'import hashlib,os; print(int(hashlib.sha256(((os.environ.get("CIRCLE_SHA1") or os.environ.get("GITHUB_SHA") or "local") + (os.environ.get("CIRCLE_WORKFLOW_ID") or os.environ.get("GITHUB_RUN_ID") or "local")).encode()).hexdigest()[:8],16))')"
export INTEGRATION_ORDER_SEED="$INTEGRATION_SEED"
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma > "$results/prisma-generate.log" 2>&1
@ -107,7 +107,7 @@ sudo iptables -N integration_only
guard_created=true
sudo iptables -A integration_only -o lo -j ACCEPT
sudo iptables -A integration_only -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT
for service in postgres-db redis-cache; do
for service in "${INTEGRATION_POSTGRES_CONTAINER:-postgres-db}" "${INTEGRATION_REDIS_CONTAINER:-redis-cache}"; do
address="$(docker inspect --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$service")"
sudo iptables -A integration_only -d "$address" -j ACCEPT
done
@ -145,6 +145,9 @@ fi
if [ "$suite" = mcp ]; then
export INTEGRATION_WORKERS=2 INTEGRATION_COVERAGE=1
fi
if [ -n "$requested_workers" ]; then
export INTEGRATION_WORKERS="$requested_workers"
fi
isolated_workers=false
if [ "${INTEGRATION_WORKERS:-1}" -gt 1 ]; then
isolated_workers=true
@ -185,7 +188,7 @@ start_proxy() {
"${database_env[@]}" REDIS_HOST="$REDIS_HOST" REDIS_PORT="$REDIS_PORT" \
INTEGRATION_UPSTREAM_URL="$INTEGRATION_UPSTREAM_URL" \
LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" LITELLM_SALT_KEY="$LITELLM_SALT_KEY" LITELLM_UI_PATH="$LITELLM_UI_PATH" PROXY_BASE_URL="http://127.0.0.1:$port" \
LITELLM_LICENSE="${LITELLM_LICENSE:-}" \
"${license_env[@]}" \
LITELLM_MODE=PRODUCTION STORE_MODEL_IN_DB=True LITELLM_ENABLE_MCP_STDIO=true "${cost_map_env[@]}" \
AWS_EC2_METADATA_DISABLED=true DO_NOT_TRACK=1 COVERAGE_FILE="$coverage_data" \
"${proxy_command[@]}" --config tests/integration/proxy_config.yaml \
@ -257,7 +260,7 @@ env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \
"${shared_services[@]}" "${worker_coverage[@]}" \
INTEGRATION_WORKERS="${INTEGRATION_WORKERS:-1}" \
LITELLM_MODE=PRODUCTION \
LITELLM_LICENSE="${LITELLM_LICENSE:-}" \
"${license_env[@]}" \
INTEGRATION_SEED="$INTEGRATION_SEED" \
INTEGRATION_ORDER_SEED="$INTEGRATION_ORDER_SEED" \
INTEGRATION_CASE_SHARDS="${CIRCLE_NODE_TOTAL:-1}" INTEGRATION_CASE_SHARD_INDEX="${CIRCLE_NODE_INDEX:-0}" \

119
.github/scripts/assert_secret_absent.py vendored Normal file
View file

@ -0,0 +1,119 @@
import base64
import io
import os
import sys
import urllib.parse
import zipfile
from collections.abc import Callable, Iterator, Mapping
from dataclasses import dataclass
from functools import reduce
from itertools import chain
from pathlib import Path
from types import MappingProxyType
from typing import Final, TextIO
SECRET_VARIABLE: Final = "SCANNED_SECRET"
MIN_ENCODED_LENGTH: Final = 12
MAX_ARCHIVE_DEPTH: Final = 3
MASK: Final = "***"
Forms = tuple[tuple[str, str], ...]
@dataclass(frozen=True, slots=True)
class Hit:
location: str
form: str
def _aligned_encodings(raw: bytes, name: str, encode: Callable[[bytes], bytes]) -> Iterator[tuple[str, str]]:
for offset in range(3):
chunk: Final = raw[offset:]
yield f"{name} offset {offset}", encode(chunk[: len(chunk) - len(chunk) % 3]).decode()
def secret_forms(secret: str) -> Forms:
encoded: Final = (
("url-encoded", urllib.parse.quote(secret, safe="")),
*_aligned_encodings(secret.encode(), "base64", base64.b64encode),
*_aligned_encodings(secret.encode(), "base64url", base64.urlsafe_b64encode),
)
by_value: Final = MappingProxyType(
{value: name for name, value in encoded if value != secret and len(value) >= MIN_ENCODED_LENGTH}
)
return (("raw", secret), *((name, value) for value, name in by_value.items()))
def mask(text: str, forms: Forms) -> str:
return reduce(lambda masked, form: masked.replace(form[1], MASK), forms, text)
def _name_hits(location: str, name: str, forms: Forms) -> Iterator[Hit]:
for form, value in forms:
if value in name:
yield Hit(mask(location, forms), f"{form} in name")
def _content_hits(location: str, data: bytes, forms: Forms, depth: int) -> Iterator[Hit]:
for form, value in forms:
if value.encode() in data:
yield Hit(mask(location, forms), form)
if depth >= MAX_ARCHIVE_DEPTH or not zipfile.is_zipfile(io.BytesIO(data)):
return
with zipfile.ZipFile(io.BytesIO(data)) as archive:
for member in archive.infolist():
if member.is_dir():
continue
yield from _content_hits(f"{location}!{member.filename}", archive.read(member), forms, depth + 1)
def _walk(root: Path) -> Iterator[tuple[Path, bool]]:
for directory, subdirectories, files in os.walk(root):
yield from ((Path(directory, name), True) for name in subdirectories if Path(directory, name).is_symlink())
yield from ((Path(directory, name), False) for name in files)
def _path_hits(root: Path, path: Path, linked_directory: bool, forms: Forms) -> Iterator[Hit]:
location: Final = str(path.relative_to(root))
if linked_directory:
yield Hit(mask(location, forms), "directory symlink, not scanned")
return
yield from _name_hits(location, location, forms)
yield from _content_hits(location, path.read_bytes(), forms, 0)
def scan(root: Path, secret: str) -> tuple[int, tuple[Hit, ...]]:
forms: Final = secret_forms(secret)
entries: Final = tuple(_walk(root))
hits: Final = tuple(chain.from_iterable(_path_hits(root, path, linked, forms) for path, linked in entries))
return len(entries), hits
def main(argv: tuple[str, ...], environ: Mapping[str, str], out: TextIO) -> int:
if len(argv) != 1:
print(f"usage: assert_secret_absent.py <directory>, with the secret in ${SECRET_VARIABLE}", file=out)
return 2
secret: Final = environ.get(SECRET_VARIABLE, "")
root: Final = Path(argv[0])
if not secret:
print(f"{SECRET_VARIABLE} is empty, nothing to scan for", file=out)
return 0
if not root.is_dir():
print(f"{root} is not a directory, nothing to scan", file=out)
return 0
try:
scanned, hits = scan(root, secret)
except Exception as error:
print(f"scan failed with {type(error).__name__}, treating the results as unsafe", file=out)
return 1
for hit in hits:
print(f"secret found: {hit.location} ({hit.form})", file=out)
if hits:
print(f"{len(hits)} copies of the secret under {root}, refusing to upload", file=out)
return 1
print(f"no copy of the secret in {scanned} files under {root}", file=out)
return 0
if __name__ == "__main__":
sys.exit(main(tuple(sys.argv[1:]), os.environ, sys.stdout))

View file

@ -13,8 +13,10 @@ on:
- migrations/Dockerfile
- migrations/run.py
- gateway/Dockerfile
- gateway/launch.py
- gateway/main.py
- gateway/routes/allowlist.py
- litellm_cli/**
- backend/Dockerfile
- backend/main.py
- litellm-rust/**

199
.github/workflows/test-integration.yml vendored Normal file
View file

@ -0,0 +1,199 @@
name: integration
on:
pull_request:
types: [labeled]
schedule:
- cron: "5 8-20/3 * * 1-6"
timezone: America/Los_Angeles
workflow_dispatch:
inputs:
suite:
description: Integration group to run
type: choice
options:
- all
- providers
- extensions
- accounting
- management
- management-replica
- database
- database-replica
- mcp
- security
- cost
- sdk
- browser
default: all
workers:
description: Workers per group, blank keeps each group's default
type: string
default: ""
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.event.label.name }}-${{ inputs.suite }}-${{ inputs.workers }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
integration:
name: integration ${{ matrix.suite }}
if: github.event_name != 'pull_request' || github.event.label.name == 'run-ci'
environment: integration-license
runs-on: ${{ fromJSON('{"providers":"ubuntu-latest-64-cores","extensions":"ubuntu-latest-64-cores","accounting":"ubuntu-latest-32-cores","management":"ubuntu-latest-32-cores","management-replica":"ubuntu-latest-32-cores"}')[matrix.suite] || 'ubuntu-latest-16-cores' }}
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
suite: ${{ fromJSON((inputs.suite || 'all') == 'all' && '["providers","extensions","accounting","management","management-replica","database","database-replica","mcp","security","cost","sdk","browser"]' || format('["{0}"]', inputs.suite)) }}
env:
UV_PYTHON: "3.12"
INTEGRATION_WORKERS: ${{ inputs.workers || fromJSON('{"providers":"32","extensions":"48","accounting":"16","management":"16","management-replica":"16","cost":"16","mcp":"8","security":"8","database":"4","database-replica":"4"}')[matrix.suite] || '' }}
services:
postgres:
image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
command: -c max_connections=1000 -c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c pg_stat_statements.max=20000
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: circle_test
ports:
- 5432:5432
options: >-
--shm-size 1g
--health-cmd "pg_isready -U postgres"
--health-interval 5s
--health-timeout 5s
--health-retries 20
redis:
image: redis:7-alpine@sha256:7aec734b2bb298a1d769fd8729f13b8514a41bf90fcdd1f38ec52267fbaa8ee6
ports:
- 6379:6379
options: >-
--health-cmd "redis-cli ping"
--health-interval 5s
--health-timeout 5s
--health-retries 20
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Set up Python
timeout-minutes: 3
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: ${{ env.UV_PYTHON }}
- name: Set up uv
timeout-minutes: 3
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Cache the Rust build
timeout-minutes: 5
uses: ./.github/actions/cache-cargo-build
- name: Cache Prisma binaries
timeout-minutes: 5
uses: ./.github/actions/cache-prisma-binaries
- name: Install dependencies
timeout-minutes: 20
run: .github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras
- name: Install redis-server 7.2.16
timeout-minutes: 3
run: |
mkdir -p "$RUNNER_TEMP/redis-bin"
cid="$(docker create redis:7.2.16@sha256:0637954999d01b7c9ce9167db2da50656e2590d3b884f1c600c5f63bb6e6773c)"
docker cp "${cid}:/usr/local/bin/redis-server" "$RUNNER_TEMP/redis-bin/redis-server"
docker rm "$cid"
"$RUNNER_TEMP/redis-bin/redis-server" --version | grep -q 'v=7.2.16'
echo "$RUNNER_TEMP/redis-bin" >> "$GITHUB_PATH"
- name: Install MCP SDK1 peer
if: matrix.suite == 'mcp'
timeout-minutes: 5
run: |
uv venv --python 3.12 .venv-mcp-peer
uv pip install --no-config --exclude-newer "3 days" --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1'
echo "MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$GITHUB_ENV"
- name: Set up Node.js
if: matrix.suite == 'browser'
timeout-minutes: 5
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
with:
node-version-file: ui/litellm-dashboard/.nvmrc
cache: "npm"
cache-dependency-path: |
ui/litellm-dashboard/package-lock.json
tests/e2e/ui/package-lock.json
- name: Install browser dependencies and build the dashboard
if: matrix.suite == 'browser'
timeout-minutes: 20
run: |
(cd ui/litellm-dashboard && npm ci)
(cd tests/e2e/ui && npm ci)
sudo env PATH="$PATH" DEBIAN_FRONTEND=noninteractive \
node tests/e2e/ui/node_modules/@playwright/test/cli.js install-deps chromium
node tests/e2e/ui/node_modules/@playwright/test/cli.js install chromium
(cd ui/litellm-dashboard && NEXT_TELEMETRY_DISABLED=1 npm run build)
- name: Run integration contracts
timeout-minutes: 45
env:
SUITE: ${{ matrix.suite }}
INTEGRATION_POSTGRES_CONTAINER: ${{ job.services.postgres.id }}
INTEGRATION_REDIS_CONTAINER: ${{ job.services.redis.id }}
LITELLM_LICENSE: ${{ secrets.LITELLM_LICENSE }}
run: |
case "$SUITE" in
*-replica) bash .circleci/scripts/run_integration.sh "${SUITE%-replica}" replica ;;
*) bash .circleci/scripts/run_integration.sh "$SUITE" ;;
esac
- name: Verify every case ran exactly once
if: ${{ !cancelled() && matrix.suite != 'browser' }}
timeout-minutes: 2
run: |
record="$(find test-results -mindepth 2 -maxdepth 2 -name execution.json -print -quit)"
test -n "$record"
mkdir -p "$RUNNER_TEMP/shards/0"
cp "$record" "$RUNNER_TEMP/shards/0/execution.json"
.venv/bin/python -m tests.integration._support.sharding "$RUNNER_TEMP/shards" --shards 1
- name: Report MCP coverage
if: ${{ !cancelled() && matrix.suite == 'mcp' }}
timeout-minutes: 5
run: |
coverage_dir=test-results/integration-mcp/coverage
test -f "$coverage_dir/coverage.txt" && exit 0
export COVERAGE_FILE="$PWD/$coverage_dir/data"
.venv/bin/python -m coverage combine --rcfile=tests/integration/mcp_coverage.toml
.venv/bin/python -m coverage report --rcfile=tests/integration/mcp_coverage.toml > "$coverage_dir/coverage.txt"
.venv/bin/python -m coverage html --rcfile=tests/integration/mcp_coverage.toml -d "$coverage_dir/html"
tail -n 1 "$coverage_dir/coverage.txt"
- name: Check results for the license
id: secret-scan
if: ${{ !cancelled() }}
timeout-minutes: 5
env:
SCANNED_SECRET: ${{ secrets.LITELLM_LICENSE }}
run: python .github/scripts/assert_secret_absent.py test-results
- name: Upload results
if: ${{ !cancelled() && steps.secret-scan.outcome == 'success' }}
timeout-minutes: 10
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
with:
name: integration-${{ matrix.suite }}-${{ github.run_id }}-${{ github.run_attempt }}
path: test-results/
retention-days: 14

View file

@ -1003,9 +1003,9 @@ jobs:
run: |
diff -u model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils
else
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
uv run --no-sync python -c "import importlib.metadata; import litellm.rust_bridge._native; print(importlib.metadata.version('litellm'))"
@ -1451,9 +1451,9 @@ jobs:
RUST_BRIDGE_ARTIFACT: ${{ needs.rust-bridge.outputs.artifact }}
run: |
if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils
else
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
fi

View file

@ -153,7 +153,7 @@ lint-install:
# the same set CI's ':(glob)litellm/**/*.py' selects.
lint-format-check-changed: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
@base_ref=$$($(RESOLVE_BASE)) && \
changed=$$(git diff --name-only --diff-filter=ACMR "$$base_ref...HEAD" -- 'litellm/*.py') && \
changed=$$(git diff --name-only --diff-filter=ACMR --merge-base "$$base_ref" -- 'litellm/*.py') && \
files=$$(printf '%s\n' "$$changed" | grep -v '^litellm/enterprise/' || true) || exit $$?; \
if [ -z "$$files" ]; then \
echo "No changed litellm Python files to format-check."; \

View file

@ -141,12 +141,56 @@ response = client.chat.completions.create(
)
```
[**Docs: LLM Providers**](https://docs.litellm.ai/docs/providers)
</details>
<details>
<summary><b>Auto Routing</b> - Choose a model for each prompt (Python SDK + AI Gateway)</summary>
### Python SDK
[**Docs: Autorouter SDK**](https://docs.litellm.ai/docs/autorouter_sdk)
Install with `uv add litellm` and set `OPENAI_API_KEY` for the models below.
```python
import asyncio
from litellm import Router
router = Router(model_list=[
{"model_name": "fast", "litellm_params": {"model": "openai/gpt-6-luna"}},
{"model_name": "capable", "litellm_params": {"model": "openai/gpt-6.1-sol"}},
{"model_name": "auto", "litellm_params": {
"model": "auto_router/complexity_router",
"complexity_router_config": {
"classifier_type": "heuristic",
"tiers": {"SIMPLE": "fast", "MEDIUM": "fast",
"COMPLEX": "capable", "REASONING": "capable"},
"return_raw_model_name": True,
},
"complexity_router_default_model": "capable",
}},
])
response = asyncio.run(router.acompletion(
model="auto", messages=[{"role": "user", "content": "What is 2+2?"}],
))
print(response.model, response.choices[0].message.content)
```
### AI Gateway (Proxy Server)
[**Create an Auto Router**](https://docs.litellm.ai/docs/auto_router/setup), then call its model name. Replace `auto` and the URL below with yours, and set `LITELLM_API_KEY` to your virtual key.
```shell
curl http://localhost:4000/v1/chat/completions \
-H "Authorization: Bearer $LITELLM_API_KEY" \
-H "Content-Type: application/json" \
-d '{"model":"auto","messages":[{"role":"user","content":"What is 2+2?"}]}'
```
</details>
<details>
<summary><b>Agents</b> - Invoke A2A Agents (Python SDK + AI Gateway)</summary>
<summary><b>A2A</b> - Invoke A2A Agents (Python SDK + AI Gateway)</summary>
[**Supported Providers**](https://docs.litellm.ai/docs/a2a#add-a2a-agents) - LangGraph, Vertex AI Agent Engine, Azure AI Foundry, Bedrock AgentCore, Pydantic AI
@ -354,6 +398,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
| [Eden AI (`edenai`)](https://docs.litellm.ai/docs/providers/edenai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | |
| [ElevenLabs (`elevenlabs`)](https://docs.litellm.ai/docs/providers/elevenlabs) | ✅ | ✅ | ✅ | | | ✅ | ✅ | | | |
| [Empower (`empower`)](https://docs.litellm.ai/docs/providers/empower) | ✅ | ✅ | ✅ | | | | | | | |
| [eu/jev Decisions API (`eu_jev`)](https://docs.litellm.ai/docs/pass_through/eu_jev) | | | | | | | | | | |
| [Fal AI (`fal_ai`)](https://docs.litellm.ai/docs/providers/fal_ai) | ✅ | ✅ | ✅ | | ✅ | | | | | |
| [Featherless AI (`featherless_ai`)](https://docs.litellm.ai/docs/providers/featherless_ai) | ✅ | ✅ | ✅ | | | | | | | |
| [Fireworks AI (`fireworks_ai`)](https://docs.litellm.ai/docs/providers/fireworks_ai) | ✅ | ✅ | ✅ | | | | | | | |
@ -442,7 +487,7 @@ You can use LiteLLM through either the Proxy Server or Python SDK. Both give you
<tr>
<th style={{width: '14%'}}></th>
<th style={{width: '43%'}}><strong><a href="https://docs.litellm.ai/docs/simple_proxy">LiteLLM AI Gateway</a></strong></th>
<th style={{width: '43%'}}><strong><a href="https://docs.litellm.ai/docs/">LiteLLM Python SDK</a></strong></th>
<th style={{width: '43%'}}><strong><a href="https://docs.litellm.ai/docs/python_sdk">LiteLLM Python SDK</a></strong></th>
</tr>
</thead>
<tbody>

View file

@ -398,7 +398,7 @@ class CheckBatchCost:
deployment. Once that deployment leaves the router, default fallbacks can
silently send the retrieve to a provider that never saw the batch, so its
404 must not retire a potentially recoverable row."""
return self.llm_router.get_deployment(model_id=model_id) is not None
return self.llm_router.registry().deployment(model_id) is not None
@staticmethod
def _is_output_file_gone_at_provider(error: Exception, output_file_id: Optional[str]) -> bool:
@ -564,7 +564,7 @@ class CheckBatchCost:
def _get_deployment_id_for_bare_model(
self, bare_model_name: str, llm_provider: str
) -> Optional[str]:
model_group = self.llm_router.resolve_model_name_from_model_id(bare_model_name)
model_group = self.llm_router.registry().resolve(bare_model_name, kind="from_id")
deployment_id = (
self._get_deployment_id_for_provider(model_group, llm_provider) if model_group else None
)
@ -580,7 +580,7 @@ class CheckBatchCost:
) -> Optional[str]:
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
for deployment in self.llm_router.get_model_list(model_name=None) or []:
for deployment in self.llm_router.registry().deployments() or []:
litellm_params = deployment.get("litellm_params") or {}
actual_model = litellm_params.get("model")
if not isinstance(actual_model, str):
@ -623,8 +623,9 @@ class CheckBatchCost:
"""
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
for deployment_id in self.llm_router.get_model_ids(model_name=model_group):
deployment_info = self.llm_router.get_deployment(model_id=deployment_id)
registry = self.llm_router.registry()
for deployment_id in registry.names(model_group, kind="ids"):
deployment_info = registry.deployment(deployment_id)
if deployment_info is None:
continue
try:
@ -743,7 +744,7 @@ class CheckBatchCost:
except (IndexError, AttributeError):
pass
credentials: Final = self.llm_router.get_deployment_credentials_with_provider(model_id) or {}
credentials: Final = self.llm_router.registry().credentials(model_id, form="with_provider") or {}
_file_content = await afile_content(
file_id=cast(str, raw_output_file_id),
_litellm_internal_model_credentials=MappingProxyType(dict(credentials)),
@ -766,7 +767,7 @@ class CheckBatchCost:
except Exception:
pass
deployment_info = self.llm_router.get_deployment(model_id=model_id)
deployment_info = self.llm_router.registry().deployment(model_id)
if deployment_info is None:
verbose_proxy_logger.info(
f"Skipping job {job.unified_object_id} because it is not a valid deployment info"
@ -996,7 +997,7 @@ class CheckBatchCost:
)
try:
deployment_info: Final = self.llm_router.get_deployment(model_id=model_id)
deployment_info: Final = self.llm_router.registry().deployment(model_id)
if deployment_info is None:
verbose_proxy_logger.error(
f"CheckBatchCost: deployment {model_id} is no longer in the router, so the line items of batch "
@ -1009,7 +1010,7 @@ class CheckBatchCost:
custom_llm_provider=deployment_info.litellm_params.custom_llm_provider,
)
cost_provider: Final = cast(BatchFileProvider, llm_provider)
credentials: Final = self.llm_router.get_deployment_credentials_with_provider(model_id) or {}
credentials: Final = self.llm_router.registry().credentials(model_id, form="with_provider") or {}
file_access: Final = {
**credentials,
"_litellm_internal_model_credentials": MappingProxyType(dict(credentials)),

View file

@ -273,7 +273,7 @@ class CheckResponsesCost:
def _resolve_deployment(self, response_id: str) -> Deployment | None:
model_id: str | None = ResponsesAPIRequestUtils.get_model_id_from_response_id(response_id)
return self.llm_router.get_deployment(model_id=model_id) if model_id is not None else None
return self.llm_router.registry().deployment(model_id) if model_id is not None else None
async def _get_response(
self,
@ -415,7 +415,7 @@ class CheckResponsesCost:
if target.deployment is None or not isinstance(requested, str):
return MappingProxyType({})
routed: Final = _ROUTED_DEPLOYMENTS.validate_python(
self.llm_router.get_model_list(model_name=requested, team_id=team_id) or ()
self.llm_router.registry().deployments(requested, team_id=team_id) or ()
)
serving_id: Final = target.deployment.model_info.id
return MappingProxyType(

View file

@ -245,7 +245,7 @@ def _provider_file_retrieve_credentials(
if llm_router is None or model_id is None:
return None
try:
credentials: Final = llm_router.get_deployment_credentials_with_provider(model_id)
credentials: Final = llm_router.registry().credentials(model_id, form="with_provider")
except Exception as error:
verbose_logger.warning(
"Failed to retrieve credentials for provider file "
@ -2650,7 +2650,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
k: v for k, v in data.items() if k not in ("model", "file_id", "_litellm_internal_model_credentials")
}
for model_id, model_file_id in specific_model_file_id_mapping.items():
credentials = llm_router.get_deployment_credentials_with_provider(model_id=model_id)
credentials = llm_router.registry().credentials(model_id, form="with_provider")
delete_data = {
**filtered_data,
**(
@ -2694,7 +2694,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
# against the deployment's configured bucket, which they only
# trust from this immutable server-side snapshot, never from
# request params.
credentials = llm_router.get_deployment_credentials_with_provider(model_id=model_id)
credentials = llm_router.registry().credentials(model_id, form="with_provider")
if credentials is not None:
data["_litellm_internal_model_credentials"] = cast(Dict, MappingProxyType(dict(credentials)))
else:

View file

@ -290,7 +290,7 @@ def _check_team_project_limits(
def _router_access_group_names(llm_router: "Router | None") -> frozenset[str]:
return frozenset(llm_router.get_model_access_groups()) if llm_router is not None else frozenset()
return frozenset(llm_router.registry().access_groups()) if llm_router is not None else frozenset()
def _project_models_expanding_at_request_time(

View file

@ -1,6 +1,6 @@
[project]
name = "litellm-enterprise"
version = "0.1.80"
version = "0.1.81"
description = "Package for LiteLLM Enterprise features"
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.1.80"
version = "0.1.81"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-enterprise==",

View file

@ -145,6 +145,7 @@ COPY --from=builder --chown=nonroot:nonroot /app/.venv /app/.venv
COPY --from=builder --chown=nonroot:nonroot /app/litellm /app/litellm
COPY --from=builder --chown=nonroot:nonroot /app/enterprise /app/enterprise
COPY --from=builder --chown=nonroot:nonroot /app/gateway /app/gateway
COPY --from=builder --chown=nonroot:nonroot /app/litellm_cli /app/litellm_cli
COPY --from=builder --chown=nonroot:nonroot /app/docker/component_entrypoint.sh /app/docker/component_entrypoint.sh
COPY --from=builder --chown=nonroot:nonroot /app/schema.prisma /app/schema.prisma
COPY --from=builder /opt/prisma /opt/prisma

View file

@ -8,6 +8,11 @@ handed the loopback URL it listens on. A pre-existing ``DATABASE_URL`` wins in
wins under token auth too, so exporting it here is enough for every worker to
pick the pooled URL up unchanged.
LiteLLM imports wait for ``main``: every uvicorn worker imports this module again
before it can answer its first 5 s health ping, so a slow host would kill each one.
Workers then load the gateway app through ``worker_app`` without garbage collection
pauses, which hold the GIL long enough to miss a ping.
Run with:
python -m gateway.launch --workers 4 --host 0.0.0.0 --port 4000
"""
@ -15,19 +20,19 @@ Run with:
import os
import sys
from collections.abc import Callable, Mapping, Sequence
from typing import Final
from typing import TYPE_CHECKING, Final
from uvicorn.main import main as uvicorn_main
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
from litellm.proxy.db.pgbouncer import (
PgBouncerError,
PgBouncerSettings,
export_pooled_database_url,
start_in_container_pgbouncer,
)
from litellm_cli.main import without_gc_pauses
GATEWAY_APP: Final = "gateway.main:app"
if TYPE_CHECKING:
from fastapi import FastAPI
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings
GATEWAY_APP: Final = "gateway.launch:worker_app"
KEEPALIVE_FLAG: Final = "--timeout-keep-alive"
@ -35,21 +40,23 @@ def uvicorn_argv(argv: Sequence[str], environ: Mapping[str, str]) -> tuple[str,
"""Honor ``KEEPALIVE_TIMEOUT`` like ``proxy_cli.py`` does, unless the flag was passed explicitly."""
keepalive: Final = environ.get("KEEPALIVE_TIMEOUT")
if keepalive is None or any(arg == KEEPALIVE_FLAG or arg.startswith(f"{KEEPALIVE_FLAG}=") for arg in argv):
return (GATEWAY_APP, *argv)
return (GATEWAY_APP, *argv, KEEPALIVE_FLAG, keepalive)
return (GATEWAY_APP, "--factory", *argv)
return (GATEWAY_APP, "--factory", *argv, KEEPALIVE_FLAG, keepalive)
def pool_database_url(
settings: DatabaseURLSettings,
pgbouncer: PgBouncerSettings,
settings: "DatabaseURLSettings",
pgbouncer: "PgBouncerSettings",
environ: Mapping[str, str],
) -> str | PgBouncerError | None:
) -> "str | PgBouncerError | None":
"""Start the in-container PgBouncer and return its loopback URL, or None when ``pgbouncer.enabled`` is off.
The upstream URL is whatever ``apply_to_env`` assembled from the discrete
``DATABASE_*`` vars (or an operator-pinned ``DATABASE_URL``). Under token
auth the pooler mints and renews the upstream token itself.
"""
from litellm.proxy.db.pgbouncer import PgBouncerError, start_in_container_pgbouncer
if not pgbouncer.enabled:
return None
upstream_url: Final = environ.get("DATABASE_URL")
@ -58,11 +65,20 @@ def pool_database_url(
return start_in_container_pgbouncer(pgbouncer, upstream_url, token_auth=settings.token_auth())
def worker_app() -> "FastAPI":
with without_gc_pauses():
from gateway.main import app
return app
def _serve(argv: Sequence[str]) -> None:
uvicorn_main(tuple(argv), prog_name="uvicorn")
def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None:
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings, export_pooled_database_url
settings: Final = DatabaseURLSettings.from_env()
settings.apply_to_env()
pooled_url: Final = pool_database_url(settings, PgBouncerSettings(), os.environ)

View file

@ -142,6 +142,7 @@ GATEWAY_EXACT_PATHS: frozenset[str] = frozenset(
"/test",
"/debug/memory/summary",
"/api/event_logging/batch",
"/eu_jev/v1/systemone",
}
)

File diff suppressed because one or more lines are too long

View file

@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
version = "0.4.111"
version = "0.4.112"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@ -30,7 +30,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.4.111"
version = "0.4.112"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-proxy-extras==",

View file

@ -520,7 +520,7 @@ aiohttp_trust_env: bool = False # set to true to use HTTP_ Proxy settings
disable_aiohttp_transport: bool = False # Set this to true to use httpx instead
disable_aiohttp_trust_env: bool = False # When False, aiohttp will respect HTTP(S)_PROXY env vars
force_ipv4: bool = False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6.
http2: bool = False
http2: bool | None = None
network_mock: bool = False # When True, use mock transport — no real network calls
####### STOP SEQUENCE LIMIT #######
@ -1783,6 +1783,9 @@ if TYPE_CHECKING:
from .llms.bespoke.decisions.transformation import (
BespokeDecisionsConfig as BespokeDecisionsConfig,
)
from .llms.eu_jev.decisions.transformation import (
EuJevDecisionsConfig as EuJevDecisionsConfig,
)
from .llms.openai.decisions.transformation import (
OpenAIDecisionsConfig as OpenAIDecisionsConfig,
)

View file

@ -174,6 +174,7 @@ LLM_CONFIG_NAMES: Final = (
"OllamaDecisionsConfig",
"LayaDecisionsConfig",
"BespokeDecisionsConfig",
"EuJevDecisionsConfig",
"OpenAIDecisionsConfig",
"AzureDecisionsConfig",
"AzureAIDecisionsConfig",
@ -753,6 +754,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
"OllamaDecisionsConfig": (".llms.ollama.decisions.transformation", "OllamaDecisionsConfig"),
"LayaDecisionsConfig": (".llms.laya.decisions.transformation", "LayaDecisionsConfig"),
"BespokeDecisionsConfig": (".llms.bespoke.decisions.transformation", "BespokeDecisionsConfig"),
"EuJevDecisionsConfig": (".llms.eu_jev.decisions.transformation", "EuJevDecisionsConfig"),
"OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"),
"AzureDecisionsConfig": (".llms.azure.decisions.transformation", "AzureDecisionsConfig"),
"AzureAIDecisionsConfig": (".llms.azure_ai.decisions.transformation", "AzureAIDecisionsConfig"),

View file

@ -28,9 +28,11 @@ from litellm.litellm_core_utils.env_utils import get_env_int
from litellm.litellm_core_utils.safe_json_dumps import UNSERIALIZABLE_OBJECT, safe_dumps, safe_json_structure
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
from litellm.litellm_core_utils.secret_redaction import (
REDACTED,
python_redact_string,
python_redact_structured_value,
redact_internal_details,
redact_structured_value,
)
from litellm.litellm_core_utils.secret_redaction import (
redact_string as redact_secret_string,
@ -140,6 +142,32 @@ def redact_internal_details_from_client_message(value: str) -> str:
return redact_internal_details(value)
_CLIENT_FIELDS: Final = TypeAdapter(dict[str, object])
_CLIENT_ITEMS: Final = TypeAdapter(list[object])
def _redacted_client_field(key: str | None, value: object) -> object:
if isinstance(value, str):
return redact_internal_details(redact_structured_value(key, value))
if isinstance(value, dict):
return {
child_key: _redacted_client_field(child_key, child)
for child_key, child in _CLIENT_FIELDS.validate_python(value).items()
}
if isinstance(value, (list, tuple)):
return [_redacted_client_field(key, child) for child in _CLIENT_ITEMS.validate_python(value)]
if not isinstance(value, (int, float)):
return value
text: Final = str(value)
return value if redact_structured_value(key, text) == text else REDACTED
def redact_internal_details_from_client_fields(fields: Mapping[str, object]) -> object:
if not _ENABLE_SECRET_REDACTION:
return fields
return _redacted_client_field(None, dict(fields))
def _substituted_color_message(record: logging.LogRecord) -> str | None:
"""Render a record's ``color_message`` against its args, or None if absent.

View file

@ -8,10 +8,9 @@ The following caching mechanisms are supported:
2. **RedisSemanticCache**
3. **QdrantSemanticCache**
4. **InMemoryCache**
5. **DiskCache**
6. **S3Cache**
7. **AzureBlobCache**
8. **DualCache** (updates both Redis and an in-memory cache simultaneously)
5. **S3Cache**
6. **AzureBlobCache**
7. **DualCache** (updates both Redis and an in-memory cache simultaneously)
## Folder Structure
@ -20,7 +19,6 @@ litellm/caching/
├── base_cache.py
├── caching.py
├── caching_handler.py
├── disk_cache.py
├── dual_cache.py
├── in_memory_cache.py
├── qdrant_semantic_cache.py

View file

@ -1,6 +1,5 @@
from .azure_blob_cache import AzureBlobCache
from .caching import Cache, LiteLLMCacheType
from .disk_cache import DiskCache
from .dual_cache import DualCache
from .gcs_cache import GCSCache
from .in_memory_cache import InMemoryCache

View file

@ -57,7 +57,7 @@ def resolve_embedding_max_input_tokens(
return configured_max_input_tokens
if router is None:
return None
deployment_max_input_tokens, _ = router.get_configured_token_limits(embedding_model)
deployment_max_input_tokens, _ = router.registry().group(embedding_model, field="token_limits")
return deployment_max_input_tokens

View file

@ -31,7 +31,6 @@ from litellm.types.utils import EmbeddingResponse, is_litellm_owned_kwarg
from .azure_blob_cache import AzureBlobCache
from .base_cache import BaseCache
from .disk_cache import DiskCache
from .dual_cache import DualCache # noqa: F401 # re-exported, callers import DualCache from litellm.caching.caching
from .gcs_cache import GCSCache
from .in_memory_cache import InMemoryCache
@ -153,7 +152,7 @@ class Cache:
Initializes the cache based on the given type.
Args:
type (str, optional): The type of cache to initialize. Can be "local", "redis", "redis-semantic", "qdrant-semantic", "s3" or "disk". Defaults to "local".
type (str, optional): The type of cache to initialize. Can be "local", "redis", "redis-semantic", "qdrant-semantic" or "s3". Defaults to "local".
# Redis Cache Args
host (str, optional): The host address for the Redis cache. Required if type is "redis".
@ -174,9 +173,6 @@ class Cache:
semantic_cache_scope (str, optional): "key" isolates semantic-cache buckets per key/team/org. "end_user" additionally isolates per end user (falls back to the key scope when the request carries no end-user id). Defaults to "key".
max_messages (int, optional): Requests with more `messages` (or Responses API `input` items) than this are neither looked up nor stored, so long agent conversations never serve or create a cache entry. None disables the limit. Defaults to 4.
# Disk Cache Args
disk_cache_dir (str, optional): The directory for the disk cache. Defaults to None.
# S3 Cache Args
s3_bucket_name (str, optional): The bucket name for the s3 cache. Defaults to None.
s3_region_name (str, optional): The region name for the s3 cache. Defaults to None.
@ -306,8 +302,11 @@ class Cache:
account_url=azure_account_url,
container=azure_blob_container,
)
elif type == LiteLLMCacheType.DISK:
self.cache = DiskCache(disk_cache_dir=disk_cache_dir)
else:
supported_types: Final = ", ".join(member.value for member in LiteLLMCacheType)
raise ValueError(f"Unsupported cache type {type!r}. Supported types: {supported_types}")
if disk_cache_dir is not None:
verbose_logger.warning("Ignoring disk_cache_dir, the disk cache backend was removed")
if "cache" not in litellm.input_callback:
litellm.input_callback.append("cache")
if "cache" not in litellm.success_callback:
@ -1044,7 +1043,7 @@ def enable_cache(
Enable cache with the specified configuration.
Args:
type (Optional[Literal["local", "redis", "s3", "disk"]]): The type of cache to enable. Defaults to "local".
type (Optional[Literal["local", "redis", "s3"]]): The type of cache to enable. Defaults to "local".
host (Optional[str]): The host address of the cache server. Defaults to None.
port (Optional[str]): The port number of the cache server. Defaults to None.
password (Optional[str]): The password for the cache server. Defaults to None.
@ -1056,7 +1055,7 @@ def enable_cache(
None
Raises:
None
ValueError: If no cache is set yet and the cache type is unsupported.
"""
print_verbose("LiteLLM: Enabling Cache")
if "cache" not in litellm.input_callback:
@ -1091,7 +1090,7 @@ def update_cache(
Update the cache for LiteLLM.
Args:
type (Optional[Literal["local", "redis", "s3", "disk"]]): The type of cache. Defaults to "local".
type (Optional[Literal["local", "redis", "s3"]]): The type of cache. Defaults to "local".
host (Optional[str]): The host of the cache. Defaults to None.
port (Optional[str]): The port of the cache. Defaults to None.
password (Optional[str]): The password for the cache. Defaults to None.

View file

@ -9,7 +9,7 @@ This file is a wrapper around caching.py
This class is used to handle caching logic specific for LLM API requests (completion / embedding / text_completion / transcription etc)
It utilizes the (RedisCache, s3Cache, RedisSemanticCache, QdrantSemanticCache, InMemoryCache, DiskCache) based on what the user has setup
It utilizes the (RedisCache, s3Cache, RedisSemanticCache, QdrantSemanticCache, InMemoryCache) based on what the user has setup
In each method it will call the appropriate method from caching.py
"""
@ -830,6 +830,13 @@ class LLMCachingHandler:
if is_internal_call.get():
return
if logging_obj.defer_async_logging and not logging_obj.stream:
from litellm.utils import dispatch_success_logging
logging_obj.model_call_details["cache_hit"] = cache_hit # rebind-ok: capture cache status before deferring
dispatch_success_logging(logging_obj, cached_result, start_time, end_time, False, False)
return
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(

View file

@ -1,92 +0,0 @@
import json
from typing import TYPE_CHECKING, Any, Final
from pydantic import TypeAdapter
from .base_cache import BaseCache
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
Span = _Span
else:
Span = Any
_DECODED_DISK_VALUE: Final = TypeAdapter(object)
class DiskCache(BaseCache):
def __init__(self, disk_cache_dir: str | None = None):
try:
import diskcache as dc
except ModuleNotFoundError as e:
raise ModuleNotFoundError("Please install litellm with `litellm[caching]` to use disk caching.") from e
# if users don't provider one, use the default litellm cache
if disk_cache_dir is None:
self.disk_cache = dc.Cache(".litellm_cache")
else:
self.disk_cache = dc.Cache(disk_cache_dir)
def set_cache(self, key, value, **kwargs):
if "ttl" in kwargs:
self.disk_cache.set(key, value, expire=kwargs["ttl"])
else:
self.disk_cache.set(key, value)
async def async_set_cache(self, key, value, **kwargs):
self.set_cache(key=key, value=value, **kwargs)
async def async_set_cache_pipeline(self, cache_list, **kwargs):
for cache_key, cache_value in cache_list:
if "ttl" in kwargs:
self.set_cache(key=cache_key, value=cache_value, ttl=kwargs["ttl"])
else:
self.set_cache(key=cache_key, value=cache_value)
def get_cache(self, key, **kwargs):
original_cached_response: Final = self.disk_cache.get(key)
if original_cached_response:
try:
cached_response = _DECODED_DISK_VALUE.validate_python(json.loads(original_cached_response))
except Exception:
cached_response = original_cached_response
return cached_response
return None
def batch_get_cache(self, keys: list, **kwargs):
return_val: Final = []
for k in keys:
val = self.get_cache(key=k, **kwargs)
return_val.append(val)
return return_val
def increment_cache(self, key, value: int, **kwargs) -> int:
with self.disk_cache.transact():
cached_value: Final = self.get_cache(key=key)
init_value: Final = cached_value if isinstance(cached_value, int) else 0
new_value: Final = init_value + value
self.set_cache(key, new_value, **kwargs)
return new_value
async def async_get_cache(self, key, **kwargs):
return self.get_cache(key=key, **kwargs)
async def async_batch_get_cache(self, keys: list, **kwargs):
return_val: Final = []
for k in keys:
val = self.get_cache(key=k, **kwargs)
return_val.append(val)
return return_val
async def async_increment(self, key, value: int, **kwargs) -> int:
return self.increment_cache(key=key, value=value, **kwargs)
def flush_cache(self):
self.disk_cache.clear()
async def disconnect(self):
pass
def delete_cache(self, key):
self.disk_cache.pop(key)

View file

@ -29,6 +29,9 @@ it does not belong to. Queued clients are therefore bucketed by what it takes to
close them, and each bucket is ordered by deadline, so a reap walks the entries
that are due rather than the whole queue.
A bucket whose loop has been collected is dropped, and its clients are left to the
collector, as they are when the queue is full.
The queue holds its clients weakly, so waiting out a grace window never keeps
alive anything the collector would have reclaimed first.
"""
@ -36,6 +39,7 @@ alive anything the collector would have reclaimed first.
import asyncio
import contextlib
import inspect
import itertools
import threading
import time
import weakref
@ -44,6 +48,8 @@ from collections.abc import Awaitable, Callable, Iterator
from dataclasses import dataclass, replace
from typing import Final
import httpx
from litellm.constants import (
EVICTED_LLM_CLIENT_CLOSE_GRACE_SECONDS,
EVICTED_LLM_CLIENT_CLOSE_MAX_PENDING,
@ -53,6 +59,9 @@ _CLOSABLE_ANYWHERE: Final = "closable-anywhere"
_CLOSABLE_ON_ANY_LOOP: Final = "closable-on-any-loop"
_BucketKey = str | int
_LOOP_TOKENS: Final[weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, int]] = weakref.WeakKeyDictionary()
_LOOP_TOKEN_COUNTER: Final = itertools.count()
_LOOP_TOKEN_LOCK: Final = threading.Lock()
@dataclass(frozen=True, slots=True)
@ -63,12 +72,12 @@ class _PendingClose:
collector would otherwise have reclaimed first.
``needs_loop`` is set for a client whose close is a coroutine; those can only
be closed from the event loop they were evicted on, recorded in ``loop_id``.
be closed from the event loop they were evicted on, recorded in ``loop_token``.
A client that closes synchronously carries neither constraint.
"""
client_ref: "weakref.ref[object]"
loop_id: int | None
loop_token: int | None
needs_loop: bool
close_after: float
@ -77,16 +86,24 @@ def _bucket_key(pending: _PendingClose) -> _BucketKey:
"""Which reaps can close this entry: any at all, any running a loop, or one loop's."""
if not pending.needs_loop:
return _CLOSABLE_ANYWHERE
if pending.loop_id is None:
if pending.loop_token is None:
return _CLOSABLE_ON_ANY_LOOP
return pending.loop_id
return pending.loop_token
def _running_loop_id() -> int | None:
def running_loop_token() -> int | None:
try:
return id(asyncio.get_running_loop())
loop: Final = asyncio.get_running_loop()
except RuntimeError:
return None
try:
token: Final[int | None] = _LOOP_TOKENS.get(loop)
if token is not None:
return token
with _LOOP_TOKEN_LOCK:
return _LOOP_TOKENS.setdefault(loop, next(_LOOP_TOKEN_COUNTER))
except TypeError:
return id(loop)
def _close_function(client: object) -> Callable[[], object] | None:
@ -94,13 +111,24 @@ def _close_function(client: object) -> Callable[[], object] | None:
return close_fn
def _transport_of(client: object) -> object:
"""The httpx transport behind an SDK wrapper, a litellm handler, or a bare client."""
def _mounted_transports(holder: object) -> tuple[object, ...]:
mounts: Final[object] = getattr(holder, "_mounts", None)
if not isinstance(mounts, dict):
return ()
return tuple(
mounted
for mounted in mounts.values() # pyright: ignore[reportUnknownVariableType] # untyped httpx mount map
if isinstance(mounted, (httpx.AsyncBaseTransport, httpx.BaseTransport))
)
def _transports_of(client: object) -> tuple[object, ...]:
"""The httpx transports behind an SDK wrapper, a litellm handler, or a bare client, proxy mounts included."""
for holder in (getattr(client, "_client", None), getattr(client, "client", None), client):
transport: object = getattr(holder, "_transport", None)
if transport is not None:
return transport
return None
return (transport, *_mounted_transports(holder))
return ()
def _connection_is_idle(connection: object) -> bool:
@ -123,6 +151,17 @@ def _pool_has_busy_connection(transport: object) -> bool | None:
)
def _transport_in_flight(transport: object) -> bool:
spread_in_flight: Final[object] = getattr(transport, "has_request_in_flight", None)
if callable(spread_in_flight):
return bool(spread_in_flight())
pooled_busy: Final = _pool_has_busy_connection(transport)
if pooled_busy is not None:
return pooled_busy
session: Final[object] = getattr(transport, "client", None)
return bool(getattr(getattr(session, "connector", None), "_acquired", None))
def _has_connection_in_flight(client: object) -> bool:
"""Whether the client is servicing a request right now.
@ -138,12 +177,7 @@ def _has_connection_in_flight(client: object) -> bool:
try:
if getattr(getattr(client, "connection_pool", None), "_in_use_connections", None):
return True
transport: Final = _transport_of(client)
pooled_busy: Final = _pool_has_busy_connection(transport)
if pooled_busy is not None:
return pooled_busy
session: Final[object] = getattr(transport, "client", None)
return bool(getattr(getattr(session, "connector", None), "_acquired", None))
return any(_transport_in_flight(transport) for transport in _transports_of(client))
except Exception: # noqa: BLE001 - a client that cannot report its state is treated as idle
return False
@ -167,6 +201,8 @@ class EvictedClientCloser:
self._clock = clock
self._owned: weakref.WeakSet[object] = weakref.WeakSet()
self._buckets: dict[_BucketKey, deque[_PendingClose]] = {} # mutable-ok: deadline-ordered queues
self._dead_loop_tokens: deque[int] = deque() # mutable-ok: loop finalizers append without taking the queue lock
self._watched_loop_tokens: set[int] = set() # mutable-ok: tracks tokens until their dead buckets are discarded
self._pending_count = 0
self._queue_lock = threading.Lock() # the cache is reachable from every worker thread's loop
self._close_tasks: set[asyncio.Task[None]] = set() # mutable-ok: strong refs to running closes
@ -197,13 +233,19 @@ class EvictedClientCloser:
close_fn: Final = _close_function(client)
if close_fn is None:
return
loop_token: Final = running_loop_token()
needs_loop: Final = inspect.iscoroutinefunction(close_fn)
if self._pending_count >= self._max_pending:
return
self._discard_dead_loops()
if self._pending_count >= self._max_pending:
return
if needs_loop and loop_token is not None:
self._watch_loop(loop_token)
self._enqueue(
_PendingClose(
client_ref=weakref.ref(client),
loop_id=_running_loop_id(),
needs_loop=inspect.iscoroutinefunction(close_fn),
loop_token=loop_token,
needs_loop=needs_loop,
close_after=self._clock() + self._grace_seconds,
)
)
@ -217,7 +259,7 @@ class EvictedClientCloser:
if not self._pending_count:
return
now: Final = self._clock()
for pending in self._take_due(_running_loop_id(), now):
for pending in self._take_due(running_loop_token(), now):
client = pending.client_ref()
if client is None:
continue
@ -230,6 +272,29 @@ class EvictedClientCloser:
def pending_count(self) -> int:
return self._pending_count
def _watch_loop(self, loop_token: int) -> None:
if loop_token in self._watched_loop_tokens:
return
try:
weakref.finalize(asyncio.get_running_loop(), self._dead_loop_tokens.append, loop_token)
except TypeError:
return
self._watched_loop_tokens.add(loop_token)
def _discard_dead_loops(self) -> None:
with self._queue_lock:
self._discard_dead_loops_locked()
def _discard_dead_loops_locked(self) -> None:
while self._dead_loop_tokens:
self._discard_dead_loop_token_locked(self._dead_loop_tokens.popleft())
def _discard_dead_loop_token_locked(self, loop_token: int) -> None:
self._watched_loop_tokens.discard(loop_token)
bucket: Final = self._buckets.pop(loop_token, None)
if bucket is not None:
self._pending_count -= len(bucket)
def _enqueue(self, pending: _PendingClose) -> None:
"""Append to the entry's bucket, dropping any dead entries it queues behind.
@ -238,6 +303,7 @@ class EvictedClientCloser:
the front rather than having to be searched for.
"""
with self._queue_lock:
self._discard_dead_loops_locked()
bucket: Final = self._buckets.setdefault(_bucket_key(pending), deque())
while bucket and bucket[0].client_ref() is None:
bucket.popleft()
@ -245,9 +311,12 @@ class EvictedClientCloser:
bucket.append(pending)
self._pending_count += 1
def _take_due(self, loop_id: int | None, now: float) -> tuple[_PendingClose, ...]:
buckets = (_CLOSABLE_ANYWHERE,) if loop_id is None else (_CLOSABLE_ANYWHERE, _CLOSABLE_ON_ANY_LOOP, loop_id)
def _take_due(self, loop_token: int | None, now: float) -> tuple[_PendingClose, ...]:
buckets = (
(_CLOSABLE_ANYWHERE,) if loop_token is None else (_CLOSABLE_ANYWHERE, _CLOSABLE_ON_ANY_LOOP, loop_token)
)
with self._queue_lock:
self._discard_dead_loops_locked()
return tuple(pending for key in buckets for pending in self._drain_locked(key, now))
def _drain_locked(self, key: _BucketKey, now: float) -> Iterator[_PendingClose]:

View file

@ -2,10 +2,9 @@
Add the event loop to the cache key, to prevent event loop closed errors.
"""
import asyncio
from typing import Final
from .evicted_client_closer import EvictedClientCloser, default_evicted_client_closer
from .evicted_client_closer import EvictedClientCloser, default_evicted_client_closer, running_loop_token
from .in_memory_cache import InMemoryCache
@ -43,24 +42,22 @@ class LLMClientCache(InMemoryCache):
self.evicted_client_closer.schedule(evicted)
self.evicted_client_closer.reap()
def update_cache_key_with_event_loop(self, key):
def update_cache_key_with_event_loop(self, key: str) -> str:
"""
Add the event loop to the cache key, to prevent event loop closed errors.
If none, use the key as is.
"""
try:
event_loop: Final = asyncio.get_running_loop()
stringified_event_loop: Final = str(id(event_loop))
return f"{key}-{stringified_event_loop}"
except RuntimeError: # handle no current running event loop
return key
token: Final = running_loop_token()
return f"{key}-{token}" if token is not None else key
def set_cache(self, key: str, value: object, litellm_owned_client: bool = False, **kwargs):
def _scoped_key(self, key: str, loop_bound: bool) -> str:
return self.update_cache_key_with_event_loop(key) if loop_bound else key
def set_cache(self, key: str, value: object, litellm_owned_client: bool = False, loop_bound: bool = True, **kwargs):
"""``litellm_owned_client`` marks a client litellm built, so it may be closed once evicted."""
if litellm_owned_client:
self.evicted_client_closer.mark_owned(value)
key = self.update_cache_key_with_event_loop(key)
return super().set_cache(key, value, **kwargs)
return super().set_cache(self._scoped_key(key, loop_bound), value, **kwargs)
async def async_set_cache(self, key: str, value: object, litellm_owned_client: bool = False, **kwargs):
if litellm_owned_client:
@ -68,13 +65,13 @@ class LLMClientCache(InMemoryCache):
key = self.update_cache_key_with_event_loop(key)
return await super().async_set_cache(key, value, **kwargs)
def get_cache(self, key, **kwargs):
key = self.update_cache_key_with_event_loop(key)
def get_cache(self, key: str, loop_bound: bool = True, **kwargs):
scoped_key: Final = self._scoped_key(key, loop_bound)
self.evicted_client_closer.reap()
return super().get_cache(key, **kwargs)
return super().get_cache(scoped_key, **kwargs)
async def async_get_cache(self, key, **kwargs):
async def async_get_cache(self, key: str, **kwargs):
key = self.update_cache_key_with_event_loop(key)
return await super().async_get_cache(key, **kwargs)

View file

@ -1887,7 +1887,10 @@ def completion_cost(
video_resolution=video_resolution,
)
elif call_type in _SPEECH_CALL_TYPES:
prompt_characters = litellm.utils.count_characters(text=prompt)
prompt_characters = _speech_billed_characters(
completion_response if isinstance(completion_response, HttpxBinaryResponseContent) else None,
prompt,
)
elif call_type in _TRANSCRIPTION_CALL_TYPES:
# Check _hidden_params first (duration stored there to
# avoid polluting the response body), then fall back to
@ -2366,6 +2369,11 @@ def response_cost_calculator(
raise e
def _speech_billed_characters(speech_response: HttpxBinaryResponseContent | None, prompt: str) -> int:
reported: Final = speech_response.hidden_params.get("billed_characters") if speech_response is not None else None
return reported if isinstance(reported, int) else litellm.utils.count_characters(text=prompt)
def _video_input_image_cost(
model: str,
custom_llm_provider: str | None,

View file

@ -53,7 +53,7 @@ from litellm.main import (
from litellm.secret_managers.main import get_secret_str
from litellm.types.images.main import ImageEditOptionalRequestParams
from litellm.types.llms.openai import ImageGenerationRequestQuality
from litellm.types.router import GenericLiteLLMParams
from litellm.types.router import CustomPricingLiteLLMParams, GenericLiteLLMParams
from litellm.types.utils import (
LITELLM_IMAGE_VARIATION_PROVIDERS,
LlmProviders,
@ -1010,6 +1010,7 @@ def image_edit(
optional_params=dict(image_edit_request_params),
litellm_params={
**image_edit_request_params,
**litellm_params.model_dump(include=set(CustomPricingLiteLLMParams.model_fields), exclude_none=True),
"litellm_call_id": litellm_call_id,
"model_info": model_info,
"vertex_location": litellm_params.vertex_location,

View file

@ -349,7 +349,7 @@ class SlackAlerting(CustomBatchLogger):
except Exception:
return 0
async def send_daily_reports(self, router) -> bool:
async def send_daily_reports(self, router: Router) -> bool:
"""
Send a daily report on:
- Top 5 deployments with most failed requests
@ -365,7 +365,7 @@ class SlackAlerting(CustomBatchLogger):
False -> if not sent
"""
ids: Final = router.get_model_ids()
ids: Final = router.registry().names(kind="ids")
# get keys
failed_request_keys: Final = [f"{id}:{SlackAlertingCacheKeys.failed_requests_key.value}" for id in ids]
@ -430,7 +430,7 @@ class SlackAlerting(CustomBatchLogger):
message += "\tNone\n"
for i in range(len(top_5_failed)):
key = failed_request_keys[top_5_failed[i]].split(":")[0]
_deployment = router.get_model_info(key)
_deployment = router.registry().deployment_dict(key)
if isinstance(_deployment, dict):
deployment_name = _deployment["litellm_params"].get("model", "")
else:
@ -452,7 +452,7 @@ class SlackAlerting(CustomBatchLogger):
message += "\tNone\n"
for i in range(len(top_5_slowest)):
key = latency_keys[top_5_slowest[i]].split(":")[0]
_deployment = router.get_model_info(key)
_deployment = router.registry().deployment_dict(key)
if _deployment is not None:
deployment_name = _deployment["litellm_params"].get("model", "")
else:
@ -802,7 +802,7 @@ class SlackAlerting(CustomBatchLogger):
if self.llm_router is None:
return
deployment: Final = self.llm_router.get_deployment(model_id=deployment_id)
deployment: Final = self.llm_router.registry().deployment(deployment_id)
if deployment is None:
return
@ -953,7 +953,7 @@ class SlackAlerting(CustomBatchLogger):
return
### EXTRACT MODEL DETAILS ###
deployment: Final = self.llm_router.get_deployment(model_id=deployment_id)
deployment: Final = self.llm_router.registry().deployment(deployment_id)
if deployment is None:
return

View file

@ -102,18 +102,14 @@ class AgentOps(OpenTelemetry):
"Connection": "keep-alive",
}
client: Final = get_httpx_client()
try:
response: Final = client.post(
url=auth_endpoint,
headers=headers,
json={"api_key": api_key},
timeout=10,
)
response: Final = get_httpx_client().post(
url=auth_endpoint,
headers=headers,
json={"api_key": api_key},
timeout=10,
)
if response.status_code != 200:
raise Exception(f"Failed to fetch auth token: {response.text}")
if response.status_code != 200:
raise Exception(f"Failed to fetch auth token: {response.text}")
return _DECODED_JSON.validate_python(response.json())
finally:
client.close()
return _DECODED_JSON.validate_python(response.json())

View file

@ -35,6 +35,8 @@ from litellm.types.utils import StandardCallbackDynamicParams
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
from litellm.proxy._types import UserAPIKeyAuth
Span = _Span
else:
Span = Any
@ -126,6 +128,26 @@ def _starts_its_own_trace(span: object, kwargs: Mapping[str, object]) -> bool:
)
def _a_destination_records_the_call(kwargs: Mapping[str, object]) -> bool:
"""Whether a key's or team's ``langfuse_otel`` destination gets this call from the OTel v2 fan-out, which records
every call but one whose logging payload shows it never reached a provider."""
from litellm.integrations.otel.model.metadata import LLMCallEvent
if not _a_destination_carries_the_trace():
return False
call: Final = LLMCallEvent.from_dict(kwargs)
return call.payload is None or call.reached_a_provider
def _a_destination_carries_the_trace() -> bool:
"""Whether this request resolved a key's or team's ``langfuse_otel`` destination, whose fan-out sends it the whole
trace."""
from litellm.integrations.otel.model.config import ExporterOwner
from litellm.integrations.otel.plumbing.context import destination_backends
return ExporterOwner.LANGFUSE_OTEL in destination_backends()
class LangfuseOtelLogger(OpenTelemetry):
def __init__(self, config=None, *args, **kwargs):
# Prevent LangfuseOtelLogger from modifying global environment variables by constructing config manually
@ -541,6 +563,27 @@ class LangfuseOtelLogger(OpenTelemetry):
"""
return None
def _emit_once(
self,
kwargs: dict[str, object], # mutable-ok: the v1 dedupe marker is written into the request's kwargs
*scope: object,
) -> bool:
"""Whether to emit this span: once per request, and never for a call a Langfuse destination records."""
return not _a_destination_records_the_call(kwargs) and super()._emit_once(kwargs, *scope) # pyright: ignore[reportUnknownMemberType] # the v1 hook takes a bare dict
async def async_post_call_failure_hook(
self,
request_data: dict[str, object], # mutable-ok: the v1 hook contract
original_exception: Exception,
user_api_key_dict: "UserAPIKeyAuth",
traceback_str: str | None = None,
) -> None:
"""Record the proxy failure of a request no Langfuse destination carries."""
if not _a_destination_carries_the_trace():
await super().async_post_call_failure_hook( # pyright: ignore[reportUnknownMemberType] # the v1 hook takes a bare dict
request_data, original_exception, user_api_key_dict, traceback_str
)
async def async_service_success_hook(self, *args, **kwargs):
"""
Langfuse should not receive service success logs.

View file

@ -13,9 +13,9 @@ from opentelemetry.trace.status import Status, StatusCode
from litellm.integrations.otel.mappers import resolve_mappers
from litellm.integrations.otel.mappers.base import AttributeMapper, AttrValue, SpanData
from litellm.integrations.otel.mappers.langfuse import LANGFUSE_REQUEST_METADATA_ATTRIBUTES
from litellm.integrations.otel.mappers.langfuse import LANGFUSE_REQUEST_METADATA_ATTRIBUTES, LangfuseMapper
from litellm.integrations.otel.mappers.openinference import fit_indexed_messages
from litellm.integrations.otel.model.config import OpenTelemetryV2Config
from litellm.integrations.otel.model.config import ExporterOwner, OpenTelemetryV2Config
from litellm.integrations.otel.model.payloads import (
GuardrailSpanData,
LLMCallSpanData,
@ -34,6 +34,7 @@ from litellm.integrations.otel.model.spans import (
mcp_tool_call_span_name,
service_span_name,
)
from litellm.integrations.otel.plumbing.context import destination_backends
from litellm.integrations.otel.plumbing.events import GenAIEventRecorder
from litellm.integrations.otel.plumbing.providers import to_otel_span_kind
@ -168,6 +169,9 @@ class SpanEmitter:
self._mappers: list[AttributeMapper] = (
list(mappers) if mappers is not None else resolve_mappers(config.mapper_names)
)
self._langfuse: Final = (
None if any(isinstance(mapper, LangfuseMapper) for mapper in self._mappers) else LangfuseMapper()
)
# Bounded LRU (ordered by insertion / most-recent touch). Storing keys
# only — the value is unused — so it behaves like a capped set.
self._emitted: OrderedDict[tuple[str, SpanRole], None] = OrderedDict()
@ -295,17 +299,27 @@ class SpanEmitter:
)
else None
)
mapped: Final = MappingProxyType(
{key: value for mapper in self._mappers for key, value in mapper.map(data).items()}
configured: Final = {key: value for mapper in self._mappers for key, value in mapper.map(data).items()}
for_langfuse: Final = (
dict(
sorted(
self._langfuse.map(data).items(),
key=lambda item: item[0] not in LANGFUSE_REQUEST_METADATA_ATTRIBUTES,
)
)
if self._langfuse is not None and ExporterOwner.LANGFUSE_OTEL in destination_backends()
else _NO_ATTRIBUTES
)
mapped: Final = MappingProxyType({**configured, **for_langfuse})
stamped_later: Final = error_attributes(error) if error else _NO_ATTRIBUTES
reserved: Final = len(stamped_later.keys() - mapped.keys())
overwrites: Final = carried_keys(span) & mapped.keys()
# The Langfuse request metadata sheds first, so a span at its attribute limit loses nothing else to it.
# The Langfuse request metadata, and the Langfuse vocabulary only a destination asked for, shed first,
# so a span at its attribute limit loses nothing else to them.
fitted: Final = fit_indexed_messages(
MappingProxyType({key: value for key, value in mapped.items() if key not in overwrites}),
attribute_budget(span, reserved),
LANGFUSE_REQUEST_METADATA_ATTRIBUTES,
LANGFUSE_REQUEST_METADATA_ATTRIBUTES.union(for_langfuse),
)
for key, value in mapped.items():
if key in overwrites or key in fitted:

View file

@ -198,6 +198,7 @@ class OpenTelemetryV2(CustomLogger):
tracer_provider: TracerProvider | None = None,
logger_provider: LoggerProvider | None = None,
meter_provider: "MeterProvider | None" = None,
console_default: bool = False,
**kwargs: Unpack[_CustomLoggerOptions],
) -> None:
super().__init__(**kwargs)
@ -206,7 +207,7 @@ class OpenTelemetryV2(CustomLogger):
self._tracer_provider: TracerProvider = (
tracer_provider
if tracer_provider is not None
else build_tracer_provider(self.config, tenant_overrides=True)
else build_tracer_provider(self.config, tenant_overrides=True, console_default=console_default)
)
self.tracer: Tracer = get_tracer(self._tracer_provider, LITELLM_TRACER_NAME)
self._metrics_recorder = self._init_metrics(meter_provider)
@ -217,7 +218,9 @@ class OpenTelemetryV2(CustomLogger):
mappers=resolve_mappers(self.config.mapper_names),
event_recorder=self._init_events(logger_provider),
)
self._tenant_tracers = TenantTracerCache(self.config, callback_name, LITELLM_TRACER_NAME)
self._tenant_tracers = TenantTracerCache(
self.config, callback_name, LITELLM_TRACER_NAME, console_default=console_default
)
self._open_llm_calls: OrderedDict[str, _LLMCallSpan] = OrderedDict()
self._init_otel_logger_on_litellm_proxy()
@ -541,9 +544,7 @@ class OpenTelemetryV2(CustomLogger):
# gate rejection carries ``is_no_upstream_call`` and gets no span. A cache
# hit never hands off to a provider, so ``cache_hit`` substitutes for the
# upstream stamp, since the payload proves the call completed.
if carrier is None and (
call.is_no_upstream_call or call.payload is None or (not call.upstream_started and not call.cache_hit)
):
if carrier is None and not call.reached_a_provider:
return None
try:
return self._finish_carrier(carrier, call, start_time, end_time)
@ -1091,6 +1092,7 @@ def build_otel_v2_logger(
logger_provider: LoggerProvider | None = None,
meter_provider: "MeterProvider | None" = None,
settings: Mapping[str, object] = MappingProxyType({}),
console_default: bool = False,
) -> OpenTelemetryV2:
return _logger_class(config)(
config=config,
@ -1098,6 +1100,7 @@ def build_otel_v2_logger(
tracer_provider=tracer_provider,
logger_provider=logger_provider,
meter_provider=meter_provider,
console_default=console_default,
**settings,
)

View file

@ -103,6 +103,18 @@ class LangfuseMapper:
),
}
#: Every attribute this mapper can write on a model call.
LLM_CALL_ATTRIBUTES: Final[frozenset[str]] = frozenset(
{
*_LLM_CALL_ATTRS,
*_BLOB_ATTRS,
LANGFUSE_TRACE_NAME,
LANGFUSE_TRACE_USER_ID,
LANGFUSE_TRACE_SESSION_ID,
LANGFUSE_TRACE_TAGS,
}
)
def map(self, data: SpanData) -> AttributeMap:
match data:
case LLMCallSpanData():

View file

@ -121,6 +121,17 @@ class ExporterSpec(LiteLLMBaseModel):
)
def is_unconfigured_placeholder(spec: ExporterSpec) -> bool:
"""Whether ``spec`` is the one ``_normalize`` folds in when nothing was configured.
No field set is what says the operator asked for nothing: an exporter they did
configure survives, even ``OTEL_EXPORTER=console`` whose value matches the default,
and so does the gated spec ``credential_gated_exporters`` appends, which would
otherwise eat itself when one preset layers onto another.
"""
return not spec.model_fields_set
class _EnvWithoutBareExcludedServices(PydanticBaseSettingsSource):
def __init__(self, settings_cls: type[BaseSettings], env_settings: PydanticBaseSettingsSource) -> None:
super().__init__(settings_cls)

View file

@ -272,6 +272,11 @@ class LLMCallEvent:
trace: TraceControls
session_id: str | None
@property
def reached_a_provider(self) -> bool:
"""Whether the call was handed to a provider or answered from the response cache."""
return not self.is_no_upstream_call and self.payload is not None and (self.upstream_started or self.cache_hit)
@classmethod
def from_dict(cls, kwargs: Mapping[str, object]) -> LLMCallEvent:
raw_payload: Final = kwargs.get("standard_logging_object")

View file

@ -47,6 +47,7 @@ from litellm.integrations.otel.mappers.langfuse import (
LANGFUSE_OBSERVATION_OUTPUT,
LANGFUSE_REQUEST_METADATA_ATTRIBUTES,
LANGFUSE_TRACE_NAME,
LangfuseMapper,
)
from litellm.integrations.otel.model.config import (
CaptureMessageContent,
@ -55,6 +56,7 @@ from litellm.integrations.otel.model.config import (
OpenTelemetryV2Config,
capture_message_content_from_env,
captures_span_content,
is_unconfigured_placeholder,
)
from litellm.integrations.otel.model.semconv import (
DB,
@ -490,19 +492,25 @@ def _is_litellm_span(span: ReadableSpan) -> bool:
return scope is not None and scope.name == LITELLM_TRACER_NAME
def without_langfuse_request_metadata(span: ReadableSpan) -> ReadableSpan:
"""``span`` without the request metadata the Langfuse mapper writes for a Langfuse destination.
def without_destination_langfuse_attributes(span: ReadableSpan, langfuse_mapped: bool) -> ReadableSpan:
"""``span`` without the Langfuse attributes only a Langfuse destination asked for.
Only LiteLLM's own spans lose it: any other span keeps every key its own code set.
That is the request metadata the Langfuse mapper writes and, on a model call written by
a logger whose own mappers leave the Langfuse vocabulary out (``langfuse_mapped`` off),
every Langfuse attribute, since that logger wrote them for a key's or team's Langfuse
destination alone. Only LiteLLM's own spans lose them: any other span keeps every key
its own code set.
"""
attributes: Final = span.attributes or _NO_ATTRIBUTES
if not (_is_litellm_span(span) and any(key in LANGFUSE_REQUEST_METADATA_ATTRIBUTES for key in attributes)):
dropped: Final = (
LANGFUSE_REQUEST_METADATA_ATTRIBUTES
if langfuse_mapped or not is_llm_call_span(span)
else LangfuseMapper.LLM_CALL_ATTRIBUTES
)
if not (_is_litellm_span(span) and any(key in dropped for key in attributes)):
return span
return _CountedSpanView(
span,
MappingProxyType(
{key: value for key, value in attributes.items() if key not in LANGFUSE_REQUEST_METADATA_ATTRIBUTES}
),
span, MappingProxyType({key: value for key, value in attributes.items() if key not in dropped})
)
@ -658,6 +666,7 @@ def _for_destination(
span: ReadableSpan,
destination: "OtelDestination",
default_capture: str,
langfuse_mapped: bool,
) -> ReadableSpan:
"""The view of ``span`` a tenant destination receives.
@ -672,13 +681,16 @@ def _for_destination(
and the query string of the request URL, which can hold the same key. A
destination whose effective setting disables span content loses the prompt and
response content too, and a destination for any backend but Langfuse loses the
request metadata the Langfuse mapper writes on LiteLLM's own spans. The span itself
stays, so the tenant still gets the whole trace tree.
Langfuse attributes only a Langfuse destination asked for (see
:func:`without_destination_langfuse_attributes`). The span itself stays, so the
tenant still gets the whole trace tree.
"""
extra: Final = destination.resource_attributes
defaults: Final = destination.resource_defaults
source: Final = (
span if destination.callback_name == ExporterOwner.LANGFUSE_OTEL else without_langfuse_request_metadata(span)
span
if destination.callback_name == ExporterOwner.LANGFUSE_OTEL
else without_destination_langfuse_attributes(span, langfuse_mapped)
)
attributes: Final = source.attributes or _NO_ATTRIBUTES
database: Final = _is_database_span(attributes)
@ -728,8 +740,10 @@ class TenantFanOutSpanProcessor(SpanProcessor):
default_capture_by_backend: Mapping[str, str] = MappingProxyType({}),
pending_drains: int = _MAX_PENDING_DRAINS,
drain_pool: _DrainPool | None = None,
langfuse_mapped: bool = True,
) -> None:
self._operator_sinks: Final = operator_sinks
self._langfuse_mapped: Final = langfuse_mapped
self._excluded_db_systems: Final = excluded_db_systems
self._default_capture: Final = default_capture
self._default_capture_by_backend: Final = default_capture_by_backend
@ -822,7 +836,8 @@ class TenantFanOutSpanProcessor(SpanProcessor):
try:
processor.on_end(
_scoped(
_for_destination(span, destination, self._default_capture_for(destination)), destination.span_scope
_for_destination(span, destination, self._default_capture_for(destination), self._langfuse_mapped),
destination.span_scope,
)
)
except Exception as exc: # noqa: BLE001 # one destination's failure must not cost the others their span
@ -1198,17 +1213,19 @@ class _OverriddenBackendFilter(SpanProcessor):
return self._inner.force_flush(timeout_millis)
class _LangfuseRequestMetadataFilter(SpanProcessor):
"""Strip the Langfuse request metadata from LiteLLM's own spans before ``inner`` exports them."""
class _DestinationLangfuseAttributeFilter(SpanProcessor):
"""Strip the Langfuse attributes only a Langfuse destination asked for from LiteLLM's own spans before ``inner``
exports them."""
def __init__(self, inner: SpanProcessor) -> None:
def __init__(self, inner: SpanProcessor, langfuse_mapped: bool) -> None:
self._inner: Final = inner
self._langfuse_mapped: Final = langfuse_mapped
def on_start(self, span: SDKSpan, parent_context: Context | None = None) -> None:
self._inner.on_start(span, parent_context)
def on_end(self, span: ReadableSpan) -> None:
self._inner.on_end(without_langfuse_request_metadata(span))
self._inner.on_end(without_destination_langfuse_attributes(span, self._langfuse_mapped))
def shutdown(self) -> None:
self._inner.shutdown()
@ -1491,13 +1508,18 @@ def _spec_processor(
spec: ExporterSpec,
use_simple_processor: bool | None,
strip_message_content: bool,
strip_langfuse_request_metadata: bool,
strip_destination_langfuse_attributes: bool,
langfuse_mapped: bool,
) -> SpanProcessor:
exporting: Final = _processor_for(
_exporter_from_spec(spec),
(spec.use_simple_processor if spec.use_simple_processor is not None else use_simple_processor),
)
scoped: Final = _LangfuseRequestMetadataFilter(exporting) if strip_langfuse_request_metadata else exporting
scoped: Final = (
_DestinationLangfuseAttributeFilter(exporting, langfuse_mapped)
if strip_destination_langfuse_attributes
else exporting
)
return _MessageContentFilter(scoped) if strip_message_content else scoped
@ -1530,6 +1552,7 @@ def build_tracer_provider(
use_simple_processor: bool | None = None,
tenant_overrides: bool = False,
strip_message_content: bool = False,
console_default: bool = False,
) -> TracerProvider:
"""Build the shared :class:`TracerProvider`.
@ -1548,11 +1571,16 @@ def build_tracer_provider(
``config.langfuse_span_scope`` narrows the exporter owned by ``langfuse_otel``
alone; a collector or any other backend in the same config keeps the full tree.
In a config that writes the Langfuse vocabulary, every exporter but ``langfuse_otel``'s
goes without the request metadata the Langfuse mapper writes on LiteLLM's own spans.
Every exporter but ``langfuse_otel``'s goes without the Langfuse attributes only a
Langfuse destination asked for (see :func:`without_destination_langfuse_attributes`),
which any logger writes for a request with a Langfuse destination.
``strip_message_content`` removes prompt and response content before every
exporter, for a routed provider whose spans can carry content another
destination asked for.
The console spec ``config._normalize`` folds in when nothing was configured
prints only under ``console_default``, which the ``otel`` callback passes: a
vendor preset or the proxy's background global logger was never asked to print.
"""
provider: Final = TracerProvider(resource=build_resource(config))
provider.sampler = _RequestExclusionSampler(provider.sampler)
@ -1564,19 +1592,20 @@ def build_tracer_provider(
provider.add_span_processor(_processor_for(exporter, use_simple_processor))
return provider
langfuse_mapped: Final = _maps_langfuse(config)
# ``config._normalize`` guarantees at least one spec (it folds the top-level
# ``exporter``/``endpoint``/``headers`` fields in when ``exporters`` is empty).
writes_langfuse: Final = "langfuse" in config.mapper_names or any(
spec.owner is ExporterOwner.LANGFUSE_OTEL for spec in config.exporters
)
for spec in config.exporters:
if spec.requires_headers and not spec.headers:
continue
if is_unconfigured_placeholder(spec) and not console_default:
continue
processor = _spec_processor(
spec,
use_simple_processor,
strip_message_content,
strip_langfuse_request_metadata=writes_langfuse and spec.owner is not ExporterOwner.LANGFUSE_OTEL,
strip_destination_langfuse_attributes=spec.owner is not ExporterOwner.LANGFUSE_OTEL,
langfuse_mapped=langfuse_mapped,
)
owner = spec.owner.value if tenant_overrides and spec.owner is not None else None
scope = _operator_scope(config, spec)
@ -1613,8 +1642,9 @@ def attach_tenant_fan_out(
so exactly one fan-out lands. ``configs`` name the operator's own exporters, one
config per v2 logger since each keeps its own provider and still writes its
account, so an additive destination pointing at any of them is delivered once
rather than twice. ``excluded_db_systems`` only filters what the fan-out
delivers, never the operator's own exporters.
rather than twice. The first is the config of the logger that owns ``provider``,
whose model calls the fan-out carries. ``excluded_db_systems`` only filters what
the fan-out delivers, never the operator's own exporters.
"""
with _FAN_OUT_ATTACH_LOCK:
if any(isinstance(processor, TenantFanOutSpanProcessor) for processor in _attached_processors(provider)):
@ -1625,10 +1655,16 @@ def attach_tenant_fan_out(
excluded_db_systems=excluded_db_systems,
default_capture=_operator_capture_mode(configs),
default_capture_by_backend=MappingProxyType(dict(_owned_capture_modes(configs))),
langfuse_mapped=any(_maps_langfuse(config) for config in configs[:1]),
)
)
def _maps_langfuse(config: OpenTelemetryV2Config) -> bool:
"""Whether a logger built from ``config`` writes the Langfuse vocabulary on every model call."""
return "langfuse" in config.mapper_names
def _operator_capture_mode(configs: Sequence[OpenTelemetryV2Config]) -> str:
"""The mode of the operator's generic ``otel`` callback, so ``callback_settings.otel`` counts as the global default"""
return next(

View file

@ -163,10 +163,12 @@ class TenantTracerCache:
config: OpenTelemetryV2Config,
callback_name: str | None,
tracer_name: str,
console_default: bool = False,
) -> None:
self._config = config
self._callback_name = callback_name
self._tracer_name = tracer_name
self._console_default = console_default
# Guards the three mutable structures below: ``pre_call`` can run on
# thread-pool workers concurrently with the event loop, so cache
# updates, span counts, and retirement must be atomic.
@ -284,6 +286,7 @@ class TenantTracerCache:
built: Final = build_tracer_provider(
self._routed_config(credential_headers, project_headers, endpoint, service_name),
strip_message_content=not self._config.capture_span_content,
console_default=self._console_default,
)
self._providers[cache_key] = built
return built

View file

@ -111,7 +111,7 @@ _REQUIRED_HEADERS_BY_CALLBACK: Final[Mapping[str, frozenset[str]]] = MappingProx
_NO_ATTRS: Final[Mapping[str, str]] = MappingProxyType({})
def _arize_project() -> Mapping[str, str]:
def _arize_project(_params: StandardCallbackDynamicParams) -> Mapping[str, str]:
from litellm.integrations.arize.arize import ArizeLogger
project: Final = ArizeLogger.get_arize_config().project_name
@ -124,10 +124,27 @@ def _arize_default_project() -> Mapping[str, str]:
return MappingProxyType({"model_id": ARIZE_DEFAULT_PROJECT})
#: Resource attributes the operator chose for a backend, applied to every export the
#: team receives over the span's own resource: ``ARIZE_PROJECT_NAME`` for Arize.
_RESOURCE_ATTRIBUTES_BY_CALLBACK: Final[Mapping[str, Callable[[], Mapping[str, str]]]] = MappingProxyType(
{"arize": _arize_project}
def _stands_in_for_the_legacy_langfuse_logger() -> bool:
"""Whether a Langfuse destination replaces the older Langfuse logger's copy, which a proxy without Langfuse keys
of its own sends for a key's or team's ``langfuse_otel``."""
return not (os.environ.get("LANGFUSE_PUBLIC_KEY") and os.environ.get("LANGFUSE_SECRET_KEY"))
def _langfuse_environment(params: StandardCallbackDynamicParams) -> Mapping[str, str]:
"""The key's or team's ``langfuse_environment``, else ``LANGFUSE_TRACING_ENVIRONMENT``, where the older Langfuse
logger wrote it."""
from litellm.types.integrations.langfuse_otel import LangfuseSpanAttributes
if not _stands_in_for_the_legacy_langfuse_logger():
return _NO_ATTRS
environment: Final = params.get("langfuse_environment") or os.environ.get("LANGFUSE_TRACING_ENVIRONMENT")
return MappingProxyType({LangfuseSpanAttributes.LANGFUSE_ENVIRONMENT.value: environment} if environment else {})
#: Resource attributes a backend gets on every export the team receives, over the
#: span's own resource: ``ARIZE_PROJECT_NAME`` for Arize, the Langfuse environment.
_RESOURCE_ATTRIBUTES_BY_CALLBACK: Final[Mapping[str, Callable[[StandardCallbackDynamicParams], Mapping[str, str]]]] = (
MappingProxyType({"arize": _arize_project, "langfuse_otel": _langfuse_environment})
)
#: Resource attributes a backend needs on every export to accept it, filled only where
@ -137,12 +154,14 @@ _RESOURCE_DEFAULTS_BY_CALLBACK: Final[Mapping[str, Callable[[], Mapping[str, str
)
def _resource_attributes(callback_name: str, service_name: str | None) -> Mapping[str, str]:
def _resource_attributes(
callback_name: str, service_name: str | None, params: StandardCallbackDynamicParams
) -> Mapping[str, str]:
backend: Final = _RESOURCE_ATTRIBUTES_BY_CALLBACK.get(callback_name)
return MappingProxyType(
{
**({"service.name": service_name} if service_name else {}),
**(backend() if backend is not None else {}),
**(backend(params) if backend is not None else {}),
}
)
@ -214,7 +233,7 @@ def destination_for(
return OtelDestination(
endpoint=endpoint,
headers=MappingProxyType(dict(headers)),
resource_attributes=_resource_attributes(callback_name, service_name),
resource_attributes=_resource_attributes(callback_name, service_name, params),
resource_defaults=_resource_defaults(callback_name),
callback_name=callback_name,
protocol=protocol,

View file

@ -3,7 +3,7 @@
from collections.abc import Iterable
from typing import Final
from litellm.integrations.otel.model.config import ExporterOwner, ExporterSpec
from litellm.integrations.otel.model.config import ExporterOwner, ExporterSpec, is_unconfigured_placeholder
def ensure_mappers(mapper_names: Iterable[str], *names: str) -> list[str]:
@ -35,14 +35,3 @@ def credential_gated_exporters(
*(spec for spec in exporters if not is_unconfigured_placeholder(spec)),
ExporterSpec(owner=owner, requires_headers=True),
)
def is_unconfigured_placeholder(spec: "ExporterSpec") -> bool:
"""Whether ``spec`` is the one ``_normalize`` folds in when nothing was configured.
No field set is what says the operator asked for nothing: an exporter they did
configure survives, even ``OTEL_EXPORTER=console`` whose value matches the default,
and so does the gated spec this module appends, which would otherwise eat itself
when one preset layers onto another.
"""
return not spec.model_fields_set

View file

@ -468,15 +468,16 @@ def _bounded_requested_model_label(requested_model: object, router_originated: b
llm_router: Final = _get_proxy_llm_router()
if llm_router is None:
return requested_model if router_originated else UNRECOGNIZED_REQUESTED_MODEL_LABEL
if llm_router.is_recognized_model(requested_model):
registry: Final = llm_router.registry()
if registry.resolve(requested_model, kind="recognized"):
return requested_model
if requested_model in llm_router.team_public_model_names:
if requested_model in registry.names(kind="team_public"):
return requested_model
if llm_router.pattern_router.route(requested_model) is not None:
if registry.patterns().route(requested_model) is not None:
return requested_model
if any(
team_pattern_router.route(requested_model) is not None
for team_pattern_router in llm_router.team_pattern_routers.values()
for team_pattern_router in registry.patterns(team=True).values()
):
return requested_model
return UNRECOGNIZED_REQUESTED_MODEL_LABEL
@ -3410,7 +3411,7 @@ class PrometheusLogger(CustomLogger):
OpenAI / Anthropic / Azure return remaining tokens/requests in response
headers, but Bedrock and Vertex AI do not. This fallback computes
``configured_limit - current_usage`` via
``Router.get_remaining_model_group_usage`` so the same gauges are
``Router.health().usage(model_group, kind="remaining")`` so the same gauges are
emitted for every provider when tpm/rpm is configured on the
deployment.
"""
@ -3437,7 +3438,7 @@ class PrometheusLogger(CustomLogger):
return
try:
remaining_usage: Final = await llm_router.get_remaining_model_group_usage(model_group)
remaining_usage: Final = await llm_router.health().usage(model_group, kind="remaining")
except Exception as e:
verbose_logger.exception(
"Prometheus: get_remaining_model_group_usage failed for model_group=%s: %s",

View file

@ -40,7 +40,6 @@ from litellm.litellm_core_utils.llm_judge import (
)
from litellm.litellm_core_utils.redact_messages import should_redact_message_logging
from litellm.llms.base_llm.base_utils import type_to_response_format_param
from litellm.router_utils.common_utils import resolve_model_group_alias
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.management_endpoints.auto_router_endpoints import ShadowEvalDirection
from litellm.types.utils import SHADOW_EVAL_JUDGE_CALL_ORIGIN, SHADOW_EVAL_ROUTER_CALL_ORIGIN
@ -790,9 +789,7 @@ class ActiveShadowEvalJob(LiteLLMBaseModel):
def _canonical_group(router: "Router | None", model_group: str) -> str:
"""A model group in the one spelling both a job's scope and a request's model compare
under: an alias resolves to its target so the two never fail to match on spelling."""
return (
resolve_model_group_alias(router.model_group_alias, model_group) if router is not None else None
) or model_group
return (router.registry().resolve(model_group, kind="alias") if router is not None else None) or model_group
def _scope_admits(router: "Router | None", job: "ActiveShadowEvalJob", model_group: str) -> bool:

View file

@ -80,6 +80,7 @@ if TYPE_CHECKING:
from litellm.llms.base_llm.anthropic_messages.transformation import (
BaseAnthropicMessagesConfig,
)
from litellm.router import Router
from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,
)
@ -133,7 +134,7 @@ class _SearchToolLitellmParams(TypedDict, total=False):
class _SearchToolConfig(TypedDict, total=False):
search_tool_name: str
search_tool_name: ReadOnly[str]
litellm_params: ReadOnly[_SearchToolLitellmParams | None]
@ -1772,10 +1773,13 @@ class WebSearchInterceptionLogger(CustomLogger):
return None
def _select_search_tool_from_router(self, llm_router: object) -> "_SearchToolConfig | None":
if llm_router is None or not hasattr(llm_router, "search_tools"):
def _select_search_tool_from_router(self, llm_router: "Router | None") -> "_SearchToolConfig | None":
if llm_router is None:
return None
search_tools: Final = tuple(getattr(llm_router, "search_tools", None) or ())
settings: Final = llm_router.settings()
if not settings.is_set("search_tools"):
return None
search_tools: Final = tuple(settings.search_tools or ())
return self._select_search_tool_from_list(search_tools=search_tools, source="router")
def _select_search_tool_from_list(

View file

@ -1,6 +1,7 @@
# What is this?
## Common Utility file for Logging handler
# Logging function -> log the exact model details + what's being sent | Non-Blocking
import asyncio
import copy
import datetime
import functools
@ -96,6 +97,7 @@ from litellm.litellm_core_utils.internal_call_metadata import (
MODEL_ACCESS_GROUP_METADATA_KEY,
is_unbilled_non_inference_call,
)
from litellm.litellm_core_utils.llm_cost_calc.finalization import FinalizedCost, PreparedCost
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import (
bedrock_guardrail_trace,
cost_breakdown_with_guardrail,
@ -813,6 +815,11 @@ class Logging(LiteLLMLoggingBaseClass):
# Initialize cost breakdown field
self.cost_breakdown: CostBreakdown | None = None
self.finalized_cost: FinalizedCost | None = None
self.prepared_cost: PreparedCost | None = None
self.cost_allowance_discount: float | None = None
self.cost_finalization_lock = asyncio.Lock()
self.deferred_success_result: tuple[object, datetime.datetime, datetime.datetime] | None = None
self.billed_token_rates: BilledTokenRates | None = None
self._provider_reported_cost: float | None = None
@ -874,6 +881,11 @@ class Logging(LiteLLMLoggingBaseClass):
"current_attempt_api_call_start_time": None,
"litellm_params": litellm_params,
}
new_call.finalized_cost = None
new_call.prepared_cost = None
new_call.cost_allowance_discount = None
new_call.cost_finalization_lock = asyncio.Lock()
new_call.deferred_success_result = None
new_call.streaming_chunks = []
new_call.sync_streaming_chunks = []
return new_call
@ -2462,6 +2474,55 @@ class Logging(LiteLLMLoggingBaseClass):
**kwargs,
)
def prepare_response_cost(self, response: object, now: datetime.datetime) -> PreparedCost | None:
if self.prepared_cost is not None:
return self.prepared_cost
recorded_cache_hit: Final[object] = self.model_call_details.get("cache_hit")
deferred: Final = self.deferred_success_result
prepared: Final = self.prepare_success_logging(
result=deferred[0] if deferred is not None else response,
start_time=deferred[1] if deferred is not None else self.start_time,
end_time=deferred[2] if deferred is not None else now.replace(tzinfo=None),
cache_hit=recorded_cache_hit if isinstance(recorded_cache_hit, bool) else None,
standard_logging_object=None,
build_logging_payload=True,
publish_logging_payload=False,
)
payload: Final = prepared[3]
raw_cost: Final[object] = self.model_call_details.get("response_cost")
if payload is None or not isinstance(raw_cost, (int, float)):
return None
recorded_usage: Final[object] = self.model_call_details.get("combined_usage_object")
response_usage: Final[object] = getattr(prepared[2], "usage", None)
normalized_usage: Final = (
recorded_usage
if isinstance(recorded_usage, Usage)
else response_usage
if isinstance(response_usage, Usage)
else None
)
usage: Final = StandardLoggingPayloadSetup.get_usage_from_response_obj(
_extract_response_obj_and_hidden_params(prepared[2], None)[0],
combined_usage_object=normalized_usage,
).model_copy(deep=True)
self.prepared_cost = PreparedCost(
raw_cost,
{
**payload,
"response": None,
"messages": None,
"model_map_information": copy.deepcopy(payload["model_map_information"]),
"cost_breakdown": copy.deepcopy(payload["cost_breakdown"]),
},
usage,
)
self.record_partial_usage_for_failure(usage, raw_cost)
return self.prepared_cost
def record_finalized_cost(self, finalized: FinalizedCost) -> None:
self.finalized_cost = finalized
self.model_call_details["response_cost"] = finalized.amount
def record_partial_usage_for_failure(self, usage: Usage, response_cost: float) -> None:
"""Stash what an interrupted stream already consumed so the failure log bills it instead of zero."""
self.model_call_details["combined_usage_object"] = usage
@ -2812,12 +2873,16 @@ class Logging(LiteLLMLoggingBaseClass):
logging_result, "_hidden_params", {}
)
if self.model_call_details.get("cache_hit") is True:
if self.finalized_cost is not None:
self.model_call_details["response_cost"] = self.finalized_cost.amount
elif self.model_call_details.get("cache_hit") is True:
self.model_call_details["response_cost"] = 0.0
elif "response_cost" in hidden_params:
self.model_call_details["response_cost"] = hidden_params["response_cost"]
self._record_zero_cost_diagnostic(logging_result, hidden_params["response_cost"])
elif (existing_cost := self.model_call_details.get("response_cost")) is not None and existing_cost != 0:
elif (existing_cost := self.model_call_details.get("response_cost")) is not None and (
existing_cost != 0 or self.call_type == CallTypes.pass_through.value # pyright: ignore[reportUnknownMemberType] # Logging.call_type is untyped
):
# Preserve response_cost if already calculated (e.g., by pass-through
# handlers like Gemini/Vertex which call completion_cost directly).
# Do not preserve 0 from failure_handler on intermediate router retries.
@ -2936,7 +3001,7 @@ class Logging(LiteLLMLoggingBaseClass):
standard_logging_object: StandardLoggingPayload | None = None,
build_logging_payload: bool = True,
) -> tuple[datetime.datetime, datetime.datetime, object]:
prepared: Final = self._prepare_success_logging(
prepared: Final = self.prepare_success_logging(
result=result,
start_time=start_time,
end_time=end_time,
@ -2947,7 +3012,7 @@ class Logging(LiteLLMLoggingBaseClass):
)
return prepared[0], prepared[1], prepared[2]
def _prepare_success_logging(
def prepare_success_logging(
self,
result: object,
start_time: datetime.datetime | None,
@ -2968,7 +3033,11 @@ class Logging(LiteLLMLoggingBaseClass):
self.model_call_details["log_event_type"] = "successful_api_call"
self.model_call_details["end_time"] = end_time
self.model_call_details["cache_hit"] = cache_hit
self.model_call_details["cache_hit"] = (
self.prepared_cost.payload["cache_hit"]
if cache_hit is None and self.prepared_cost is not None
else cache_hit
)
if self.call_type == CallTypes.anthropic_messages.value:
result = self._anthropic_messages_logged_response(result=result)
@ -3007,6 +3076,7 @@ class Logging(LiteLLMLoggingBaseClass):
if (
litellm.max_budget
and publish_logging_payload
and self.stream is False
and result is not None
and isinstance(result, dict)
@ -3037,10 +3107,17 @@ class Logging(LiteLLMLoggingBaseClass):
publish_logging_payload: bool,
) -> StandardLoggingPayload | None:
if standard_logging_object is not None:
effective_payload: Final = (
self.finalized_cost.logging_payload(standard_logging_object)
if self.finalized_cost is not None
else standard_logging_object
)
if self.finalized_cost is not None:
self.model_call_details["response_cost"] = self.finalized_cost.amount
if not publish_logging_payload:
return standard_logging_object.copy()
self.model_call_details["standard_logging_object"] = standard_logging_object
return standard_logging_object
return effective_payload.copy()
self.model_call_details["standard_logging_object"] = effective_payload
return effective_payload
if result is None or self.stream is True:
# Streaming reaches here before its cost is known, so the cost
# is seeded to None, but only when nothing has already
@ -3281,7 +3358,7 @@ class Logging(LiteLLMLoggingBaseClass):
verbose_logger.debug("Logging Details LiteLLM-Success Call: Cache_hit=%s", cache_hit)
if not self.should_run_logging(event_type="sync_success"): # prevent double logging
return
start_time, end_time, result, request_logging_payload = self._prepare_success_logging(
start_time, end_time, result, request_logging_payload = self.prepare_success_logging(
start_time=start_time,
end_time=end_time,
result=result,
@ -4204,7 +4281,9 @@ class Logging(LiteLLMLoggingBaseClass):
# chunks already delivered; the router stashes that recovered usage as
# ``combined_usage_object`` and pre-computes its cost, so preserve it
# here instead of zeroing the spend on an otherwise-failed request.
if self.model_call_details.get("combined_usage_object") is None:
if self.finalized_cost is not None:
self.model_call_details["response_cost"] = self.finalized_cost.amount
elif self.model_call_details.get("combined_usage_object") is None:
self.model_call_details["response_cost"] = 0
if hasattr(exception, "headers") and isinstance(exception.headers, dict):
@ -5544,7 +5623,7 @@ def build_custom_logger_compatible_class(
return callback
otel_settings: Final = _get_custom_logger_settings_from_proxy_server(callback_name=logging_integration)
otel_logger_v2: Final = build_otel_v2_logger(
config=OpenTelemetryV2Config(**otel_settings), settings=otel_settings
config=OpenTelemetryV2Config(**otel_settings), settings=otel_settings, console_default=True
)
_in_memory_loggers.append(otel_logger_v2)
_maybe_auto_initialize_arize_phoenix(_in_memory_loggers)
@ -7367,6 +7446,12 @@ def _cache_hit_cost_breakdown(served: CostBreakdown | None) -> CostBreakdown:
)
def _finalized_cost_of(logging_obj: object) -> FinalizedCost | None:
"""Duck-typed logging objects (and test doubles) carry no finalized cost."""
finalized: Final = getattr(logging_obj, "finalized_cost", None)
return finalized if isinstance(finalized, FinalizedCost) else None
def get_standard_logging_object_payload(
kwargs: dict | None,
init_response_obj: object,
@ -7470,7 +7555,10 @@ def get_standard_logging_object_payload(
raw_response_cost: Final = kwargs.get("response_cost")
llm_response_cost: Final[float] = raw_response_cost or 0.0
guardrail_cost: Final = guardrail_information_cost(metadata.get("standard_logging_guardrail_information"))
response_cost: Final[float] = llm_response_cost + guardrail_cost
finalized_cost: Final = _finalized_cost_of(logging_obj)
response_cost: Final[float] = (
finalized_cost.amount if finalized_cost is not None else llm_response_cost + guardrail_cost
)
# clean up litellm hidden params
clean_hidden_params: Final = StandardLoggingPayloadSetup.get_hidden_params(hidden_params)
@ -7537,13 +7625,17 @@ def get_standard_logging_object_payload(
):
model_name = response_model_name
request_cost_breakdown: Final = cost_breakdown_with_guardrail(
(
_cache_hit_cost_breakdown(uncached_price.cost_breakdown)
if uncached_price is not None
else logging_obj.cost_breakdown
),
guardrail_cost,
request_cost_breakdown: Final = (
finalized_cost.logging_breakdown()
if finalized_cost is not None
else cost_breakdown_with_guardrail(
(
_cache_hit_cost_breakdown(uncached_price.cost_breakdown)
if uncached_price is not None
else logging_obj.cost_breakdown
),
guardrail_cost,
)
)
captured_baseline: Final = logging_obj.baseline_observation
autorouter_savings: Final = (
@ -7600,6 +7692,7 @@ def get_standard_logging_object_payload(
model=model_name,
metadata=clean_metadata,
cache_key=clean_hidden_params["cache_key"],
cost_finalized=finalized_cost is not None,
response_cost=response_cost,
cost_breakdown=request_cost_breakdown,
autorouter_savings=autorouter_savings,

View file

@ -0,0 +1,61 @@
from collections.abc import Mapping
from dataclasses import dataclass
from types import MappingProxyType
from typing import Final, TypeAlias
from pydantic import TypeAdapter
from litellm.types.utils import CostBreakdown, StandardLoggingPayload, Usage
BreakdownValue: TypeAlias = float | str | None | Mapping[str, float]
_BREAKDOWN_FIELDS: Final = TypeAdapter(Mapping[str, BreakdownValue])
_COST_BREAKDOWN: Final = TypeAdapter(CostBreakdown)
@dataclass(frozen=True, slots=True)
class PreparedCost:
amount: float
payload: StandardLoggingPayload
usage: Usage
@dataclass(frozen=True, slots=True)
class FinalizedCost:
amount: float
breakdown: Mapping[str, BreakdownValue] | None
allowance_discount: float
def logging_payload(self, payload: StandardLoggingPayload) -> StandardLoggingPayload:
return {
**payload,
"response_cost": self.amount,
"cost_breakdown": self.logging_breakdown(),
"cost_finalized": True,
}
def logging_breakdown(self) -> CostBreakdown | None:
return _COST_BREAKDOWN.validate_python(self.breakdown) if self.breakdown is not None else None
def finalize_cost(amount: float, breakdown: CostBreakdown | None, allowance_discount: float = 0.0) -> FinalizedCost:
discount: Final = min(max(amount, 0.0), max(allowance_discount, 0.0))
total: Final = amount - discount
existing: Final = _BREAKDOWN_FIELDS.validate_python(breakdown) if breakdown is not None else None
adjusted: Final = (
{
**(existing or {}),
"total_cost": total,
"original_cost": (breakdown or {}).get("original_cost", amount),
"discount_amount": (breakdown or {}).get("discount_amount", 0.0) + discount,
}
if discount > 0
else existing
)
frozen: Final = (
MappingProxyType(
{key: MappingProxyType(value) if isinstance(value, Mapping) else value for key, value in adjusted.items()}
)
if adjusted is not None
else None
)
return FinalizedCost(amount=total, breakdown=frozen, allowance_discount=discount)

View file

@ -75,6 +75,14 @@ _IMAGE_TOKEN_RATE_KEYS: Final[tuple[str, ...]] = (
"output_cost_per_image_token",
)
def _deployment_prices_images_without_an_output_price(pricing: ModelInfo | None) -> bool:
return pricing is not None and (
"input_cost_per_pixel" in pricing
or ("input_cost_per_image" in pricing and "output_cost_per_image" not in pricing)
)
# Pre-resolved service-tier cost-key suffixes (e.g. "_priority"). Used per
# request in the cost-calc path, so the f-strings are built once here instead
# of being rebuilt for every model_info key on every call. Longest-first so a
@ -127,7 +135,7 @@ def _get_token_detail_value(details: object, key: str) -> int | None:
return value if isinstance(value, int) else None
_IMAGE_SIZE_PATTERN: Final = re.compile(r"\d+(?:x|-x-)\d+")
_IMAGE_SIZE_PATTERN: Final = re.compile(r"\d+(?:x|-x-|\*)\d+")
def _requested_image_param(optional_params: Mapping[str, object] | None, key: str) -> str | None:
@ -137,7 +145,7 @@ def _requested_image_param(optional_params: Mapping[str, object] | None, key: st
def _requested_image_size(optional_params: Mapping[str, object] | None) -> str | None:
value: Final = _requested_image_param(optional_params, "size")
return value if value is not None and _IMAGE_SIZE_PATTERN.fullmatch(value) else None
return value.replace("*", "x") if value is not None and _IMAGE_SIZE_PATTERN.fullmatch(value) else None
def get_web_search_requests(server_tool_use: object) -> int | None:
@ -2090,7 +2098,7 @@ class _ModelPricedImageCostCalculator(Protocol):
def _model_priced_image_cost_calculator(
custom_llm_provider: str | None, model: str
custom_llm_provider: str | None, model: str, pricing: ModelInfo | None
) -> _ModelPricedImageCostCalculator | None:
match custom_llm_provider:
case litellm.LlmProviders.RECRAFT.value:
@ -2116,6 +2124,14 @@ def _model_priced_image_cost_calculator(
from litellm.llms.tencent.image_generation.transformation import is_tokenhub_image_model
return tencent_image_cost_calculator if is_tokenhub_image_model(model) else None
case litellm.LlmProviders.DASHSCOPE.value:
if _deployment_prices_images_without_an_output_price(pricing):
return None
from litellm.llms.dashscope.image_generation.cost_calculator import (
cost_calculator as dashscope_image_cost_calculator,
)
return dashscope_image_cost_calculator
case _:
return None
@ -2178,7 +2194,7 @@ class CostCalculatorUtils:
)
resolved_n: Final = n if n is not None else (len(completion_response.data) if completion_response.data else 0)
pricing: Final = deployment_pricing(model_info)
model_priced_calculator: Final = _model_priced_image_cost_calculator(custom_llm_provider, model)
model_priced_calculator: Final = _model_priced_image_cost_calculator(custom_llm_provider, model, pricing)
if model_priced_calculator is not None:
return model_priced_calculator(model=model, image_response=completion_response, model_info=pricing)

View file

@ -108,7 +108,7 @@ def judge_target(router: Router | None, model: str, team_id: str | None = None)
only for its own team and a team's own deployment resolves for nobody else, so asking
without it answers for a caller who does not exist.
"""
served: Final = router.resolved_litellm_models(model, team_id=team_id) if router is not None else ()
served: Final = router.registry().names(model, kind="litellm_models", team_id=team_id) if router is not None else ()
if served:
return JudgeTarget("router", frozenset(_provider_qualified(m) or m for m in served))
qualified: Final = _provider_qualified(model)
@ -148,7 +148,7 @@ def attached_auth(request_data: object) -> UserAPIKeyAuth | None:
def _named_route(router: Router, deployment_id: str) -> bool:
deployment: Final = router.get_deployment(model_id=deployment_id)
deployment: Final = router.registry().deployment(deployment_id)
if deployment is None:
return False
public_names: Final = (deployment.model_name, deployment.model_info.team_public_model_name or "")
@ -167,7 +167,7 @@ def caller_team_for(router: Router | None, model: str, auth: UserAPIKeyAuth | No
team_id: Final = auth.team_id if auth is not None else None
if team_id is None or router is None or judge_target(router, model).via != "nothing":
return None
team_route: Final = router.get_candidate_model_ids_for_route(model, team_id)
team_route: Final = router.registry().names(model, kind="route_candidates", team_id=team_id)
return team_id if team_route and all(_named_route(router, deployment_id) for deployment_id in team_route) else None

View file

@ -0,0 +1,45 @@
"""How strong each reasoning_effort level is, and rounding a requested level onto a declared set.
Imports nothing from the SDK, so the `lite` CLI, which loads without initializing litellm, can use it.
"""
from collections.abc import Sequence
from types import MappingProxyType
from typing import Final, Literal, TypeAlias
from typing_extensions import assert_never
ReasoningEffortRounding: TypeAlias = Literal["up", "down"]
REASONING_EFFORT_STRENGTH_ORDER: Final = ("minimal", "low", "medium", "high", "xhigh", "max")
REASONING_EFFORT_STRENGTH_RANK: Final = MappingProxyType(
{effort: rank for rank, effort in enumerate(REASONING_EFFORT_STRENGTH_ORDER)}
)
def nearest_declared_reasoning_effort(
requested: str, declared: Sequence[str], rounding: ReasoningEffortRounding = "up"
) -> str:
"""Rounding up takes the weakest declared level at least as strong as the request, else the
strongest declared, so the caller gets no less reasoning than it asked for instead of a rejected
call. Rounding down mirrors it: the strongest declared level at most as strong, else the
weakest. none is the off switch rather than a strength, so it is never rounded onto the ladder
and no level is rounded to it: a caller who turned reasoning off must not be billed for it, and
a model that cannot turn it off says so itself. A level outside the strength order is likewise
returned as is for upstream to judge."""
ranked: Final = sorted(
(effort for effort in declared if effort in REASONING_EFFORT_STRENGTH_RANK),
key=lambda effort: REASONING_EFFORT_STRENGTH_RANK[effort],
)
if requested in ranked or requested not in REASONING_EFFORT_STRENGTH_RANK or not ranked:
return requested
strength: Final = REASONING_EFFORT_STRENGTH_RANK[requested]
match rounding:
case "up":
return next((effort for effort in ranked if REASONING_EFFORT_STRENGTH_RANK[effort] >= strength), ranked[-1])
case "down":
return next(
(effort for effort in reversed(ranked) if REASONING_EFFORT_STRENGTH_RANK[effort] <= strength), ranked[0]
)
case _:
assert_never(rounding)

View file

@ -0,0 +1,27 @@
from collections.abc import Iterable
from typing import Final
from litellm.types.utils import ModelResponseStream, StreamingChoices
def is_empty_streaming_choice(choice: StreamingChoices) -> bool:
if choice.finish_reason is not None:
return False
if getattr(choice, "logprobs", None) is not None:
return False
delta: Final = choice.delta if hasattr(choice, "delta") else None
if delta is None:
return True
return _all_values_none(delta.model_dump().values())
def _all_values_none(values: Iterable[object]) -> bool:
return all(value is None for value in values)
def is_injected_stream_usage_artifact(chunk: object) -> bool:
if not isinstance(chunk, ModelResponseStream):
return False
if chunk.provider_specific_fields is not None:
return False
return all(is_empty_streaming_choice(choice) for choice in chunk.choices or [])

View file

@ -1224,10 +1224,6 @@ def _validate_anthropic_content(content: Mapping[str, object]) -> type:
if expected_cls is None:
raise ValueError(f"Unknown Anthropic content type: '{content_type}'")
missing: Final = [k for k in getattr(expected_cls, "__required_keys__", set()) if k not in content]
if missing:
raise ValueError(f"Missing required fields in {content_type} block: {', '.join(missing)}")
return expected_cls
@ -1552,12 +1548,13 @@ def _format_single_function_definition(function: Mapping[str, object]) -> tuple[
def _format_object_parameters(parameters, indent):
properties: Final = parameters.get("properties")
if not properties:
if not isinstance(properties, Mapping) or not properties:
return ""
required_params: Final = parameters.get("required", [])
required: Final = parameters.get("required")
required_params: Final = tuple(required) if isinstance(required, list) else ()
lines: Final = []
for key, props in properties.items():
description = props.get("description")
description = props.get("description") if isinstance(props, Mapping) else None
if description:
lines.append(f"// {description}")
question = "?"
@ -1568,19 +1565,21 @@ def _format_object_parameters(parameters, indent):
def _format_type(props, indent):
if not isinstance(props, Mapping):
return "any"
type: Final = props.get("type")
enum: Final = props.get("enum")
if type == "string":
if "enum" in props:
return " | ".join([f'"{item}"' for item in props["enum"]])
if isinstance(enum, list):
return " | ".join([f'"{item}"' for item in enum])
return "string"
elif type == "array":
# items is required, OpenAI throws an error if it's missing
return f"{_format_type(props['items'], indent)}[]"
return f"{_format_type(props.get('items'), indent)}[]"
elif type == "object":
return f"{{\n{_format_object_parameters(props, indent + 2)}\n}}"
elif type in ["integer", "number"]:
if "enum" in props:
return " | ".join([f'"{item}"' for item in props["enum"]])
if isinstance(enum, list):
return " | ".join([f'"{item}"' for item in enum])
return "number"
elif type == "boolean":
return "boolean"

View file

@ -6,6 +6,7 @@ import copy
import json
import re
from collections.abc import Container, Mapping, MutableMapping, Sequence
from dataclasses import dataclass
from datetime import datetime, timezone
from types import MappingProxyType
from typing import Annotated, Any, ClassVar, Final, Literal, TypeVar
@ -1774,6 +1775,48 @@ def anthropic_request_errors_on_thinking_binding_mismatch(request_body: object)
return binding is not None and binding.get("prefix_mismatch_behavior") == "error"
_CLAUDE_STRICT_TOOLS_REJECTION: Final = re.compile(
r"Too many strict tools \(\d+\)"
r"|Schemas contains too many (?:optional parameters|parameters with union types)"
r"|[Cc]ompiled grammar (?:is too large|size)"
r"|tools\.\d+\.custom(?:\.strict)?: "
r"|Grammar compilation timed out"
)
_CLAUDE_TOOLS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...])
def is_claude_strict_tools_rejection(status_code: int, error_text: str) -> bool:
return status_code == 400 and _CLAUDE_STRICT_TOOLS_REJECTION.search(error_text) is not None
@dataclass(frozen=True, slots=True)
class ToolsWithoutStrict:
tools: tuple[Mapping[str, object], ...]
dropped_count: int
def tools_without_strict(tools: object) -> ToolsWithoutStrict | None:
try:
validated: Final = _CLAUDE_TOOLS_ADAPTER.validate_python(tools)
except ValidationError:
return None
dropped_count: Final = sum(tool.get("strict") is True for tool in validated)
if dropped_count == 0:
return None
return ToolsWithoutStrict(
tools=tuple({key: value for key, value in tool.items() if key != "strict"} for tool in validated),
dropped_count=dropped_count,
)
def warn_claude_strict_tools_dropped(dropped_count: int) -> None:
litellm.verbose_logger.warning(
"Claude rejected the request's strict schemas, so litellm removed `strict` from every tool (%d had "
"strict true) and sent the request again because drop_params is set.",
dropped_count,
)
def strip_thinking_blocks_from_anthropic_messages(messages: Sequence[object]) -> list[object]:
"""
Return a new message list with thinking / redacted_thinking content blocks removed

View file

@ -160,6 +160,7 @@ async def prepare_context_managed_request(
"""Apply client compaction history, then optional context_management polyfill."""
from litellm.llms.anthropic.messages.context_management.editors.compact import (
apply_client_compaction_block_history,
slice_around_compaction_block,
)
# Skip the client-history pre-processing when a ``compact_20260112``
@ -175,7 +176,7 @@ async def prepare_context_managed_request(
if polyfill_will_run:
history_result: PolyfillResult | None = None
working_messages: _AnthropicMessages = messages
working_messages: _AnthropicMessages = list(slice_around_compaction_block(messages)[0])
working_system: _AnthropicSystem = system
else:
history_result = apply_client_compaction_block_history(

View file

@ -1,6 +1,7 @@
# What is this?
## Translates OpenAI call to Anthropic `/v1/messages` format
import copy
import io
import itertools
import json
import traceback
@ -181,6 +182,44 @@ class _ToolCallPart:
)
@dataclass(frozen=True, slots=True)
class _ThinkingPiece:
text: str
signature: str
index: int | None
@staticmethod
def of(chunk: "ModelResponseStream") -> "_ThinkingPiece | None":
choices: Final = _optional_attr_sequence(chunk, "choices")
if len(choices) != 1 or _optional_attr(choices[0], "finish_reason") is not None:
return None
delta: Final = _optional_attr(choices[0], "delta")
if _optional_attr(delta, "content") or _optional_attr(delta, "tool_calls"):
return None
blocks: Final = _optional_attr_sequence(delta, "thinking_blocks")
if not blocks:
reasoning: Final = _optional_str_attr(delta, "reasoning_content")
return _ThinkingPiece(text=reasoning, signature="", index=None) if reasoning else None
block: Final = as_str_mapping(blocks[0]) if len(blocks) == 1 else None
if block is None or block.get("type") != "thinking":
return None
text: Final = block.get("thinking")
signature: Final = block.get("signature")
index: Final = block.get("index")
return _ThinkingPiece(
text=text if isinstance(text, str) else "",
signature=signature if isinstance(signature, str) else "",
index=index if isinstance(index, int) else None,
)
def block(self, *, text: str, signature: str) -> ChatCompletionThinkingBlock:
if self.index is None:
return ChatCompletionThinkingBlock(type="thinking", thinking=text, signature=signature or None)
return ChatCompletionThinkingBlock(
type="thinking", thinking=text, signature=signature or None, index=self.index
)
class _CombinedChunkSplitter:
"""
Splits a streaming chunk that carries BOTH response content and a
@ -209,6 +248,10 @@ class _CombinedChunkSplitter:
self._buffer: deque[ModelResponseStream] = deque()
self._held_calls: Mapping[int, tuple[ModelResponseStream, _ToolCallPart]] = MappingProxyType({})
self._live_call_index: int | None = None
self._held_signature: ModelResponseStream | None = None
self._unsigned_thinking = io.StringIO()
self._unsigned_thinking_length = 0
self._thinking_index: int | None = None
self._closed = False
async def aclose(self) -> None:
@ -458,7 +501,45 @@ class _CombinedChunkSplitter:
for split_chunk in self._split_by_payload_kind(combined_chunk) # comprehension-ok: flattens nested splits
)
block_chunks: Final = tuple(itertools.chain.from_iterable(map(self._split_thinking_blocks, split_chunks)))
return tuple(itertools.chain.from_iterable(map(self._route_tool_calls, block_chunks)))
signed_chunks: Final = tuple(itertools.chain.from_iterable(map(self._defer_signature, block_chunks)))
return tuple(itertools.chain.from_iterable(map(self._route_tool_calls, signed_chunks)))
def _end_thinking_run(self) -> "tuple[ModelResponseStream, ...]":
held: Final = self._held_signature
self._held_signature = None
self._unsigned_thinking = io.StringIO()
self._unsigned_thinking_length = 0
self._thinking_index = None
return (held,) if held is not None else ()
def _defer_signature(self, chunk: "ModelResponseStream") -> "tuple[ModelResponseStream, ...]":
"""Stream a signed thinking chunk's new text as thinking and send its signature once, when the block ends.
Anthropic-shaped producers stream unsigned text and then sign it, either with no text or
restating the unsigned text, which passes through untouched. Mistral signs every chunk while
each one still carries new text, which would otherwise collapse into ``signature_delta``
events and lose the text.
"""
piece: Final = _ThinkingPiece.of(chunk)
released: Final = self._end_thinking_run() if piece is None or piece.index != self._thinking_index else ()
if piece is None:
return (*released, chunk)
self._thinking_index = piece.index
if not piece.signature:
self._unsigned_thinking_length += self._unsigned_thinking.write(piece.text)
return (*released, chunk)
self._held_signature = None
if not piece.text or self._restates_unsigned_thinking(piece.text):
return (*released, chunk)
signature_chunk: Final = self._with_thinking_block(
chunk, piece.block(text="", signature=piece.signature), keep_role=False
)
self._clear_usage(signature_chunk)
self._held_signature = signature_chunk
return (*released, self._with_thinking_block(chunk, piece.block(text=piece.text, signature=""), keep_role=True))
def _restates_unsigned_thinking(self, text: str) -> bool:
return len(text) == self._unsigned_thinking_length and text == self._unsigned_thinking.getvalue()
def __iter__(self) -> "Iterator[ModelResponseStream]":
return self
@ -493,7 +574,7 @@ class _CombinedChunkSplitter:
try:
chunk = next(self._sync_iter)
except StopIteration:
self._buffer.extend(self._release_held_calls())
self._buffer.extend((*self._end_thinking_run(), *self._release_held_calls()))
if not self._buffer:
raise
break
@ -512,7 +593,7 @@ class _CombinedChunkSplitter:
try:
chunk = await self._async_iter.__anext__()
except StopAsyncIteration:
self._buffer.extend(self._release_held_calls())
self._buffer.extend((*self._end_thinking_run(), *self._release_held_calls()))
if not self._buffer:
raise
break

View file

@ -8,6 +8,7 @@ from litellm._logging import verbose_logger
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.types.llms.anthropic import AppliedEdit
from ..convertible_blocks import drop_non_string_texts
from .constants import CLEAR_TOOL_USES_EDIT_TYPE, COMPACT_EDIT_TYPE
from .editors import (
CheckedEdit,
@ -103,7 +104,7 @@ async def apply_context_management(
Every ``clear_tool_uses_20250919`` edit is validated before any editor runs, so a malformed one raises its 400
before a ``compact_20260112`` summary call is paid for, or runs without its malformed fields under
``drop_invalid_edits``.
``drop_invalid_edits``, which also drops non-string user and ``tool_result`` text first, as the provider never receives it.
The dispatcher is async so async editors (``compact_20260112``) can
``await`` the configured summarization model. Sync editors run in a
@ -119,7 +120,7 @@ async def apply_context_management(
raise first_error
well_formed_edits: Final = tuple(check.edit for check in checked)
current_messages = messages
current_messages = list(drop_non_string_texts(messages, drop_invalid_edits))
current_system = system
aggregated_applied: Final[list[AppliedEdit]] = []
aggregated_compaction_block = None

View file

@ -39,6 +39,21 @@ if TYPE_CHECKING:
T = TypeVar("T")
# api.anthropic.com, measured 2026-10-19: text results of 1 to 46 bytes are never cleared or counted toward keep
_TINY_RESULT_BYTES: Final = 46
_TOKEN_COUNTER_BLOCK_TYPES: Final = frozenset(
{
"text",
"image_url",
"image",
"document",
"file",
"tool_use",
"tool_result",
"thinking",
"redacted_thinking",
"tool_reference",
"search_result",
}
)
ToolNames = Literal["all"] | frozenset[str]
@ -52,16 +67,19 @@ class _ClearAtLeast(BaseModel):
class _ClearToolUsesFields(BaseModel):
model_config = ConfigDict(frozen=True)
trigger: Mapping[str, object] | None = None
keep: Mapping[str, object] | None = None
clear_at_least: _ClearAtLeast | None = None
exclude_tools: tuple[str, ...] | None = None
clear_tool_inputs: bool | tuple[str, ...] | None = None
_BLOCKS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
# api.anthropic.com, measured 2026-10-19: its wording for the two shapes pydantic words differently
# api.anthropic.com, measured 2026-10-19 and 2026-10-28: its wording for the shapes pydantic words differently
_API_MESSAGES: Final = MappingProxyType(
{
"tuple_type": "Input should be a valid array",
"dict_type": "Input should be an object",
"model_type": "Input does not match the expected shape.",
}
)
@ -178,10 +196,12 @@ def _cleared_result(block: Mapping[str, object]) -> dict[str, object]:
def _freed_tokens(model: str, result: Mapping[str, object], use: Mapping[str, object], clears_input: bool) -> int:
"""Tokens that clearing this use frees: its result, and its input when ``clear_tool_inputs`` covers it."""
before: Final = [result, use] if clears_input else [result]
after: Final = [_cleared_result(result), {**use, "input": {}}] if clears_input else [_cleared_result(result)]
before: Final[Sequence[Mapping[str, object]]] = (result, use) if clears_input else (result,)
after: Final[Sequence[Mapping[str, object]]] = (
(_cleared_result(result), {**use, "input": {}}) if clears_input else (_cleared_result(result),)
)
counts: Final = tuple(
litellm.token_counter(model=model, messages=[{"role": "user", "content": blocks}]) for blocks in (before, after)
_count_tokens(model, [{"role": "user", "content": blocks}], None) for blocks in (before, after)
)
return max(counts[0] - counts[1], 0)
@ -199,6 +219,35 @@ def _select_in_batches(candidates: Sequence[str], tokens: Mapping[str, int], cle
return tuple(candidates[:batch_end])
def _without_uncountable_blocks(block: Mapping[str, object]) -> Mapping[str, object]:
nested: Final = _parse(_BLOCKS, block.get("content"))
if nested is None:
return block
return {**block, "content": [inner for inner in nested if inner.get("type") in _TOKEN_COUNTER_BLOCK_TYPES]}
def _countable_message(message: Mapping[str, object]) -> Mapping[str, object]:
blocks: Final = _parse(_BLOCKS, message.get("content"))
if blocks is None:
return message
return {
**message,
"content": [
_without_uncountable_blocks(block) for block in blocks if block.get("type") in _TOKEN_COUNTER_BLOCK_TYPES
],
}
def _count_tokens(
model: str, messages: Sequence[Mapping[str, object]], tools: Sequence[Mapping[str, object]] | None
) -> int:
return litellm.token_counter(
model=model,
messages=[_countable_message(message) for message in messages],
tools=cast("list[ChatCompletionToolParam] | None", tools),
)
def _trigger_met(
trigger: dict[str, object],
model: str,
@ -218,11 +267,7 @@ def _trigger_met(
if not isinstance(threshold, int):
threshold = DEFAULT_INPUT_TOKENS_TRIGGER
message_tokens: Final = litellm.token_counter(
model=model,
messages=messages,
tools=cast("list[ChatCompletionToolParam] | None", tools),
)
message_tokens: Final = _count_tokens(model, messages, tools)
if message_tokens > threshold:
return True, message_tokens
current_tokens: Final = message_tokens + count_system_tokens(model, system)
@ -375,17 +420,9 @@ def apply_clear_tool_uses_20250919(
[_message_without_inputs(message, ids_to_clear, input_tools) for message in edited] if input_tools else edited
)
tokens_before: Final = (
counted_tokens
if counted_tokens is not None
else litellm.token_counter(
model=model, messages=kept, tools=cast("list[ChatCompletionToolParam] | None", tools)
)
)
tokens_before: Final = counted_tokens if counted_tokens is not None else _count_tokens(model, kept, tools)
kept_after, _ = history_after_client_compaction(final_messages, system)
tokens_after: Final = litellm.token_counter(
model=model, messages=kept_after, tools=cast("list[ChatCompletionToolParam] | None", tools)
)
tokens_after: Final = _count_tokens(model, kept_after, tools)
cleared_input_tokens: Final = max(tokens_before - tokens_after, 0)
return final_messages, {

View file

@ -137,7 +137,7 @@ def _find_latest_compaction_index(
return None, None
def _slice_around_compaction_block(
def slice_around_compaction_block(
messages: Sequence[_MsgT],
) -> tuple[Sequence[_MsgT | dict[str, object]], dict[str, object] | None]:
"""Apply Anthropic's "drop everything before the compaction block" rule.
@ -194,7 +194,7 @@ def history_after_client_compaction(
system: str | Sequence[Mapping[str, object]] | None,
) -> tuple[list[dict[str, object]], str | Sequence[Mapping[str, object]] | None]:
"""Return the history the API keeps after the latest compaction block, without it, and the system with its summary."""
sliced, block = _slice_around_compaction_block(messages)
sliced, block = slice_around_compaction_block(messages)
if block is None:
return list(messages), system
tail: Final = _strip_compaction_blocks(sliced)
@ -615,7 +615,7 @@ def apply_client_compaction_block_history(
tail is forwarded unchanged (with compaction blocks stripped) so recent
turns the summary does not cover are preserved.
"""
effective_messages, prior_compaction_block = _slice_around_compaction_block(messages)
effective_messages, prior_compaction_block = slice_around_compaction_block(messages)
if prior_compaction_block is None:
return None
@ -686,7 +686,7 @@ async def apply_compact_20260112(
# opt-in gate below so that even when summarization is disabled we still
# strip Anthropic-only ``compaction`` blocks from messages going to
# non-Anthropic backends (which would reject them).
effective_messages, prior_compaction_block = _slice_around_compaction_block(messages)
effective_messages, prior_compaction_block = slice_around_compaction_block(messages)
prior_summary_text: Final = prior_compaction_block.get("content") if prior_compaction_block else None
augmented_system: str | list[dict[str, object]] | None = system
if isinstance(prior_summary_text, str) and prior_summary_text:

View file

@ -396,7 +396,8 @@ def _resolve_advisor_router(advisor_model: str) -> "Router | None":
return None
if llm_router is None:
return None
if llm_router.is_recognized_model(advisor_model) or llm_router.pattern_router.route(advisor_model):
registry: Final = llm_router.registry()
if registry.resolve(advisor_model, kind="recognized") or registry.patterns().route(advisor_model):
return llm_router
return None

View file

@ -10,11 +10,15 @@ from typing import Final, TypeAlias
from pydantic import TypeAdapter, ValidationError
import litellm
from litellm._logging import verbose_logger
from litellm.integrations.anthropic_cache_control_hook import (
AnthropicCacheControlHook,
supports_openai_prompt_cache_breakpoint,
)
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.asyncify import (
run_async_function, # pyright: ignore[reportUnknownVariableType] # untyped helper
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params
from litellm.litellm_core_utils.prompt_templates.common_utils import (
cache_control_as_prompt_cache_breakpoint,
prompt_cache_breakpoint_for_wire,
@ -28,14 +32,19 @@ from litellm.types.llms.anthropic import (
AnthropicMessagesRequest,
AnthropicOutputConfig,
AnthropicOutputSchema,
ContextManagementResponse,
)
from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,
)
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.utils import CountContextManagement
from litellm.utils import ProviderConfigManager
from ..adapters.handler import request_safeguards_evaluator
from ..adapters.handler import context_management_explicitly_dropped, request_safeguards_evaluator
from ..context_management import AnthropicContextManagementError, PolyfillResult, apply_context_management
from ..context_management.constants import COMPACT_EDIT_TYPE
from ..context_management.dispatcher import normalize_spec
from ..convertible_blocks import reject_malformed_text
from ..safeguards import with_safeguard_results
from ..utils import litellm_logging_obj_from_kwargs, local_model_name
@ -45,10 +54,111 @@ from .transformation import LiteLLMAnthropicToResponsesAPIAdapter
AnthropicRequestMessages: TypeAlias = list[AllAnthropicMessageValues] | list[dict[str, object]]
BlockRewrite: TypeAlias = Callable[[Mapping[str, object]], Mapping[str, object]]
_BLOCK_LIST: Final = TypeAdapter(list[dict[str, object]])
_SYSTEM_ITEMS: Final = TypeAdapter(list[object])
_SPEC: Final = TypeAdapter(dict[str, object])
_SplitEdits: TypeAlias = tuple[dict[str, object] | None, Sequence[Mapping[str, object]]]
AnthropicContextManagementSpec: TypeAlias = dict[str, object] | list[dict[str, object]] | None
_ADAPTER: Final = LiteLLMAnthropicToResponsesAPIAdapter()
def _split_context_management(
spec: AnthropicContextManagementSpec,
kwargs: Mapping[str, object],
) -> _SplitEdits:
"""Return (the edits the Responses API runs natively, the edits the gateway polyfill runs).
The Responses API compacts natively (``compact_20260112`` becomes its ``compaction``); any other
edit, such as ``clear_tool_uses_20250919``, has no Responses API form and runs in the polyfill, as
it does on the chat completions path. An OpenAI-style list spec is not Anthropic's shape and stays
unsent, as it was before the polyfill ran here.
"""
if not isinstance(spec, dict):
return None, ()
if not spec or context_management_explicitly_dropped(kwargs.get("additional_drop_params")):
return spec, ()
edits: Final = normalize_spec(spec) or []
native: Final = [edit for edit in edits if edit.get("type") == COMPACT_EDIT_TYPE]
return ({"edits": native} if native else None), tuple(
edit for edit in edits if edit.get("type") != COMPACT_EDIT_TYPE
)
def _system_blocks(system: object) -> str | Sequence[Mapping[str, object]] | None:
if system is None or isinstance(system, str):
return system
try:
items: Final = _SYSTEM_ITEMS.validate_python(system)
except ValidationError:
return None
return tuple(block for block in map(as_str_mapping, items) if block is not None)
async def _polyfilled(
*,
model: str,
messages: AnthropicRequestMessages,
tools: Sequence[AllAnthropicToolsValues | Mapping[str, object]] | None,
system: object,
edits: Sequence[Mapping[str, object]],
kwargs: Mapping[str, object],
) -> PolyfillResult | None:
"""Run the edits the Responses API can't run natively; raise like the chat completions path does."""
if not edits:
return None
reject_malformed_text(messages, model, kwargs)
drop_invalid_edits: Final = drops_unsupported_params(kwargs.get("drop_params"))
blocks: Final = _system_blocks(system)
try:
return await apply_context_management(
model=model,
messages=_BLOCK_LIST.validate_python(messages),
tools=_BLOCK_LIST.validate_python(tools) if tools else None,
system=blocks if blocks is None or isinstance(blocks, str) else _BLOCK_LIST.validate_python(blocks),
context_management_spec={"edits": [dict(edit) for edit in edits]},
drop_invalid_edits=drop_invalid_edits,
)
except AnthropicContextManagementError:
raise
except Exception as e:
verbose_logger.exception("context_management polyfill failed on the Responses API path: %s", e)
raise AnthropicContextManagementError(
status_code=500, message=f"context_management polyfill failed: {e}"
) from e
async def count_view_messages(
*,
model: str,
messages: Sequence[Mapping[str, object]],
tools: Sequence[Mapping[str, object]] | None,
system: object,
context_management: CountContextManagement | None,
) -> Sequence[Mapping[str, object]]:
"""The messages /v1/messages sends the Responses API once the polyfill has run the edits it can't run natively."""
if context_management is None or not isinstance(context_management.spec, dict):
return messages
_, edits = _split_context_management(_SPEC.validate_python(context_management.spec), {})
result: Final = await _polyfilled(
model=model,
messages=_BLOCK_LIST.validate_python(messages),
tools=tools,
system=system,
edits=edits,
kwargs={"drop_params": context_management.drop_invalid_edits},
)
return messages if result is None else result.messages
def _with_applied_edits(
response: AnthropicMessagesResponse, polyfill_result: PolyfillResult | None
) -> AnthropicMessagesResponse:
applied_edits: Final = polyfill_result.applied_edits_for_response() if polyfill_result else None
if not applied_edits:
return response
return {**response, "context_management": ContextManagementResponse(applied_edits=list(applied_edits))}
def _record_bridged_response(responses_kwargs: Mapping[str, object], result: ResponsesAPIResponse) -> None:
logging_obj: Final = litellm_logging_obj_from_kwargs(responses_kwargs)
if logging_obj is not None:
@ -281,7 +391,7 @@ class LiteLLMMessagesToResponsesAPIHandler:
max_tokens: int,
messages: AnthropicRequestMessages,
model: str,
context_management: dict[str, object] | None = None,
context_management: AnthropicContextManagementSpec = None,
metadata: dict[str, object] | None = None,
output_config: AnthropicOutputConfig | None = None,
stop_sequences: list[str] | None = None,
@ -296,11 +406,15 @@ class LiteLLMMessagesToResponsesAPIHandler:
output_format: AnthropicOutputSchema | None = None,
**kwargs: object,
) -> AnthropicMessagesResponse | AsyncIterator[bytes]:
native_context_management, polyfilled_edits = _split_context_management(context_management, kwargs)
polyfill_result: Final = await _polyfilled(
model=model, messages=messages, tools=tools, system=system, edits=polyfilled_edits, kwargs=kwargs
)
responses_kwargs: Final = _build_responses_kwargs(
max_tokens=max_tokens,
messages=messages,
messages=polyfill_result.messages if polyfill_result is not None else messages,
model=model,
context_management=context_management,
context_management=native_context_management,
metadata=metadata,
output_config=output_config,
stop_sequences=stop_sequences,
@ -325,13 +439,16 @@ class LiteLLMMessagesToResponsesAPIHandler:
model=local_model_name(model, kwargs.get("custom_llm_provider")),
litellm_logging_obj=litellm_logging_obj_from_kwargs(responses_kwargs),
safeguards_evaluator=safeguards_evaluator,
applied_edits=polyfill_result.applied_edits_for_response() if polyfill_result else None,
)
return wrapper.async_anthropic_sse_wrapper()
if not isinstance(result, ResponsesAPIResponse):
raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}")
served: Final = await with_safeguard_results(_ADAPTER.translate_response(result), safeguards_evaluator)
served: Final = await with_safeguard_results(
_with_applied_edits(_ADAPTER.translate_response(result), polyfill_result), safeguards_evaluator
)
_record_bridged_response(responses_kwargs, result)
return served
@ -340,7 +457,7 @@ class LiteLLMMessagesToResponsesAPIHandler:
max_tokens: int,
messages: AnthropicRequestMessages,
model: str,
context_management: dict[str, object] | None = None,
context_management: AnthropicContextManagementSpec = None,
metadata: dict[str, object] | None = None,
output_config: AnthropicOutputConfig | None = None,
stop_sequences: list[str] | None = None,
@ -382,11 +499,26 @@ class LiteLLMMessagesToResponsesAPIHandler:
)
# Sync path
native_context_management, polyfilled_edits = _split_context_management(context_management, kwargs)
ran: Final[object] = ( # pyright: ignore[reportUnknownVariableType] # run_async_function is untyped
run_async_function(
_polyfilled,
model=model,
messages=messages,
tools=tools,
system=system,
edits=polyfilled_edits,
kwargs=kwargs,
)
if polyfilled_edits
else None
)
polyfill_result: Final = ran if isinstance(ran, PolyfillResult) else None
responses_kwargs: Final = _build_responses_kwargs(
max_tokens=max_tokens,
messages=messages,
messages=polyfill_result.messages if polyfill_result is not None else messages,
model=model,
context_management=context_management,
context_management=native_context_management,
metadata=metadata,
output_config=output_config,
stop_sequences=stop_sequences,
@ -409,10 +541,11 @@ class LiteLLMMessagesToResponsesAPIHandler:
responses_stream=result,
model=local_model_name(model, kwargs.get("custom_llm_provider")),
litellm_logging_obj=litellm_logging_obj_from_kwargs(responses_kwargs),
applied_edits=polyfill_result.applied_edits_for_response() if polyfill_result else None,
)
return wrapper.async_anthropic_sse_wrapper()
if not isinstance(result, ResponsesAPIResponse):
raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}")
return _ADAPTER.translate_response(result)
return _with_applied_edits(_ADAPTER.translate_response(result), polyfill_result)

View file

@ -2,7 +2,7 @@
## Translates OpenAI call to Anthropic `/v1/messages` format
import asyncio
from collections import deque
from collections.abc import AsyncGenerator, Awaitable, Callable, Iterator, Mapping
from collections.abc import AsyncGenerator, Awaitable, Callable, Iterator, Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final
import anyio
@ -27,6 +27,7 @@ from litellm.responses.streaming_iterator import (
partial_responses_incomplete_event_on_disconnect,
stream_error_status_and_message,
)
from litellm.types.llms.anthropic import AppliedEdit, ContextManagementResponse
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage
from litellm.types.llms.base import LiteLLMBaseModel
@ -129,8 +130,10 @@ class AnthropicResponsesStreamWrapper:
model: str,
litellm_logging_obj: "LiteLLMLoggingObject | None" = None,
safeguards_evaluator: "SafeguardsEvaluator | None" = None,
applied_edits: Sequence[AppliedEdit] | None = None,
) -> None:
self.responses_stream = responses_stream
self._applied_edits: Final = tuple(applied_edits or ())
self.model = model
self.litellm_logging_obj = litellm_logging_obj
self.safeguards_evaluator: Final = safeguards_evaluator
@ -445,6 +448,11 @@ class AnthropicResponsesStreamWrapper:
"type": "message_delta",
"delta": message_delta_payload,
"usage": dict(anthropic_usage),
**(
{"context_management": ContextManagementResponse(applied_edits=list(self._applied_edits))}
if self._applied_edits
else {}
),
}
)
self._chunk_queue.append({"type": "message_stop"})

View file

@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Final
import httpx
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,
)
@ -201,11 +202,23 @@ class BaseAnthropicMessagesConfig(ABC):
anthropic_request_errors_on_thinking_binding_mismatch,
is_anthropic_invalid_thinking_block_error,
is_anthropic_thinking_binding_mismatch_error,
is_claude_strict_tools_rejection,
tools_without_strict,
validated_object_mapping,
)
if e.response.status_code != 400:
return False
error_text: Final[str] = e.response.text
body: Final = validated_object_mapping(request_body)
drops_rejected_strict_tools: Final = (
drops_unsupported_params(litellm_params.get("drop_params"))
and is_claude_strict_tools_rejection(e.response.status_code, error_text)
and body is not None
and tools_without_strict(body.get("tools")) is not None
)
if drops_rejected_strict_tools:
return True
caller_asked_for_this_error: Final = anthropic_request_errors_on_thinking_binding_mismatch(
request_body
) and is_anthropic_thinking_binding_mismatch_error(error_text)
@ -219,11 +232,22 @@ class BaseAnthropicMessagesConfig(ABC):
"""
from litellm.llms.anthropic.common_utils import (
is_anthropic_invalid_thinking_block_error,
is_claude_strict_tools_rejection,
strip_thinking_blocks_from_anthropic_messages_request_dict,
tools_without_strict,
warn_claude_strict_tools_dropped,
)
if e.response.status_code == 400 and is_anthropic_invalid_thinking_block_error(e.response.text):
strip_thinking_blocks_from_anthropic_messages_request_dict(request_data)
without_strict: Final = (
tools_without_strict(request_data.get("tools"))
if is_claude_strict_tools_rejection(e.response.status_code, e.response.text)
else None
)
if without_strict is not None:
warn_claude_strict_tools_dropped(without_strict.dropped_count)
request_data["tools"] = list(without_strict.tools) # rebind-ok: the retry loop re-signs this same dict
return request_data
def native_output_fallback_litellm_params(

View file

@ -135,6 +135,18 @@ class BaseTextToSpeechConfig(ABC):
Transform provider response to standard format
"""
async def async_transform_text_to_speech_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
) -> "HttpxBinaryResponseContent":
return self.transform_text_to_speech_response(
model=model,
raw_response=raw_response,
logging_obj=logging_obj,
)
def get_error_class(self, error_message: str, status_code: int, headers: dict) -> BaseLLMException:
from ..chat.transformation import BaseLLMException

View file

@ -93,9 +93,12 @@ class RouterVectorStoreEmbeddingExecutor:
def _router_serves(self, model: str) -> bool:
team_id: Final = self.metadata.get("user_api_key_team_id")
resolved: Final = self.router.resolved_litellm_models(model, team_id if isinstance(team_id, str) else None)
registry: Final = self.router.registry()
resolved: Final = registry.names(
model, kind="litellm_models", team_id=team_id if isinstance(team_id, str) else None
)
deployment_models: Final = (
deployment.get("litellm_params", {}).get("model") for deployment in self.router.get_model_list() or ()
deployment.get("litellm_params", {}).get("model") for deployment in registry.deployments() or ()
)
return bool(resolved) or model in deployment_models
@ -124,6 +127,9 @@ class BaseVectorStoreConfig:
def validate_create_vector_store(self) -> None:
return None
def get_async_httpx_client_params(self) -> Mapping[str, object]:
return MappingProxyType({})
def get_supported_openai_params(self, model: str) -> list[VECTOR_STORE_OPENAI_PARAMS]:
return []

View file

@ -2,11 +2,11 @@ from collections.abc import Callable, Coroutine
from functools import partial
from typing import Final, TypeAlias
from litellm.llms.anthropic.common_utils import is_claude_strict_tools_rejection
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.bedrock.common_utils import (
NativeOutputFallback,
is_bedrock_native_output_rejection,
is_bedrock_strict_tools_rejection,
warn_bedrock_native_output_rejected,
warn_bedrock_strict_tools_dropped,
)
@ -34,7 +34,7 @@ def _native_output_retry(
def _strict_tools_retry(
error: BaseLLMException, fallback: NativeOutputFallbackFactory | None, model: str
) -> NativeOutputFallback | None:
if fallback is None or not is_bedrock_strict_tools_rejection(error.status_code, error.message):
if fallback is None or not is_claude_strict_tools_rejection(error.status_code, error.message):
return None
retry: Final = fallback()
if retry is not None:

View file

@ -366,21 +366,10 @@ class NativeOutputFallback:
litellm_params: Mapping[str, object]
_BEDROCK_STRICT_TOOLS_REJECTION: Final = re.compile(
r"Too many strict tools \(\d+\)"
r"|Schemas contains too many (?:optional parameters|parameters with union types)"
r"|[Cc]ompiled grammar (?:is too large|size)"
r"|tools\.\d+\.custom(?:\.strict)?: "
r"|Grammar compilation timed out"
)
_TOOLS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...])
_TOOL_SPEC_ADAPTER: Final = TypeAdapter(Mapping[str, object])
def is_bedrock_strict_tools_rejection(status_code: int, error_text: str) -> bool:
return status_code == 400 and _BEDROCK_STRICT_TOOLS_REJECTION.search(error_text) is not None
def warn_bedrock_strict_tools_dropped(model: str) -> None:
verbose_logger.warning(
"Bedrock rejected the strict tool schemas for model=%s, so litellm sent the request again without `strict` "

View file

@ -22,6 +22,7 @@ import httpx
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.responses.codex_compat import drop_unsupported_tools, normalize_codex_input_items
@ -196,7 +197,9 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
params: Final = cast( # cast-ok: the base signature leaves the params dict untyped
"ResponsesAPIOptionalRequestParams", response_api_optional_request_params
)
hoisted: Final = hoist_additional_tools(input, params.get("tools"))
hoisted: Final = hoist_additional_tools(
input, params.get("tools"), drop_params=drops_unsupported_params(litellm_params.drop_params)
)
normalized_input, rewritten_types = normalize_codex_input_items(hoisted.input)
if rewritten_types:
verbose_logger.warning(

View file

@ -0,0 +1,128 @@
import weakref
from collections.abc import AsyncIterator, Callable
from typing import Final
import httpx
HTTP2_STREAMS_PER_CONNECTION: Final = 100
class _Lane:
__slots__ = ("in_flight", "transport")
def __init__(self, transport: httpx.AsyncBaseTransport) -> None:
self.transport: Final = transport
self.in_flight = 0
def acquire(self) -> None:
self.in_flight += 1
def release(self) -> None:
self.in_flight -= 1
class _Lanes:
def __init__(
self,
new_transport: Callable[[httpx.Limits], httpx.AsyncBaseTransport],
limits: httpx.Limits,
streams_per_connection: int,
) -> None:
self._new_transport: Final = new_transport
self._limits: Final = limits
self._streams_per_connection: Final = streams_per_connection
self.lanes: tuple[_Lane, ...] = ()
def acquire(self) -> _Lane:
free: Final = next((lane for lane in self.lanes if lane.in_flight < self._streams_per_connection), None)
lane: Final = free if free is not None else self._open_or_share()
lane.acquire()
return lane
def _open_or_share(self) -> _Lane:
cap: Final = self._limits.max_connections
if cap is not None and len(self.lanes) * self._streams_per_connection >= cap:
return min(self.lanes, key=lambda lane: lane.in_flight)
return self._open()
def _open(self) -> _Lane:
lane: Final = _Lane(self._new_transport(self._next_lane_limits()))
self.lanes = (*self.lanes, lane)
return lane
def _next_lane_limits(self) -> httpx.Limits:
cap: Final = self._limits.max_connections
if cap is None:
return self._limits
return httpx.Limits(
max_connections=min(self._streams_per_connection, cap - len(self.lanes) * self._streams_per_connection),
max_keepalive_connections=self._limits.max_keepalive_connections,
keepalive_expiry=self._limits.keepalive_expiry,
)
class _AsyncReleasingStream(httpx.AsyncByteStream):
def __init__(self, stream: httpx.AsyncByteStream, release: Callable[[], None]) -> None:
self._stream: Final = stream
self._release: Final = weakref.finalize(self, release)
async def __aiter__(self) -> AsyncIterator[bytes]:
try:
async for chunk in self._stream:
yield chunk
except BaseException:
self._release()
raise
async def aclose(self) -> None:
self._release()
await self._stream.aclose()
def _allowed_over_http2(name: bytes, value: bytes) -> bool:
return name.lower() != b"te" or value.strip().lower() == b"trailers"
def _without_te_codings(request: httpx.Request) -> httpx.Request:
if request.url.scheme != "https":
return request
kept: Final = tuple((name, value) for name, value in request.headers.raw if _allowed_over_http2(name, value))
if len(kept) == len(request.headers.raw):
return request
return httpx.Request(
request.method, request.url, headers=kept, stream=request.stream, extensions=request.extensions
)
def _with_stream(response: httpx.Response, stream: httpx.AsyncByteStream) -> httpx.Response:
return httpx.Response(response.status_code, headers=response.headers, stream=stream, extensions=response.extensions)
class AsyncHTTP2ConnectionSpread(httpx.AsyncBaseTransport):
def __init__(
self,
new_transport: Callable[[httpx.Limits], httpx.AsyncBaseTransport],
limits: httpx.Limits,
streams_per_connection: int = HTTP2_STREAMS_PER_CONNECTION,
) -> None:
self._lanes: Final = _Lanes(new_transport, limits, streams_per_connection)
async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
lane: Final = self._lanes.acquire()
try:
response: Final = await lane.transport.handle_async_request(_without_te_codings(request))
except BaseException:
lane.release()
raise
stream: Final = response.stream
if not isinstance(stream, httpx.AsyncByteStream):
lane.release()
return response
return _with_stream(response, _AsyncReleasingStream(stream, lane.release))
def has_request_in_flight(self) -> bool:
return any(lane.in_flight > 0 for lane in self._lanes.lanes)
async def aclose(self) -> None:
for lane in self._lanes.lanes:
await lane.transport.aclose()

View file

@ -18,6 +18,7 @@ import certifi
import httpx
from aiohttp import ClientSession, DummyCookieJar, TCPConnector
from httpx import USE_CLIENT_DEFAULT, AsyncHTTPTransport, HTTPTransport
from httpx._config import DEFAULT_LIMITS
from httpx._types import CertTypes, RequestFiles
from httpx._utils import get_environment_proxies
from pydantic import TypeAdapter
@ -43,6 +44,7 @@ from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
from litellm.litellm_core_utils.request_timeout_resolver import (
get_configured_request_timeout,
)
from litellm.llms.custom_httpx.http2_connection_spread import AsyncHTTP2ConnectionSpread
from litellm.types.llms.custom_http import *
if TYPE_CHECKING:
@ -73,7 +75,7 @@ _RequestContent: TypeAlias = str | bytes | Iterable[bytes] | AsyncIterable[bytes
_IPV4_LOCAL_ADDRESS: Final = "0.0.0.0"
_HttpxTransportT = TypeVar("_HttpxTransportT", HTTPTransport, AsyncHTTPTransport)
_HttpxTransportT: Final = TypeVar("_HttpxTransportT", httpx.BaseTransport, httpx.AsyncBaseTransport)
HTTPEventHooks: TypeAlias = Mapping[str, list[Callable[..., object]]]
@ -84,6 +86,36 @@ def http2_enabled() -> bool:
return litellm.http2 is True or str_to_bool(os.getenv("LITELLM_HTTP2", "False")) is True
_HTTP2_OFF_VALUES: Final = frozenset({"false", "0", "no", "off"})
def _is_http2_off(value: bool | int | str | None) -> bool:
return str(value).strip().lower() in _HTTP2_OFF_VALUES
def http2_disabled() -> bool:
return _is_http2_off(litellm.http2) or _is_http2_off(os.getenv("LITELLM_HTTP2"))
def _resolve_http2(provider_default: bool, ssl_verify: VerifyTypes | None) -> bool:
if http2_enabled():
return True
return provider_default and not http2_disabled() and not isinstance(ssl_verify, ssl.SSLContext)
def _aiohttp_env_proxies_disabled() -> bool:
from litellm.secret_managers.main import str_to_bool
return litellm.disable_aiohttp_trust_env or str_to_bool(os.getenv("DISABLE_AIOHTTP_TRUST_ENV", "False")) is True
_AIOHTTP_EQUIVALENT_LIMITS: Final = httpx.Limits(
max_connections=AIOHTTP_CONNECTOR_LIMIT if AIOHTTP_CONNECTOR_LIMIT > 0 else None,
max_keepalive_connections=None,
keepalive_expiry=AIOHTTP_KEEPALIVE_TIMEOUT,
)
def _environment_proxy_mounts(
build_proxy_transport: Callable[[str], _HttpxTransportT],
) -> Mapping[str, _HttpxTransportT | None]:
@ -145,7 +177,7 @@ def build_aiohttp_keepalive_socket_factory() -> Callable[[_AddrInfo], socket.soc
_build_aiohttp_keepalive_socket_factory = build_aiohttp_keepalive_socket_factory
def get_default_headers() -> dict:
def get_default_headers() -> dict[str, str]:
"""
Get default headers for HTTP requests.
@ -280,9 +312,7 @@ def _prepare_request_data_and_content(
# Cache for SSL contexts to avoid creating duplicate contexts with the same configuration
# Key: tuple of (cafile, ssl_security_level, ssl_ecdh_curve)
# Value: ssl.SSLContext
_ssl_context_cache: Final[dict[tuple[str | None, str | None, str | None], ssl.SSLContext]] = {}
_ssl_context_cache: Final[dict[tuple[str | None, str | None, str | None, bool], ssl.SSLContext]] = {}
def _create_ssl_context(
@ -375,6 +405,7 @@ def get_ssl_verify(
def get_ssl_configuration(
ssl_verify: VerifyTypes | None = None,
http2: bool = False,
) -> bool | str | ssl.SSLContext:
"""
Unified SSL configuration function that handles ssl_context and ssl_verify logic.
@ -397,6 +428,8 @@ def get_ssl_configuration(
- False: Disable SSL verification
- True: Enable SSL verification
- str: Path to CA bundle file
http2: Whether the context is for an HTTP/2 client. httpx sets ALPN on the context it is given,
so HTTP/2 clients get their own cached context instead of sharing the HTTP/1.1 one
Returns:
Union[bool, str, ssl.SSLContext]: Appropriate SSL configuration
@ -423,7 +456,7 @@ def get_ssl_configuration(
if ssl_verify is not False:
# Create cache key from configuration parameters
cache_key: Final = (cafile, ssl_security_level, ssl_ecdh_curve)
cache_key: Final = (cafile, ssl_security_level, ssl_ecdh_curve, http2)
# Check if we have a cached SSL context for this configuration
if cache_key not in _ssl_context_cache:
@ -439,6 +472,11 @@ def get_ssl_configuration(
return ssl_verify
def _httpx_default_verify_unless_configured(verify: VerifyTypes) -> VerifyTypes:
litellm_defaults: Final = (get_ssl_configuration(True), get_ssl_configuration(True, http2=True))
return True if any(verify is default for default in litellm_defaults) else verify
_shared_realtime_ssl_context: bool | str | ssl.SSLContext | None = None
@ -662,6 +700,7 @@ class AsyncHTTPHandler:
shared_session: Optional["ClientSession"] = None,
transport: httpx.AsyncBaseTransport | None = None,
follow_redirects: bool = True,
http2: bool = False,
):
self.timeout = timeout
self.event_hooks = event_hooks
@ -669,6 +708,7 @@ class AsyncHTTPHandler:
self.shared_session = shared_session
self.transport = transport
self.follow_redirects = follow_redirects
self.http2 = http2
self._owns_client = True
self._client = self.create_client(
timeout=timeout,
@ -711,8 +751,11 @@ class AsyncHTTPHandler:
follow_redirects=self.follow_redirects,
trust_env=False,
)
use_http2: Final = _resolve_http2(self.http2, ssl_verify)
replaces_aiohttp: Final = use_http2 and AsyncHTTPHandler.should_use_aiohttp_transport()
limits: Final = _AIOHTTP_EQUIVALENT_LIMITS if replaces_aiohttp else DEFAULT_LIMITS
# Get unified SSL configuration
ssl_config: Final = get_ssl_configuration(ssl_verify)
ssl_config: Final = get_ssl_configuration(ssl_verify, http2=use_http2)
# An SSL certificate used by the requested host to authenticate the client.
# /path/to/client.pem
@ -726,14 +769,23 @@ class AsyncHTTPHandler:
ssl_context=ssl_config if isinstance(ssl_config, ssl.SSLContext) else None,
ssl_verify=ssl_config if isinstance(ssl_config, bool) else None,
shared_session=shared_session,
http2=use_http2,
cert=cert,
limits=limits,
)
# Get default headers (User-Agent, overridable via LITELLM_USER_AGENT)
default_headers: Final = get_default_headers()
trust_env: Final = not (replaces_aiohttp and _aiohttp_env_proxies_disabled())
return httpx.AsyncClient(
transport=transport,
mounts=AsyncHTTPHandler.create_httpx_proxy_mounts(transport, verify=ssl_config, cert=cert),
mounts=AsyncHTTPHandler.create_httpx_proxy_mounts(
transport, verify=ssl_config, cert=cert, http2=use_http2, limits=limits
)
if trust_env
else None,
limits=limits,
event_hooks=event_hooks,
timeout=timeout,
verify=ssl_config,
@ -741,7 +793,8 @@ class AsyncHTTPHandler:
headers=default_headers,
cookies=blocked_cookie_jar(),
follow_redirects=self.follow_redirects,
http2=http2_enabled(),
http2=use_http2,
trust_env=trust_env,
)
async def close(self):
@ -1152,6 +1205,17 @@ class AsyncHTTPHandler:
# Mirrors LiteLLMAiohttpTransport._background_close_tasks.
_finalizer_close_tasks: ClassVar[set["asyncio.Task[None]"]] = set() # mutable-ok: strong refs for pending closes
@classmethod
async def _close_client_at_finalization(cls, client: httpx.AsyncClient) -> None:
try:
await client.aclose()
except (OSError, RuntimeError) as exc:
verbose_logger.debug("Error closing client at finalization: %s", exc)
finally:
task: Final = asyncio.current_task()
if task is not None:
cls._finalizer_close_tasks.discard(task)
@classmethod
def _on_finalizer_close_done(cls, task: "asyncio.Task[None]") -> None:
cls._finalizer_close_tasks.discard(task)
@ -1219,8 +1283,8 @@ class AsyncHTTPHandler:
# here is the cross-loop path the transport refuses.
self._dispose_wrapped_aiohttp_session()
return
task: Final = loop.create_task(self._client.aclose())
cls: Final = type(self)
task: Final = loop.create_task(cls._close_client_at_finalization(self._client))
cls._finalizer_close_tasks.add(task)
task.add_done_callback(cls._on_finalizer_close_done)
except Exception:
@ -1231,7 +1295,10 @@ class AsyncHTTPHandler:
ssl_context: ssl.SSLContext | None = None,
ssl_verify: bool | None = None,
shared_session: Optional["ClientSession"] = None,
) -> LiteLLMAiohttpTransport | AsyncHTTPTransport | None:
http2: bool = False,
cert: CertTypes | None = None,
limits: httpx.Limits = DEFAULT_LIMITS,
) -> LiteLLMAiohttpTransport | AsyncHTTPTransport | AsyncHTTP2ConnectionSpread | None:
"""
- Creates a transport for httpx.AsyncClient
- if litellm.force_ipv4 is True, it will return AsyncHTTPTransport with local_address="0.0.0.0"
@ -1249,7 +1316,7 @@ class AsyncHTTPHandler:
#########################################################
# AIOHTTP TRANSPORT is off by default
#########################################################
if AsyncHTTPHandler.should_use_aiohttp_transport():
if not http2 and AsyncHTTPHandler.should_use_aiohttp_transport():
return AsyncHTTPHandler.create_aiohttp_transport(
ssl_context=ssl_context,
ssl_verify=ssl_verify,
@ -1259,7 +1326,12 @@ class AsyncHTTPHandler:
#########################################################
# HTTPX TRANSPORT is used when aiohttp is not installed
#########################################################
return AsyncHTTPHandler._create_httpx_transport()
return AsyncHTTPHandler._create_httpx_transport(
http2=http2,
verify=ssl_context if ssl_context is not None else ssl_verify is not False,
cert=cert,
limits=limits,
)
_create_async_transport = create_async_transport
@ -1406,28 +1478,57 @@ class AsyncHTTPHandler:
_create_aiohttp_transport = create_aiohttp_transport
@staticmethod
def _create_httpx_transport() -> AsyncHTTPTransport | None:
def _create_httpx_transport(
http2: bool = False,
verify: ssl.SSLContext | bool = True,
cert: CertTypes | None = None,
limits: httpx.Limits = DEFAULT_LIMITS,
) -> AsyncHTTPTransport | AsyncHTTP2ConnectionSpread | None:
"""
Creates an AsyncHTTPTransport
- If HTTP/2 is on, it spreads requests over as many HTTP/2 connections as the stream limit needs
- If force_ipv4 is True, it will create an AsyncHTTPTransport with local_address set to "0.0.0.0"
- [Default] If force_ipv4 is False, it will return None
- [Default] Otherwise it will return None
"""
local_address: Final = _IPV4_LOCAL_ADDRESS if litellm.force_ipv4 else None
transport_verify: Final = (
_httpx_default_verify_unless_configured(verify)
if litellm.force_ipv4 and not AsyncHTTPHandler.should_use_aiohttp_transport()
else verify
)
if http2 or http2_enabled():
return AsyncHTTP2ConnectionSpread(
lambda lane_limits: AsyncHTTPTransport(
local_address=local_address, verify=transport_verify, cert=cert, http2=True, limits=lane_limits
),
limits,
)
if litellm.force_ipv4:
return AsyncHTTPTransport(local_address=_IPV4_LOCAL_ADDRESS, http2=http2_enabled())
else:
return None
return AsyncHTTPTransport(local_address=local_address, verify=transport_verify, cert=cert, limits=limits)
return None
@staticmethod
def create_httpx_proxy_mounts(
transport: LiteLLMAiohttpTransport | AsyncHTTPTransport | None,
transport: LiteLLMAiohttpTransport | AsyncHTTPTransport | AsyncHTTP2ConnectionSpread | None,
verify: VerifyTypes,
cert: CertTypes | None,
) -> Mapping[str, AsyncHTTPTransport | None] | None:
if not isinstance(transport, AsyncHTTPTransport):
http2: bool = False,
limits: httpx.Limits = DEFAULT_LIMITS,
) -> Mapping[str, httpx.AsyncBaseTransport | None] | None:
if not isinstance(transport, (AsyncHTTPTransport, AsyncHTTP2ConnectionSpread)):
return None
if http2 or http2_enabled():
return _environment_proxy_mounts(
lambda proxy_url: AsyncHTTP2ConnectionSpread(
lambda lane_limits: AsyncHTTPTransport(
proxy=proxy_url, verify=verify, cert=cert, http2=True, limits=lane_limits
),
limits,
)
)
return _environment_proxy_mounts(
lambda proxy_url: AsyncHTTPTransport(proxy=proxy_url, verify=verify, cert=cert, http2=http2_enabled())
lambda proxy_url: AsyncHTTPTransport(proxy=proxy_url, verify=verify, cert=cert, limits=limits)
)
_create_httpx_proxy_mounts = create_httpx_proxy_mounts
@ -1453,8 +1554,9 @@ class HTTPHandler:
self._client = self.create_client() if client is None else client
def create_client(self) -> httpx.Client:
use_http2: Final = http2_enabled()
# Get unified SSL configuration
ssl_config: Final = get_ssl_configuration(self.ssl_verify)
ssl_config: Final = get_ssl_configuration(self.ssl_verify, http2=use_http2)
# An SSL certificate used by the requested host to authenticate the client.
# /path/to/client.pem
@ -1463,17 +1565,19 @@ class HTTPHandler:
# Get default headers (User-Agent, overridable via LITELLM_USER_AGENT)
default_headers: Final = get_default_headers() if not self.disable_default_headers else None
transport: Final = self._create_sync_transport(verify=ssl_config, cert=cert, http2=use_http2)
# Create a client with a connection pool
return httpx.Client(
transport=self._create_sync_transport(),
mounts=self._create_sync_proxy_mounts(verify=ssl_config, cert=cert),
transport=transport,
mounts=self._create_sync_proxy_mounts(transport, verify=ssl_config, cert=cert, http2=use_http2),
timeout=self.timeout if self.timeout is not None else _DEFAULT_TIMEOUT,
verify=ssl_config,
cert=cert,
headers=default_headers,
cookies=blocked_cookie_jar(),
follow_redirects=self.follow_redirects,
http2=http2_enabled(),
http2=use_http2,
)
@property
@ -1769,27 +1873,34 @@ class HTTPHandler:
except Exception:
pass
def _create_sync_transport(self) -> HTTPTransport | None:
@staticmethod
def _create_sync_transport(verify: VerifyTypes, cert: CertTypes | None, http2: bool) -> HTTPTransport | None:
"""
Create an HTTP transport with IPv4 only if litellm.force_ipv4 is True.
Otherwise, return None.
Otherwise, return litellm.sync_transport or None.
Some users have seen httpx ConnectionError when using ipv6 - forcing ipv4 resolves the issue for them
"""
if litellm.force_ipv4:
return HTTPTransport(local_address=_IPV4_LOCAL_ADDRESS, http2=http2_enabled())
else:
return getattr(litellm, "sync_transport", None)
return HTTPTransport(
local_address=_IPV4_LOCAL_ADDRESS,
verify=_httpx_default_verify_unless_configured(verify),
cert=cert,
http2=http2,
)
return getattr(litellm, "sync_transport", None)
@staticmethod
def _create_sync_proxy_mounts(
transport: httpx.BaseTransport | None,
verify: VerifyTypes,
cert: CertTypes | None,
) -> Mapping[str, HTTPTransport | None] | None:
if not litellm.force_ipv4:
http2: bool,
) -> Mapping[str, httpx.BaseTransport | None] | None:
if transport is None or transport is getattr(litellm, "sync_transport", None):
return None
return _environment_proxy_mounts(
lambda proxy_url: HTTPTransport(proxy=proxy_url, verify=verify, cert=cert, http2=http2_enabled())
lambda proxy_url: HTTPTransport(proxy=proxy_url, verify=verify, cert=cert, http2=http2)
)
@ -1873,7 +1984,7 @@ def get_httpx_client(params: dict | None = None) -> HTTPHandler:
cache = LLMClientCache()
setattr(litellm, "in_memory_llm_clients_cache", cache)
_cached_client: Final = cache.get_cache(_cache_key_name)
_cached_client: Final = cache.get_cache(_cache_key_name, loop_bound=False)
if _cached_client:
return _cached_client
@ -1889,6 +2000,7 @@ def get_httpx_client(params: dict | None = None) -> HTTPHandler:
value=_new_client,
ttl=_DEFAULT_TTL_FOR_HTTPX_CLIENTS,
litellm_owned_client=True,
loop_bound=False,
)
return _new_client

View file

@ -26,6 +26,7 @@ from typing import (
from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
import httpx
from h2.events import ConnectionTerminated
from httpx import USE_CLIENT_DEFAULT
from httpx._types import FileContent
from openai.types.file_deleted import FileDeleted
@ -371,6 +372,34 @@ class _SignedBodyKwargs(TypedDict, total=False):
json: ReadOnly[dict[str, object]]
_GOAWAY_RETRY_MESSAGE: Final = "Vector store search connection was closed by an HTTP/2 GOAWAY, retrying once"
def _closed_by_http2_goaway(error: httpx.RemoteProtocolError) -> bool:
cause: Final = error.__cause__
return cause is not None and any(isinstance(arg, ConnectionTerminated) for arg in cause.args)
async def _asend_with_goaway_retry(send: Callable[[], Awaitable[httpx.Response]]) -> httpx.Response:
try:
return await send()
except httpx.RemoteProtocolError as error:
if not _closed_by_http2_goaway(error):
raise
verbose_logger.warning(_GOAWAY_RETRY_MESSAGE)
return await send()
def _send_with_goaway_retry(send: Callable[[], httpx.Response]) -> httpx.Response:
try:
return send()
except httpx.RemoteProtocolError as error:
if not _closed_by_http2_goaway(error):
raise
verbose_logger.warning(_GOAWAY_RETRY_MESSAGE)
return send()
def _is_sized_json_body(response: httpx.Response) -> bool:
headers: Final = httpx.Headers(response.headers)
media_types: Final = (value.split(";")[0].strip().lower() for value in headers.get_list("content-type"))
@ -2438,8 +2467,7 @@ class BaseLLMHTTPHandler:
)
else:
verbose_logger.debug(
"Anthropic /v1/messages: invalid thinking signature; "
"stripping thinking blocks and retrying (attempt %s/%s).",
"Anthropic /v1/messages: retrying after a recoverable HTTP error (attempt %s/%s).",
len(recovered_errors) + 2,
max_attempts,
)
@ -10627,7 +10655,10 @@ class BaseLLMHTTPHandler:
if client is None or not isinstance(client, AsyncHTTPHandler):
async_httpx_client = get_async_httpx_client(
llm_provider=litellm.LlmProviders(custom_llm_provider),
params={"ssl_verify": litellm_params.get("ssl_verify", None)},
params={
"ssl_verify": litellm_params.get("ssl_verify", None),
**vector_store_provider_config.get_async_httpx_client_params(),
},
)
else:
async_httpx_client = client
@ -10699,11 +10730,8 @@ class BaseLLMHTTPHandler:
request_data: Final = json.dumps(request_body) if signed_json_body is None else signed_json_body
try:
response: Final = await async_httpx_client.post(
url=url,
headers=headers,
data=request_data,
timeout=timeout,
response: Final = await _asend_with_goaway_retry(
partial(async_httpx_client.post, url=url, headers=headers, data=request_data, timeout=timeout)
)
except httpx.TimeoutException:
raise vector_store_provider_config.get_error_class(
@ -10836,11 +10864,8 @@ class BaseLLMHTTPHandler:
request_data: Final = json.dumps(request_body) if signed_json_body is None else signed_json_body
try:
response: Final = sync_httpx_client.post(
url=url,
headers=headers,
data=request_data,
timeout=timeout,
response: Final = _send_with_goaway_retry(
partial(sync_httpx_client.post, url=url, headers=headers, data=request_data, timeout=timeout)
)
except httpx.TimeoutException:
raise vector_store_provider_config.get_error_class(
@ -12824,7 +12849,7 @@ class BaseLLMHTTPHandler:
provider_config=text_to_speech_provider_config,
)
speech_response: Final = text_to_speech_provider_config.transform_text_to_speech_response(
speech_response: Final = await text_to_speech_provider_config.async_transform_text_to_speech_response(
model=model,
raw_response=response,
logging_obj=logging_obj,

View file

@ -24,7 +24,8 @@ DASHSCOPE_ANTHROPIC_PATH: Final = "/apps/anthropic"
DASHSCOPE_NATIVE_V1_PATH: Final = "/api/v1"
DASHSCOPE_COMPATIBLE_API_PATH: Final = "/compatible-api/v1"
DASHSCOPE_RERANK_PATH: Final = f"{DASHSCOPE_COMPATIBLE_API_PATH}/reranks"
DASHSCOPE_IMAGE_PATH: Final = "/api/v1/services/aigc/multimodal-generation/generation"
DASHSCOPE_MULTIMODAL_GENERATION_PATH: Final = "/api/v1/services/aigc/multimodal-generation/generation"
DASHSCOPE_IMAGE_PATH: Final = DASHSCOPE_MULTIMODAL_GENERATION_PATH
DASHSCOPE_DEFAULT_ORIGIN: Final = "https://dashscope.aliyuncs.com"
_STANDARD_DASHSCOPE_HOSTS: Final = frozenset(
@ -100,6 +101,21 @@ def is_dashscope_native_endpoint_url(api_base: str) -> bool:
return urlparse(api_base).path.rstrip("/").startswith(f"{DASHSCOPE_NATIVE_V1_PATH}/")
def dashscope_api_root(api_base: str) -> str:
endpoints: Final = resolve_dashscope_endpoints(api_base)
if endpoints is not None:
return endpoints.origin
trimmed: Final = api_base.rstrip("/")
if trimmed.endswith(DASHSCOPE_CHAT_COMPATIBLE_PATH):
return trimmed.removesuffix(DASHSCOPE_CHAT_COMPATIBLE_PATH)
root, separator, _ = trimmed.partition(DASHSCOPE_NATIVE_V1_PATH)
return root if separator else trimmed
def is_aliyun_host(host: str | None) -> bool:
return host is not None and (host == "aliyuncs.com" or host.endswith(".aliyuncs.com"))
def resolve_dashscope_openai_api_base(
api_base: str | None,
env_var: str = "DASHSCOPE_API_BASE",
@ -140,9 +156,7 @@ def _chat_shaped_aliyun_origin(api_base: str | None) -> str | None:
if api_base is None:
return None
parsed: Final = urlparse(api_base)
host: Final = parsed.hostname or ""
on_aliyun_host: Final = host == "aliyuncs.com" or host.endswith(".aliyuncs.com")
if not on_aliyun_host or parsed.path.rstrip("/") != DASHSCOPE_CHAT_COMPATIBLE_PATH:
if not is_aliyun_host(parsed.hostname) or parsed.path.rstrip("/") != DASHSCOPE_CHAT_COMPATIBLE_PATH:
return None
return f"{parsed.scheme}://{parsed.netloc}"

View file

@ -0,0 +1,160 @@
from collections.abc import Mapping
from typing import TYPE_CHECKING, Final
import httpx
from httpx._types import RequestFiles
from pydantic import TypeAdapter
from typing_extensions import override
import litellm
from litellm.images.utils import image_to_data_url
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
from litellm.llms.dashscope.common_utils import (
DASHSCOPE_CHAT_COMPATIBLE_PATH,
DASHSCOPE_MULTIMODAL_GENERATION_PATH,
dashscope_api_root,
is_dashscope_native_endpoint_url,
resolve_dashscope_endpoints,
)
from litellm.llms.dashscope.image_generation.transformation import (
DEFAULT_API_BASE,
OPENAI_TO_DASHSCOPE_SIZE,
DashScopeImageGenerationConfig,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.images.main import ImageEditOptionalRequestParams
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import ImageResponse
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
MAX_INPUT_IMAGES: Final = 3
DASHSCOPE_IMAGE_EDIT_PARAMS: Final = ("n", "size", "negative_prompt", "prompt_extend", "watermark", "seed")
_IMAGES: Final = TypeAdapter(tuple[object, ...])
def _dashscope_size(size: object) -> object:
if not isinstance(size, str):
return size
return OPENAI_TO_DASHSCOPE_SIZE.get(size, size.replace("x", "*"))
def _deployment_edit_url(api_base: str) -> str:
derives_from_root: Final = not is_dashscope_native_endpoint_url(api_base) and (
resolve_dashscope_endpoints(api_base) is not None
or api_base.rstrip("/").endswith(DASHSCOPE_CHAT_COMPATIBLE_PATH)
)
return f"{dashscope_api_root(api_base)}{DASHSCOPE_MULTIMODAL_GENERATION_PATH}" if derives_from_root else api_base
class DashScopeImageEditConfig(BaseImageEditConfig):
def __init__(self, generation_config: DashScopeImageGenerationConfig | None = None) -> None:
super().__init__()
self._generation_config: Final = generation_config or DashScopeImageGenerationConfig()
@override
def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: mirrors override contract
return list(DASHSCOPE_IMAGE_EDIT_PARAMS)
@override
def map_openai_params(
self,
image_edit_optional_params: ImageEditOptionalRequestParams,
model: str,
drop_params: bool,
) -> dict[str, object]: # mutable-ok: mirrors override contract
return {
key: _dashscope_size(value) if key == "size" else value
for key, value in image_edit_optional_params.items()
if key in DASHSCOPE_IMAGE_EDIT_PARAMS
}
@override
def validate_environment(
self,
headers: dict[str, str], # mutable-ok: mirrors override contract
model: str,
api_key: str | None = None,
litellm_params: dict[str, object] | None = None, # mutable-ok: mirrors override contract
api_base: str | None = None,
) -> dict[str, str]: # mutable-ok: mirrors override contract
return self._generation_config.validate_environment(
headers=dict(headers),
model=model,
messages=[],
optional_params={},
litellm_params=dict(litellm_params or {}),
api_key=api_key,
api_base=api_base,
)
@override
def get_complete_url(
self,
model: str,
api_base: str | None,
litellm_params: dict[str, object], # mutable-ok: mirrors override contract
) -> str:
if api_base is not None:
return _deployment_edit_url(api_base)
env_api_base: Final = get_secret_str("DASHSCOPE_API_BASE")
derived: Final = (
f"{dashscope_api_root(env_api_base)}{DASHSCOPE_MULTIMODAL_GENERATION_PATH}" if env_api_base else None
)
return get_secret_str("DASHSCOPE_API_BASE_IMAGE") or derived or DEFAULT_API_BASE
@override
def use_multipart_form_data(self) -> bool:
return False
@override
def transform_image_edit_request(
self,
model: str,
prompt: str | None,
image: object,
image_edit_optional_request_params: Mapping[str, object],
litellm_params: GenericLiteLLMParams,
headers: Mapping[str, str],
) -> tuple[dict[str, object], RequestFiles]: # mutable-ok: mirrors override contract
candidates: Final = _IMAGES.validate_python(image) if isinstance(image, list) else (image,)
images: Final = tuple(candidate for candidate in candidates if candidate is not None)
if len(images) > MAX_INPUT_IMAGES:
raise litellm.BadRequestError(
message=f"DashScope image edit accepts at most {MAX_INPUT_IMAGES} input images, got {len(images)}",
model=model,
llm_provider="dashscope",
)
prompt_content: Final = ({"text": prompt},) if prompt is not None else ()
return {
"model": model,
"input": {
"messages": [
{
"role": "user",
"content": [*({"image": image_to_data_url(item)} for item in images), *prompt_content],
}
]
},
"parameters": dict(image_edit_optional_request_params),
}, ()
@override
def transform_image_edit_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: "LiteLLMLoggingObj",
) -> ImageResponse:
return self._generation_config.transform_image_generation_response(
model=model,
raw_response=raw_response,
model_response=ImageResponse(),
logging_obj=logging_obj,
request_data={},
optional_params={},
litellm_params={},
encoding=None,
)

View file

@ -0,0 +1,19 @@
from typing import Final
import litellm
from litellm.litellm_core_utils.llm_cost_calc.utils import resolve_image_model_info
from litellm.types.utils import ImageResponse, ModelInfo
def cost_calculator(model: str, image_response: ImageResponse, model_info: ModelInfo | None = None) -> float:
prices: Final = resolve_image_model_info(
model=model, custom_llm_provider=litellm.LlmProviders.DASHSCOPE.value, model_info=model_info
)
output_cost_per_image: Final = prices.get("output_cost_per_image")
if output_cost_per_image is None:
raise ValueError(f"No output_cost_per_image found for dashscope/{model}")
input_images: Final = image_response.hidden_params.get("billable_input_image_count")
input_image_cost: Final = (prices.get("input_cost_per_image") or 0.0) * (
input_images if isinstance(input_images, int) else 0
)
return output_cost_per_image * len(image_response.data or ()) + input_image_cost

View file

@ -23,7 +23,7 @@ Response format:
}
"""
from collections.abc import Iterable, Mapping
from collections.abc import Iterable, Iterator, Mapping
from typing import TYPE_CHECKING, Any, Final
import httpx
@ -61,7 +61,7 @@ _DECODED_JSON: Final = TypeAdapter(object)
DEFAULT_API_BASE: Final = "https://dashscope-intl.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation"
# Maps OpenAI size strings (WxH) to DashScope size strings (W*H)
OPENAI_TO_DASHSCOPE_SIZE: Final[dict] = {
OPENAI_TO_DASHSCOPE_SIZE: Final[Mapping[str, str]] = {
"256x256": "256*256",
"512x512": "512*512",
"1024x1024": "1024*1024",
@ -71,6 +71,24 @@ OPENAI_TO_DASHSCOPE_SIZE: Final[dict] = {
}
def _message_image_urls(message: Mapping[str, object]) -> Iterator[str]:
contents: Final = _JSON_OBJECTS.validate_python(message.get("content", []))
return (url for url in (item.get("image") for item in contents) if isinstance(url, str) and url)
def _image_urls(response_object: Mapping[str, object]) -> Iterator[str]:
output: Final = _JSON_OBJECT.validate_python(response_object.get("output", {}))
choices: Final = _JSON_OBJECTS.validate_python(output.get("choices", []))
for choice in choices:
yield from _message_image_urls(_JSON_OBJECT.validate_python(choice.get("message", {})))
def _dashscope_size(size: object) -> object:
if not isinstance(size, str):
return size
return OPENAI_TO_DASHSCOPE_SIZE.get(size, size.replace("x", "*"))
class DashScopeImageGenerationConfig(BaseImageGenerationConfig):
"""
Configuration for DashScope image generation (qwen-image-2.0, qwen-image-2.0-pro,
@ -82,24 +100,16 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig):
def map_openai_params(
self,
non_default_params: dict,
optional_params: dict,
non_default_params: dict[str, object],
optional_params: dict[str, object],
model: str,
drop_params: bool,
) -> dict:
) -> dict[str, object]:
supported_params: Final = self.get_supported_openai_params(model)
mapped: Final[dict] = {}
for k, v in non_default_params.items():
if k in optional_params:
continue
if k not in supported_params:
continue
if k == "size":
# Convert "WxH" → "W*H"
mapped["size"] = OPENAI_TO_DASHSCOPE_SIZE.get(v, v.replace("x", "*"))
else:
mapped[k] = v
return mapped
requested: Final = {
k: v for k, v in non_default_params.items() if k in supported_params and k not in optional_params
}
return {k: _dashscope_size(v) if k == "size" else v for k, v in requested.items()}
def _resolve_api_key(self, api_key: str | None) -> str:
resolved_api_key: Final = api_key or get_secret_str("DASHSCOPE_API_KEY")
@ -121,8 +131,8 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig):
api_base: str | None,
api_key: str | None,
model: str,
optional_params: dict,
litellm_params: dict,
optional_params: dict[str, object],
litellm_params: dict[str, object],
stream: bool | None = None,
) -> str:
image_env_api_base: Final = self._image_api_base_env()
@ -140,14 +150,14 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig):
def validate_environment(
self,
headers: dict,
headers: dict[str, str],
model: str,
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
optional_params: dict[str, object],
litellm_params: dict[str, object],
api_key: str | None = None,
api_base: str | None = None,
) -> dict:
) -> dict[str, str]:
headers["Authorization"] = f"Bearer {self._resolve_api_key(api_key)}"
headers["Content-Type"] = "application/json"
return headers
@ -186,9 +196,9 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig):
raw_response: httpx.Response,
model_response: ImageResponse,
logging_obj: LiteLLMLoggingObj,
request_data: dict,
optional_params: dict,
litellm_params: dict,
request_data: dict[str, object],
optional_params: dict[str, object],
litellm_params: dict[str, object],
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
@ -225,17 +235,21 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig):
headers=raw_response.headers,
)
if not model_response.data:
model_response.data = []
output: Final = _JSON_OBJECT.validate_python(response_object.get("output", {}))
choices: Final = _JSON_OBJECTS.validate_python(output.get("choices", []))
for choice in choices:
message = _JSON_OBJECT.validate_python(choice.get("message", {}))
content_list = _JSON_OBJECTS.validate_python(message.get("content", []))
for content_item in content_list:
image_url = content_item.get("image")
if image_url:
model_response.data.append(ImageObject(url=image_url))
return model_response
images: Final = [*(model_response.data or ()), *(ImageObject(url=url) for url in _image_urls(response_object))]
usage: Final = response_object.get("usage")
usage_fields: Final[Mapping[str, object]] = usage if isinstance(usage, Mapping) else {}
width: Final = usage_fields.get("output_width", usage_fields.get("width"))
height: Final = usage_fields.get("output_height", usage_fields.get("height"))
image_response: Final = model_response.model_copy(
update={
"data": images,
**({"size": f"{width}x{height}"} if isinstance(width, int) and isinstance(height, int) else {}),
}
)
input_image_count: Final = usage_fields.get("input_image_count")
if isinstance(input_image_count, int) and input_image_count > 0:
image_response.hidden_params = {
**model_response.hidden_params,
"billable_input_image_count": input_image_count,
}
return image_response

View file

@ -0,0 +1,297 @@
from collections.abc import Callable, Mapping
from dataclasses import dataclass
from itertools import chain
from typing import TYPE_CHECKING, Final
from urllib.parse import urlparse
import httpx
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from typing_extensions import override
import litellm
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.url_utils import SSRFError, async_safe_get, safe_get
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.text_to_speech.transformation import (
BaseTextToSpeechConfig,
TextToSpeechRequestData,
)
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # factory stub leaves params untyped
get_httpx_client, # pyright: ignore[reportUnknownVariableType] # factory stub leaves params untyped
)
from litellm.llms.dashscope.common_utils import (
DASHSCOPE_MULTIMODAL_GENERATION_PATH,
DashScopeError,
dashscope_api_root,
is_aliyun_host,
missing_dashscope_family_key_message,
resolve_dashscope_family_api_key,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import HttpxBinaryResponseContent
from litellm.types.utils import Usage
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
DASHSCOPE_TTS_DEFAULT_API_BASE: Final = "https://dashscope-intl.aliyuncs.com"
_SPEECH_SYNTHESIZER_MODEL_PREFIX: Final = "qwen-audio-"
_TIMEOUT_PHASES: Final = TypeAdapter(dict[str, float | None])
@dataclass(frozen=True, slots=True)
class _TextToSpeechFamily:
path: str
response_formats: frozenset[str]
sends_format: bool
takes_instructions: bool
MULTIMODAL_GENERATION_FAMILY: Final = _TextToSpeechFamily(
path=DASHSCOPE_MULTIMODAL_GENERATION_PATH,
response_formats=frozenset({"wav"}),
sends_format=False,
takes_instructions=False,
)
SPEECH_SYNTHESIZER_FAMILY: Final = _TextToSpeechFamily(
path="/api/v1/services/audio/tts/SpeechSynthesizer",
response_formats=frozenset({"mp3", "wav", "opus", "pcm"}),
sends_format=True,
takes_instructions=True,
)
class _Audio(BaseModel):
model_config = ConfigDict(frozen=True)
url: str
class _Output(BaseModel):
model_config = ConfigDict(frozen=True)
audio: _Audio
class _Usage(BaseModel):
model_config = ConfigDict(frozen=True)
input_tokens: int | None = None
output_tokens: int | None = None
characters: int | None = None
class _SpeechResponse(BaseModel):
model_config = ConfigDict(frozen=True)
output: _Output | None = None
usage: _Usage | None = None
code: str | None = None
message: str | None = None
def text_to_speech_family(model: str) -> _TextToSpeechFamily:
if model.startswith(_SPEECH_SYNTHESIZER_MODEL_PREFIX):
return SPEECH_SYNTHESIZER_FAMILY
return MULTIMODAL_GENERATION_FAMILY
def _mapped_field(family: _TextToSpeechFamily, key: str, value: object) -> Mapping[str, object] | None:
match key:
case "response_format" if value in family.response_formats:
return {"format": value} if family.sends_format else {}
case "instructions" if family.takes_instructions:
return {"instruction": value}
case _:
return None
def _unsupported(model: str, unsupported: tuple[str, ...]) -> UnsupportedParamsError:
return UnsupportedParamsError(
message=f"DashScope {model} does not support {', '.join(unsupported)}, set drop_params=True to ignore it",
model=model,
llm_provider="dashscope",
)
def _speech_response(raw_response: httpx.Response) -> _SpeechResponse:
try:
return _SpeechResponse.model_validate_json(raw_response.content)
except ValidationError as e:
raise DashScopeError(
status_code=502,
message=f"Unexpected DashScope text-to-speech response: {e}",
headers=raw_response.headers,
) from e
def _https_on_aliyun(url: str) -> str:
parsed: Final = urlparse(url)
return (
parsed._replace(scheme="https").geturl() if parsed.scheme == "http" and is_aliyun_host(parsed.hostname) else url
)
def _audio_url(raw_response: httpx.Response) -> tuple[str, _Usage | None]:
parsed: Final = _speech_response(raw_response)
if parsed.output is None:
raise DashScopeError(
status_code=max(raw_response.status_code, 400),
message=parsed.message or parsed.code or raw_response.text,
headers=raw_response.headers,
)
return _https_on_aliyun(parsed.output.audio.url), parsed.usage
def _request_timeout(raw_response: httpx.Response) -> httpx.Timeout | None:
extension: Final = raw_response.request.extensions.get("timeout")
if extension is None:
return None
phases: Final = _TIMEOUT_PHASES.validate_python(extension)
return httpx.Timeout(
connect=phases.get("connect"), read=phases.get("read"), write=phases.get("write"), pool=phases.get("pool")
)
def _token_usage(usage: _Usage | None) -> Usage | None:
if usage is None or usage.input_tokens is None or usage.output_tokens is None:
return None
return Usage(
prompt_tokens=usage.input_tokens,
completion_tokens=usage.output_tokens,
total_tokens=usage.input_tokens + usage.output_tokens,
)
def _refused_audio_url(error: SSRFError) -> DashScopeError:
return DashScopeError(status_code=502, message=f"DashScope returned an audio URL that was refused: {error}")
def _binary_audio(audio_response: httpx.Response, usage: _Usage | None) -> HttpxBinaryResponseContent:
if audio_response.status_code != 200:
raise DashScopeError(
status_code=502,
message=f"DashScope audio download failed with HTTP {audio_response.status_code}",
headers=audio_response.headers,
)
binary: Final = HttpxBinaryResponseContent(audio_response)
binary.usage = _token_usage(usage)
if usage is not None and usage.characters is not None:
binary.hidden_params = {**binary.hidden_params, "billed_characters": usage.characters}
return binary
def _shared_async_client() -> AsyncHTTPHandler:
return get_async_httpx_client(llm_provider=litellm.LlmProviders.DASHSCOPE)
class DashScopeTextToSpeechConfig(BaseTextToSpeechConfig):
def __init__(
self,
sync_client: Callable[[], HTTPHandler] = get_httpx_client,
async_client: Callable[[], AsyncHTTPHandler] = _shared_async_client,
) -> None:
super().__init__()
self._sync_client: Final = sync_client
self._async_client: Final = async_client
@override
def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: mirrors override contract
family: Final = text_to_speech_family(model)
return ["voice", "response_format", *(("instructions",) if family.takes_instructions else ())]
@override
def map_openai_params(
self,
model: str,
optional_params: dict[str, object], # mutable-ok: mirrors override contract
voice: str | dict[str, object] | None = None, # mutable-ok: mirrors override contract
drop_params: bool = False,
kwargs: Mapping[str, object] | None = None,
) -> tuple[str | None, dict[str, object]]: # mutable-ok: mirrors override contract
family: Final = text_to_speech_family(model)
fields: Final = tuple((key, value, _mapped_field(family, key, value)) for key, value in optional_params.items())
unsupported: Final = (
*(f"{key}={value!r}" for key, value, field in fields if field is None),
*(("voice as an object",) if isinstance(voice, dict) else ()),
)
if unsupported and not (drop_params or drops_unsupported_params((kwargs or {}).get("drop_params"))):
raise _unsupported(model, unsupported)
supported: Final = tuple(field for _, _, field in fields if field is not None)
return voice if isinstance(voice, str) else None, dict(chain.from_iterable(f.items() for f in supported))
@override
def validate_environment(
self,
headers: dict[str, str], # mutable-ok: mirrors override contract
model: str,
api_key: str | None = None,
api_base: str | None = None,
) -> dict[str, str]: # mutable-ok: mirrors override contract
resolved_api_key: Final = resolve_dashscope_family_api_key("dashscope", api_key)
if resolved_api_key is None:
raise ValueError(missing_dashscope_family_key_message("dashscope"))
return {**headers, "Authorization": f"Bearer {resolved_api_key}", "Content-Type": "application/json"}
@override
def get_complete_url(
self,
model: str,
api_base: str | None,
litellm_params: dict[str, object], # mutable-ok: mirrors override contract
) -> str:
configured: Final = api_base or get_secret_str("DASHSCOPE_API_BASE")
root: Final = dashscope_api_root(configured) if configured else DASHSCOPE_TTS_DEFAULT_API_BASE
return f"{root}{text_to_speech_family(model).path}"
@override
def transform_text_to_speech_request(
self,
model: str,
input: str,
voice: str | None,
optional_params: dict[str, object], # mutable-ok: mirrors override contract
litellm_params: dict[str, object], # mutable-ok: mirrors override contract
headers: dict[str, str], # mutable-ok: mirrors override contract
) -> TextToSpeechRequestData:
voice_field: Final = {"voice": voice} if voice is not None else {}
return {"dict_body": {"model": model, "input": {"text": input, **voice_field, **optional_params}}}
@override
def transform_text_to_speech_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: "LiteLLMLoggingObj",
) -> HttpxBinaryResponseContent:
audio_url, usage = _audio_url(raw_response)
try:
audio_response: Final = safe_get(self._sync_client(), audio_url, timeout=_request_timeout(raw_response))
except SSRFError as e:
raise _refused_audio_url(e) from e
return _binary_audio(audio_response, usage)
@override
async def async_transform_text_to_speech_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: "LiteLLMLoggingObj",
) -> HttpxBinaryResponseContent:
audio_url, usage = _audio_url(raw_response)
try:
audio_response: Final = await async_safe_get(
self._async_client(), audio_url, timeout=_request_timeout(raw_response)
)
except SSRFError as e:
raise _refused_audio_url(e) from e
return _binary_audio(audio_response, usage)
@override
def get_error_class(
self,
error_message: str,
status_code: int,
headers: dict[str, str] | httpx.Headers, # mutable-ok: mirrors override contract
) -> BaseLLMException:
return DashScopeError(status_code=status_code, message=error_message, headers=httpx.Headers(headers))

View file

View file

@ -0,0 +1,34 @@
from collections.abc import Mapping
from types import MappingProxyType
from typing import Final
EU_JEV_LATEST_MODEL: Final = "eu-jeff-latest"
_EU_JEV_CURRENT_RELEASE_NAMES: Final = frozenset((EU_JEV_LATEST_MODEL, "jeff-latest", "jeff-1.0.0"))
_EU_JEV_PREFIX: Final = "eu_jev/"
def eu_jev_registry_model(requested_model: str | None) -> str:
release: Final = requested_model.removeprefix(_EU_JEV_PREFIX) if requested_model is not None else None
if release is None or release in _EU_JEV_CURRENT_RELEASE_NAMES:
return EU_JEV_LATEST_MODEL
return release
def eu_jev_scoped_model(decision_body: Mapping[str, object]) -> str:
requested_model: Final = decision_body.get("model")
return f"{_EU_JEV_PREFIX}{eu_jev_registry_model(requested_model if isinstance(requested_model, str) else None)}"
def eu_jev_upstream_body(hook_body: Mapping[str, object], client_body: Mapping[str, object]) -> Mapping[str, object]:
hook_model: Final = hook_body.get("model")
without_model: Final = {key: value for key, value in hook_body.items() if key != "model"}
if hook_model == eu_jev_scoped_model(client_body):
client_model: Final = client_body.get("model")
if client_model is None:
return MappingProxyType(without_model)
if isinstance(client_model, str):
return MappingProxyType({**without_model, "model": client_model.removeprefix(_EU_JEV_PREFIX)})
return MappingProxyType({**without_model, "model": client_model})
if isinstance(hook_model, str):
return MappingProxyType({**without_model, "model": hook_model.removeprefix(_EU_JEV_PREFIX)})
return hook_body

View file

@ -0,0 +1,9 @@
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
class EuJevDecisionsConfig(BaseDecisionsConfig):
api_key_env = ("EU_JEV_API_KEY",)
api_base_env = ("EU_JEV_API_BASE",)
def get_default_api_base(self) -> str | None:
return "https://eu-jev.com"

View file

@ -5,11 +5,30 @@ LiteLLM Proxy supports the OpenAI Responses API natively when the underlying mod
This config enables pass-through behavior to the proxy's /v1/responses endpoint.
"""
from typing import Final
import httpx
from pydantic import TypeAdapter, ValidationError
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.utils import LlmProviders
_ERROR_BODY: Final = TypeAdapter(dict[str, object])
def _openai_sdk_error_body(
error_message: str,
) -> dict[str, object] | None: # mutable-ok: BaseLLMException.body is a dict
try:
decoded: Final = _ERROR_BODY.validate_json(error_message)
except ValidationError:
return None
nested: Final = decoded.get("error")
return _ERROR_BODY.validate_python(nested) if isinstance(nested, dict) else decoded
class LiteLLMProxyResponsesAPIConfig(OpenAIResponsesAPIConfig):
"""
@ -43,6 +62,19 @@ class LiteLLMProxyResponsesAPIConfig(OpenAIResponsesAPIConfig):
return join_api_base(api_base, "responses")
def get_error_class(
self,
error_message: str,
status_code: int,
headers: dict[str, object] | httpx.Headers, # mutable-ok: inherited contract
) -> BaseLLMException:
return BaseLLMException(
status_code=status_code,
message=error_message,
headers=headers,
body=_openai_sdk_error_body(error_message),
)
def supports_native_websocket(self) -> bool:
"""LiteLLM Proxy does not support native WebSocket for Responses API"""
return False

View file

@ -275,6 +275,10 @@ def build_output_token_limit_response(
return e.response.headers, (_aiter_once(chunk) if is_async else _iter_once(chunk))
def _is_loop_bound(client_initialization_params: Mapping[str, object]) -> bool:
return client_initialization_params.get("is_async") is not False
class BaseOpenAILLM:
"""
Base class for OpenAI LLMs for getting their httpx clients and SSL verification settings
@ -289,7 +293,9 @@ class BaseOpenAILLM:
client_initialization_params=client_initialization_params,
client_type=client_type,
)
_cached_client: Final = litellm.in_memory_llm_clients_cache.get_cache(_cache_key)
_cached_client: Final = litellm.in_memory_llm_clients_cache.get_cache(
_cache_key, loop_bound=_is_loop_bound(client_initialization_params)
)
return _cached_client
@staticmethod
@ -330,6 +336,7 @@ class BaseOpenAILLM:
value=openai_client,
ttl=_DEFAULT_TTL_FOR_HTTPX_CLIENTS,
litellm_owned_client=litellm_owned_client,
loop_bound=_is_loop_bound(client_initialization_params),
)
@staticmethod

View file

@ -15,7 +15,9 @@ from litellm._logging import verbose_logger
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.tool_fields import DropPolicy
from litellm.llms.anthropic.messages.context_management import AnthropicContextManagementError
from litellm.llms.anthropic.messages.convertible_blocks import deployment_drops_params, unconvertible_blocks_error
from litellm.llms.anthropic.messages.responses_adapters.handler import count_view_messages
from litellm.llms.anthropic.messages.responses_adapters.transformation import (
LiteLLMAnthropicToResponsesAPIAdapter,
)
@ -95,6 +97,32 @@ def _count_error(model_to_use: str, request_model: str, message: str, status_cod
)
async def _sent_messages(
model: str,
messages: Sequence[Mapping[str, object]],
tools: Sequence[Mapping[str, object]] | None,
system: object,
messages_format: TokenCountMessagesFormat,
context_management: CountContextManagement | None,
) -> Sequence[Mapping[str, object]] | AnthropicContextManagementError:
match messages_format:
case "openai":
return messages
case "anthropic":
try:
return await count_view_messages(
model=model,
messages=messages,
tools=tools,
system=system,
context_management=context_management,
)
except AnthropicContextManagementError as error:
return error
case _:
assert_never(messages_format)
class OpenAITokenCounter(BaseTokenCounter):
"""Token counter implementation for OpenAI provider using the Responses API."""
@ -175,13 +203,28 @@ class OpenAITokenCounter(BaseTokenCounter):
rejected_before_provider_call=True,
)
sent_messages: Final = await _sent_messages(
model_to_use, messages, tools, system, messages_format, context_management
)
if isinstance(sent_messages, AnthropicContextManagementError):
return TokenCountResponse(
total_tokens=0,
request_model=request_model,
model_used=model_to_use,
tokenizer_type="openai_api",
error=True,
error_message=sent_messages.message,
status_code=sent_messages.status_code,
rejected_before_provider_call=True,
)
resolved: Final = _count_credential(deployment, request_model or model_to_use)
if isinstance(resolved, OpenAIError):
return _count_error(model_to_use, request_model, resolved.message, resolved.status_code)
input_items, instructions, counted_tools, text = self._responses_request_parts(
model_to_use=model_to_use,
messages=messages,
messages=sent_messages,
tools=tools,
system=system,
messages_format=messages_format,

View file

@ -11,6 +11,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.litellm_logging import (
get_standard_logging_object_payload, # pyright: ignore[reportUnknownVariableType] # legacy helper has an untyped signature
)
from litellm.llms.eu_jev.common_utils import EU_JEV_LATEST_MODEL, eu_jev_registry_model
from litellm.llms.laya.common_utils import laya_response_model
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.utils import ModelResponse, StandardPassThroughResponseObject, Usage
@ -82,13 +83,21 @@ class TypeSafePassthroughLoggingHandler:
response_model: Final = (
laya_response_model(response_body, request_model) if custom_llm_provider == "laya" else response.model
)
logged_model: Final = response_model or request_model or "unknown"
registry_request_model: Final = (
eu_jev_registry_model(request_model) if custom_llm_provider == "eu_jev" else request_model
)
logged_model: Final = response_model or registry_request_model or "unknown"
model_name: Final = f"{custom_llm_provider}/{logged_model}"
usage: Final = response.usage or _TypeSafeUsage()
input_tokens: Final = usage.input_tokens
output_tokens: Final = usage.output_tokens
candidate_models: Final = (
(response_model, registry_request_model, EU_JEV_LATEST_MODEL)
if custom_llm_provider == "eu_jev"
else (response_model, registry_request_model)
)
candidate_model_keys: Final = tuple(
f"{custom_llm_provider}/{model}" for model in (response_model, request_model) if model is not None
f"{custom_llm_provider}/{model}" for model in candidate_models if model is not None
)
pricing: Final = _pricing_for(candidate_model_keys)
response_cost: Final = (

View file

@ -234,6 +234,9 @@ class VertexSearchAPIVectorStoreConfig(BaseVectorStoreConfig, VertexBase):
def __init__(self):
super().__init__()
def get_async_httpx_client_params(self) -> Mapping[str, object]:
return MappingProxyType({"http2": True})
@staticmethod
def get_supported_extra_body_fields(is_engine: bool = False) -> frozenset[str]:
"""
@ -458,3 +461,6 @@ class VertexSearchAPIVectorStoreConfig(BaseVectorStoreConfig, VertexBase):
input_cost_per_query: Final = model_info.get("input_cost_per_query") or 0.0
return input_cost_per_query, 0.0
vertex_search_api_vector_store_config: Final = VertexSearchAPIVectorStoreConfig()

View file

@ -7975,6 +7975,8 @@ async def atext_completion(*args, **kwargs) -> TextCompletionResponse | TextComp
model=model,
custom_llm_provider=custom_llm_provider,
stream_options=kwargs.get("stream_options"),
usage_injected_by_proxy="_litellm_strip_stream_usage" in kwargs
and kwargs["_litellm_strip_stream_usage"] is True,
)
else:
## OpenAI / Azure Text Completion Returns here
@ -8824,8 +8826,9 @@ def speech(
extra_headers: Final = kwargs.get("extra_headers", None)
model_info: Final = kwargs.get("model_info", None)
shared_session: Final = kwargs.get("shared_session", None)
caller_api_base: Final = api_base or base_url
model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider(
model=model, custom_llm_provider=custom_llm_provider, api_base=api_base or base_url
model=model, custom_llm_provider=custom_llm_provider, api_base=caller_api_base
)
kwargs.pop("tags", [])
@ -8880,13 +8883,13 @@ def speech(
custom_llm_provider=custom_llm_provider,
)
response: HttpxBinaryResponseContent | Coroutine[object, object, HttpxBinaryResponseContent] | None = None
if custom_llm_provider == "edenai":
litellm_params_dict["api_base"] = api_base
if custom_llm_provider in ("edenai", "dashscope") and text_to_speech_provider_config is not None:
litellm_params_dict["api_base"] = caller_api_base if custom_llm_provider == "dashscope" else api_base
response = base_llm_http_handler.text_to_speech_handler(
model=model,
input=input,
voice=voice if isinstance(voice, str) else None,
text_to_speech_provider_config=text_to_speech_provider_config or litellm.EdenAITextToSpeechConfig(),
text_to_speech_provider_config=text_to_speech_provider_config,
text_to_speech_optional_params=optional_params,
custom_llm_provider=custom_llm_provider,
litellm_params=litellm_params_dict,

View file

@ -121,6 +121,7 @@ _ASYNC_FEATURES: Final[detectors.Detectors] = MappingProxyType(
Feature.PRE_REQUEST_HOOK: route_detectors.pre_request_hook,
Feature.WEBSEARCH_INTERCEPTION: route_detectors.websearch_interception,
Feature.MESSAGES_INTERCEPTOR: route_detectors.intercepted,
Feature.STRICT_TOOLS_RETRY: route_detectors.strict_tools_retry,
}
)

View file

@ -18346,25 +18346,31 @@
"dashscope/qwen-image-2.0": {
"litellm_provider": "dashscope",
"mode": "image_generation",
"source": "https://www.alibabacloud.com/help/en/model-studio/models",
"output_cost_per_image": 0.035,
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supported_endpoints": [
"/v1/images/generations"
"/v1/images/generations",
"/v1/images/edits"
]
},
"dashscope/qwen-image-2.0-pro": {
"litellm_provider": "dashscope",
"mode": "image_generation",
"source": "https://www.alibabacloud.com/help/en/model-studio/models",
"output_cost_per_image": 0.075,
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supported_endpoints": [
"/v1/images/generations"
"/v1/images/generations",
"/v1/images/edits"
]
},
"dashscope/qwen-image-3.0": {
"litellm_provider": "dashscope",
"mode": "image_generation",
"source": "https://www.alibabacloud.com/help/en/model-studio/models",
"output_cost_per_image": 0.03,
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supported_endpoints": [
"/v1/images/generations"
"/v1/images/generations",
"/v1/images/edits"
]
},
"dashscope/qwen-image-3.0-pro": {
@ -18372,7 +18378,54 @@
"mode": "image_generation",
"source": "https://www.alibabacloud.com/help/en/model-studio/models",
"supported_endpoints": [
"/v1/images/generations"
"/v1/images/generations",
"/v1/images/edits"
]
},
"dashscope/qwen3-tts-flash": {
"input_cost_per_character": 1e-05,
"litellm_provider": "dashscope",
"mode": "audio_speech",
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"dashscope/qwen-audio-3.1-tts-flash": {
"input_cost_per_token": 2.3e-07,
"output_cost_per_token": 1.87e-06,
"litellm_provider": "dashscope",
"mode": "audio_speech",
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"dashscope/qwen-audio-3.1-tts-plus": {
"input_cost_per_character": 3e-05,
"litellm_provider": "dashscope",
"mode": "audio_speech",
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"dashscope/qwen-audio-3.0-tts-plus": {
"input_cost_per_character": 2e-05,
"litellm_provider": "dashscope",
"mode": "audio_speech",
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"dashscope/qwen-audio-3.0-tts-flash": {
"input_cost_per_character": 1.5e-05,
"litellm_provider": "dashscope",
"mode": "audio_speech",
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"dashscope/wan3.0-video": {
@ -72330,6 +72383,28 @@
"source": "https://docs.typesafe.ai/models",
"supports_box_questions": true
},
"eu_jev/eu-jeff-1.0.0": {
"input_cost_per_token": 4.15e-08,
"litellm_provider": "eu_jev",
"max_input_tokens": 32768,
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://eu-jev.com/docs",
"supported_endpoints": [
"/v1/systemone"
]
},
"eu_jev/eu-jeff-latest": {
"input_cost_per_token": 4.15e-08,
"litellm_provider": "eu_jev",
"max_input_tokens": 32768,
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://eu-jev.com/docs",
"supported_endpoints": [
"/v1/systemone"
]
},
"wandb/deepseek-ai/DeepSeek-V4.1-Flash": {
"cache_read_input_token_cost": 3e-08,
"input_cost_per_token": 2e-07,

View file

@ -1148,9 +1148,9 @@
"text_completion": true,
"embeddings": false,
"image_generations": true,
"image_edits": false,
"image_edits": true,
"audio_transcriptions": false,
"audio_speech": false,
"audio_speech": true,
"rerank": false,
"ocr": false,
"search": false,
@ -2241,6 +2241,32 @@
"generateContent": false
}
},
"eu_jev": {
"display_name": "eu/jev (`eu_jev`)",
"url": "https://docs.litellm.ai/docs/pass_through/eu_jev",
"endpoints": {
"chat_completions": false,
"responses": false,
"text_completion": false,
"embeddings": false,
"image_generations": false,
"image_edits": false,
"audio_transcriptions": false,
"audio_speech": false,
"rerank": false,
"ocr": false,
"search": false,
"moderations": false,
"realtime": false,
"video_generations": false,
"batches": false,
"messages": false,
"interactions": false,
"systemone": true,
"a2a": false,
"generateContent": false
}
},
"laya": {
"display_name": "Laya (`laya`)",
"url": "https://docs.litellm.ai/docs/auto_router/decision_classifiers",

View file

@ -238,6 +238,7 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = (
"/tinyfish/",
"/transcribe",
"/typesafe/",
"/eu_jev/",
"/laya/",
"/bespoke/",
"/openrouter/",

View file

@ -18332,6 +18332,7 @@
{
"enum": [
"block",
"mask",
"monitor",
"inject_system_message"
],
@ -18342,7 +18343,7 @@
}
],
"default": "block",
"description": "Action to take when content is flagged: 'block' (raise exception), 'monitor' (log only), or 'inject_system_message' (append an advisory system message and let the LLM decide)",
"description": "Action to take when content is flagged: 'block' (raise exception), 'mask' (mask PII-only detections and allow, block anything else), 'monitor' (log only), or 'inject_system_message' (append an advisory system message and let the LLM decide)",
"title": "On Flagged"
},
"on_flagged_action": {
@ -23169,6 +23170,39 @@
"title": "CachedTokensDetails",
"type": "object"
},
"ChoiceAnswer": {
"additionalProperties": true,
"properties": {
"choice": {
"title": "Choice",
"type": "string"
},
"confidence": {
"title": "Confidence",
"type": "number"
},
"probabilities": {
"additionalProperties": {
"type": "number"
},
"title": "Probabilities",
"type": "object"
},
"type": {
"const": "choice",
"title": "Type",
"type": "string"
}
},
"required": [
"type",
"choice",
"confidence",
"probabilities"
],
"title": "ChoiceAnswer",
"type": "object"
},
"Click": {
"additionalProperties": true,
"description": "A click action.",
@ -23933,6 +23967,112 @@
"title": "ErrorResponse",
"type": "object"
},
"EuJevDecisionResponse": {
"properties": {
"answers": {
"additionalProperties": {
"discriminator": {
"mapping": {
"choice": "#/components/schemas/ChoiceAnswer",
"noul": "#/components/schemas/NoulAnswer",
"score": "#/components/schemas/ScoreAnswer"
},
"propertyName": "type"
},
"oneOf": [
{
"$ref": "#/components/schemas/ChoiceAnswer"
},
{
"$ref": "#/components/schemas/NoulAnswer"
},
{
"$ref": "#/components/schemas/ScoreAnswer"
}
]
},
"description": "One answer per question, under the same names",
"title": "Answers",
"type": "object"
},
"meta": {
"$ref": "#/components/schemas/EuJevMeta"
},
"model": {
"description": "The release that answered, such as eu-jeff-1.0.0",
"title": "Model",
"type": "string"
},
"usage": {
"$ref": "#/components/schemas/EuJevUsage"
}
},
"required": [
"model",
"answers",
"usage",
"meta"
],
"title": "EuJevDecisionResponse",
"type": "object"
},
"EuJevMeta": {
"properties": {
"cost_eur": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"description": "The exact charge in euros, on authenticated requests",
"title": "Cost Eur"
},
"latency_ms": {
"description": "Server-side time for token quoting, credit reservation and inference",
"title": "Latency Ms",
"type": "integer"
},
"mode": {
"const": "live",
"title": "Mode",
"type": "string"
},
"request_id": {
"title": "Request Id",
"type": "string"
}
},
"required": [
"request_id",
"mode",
"latency_ms"
],
"title": "EuJevMeta",
"type": "object"
},
"EuJevUsage": {
"properties": {
"input_tokens": {
"minimum": 0.0,
"title": "Input Tokens",
"type": "integer"
},
"output_tokens": {
"minimum": 0.0,
"title": "Output Tokens",
"type": "integer"
}
},
"required": [
"input_tokens",
"output_tokens"
],
"title": "EuJevUsage",
"type": "object"
},
"FileSearchTool": {
"additionalProperties": true,
"description": "A tool that searches for relevant content from uploaded files.\n\nLearn more about the [file search tool](https://platform.openai.com/docs/guides/tools-file-search).",
@ -25593,6 +25733,26 @@
"title": "NamespaceTool",
"type": "object"
},
"NoulAnswer": {
"additionalProperties": true,
"properties": {
"noul": {
"title": "Noul",
"type": "number"
},
"type": {
"const": "noul",
"title": "Type",
"type": "string"
}
},
"required": [
"type",
"noul"
],
"title": "NoulAnswer",
"type": "object"
},
"OperationCreateFile": {
"additionalProperties": true,
"description": "Instruction describing how to create a file via the apply_patch tool.",
@ -28865,6 +29025,59 @@
"title": "Result",
"type": "object"
},
"ScoreAnswer": {
"additionalProperties": true,
"properties": {
"confidence": {
"title": "Confidence",
"type": "number"
},
"legend": {
"additionalProperties": {
"anyOf": [
{
"type": "string"
},
{
"additionalProperties": true,
"type": "object"
},
{
"items": {},
"type": "array"
}
]
},
"title": "Legend",
"type": "object"
},
"probabilities": {
"additionalProperties": {
"type": "number"
},
"title": "Probabilities",
"type": "object"
},
"score": {
"title": "Score",
"type": "number"
},
"type": {
"const": "score",
"title": "Type",
"type": "string"
}
},
"required": [
"type",
"score",
"confidence",
"legend",
"probabilities"
],
"title": "ScoreAnswer",
"type": "object"
},
"Screenshot": {
"additionalProperties": true,
"description": "A screenshot action.",
@ -31757,6 +31970,315 @@
]
}
},
"/eu_jev/v1/systemone": {
"post": {
"description": "Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`.\n\n`POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`)\nand returns one answer per question name, with spend booked from the reported `usage`. `custom_body` and\n`stream` are rejected with a 400. A pass-through endpoint configured at this path keeps serving it.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)",
"operationId": "eu_jev_proxy_route_eu_jev_v1_systemone_post",
"requestBody": {
"content": {
"application/json": {
"example": {
"model": "eu-jeff-latest",
"questions": {
"frustration": {
"criteria": [
"calm",
"mildly annoyed",
"angry"
],
"instructions": "How frustrated is the customer?",
"type": "score"
},
"is_refund_request": {
"instructions": "Is the customer asking for a refund?",
"type": "noul"
},
"team": {
"criteria": {
"billing": "Payments, invoices, and refunds",
"support": "Technical issues and bugs"
},
"instructions": "Which team should handle this?",
"type": "choice"
}
},
"state": "I was charged twice. Can I get a refund?"
},
"schema": {
"properties": {
"model": {
"default": "eu-jeff-latest",
"description": "eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it",
"title": "Model",
"type": "string"
},
"questions": {
"additionalProperties": {
"discriminator": {
"propertyName": "type"
},
"oneOf": [
{
"properties": {
"criteria": {
"additionalProperties": {
"anyOf": [
{
"type": "string"
},
{
"additionalProperties": true,
"type": "object"
},
{
"items": {},
"type": "array"
},
{
"type": "null"
}
],
"description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference"
},
"description": "Two to 64 options, option name to description. A null description uses the option name",
"maxProperties": 64,
"minProperties": 2,
"title": "Criteria",
"type": "object"
},
"instructions": {
"anyOf": [
{
"type": "string"
},
{
"additionalProperties": true,
"type": "object"
},
{
"items": {},
"type": "array"
},
{
"type": "null"
}
],
"description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference",
"title": "Instructions"
},
"type": {
"const": "choice",
"title": "Type",
"type": "string"
}
},
"required": [
"type",
"instructions",
"criteria"
],
"title": "EuJevChoiceQuestion",
"type": "object"
},
{
"properties": {
"criteria": {
"anyOf": [
{
"additionalProperties": {
"anyOf": [
{
"type": "string"
},
{
"additionalProperties": true,
"type": "object"
},
{
"items": {},
"type": "array"
},
{
"type": "null"
}
]
},
"propertyNames": {
"enum": [
"true",
"false"
]
},
"type": "object"
},
{
"type": "null"
}
],
"description": "Optional descriptions of what true and false mean",
"title": "Criteria"
},
"instructions": {
"anyOf": [
{
"type": "string"
},
{
"additionalProperties": true,
"type": "object"
},
{
"items": {},
"type": "array"
},
{
"type": "null"
}
],
"description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference",
"title": "Instructions"
},
"type": {
"const": "noul",
"title": "Type",
"type": "string"
}
},
"required": [
"type",
"instructions"
],
"title": "EuJevNoulQuestion",
"type": "object"
},
{
"properties": {
"criteria": {
"description": "Two to 10 level descriptions, ordered from low to high",
"items": {
"anyOf": [
{
"type": "string"
},
{
"additionalProperties": true,
"type": "object"
},
{
"items": {},
"type": "array"
},
{
"type": "null"
}
],
"description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference"
},
"maxItems": 10,
"minItems": 2,
"title": "Criteria",
"type": "array"
},
"instructions": {
"anyOf": [
{
"type": "string"
},
{
"additionalProperties": true,
"type": "object"
},
{
"items": {},
"type": "array"
},
{
"type": "null"
}
],
"description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference",
"title": "Instructions"
},
"type": {
"const": "score",
"title": "Type",
"type": "string"
}
},
"required": [
"type",
"instructions",
"criteria"
],
"title": "EuJevScoreQuestion",
"type": "object"
}
]
},
"description": "One to eight questions, keyed by names of up to 80 characters",
"maxProperties": 8,
"minProperties": 1,
"propertyNames": {
"maxLength": 80,
"minLength": 1
},
"title": "Questions",
"type": "object"
},
"state": {
"anyOf": [
{
"maxLength": 100000,
"minLength": 1,
"type": "string"
},
{
"additionalProperties": true,
"type": "object"
},
{
"items": {},
"type": "array"
}
],
"description": "The content to evaluate, a string of up to 100,000 characters or a JSON object or array",
"title": "State"
}
},
"required": [
"state",
"questions"
],
"title": "EuJevDecisionRequest",
"type": "object"
}
}
},
"required": true
},
"responses": {
"200": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/EuJevDecisionResponse"
}
}
},
"description": "Successful Response"
}
},
"security": [
{
"APIKeyHeader": []
}
],
"summary": "Eu Jev Proxy Route",
"tags": [
"llm_passthrough"
]
}
},
"/fal_ai/{endpoint}": {
"delete": {
"operationId": "fal_ai_proxy_route_fal_ai__endpoint__delete",

View file

@ -0,0 +1,81 @@
"""
The proxy's RouterHost: answers the router's questions about pass-through credentials, member
auto-router access and auto-router prompt compression with the proxy's own modules.
Installed by litellm.router_host the first time the router asks for its host. Each method looks its
target up at call time, so patches of the proxy functions apply.
"""
from collections.abc import Mapping, Sequence
from typing import TYPE_CHECKING
from litellm.router_host import CompressionPolicy, RouterHost
if TYPE_CHECKING:
from litellm.router import Router
from litellm.types.llms.vertex_ai import VERTEX_CREDENTIALS_TYPES
class ProxyRouterHost(RouterHost):
def add_vertex_pass_through_credentials(
self,
project_id: str,
location: str,
vertex_credentials: "VERTEX_CREDENTIALS_TYPES | None",
) -> None:
from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import passthrough_endpoint_router
passthrough_endpoint_router.add_vertex_credentials(
project_id=project_id,
location=location,
vertex_credentials=vertex_credentials,
)
async def authorize_strategy_inference(
self,
deployment: Mapping[str, object] | None,
request_kwargs: Mapping[str, object],
llm_router: "Router",
) -> None:
from litellm.proxy.auth import auto_router_checks
await auto_router_checks.authorize_member_auto_router_inference(
deployment=deployment,
request_kwargs=request_kwargs,
llm_router=llm_router,
)
def compression_policy(
self,
llm_router: "Router | None",
model_alias: str,
request_kwargs: Mapping[str, object],
request_tags: Sequence[str],
) -> CompressionPolicy | None:
from litellm.proxy.guardrails import auto_router_compression
return auto_router_compression.policy_for_model(
llm_router=llm_router,
model_alias=model_alias,
request_kwargs=request_kwargs,
request_tags=request_tags,
)
def model_hop_compression_armed(self) -> bool:
from litellm.proxy.guardrails import auto_router_compression
return auto_router_compression.model_hop_compression_armed()
async def messages_for_routing(
self,
policy: CompressionPolicy | None,
messages: list[dict[str, object]] | None, # mutable-ok: shape fixed by the pre-routing hook protocol
request_kwargs: Mapping[str, object],
) -> list[dict[str, object]] | None: # mutable-ok: shape fixed by the pre-routing hook protocol
from litellm.proxy.guardrails import auto_router_compression
return await auto_router_compression.messages_for_routing(
policy=policy, # pyright: ignore[reportArgumentType] # the proxy's own policy object, from compression_policy
messages=messages,
request_kwargs=request_kwargs,
)

View file

@ -536,7 +536,7 @@ def _get_router_zero_cost_cache(llm_router: Router) -> dict[str, bool] | None:
* dies with the router itself — no risk of CPython reusing the
previous router's ``id()`` and serving its cached entries.
"""
cache: Final = getattr(llm_router, "_zero_cost_cache", None)
cache: Final = llm_router.registry().zero_cost()
return cache if isinstance(cache, dict) else None
@ -544,7 +544,7 @@ def is_model_cost_zero(model: str | list[str] | None, llm_router: Router | None)
"""
Check if a model has zero cost (no configured pricing).
Uses the router's get_model_group_info method to get pricing information.
Uses the router registry's group info to get pricing information.
Args:
model: The model name or list of model names
@ -569,8 +569,8 @@ def is_model_cost_zero(model: str | list[str] | None, llm_router: Router | None)
return False
continue
try:
# Use router's get_model_group_info method directly for better reliability
model_group_info = llm_router.get_model_group_info(model_group=model_name, include_hidden=True)
# Use the router registry's group info directly for better reliability
model_group_info = llm_router.registry().group(model_name, include_hidden=True)
if model_group_info is None:
# Model not found or no pricing info available
@ -677,15 +677,16 @@ def _deployments_served_for(model: str, llm_router: "Router") -> Sequence[Deploy
group count, or the wildcard route serving the group when none carries its name, the same
fallback ``get_model_list()`` takes.
"""
routed_group: Final = llm_router.routable_model_group(model)
if llm_router.routable_model_group(routed_group) == routed_group:
return llm_router.get_model_list(model_name=routed_group) or ()
named: Final = llm_router.get_model_list_from_model_alias(model_name=model)
registry: Final = llm_router.registry()
routed_group: Final = registry.resolve(model)
if registry.resolve(routed_group) == routed_group:
return registry.deployments(routed_group) or ()
named: Final = registry.deployments(model, source="alias")
if named:
return named
return tuple(
DeploymentTypedDict(**deployment)
for deployment in llm_router.pattern_router.get_deployments_by_pattern(model=routed_group)
for deployment in registry.patterns().get_deployments_by_pattern(model=routed_group)
)
@ -776,7 +777,8 @@ def _model_group_has_pricing(model: str, llm_router: "Router") -> bool:
tiered_pricing or a positive price on any billed metric (tokens, characters, seconds, pages,
images, queries, ...), so models billed by a non-token metric are not treated as unpriced.
"""
for deployment in llm_router.get_model_list(model_name=model) or ():
registry: Final = llm_router.registry()
for deployment in registry.deployments(model) or ():
litellm_params = deployment.get("litellm_params") or _EMPTY_COST_ENTRY
if _entry_declares_price(litellm_params):
return True
@ -785,9 +787,7 @@ def _model_group_has_pricing(model: str, llm_router: "Router") -> bool:
if model_id is None:
continue
model_info = llm_router.get_deployment_model_info(
model_id=model_id, model_name=litellm_params.get("model") or ""
)
model_info = registry.deployment_model_info(model_id, litellm_params.get("model") or "")
if model_info is not None and _entry_has_priced_metric(model_info):
return True
@ -798,13 +798,13 @@ def model_has_no_cost_mapping(model: str | None, llm_router: Router | None) -> b
if not model or llm_router is None:
return False
if llm_router.get_model_group_info(model_group=model) is None:
if llm_router.registry().group(model) is None:
return False
if _model_group_has_pricing(model=model, llm_router=llm_router):
return False
return not _is_cost_explicitly_configured(llm_router.get_model_list(model_name=model) or ())
return not _is_cost_explicitly_configured(llm_router.registry().deployments(model) or ())
def _unpriced_models_in_request(model: str | list[str] | None, llm_router: Router | None) -> tuple[str, ...]:
@ -2785,7 +2785,7 @@ def model_in_access_group(model: str, team_models: list[str] | None, llm_router:
access_groups: dict[str, list[str]] = defaultdict(list)
if llm_router:
access_groups = llm_router.get_model_access_groups(model_name=model)
access_groups = llm_router.registry().access_groups(model_name=model)
if len(access_groups) > 0: # check if token contains any model access groups
for idx, m in enumerate(
@ -4829,7 +4829,7 @@ def _resolve_all_team_model_sentinel_for_auth_check(
) -> list[str]:
if SpecialModelNames.all_team_models.value not in models or team_id is None or llm_router is None:
return models
proxy_models: Final = llm_router.get_model_names()
proxy_models: Final = llm_router.registry().names()
non_sentinel_models: Final = [model for model in models if model != SpecialModelNames.all_team_models.value]
if not proxy_models:
return non_sentinel_models or models
@ -4849,7 +4849,7 @@ def check_model_access_helper(
access_groups: dict[str, list[str]] = defaultdict(list)
if llm_router:
access_groups = llm_router.get_model_access_groups(model_name=model, team_id=team_id)
access_groups = llm_router.registry().access_groups(model_name=model, team_id=team_id)
models = _resolve_all_team_model_sentinel_for_auth_check(
models=models,
@ -4895,7 +4895,7 @@ def _owning_team_names_for_team_deployment(
) -> tuple[str, ...]:
if llm_router is None or team_id is None or not model.startswith(f"model_name_{team_id}_"):
return ()
deployment: Final = llm_router.get_deployment_by_model_group_name(model_group_name=model)
deployment: Final = llm_router.registry().deployments(model, source="first_in_group")
public_name: Final = (
deployment.model_info.team_public_model_name
if deployment is not None and deployment.model_info.team_id == team_id
@ -4954,8 +4954,8 @@ def _can_object_call_model(
litellm.model_alias_map[model]
if model in litellm.model_alias_map
else (
llm_router.get_model_from_alias(model)
if llm_router is not None and model in llm_router.model_group_alias
llm_router.registry().resolve(model, kind="alias")
if llm_router is not None and model in llm_router.registry().aliases()
else None
)
)
@ -5042,7 +5042,7 @@ def _live_team_alias_target(
deleted_team_deployment: Final = (
llm_router is not None
and target.startswith(f"model_name_{team_id}_")
and target not in llm_router.model_name_to_deployment_indices
and target not in llm_router.registry().index("model_name")
)
return model if deleted_team_deployment else target
@ -5234,7 +5234,7 @@ def _model_access_groups_serving_model(
return frozenset(
group
for requested_model in requested
for group in llm_router.get_model_access_groups(model_name=requested_model, team_id=team_id)
for group in llm_router.registry().access_groups(model_name=requested_model, team_id=team_id)
)

View file

@ -33,6 +33,7 @@ from litellm.litellm_core_utils.url_utils import (
from litellm.llms.azure.passthrough.transformation import azure_router_model_in_endpoint
from litellm.llms.nvidia_nim.passthrough.transformation import nvidia_nim_model_group_in_path
from litellm.proxy._types import *
from litellm.proxy.auth.route_checks import RouteChecks
from litellm.proxy.common_utils.http_parsing_utils import extract_nested_form_metadata
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
LITELLM_PASS_THROUGH_ENDPOINT_MARKER,
@ -222,11 +223,12 @@ def _allow_model_level_clientside_configurable_parameters(
if llm_router is None:
return False
# check if model is set
model_info = llm_router.get_model_group_info(model_group=model)
registry: Final = llm_router.registry()
model_info = registry.group(model)
if model_info is None:
# check if wildcard model is set
if model.split("/", 1)[0] in provider_list:
model_info = llm_router.get_model_group_info(model_group=model.split("/", 1)[0])
model_info = registry.group(model.split("/", 1)[0])
if model_info is None:
return False
@ -883,7 +885,6 @@ def route_in_additonal_public_routes(current_route: str):
public_routes: ["LiteLLMRoutes.public_routes", "/spend/calculate", "/api/*"]
```
"""
from litellm.proxy.auth.route_checks import RouteChecks
from litellm.proxy.proxy_server import general_settings, premium_user
try:
@ -1208,7 +1209,7 @@ def _get_deployment_default_limit(model_name: str, field: str) -> int | None:
if llm_router is None:
return None
deployments: Final = llm_router.get_model_list(model_name=model_name)
deployments: Final = llm_router.registry().deployments(model_name)
if not deployments:
return None
limits: Final = []
@ -2124,7 +2125,7 @@ def _resolve_model_id_with_router(model_id: str | None, llm_router: Router | Non
if model_id is None or llm_router is None:
return model_id
try:
return llm_router.resolve_model_name_from_model_id(model_id) or model_id
return llm_router.registry().resolve(model_id, kind="from_id") or model_id
except Exception as e:
verbose_proxy_logger.debug("Unable to resolve model_id from managed resource ID: %s", str(e))
return model_id
@ -2133,8 +2134,9 @@ def _resolve_model_id_with_router(model_id: str | None, llm_router: Router | Non
def get_cache_prediction_deployments(
*, current_deployment_id: str, candidate_deployment_id: str, llm_router: Router, team_id: str | None
) -> tuple[Deployment, Deployment] | None:
current: Final = llm_router.get_deployment(current_deployment_id)
candidate: Final = llm_router.get_deployment(candidate_deployment_id)
registry: Final = llm_router.registry()
current: Final = registry.deployment(current_deployment_id)
candidate: Final = registry.deployment(candidate_deployment_id)
if current is None or candidate is None:
return None
if any(deployment.model_info.team_id not in (None, team_id) for deployment in (current, candidate)):
@ -2173,6 +2175,13 @@ def _extract_model_candidates_from_request(
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
return _dedupe_model_candidates((f"{provider}/{decision_model}",))
if RouteChecks.is_builtin_eu_jev_decision_route(route=route):
from litellm.llms.eu_jev.common_utils import eu_jev_registry_model
requested_model: Final = TypeAdapter(Mapping[str, object]).validate_python(request_data).get("model")
if requested_model is not None and not isinstance(requested_model, str):
raise HTTPException(status_code=400, detail="eu_jev model must be a string")
return [f"eu_jev/{eu_jev_registry_model(requested_model)}"]
if route == "/cost/predict-cache":
prediction_models: Final = _cache_prediction_model_candidates(request_data, llm_router, team_id) # pyright: ignore[reportUnknownArgumentType] # the typed reader validates each deployment ID from this legacy payload
return _dedupe_model_candidates(prediction_models)
@ -2346,7 +2355,7 @@ def get_model_from_request(
if route.lower().startswith("/nvidia_nim/"):
nvidia_nim_model: Final = (
nvidia_nim_model_group_in_path(route, llm_router.get_model_list()) if llm_router else None
nvidia_nim_model_group_in_path(route, llm_router.registry().deployments()) if llm_router else None
)
return model if nvidia_nim_model is None else nvidia_nim_model
@ -2357,7 +2366,7 @@ def _router_model_from_azure_route(route: str, llm_router: Router | None) -> str
if llm_router is None:
return None
endpoint: Final = re.sub(r"^/azure(?:_ai)?/", "", route, flags=re.IGNORECASE)
return azure_router_model_in_endpoint(endpoint, frozenset(llm_router.get_model_names()))
return azure_router_model_in_endpoint(endpoint, frozenset(llm_router.registry().names()))
def _model_from_bedrock_route(route: str) -> str | None:

View file

@ -193,7 +193,7 @@ class RouterFallbackBudgetCheck:
valid_token: Final = _user_api_key_auth_from_request(request_kwargs)
if valid_token is None:
return True
budget_models: Final = tuple(dict.fromkeys((model, llm_router.routable_model_group(model))))
budget_models: Final = tuple(dict.fromkeys((model, llm_router.registry().resolve(model))))
for budget_model in budget_models:
try:
if not await is_token_within_budget_for_model(

View file

@ -5,6 +5,7 @@ import functools
import itertools
import re
from collections.abc import Callable, Mapping, Sequence
from types import MappingProxyType
from typing import Final
import litellm
@ -385,7 +386,15 @@ def _deployment_team_id(deployment: Mapping[str, object]) -> str | None:
def _cached_wildcard_routing(llm_router: Router | None) -> Callable[[str, str | None], frozenset[str]] | None:
return None if llm_router is None else functools.cache(llm_router.wildcard_routed_models)
if llm_router is None:
return None
registry: Final = llm_router.registry()
def wildcard_routed_models(model: str, team_id: str | None) -> frozenset[str]:
routed: Final[frozenset[str]] = registry.names(model, kind="wildcard", team_id=team_id)
return routed
return functools.cache(wildcard_routed_models)
def _routes_to_model(
@ -550,7 +559,7 @@ def _get_wildcard_models(
models_to_remove.add(model)
model_list = llm_router.get_model_list(model_name=model, team_id=team_id) if llm_router else None
model_list = llm_router.registry().deployments(model, team_id=team_id) if llm_router else None
if model_list:
for router_model in model_list:
all_wildcard_models.extend(
@ -569,6 +578,15 @@ def _get_wildcard_models(
return all_wildcard_models
_FALLBACK_SETTING_BY_TYPE: Final[Mapping[str, str]] = MappingProxyType(
{
"general": "fallbacks",
"context_window": "context_window_fallbacks",
"content_policy": "content_policy_fallbacks",
}
)
def get_all_fallbacks(
model: str,
llm_router: Router | None = None,
@ -589,16 +607,15 @@ def get_all_fallbacks(
return []
# Get the appropriate fallback list based on type
fallbacks_config: list = []
if fallback_type == "general":
fallbacks_config = getattr(llm_router, "fallbacks", [])
elif fallback_type == "context_window":
fallbacks_config = getattr(llm_router, "context_window_fallbacks", [])
elif fallback_type == "content_policy":
fallbacks_config = getattr(llm_router, "content_policy_fallbacks", [])
else:
setting_name: Final = _FALLBACK_SETTING_BY_TYPE.get(fallback_type)
if setting_name is None:
verbose_proxy_logger.warning("Unknown fallback_type: %s", fallback_type)
return []
settings: Final = llm_router.settings()
# An unset setting reads as no fallbacks, as the router attribute's absence did.
if not settings.is_set(setting_name):
return []
fallbacks_config: list = getattr(settings, setting_name)
if not fallbacks_config:
return []

View file

@ -12,6 +12,11 @@ from litellm.llms.custom_httpx.http_handler import (
from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth
class OAuth2TokenValidationError(ValueError):
"""The token-info or introspection endpoint did not vouch for the token: it rejected it, reported it
inactive, or could not be reached."""
class Oauth2Handler:
"""
Handles OAuth2 token validation.
@ -207,7 +212,7 @@ class Oauth2Handler:
)
except httpx.HTTPStatusError as e:
# This will catch any 4xx or 5xx errors
raise ValueError(f"Oauth 2.0 Token validation failed: {e}")
raise OAuth2TokenValidationError(f"Oauth 2.0 Token validation failed: {e}")
except Exception as e:
# This will catch any other errors (like network issues)
raise ValueError(f"An error occurred during token validation: {e}")
raise OAuth2TokenValidationError(f"An error occurred during token validation: {e}")

View file

@ -73,6 +73,7 @@ _PROXY_ADMIN_VIEW_ONLY_BLOCKED_ROUTES: Final = frozenset(
_PROXY_ADMIN_VIEW_ONLY_BLOCKED_KEY_SUFFIXES: Final = ("/regenerate", "/reset_spend")
_LLM_API_ROUTE_GROUPS: Final = frozenset(("llm_api_routes", "llm_api_routes_without_mcp"))
_EU_JEV_DECISION_ROUTE: Final = "/eu_jev/v1/systemone"
_AUTH_ENFORCED_PASS_THROUGH_ROUTE_GROUPS: Final = frozenset(("openai_routes", *_LLM_API_ROUTE_GROUPS))
# Writes that only touch the caller's own credential row (BYOK connections and
@ -503,7 +504,19 @@ class RouteChecks:
for _llm_passthrough_route in LiteLLMRoutes.mapped_pass_through_routes.value:
if route == _llm_passthrough_route or route.startswith(_llm_passthrough_route + "/"):
return True
return False
return RouteChecks.is_builtin_eu_jev_decision_route(route=route)
@staticmethod
def is_builtin_eu_jev_decision_route(route: str) -> bool:
if route.rstrip("/") != _EU_JEV_DECISION_ROUTE:
return False
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( # noqa: PLC0415 # pass_through_endpoints imports the auth stack
InitPassThroughEndpointHelpers,
)
return not InitPassThroughEndpointHelpers.has_configured_route_ignoring_trailing_slash(
route=_EU_JEV_DECISION_ROUTE
)
@staticmethod
def _is_get_mcp_server_discovery_route(route: str, request: Request | None) -> bool:

View file

@ -463,13 +463,9 @@ def _token_model_budget_fields(valid_token: _TokenModelBudgetFields) -> _TokenMo
return valid_token
class _RouterFallbacks(Protocol):
@property
def fallbacks(self) -> Sequence[object] | None: ...
def _router_fallbacks(router: _RouterFallbacks) -> Sequence[object] | None:
return router.fallbacks
def _router_fallbacks(router: litellm.Router) -> Sequence[object] | None:
fallbacks: Final[Sequence[object] | None] = router.settings().fallbacks
return fallbacks
async def _read_user_model_max_budget(
@ -977,7 +973,7 @@ async def _try_team_member_model_budget_fallback(
fallback_candidate,
fallback_model,
served_model,
llm_router.routable_model_group(served_model),
llm_router.registry().resolve(served_model),
)
)
),
@ -1164,6 +1160,24 @@ def _matches_routing_override(token_claims: dict, override: "JWTRoutingOverride"
)
def oauth2_introspection_applies(
api_key: str, route: str, *, enable_oauth2_auth: bool, enable_jwt_auth: bool, jwt_handler: JWTHandler
) -> bool:
"""Whether LiteLLM validates ``api_key`` on ``route`` by OAuth2 token introspection, on LLM API, info and
lens routes only: a non-JWT token when OAuth2 auth is on globally, or a JWT matched by a routing override."""
is_jwt: Final = enable_jwt_auth and jwt_handler.is_jwt(token=api_key)
# Routing uses unverified JWT claims only to choose auth path.
# Final authentication is enforced by the selected validator.
route_jwt_to_oauth2: Final = is_jwt and _should_route_jwt_to_oauth2_override(token=api_key, jwt_handler=jwt_handler)
covered_route: Final = (
RouteChecks.is_llm_api_route(route=route)
or RouteChecks.is_info_route(route=route)
or route == "/lens"
or route.startswith("/lens/")
)
return covered_route and ((enable_oauth2_auth and not is_jwt) or route_jwt_to_oauth2)
def _should_route_jwt_to_oauth2_override(token: str, jwt_handler: JWTHandler) -> bool:
routing_overrides: Final = jwt_handler.litellm_jwtauth.routing_overrides
if not routing_overrides:
@ -2282,23 +2296,13 @@ async def user_api_key_auth_builder(
enable_oauth2_auth: Final = general_settings.get("enable_oauth2_auth", False) is True
enable_jwt_auth: Final = general_settings.get("enable_jwt_auth", False) is True
is_jwt = jwt_handler.is_jwt(token=api_key) if enable_jwt_auth else False
# Routing uses unverified JWT claims only to choose auth path.
# Final authentication is enforced by the selected validator.
route_jwt_to_oauth2 = is_jwt and _should_route_jwt_to_oauth2_override(token=api_key, jwt_handler=jwt_handler)
# OAuth2 applies for:
# 1) when global OAuth2 auth is enabled on LLM + info routes
# 2) JWT tokens that explicitly match routing_overrides on LLM + info routes
route_is_lens: Final = route == "/lens" or route.startswith("/lens/")
should_apply_override_oauth2: Final = route_jwt_to_oauth2 and (
RouteChecks.is_llm_api_route(route=route) or RouteChecks.is_info_route(route=route) or route_is_lens
)
should_apply_global_oauth2: Final = enable_oauth2_auth and (
RouteChecks.is_llm_api_route(route=route) or RouteChecks.is_info_route(route=route) or route_is_lens
)
if (should_apply_global_oauth2 and not is_jwt) or should_apply_override_oauth2:
if oauth2_introspection_applies(
api_key,
route,
enable_oauth2_auth=enable_oauth2_auth,
enable_jwt_auth=enable_jwt_auth,
jwt_handler=jwt_handler,
):
from litellm.proxy.proxy_server import premium_user
if premium_user is not True:

View file

@ -621,7 +621,7 @@ async def retrieve_batch(
unified_model_id: Final = get_model_id_from_unified_batch_id(unified_batch_id) if unified_batch_id else None
if unified_model_id is not None:
resolved_unified_model: Final = (
llm_router.resolve_model_name_from_model_id(unified_model_id) if llm_router is not None else None
llm_router.registry().resolve(unified_model_id, kind="from_id") if llm_router is not None else None
)
await authorize_model_for_key(
model_id=resolved_unified_model or unified_model_id,
@ -1130,7 +1130,7 @@ async def cancel_batch(
unified_model_id: Final = get_model_id_from_unified_batch_id(unified_batch_id) if unified_batch_id else None
if unified_model_id is not None:
resolved_unified_model: Final = (
llm_router.resolve_model_name_from_model_id(unified_model_id) if llm_router is not None else None
llm_router.registry().resolve(unified_model_id, kind="from_id") if llm_router is not None else None
)
await authorize_model_for_key(
model_id=resolved_unified_model or unified_model_id,

Some files were not shown because too many files have changed in this diff Show more