mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge remote-tracking branch 'origin/main' into litellm_responses_drop_malformed_values
# Conflicts: # litellm/llms/openai/responses/count_tokens/token_counter.py
This commit is contained in:
commit
814eb8c3fc
362 changed files with 21478 additions and 2860 deletions
|
|
@ -1,14 +1,14 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if [ "${GITHUB_ACTIONS:-}" = true ]; then
|
||||
echo "Integration contracts are owned by CircleCI" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
suite="${1:?integration suite required}"
|
||||
mode="${2:-standard}"
|
||||
side="${3:-}"
|
||||
requested_workers="${INTEGRATION_WORKERS:-}"
|
||||
license_env=()
|
||||
if [ -n "${LITELLM_LICENSE:-}" ]; then
|
||||
license_env=("LITELLM_LICENSE=$LITELLM_LICENSE")
|
||||
fi
|
||||
if [ "$mode" = replica ]; then
|
||||
results="test-results/integration-${suite}-replica"
|
||||
elif [ "$mode" = parity ]; then
|
||||
|
|
@ -84,7 +84,7 @@ if [ "$suite" = browser ]; then
|
|||
export LITELLM_UI_PATH="$PWD/ui/litellm-dashboard/out"
|
||||
test -f "$LITELLM_UI_PATH/index.html"
|
||||
fi
|
||||
export INTEGRATION_SEED="$(.venv/bin/python -c 'import hashlib,os; print(int(hashlib.sha256((os.environ.get("CIRCLE_SHA1", "local") + os.environ.get("CIRCLE_WORKFLOW_ID", "local")).encode()).hexdigest()[:8],16))')"
|
||||
export INTEGRATION_SEED="$(.venv/bin/python -c 'import hashlib,os; print(int(hashlib.sha256(((os.environ.get("CIRCLE_SHA1") or os.environ.get("GITHUB_SHA") or "local") + (os.environ.get("CIRCLE_WORKFLOW_ID") or os.environ.get("GITHUB_RUN_ID") or "local")).encode()).hexdigest()[:8],16))')"
|
||||
export INTEGRATION_ORDER_SEED="$INTEGRATION_SEED"
|
||||
|
||||
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma > "$results/prisma-generate.log" 2>&1
|
||||
|
|
@ -107,7 +107,7 @@ sudo iptables -N integration_only
|
|||
guard_created=true
|
||||
sudo iptables -A integration_only -o lo -j ACCEPT
|
||||
sudo iptables -A integration_only -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT
|
||||
for service in postgres-db redis-cache; do
|
||||
for service in "${INTEGRATION_POSTGRES_CONTAINER:-postgres-db}" "${INTEGRATION_REDIS_CONTAINER:-redis-cache}"; do
|
||||
address="$(docker inspect --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$service")"
|
||||
sudo iptables -A integration_only -d "$address" -j ACCEPT
|
||||
done
|
||||
|
|
@ -145,6 +145,9 @@ fi
|
|||
if [ "$suite" = mcp ]; then
|
||||
export INTEGRATION_WORKERS=2 INTEGRATION_COVERAGE=1
|
||||
fi
|
||||
if [ -n "$requested_workers" ]; then
|
||||
export INTEGRATION_WORKERS="$requested_workers"
|
||||
fi
|
||||
isolated_workers=false
|
||||
if [ "${INTEGRATION_WORKERS:-1}" -gt 1 ]; then
|
||||
isolated_workers=true
|
||||
|
|
@ -185,7 +188,7 @@ start_proxy() {
|
|||
"${database_env[@]}" REDIS_HOST="$REDIS_HOST" REDIS_PORT="$REDIS_PORT" \
|
||||
INTEGRATION_UPSTREAM_URL="$INTEGRATION_UPSTREAM_URL" \
|
||||
LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" LITELLM_SALT_KEY="$LITELLM_SALT_KEY" LITELLM_UI_PATH="$LITELLM_UI_PATH" PROXY_BASE_URL="http://127.0.0.1:$port" \
|
||||
LITELLM_LICENSE="${LITELLM_LICENSE:-}" \
|
||||
"${license_env[@]}" \
|
||||
LITELLM_MODE=PRODUCTION STORE_MODEL_IN_DB=True LITELLM_ENABLE_MCP_STDIO=true "${cost_map_env[@]}" \
|
||||
AWS_EC2_METADATA_DISABLED=true DO_NOT_TRACK=1 COVERAGE_FILE="$coverage_data" \
|
||||
"${proxy_command[@]}" --config tests/integration/proxy_config.yaml \
|
||||
|
|
@ -257,7 +260,7 @@ env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \
|
|||
"${shared_services[@]}" "${worker_coverage[@]}" \
|
||||
INTEGRATION_WORKERS="${INTEGRATION_WORKERS:-1}" \
|
||||
LITELLM_MODE=PRODUCTION \
|
||||
LITELLM_LICENSE="${LITELLM_LICENSE:-}" \
|
||||
"${license_env[@]}" \
|
||||
INTEGRATION_SEED="$INTEGRATION_SEED" \
|
||||
INTEGRATION_ORDER_SEED="$INTEGRATION_ORDER_SEED" \
|
||||
INTEGRATION_CASE_SHARDS="${CIRCLE_NODE_TOTAL:-1}" INTEGRATION_CASE_SHARD_INDEX="${CIRCLE_NODE_INDEX:-0}" \
|
||||
|
|
|
|||
119
.github/scripts/assert_secret_absent.py
vendored
Normal file
119
.github/scripts/assert_secret_absent.py
vendored
Normal file
|
|
@ -0,0 +1,119 @@
|
|||
import base64
|
||||
import io
|
||||
import os
|
||||
import sys
|
||||
import urllib.parse
|
||||
import zipfile
|
||||
from collections.abc import Callable, Iterator, Mapping
|
||||
from dataclasses import dataclass
|
||||
from functools import reduce
|
||||
from itertools import chain
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final, TextIO
|
||||
|
||||
SECRET_VARIABLE: Final = "SCANNED_SECRET"
|
||||
MIN_ENCODED_LENGTH: Final = 12
|
||||
MAX_ARCHIVE_DEPTH: Final = 3
|
||||
MASK: Final = "***"
|
||||
|
||||
Forms = tuple[tuple[str, str], ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Hit:
|
||||
location: str
|
||||
form: str
|
||||
|
||||
|
||||
def _aligned_encodings(raw: bytes, name: str, encode: Callable[[bytes], bytes]) -> Iterator[tuple[str, str]]:
|
||||
for offset in range(3):
|
||||
chunk: Final = raw[offset:]
|
||||
yield f"{name} offset {offset}", encode(chunk[: len(chunk) - len(chunk) % 3]).decode()
|
||||
|
||||
|
||||
def secret_forms(secret: str) -> Forms:
|
||||
encoded: Final = (
|
||||
("url-encoded", urllib.parse.quote(secret, safe="")),
|
||||
*_aligned_encodings(secret.encode(), "base64", base64.b64encode),
|
||||
*_aligned_encodings(secret.encode(), "base64url", base64.urlsafe_b64encode),
|
||||
)
|
||||
by_value: Final = MappingProxyType(
|
||||
{value: name for name, value in encoded if value != secret and len(value) >= MIN_ENCODED_LENGTH}
|
||||
)
|
||||
return (("raw", secret), *((name, value) for value, name in by_value.items()))
|
||||
|
||||
|
||||
def mask(text: str, forms: Forms) -> str:
|
||||
return reduce(lambda masked, form: masked.replace(form[1], MASK), forms, text)
|
||||
|
||||
|
||||
def _name_hits(location: str, name: str, forms: Forms) -> Iterator[Hit]:
|
||||
for form, value in forms:
|
||||
if value in name:
|
||||
yield Hit(mask(location, forms), f"{form} in name")
|
||||
|
||||
|
||||
def _content_hits(location: str, data: bytes, forms: Forms, depth: int) -> Iterator[Hit]:
|
||||
for form, value in forms:
|
||||
if value.encode() in data:
|
||||
yield Hit(mask(location, forms), form)
|
||||
if depth >= MAX_ARCHIVE_DEPTH or not zipfile.is_zipfile(io.BytesIO(data)):
|
||||
return
|
||||
with zipfile.ZipFile(io.BytesIO(data)) as archive:
|
||||
for member in archive.infolist():
|
||||
if member.is_dir():
|
||||
continue
|
||||
yield from _content_hits(f"{location}!{member.filename}", archive.read(member), forms, depth + 1)
|
||||
|
||||
|
||||
def _walk(root: Path) -> Iterator[tuple[Path, bool]]:
|
||||
for directory, subdirectories, files in os.walk(root):
|
||||
yield from ((Path(directory, name), True) for name in subdirectories if Path(directory, name).is_symlink())
|
||||
yield from ((Path(directory, name), False) for name in files)
|
||||
|
||||
|
||||
def _path_hits(root: Path, path: Path, linked_directory: bool, forms: Forms) -> Iterator[Hit]:
|
||||
location: Final = str(path.relative_to(root))
|
||||
if linked_directory:
|
||||
yield Hit(mask(location, forms), "directory symlink, not scanned")
|
||||
return
|
||||
yield from _name_hits(location, location, forms)
|
||||
yield from _content_hits(location, path.read_bytes(), forms, 0)
|
||||
|
||||
|
||||
def scan(root: Path, secret: str) -> tuple[int, tuple[Hit, ...]]:
|
||||
forms: Final = secret_forms(secret)
|
||||
entries: Final = tuple(_walk(root))
|
||||
hits: Final = tuple(chain.from_iterable(_path_hits(root, path, linked, forms) for path, linked in entries))
|
||||
return len(entries), hits
|
||||
|
||||
|
||||
def main(argv: tuple[str, ...], environ: Mapping[str, str], out: TextIO) -> int:
|
||||
if len(argv) != 1:
|
||||
print(f"usage: assert_secret_absent.py <directory>, with the secret in ${SECRET_VARIABLE}", file=out)
|
||||
return 2
|
||||
secret: Final = environ.get(SECRET_VARIABLE, "")
|
||||
root: Final = Path(argv[0])
|
||||
if not secret:
|
||||
print(f"{SECRET_VARIABLE} is empty, nothing to scan for", file=out)
|
||||
return 0
|
||||
if not root.is_dir():
|
||||
print(f"{root} is not a directory, nothing to scan", file=out)
|
||||
return 0
|
||||
try:
|
||||
scanned, hits = scan(root, secret)
|
||||
except Exception as error:
|
||||
print(f"scan failed with {type(error).__name__}, treating the results as unsafe", file=out)
|
||||
return 1
|
||||
for hit in hits:
|
||||
print(f"secret found: {hit.location} ({hit.form})", file=out)
|
||||
if hits:
|
||||
print(f"{len(hits)} copies of the secret under {root}, refusing to upload", file=out)
|
||||
return 1
|
||||
print(f"no copy of the secret in {scanned} files under {root}", file=out)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(tuple(sys.argv[1:]), os.environ, sys.stdout))
|
||||
10
.github/workflows/cost-map-guard.yml
vendored
10
.github/workflows/cost-map-guard.yml
vendored
|
|
@ -8,6 +8,7 @@ on:
|
|||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
|
||||
|
|
@ -21,16 +22,19 @@ jobs:
|
|||
with:
|
||||
ref: ${{ github.event.pull_request.base.sha }}
|
||||
persist-credentials: false
|
||||
- name: Fetch the pull request head, its merge base, and the base branch tip
|
||||
- name: Check out the base branch tip and fetch the pull request head and its merge base
|
||||
id: revisions
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
BASE_REF: ${{ github.event.pull_request.base.ref }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
run: |
|
||||
target="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${BASE_REF}" --jq '.commit.sha')"
|
||||
base_ref="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name)" || base_ref="$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)"
|
||||
target="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${base_ref}" --jq '.commit.sha')"
|
||||
merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${target}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
|
||||
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA" "$target"
|
||||
git checkout -q --detach "$target"
|
||||
echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT"
|
||||
echo "target=$target" >> "$GITHUB_OUTPUT"
|
||||
- name: Set up uv
|
||||
|
|
|
|||
6
.github/workflows/spend-logs-guard.yml
vendored
6
.github/workflows/spend-logs-guard.yml
vendored
|
|
@ -19,17 +19,17 @@ jobs:
|
|||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.base.sha }}
|
||||
ref: ${{ github.event.pull_request.base.ref }}
|
||||
persist-credentials: false
|
||||
- name: Fetch the pull request head, its merge base and its reviews
|
||||
id: revisions
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
BASE_REF: ${{ github.event.pull_request.base.ref }}
|
||||
run: |
|
||||
merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_SHA}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
|
||||
merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_REF}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
|
||||
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
|
||||
gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/reviews" --paginate --slurp | jq 'add // []' > "$RUNNER_TEMP/reviews.json"
|
||||
echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT"
|
||||
|
|
|
|||
199
.github/workflows/test-integration.yml
vendored
Normal file
199
.github/workflows/test-integration.yml
vendored
Normal file
|
|
@ -0,0 +1,199 @@
|
|||
name: integration
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [labeled]
|
||||
schedule:
|
||||
- cron: "5 8-20/3 * * 1-6"
|
||||
timezone: America/Los_Angeles
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
suite:
|
||||
description: Integration group to run
|
||||
type: choice
|
||||
options:
|
||||
- all
|
||||
- providers
|
||||
- extensions
|
||||
- accounting
|
||||
- management
|
||||
- management-replica
|
||||
- database
|
||||
- database-replica
|
||||
- mcp
|
||||
- security
|
||||
- cost
|
||||
- sdk
|
||||
- browser
|
||||
default: all
|
||||
workers:
|
||||
description: Workers per group, blank keeps each group's default
|
||||
type: string
|
||||
default: ""
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.event.label.name }}-${{ inputs.suite }}-${{ inputs.workers }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
jobs:
|
||||
integration:
|
||||
name: integration ${{ matrix.suite }}
|
||||
if: github.event_name != 'pull_request' || github.event.label.name == 'run-ci'
|
||||
environment: integration-license
|
||||
runs-on: ${{ fromJSON('{"providers":"ubuntu-latest-64-cores","extensions":"ubuntu-latest-64-cores","accounting":"ubuntu-latest-32-cores","management":"ubuntu-latest-32-cores","management-replica":"ubuntu-latest-32-cores"}')[matrix.suite] || 'ubuntu-latest-16-cores' }}
|
||||
timeout-minutes: 60
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
suite: ${{ fromJSON((inputs.suite || 'all') == 'all' && '["providers","extensions","accounting","management","management-replica","database","database-replica","mcp","security","cost","sdk","browser"]' || format('["{0}"]', inputs.suite)) }}
|
||||
env:
|
||||
UV_PYTHON: "3.12"
|
||||
INTEGRATION_WORKERS: ${{ inputs.workers || fromJSON('{"providers":"32","extensions":"48","accounting":"16","management":"16","management-replica":"16","cost":"16","mcp":"8","security":"8","database":"4","database-replica":"4"}')[matrix.suite] || '' }}
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
|
||||
command: -c max_connections=1000 -c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c pg_stat_statements.max=20000
|
||||
env:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: circle_test
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: >-
|
||||
--shm-size 1g
|
||||
--health-cmd "pg_isready -U postgres"
|
||||
--health-interval 5s
|
||||
--health-timeout 5s
|
||||
--health-retries 20
|
||||
redis:
|
||||
image: redis:7-alpine@sha256:7aec734b2bb298a1d769fd8729f13b8514a41bf90fcdd1f38ec52267fbaa8ee6
|
||||
ports:
|
||||
- 6379:6379
|
||||
options: >-
|
||||
--health-cmd "redis-cli ping"
|
||||
--health-interval 5s
|
||||
--health-timeout 5s
|
||||
--health-retries 20
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python
|
||||
timeout-minutes: 3
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: ${{ env.UV_PYTHON }}
|
||||
|
||||
- name: Set up uv
|
||||
timeout-minutes: 3
|
||||
uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
|
||||
- name: Cache the Rust build
|
||||
timeout-minutes: 5
|
||||
uses: ./.github/actions/cache-cargo-build
|
||||
|
||||
- name: Cache Prisma binaries
|
||||
timeout-minutes: 5
|
||||
uses: ./.github/actions/cache-prisma-binaries
|
||||
|
||||
- name: Install dependencies
|
||||
timeout-minutes: 20
|
||||
run: .github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras
|
||||
|
||||
- name: Install redis-server 7.2.16
|
||||
timeout-minutes: 3
|
||||
run: |
|
||||
mkdir -p "$RUNNER_TEMP/redis-bin"
|
||||
cid="$(docker create redis:7.2.16@sha256:0637954999d01b7c9ce9167db2da50656e2590d3b884f1c600c5f63bb6e6773c)"
|
||||
docker cp "${cid}:/usr/local/bin/redis-server" "$RUNNER_TEMP/redis-bin/redis-server"
|
||||
docker rm "$cid"
|
||||
"$RUNNER_TEMP/redis-bin/redis-server" --version | grep -q 'v=7.2.16'
|
||||
echo "$RUNNER_TEMP/redis-bin" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Install MCP SDK1 peer
|
||||
if: matrix.suite == 'mcp'
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
uv venv --python 3.12 .venv-mcp-peer
|
||||
uv pip install --no-config --exclude-newer "3 days" --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1'
|
||||
echo "MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Set up Node.js
|
||||
if: matrix.suite == 'browser'
|
||||
timeout-minutes: 5
|
||||
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
|
||||
with:
|
||||
node-version-file: ui/litellm-dashboard/.nvmrc
|
||||
cache: "npm"
|
||||
cache-dependency-path: |
|
||||
ui/litellm-dashboard/package-lock.json
|
||||
tests/e2e/ui/package-lock.json
|
||||
|
||||
- name: Install browser dependencies and build the dashboard
|
||||
if: matrix.suite == 'browser'
|
||||
timeout-minutes: 20
|
||||
run: |
|
||||
(cd ui/litellm-dashboard && npm ci)
|
||||
(cd tests/e2e/ui && npm ci)
|
||||
sudo env PATH="$PATH" DEBIAN_FRONTEND=noninteractive \
|
||||
node tests/e2e/ui/node_modules/@playwright/test/cli.js install-deps chromium
|
||||
node tests/e2e/ui/node_modules/@playwright/test/cli.js install chromium
|
||||
(cd ui/litellm-dashboard && NEXT_TELEMETRY_DISABLED=1 npm run build)
|
||||
|
||||
- name: Run integration contracts
|
||||
timeout-minutes: 45
|
||||
env:
|
||||
SUITE: ${{ matrix.suite }}
|
||||
INTEGRATION_POSTGRES_CONTAINER: ${{ job.services.postgres.id }}
|
||||
INTEGRATION_REDIS_CONTAINER: ${{ job.services.redis.id }}
|
||||
LITELLM_LICENSE: ${{ secrets.LITELLM_LICENSE }}
|
||||
run: |
|
||||
case "$SUITE" in
|
||||
*-replica) bash .circleci/scripts/run_integration.sh "${SUITE%-replica}" replica ;;
|
||||
*) bash .circleci/scripts/run_integration.sh "$SUITE" ;;
|
||||
esac
|
||||
|
||||
- name: Verify every case ran exactly once
|
||||
if: ${{ !cancelled() && matrix.suite != 'browser' }}
|
||||
timeout-minutes: 2
|
||||
run: |
|
||||
record="$(find test-results -mindepth 2 -maxdepth 2 -name execution.json -print -quit)"
|
||||
test -n "$record"
|
||||
mkdir -p "$RUNNER_TEMP/shards/0"
|
||||
cp "$record" "$RUNNER_TEMP/shards/0/execution.json"
|
||||
.venv/bin/python -m tests.integration._support.sharding "$RUNNER_TEMP/shards" --shards 1
|
||||
|
||||
- name: Report MCP coverage
|
||||
if: ${{ !cancelled() && matrix.suite == 'mcp' }}
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
coverage_dir=test-results/integration-mcp/coverage
|
||||
test -f "$coverage_dir/coverage.txt" && exit 0
|
||||
export COVERAGE_FILE="$PWD/$coverage_dir/data"
|
||||
.venv/bin/python -m coverage combine --rcfile=tests/integration/mcp_coverage.toml
|
||||
.venv/bin/python -m coverage report --rcfile=tests/integration/mcp_coverage.toml > "$coverage_dir/coverage.txt"
|
||||
.venv/bin/python -m coverage html --rcfile=tests/integration/mcp_coverage.toml -d "$coverage_dir/html"
|
||||
tail -n 1 "$coverage_dir/coverage.txt"
|
||||
|
||||
- name: Check results for the license
|
||||
id: secret-scan
|
||||
if: ${{ !cancelled() }}
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
SCANNED_SECRET: ${{ secrets.LITELLM_LICENSE }}
|
||||
run: python .github/scripts/assert_secret_absent.py test-results
|
||||
|
||||
- name: Upload results
|
||||
if: ${{ !cancelled() && steps.secret-scan.outcome == 'success' }}
|
||||
timeout-minutes: 10
|
||||
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
|
||||
with:
|
||||
name: integration-${{ matrix.suite }}-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: test-results/
|
||||
retention-days: 14
|
||||
25
.github/workflows/test-linting.yml
vendored
25
.github/workflows/test-linting.yml
vendored
|
|
@ -48,11 +48,13 @@ jobs:
|
|||
if: steps.changes.outputs.decision != 'skip'
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
run: |
|
||||
retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; }
|
||||
MERGE_BASE=$(retry gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
|
||||
BASE_REF=$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name) || BASE_REF=$(retry gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)
|
||||
MERGE_BASE=$(retry gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_REF}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
|
||||
test -n "$MERGE_BASE"
|
||||
retry git fetch --no-tags --depth=1 origin "$MERGE_BASE"
|
||||
echo "GATE_BASE_SHA=$MERGE_BASE" >> "$GITHUB_ENV"
|
||||
|
|
@ -264,6 +266,7 @@ jobs:
|
|||
name: ui
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 8
|
||||
defaults:
|
||||
|
|
@ -281,16 +284,12 @@ jobs:
|
|||
id: changed
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
run: |
|
||||
# base.sha is the base branch tip from when the PR was opened, while
|
||||
# actions/checkout leaves HEAD on a merge of the PR into the *current*
|
||||
# base tip. "$BASE_SHA"...HEAD therefore spans every base-branch commit
|
||||
# landed since, so a PR that touches no UI file still gets linted
|
||||
# against hundreds of other people's files. Diff the PR head against its
|
||||
# own merge base instead, which is exactly what this PR changed.
|
||||
merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
|
||||
base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name) || base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)
|
||||
merge_base=$(gh api "repos/${GITHUB_REPOSITORY}/compare/${base_ref}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
|
||||
test -n "$merge_base"
|
||||
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
|
||||
: > "$RUNNER_TEMP/prettier_files.txt"
|
||||
|
|
@ -630,6 +629,12 @@ jobs:
|
|||
- name: check_migrations_lock_timeout
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_migrations_lock_timeout.py
|
||||
|
||||
- name: check_no_none_string_error_fields
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_no_none_string_error_fields.py
|
||||
|
||||
- name: test_check_no_none_string_error_fields
|
||||
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_check_no_none_string_error_fields.py
|
||||
|
||||
- name: check_no_publicly_known_master_key
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_no_publicly_known_master_key.py
|
||||
|
||||
|
|
|
|||
17
.github/workflows/test-unit.yml
vendored
17
.github/workflows/test-unit.yml
vendored
|
|
@ -245,6 +245,7 @@ jobs:
|
|||
tests/unit/llms/chatgpt
|
||||
tests/unit/llms/clarifai
|
||||
tests/unit/llms/claude_code
|
||||
tests/unit/llms/clf_ai_gateway
|
||||
tests/unit/llms/cloudflare
|
||||
tests/unit/llms/codestral
|
||||
tests/unit/llms/codex
|
||||
|
|
@ -1003,9 +1004,9 @@ jobs:
|
|||
run: |
|
||||
diff -u model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
|
||||
if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils
|
||||
else
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
|
||||
uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
|
||||
cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
|
||||
uv run --no-sync python -c "import importlib.metadata; import litellm.rust_bridge._native; print(importlib.metadata.version('litellm'))"
|
||||
|
|
@ -1150,18 +1151,20 @@ jobs:
|
|||
env:
|
||||
CI: "true"
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
run: |
|
||||
full_suite() { npm run test -- --run --pool forks --maxWorkers=14; }
|
||||
|
||||
if [ -z "$BASE_SHA" ]; then
|
||||
if [ -z "$PR_NUMBER" ]; then
|
||||
echo "Push to $GITHUB_REF_NAME: running the full suite"
|
||||
full_suite
|
||||
exit 0
|
||||
fi
|
||||
|
||||
merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
|
||||
base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name) || base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)
|
||||
merge_base=$(gh api "repos/${GITHUB_REPOSITORY}/compare/${base_ref}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
|
||||
test -n "$merge_base"
|
||||
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
|
||||
changed_files=()
|
||||
|
|
@ -1451,9 +1454,9 @@ jobs:
|
|||
RUST_BRIDGE_ARTIFACT: ${{ needs.rust-bridge.outputs.artifact }}
|
||||
run: |
|
||||
if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils
|
||||
else
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
|
||||
uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
|
||||
cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
|
||||
fi
|
||||
|
|
|
|||
2
Makefile
2
Makefile
|
|
@ -153,7 +153,7 @@ lint-install:
|
|||
# the same set CI's ':(glob)litellm/**/*.py' selects.
|
||||
lint-format-check-changed: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
|
||||
@base_ref=$$($(RESOLVE_BASE)) && \
|
||||
changed=$$(git diff --name-only --diff-filter=ACMR "$$base_ref...HEAD" -- 'litellm/*.py') && \
|
||||
changed=$$(git diff --name-only --diff-filter=ACMR --merge-base "$$base_ref" -- 'litellm/*.py') && \
|
||||
files=$$(printf '%s\n' "$$changed" | grep -v '^litellm/enterprise/' || true) || exit $$?; \
|
||||
if [ -z "$$files" ]; then \
|
||||
echo "No changed litellm Python files to format-check."; \
|
||||
|
|
|
|||
|
|
@ -379,6 +379,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
|
|||
| [Bytez (`bytez`)](https://docs.litellm.ai/docs/providers/bytez) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Cerebras (`cerebras`)](https://docs.litellm.ai/docs/providers/cerebras) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Clarifai (`clarifai`)](https://docs.litellm.ai/docs/providers/clarifai) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [CLF AI Gateway (`clf_ai_gateway`)](https://docs.litellm.ai/docs/providers/clf_ai_gateway) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Cloudflare AI Workers (`cloudflare`)](https://docs.litellm.ai/docs/providers/cloudflare_workers) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Codestral (`codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Cognition (`cognition`)](https://docs.litellm.ai/docs/providers/cognition) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
|
|
@ -398,6 +399,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
|
|||
| [Eden AI (`edenai`)](https://docs.litellm.ai/docs/providers/edenai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | |
|
||||
| [ElevenLabs (`elevenlabs`)](https://docs.litellm.ai/docs/providers/elevenlabs) | ✅ | ✅ | ✅ | | | ✅ | ✅ | | | |
|
||||
| [Empower (`empower`)](https://docs.litellm.ai/docs/providers/empower) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [eu/jev Decisions API (`eu_jev`)](https://docs.litellm.ai/docs/pass_through/eu_jev) | | | | | | | | | | |
|
||||
| [Fal AI (`fal_ai`)](https://docs.litellm.ai/docs/providers/fal_ai) | ✅ | ✅ | ✅ | | ✅ | | | | | |
|
||||
| [Featherless AI (`featherless_ai`)](https://docs.litellm.ai/docs/providers/featherless_ai) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Fireworks AI (`fireworks_ai`)](https://docs.litellm.ai/docs/providers/fireworks_ai) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[project]
|
||||
name = "litellm-enterprise"
|
||||
version = "0.1.80"
|
||||
version = "0.1.81"
|
||||
description = "Package for LiteLLM Enterprise features"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9"
|
||||
|
|
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
|
|||
module-root = ""
|
||||
|
||||
[tool.commitizen]
|
||||
version = "0.1.80"
|
||||
version = "0.1.81"
|
||||
version_files = [
|
||||
"pyproject.toml:^version",
|
||||
"../pyproject.toml:litellm-enterprise==",
|
||||
|
|
|
|||
|
|
@ -142,6 +142,7 @@ GATEWAY_EXACT_PATHS: frozenset[str] = frozenset(
|
|||
"/test",
|
||||
"/debug/memory/summary",
|
||||
"/api/event_logging/batch",
|
||||
"/eu_jev/v1/systemone",
|
||||
}
|
||||
)
|
||||
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -1,6 +1,6 @@
|
|||
[project]
|
||||
name = "litellm-proxy-extras"
|
||||
version = "0.4.111"
|
||||
version = "0.4.112"
|
||||
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9"
|
||||
|
|
@ -30,7 +30,7 @@ required-version = ">=0.10.9"
|
|||
module-root = ""
|
||||
|
||||
[tool.commitizen]
|
||||
version = "0.4.111"
|
||||
version = "0.4.112"
|
||||
version_files = [
|
||||
"pyproject.toml:^version",
|
||||
"../pyproject.toml:litellm-proxy-extras==",
|
||||
|
|
|
|||
|
|
@ -238,7 +238,8 @@ impl AnthropicBeta {
|
|||
Self::ComputerUse20250124
|
||||
| Self::ComputerUse20251124
|
||||
| Self::Context1m20250807
|
||||
| Self::ThinkingBindingControls20260801 => &[],
|
||||
| Self::ThinkingBindingControls20260801
|
||||
| Self::ThinkingDisplayUpdates20260818 => &[],
|
||||
Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse],
|
||||
Self::Effort20251124 => &[BetaProvider::VertexAi],
|
||||
Self::FastMode20260201 | Self::Oauth20250420 => &[
|
||||
|
|
@ -275,7 +276,6 @@ impl AnthropicBeta {
|
|||
BetaProvider::Databricks,
|
||||
],
|
||||
Self::StructuredOutputs20251113 => &[BetaProvider::Bedrock, BetaProvider::VertexAi],
|
||||
Self::ThinkingDisplayUpdates20260818 => &[BetaProvider::Databricks],
|
||||
Self::ToolSearchTool20251019 => &[
|
||||
BetaProvider::Anthropic,
|
||||
BetaProvider::AzureAi,
|
||||
|
|
|
|||
|
|
@ -633,6 +633,7 @@ fireworks_ai_models: Set = set()
|
|||
baseten_models: Set[str] = set()
|
||||
fireworks_ai_embedding_models: Set = set()
|
||||
deepinfra_models: Set = set()
|
||||
clf_ai_gateway_models: set[str] = set() # mutable-ok: filled at import like sibling sets
|
||||
perplexity_models: Set = set()
|
||||
watsonx_models: Set = set()
|
||||
gemini_models: Set = set()
|
||||
|
|
@ -859,6 +860,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
|
|||
bedrock_converse_models.add(key)
|
||||
elif value.get("litellm_provider") == "deepinfra":
|
||||
deepinfra_models.add(key)
|
||||
elif value.get("litellm_provider") == "clf_ai_gateway": # pyright: ignore[reportUnknownMemberType] # raw cost map
|
||||
clf_ai_gateway_models.add(key) # pyright: ignore[reportUnknownArgumentType] # raw cost map key
|
||||
elif value.get("litellm_provider") == "perplexity":
|
||||
perplexity_models.add(key)
|
||||
elif value.get("litellm_provider") == "watsonx":
|
||||
|
|
@ -1110,6 +1113,7 @@ model_list = list(
|
|||
| set(ollama_models)
|
||||
| bedrock_models
|
||||
| deepinfra_models
|
||||
| clf_ai_gateway_models
|
||||
| perplexity_models
|
||||
| set(maritalk_models)
|
||||
| runwayml_models
|
||||
|
|
@ -1235,6 +1239,7 @@ def _build_models_by_provider() -> dict:
|
|||
"ollama": ollama_models,
|
||||
"ollama_chat": ollama_models,
|
||||
"deepinfra": deepinfra_models,
|
||||
"clf_ai_gateway": clf_ai_gateway_models,
|
||||
"perplexity": perplexity_models,
|
||||
"maritalk": maritalk_models,
|
||||
"watsonx": watsonx_models,
|
||||
|
|
@ -1783,6 +1788,9 @@ if TYPE_CHECKING:
|
|||
from .llms.bespoke.decisions.transformation import (
|
||||
BespokeDecisionsConfig as BespokeDecisionsConfig,
|
||||
)
|
||||
from .llms.eu_jev.decisions.transformation import (
|
||||
EuJevDecisionsConfig as EuJevDecisionsConfig,
|
||||
)
|
||||
from .llms.openai.decisions.transformation import (
|
||||
OpenAIDecisionsConfig as OpenAIDecisionsConfig,
|
||||
)
|
||||
|
|
@ -2108,6 +2116,9 @@ if TYPE_CHECKING:
|
|||
LiteLLMProxyChatConfig as _LiteLLMProxyChatConfig,
|
||||
)
|
||||
from .llms.deepinfra.chat.transformation import DeepInfraConfig as _DeepInfraConfig
|
||||
from .llms.clf_ai_gateway.chat.transformation import (
|
||||
ClfAiGatewayConfig as _ClfAiGatewayConfig,
|
||||
)
|
||||
from .llms.llamafile.chat.transformation import (
|
||||
LlamafileChatConfig as _LlamafileChatConfig,
|
||||
)
|
||||
|
|
@ -2137,6 +2148,7 @@ if TYPE_CHECKING:
|
|||
IBMWatsonXAIConfig: Type[_IBMWatsonXAIConfig]
|
||||
LiteLLMProxyChatConfig: Type[_LiteLLMProxyChatConfig]
|
||||
DeepInfraConfig: Type[_DeepInfraConfig]
|
||||
ClfAiGatewayConfig: type[_ClfAiGatewayConfig]
|
||||
LlamafileChatConfig: Type[_LlamafileChatConfig]
|
||||
LMStudioChatConfig: Type[_LMStudioChatConfig]
|
||||
LmStudioEmbeddingConfig: Type[_LmStudioEmbeddingConfig]
|
||||
|
|
|
|||
|
|
@ -174,6 +174,7 @@ LLM_CONFIG_NAMES: Final = (
|
|||
"OllamaDecisionsConfig",
|
||||
"LayaDecisionsConfig",
|
||||
"BespokeDecisionsConfig",
|
||||
"EuJevDecisionsConfig",
|
||||
"OpenAIDecisionsConfig",
|
||||
"AzureDecisionsConfig",
|
||||
"AzureAIDecisionsConfig",
|
||||
|
|
@ -244,6 +245,7 @@ LLM_CONFIG_NAMES: Final = (
|
|||
"MistralEmbeddingConfig",
|
||||
"OpenAIImageVariationConfig",
|
||||
"DeepInfraConfig",
|
||||
"ClfAiGatewayConfig",
|
||||
"DeepgramAudioTranscriptionConfig",
|
||||
"TopazImageVariationConfig",
|
||||
"OpenAITextCompletionConfig",
|
||||
|
|
@ -753,6 +755,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
|
|||
"OllamaDecisionsConfig": (".llms.ollama.decisions.transformation", "OllamaDecisionsConfig"),
|
||||
"LayaDecisionsConfig": (".llms.laya.decisions.transformation", "LayaDecisionsConfig"),
|
||||
"BespokeDecisionsConfig": (".llms.bespoke.decisions.transformation", "BespokeDecisionsConfig"),
|
||||
"EuJevDecisionsConfig": (".llms.eu_jev.decisions.transformation", "EuJevDecisionsConfig"),
|
||||
"OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"),
|
||||
"AzureDecisionsConfig": (".llms.azure.decisions.transformation", "AzureDecisionsConfig"),
|
||||
"AzureAIDecisionsConfig": (".llms.azure_ai.decisions.transformation", "AzureAIDecisionsConfig"),
|
||||
|
|
@ -973,6 +976,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
|
|||
"OpenAIImageVariationConfig",
|
||||
),
|
||||
"DeepInfraConfig": (".llms.deepinfra.chat.transformation", "DeepInfraConfig"),
|
||||
"ClfAiGatewayConfig": (
|
||||
".llms.clf_ai_gateway.chat.transformation",
|
||||
"ClfAiGatewayConfig",
|
||||
),
|
||||
"DeepgramAudioTranscriptionConfig": (
|
||||
".llms.deepgram.audio_transcription.transformation",
|
||||
"DeepgramAudioTranscriptionConfig",
|
||||
|
|
|
|||
|
|
@ -28,9 +28,11 @@ from litellm.litellm_core_utils.env_utils import get_env_int
|
|||
from litellm.litellm_core_utils.safe_json_dumps import UNSERIALIZABLE_OBJECT, safe_dumps, safe_json_structure
|
||||
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
|
||||
from litellm.litellm_core_utils.secret_redaction import (
|
||||
REDACTED,
|
||||
python_redact_string,
|
||||
python_redact_structured_value,
|
||||
redact_internal_details,
|
||||
redact_structured_value,
|
||||
)
|
||||
from litellm.litellm_core_utils.secret_redaction import (
|
||||
redact_string as redact_secret_string,
|
||||
|
|
@ -140,6 +142,32 @@ def redact_internal_details_from_client_message(value: str) -> str:
|
|||
return redact_internal_details(value)
|
||||
|
||||
|
||||
_CLIENT_FIELDS: Final = TypeAdapter(dict[str, object])
|
||||
_CLIENT_ITEMS: Final = TypeAdapter(list[object])
|
||||
|
||||
|
||||
def _redacted_client_field(key: str | None, value: object) -> object:
|
||||
if isinstance(value, str):
|
||||
return redact_internal_details(redact_structured_value(key, value))
|
||||
if isinstance(value, dict):
|
||||
return {
|
||||
child_key: _redacted_client_field(child_key, child)
|
||||
for child_key, child in _CLIENT_FIELDS.validate_python(value).items()
|
||||
}
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [_redacted_client_field(key, child) for child in _CLIENT_ITEMS.validate_python(value)]
|
||||
if not isinstance(value, (int, float)):
|
||||
return value
|
||||
text: Final = str(value)
|
||||
return value if redact_structured_value(key, text) == text else REDACTED
|
||||
|
||||
|
||||
def redact_internal_details_from_client_fields(fields: Mapping[str, object]) -> object:
|
||||
if not _ENABLE_SECRET_REDACTION:
|
||||
return fields
|
||||
return _redacted_client_field(None, dict(fields))
|
||||
|
||||
|
||||
def _substituted_color_message(record: logging.LogRecord) -> str | None:
|
||||
"""Render a record's ``color_message`` against its args, or None if absent.
|
||||
|
||||
|
|
|
|||
|
|
@ -8,10 +8,9 @@ The following caching mechanisms are supported:
|
|||
2. **RedisSemanticCache**
|
||||
3. **QdrantSemanticCache**
|
||||
4. **InMemoryCache**
|
||||
5. **DiskCache**
|
||||
6. **S3Cache**
|
||||
7. **AzureBlobCache**
|
||||
8. **DualCache** (updates both Redis and an in-memory cache simultaneously)
|
||||
5. **S3Cache**
|
||||
6. **AzureBlobCache**
|
||||
7. **DualCache** (updates both Redis and an in-memory cache simultaneously)
|
||||
|
||||
## Folder Structure
|
||||
|
||||
|
|
@ -20,7 +19,6 @@ litellm/caching/
|
|||
├── base_cache.py
|
||||
├── caching.py
|
||||
├── caching_handler.py
|
||||
├── disk_cache.py
|
||||
├── dual_cache.py
|
||||
├── in_memory_cache.py
|
||||
├── qdrant_semantic_cache.py
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
from .azure_blob_cache import AzureBlobCache
|
||||
from .caching import Cache, LiteLLMCacheType
|
||||
from .disk_cache import DiskCache
|
||||
from .dual_cache import DualCache
|
||||
from .gcs_cache import GCSCache
|
||||
from .in_memory_cache import InMemoryCache
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ from contextlib import contextmanager
|
|||
from enum import Enum
|
||||
from typing import Any, Final, Literal, cast
|
||||
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._internal_context import current_service_target, service_target
|
||||
|
|
@ -31,7 +31,6 @@ from litellm.types.utils import EmbeddingResponse, is_litellm_owned_kwarg
|
|||
|
||||
from .azure_blob_cache import AzureBlobCache
|
||||
from .base_cache import BaseCache
|
||||
from .disk_cache import DiskCache
|
||||
from .dual_cache import DualCache # noqa: F401 # re-exported, callers import DualCache from litellm.caching.caching
|
||||
from .gcs_cache import GCSCache
|
||||
from .in_memory_cache import InMemoryCache
|
||||
|
|
@ -44,6 +43,9 @@ from .s3_cache import S3Cache
|
|||
RESPONSE_CACHE_TARGET: Final = "llm_response"
|
||||
_ALWAYS_KEYED_PROVIDER_PARAMS: Final = frozenset({"continuation_token"})
|
||||
_DECODED_LITERAL: Final = TypeAdapter(object)
|
||||
_REDIS_STARTUP_NODES_ADAPTER: Final = TypeAdapter(
|
||||
list[dict[str, object]], config=ConfigDict(strict=True, hide_input_in_errors=True)
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
|
|
@ -153,7 +155,7 @@ class Cache:
|
|||
Initializes the cache based on the given type.
|
||||
|
||||
Args:
|
||||
type (str, optional): The type of cache to initialize. Can be "local", "redis", "redis-semantic", "qdrant-semantic", "s3" or "disk". Defaults to "local".
|
||||
type (str, optional): The type of cache to initialize. Can be "local", "redis", "redis-semantic", "qdrant-semantic" or "s3". Defaults to "local".
|
||||
|
||||
# Redis Cache Args
|
||||
host (str, optional): The host address for the Redis cache. Required if type is "redis".
|
||||
|
|
@ -174,9 +176,6 @@ class Cache:
|
|||
semantic_cache_scope (str, optional): "key" isolates semantic-cache buckets per key/team/org. "end_user" additionally isolates per end user (falls back to the key scope when the request carries no end-user id). Defaults to "key".
|
||||
max_messages (int, optional): Requests with more `messages` (or Responses API `input` items) than this are neither looked up nor stored, so long agent conversations never serve or create a cache entry. None disables the limit. Defaults to 4.
|
||||
|
||||
# Disk Cache Args
|
||||
disk_cache_dir (str, optional): The directory for the disk cache. Defaults to None.
|
||||
|
||||
# S3 Cache Args
|
||||
s3_bucket_name (str, optional): The bucket name for the s3 cache. Defaults to None.
|
||||
s3_region_name (str, optional): The region name for the s3 cache. Defaults to None.
|
||||
|
|
@ -211,7 +210,7 @@ class Cache:
|
|||
if not redis_startup_nodes:
|
||||
_env_cluster_nodes: Final = litellm.get_secret("REDIS_CLUSTER_NODES")
|
||||
if _env_cluster_nodes is not None and isinstance(_env_cluster_nodes, str):
|
||||
redis_startup_nodes = json.loads(_env_cluster_nodes)
|
||||
redis_startup_nodes = _REDIS_STARTUP_NODES_ADAPTER.validate_python(json.loads(_env_cluster_nodes))
|
||||
|
||||
if redis_startup_nodes:
|
||||
# Only pass GCP parameters if they are provided
|
||||
|
|
@ -306,8 +305,11 @@ class Cache:
|
|||
account_url=azure_account_url,
|
||||
container=azure_blob_container,
|
||||
)
|
||||
elif type == LiteLLMCacheType.DISK:
|
||||
self.cache = DiskCache(disk_cache_dir=disk_cache_dir)
|
||||
else:
|
||||
supported_types: Final = ", ".join(member.value for member in LiteLLMCacheType)
|
||||
raise ValueError(f"Unsupported cache type {type!r}. Supported types: {supported_types}")
|
||||
if disk_cache_dir is not None:
|
||||
verbose_logger.warning("Ignoring disk_cache_dir, the disk cache backend was removed")
|
||||
if "cache" not in litellm.input_callback:
|
||||
litellm.input_callback.append("cache")
|
||||
if "cache" not in litellm.success_callback:
|
||||
|
|
@ -1044,7 +1046,7 @@ def enable_cache(
|
|||
Enable cache with the specified configuration.
|
||||
|
||||
Args:
|
||||
type (Optional[Literal["local", "redis", "s3", "disk"]]): The type of cache to enable. Defaults to "local".
|
||||
type (Optional[Literal["local", "redis", "s3"]]): The type of cache to enable. Defaults to "local".
|
||||
host (Optional[str]): The host address of the cache server. Defaults to None.
|
||||
port (Optional[str]): The port number of the cache server. Defaults to None.
|
||||
password (Optional[str]): The password for the cache server. Defaults to None.
|
||||
|
|
@ -1056,7 +1058,7 @@ def enable_cache(
|
|||
None
|
||||
|
||||
Raises:
|
||||
None
|
||||
ValueError: If no cache is set yet and the cache type is unsupported.
|
||||
"""
|
||||
print_verbose("LiteLLM: Enabling Cache")
|
||||
if "cache" not in litellm.input_callback:
|
||||
|
|
@ -1091,7 +1093,7 @@ def update_cache(
|
|||
Update the cache for LiteLLM.
|
||||
|
||||
Args:
|
||||
type (Optional[Literal["local", "redis", "s3", "disk"]]): The type of cache. Defaults to "local".
|
||||
type (Optional[Literal["local", "redis", "s3"]]): The type of cache. Defaults to "local".
|
||||
host (Optional[str]): The host of the cache. Defaults to None.
|
||||
port (Optional[str]): The port of the cache. Defaults to None.
|
||||
password (Optional[str]): The password for the cache. Defaults to None.
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ This file is a wrapper around caching.py
|
|||
|
||||
This class is used to handle caching logic specific for LLM API requests (completion / embedding / text_completion / transcription etc)
|
||||
|
||||
It utilizes the (RedisCache, s3Cache, RedisSemanticCache, QdrantSemanticCache, InMemoryCache, DiskCache) based on what the user has setup
|
||||
It utilizes the (RedisCache, s3Cache, RedisSemanticCache, QdrantSemanticCache, InMemoryCache) based on what the user has setup
|
||||
|
||||
In each method it will call the appropriate method from caching.py
|
||||
"""
|
||||
|
|
@ -830,6 +830,13 @@ class LLMCachingHandler:
|
|||
if is_internal_call.get():
|
||||
return
|
||||
|
||||
if logging_obj.defer_async_logging and not logging_obj.stream:
|
||||
from litellm.utils import dispatch_success_logging
|
||||
|
||||
logging_obj.model_call_details["cache_hit"] = cache_hit # rebind-ok: capture cache status before deferring
|
||||
dispatch_success_logging(logging_obj, cached_result, start_time, end_time, False, False)
|
||||
return
|
||||
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
|
||||
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(
|
||||
|
|
|
|||
|
|
@ -1,92 +0,0 @@
|
|||
import json
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from .base_cache import BaseCache
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from opentelemetry.trace import Span as _Span
|
||||
|
||||
Span = _Span
|
||||
else:
|
||||
Span = Any
|
||||
|
||||
_DECODED_DISK_VALUE: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
class DiskCache(BaseCache):
|
||||
def __init__(self, disk_cache_dir: str | None = None):
|
||||
try:
|
||||
import diskcache as dc
|
||||
except ModuleNotFoundError as e:
|
||||
raise ModuleNotFoundError("Please install litellm with `litellm[caching]` to use disk caching.") from e
|
||||
|
||||
# if users don't provider one, use the default litellm cache
|
||||
if disk_cache_dir is None:
|
||||
self.disk_cache = dc.Cache(".litellm_cache")
|
||||
else:
|
||||
self.disk_cache = dc.Cache(disk_cache_dir)
|
||||
|
||||
def set_cache(self, key, value, **kwargs):
|
||||
if "ttl" in kwargs:
|
||||
self.disk_cache.set(key, value, expire=kwargs["ttl"])
|
||||
else:
|
||||
self.disk_cache.set(key, value)
|
||||
|
||||
async def async_set_cache(self, key, value, **kwargs):
|
||||
self.set_cache(key=key, value=value, **kwargs)
|
||||
|
||||
async def async_set_cache_pipeline(self, cache_list, **kwargs):
|
||||
for cache_key, cache_value in cache_list:
|
||||
if "ttl" in kwargs:
|
||||
self.set_cache(key=cache_key, value=cache_value, ttl=kwargs["ttl"])
|
||||
else:
|
||||
self.set_cache(key=cache_key, value=cache_value)
|
||||
|
||||
def get_cache(self, key, **kwargs):
|
||||
original_cached_response: Final = self.disk_cache.get(key)
|
||||
if original_cached_response:
|
||||
try:
|
||||
cached_response = _DECODED_DISK_VALUE.validate_python(json.loads(original_cached_response))
|
||||
except Exception:
|
||||
cached_response = original_cached_response
|
||||
return cached_response
|
||||
return None
|
||||
|
||||
def batch_get_cache(self, keys: list, **kwargs):
|
||||
return_val: Final = []
|
||||
for k in keys:
|
||||
val = self.get_cache(key=k, **kwargs)
|
||||
return_val.append(val)
|
||||
return return_val
|
||||
|
||||
def increment_cache(self, key, value: int, **kwargs) -> int:
|
||||
with self.disk_cache.transact():
|
||||
cached_value: Final = self.get_cache(key=key)
|
||||
init_value: Final = cached_value if isinstance(cached_value, int) else 0
|
||||
new_value: Final = init_value + value
|
||||
self.set_cache(key, new_value, **kwargs)
|
||||
return new_value
|
||||
|
||||
async def async_get_cache(self, key, **kwargs):
|
||||
return self.get_cache(key=key, **kwargs)
|
||||
|
||||
async def async_batch_get_cache(self, keys: list, **kwargs):
|
||||
return_val: Final = []
|
||||
for k in keys:
|
||||
val = self.get_cache(key=k, **kwargs)
|
||||
return_val.append(val)
|
||||
return return_val
|
||||
|
||||
async def async_increment(self, key, value: int, **kwargs) -> int:
|
||||
return self.increment_cache(key=key, value=value, **kwargs)
|
||||
|
||||
def flush_cache(self):
|
||||
self.disk_cache.clear()
|
||||
|
||||
async def disconnect(self):
|
||||
pass
|
||||
|
||||
def delete_cache(self, key):
|
||||
self.disk_cache.pop(key)
|
||||
|
|
@ -29,6 +29,9 @@ it does not belong to. Queued clients are therefore bucketed by what it takes to
|
|||
close them, and each bucket is ordered by deadline, so a reap walks the entries
|
||||
that are due rather than the whole queue.
|
||||
|
||||
A bucket whose loop has been collected is dropped, and its clients are left to the
|
||||
collector, as they are when the queue is full.
|
||||
|
||||
The queue holds its clients weakly, so waiting out a grace window never keeps
|
||||
alive anything the collector would have reclaimed first.
|
||||
"""
|
||||
|
|
@ -36,6 +39,7 @@ alive anything the collector would have reclaimed first.
|
|||
import asyncio
|
||||
import contextlib
|
||||
import inspect
|
||||
import itertools
|
||||
import threading
|
||||
import time
|
||||
import weakref
|
||||
|
|
@ -55,6 +59,9 @@ _CLOSABLE_ANYWHERE: Final = "closable-anywhere"
|
|||
_CLOSABLE_ON_ANY_LOOP: Final = "closable-on-any-loop"
|
||||
|
||||
_BucketKey = str | int
|
||||
_LOOP_TOKENS: Final[weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, int]] = weakref.WeakKeyDictionary()
|
||||
_LOOP_TOKEN_COUNTER: Final = itertools.count()
|
||||
_LOOP_TOKEN_LOCK: Final = threading.Lock()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -65,12 +72,12 @@ class _PendingClose:
|
|||
collector would otherwise have reclaimed first.
|
||||
|
||||
``needs_loop`` is set for a client whose close is a coroutine; those can only
|
||||
be closed from the event loop they were evicted on, recorded in ``loop_id``.
|
||||
be closed from the event loop they were evicted on, recorded in ``loop_token``.
|
||||
A client that closes synchronously carries neither constraint.
|
||||
"""
|
||||
|
||||
client_ref: "weakref.ref[object]"
|
||||
loop_id: int | None
|
||||
loop_token: int | None
|
||||
needs_loop: bool
|
||||
close_after: float
|
||||
|
||||
|
|
@ -79,16 +86,24 @@ def _bucket_key(pending: _PendingClose) -> _BucketKey:
|
|||
"""Which reaps can close this entry: any at all, any running a loop, or one loop's."""
|
||||
if not pending.needs_loop:
|
||||
return _CLOSABLE_ANYWHERE
|
||||
if pending.loop_id is None:
|
||||
if pending.loop_token is None:
|
||||
return _CLOSABLE_ON_ANY_LOOP
|
||||
return pending.loop_id
|
||||
return pending.loop_token
|
||||
|
||||
|
||||
def _running_loop_id() -> int | None:
|
||||
def running_loop_token() -> int | None:
|
||||
try:
|
||||
return id(asyncio.get_running_loop())
|
||||
loop: Final = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
return None
|
||||
try:
|
||||
token: Final[int | None] = _LOOP_TOKENS.get(loop)
|
||||
if token is not None:
|
||||
return token
|
||||
with _LOOP_TOKEN_LOCK:
|
||||
return _LOOP_TOKENS.setdefault(loop, next(_LOOP_TOKEN_COUNTER))
|
||||
except TypeError:
|
||||
return id(loop)
|
||||
|
||||
|
||||
def _close_function(client: object) -> Callable[[], object] | None:
|
||||
|
|
@ -186,6 +201,8 @@ class EvictedClientCloser:
|
|||
self._clock = clock
|
||||
self._owned: weakref.WeakSet[object] = weakref.WeakSet()
|
||||
self._buckets: dict[_BucketKey, deque[_PendingClose]] = {} # mutable-ok: deadline-ordered queues
|
||||
self._dead_loop_tokens: deque[int] = deque() # mutable-ok: loop finalizers append without taking the queue lock
|
||||
self._watched_loop_tokens: set[int] = set() # mutable-ok: tracks tokens until their dead buckets are discarded
|
||||
self._pending_count = 0
|
||||
self._queue_lock = threading.Lock() # the cache is reachable from every worker thread's loop
|
||||
self._close_tasks: set[asyncio.Task[None]] = set() # mutable-ok: strong refs to running closes
|
||||
|
|
@ -216,13 +233,19 @@ class EvictedClientCloser:
|
|||
close_fn: Final = _close_function(client)
|
||||
if close_fn is None:
|
||||
return
|
||||
loop_token: Final = running_loop_token()
|
||||
needs_loop: Final = inspect.iscoroutinefunction(close_fn)
|
||||
if self._pending_count >= self._max_pending:
|
||||
return
|
||||
self._discard_dead_loops()
|
||||
if self._pending_count >= self._max_pending:
|
||||
return
|
||||
if needs_loop and loop_token is not None:
|
||||
self._watch_loop(loop_token)
|
||||
self._enqueue(
|
||||
_PendingClose(
|
||||
client_ref=weakref.ref(client),
|
||||
loop_id=_running_loop_id(),
|
||||
needs_loop=inspect.iscoroutinefunction(close_fn),
|
||||
loop_token=loop_token,
|
||||
needs_loop=needs_loop,
|
||||
close_after=self._clock() + self._grace_seconds,
|
||||
)
|
||||
)
|
||||
|
|
@ -236,7 +259,7 @@ class EvictedClientCloser:
|
|||
if not self._pending_count:
|
||||
return
|
||||
now: Final = self._clock()
|
||||
for pending in self._take_due(_running_loop_id(), now):
|
||||
for pending in self._take_due(running_loop_token(), now):
|
||||
client = pending.client_ref()
|
||||
if client is None:
|
||||
continue
|
||||
|
|
@ -249,6 +272,29 @@ class EvictedClientCloser:
|
|||
def pending_count(self) -> int:
|
||||
return self._pending_count
|
||||
|
||||
def _watch_loop(self, loop_token: int) -> None:
|
||||
if loop_token in self._watched_loop_tokens:
|
||||
return
|
||||
try:
|
||||
weakref.finalize(asyncio.get_running_loop(), self._dead_loop_tokens.append, loop_token)
|
||||
except TypeError:
|
||||
return
|
||||
self._watched_loop_tokens.add(loop_token)
|
||||
|
||||
def _discard_dead_loops(self) -> None:
|
||||
with self._queue_lock:
|
||||
self._discard_dead_loops_locked()
|
||||
|
||||
def _discard_dead_loops_locked(self) -> None:
|
||||
while self._dead_loop_tokens:
|
||||
self._discard_dead_loop_token_locked(self._dead_loop_tokens.popleft())
|
||||
|
||||
def _discard_dead_loop_token_locked(self, loop_token: int) -> None:
|
||||
self._watched_loop_tokens.discard(loop_token)
|
||||
bucket: Final = self._buckets.pop(loop_token, None)
|
||||
if bucket is not None:
|
||||
self._pending_count -= len(bucket)
|
||||
|
||||
def _enqueue(self, pending: _PendingClose) -> None:
|
||||
"""Append to the entry's bucket, dropping any dead entries it queues behind.
|
||||
|
||||
|
|
@ -257,6 +303,7 @@ class EvictedClientCloser:
|
|||
the front rather than having to be searched for.
|
||||
"""
|
||||
with self._queue_lock:
|
||||
self._discard_dead_loops_locked()
|
||||
bucket: Final = self._buckets.setdefault(_bucket_key(pending), deque())
|
||||
while bucket and bucket[0].client_ref() is None:
|
||||
bucket.popleft()
|
||||
|
|
@ -264,9 +311,12 @@ class EvictedClientCloser:
|
|||
bucket.append(pending)
|
||||
self._pending_count += 1
|
||||
|
||||
def _take_due(self, loop_id: int | None, now: float) -> tuple[_PendingClose, ...]:
|
||||
buckets = (_CLOSABLE_ANYWHERE,) if loop_id is None else (_CLOSABLE_ANYWHERE, _CLOSABLE_ON_ANY_LOOP, loop_id)
|
||||
def _take_due(self, loop_token: int | None, now: float) -> tuple[_PendingClose, ...]:
|
||||
buckets = (
|
||||
(_CLOSABLE_ANYWHERE,) if loop_token is None else (_CLOSABLE_ANYWHERE, _CLOSABLE_ON_ANY_LOOP, loop_token)
|
||||
)
|
||||
with self._queue_lock:
|
||||
self._discard_dead_loops_locked()
|
||||
return tuple(pending for key in buckets for pending in self._drain_locked(key, now))
|
||||
|
||||
def _drain_locked(self, key: _BucketKey, now: float) -> Iterator[_PendingClose]:
|
||||
|
|
|
|||
|
|
@ -2,10 +2,9 @@
|
|||
Add the event loop to the cache key, to prevent event loop closed errors.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from typing import Final
|
||||
|
||||
from .evicted_client_closer import EvictedClientCloser, default_evicted_client_closer
|
||||
from .evicted_client_closer import EvictedClientCloser, default_evicted_client_closer, running_loop_token
|
||||
from .in_memory_cache import InMemoryCache
|
||||
|
||||
|
||||
|
|
@ -43,24 +42,22 @@ class LLMClientCache(InMemoryCache):
|
|||
self.evicted_client_closer.schedule(evicted)
|
||||
self.evicted_client_closer.reap()
|
||||
|
||||
def update_cache_key_with_event_loop(self, key):
|
||||
def update_cache_key_with_event_loop(self, key: str) -> str:
|
||||
"""
|
||||
Add the event loop to the cache key, to prevent event loop closed errors.
|
||||
If none, use the key as is.
|
||||
"""
|
||||
try:
|
||||
event_loop: Final = asyncio.get_running_loop()
|
||||
stringified_event_loop: Final = str(id(event_loop))
|
||||
return f"{key}-{stringified_event_loop}"
|
||||
except RuntimeError: # handle no current running event loop
|
||||
return key
|
||||
token: Final = running_loop_token()
|
||||
return f"{key}-{token}" if token is not None else key
|
||||
|
||||
def set_cache(self, key: str, value: object, litellm_owned_client: bool = False, **kwargs):
|
||||
def _scoped_key(self, key: str, loop_bound: bool) -> str:
|
||||
return self.update_cache_key_with_event_loop(key) if loop_bound else key
|
||||
|
||||
def set_cache(self, key: str, value: object, litellm_owned_client: bool = False, loop_bound: bool = True, **kwargs):
|
||||
"""``litellm_owned_client`` marks a client litellm built, so it may be closed once evicted."""
|
||||
if litellm_owned_client:
|
||||
self.evicted_client_closer.mark_owned(value)
|
||||
key = self.update_cache_key_with_event_loop(key)
|
||||
return super().set_cache(key, value, **kwargs)
|
||||
return super().set_cache(self._scoped_key(key, loop_bound), value, **kwargs)
|
||||
|
||||
async def async_set_cache(self, key: str, value: object, litellm_owned_client: bool = False, **kwargs):
|
||||
if litellm_owned_client:
|
||||
|
|
@ -68,13 +65,13 @@ class LLMClientCache(InMemoryCache):
|
|||
key = self.update_cache_key_with_event_loop(key)
|
||||
return await super().async_set_cache(key, value, **kwargs)
|
||||
|
||||
def get_cache(self, key, **kwargs):
|
||||
key = self.update_cache_key_with_event_loop(key)
|
||||
def get_cache(self, key: str, loop_bound: bool = True, **kwargs):
|
||||
scoped_key: Final = self._scoped_key(key, loop_bound)
|
||||
self.evicted_client_closer.reap()
|
||||
|
||||
return super().get_cache(key, **kwargs)
|
||||
return super().get_cache(scoped_key, **kwargs)
|
||||
|
||||
async def async_get_cache(self, key, **kwargs):
|
||||
async def async_get_cache(self, key: str, **kwargs):
|
||||
key = self.update_cache_key_with_event_loop(key)
|
||||
|
||||
return await super().async_get_cache(key, **kwargs)
|
||||
|
|
|
|||
|
|
@ -1979,6 +1979,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
|
|||
self._tool_call_index_map: dict[int, int] = {} # mutable-ok: per-stream accumulator state
|
||||
self._structured_output: Final = structured_output
|
||||
self._first_message_item_done: bool = False
|
||||
self._reasoning_summary_streamed: bool = False
|
||||
|
||||
def _handle_string_chunk(
|
||||
self, str_line: Union[str, "BaseModel"]
|
||||
|
|
@ -2021,6 +2022,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
|
|||
tool_call_index_map: dict[int, int] | None = None, # mutable-ok: per-stream state, remapped in place
|
||||
replays_openai_history: bool = False,
|
||||
structured_output: bool = False,
|
||||
reasoning_summary_streamed: bool = False,
|
||||
) -> "ModelResponseStream":
|
||||
"""
|
||||
Translate a Responses API streaming chunk to OpenAI chat completion streaming format.
|
||||
|
|
@ -2030,6 +2032,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
|
|||
tool_call_index_map: Per-stream output_index -> sequential tool_call index map
|
||||
replays_openai_history: Record the completed output's item sequence for the next turn
|
||||
structured_output: Only the first message item became content
|
||||
reasoning_summary_streamed: Whether earlier chunks already streamed reasoning summary text
|
||||
|
||||
Returns:
|
||||
ModelResponseStream: OpenAI-formatted streaming chunk
|
||||
|
|
@ -2210,13 +2213,15 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
|
|||
)
|
||||
else:
|
||||
raise ValueError(f"Chat provider: Invalid text delta {parsed_chunk}")
|
||||
elif event_type == "response.reasoning_summary_part.added" and reasoning_summary_streamed:
|
||||
return ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(reasoning_content=" "))])
|
||||
elif event_type == "response.reasoning_summary_text.delta":
|
||||
content_part = parsed_chunk.get("delta", None)
|
||||
if content_part:
|
||||
return ModelResponseStream(
|
||||
choices=[
|
||||
StreamingChoices(
|
||||
index=cast(int, parsed_chunk.get("summary_index")),
|
||||
index=0,
|
||||
delta=Delta(reasoning_content=content_part),
|
||||
)
|
||||
]
|
||||
|
|
@ -2317,10 +2322,13 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
|
|||
tool_call_index_map=self._tool_call_index_map,
|
||||
replays_openai_history=self._replays_openai_history,
|
||||
structured_output=self._structured_output,
|
||||
reasoning_summary_streamed=self._reasoning_summary_streamed,
|
||||
)
|
||||
)
|
||||
if self._structured_output and _is_message_item_done(chunk):
|
||||
self._first_message_item_done = True
|
||||
if _responses_event_type(chunk) == "response.reasoning_summary_text.delta" and chunk.get("delta"):
|
||||
self._reasoning_summary_streamed = True
|
||||
return self._with_served_service_tier(self._with_stream_scoped_id(translated))
|
||||
|
||||
def _is_text_delta_after_structured_answer(self, chunk: Mapping[str, object]) -> bool:
|
||||
|
|
|
|||
|
|
@ -753,7 +753,13 @@ ANTHROPIC_BATCHES_ROUTE: Final = "/v1/messages/batches"
|
|||
ANTHROPIC_IMAGE_MAX_LONG_EDGE_PX: Final = 1568
|
||||
ANTHROPIC_IMAGE_MAX_PIXELS: Final = 1_150_000
|
||||
ANTHROPIC_IMAGE_PIXELS_PER_TOKEN: Final = 750
|
||||
OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS: Final = 13
|
||||
OPENAI_CHAT_PDF_FILE_TOKENS: Final = 192
|
||||
PDF_DATA_URL_PREFIX: Final = "data:application/pdf;base64,"
|
||||
PDF_MAX_FORM_XOBJECT_DEPTH: Final = 32
|
||||
PDF_MAX_XOBJECT_DRAWS_PER_PAGE: Final = 10_000
|
||||
OPENAI_MODEL_NAME_MAX_DASHES: Final = 16
|
||||
OPENAI_CHAT_MODES: Final = frozenset({"chat", "responses"})
|
||||
VERTEX_BATCH_PREDICTION_JOBS_ROUTE: Final = "batchPredictionJobs"
|
||||
ANTHROPIC_WEB_SEARCH_TOOL_MAX_USES: Final = {
|
||||
"low": 1,
|
||||
|
|
@ -813,6 +819,7 @@ LITELLM_CHAT_PROVIDERS: Final = [
|
|||
"ollama",
|
||||
"ollama_chat",
|
||||
"deepinfra",
|
||||
"clf_ai_gateway",
|
||||
"perplexity",
|
||||
"mistral",
|
||||
"groq",
|
||||
|
|
@ -1012,6 +1019,7 @@ openai_compatible_endpoints: Final[list[str]] = [
|
|||
"api.perplexity.ai",
|
||||
"api.endpoints.anyscale.com/v1",
|
||||
"api.deepinfra.com/v1/openai",
|
||||
"api.clfaigateway.dev/v1",
|
||||
"api.mistral.ai/v1",
|
||||
"codestral.mistral.ai/v1/chat/completions",
|
||||
"codestral.mistral.ai/v1/fim/completions",
|
||||
|
|
@ -1083,6 +1091,7 @@ openai_compatible_providers: Final[list[str]] = [
|
|||
"deepseek",
|
||||
"tencent",
|
||||
"deepinfra",
|
||||
"clf_ai_gateway",
|
||||
"perplexity",
|
||||
"xinference",
|
||||
"xai",
|
||||
|
|
|
|||
|
|
@ -153,7 +153,9 @@ from litellm.utils import (
|
|||
ProviderConfigManager,
|
||||
TextCompletionResponse,
|
||||
TranscriptionResponse,
|
||||
cached_cost_map_key,
|
||||
cached_get_model_info_helper,
|
||||
mapped_model_fill_missing,
|
||||
model_info_with_pricing_overrides,
|
||||
runtime_model_cost_registrations,
|
||||
token_counter,
|
||||
|
|
@ -962,21 +964,17 @@ def select_model_name_for_cost_calc(
|
|||
"""
|
||||
1. If custom pricing is true, return received model name
|
||||
2. If base_model is set (e.g. for azure models), return that
|
||||
3. If completion response has model set return that
|
||||
3. If completion response has model set return that, unless only the requested model's cost-map row has rates
|
||||
4. Check if model is passed in return that
|
||||
"""
|
||||
|
||||
return_model: str | None = None
|
||||
custom_llm_provider = _get_provider_for_cost_calc(model=model, custom_llm_provider=custom_llm_provider)
|
||||
|
||||
completion_response_model: str | None = None
|
||||
if completion_response is not None:
|
||||
if isinstance(completion_response, BaseModel):
|
||||
completion_response_model = getattr(completion_response, "model", None)
|
||||
elif isinstance(completion_response, dict):
|
||||
completion_response_model = completion_response.get("model", None)
|
||||
completion_response_model: Final = _get_response_model(completion_response)
|
||||
hidden_params: Final[dict | None] = getattr(completion_response, "_hidden_params", None)
|
||||
provider_response_model: Final = _get_hidden_str_for_cost_calc(hidden_params, "provider_response_model")
|
||||
requested_model: Final = model or _get_hidden_str_for_cost_calc(hidden_params, "litellm_model_name")
|
||||
explicit_pricing: Final = custom_pricing is True or base_model is not None
|
||||
configured_model: Final = model or (
|
||||
completion_response_model if isinstance(completion_response_model, str) else None
|
||||
|
|
@ -1002,35 +1000,72 @@ def select_model_name_for_cost_calc(
|
|||
else:
|
||||
return_model = model
|
||||
|
||||
elif base_model is not None or bedrock_pricing_model is not None or provider_response_model is not None:
|
||||
return_model = base_model if base_model is not None else bedrock_pricing_model or provider_response_model
|
||||
elif base_model is not None or bedrock_pricing_model is not None:
|
||||
return_model = base_model if base_model is not None else bedrock_pricing_model
|
||||
|
||||
elif provider_response_model is not None:
|
||||
return _served_or_priced_requested_model(
|
||||
provider_response_model, requested_model, custom_llm_provider, priced_region
|
||||
)
|
||||
|
||||
elif completion_response_model is None and hidden_params is not None:
|
||||
if hidden_params.get("model", None) is not None and len(hidden_params["model"]) > 0:
|
||||
return_model = hidden_params.get("model", model)
|
||||
|
||||
if return_model is None and completion_response_model is not None:
|
||||
return_model = completion_response_model
|
||||
return _served_or_priced_requested_model(
|
||||
completion_response_model, requested_model, custom_llm_provider, priced_region
|
||||
)
|
||||
|
||||
if return_model is None and model is not None:
|
||||
return_model = model
|
||||
|
||||
if (
|
||||
return_model is not None
|
||||
and custom_llm_provider is not None
|
||||
and not _model_contains_known_llm_provider(return_model)
|
||||
): # add provider prefix if not already present, to match model_cost
|
||||
provider_prefix: Final = (
|
||||
custom_llm_provider if priced_region is None else f"{custom_llm_provider}/{priced_region}"
|
||||
)
|
||||
return_model = _strip_unregistered_leading_segments(f"{provider_prefix}/{return_model}", priced_region)
|
||||
|
||||
return return_model
|
||||
if return_model is None:
|
||||
return None
|
||||
return _cost_map_model_name(return_model, custom_llm_provider, priced_region)
|
||||
|
||||
|
||||
_select_model_name_for_cost_calc = select_model_name_for_cost_calc
|
||||
|
||||
|
||||
def _cost_map_model_name(model: str, custom_llm_provider: str | None, region_name: str | None) -> str:
|
||||
if custom_llm_provider is None or _model_contains_known_llm_provider(model):
|
||||
return model
|
||||
provider_prefix: Final = custom_llm_provider if region_name is None else f"{custom_llm_provider}/{region_name}"
|
||||
return _strip_unregistered_leading_segments(f"{provider_prefix}/{model}", region_name)
|
||||
|
||||
|
||||
def _rule_declares_rates(model: str, custom_llm_provider: str | None) -> bool:
|
||||
rule_info: Final = unmapped_model_generalization(model, custom_llm_provider)
|
||||
return rule_info is not None and _cost_map_entry_prices_anything(rule_info)
|
||||
|
||||
|
||||
def _resolves_to_rates(model: str, custom_llm_provider: str | None) -> bool:
|
||||
key: Final = cached_cost_map_key(model, custom_llm_provider)
|
||||
if key is None:
|
||||
return False
|
||||
entry: Final = _raw_cost_map_entry(key)
|
||||
if entry is None:
|
||||
return _rule_declares_rates(model, custom_llm_provider)
|
||||
fill_missing: Final = mapped_model_fill_missing(key, entry)
|
||||
return _cost_map_entry_prices_anything(entry if fill_missing is None else {**fill_missing, **entry})
|
||||
|
||||
|
||||
def _served_or_priced_requested_model(
|
||||
served_model: str, requested_model: str | None, custom_llm_provider: str | None, priced_region: str | None
|
||||
) -> str:
|
||||
served: Final = _cost_map_model_name(served_model, custom_llm_provider, priced_region)
|
||||
if requested_model is None:
|
||||
return served
|
||||
requested: Final = _cost_map_model_name(requested_model, custom_llm_provider, None)
|
||||
if served in (requested, requested.partition("/")[2]):
|
||||
return served
|
||||
pricing_provider: Final = _get_provider_for_cost_calc(requested, custom_llm_provider)
|
||||
if _resolves_to_rates(served, pricing_provider):
|
||||
return served
|
||||
return requested if _resolves_to_rates(requested, pricing_provider) else served
|
||||
|
||||
|
||||
def _strip_unregistered_leading_segments(model: str, region_name: str | None) -> str:
|
||||
"""Resolve a provider-prefixed slash alias like "vertex_ai/vertex/claude-opus-5" to the
|
||||
registered cost key ("vertex_ai/claude-opus-5"), keeping the model unchanged when it already
|
||||
|
|
@ -1887,7 +1922,10 @@ def completion_cost(
|
|||
video_resolution=video_resolution,
|
||||
)
|
||||
elif call_type in _SPEECH_CALL_TYPES:
|
||||
prompt_characters = litellm.utils.count_characters(text=prompt)
|
||||
prompt_characters = _speech_billed_characters(
|
||||
completion_response if isinstance(completion_response, HttpxBinaryResponseContent) else None,
|
||||
prompt,
|
||||
)
|
||||
elif call_type in _TRANSCRIPTION_CALL_TYPES:
|
||||
# Check _hidden_params first (duration stored there to
|
||||
# avoid polluting the response body), then fall back to
|
||||
|
|
@ -2366,6 +2404,11 @@ def response_cost_calculator(
|
|||
raise e
|
||||
|
||||
|
||||
def _speech_billed_characters(speech_response: HttpxBinaryResponseContent | None, prompt: str) -> int:
|
||||
reported: Final = speech_response.hidden_params.get("billed_characters") if speech_response is not None else None
|
||||
return reported if isinstance(reported, int) else litellm.utils.count_characters(text=prompt)
|
||||
|
||||
|
||||
def _video_input_image_cost(
|
||||
model: str,
|
||||
custom_llm_provider: str | None,
|
||||
|
|
@ -3508,7 +3551,7 @@ def _cost_map_entry_declares_pricing(model_name: str, custom_llm_provider: str)
|
|||
lookups cannot strip that prefix, so a zero-rated override read as declaring nothing, and a
|
||||
session that should bill nothing fell through to the public rates instead.
|
||||
"""
|
||||
resolved: Final = _get_model_info_or_none(model_name, custom_llm_provider)
|
||||
resolved: Final = _cost_map_model_info(model_name, custom_llm_provider)
|
||||
entries: Final = (
|
||||
litellm.model_cost.get(resolved.get("key")) if resolved is not None else None,
|
||||
litellm.model_cost.get(model_name),
|
||||
|
|
@ -3704,9 +3747,9 @@ def handle_realtime_transcription_cost_calculation(
|
|||
return 0.0
|
||||
|
||||
model_name: Final = _get_transcription_model_name_from_results(results) or litellm_model_name
|
||||
model_info: Final = _get_model_info_or_none(model_name, custom_llm_provider)
|
||||
model_info: Final = _cost_map_model_info(model_name, custom_llm_provider)
|
||||
override_info: Final = (
|
||||
_get_model_info_or_none(custom_pricing_model, custom_llm_provider) if custom_pricing_model is not None else None
|
||||
_cost_map_model_info(custom_pricing_model, custom_llm_provider) if custom_pricing_model is not None else None
|
||||
)
|
||||
|
||||
total_cost = 0.0
|
||||
|
|
@ -3737,13 +3780,6 @@ def _get_transcription_model_name_from_results(
|
|||
return None
|
||||
|
||||
|
||||
def _get_model_info_or_none(model: str, custom_llm_provider: str) -> ModelInfo | None:
|
||||
try:
|
||||
return litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
_TranscriptionRateKey: TypeAlias = Literal[
|
||||
"input_cost_per_second", "input_cost_per_audio_token", "input_cost_per_token", "output_cost_per_token"
|
||||
]
|
||||
|
|
|
|||
|
|
@ -53,7 +53,7 @@ from litellm.main import (
|
|||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.images.main import ImageEditOptionalRequestParams
|
||||
from litellm.types.llms.openai import ImageGenerationRequestQuality
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.router import CustomPricingLiteLLMParams, GenericLiteLLMParams
|
||||
from litellm.types.utils import (
|
||||
LITELLM_IMAGE_VARIATION_PROVIDERS,
|
||||
LlmProviders,
|
||||
|
|
@ -1010,6 +1010,7 @@ def image_edit(
|
|||
optional_params=dict(image_edit_request_params),
|
||||
litellm_params={
|
||||
**image_edit_request_params,
|
||||
**litellm_params.model_dump(include=set(CustomPricingLiteLLMParams.model_fields), exclude_none=True),
|
||||
"litellm_call_id": litellm_call_id,
|
||||
"model_info": model_info,
|
||||
"vertex_location": litellm_params.vertex_location,
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ class ImageEditRequestUtils:
|
|||
image_edit_optional_params: ImageEditOptionalRequestParams,
|
||||
drop_params: bool | None = None,
|
||||
additional_drop_params: Collection[str] | None = None,
|
||||
) -> dict:
|
||||
) -> dict[str, object]:
|
||||
"""
|
||||
Get optional parameters for the image edit API.
|
||||
|
||||
|
|
|
|||
|
|
@ -102,18 +102,14 @@ class AgentOps(OpenTelemetry):
|
|||
"Connection": "keep-alive",
|
||||
}
|
||||
|
||||
client: Final = get_httpx_client()
|
||||
try:
|
||||
response: Final = client.post(
|
||||
url=auth_endpoint,
|
||||
headers=headers,
|
||||
json={"api_key": api_key},
|
||||
timeout=10,
|
||||
)
|
||||
response: Final = get_httpx_client().post(
|
||||
url=auth_endpoint,
|
||||
headers=headers,
|
||||
json={"api_key": api_key},
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
if response.status_code != 200:
|
||||
raise Exception(f"Failed to fetch auth token: {response.text}")
|
||||
if response.status_code != 200:
|
||||
raise Exception(f"Failed to fetch auth token: {response.text}")
|
||||
|
||||
return _DECODED_JSON.validate_python(response.json())
|
||||
finally:
|
||||
client.close()
|
||||
return _DECODED_JSON.validate_python(response.json())
|
||||
|
|
|
|||
|
|
@ -5,9 +5,17 @@ Arize Phoenix API client for fetching prompt versions from Arize Phoenix.
|
|||
import urllib.parse
|
||||
from typing import Final
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
|
||||
|
||||
class _PromptVersionResponse(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore", strict=True, frozen=True, hide_input_in_errors=True)
|
||||
|
||||
data: dict[str, object] | None = None
|
||||
|
||||
|
||||
def _sanitize_id(identifier: str) -> str:
|
||||
"""Reject path traversal characters and URL-encode the identifier."""
|
||||
if any(c in identifier for c in ("/", "\\", "#", "?")):
|
||||
|
|
@ -71,8 +79,7 @@ class ArizePhoenixClient:
|
|||
response: Final = self.http_handler.get(url, headers=self.headers)
|
||||
response.raise_for_status()
|
||||
|
||||
data: Final = response.json()
|
||||
return data.get("data")
|
||||
return _PromptVersionResponse.model_validate(response.json()).data
|
||||
|
||||
except Exception as e:
|
||||
# Check if it's an HTTP error
|
||||
|
|
|
|||
|
|
@ -35,6 +35,8 @@ from litellm.types.utils import StandardCallbackDynamicParams
|
|||
if TYPE_CHECKING:
|
||||
from opentelemetry.trace import Span as _Span
|
||||
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
Span = _Span
|
||||
else:
|
||||
Span = Any
|
||||
|
|
@ -126,6 +128,26 @@ def _starts_its_own_trace(span: object, kwargs: Mapping[str, object]) -> bool:
|
|||
)
|
||||
|
||||
|
||||
def _a_destination_records_the_call(kwargs: Mapping[str, object]) -> bool:
|
||||
"""Whether a key's or team's ``langfuse_otel`` destination gets this call from the OTel v2 fan-out, which records
|
||||
every call but one whose logging payload shows it never reached a provider."""
|
||||
from litellm.integrations.otel.model.metadata import LLMCallEvent
|
||||
|
||||
if not _a_destination_carries_the_trace():
|
||||
return False
|
||||
call: Final = LLMCallEvent.from_dict(kwargs)
|
||||
return call.payload is None or call.reached_a_provider
|
||||
|
||||
|
||||
def _a_destination_carries_the_trace() -> bool:
|
||||
"""Whether this request resolved a key's or team's ``langfuse_otel`` destination, whose fan-out sends it the whole
|
||||
trace."""
|
||||
from litellm.integrations.otel.model.config import ExporterOwner
|
||||
from litellm.integrations.otel.plumbing.context import destination_backends
|
||||
|
||||
return ExporterOwner.LANGFUSE_OTEL in destination_backends()
|
||||
|
||||
|
||||
class LangfuseOtelLogger(OpenTelemetry):
|
||||
def __init__(self, config=None, *args, **kwargs):
|
||||
# Prevent LangfuseOtelLogger from modifying global environment variables by constructing config manually
|
||||
|
|
@ -541,6 +563,27 @@ class LangfuseOtelLogger(OpenTelemetry):
|
|||
"""
|
||||
return None
|
||||
|
||||
def _emit_once(
|
||||
self,
|
||||
kwargs: dict[str, object], # mutable-ok: the v1 dedupe marker is written into the request's kwargs
|
||||
*scope: object,
|
||||
) -> bool:
|
||||
"""Whether to emit this span: once per request, and never for a call a Langfuse destination records."""
|
||||
return not _a_destination_records_the_call(kwargs) and super()._emit_once(kwargs, *scope) # pyright: ignore[reportUnknownMemberType] # the v1 hook takes a bare dict
|
||||
|
||||
async def async_post_call_failure_hook(
|
||||
self,
|
||||
request_data: dict[str, object], # mutable-ok: the v1 hook contract
|
||||
original_exception: Exception,
|
||||
user_api_key_dict: "UserAPIKeyAuth",
|
||||
traceback_str: str | None = None,
|
||||
) -> None:
|
||||
"""Record the proxy failure of a request no Langfuse destination carries."""
|
||||
if not _a_destination_carries_the_trace():
|
||||
await super().async_post_call_failure_hook( # pyright: ignore[reportUnknownMemberType] # the v1 hook takes a bare dict
|
||||
request_data, original_exception, user_api_key_dict, traceback_str
|
||||
)
|
||||
|
||||
async def async_service_success_hook(self, *args, **kwargs):
|
||||
"""
|
||||
Langfuse should not receive service success logs.
|
||||
|
|
|
|||
|
|
@ -72,7 +72,7 @@ if TYPE_CHECKING:
|
|||
)
|
||||
from litellm.proxy.proxy_server import UserAPIKeyAuth as _UserAPIKeyAuth
|
||||
|
||||
Span = _Span | Any
|
||||
Span = _Span
|
||||
Tracer = _Tracer
|
||||
Context = _Context
|
||||
SpanExporter = _SpanExporter
|
||||
|
|
@ -1440,7 +1440,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
span = None
|
||||
# Only set attributes if the span is still recording (not closed)
|
||||
# Note: parent_span is guaranteed to be not None here
|
||||
if hasattr(parent_span, "set_status"):
|
||||
if parent_span is not None and hasattr(parent_span, "set_status"):
|
||||
parent_span.set_status(Status(StatusCode.OK))
|
||||
self.set_attributes(parent_span, kwargs, response_obj)
|
||||
# Raw-request as direct child of parent_span
|
||||
|
|
@ -2233,7 +2233,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
# record error on parent span (keeps hierarchy shallow)
|
||||
# Only set attributes if the span is still recording (not closed)
|
||||
# Note: parent_otel_span is guaranteed to be not None here
|
||||
if parent_otel_span.is_recording():
|
||||
if parent_otel_span is not None and parent_otel_span.is_recording():
|
||||
self.set_attributes(parent_otel_span, kwargs, response_obj)
|
||||
parent_error_message: Final = self._record_exception_on_span(span=parent_otel_span, kwargs=kwargs)
|
||||
parent_otel_span.set_status(Status(StatusCode.ERROR, parent_error_message))
|
||||
|
|
|
|||
|
|
@ -13,9 +13,9 @@ from opentelemetry.trace.status import Status, StatusCode
|
|||
|
||||
from litellm.integrations.otel.mappers import resolve_mappers
|
||||
from litellm.integrations.otel.mappers.base import AttributeMapper, AttrValue, SpanData
|
||||
from litellm.integrations.otel.mappers.langfuse import LANGFUSE_REQUEST_METADATA_ATTRIBUTES
|
||||
from litellm.integrations.otel.mappers.langfuse import LANGFUSE_REQUEST_METADATA_ATTRIBUTES, LangfuseMapper
|
||||
from litellm.integrations.otel.mappers.openinference import fit_indexed_messages
|
||||
from litellm.integrations.otel.model.config import OpenTelemetryV2Config
|
||||
from litellm.integrations.otel.model.config import ExporterOwner, OpenTelemetryV2Config
|
||||
from litellm.integrations.otel.model.payloads import (
|
||||
GuardrailSpanData,
|
||||
LLMCallSpanData,
|
||||
|
|
@ -34,6 +34,7 @@ from litellm.integrations.otel.model.spans import (
|
|||
mcp_tool_call_span_name,
|
||||
service_span_name,
|
||||
)
|
||||
from litellm.integrations.otel.plumbing.context import destination_backends
|
||||
from litellm.integrations.otel.plumbing.events import GenAIEventRecorder
|
||||
from litellm.integrations.otel.plumbing.providers import to_otel_span_kind
|
||||
|
||||
|
|
@ -168,6 +169,9 @@ class SpanEmitter:
|
|||
self._mappers: list[AttributeMapper] = (
|
||||
list(mappers) if mappers is not None else resolve_mappers(config.mapper_names)
|
||||
)
|
||||
self._langfuse: Final = (
|
||||
None if any(isinstance(mapper, LangfuseMapper) for mapper in self._mappers) else LangfuseMapper()
|
||||
)
|
||||
# Bounded LRU (ordered by insertion / most-recent touch). Storing keys
|
||||
# only — the value is unused — so it behaves like a capped set.
|
||||
self._emitted: OrderedDict[tuple[str, SpanRole], None] = OrderedDict()
|
||||
|
|
@ -295,17 +299,27 @@ class SpanEmitter:
|
|||
)
|
||||
else None
|
||||
)
|
||||
mapped: Final = MappingProxyType(
|
||||
{key: value for mapper in self._mappers for key, value in mapper.map(data).items()}
|
||||
configured: Final = {key: value for mapper in self._mappers for key, value in mapper.map(data).items()}
|
||||
for_langfuse: Final = (
|
||||
dict(
|
||||
sorted(
|
||||
self._langfuse.map(data).items(),
|
||||
key=lambda item: item[0] not in LANGFUSE_REQUEST_METADATA_ATTRIBUTES,
|
||||
)
|
||||
)
|
||||
if self._langfuse is not None and ExporterOwner.LANGFUSE_OTEL in destination_backends()
|
||||
else _NO_ATTRIBUTES
|
||||
)
|
||||
mapped: Final = MappingProxyType({**configured, **for_langfuse})
|
||||
stamped_later: Final = error_attributes(error) if error else _NO_ATTRIBUTES
|
||||
reserved: Final = len(stamped_later.keys() - mapped.keys())
|
||||
overwrites: Final = carried_keys(span) & mapped.keys()
|
||||
# The Langfuse request metadata sheds first, so a span at its attribute limit loses nothing else to it.
|
||||
# The Langfuse request metadata, and the Langfuse vocabulary only a destination asked for, shed first,
|
||||
# so a span at its attribute limit loses nothing else to them.
|
||||
fitted: Final = fit_indexed_messages(
|
||||
MappingProxyType({key: value for key, value in mapped.items() if key not in overwrites}),
|
||||
attribute_budget(span, reserved),
|
||||
LANGFUSE_REQUEST_METADATA_ATTRIBUTES,
|
||||
LANGFUSE_REQUEST_METADATA_ATTRIBUTES.union(for_langfuse),
|
||||
)
|
||||
for key, value in mapped.items():
|
||||
if key in overwrites or key in fitted:
|
||||
|
|
|
|||
|
|
@ -198,6 +198,7 @@ class OpenTelemetryV2(CustomLogger):
|
|||
tracer_provider: TracerProvider | None = None,
|
||||
logger_provider: LoggerProvider | None = None,
|
||||
meter_provider: "MeterProvider | None" = None,
|
||||
console_default: bool = False,
|
||||
**kwargs: Unpack[_CustomLoggerOptions],
|
||||
) -> None:
|
||||
super().__init__(**kwargs)
|
||||
|
|
@ -206,7 +207,7 @@ class OpenTelemetryV2(CustomLogger):
|
|||
self._tracer_provider: TracerProvider = (
|
||||
tracer_provider
|
||||
if tracer_provider is not None
|
||||
else build_tracer_provider(self.config, tenant_overrides=True)
|
||||
else build_tracer_provider(self.config, tenant_overrides=True, console_default=console_default)
|
||||
)
|
||||
self.tracer: Tracer = get_tracer(self._tracer_provider, LITELLM_TRACER_NAME)
|
||||
self._metrics_recorder = self._init_metrics(meter_provider)
|
||||
|
|
@ -217,7 +218,9 @@ class OpenTelemetryV2(CustomLogger):
|
|||
mappers=resolve_mappers(self.config.mapper_names),
|
||||
event_recorder=self._init_events(logger_provider),
|
||||
)
|
||||
self._tenant_tracers = TenantTracerCache(self.config, callback_name, LITELLM_TRACER_NAME)
|
||||
self._tenant_tracers = TenantTracerCache(
|
||||
self.config, callback_name, LITELLM_TRACER_NAME, console_default=console_default
|
||||
)
|
||||
self._open_llm_calls: OrderedDict[str, _LLMCallSpan] = OrderedDict()
|
||||
self._init_otel_logger_on_litellm_proxy()
|
||||
|
||||
|
|
@ -541,9 +544,7 @@ class OpenTelemetryV2(CustomLogger):
|
|||
# gate rejection carries ``is_no_upstream_call`` and gets no span. A cache
|
||||
# hit never hands off to a provider, so ``cache_hit`` substitutes for the
|
||||
# upstream stamp, since the payload proves the call completed.
|
||||
if carrier is None and (
|
||||
call.is_no_upstream_call or call.payload is None or (not call.upstream_started and not call.cache_hit)
|
||||
):
|
||||
if carrier is None and not call.reached_a_provider:
|
||||
return None
|
||||
try:
|
||||
return self._finish_carrier(carrier, call, start_time, end_time)
|
||||
|
|
@ -1091,6 +1092,7 @@ def build_otel_v2_logger(
|
|||
logger_provider: LoggerProvider | None = None,
|
||||
meter_provider: "MeterProvider | None" = None,
|
||||
settings: Mapping[str, object] = MappingProxyType({}),
|
||||
console_default: bool = False,
|
||||
) -> OpenTelemetryV2:
|
||||
return _logger_class(config)(
|
||||
config=config,
|
||||
|
|
@ -1098,6 +1100,7 @@ def build_otel_v2_logger(
|
|||
tracer_provider=tracer_provider,
|
||||
logger_provider=logger_provider,
|
||||
meter_provider=meter_provider,
|
||||
console_default=console_default,
|
||||
**settings,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -103,6 +103,18 @@ class LangfuseMapper:
|
|||
),
|
||||
}
|
||||
|
||||
#: Every attribute this mapper can write on a model call.
|
||||
LLM_CALL_ATTRIBUTES: Final[frozenset[str]] = frozenset(
|
||||
{
|
||||
*_LLM_CALL_ATTRS,
|
||||
*_BLOB_ATTRS,
|
||||
LANGFUSE_TRACE_NAME,
|
||||
LANGFUSE_TRACE_USER_ID,
|
||||
LANGFUSE_TRACE_SESSION_ID,
|
||||
LANGFUSE_TRACE_TAGS,
|
||||
}
|
||||
)
|
||||
|
||||
def map(self, data: SpanData) -> AttributeMap:
|
||||
match data:
|
||||
case LLMCallSpanData():
|
||||
|
|
|
|||
|
|
@ -121,6 +121,17 @@ class ExporterSpec(LiteLLMBaseModel):
|
|||
)
|
||||
|
||||
|
||||
def is_unconfigured_placeholder(spec: ExporterSpec) -> bool:
|
||||
"""Whether ``spec`` is the one ``_normalize`` folds in when nothing was configured.
|
||||
|
||||
No field set is what says the operator asked for nothing: an exporter they did
|
||||
configure survives, even ``OTEL_EXPORTER=console`` whose value matches the default,
|
||||
and so does the gated spec ``credential_gated_exporters`` appends, which would
|
||||
otherwise eat itself when one preset layers onto another.
|
||||
"""
|
||||
return not spec.model_fields_set
|
||||
|
||||
|
||||
class _EnvWithoutBareExcludedServices(PydanticBaseSettingsSource):
|
||||
def __init__(self, settings_cls: type[BaseSettings], env_settings: PydanticBaseSettingsSource) -> None:
|
||||
super().__init__(settings_cls)
|
||||
|
|
|
|||
|
|
@ -272,6 +272,11 @@ class LLMCallEvent:
|
|||
trace: TraceControls
|
||||
session_id: str | None
|
||||
|
||||
@property
|
||||
def reached_a_provider(self) -> bool:
|
||||
"""Whether the call was handed to a provider or answered from the response cache."""
|
||||
return not self.is_no_upstream_call and self.payload is not None and (self.upstream_started or self.cache_hit)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, kwargs: Mapping[str, object]) -> LLMCallEvent:
|
||||
raw_payload: Final = kwargs.get("standard_logging_object")
|
||||
|
|
|
|||
|
|
@ -47,6 +47,7 @@ from litellm.integrations.otel.mappers.langfuse import (
|
|||
LANGFUSE_OBSERVATION_OUTPUT,
|
||||
LANGFUSE_REQUEST_METADATA_ATTRIBUTES,
|
||||
LANGFUSE_TRACE_NAME,
|
||||
LangfuseMapper,
|
||||
)
|
||||
from litellm.integrations.otel.model.config import (
|
||||
CaptureMessageContent,
|
||||
|
|
@ -55,6 +56,7 @@ from litellm.integrations.otel.model.config import (
|
|||
OpenTelemetryV2Config,
|
||||
capture_message_content_from_env,
|
||||
captures_span_content,
|
||||
is_unconfigured_placeholder,
|
||||
)
|
||||
from litellm.integrations.otel.model.semconv import (
|
||||
DB,
|
||||
|
|
@ -490,19 +492,25 @@ def _is_litellm_span(span: ReadableSpan) -> bool:
|
|||
return scope is not None and scope.name == LITELLM_TRACER_NAME
|
||||
|
||||
|
||||
def without_langfuse_request_metadata(span: ReadableSpan) -> ReadableSpan:
|
||||
"""``span`` without the request metadata the Langfuse mapper writes for a Langfuse destination.
|
||||
def without_destination_langfuse_attributes(span: ReadableSpan, langfuse_mapped: bool) -> ReadableSpan:
|
||||
"""``span`` without the Langfuse attributes only a Langfuse destination asked for.
|
||||
|
||||
Only LiteLLM's own spans lose it: any other span keeps every key its own code set.
|
||||
That is the request metadata the Langfuse mapper writes and, on a model call written by
|
||||
a logger whose own mappers leave the Langfuse vocabulary out (``langfuse_mapped`` off),
|
||||
every Langfuse attribute, since that logger wrote them for a key's or team's Langfuse
|
||||
destination alone. Only LiteLLM's own spans lose them: any other span keeps every key
|
||||
its own code set.
|
||||
"""
|
||||
attributes: Final = span.attributes or _NO_ATTRIBUTES
|
||||
if not (_is_litellm_span(span) and any(key in LANGFUSE_REQUEST_METADATA_ATTRIBUTES for key in attributes)):
|
||||
dropped: Final = (
|
||||
LANGFUSE_REQUEST_METADATA_ATTRIBUTES
|
||||
if langfuse_mapped or not is_llm_call_span(span)
|
||||
else LangfuseMapper.LLM_CALL_ATTRIBUTES
|
||||
)
|
||||
if not (_is_litellm_span(span) and any(key in dropped for key in attributes)):
|
||||
return span
|
||||
return _CountedSpanView(
|
||||
span,
|
||||
MappingProxyType(
|
||||
{key: value for key, value in attributes.items() if key not in LANGFUSE_REQUEST_METADATA_ATTRIBUTES}
|
||||
),
|
||||
span, MappingProxyType({key: value for key, value in attributes.items() if key not in dropped})
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -658,6 +666,7 @@ def _for_destination(
|
|||
span: ReadableSpan,
|
||||
destination: "OtelDestination",
|
||||
default_capture: str,
|
||||
langfuse_mapped: bool,
|
||||
) -> ReadableSpan:
|
||||
"""The view of ``span`` a tenant destination receives.
|
||||
|
||||
|
|
@ -672,13 +681,16 @@ def _for_destination(
|
|||
and the query string of the request URL, which can hold the same key. A
|
||||
destination whose effective setting disables span content loses the prompt and
|
||||
response content too, and a destination for any backend but Langfuse loses the
|
||||
request metadata the Langfuse mapper writes on LiteLLM's own spans. The span itself
|
||||
stays, so the tenant still gets the whole trace tree.
|
||||
Langfuse attributes only a Langfuse destination asked for (see
|
||||
:func:`without_destination_langfuse_attributes`). The span itself stays, so the
|
||||
tenant still gets the whole trace tree.
|
||||
"""
|
||||
extra: Final = destination.resource_attributes
|
||||
defaults: Final = destination.resource_defaults
|
||||
source: Final = (
|
||||
span if destination.callback_name == ExporterOwner.LANGFUSE_OTEL else without_langfuse_request_metadata(span)
|
||||
span
|
||||
if destination.callback_name == ExporterOwner.LANGFUSE_OTEL
|
||||
else without_destination_langfuse_attributes(span, langfuse_mapped)
|
||||
)
|
||||
attributes: Final = source.attributes or _NO_ATTRIBUTES
|
||||
database: Final = _is_database_span(attributes)
|
||||
|
|
@ -728,8 +740,10 @@ class TenantFanOutSpanProcessor(SpanProcessor):
|
|||
default_capture_by_backend: Mapping[str, str] = MappingProxyType({}),
|
||||
pending_drains: int = _MAX_PENDING_DRAINS,
|
||||
drain_pool: _DrainPool | None = None,
|
||||
langfuse_mapped: bool = True,
|
||||
) -> None:
|
||||
self._operator_sinks: Final = operator_sinks
|
||||
self._langfuse_mapped: Final = langfuse_mapped
|
||||
self._excluded_db_systems: Final = excluded_db_systems
|
||||
self._default_capture: Final = default_capture
|
||||
self._default_capture_by_backend: Final = default_capture_by_backend
|
||||
|
|
@ -822,7 +836,8 @@ class TenantFanOutSpanProcessor(SpanProcessor):
|
|||
try:
|
||||
processor.on_end(
|
||||
_scoped(
|
||||
_for_destination(span, destination, self._default_capture_for(destination)), destination.span_scope
|
||||
_for_destination(span, destination, self._default_capture_for(destination), self._langfuse_mapped),
|
||||
destination.span_scope,
|
||||
)
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 # one destination's failure must not cost the others their span
|
||||
|
|
@ -1198,17 +1213,19 @@ class _OverriddenBackendFilter(SpanProcessor):
|
|||
return self._inner.force_flush(timeout_millis)
|
||||
|
||||
|
||||
class _LangfuseRequestMetadataFilter(SpanProcessor):
|
||||
"""Strip the Langfuse request metadata from LiteLLM's own spans before ``inner`` exports them."""
|
||||
class _DestinationLangfuseAttributeFilter(SpanProcessor):
|
||||
"""Strip the Langfuse attributes only a Langfuse destination asked for from LiteLLM's own spans before ``inner``
|
||||
exports them."""
|
||||
|
||||
def __init__(self, inner: SpanProcessor) -> None:
|
||||
def __init__(self, inner: SpanProcessor, langfuse_mapped: bool) -> None:
|
||||
self._inner: Final = inner
|
||||
self._langfuse_mapped: Final = langfuse_mapped
|
||||
|
||||
def on_start(self, span: SDKSpan, parent_context: Context | None = None) -> None:
|
||||
self._inner.on_start(span, parent_context)
|
||||
|
||||
def on_end(self, span: ReadableSpan) -> None:
|
||||
self._inner.on_end(without_langfuse_request_metadata(span))
|
||||
self._inner.on_end(without_destination_langfuse_attributes(span, self._langfuse_mapped))
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self._inner.shutdown()
|
||||
|
|
@ -1491,13 +1508,18 @@ def _spec_processor(
|
|||
spec: ExporterSpec,
|
||||
use_simple_processor: bool | None,
|
||||
strip_message_content: bool,
|
||||
strip_langfuse_request_metadata: bool,
|
||||
strip_destination_langfuse_attributes: bool,
|
||||
langfuse_mapped: bool,
|
||||
) -> SpanProcessor:
|
||||
exporting: Final = _processor_for(
|
||||
_exporter_from_spec(spec),
|
||||
(spec.use_simple_processor if spec.use_simple_processor is not None else use_simple_processor),
|
||||
)
|
||||
scoped: Final = _LangfuseRequestMetadataFilter(exporting) if strip_langfuse_request_metadata else exporting
|
||||
scoped: Final = (
|
||||
_DestinationLangfuseAttributeFilter(exporting, langfuse_mapped)
|
||||
if strip_destination_langfuse_attributes
|
||||
else exporting
|
||||
)
|
||||
return _MessageContentFilter(scoped) if strip_message_content else scoped
|
||||
|
||||
|
||||
|
|
@ -1530,6 +1552,7 @@ def build_tracer_provider(
|
|||
use_simple_processor: bool | None = None,
|
||||
tenant_overrides: bool = False,
|
||||
strip_message_content: bool = False,
|
||||
console_default: bool = False,
|
||||
) -> TracerProvider:
|
||||
"""Build the shared :class:`TracerProvider`.
|
||||
|
||||
|
|
@ -1548,11 +1571,16 @@ def build_tracer_provider(
|
|||
|
||||
``config.langfuse_span_scope`` narrows the exporter owned by ``langfuse_otel``
|
||||
alone; a collector or any other backend in the same config keeps the full tree.
|
||||
In a config that writes the Langfuse vocabulary, every exporter but ``langfuse_otel``'s
|
||||
goes without the request metadata the Langfuse mapper writes on LiteLLM's own spans.
|
||||
Every exporter but ``langfuse_otel``'s goes without the Langfuse attributes only a
|
||||
Langfuse destination asked for (see :func:`without_destination_langfuse_attributes`),
|
||||
which any logger writes for a request with a Langfuse destination.
|
||||
``strip_message_content`` removes prompt and response content before every
|
||||
exporter, for a routed provider whose spans can carry content another
|
||||
destination asked for.
|
||||
|
||||
The console spec ``config._normalize`` folds in when nothing was configured
|
||||
prints only under ``console_default``, which the ``otel`` callback passes: a
|
||||
vendor preset or the proxy's background global logger was never asked to print.
|
||||
"""
|
||||
provider: Final = TracerProvider(resource=build_resource(config))
|
||||
provider.sampler = _RequestExclusionSampler(provider.sampler)
|
||||
|
|
@ -1564,19 +1592,20 @@ def build_tracer_provider(
|
|||
provider.add_span_processor(_processor_for(exporter, use_simple_processor))
|
||||
return provider
|
||||
|
||||
langfuse_mapped: Final = _maps_langfuse(config)
|
||||
# ``config._normalize`` guarantees at least one spec (it folds the top-level
|
||||
# ``exporter``/``endpoint``/``headers`` fields in when ``exporters`` is empty).
|
||||
writes_langfuse: Final = "langfuse" in config.mapper_names or any(
|
||||
spec.owner is ExporterOwner.LANGFUSE_OTEL for spec in config.exporters
|
||||
)
|
||||
for spec in config.exporters:
|
||||
if spec.requires_headers and not spec.headers:
|
||||
continue
|
||||
if is_unconfigured_placeholder(spec) and not console_default:
|
||||
continue
|
||||
processor = _spec_processor(
|
||||
spec,
|
||||
use_simple_processor,
|
||||
strip_message_content,
|
||||
strip_langfuse_request_metadata=writes_langfuse and spec.owner is not ExporterOwner.LANGFUSE_OTEL,
|
||||
strip_destination_langfuse_attributes=spec.owner is not ExporterOwner.LANGFUSE_OTEL,
|
||||
langfuse_mapped=langfuse_mapped,
|
||||
)
|
||||
owner = spec.owner.value if tenant_overrides and spec.owner is not None else None
|
||||
scope = _operator_scope(config, spec)
|
||||
|
|
@ -1613,8 +1642,9 @@ def attach_tenant_fan_out(
|
|||
so exactly one fan-out lands. ``configs`` name the operator's own exporters, one
|
||||
config per v2 logger since each keeps its own provider and still writes its
|
||||
account, so an additive destination pointing at any of them is delivered once
|
||||
rather than twice. ``excluded_db_systems`` only filters what the fan-out
|
||||
delivers, never the operator's own exporters.
|
||||
rather than twice. The first is the config of the logger that owns ``provider``,
|
||||
whose model calls the fan-out carries. ``excluded_db_systems`` only filters what
|
||||
the fan-out delivers, never the operator's own exporters.
|
||||
"""
|
||||
with _FAN_OUT_ATTACH_LOCK:
|
||||
if any(isinstance(processor, TenantFanOutSpanProcessor) for processor in _attached_processors(provider)):
|
||||
|
|
@ -1625,10 +1655,16 @@ def attach_tenant_fan_out(
|
|||
excluded_db_systems=excluded_db_systems,
|
||||
default_capture=_operator_capture_mode(configs),
|
||||
default_capture_by_backend=MappingProxyType(dict(_owned_capture_modes(configs))),
|
||||
langfuse_mapped=any(_maps_langfuse(config) for config in configs[:1]),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _maps_langfuse(config: OpenTelemetryV2Config) -> bool:
|
||||
"""Whether a logger built from ``config`` writes the Langfuse vocabulary on every model call."""
|
||||
return "langfuse" in config.mapper_names
|
||||
|
||||
|
||||
def _operator_capture_mode(configs: Sequence[OpenTelemetryV2Config]) -> str:
|
||||
"""The mode of the operator's generic ``otel`` callback, so ``callback_settings.otel`` counts as the global default"""
|
||||
return next(
|
||||
|
|
|
|||
|
|
@ -163,10 +163,12 @@ class TenantTracerCache:
|
|||
config: OpenTelemetryV2Config,
|
||||
callback_name: str | None,
|
||||
tracer_name: str,
|
||||
console_default: bool = False,
|
||||
) -> None:
|
||||
self._config = config
|
||||
self._callback_name = callback_name
|
||||
self._tracer_name = tracer_name
|
||||
self._console_default = console_default
|
||||
# Guards the three mutable structures below: ``pre_call`` can run on
|
||||
# thread-pool workers concurrently with the event loop, so cache
|
||||
# updates, span counts, and retirement must be atomic.
|
||||
|
|
@ -284,6 +286,7 @@ class TenantTracerCache:
|
|||
built: Final = build_tracer_provider(
|
||||
self._routed_config(credential_headers, project_headers, endpoint, service_name),
|
||||
strip_message_content=not self._config.capture_span_content,
|
||||
console_default=self._console_default,
|
||||
)
|
||||
self._providers[cache_key] = built
|
||||
return built
|
||||
|
|
|
|||
|
|
@ -111,7 +111,7 @@ _REQUIRED_HEADERS_BY_CALLBACK: Final[Mapping[str, frozenset[str]]] = MappingProx
|
|||
_NO_ATTRS: Final[Mapping[str, str]] = MappingProxyType({})
|
||||
|
||||
|
||||
def _arize_project() -> Mapping[str, str]:
|
||||
def _arize_project(_params: StandardCallbackDynamicParams) -> Mapping[str, str]:
|
||||
from litellm.integrations.arize.arize import ArizeLogger
|
||||
|
||||
project: Final = ArizeLogger.get_arize_config().project_name
|
||||
|
|
@ -124,10 +124,27 @@ def _arize_default_project() -> Mapping[str, str]:
|
|||
return MappingProxyType({"model_id": ARIZE_DEFAULT_PROJECT})
|
||||
|
||||
|
||||
#: Resource attributes the operator chose for a backend, applied to every export the
|
||||
#: team receives over the span's own resource: ``ARIZE_PROJECT_NAME`` for Arize.
|
||||
_RESOURCE_ATTRIBUTES_BY_CALLBACK: Final[Mapping[str, Callable[[], Mapping[str, str]]]] = MappingProxyType(
|
||||
{"arize": _arize_project}
|
||||
def _stands_in_for_the_legacy_langfuse_logger() -> bool:
|
||||
"""Whether a Langfuse destination replaces the older Langfuse logger's copy, which a proxy without Langfuse keys
|
||||
of its own sends for a key's or team's ``langfuse_otel``."""
|
||||
return not (os.environ.get("LANGFUSE_PUBLIC_KEY") and os.environ.get("LANGFUSE_SECRET_KEY"))
|
||||
|
||||
|
||||
def _langfuse_environment(params: StandardCallbackDynamicParams) -> Mapping[str, str]:
|
||||
"""The key's or team's ``langfuse_environment``, else ``LANGFUSE_TRACING_ENVIRONMENT``, where the older Langfuse
|
||||
logger wrote it."""
|
||||
from litellm.types.integrations.langfuse_otel import LangfuseSpanAttributes
|
||||
|
||||
if not _stands_in_for_the_legacy_langfuse_logger():
|
||||
return _NO_ATTRS
|
||||
environment: Final = params.get("langfuse_environment") or os.environ.get("LANGFUSE_TRACING_ENVIRONMENT")
|
||||
return MappingProxyType({LangfuseSpanAttributes.LANGFUSE_ENVIRONMENT.value: environment} if environment else {})
|
||||
|
||||
|
||||
#: Resource attributes a backend gets on every export the team receives, over the
|
||||
#: span's own resource: ``ARIZE_PROJECT_NAME`` for Arize, the Langfuse environment.
|
||||
_RESOURCE_ATTRIBUTES_BY_CALLBACK: Final[Mapping[str, Callable[[StandardCallbackDynamicParams], Mapping[str, str]]]] = (
|
||||
MappingProxyType({"arize": _arize_project, "langfuse_otel": _langfuse_environment})
|
||||
)
|
||||
|
||||
#: Resource attributes a backend needs on every export to accept it, filled only where
|
||||
|
|
@ -137,12 +154,14 @@ _RESOURCE_DEFAULTS_BY_CALLBACK: Final[Mapping[str, Callable[[], Mapping[str, str
|
|||
)
|
||||
|
||||
|
||||
def _resource_attributes(callback_name: str, service_name: str | None) -> Mapping[str, str]:
|
||||
def _resource_attributes(
|
||||
callback_name: str, service_name: str | None, params: StandardCallbackDynamicParams
|
||||
) -> Mapping[str, str]:
|
||||
backend: Final = _RESOURCE_ATTRIBUTES_BY_CALLBACK.get(callback_name)
|
||||
return MappingProxyType(
|
||||
{
|
||||
**({"service.name": service_name} if service_name else {}),
|
||||
**(backend() if backend is not None else {}),
|
||||
**(backend(params) if backend is not None else {}),
|
||||
}
|
||||
)
|
||||
|
||||
|
|
@ -214,7 +233,7 @@ def destination_for(
|
|||
return OtelDestination(
|
||||
endpoint=endpoint,
|
||||
headers=MappingProxyType(dict(headers)),
|
||||
resource_attributes=_resource_attributes(callback_name, service_name),
|
||||
resource_attributes=_resource_attributes(callback_name, service_name, params),
|
||||
resource_defaults=_resource_defaults(callback_name),
|
||||
callback_name=callback_name,
|
||||
protocol=protocol,
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
from collections.abc import Iterable
|
||||
from typing import Final
|
||||
|
||||
from litellm.integrations.otel.model.config import ExporterOwner, ExporterSpec
|
||||
from litellm.integrations.otel.model.config import ExporterOwner, ExporterSpec, is_unconfigured_placeholder
|
||||
|
||||
|
||||
def ensure_mappers(mapper_names: Iterable[str], *names: str) -> list[str]:
|
||||
|
|
@ -35,14 +35,3 @@ def credential_gated_exporters(
|
|||
*(spec for spec in exporters if not is_unconfigured_placeholder(spec)),
|
||||
ExporterSpec(owner=owner, requires_headers=True),
|
||||
)
|
||||
|
||||
|
||||
def is_unconfigured_placeholder(spec: "ExporterSpec") -> bool:
|
||||
"""Whether ``spec`` is the one ``_normalize`` folds in when nothing was configured.
|
||||
|
||||
No field set is what says the operator asked for nothing: an exporter they did
|
||||
configure survives, even ``OTEL_EXPORTER=console`` whose value matches the default,
|
||||
and so does the gated spec this module appends, which would otherwise eat itself
|
||||
when one preset layers onto another.
|
||||
"""
|
||||
return not spec.model_fields_set
|
||||
|
|
|
|||
|
|
@ -292,6 +292,10 @@ def get_llm_provider(
|
|||
elif endpoint == "api.deepinfra.com/v1/openai":
|
||||
custom_llm_provider = "deepinfra"
|
||||
dynamic_api_key = get_secret_str("DEEPINFRA_API_KEY")
|
||||
elif endpoint == "api.clfaigateway.dev/v1":
|
||||
custom_llm_provider = "clf_ai_gateway" # rebind-ok: endpoint inference, like every branch
|
||||
clf_env_key = get_secret_str("CLF_AI_GATEWAY_API_KEY")
|
||||
dynamic_api_key = api_key or clf_env_key
|
||||
elif endpoint == "api.mistral.ai/v1":
|
||||
custom_llm_provider = "mistral"
|
||||
dynamic_api_key = get_secret_str("MISTRAL_API_KEY")
|
||||
|
|
@ -644,6 +648,11 @@ def _get_openai_compatible_provider_info(
|
|||
api_base,
|
||||
dynamic_api_key,
|
||||
) = litellm.DeepInfraConfig().get_openai_compatible_provider_info(api_base, api_key)
|
||||
elif custom_llm_provider == "clf_ai_gateway":
|
||||
(
|
||||
api_base, # rebind-ok: resolved base/key are what this function returns, like every branch
|
||||
dynamic_api_key, # rebind-ok: see above
|
||||
) = litellm.ClfAiGatewayConfig().get_openai_compatible_provider_info(api_base, api_key)
|
||||
elif custom_llm_provider == "empower":
|
||||
api_base = api_base or get_secret("EMPOWER_API_BASE") or "https://app.empower.dev/api/v1"
|
||||
dynamic_api_key = api_key or get_secret_str("EMPOWER_API_KEY")
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
# What is this?
|
||||
## Common Utility file for Logging handler
|
||||
# Logging function -> log the exact model details + what's being sent | Non-Blocking
|
||||
import asyncio
|
||||
import copy
|
||||
import datetime
|
||||
import functools
|
||||
|
|
@ -96,6 +97,7 @@ from litellm.litellm_core_utils.internal_call_metadata import (
|
|||
MODEL_ACCESS_GROUP_METADATA_KEY,
|
||||
is_unbilled_non_inference_call,
|
||||
)
|
||||
from litellm.litellm_core_utils.llm_cost_calc.finalization import FinalizedCost, PreparedCost
|
||||
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import (
|
||||
bedrock_guardrail_trace,
|
||||
cost_breakdown_with_guardrail,
|
||||
|
|
@ -200,6 +202,7 @@ from litellm.types.utils import (
|
|||
TextCompletionResponse,
|
||||
TranscriptionResponse,
|
||||
Usage,
|
||||
batch_chat_usage,
|
||||
)
|
||||
from litellm.types.videos.main import VideoObject
|
||||
from litellm.utils import get_base_model_from_metadata, print_verbose
|
||||
|
|
@ -813,6 +816,11 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
|
||||
# Initialize cost breakdown field
|
||||
self.cost_breakdown: CostBreakdown | None = None
|
||||
self.finalized_cost: FinalizedCost | None = None
|
||||
self.prepared_cost: PreparedCost | None = None
|
||||
self.cost_allowance_discount: float | None = None
|
||||
self.cost_finalization_lock = asyncio.Lock()
|
||||
self.deferred_success_result: tuple[object, datetime.datetime, datetime.datetime] | None = None
|
||||
self.billed_token_rates: BilledTokenRates | None = None
|
||||
self._provider_reported_cost: float | None = None
|
||||
|
||||
|
|
@ -874,6 +882,11 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
"current_attempt_api_call_start_time": None,
|
||||
"litellm_params": litellm_params,
|
||||
}
|
||||
new_call.finalized_cost = None
|
||||
new_call.prepared_cost = None
|
||||
new_call.cost_allowance_discount = None
|
||||
new_call.cost_finalization_lock = asyncio.Lock()
|
||||
new_call.deferred_success_result = None
|
||||
new_call.streaming_chunks = []
|
||||
new_call.sync_streaming_chunks = []
|
||||
return new_call
|
||||
|
|
@ -2079,6 +2092,11 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
elif router_model_id is None and "model_id" in hidden_params: # use model_id if not already set
|
||||
router_model_id = hidden_params["model_id"]
|
||||
|
||||
if isinstance(priced_result, LiteLLMBatch) and (
|
||||
not batch_cost_is_final(priced_result) or self._is_batch_poll_without_logging(priced_result)
|
||||
):
|
||||
return 0.0
|
||||
|
||||
# Fallback: extract router_model_id from litellm_params when not available
|
||||
# from the result object. ResponsesAPIResponse objects (used by /v1/responses
|
||||
# streaming) don't carry _hidden_params["model_id"] like ModelResponse does.
|
||||
|
|
@ -2462,6 +2480,55 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
**kwargs,
|
||||
)
|
||||
|
||||
def prepare_response_cost(self, response: object, now: datetime.datetime) -> PreparedCost | None:
|
||||
if self.prepared_cost is not None:
|
||||
return self.prepared_cost
|
||||
recorded_cache_hit: Final[object] = self.model_call_details.get("cache_hit")
|
||||
deferred: Final = self.deferred_success_result
|
||||
prepared: Final = self.prepare_success_logging(
|
||||
result=deferred[0] if deferred is not None else response,
|
||||
start_time=deferred[1] if deferred is not None else self.start_time,
|
||||
end_time=deferred[2] if deferred is not None else now.replace(tzinfo=None),
|
||||
cache_hit=recorded_cache_hit if isinstance(recorded_cache_hit, bool) else None,
|
||||
standard_logging_object=None,
|
||||
build_logging_payload=True,
|
||||
publish_logging_payload=False,
|
||||
)
|
||||
payload: Final = prepared[3]
|
||||
raw_cost: Final[object] = self.model_call_details.get("response_cost")
|
||||
if payload is None or not isinstance(raw_cost, (int, float)):
|
||||
return None
|
||||
recorded_usage: Final[object] = self.model_call_details.get("combined_usage_object")
|
||||
response_usage: Final[object] = getattr(prepared[2], "usage", None)
|
||||
normalized_usage: Final = (
|
||||
recorded_usage
|
||||
if isinstance(recorded_usage, Usage)
|
||||
else response_usage
|
||||
if isinstance(response_usage, Usage)
|
||||
else None
|
||||
)
|
||||
usage: Final = StandardLoggingPayloadSetup.get_usage_from_response_obj(
|
||||
_extract_response_obj_and_hidden_params(prepared[2], None)[0],
|
||||
combined_usage_object=normalized_usage,
|
||||
).model_copy(deep=True)
|
||||
self.prepared_cost = PreparedCost(
|
||||
raw_cost,
|
||||
{
|
||||
**payload,
|
||||
"response": None,
|
||||
"messages": None,
|
||||
"model_map_information": copy.deepcopy(payload["model_map_information"]),
|
||||
"cost_breakdown": copy.deepcopy(payload["cost_breakdown"]),
|
||||
},
|
||||
usage,
|
||||
)
|
||||
self.record_partial_usage_for_failure(usage, raw_cost)
|
||||
return self.prepared_cost
|
||||
|
||||
def record_finalized_cost(self, finalized: FinalizedCost) -> None:
|
||||
self.finalized_cost = finalized
|
||||
self.model_call_details["response_cost"] = finalized.amount
|
||||
|
||||
def record_partial_usage_for_failure(self, usage: Usage, response_cost: float) -> None:
|
||||
"""Stash what an interrupted stream already consumed so the failure log bills it instead of zero."""
|
||||
self.model_call_details["combined_usage_object"] = usage
|
||||
|
|
@ -2812,12 +2879,16 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
logging_result, "_hidden_params", {}
|
||||
)
|
||||
|
||||
if self.model_call_details.get("cache_hit") is True:
|
||||
if self.finalized_cost is not None:
|
||||
self.model_call_details["response_cost"] = self.finalized_cost.amount
|
||||
elif self.model_call_details.get("cache_hit") is True:
|
||||
self.model_call_details["response_cost"] = 0.0
|
||||
elif "response_cost" in hidden_params:
|
||||
self.model_call_details["response_cost"] = hidden_params["response_cost"]
|
||||
self._record_zero_cost_diagnostic(logging_result, hidden_params["response_cost"])
|
||||
elif (existing_cost := self.model_call_details.get("response_cost")) is not None and existing_cost != 0:
|
||||
elif (existing_cost := self.model_call_details.get("response_cost")) is not None and (
|
||||
existing_cost != 0 or self.call_type == CallTypes.pass_through.value # pyright: ignore[reportUnknownMemberType] # Logging.call_type is untyped
|
||||
):
|
||||
# Preserve response_cost if already calculated (e.g., by pass-through
|
||||
# handlers like Gemini/Vertex which call completion_cost directly).
|
||||
# Do not preserve 0 from failure_handler on intermediate router retries.
|
||||
|
|
@ -2936,7 +3007,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
standard_logging_object: StandardLoggingPayload | None = None,
|
||||
build_logging_payload: bool = True,
|
||||
) -> tuple[datetime.datetime, datetime.datetime, object]:
|
||||
prepared: Final = self._prepare_success_logging(
|
||||
prepared: Final = self.prepare_success_logging(
|
||||
result=result,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
|
|
@ -2947,7 +3018,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
)
|
||||
return prepared[0], prepared[1], prepared[2]
|
||||
|
||||
def _prepare_success_logging(
|
||||
def prepare_success_logging(
|
||||
self,
|
||||
result: object,
|
||||
start_time: datetime.datetime | None,
|
||||
|
|
@ -2968,7 +3039,11 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
|
||||
self.model_call_details["log_event_type"] = "successful_api_call"
|
||||
self.model_call_details["end_time"] = end_time
|
||||
self.model_call_details["cache_hit"] = cache_hit
|
||||
self.model_call_details["cache_hit"] = (
|
||||
self.prepared_cost.payload["cache_hit"]
|
||||
if cache_hit is None and self.prepared_cost is not None
|
||||
else cache_hit
|
||||
)
|
||||
|
||||
if self.call_type == CallTypes.anthropic_messages.value:
|
||||
result = self._anthropic_messages_logged_response(result=result)
|
||||
|
|
@ -3007,6 +3082,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
|
||||
if (
|
||||
litellm.max_budget
|
||||
and publish_logging_payload
|
||||
and self.stream is False
|
||||
and result is not None
|
||||
and isinstance(result, dict)
|
||||
|
|
@ -3037,10 +3113,17 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
publish_logging_payload: bool,
|
||||
) -> StandardLoggingPayload | None:
|
||||
if standard_logging_object is not None:
|
||||
effective_payload: Final = (
|
||||
self.finalized_cost.logging_payload(standard_logging_object)
|
||||
if self.finalized_cost is not None
|
||||
else standard_logging_object
|
||||
)
|
||||
if self.finalized_cost is not None:
|
||||
self.model_call_details["response_cost"] = self.finalized_cost.amount
|
||||
if not publish_logging_payload:
|
||||
return standard_logging_object.copy()
|
||||
self.model_call_details["standard_logging_object"] = standard_logging_object
|
||||
return standard_logging_object
|
||||
return effective_payload.copy()
|
||||
self.model_call_details["standard_logging_object"] = effective_payload
|
||||
return effective_payload
|
||||
if result is None or self.stream is True:
|
||||
# Streaming reaches here before its cost is known, so the cost
|
||||
# is seeded to None, but only when nothing has already
|
||||
|
|
@ -3269,6 +3352,10 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
finally:
|
||||
self._restore_correlation_context()
|
||||
|
||||
def _is_batch_poll_without_logging(self, result: object) -> bool:
|
||||
litellm_metadata: Final = (self.litellm_params or {}).get("litellm_metadata") or {}
|
||||
return isinstance(result, LiteLLMBatch) and litellm_metadata.get("batch_ignore_default_logging", False) is True
|
||||
|
||||
def _success_handler_body(
|
||||
self,
|
||||
result: object = None,
|
||||
|
|
@ -3281,7 +3368,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
verbose_logger.debug("Logging Details LiteLLM-Success Call: Cache_hit=%s", cache_hit)
|
||||
if not self.should_run_logging(event_type="sync_success"): # prevent double logging
|
||||
return
|
||||
start_time, end_time, result, request_logging_payload = self._prepare_success_logging(
|
||||
start_time, end_time, result, request_logging_payload = self.prepare_success_logging(
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
result=result,
|
||||
|
|
@ -3783,11 +3870,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
|
||||
## CALCULATE COST FOR BATCH JOBS
|
||||
if self.call_type == CallTypes.aretrieve_batch.value and isinstance(result, LiteLLMBatch):
|
||||
litellm_params = self.litellm_params or {}
|
||||
litellm_metadata: Final = litellm_params.get("litellm_metadata") or {}
|
||||
if (
|
||||
litellm_metadata.get("batch_ignore_default_logging", False) is True
|
||||
): # polling job will query these frequently, don't spam db logs
|
||||
if self._is_batch_poll_without_logging(result):
|
||||
return
|
||||
|
||||
batch_cost: Final = kwargs.get("batch_cost", None)
|
||||
|
|
@ -4204,7 +4287,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
# chunks already delivered; the router stashes that recovered usage as
|
||||
# ``combined_usage_object`` and pre-computes its cost, so preserve it
|
||||
# here instead of zeroing the spend on an otherwise-failed request.
|
||||
if self.model_call_details.get("combined_usage_object") is None:
|
||||
if self.finalized_cost is not None:
|
||||
self.model_call_details["response_cost"] = self.finalized_cost.amount
|
||||
elif self.model_call_details.get("combined_usage_object") is None:
|
||||
self.model_call_details["response_cost"] = 0
|
||||
|
||||
if hasattr(exception, "headers") and isinstance(exception.headers, dict):
|
||||
|
|
@ -5544,7 +5629,7 @@ def build_custom_logger_compatible_class(
|
|||
return callback
|
||||
otel_settings: Final = _get_custom_logger_settings_from_proxy_server(callback_name=logging_integration)
|
||||
otel_logger_v2: Final = build_otel_v2_logger(
|
||||
config=OpenTelemetryV2Config(**otel_settings), settings=otel_settings
|
||||
config=OpenTelemetryV2Config(**otel_settings), settings=otel_settings, console_default=True
|
||||
)
|
||||
_in_memory_loggers.append(otel_logger_v2)
|
||||
_maybe_auto_initialize_arize_phoenix(_in_memory_loggers)
|
||||
|
|
@ -7367,6 +7452,12 @@ def _cache_hit_cost_breakdown(served: CostBreakdown | None) -> CostBreakdown:
|
|||
)
|
||||
|
||||
|
||||
def _finalized_cost_of(logging_obj: object) -> FinalizedCost | None:
|
||||
"""Duck-typed logging objects (and test doubles) carry no finalized cost."""
|
||||
finalized: Final = getattr(logging_obj, "finalized_cost", None)
|
||||
return finalized if isinstance(finalized, FinalizedCost) else None
|
||||
|
||||
|
||||
def get_standard_logging_object_payload(
|
||||
kwargs: dict | None,
|
||||
init_response_obj: object,
|
||||
|
|
@ -7396,7 +7487,8 @@ def get_standard_logging_object_payload(
|
|||
# Extract usage as a plain dict, avoiding Pydantic round-trip
|
||||
raw_usage_dict: Final = StandardLoggingPayloadSetup.get_usage_as_dict(
|
||||
response_obj=None if is_unbilled_non_inference_call(call_type, metadata, response_obj) else response_obj,
|
||||
combined_usage_object=cast(Usage | None, kwargs.get("combined_usage_object")),
|
||||
combined_usage_object=cast(Usage | None, kwargs.get("combined_usage_object"))
|
||||
or (batch_chat_usage(init_response_obj) if isinstance(init_response_obj, LiteLLMBatch) else None),
|
||||
)
|
||||
usage_dict: Final = (
|
||||
{**raw_usage_dict, "output_image_count": len(init_response_obj.data)}
|
||||
|
|
@ -7470,7 +7562,10 @@ def get_standard_logging_object_payload(
|
|||
raw_response_cost: Final = kwargs.get("response_cost")
|
||||
llm_response_cost: Final[float] = raw_response_cost or 0.0
|
||||
guardrail_cost: Final = guardrail_information_cost(metadata.get("standard_logging_guardrail_information"))
|
||||
response_cost: Final[float] = llm_response_cost + guardrail_cost
|
||||
finalized_cost: Final = _finalized_cost_of(logging_obj)
|
||||
response_cost: Final[float] = (
|
||||
finalized_cost.amount if finalized_cost is not None else llm_response_cost + guardrail_cost
|
||||
)
|
||||
|
||||
# clean up litellm hidden params
|
||||
clean_hidden_params: Final = StandardLoggingPayloadSetup.get_hidden_params(hidden_params)
|
||||
|
|
@ -7537,13 +7632,17 @@ def get_standard_logging_object_payload(
|
|||
):
|
||||
model_name = response_model_name
|
||||
|
||||
request_cost_breakdown: Final = cost_breakdown_with_guardrail(
|
||||
(
|
||||
_cache_hit_cost_breakdown(uncached_price.cost_breakdown)
|
||||
if uncached_price is not None
|
||||
else logging_obj.cost_breakdown
|
||||
),
|
||||
guardrail_cost,
|
||||
request_cost_breakdown: Final = (
|
||||
finalized_cost.logging_breakdown()
|
||||
if finalized_cost is not None
|
||||
else cost_breakdown_with_guardrail(
|
||||
(
|
||||
_cache_hit_cost_breakdown(uncached_price.cost_breakdown)
|
||||
if uncached_price is not None
|
||||
else logging_obj.cost_breakdown
|
||||
),
|
||||
guardrail_cost,
|
||||
)
|
||||
)
|
||||
captured_baseline: Final = logging_obj.baseline_observation
|
||||
autorouter_savings: Final = (
|
||||
|
|
@ -7600,6 +7699,7 @@ def get_standard_logging_object_payload(
|
|||
model=model_name,
|
||||
metadata=clean_metadata,
|
||||
cache_key=clean_hidden_params["cache_key"],
|
||||
cost_finalized=finalized_cost is not None,
|
||||
response_cost=response_cost,
|
||||
cost_breakdown=request_cost_breakdown,
|
||||
autorouter_savings=autorouter_savings,
|
||||
|
|
|
|||
61
litellm/litellm_core_utils/llm_cost_calc/finalization.py
Normal file
61
litellm/litellm_core_utils/llm_cost_calc/finalization.py
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import Final, TypeAlias
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.types.utils import CostBreakdown, StandardLoggingPayload, Usage
|
||||
|
||||
BreakdownValue: TypeAlias = float | str | None | Mapping[str, float]
|
||||
_BREAKDOWN_FIELDS: Final = TypeAdapter(Mapping[str, BreakdownValue])
|
||||
_COST_BREAKDOWN: Final = TypeAdapter(CostBreakdown)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PreparedCost:
|
||||
amount: float
|
||||
payload: StandardLoggingPayload
|
||||
usage: Usage
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class FinalizedCost:
|
||||
amount: float
|
||||
breakdown: Mapping[str, BreakdownValue] | None
|
||||
allowance_discount: float
|
||||
|
||||
def logging_payload(self, payload: StandardLoggingPayload) -> StandardLoggingPayload:
|
||||
return {
|
||||
**payload,
|
||||
"response_cost": self.amount,
|
||||
"cost_breakdown": self.logging_breakdown(),
|
||||
"cost_finalized": True,
|
||||
}
|
||||
|
||||
def logging_breakdown(self) -> CostBreakdown | None:
|
||||
return _COST_BREAKDOWN.validate_python(self.breakdown) if self.breakdown is not None else None
|
||||
|
||||
|
||||
def finalize_cost(amount: float, breakdown: CostBreakdown | None, allowance_discount: float = 0.0) -> FinalizedCost:
|
||||
discount: Final = min(max(amount, 0.0), max(allowance_discount, 0.0))
|
||||
total: Final = amount - discount
|
||||
existing: Final = _BREAKDOWN_FIELDS.validate_python(breakdown) if breakdown is not None else None
|
||||
adjusted: Final = (
|
||||
{
|
||||
**(existing or {}),
|
||||
"total_cost": total,
|
||||
"original_cost": (breakdown or {}).get("original_cost", amount),
|
||||
"discount_amount": (breakdown or {}).get("discount_amount", 0.0) + discount,
|
||||
}
|
||||
if discount > 0
|
||||
else existing
|
||||
)
|
||||
frozen: Final = (
|
||||
MappingProxyType(
|
||||
{key: MappingProxyType(value) if isinstance(value, Mapping) else value for key, value in adjusted.items()}
|
||||
)
|
||||
if adjusted is not None
|
||||
else None
|
||||
)
|
||||
return FinalizedCost(amount=total, breakdown=frozen, allowance_discount=discount)
|
||||
|
|
@ -75,6 +75,14 @@ _IMAGE_TOKEN_RATE_KEYS: Final[tuple[str, ...]] = (
|
|||
"output_cost_per_image_token",
|
||||
)
|
||||
|
||||
|
||||
def _deployment_prices_images_without_an_output_price(pricing: ModelInfo | None) -> bool:
|
||||
return pricing is not None and (
|
||||
"input_cost_per_pixel" in pricing
|
||||
or ("input_cost_per_image" in pricing and "output_cost_per_image" not in pricing)
|
||||
)
|
||||
|
||||
|
||||
# Pre-resolved service-tier cost-key suffixes (e.g. "_priority"). Used per
|
||||
# request in the cost-calc path, so the f-strings are built once here instead
|
||||
# of being rebuilt for every model_info key on every call. Longest-first so a
|
||||
|
|
@ -127,7 +135,7 @@ def _get_token_detail_value(details: object, key: str) -> int | None:
|
|||
return value if isinstance(value, int) else None
|
||||
|
||||
|
||||
_IMAGE_SIZE_PATTERN: Final = re.compile(r"\d+(?:x|-x-)\d+")
|
||||
_IMAGE_SIZE_PATTERN: Final = re.compile(r"\d+(?:x|-x-|\*)\d+")
|
||||
|
||||
|
||||
def _requested_image_param(optional_params: Mapping[str, object] | None, key: str) -> str | None:
|
||||
|
|
@ -137,7 +145,7 @@ def _requested_image_param(optional_params: Mapping[str, object] | None, key: st
|
|||
|
||||
def _requested_image_size(optional_params: Mapping[str, object] | None) -> str | None:
|
||||
value: Final = _requested_image_param(optional_params, "size")
|
||||
return value if value is not None and _IMAGE_SIZE_PATTERN.fullmatch(value) else None
|
||||
return value.replace("*", "x") if value is not None and _IMAGE_SIZE_PATTERN.fullmatch(value) else None
|
||||
|
||||
|
||||
def get_web_search_requests(server_tool_use: object) -> int | None:
|
||||
|
|
@ -2090,7 +2098,7 @@ class _ModelPricedImageCostCalculator(Protocol):
|
|||
|
||||
|
||||
def _model_priced_image_cost_calculator(
|
||||
custom_llm_provider: str | None, model: str
|
||||
custom_llm_provider: str | None, model: str, pricing: ModelInfo | None
|
||||
) -> _ModelPricedImageCostCalculator | None:
|
||||
match custom_llm_provider:
|
||||
case litellm.LlmProviders.RECRAFT.value:
|
||||
|
|
@ -2116,6 +2124,14 @@ def _model_priced_image_cost_calculator(
|
|||
from litellm.llms.tencent.image_generation.transformation import is_tokenhub_image_model
|
||||
|
||||
return tencent_image_cost_calculator if is_tokenhub_image_model(model) else None
|
||||
case litellm.LlmProviders.DASHSCOPE.value:
|
||||
if _deployment_prices_images_without_an_output_price(pricing):
|
||||
return None
|
||||
from litellm.llms.dashscope.image_generation.cost_calculator import (
|
||||
cost_calculator as dashscope_image_cost_calculator,
|
||||
)
|
||||
|
||||
return dashscope_image_cost_calculator
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
|
@ -2178,7 +2194,7 @@ class CostCalculatorUtils:
|
|||
)
|
||||
resolved_n: Final = n if n is not None else (len(completion_response.data) if completion_response.data else 0)
|
||||
pricing: Final = deployment_pricing(model_info)
|
||||
model_priced_calculator: Final = _model_priced_image_cost_calculator(custom_llm_provider, model)
|
||||
model_priced_calculator: Final = _model_priced_image_cost_calculator(custom_llm_provider, model, pricing)
|
||||
|
||||
if model_priced_calculator is not None:
|
||||
return model_priced_calculator(model=model, image_response=completion_response, model_info=pricing)
|
||||
|
|
|
|||
45
litellm/litellm_core_utils/reasoning_effort_strength.py
Normal file
45
litellm/litellm_core_utils/reasoning_effort_strength.py
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
"""How strong each reasoning_effort level is, and rounding a requested level onto a declared set.
|
||||
|
||||
Imports nothing from the SDK, so the `lite` CLI, which loads without initializing litellm, can use it.
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal, TypeAlias
|
||||
|
||||
from typing_extensions import assert_never
|
||||
|
||||
ReasoningEffortRounding: TypeAlias = Literal["up", "down"]
|
||||
|
||||
REASONING_EFFORT_STRENGTH_ORDER: Final = ("minimal", "low", "medium", "high", "xhigh", "max")
|
||||
REASONING_EFFORT_STRENGTH_RANK: Final = MappingProxyType(
|
||||
{effort: rank for rank, effort in enumerate(REASONING_EFFORT_STRENGTH_ORDER)}
|
||||
)
|
||||
|
||||
|
||||
def nearest_declared_reasoning_effort(
|
||||
requested: str, declared: Sequence[str], rounding: ReasoningEffortRounding = "up"
|
||||
) -> str:
|
||||
"""Rounding up takes the weakest declared level at least as strong as the request, else the
|
||||
strongest declared, so the caller gets no less reasoning than it asked for instead of a rejected
|
||||
call. Rounding down mirrors it: the strongest declared level at most as strong, else the
|
||||
weakest. none is the off switch rather than a strength, so it is never rounded onto the ladder
|
||||
and no level is rounded to it: a caller who turned reasoning off must not be billed for it, and
|
||||
a model that cannot turn it off says so itself. A level outside the strength order is likewise
|
||||
returned as is for upstream to judge."""
|
||||
ranked: Final = sorted(
|
||||
(effort for effort in declared if effort in REASONING_EFFORT_STRENGTH_RANK),
|
||||
key=lambda effort: REASONING_EFFORT_STRENGTH_RANK[effort],
|
||||
)
|
||||
if requested in ranked or requested not in REASONING_EFFORT_STRENGTH_RANK or not ranked:
|
||||
return requested
|
||||
strength: Final = REASONING_EFFORT_STRENGTH_RANK[requested]
|
||||
match rounding:
|
||||
case "up":
|
||||
return next((effort for effort in ranked if REASONING_EFFORT_STRENGTH_RANK[effort] >= strength), ranked[-1])
|
||||
case "down":
|
||||
return next(
|
||||
(effort for effort in reversed(ranked) if REASONING_EFFORT_STRENGTH_RANK[effort] <= strength), ranked[0]
|
||||
)
|
||||
case _:
|
||||
assert_never(rounding)
|
||||
27
litellm/litellm_core_utils/stream_usage_chunks.py
Normal file
27
litellm/litellm_core_utils/stream_usage_chunks.py
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
from collections.abc import Iterable
|
||||
from typing import Final
|
||||
|
||||
from litellm.types.utils import ModelResponseStream, StreamingChoices
|
||||
|
||||
|
||||
def is_empty_streaming_choice(choice: StreamingChoices) -> bool:
|
||||
if choice.finish_reason is not None:
|
||||
return False
|
||||
if getattr(choice, "logprobs", None) is not None:
|
||||
return False
|
||||
delta: Final = choice.delta if hasattr(choice, "delta") else None
|
||||
if delta is None:
|
||||
return True
|
||||
return _all_values_none(delta.model_dump().values())
|
||||
|
||||
|
||||
def _all_values_none(values: Iterable[object]) -> bool:
|
||||
return all(value is None for value in values)
|
||||
|
||||
|
||||
def is_injected_stream_usage_artifact(chunk: object) -> bool:
|
||||
if not isinstance(chunk, ModelResponseStream):
|
||||
return False
|
||||
if chunk.provider_specific_fields is not None:
|
||||
return False
|
||||
return all(is_empty_streaming_choice(choice) for choice in chunk.choices or [])
|
||||
|
|
@ -3,7 +3,7 @@ import builtins
|
|||
import time
|
||||
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass, replace
|
||||
from itertools import chain, groupby
|
||||
from itertools import chain, groupby, takewhile
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Optional, Protocol, TypeAlias, TypedDict, TypeVar, Union, cast
|
||||
|
||||
|
|
@ -257,7 +257,7 @@ def apply_grounding_request_counts(
|
|||
|
||||
|
||||
class ChunkProcessor:
|
||||
def __init__(self, chunks: list, messages: Sequence | None = None):
|
||||
def __init__(self, chunks: list, messages: Sequence[object] | None = None):
|
||||
self.chunks = self._sort_chunks(chunks)
|
||||
self.messages = messages
|
||||
self.first_chunk = chunks[0]
|
||||
|
|
@ -390,6 +390,30 @@ class ChunkProcessor:
|
|||
case _:
|
||||
return "assistant"
|
||||
|
||||
@staticmethod
|
||||
def _finish_reason_of_chunks(chunks: Sequence["_BaseChunk"], unreported: str = "stop") -> object:
|
||||
finish_reason = unreported
|
||||
for chunk in chunks:
|
||||
if "choices" in chunk and len(chunk["choices"]) > 0:
|
||||
chunk_finish_reason = None
|
||||
if hasattr(chunk["choices"][0], "finish_reason"):
|
||||
chunk_finish_reason = chunk["choices"][0].finish_reason
|
||||
elif "finish_reason" in chunk["choices"][0]:
|
||||
chunk_finish_reason = _CHOICE_FIELD_VALUE.validate_python(chunk["choices"][0]["finish_reason"])
|
||||
if chunk_finish_reason is not None:
|
||||
finish_reason = chunk_finish_reason
|
||||
return finish_reason
|
||||
|
||||
@staticmethod
|
||||
def build_base_choice(index: int, chunks: Sequence["_BaseChunk"]) -> Choices:
|
||||
return Choices(
|
||||
index=index,
|
||||
finish_reason=ChunkProcessor._finish_reason_of_chunks(
|
||||
chunks, unreported="tool_calls" if _carries_tool_calls(chunks) else "stop"
|
||||
),
|
||||
message={"role": ChunkProcessor._get_role_from_chunks(chunks), "content": ""},
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _get_model_from_chunks(chunks: Sequence["_BaseChunk"], first_chunk_model: str) -> str:
|
||||
"""
|
||||
|
|
@ -414,7 +438,11 @@ class ChunkProcessor:
|
|||
None,
|
||||
)
|
||||
|
||||
def build_base_response(self, chunks: Sequence["_BaseChunk"]) -> ModelResponse:
|
||||
def build_base_response(
|
||||
self,
|
||||
chunks: Sequence["_BaseChunk"],
|
||||
choice_streams: Sequence[tuple[int, Sequence["_BaseChunk"]]],
|
||||
) -> ModelResponse:
|
||||
chunk = self.first_chunk
|
||||
id: Final = ChunkProcessor._get_chunk_id(chunks)
|
||||
object: Final = chunk["object"]
|
||||
|
|
@ -426,16 +454,7 @@ class ChunkProcessor:
|
|||
service_tier: Final = ChunkProcessor._get_service_tier_from_chunks(chunks)
|
||||
|
||||
role: Final = ChunkProcessor._get_role_from_chunks(chunks)
|
||||
finish_reason = "stop"
|
||||
for chunk in chunks:
|
||||
if "choices" in chunk and len(chunk["choices"]) > 0:
|
||||
chunk_finish_reason = None
|
||||
if hasattr(chunk["choices"][0], "finish_reason"):
|
||||
chunk_finish_reason = chunk["choices"][0].finish_reason
|
||||
elif "finish_reason" in chunk["choices"][0]:
|
||||
chunk_finish_reason = _CHOICE_FIELD_VALUE.validate_python(chunk["choices"][0]["finish_reason"])
|
||||
if chunk_finish_reason is not None:
|
||||
finish_reason = chunk_finish_reason
|
||||
finish_reason: Final = ChunkProcessor._finish_reason_of_chunks(chunks)
|
||||
|
||||
# Initialize the response dictionary
|
||||
response = ModelResponse(
|
||||
|
|
@ -455,13 +474,17 @@ class ChunkProcessor:
|
|||
if service_tier is not None
|
||||
else MappingProxyType({})
|
||||
),
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": role, "content": ""},
|
||||
"finish_reason": finish_reason,
|
||||
}
|
||||
],
|
||||
"choices": (
|
||||
[ChunkProcessor.build_base_choice(index, choice_chunks) for index, choice_chunks in choice_streams]
|
||||
if len(choice_streams) > 1
|
||||
else [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": role, "content": ""},
|
||||
"finish_reason": finish_reason,
|
||||
}
|
||||
]
|
||||
),
|
||||
"usage": {
|
||||
"prompt_tokens": 0, # Modify as needed
|
||||
"completion_tokens": 0, # Modify as needed
|
||||
|
|
@ -1520,11 +1543,188 @@ def _bedrock_trace_fields(chunk: object) -> Mapping[str, object]:
|
|||
return {field: fields[field] for field in BEDROCK_TRACE_FIELDS if field in fields}
|
||||
|
||||
|
||||
def _stream_choice_index(choice: object) -> int:
|
||||
index: Final = _field_of(choice, "index")
|
||||
return index if isinstance(index, int) else 0
|
||||
|
||||
|
||||
def _carries_tool_calls(chunks: Sequence[_BaseChunk]) -> bool:
|
||||
return any(
|
||||
"tool_calls" in choice["delta"] and choice["delta"]["tool_calls"]
|
||||
for choice in chain.from_iterable(chunk.get("choices") or () for chunk in chunks)
|
||||
)
|
||||
|
||||
|
||||
def _choice_streams(chunks: Sequence[_BaseChunk]) -> tuple[tuple[int, Sequence[_BaseChunk]], ...]:
|
||||
indices: Final = sorted(
|
||||
frozenset(map(_stream_choice_index, chain.from_iterable(chunk.get("choices") or () for chunk in chunks)))
|
||||
)
|
||||
if len(indices) <= 1:
|
||||
return ((0, chunks),)
|
||||
return tuple((index, _narrowed_to_choice(chunks, index)) for index in indices)
|
||||
|
||||
|
||||
def _narrowed_to_choice(chunks: Sequence[_BaseChunk], index: int) -> tuple[_BaseChunk, ...]:
|
||||
return tuple(
|
||||
_BaseChunk(choices=[choice for choice in chunk.get("choices") or () if _stream_choice_index(choice) == index])
|
||||
for chunk in chunks
|
||||
)
|
||||
|
||||
|
||||
def _simple_text(chunks: Sequence[_BaseChunk]) -> str | None:
|
||||
choices_per_chunk: Final = tuple(chunk["choices"] for chunk in chunks if chunk.get("choices"))
|
||||
parts: Final = tuple(takewhile(lambda part: part is not None, map(_simple_text_part, choices_per_chunk)))
|
||||
return "".join(filter(None, parts)) if len(parts) == len(choices_per_chunk) else None
|
||||
|
||||
|
||||
def _assembled_message_fields(processor: ChunkProcessor, chunks: Sequence) -> Iterator[tuple[str, object]]:
|
||||
tool_call_chunks: Final = [
|
||||
chunk
|
||||
for chunk in chunks
|
||||
if any(
|
||||
"tool_calls" in choice["delta"] and choice["delta"]["tool_calls"] is not None
|
||||
for choice in chunk.get("choices") or ()
|
||||
)
|
||||
]
|
||||
|
||||
if tool_call_chunks:
|
||||
tool_calls_list: Final = processor.get_combined_tool_content(tool_call_chunks)
|
||||
yield "content", None
|
||||
yield "tool_calls", tool_calls_list
|
||||
|
||||
function_call_chunks: Final = [
|
||||
chunk
|
||||
for chunk in chunks
|
||||
if chunk.get("choices")
|
||||
and "function_call" in chunk["choices"][0]["delta"]
|
||||
and chunk["choices"][0]["delta"]["function_call"] is not None
|
||||
]
|
||||
|
||||
if function_call_chunks:
|
||||
yield "content", None
|
||||
yield "function_call", processor.get_combined_function_call_content(function_call_chunks)
|
||||
|
||||
content_chunks: Final = [
|
||||
chunk
|
||||
for chunk in chunks
|
||||
if chunk.get("choices")
|
||||
and "content" in chunk["choices"][0]["delta"]
|
||||
and chunk["choices"][0]["delta"]["content"] is not None
|
||||
]
|
||||
|
||||
if content_chunks:
|
||||
yield "content", processor.get_combined_content(content_chunks)
|
||||
|
||||
thinking_blocks: Final = [
|
||||
chunk
|
||||
for chunk in chunks
|
||||
if chunk.get("choices")
|
||||
and "thinking_blocks" in chunk["choices"][0]["delta"]
|
||||
and chunk["choices"][0]["delta"]["thinking_blocks"] is not None
|
||||
]
|
||||
|
||||
if thinking_blocks:
|
||||
yield "thinking_blocks", processor.get_combined_thinking_content(thinking_blocks)
|
||||
|
||||
reasoning_chunks: Final = [
|
||||
chunk
|
||||
for chunk in chunks
|
||||
if chunk.get("choices")
|
||||
and "reasoning_content" in chunk["choices"][0]["delta"]
|
||||
and chunk["choices"][0]["delta"]["reasoning_content"] is not None
|
||||
]
|
||||
|
||||
if reasoning_chunks:
|
||||
yield "reasoning_content", processor.get_combined_reasoning_content(reasoning_chunks)
|
||||
|
||||
typed_chunks: Final = cast(Sequence[object], chunks) # cast-ok: chunks are untyped upstream
|
||||
reasoning_items: Final = tuple(chain.from_iterable(_chunk_reasoning_items(chunk) for chunk in typed_chunks))
|
||||
if reasoning_items:
|
||||
yield "reasoning_items", list(reasoning_items)
|
||||
|
||||
annotation_chunks: Final = [
|
||||
chunk
|
||||
for chunk in chunks
|
||||
if chunk.get("choices")
|
||||
and "annotations" in chunk["choices"][0]["delta"]
|
||||
and chunk["choices"][0]["delta"]["annotations"] is not None
|
||||
]
|
||||
|
||||
message_annotations, provider_annotations = (
|
||||
partition_annotations(
|
||||
list(chain.from_iterable(chunk["choices"][0]["delta"]["annotations"] for chunk in annotation_chunks))
|
||||
)
|
||||
if annotation_chunks
|
||||
else (None, ())
|
||||
)
|
||||
if message_annotations is not None:
|
||||
yield "annotations", message_annotations
|
||||
|
||||
audio_chunks: Final = [
|
||||
chunk
|
||||
for chunk in chunks
|
||||
if chunk.get("choices")
|
||||
and "audio" in chunk["choices"][0]["delta"]
|
||||
and chunk["choices"][0]["delta"]["audio"] is not None
|
||||
]
|
||||
|
||||
if audio_chunks:
|
||||
yield "audio", processor.get_combined_audio_content(audio_chunks)
|
||||
|
||||
# Handle image chunks from models like gemini-2.5-flash-image
|
||||
# See: https://github.com/BerriAI/litellm/issues/19478
|
||||
image_chunks: Final = [
|
||||
chunk
|
||||
for chunk in chunks
|
||||
if chunk.get("choices")
|
||||
and "images" in chunk["choices"][0]["delta"]
|
||||
and chunk["choices"][0]["delta"]["images"] is not None
|
||||
]
|
||||
|
||||
if image_chunks:
|
||||
# Images come complete in a single chunk, collect all images from all chunks
|
||||
all_images: Final = []
|
||||
for chunk in image_chunks:
|
||||
all_images.extend(chunk["choices"][0]["delta"]["images"])
|
||||
yield "images", all_images
|
||||
|
||||
# Combine provider_specific_fields from streaming chunks (e.g., web_search_results, citations)
|
||||
# See: https://github.com/BerriAI/litellm/issues/17737
|
||||
provider_specific_chunks: Final = [
|
||||
chunk
|
||||
for chunk in chunks
|
||||
if chunk.get("choices")
|
||||
and "provider_specific_fields" in chunk["choices"][0]["delta"]
|
||||
and chunk["choices"][0]["delta"]["provider_specific_fields"] is not None
|
||||
]
|
||||
|
||||
provider_field_dicts: Final = tuple(
|
||||
fields
|
||||
for chunk in provider_specific_chunks
|
||||
for fields in (chunk["choices"][0]["delta"]["provider_specific_fields"],)
|
||||
if isinstance(fields, dict)
|
||||
)
|
||||
streamed_citations: Final = tuple(
|
||||
fields["citation"] for fields in provider_field_dicts if fields.get("citation") is not None
|
||||
)
|
||||
citation_fields: Final = {"citations": _joined_streamed_citations(streamed_citations)} if streamed_citations else {}
|
||||
provider_annotation_fields: Final = {"annotations": list(provider_annotations)} if provider_annotations else {}
|
||||
combined_provider_fields: Final = {
|
||||
key: value
|
||||
for fields in (provider_annotation_fields, citation_fields, *provider_field_dicts)
|
||||
for key, value in fields.items()
|
||||
if key != "citation"
|
||||
}
|
||||
|
||||
if combined_provider_fields:
|
||||
yield "provider_specific_fields", combined_provider_fields
|
||||
|
||||
|
||||
def stream_chunk_builder(
|
||||
chunks: list,
|
||||
messages: Sequence | None = None,
|
||||
start_time=None,
|
||||
end_time=None,
|
||||
start_time: object = None,
|
||||
end_time: object = None,
|
||||
logging_obj: Optional["Logging"] = None,
|
||||
count_prompt_tokens: Callable[[], int] | None = None,
|
||||
) -> ModelResponse | TextCompletionResponse | None:
|
||||
|
|
@ -1561,26 +1761,15 @@ def stream_chunk_builder(
|
|||
usage_model: Final = usage_tokenizer_model(
|
||||
model, get_base_model_from_metadata(logging_obj.model_call_details) if logging_obj is not None else None
|
||||
)
|
||||
# Initialize the response dictionary
|
||||
response: Final = processor.build_base_response(chunks)
|
||||
choice_streams: Final = _choice_streams(chunks)
|
||||
response: Final = processor.build_base_response(chunks, choice_streams)
|
||||
rebuilt_choices: Final = tuple(zip(response.choices, (choice_chunks for _, choice_chunks in choice_streams)))
|
||||
|
||||
# Fast path for the common text-only streaming case:
|
||||
# avoid repeated multi-pass list scans over chunks.
|
||||
simple_content_parts: Final[list[str]] = []
|
||||
is_simple_text_stream = True
|
||||
for chunk in chunks:
|
||||
if not chunk.get("choices"):
|
||||
continue
|
||||
|
||||
if (part := _simple_text_part(chunk["choices"])) is None:
|
||||
is_simple_text_stream = False
|
||||
break
|
||||
if part:
|
||||
simple_content_parts.append(part)
|
||||
|
||||
if is_simple_text_stream:
|
||||
if simple_content_parts:
|
||||
response["choices"][0]["message"]["content"] = "".join(simple_content_parts)
|
||||
simple_texts: Final = tuple(_simple_text(choice_chunks) for _, choice_chunks in choice_streams)
|
||||
if None not in simple_texts:
|
||||
for (choice, _), simple_text in zip(rebuilt_choices, simple_texts):
|
||||
if simple_text:
|
||||
choice.message.content = simple_text
|
||||
completion_output = get_content_from_model_response(response)
|
||||
usage = processor.calculate_usage(
|
||||
chunks=chunks,
|
||||
|
|
@ -1619,165 +1808,9 @@ def stream_chunk_builder(
|
|||
processor.apply_provider_assembled_streaming_metadata(response, chunks, logging_obj)
|
||||
return response
|
||||
|
||||
tool_call_chunks: Final = [
|
||||
chunk
|
||||
for chunk in chunks
|
||||
if any(
|
||||
"tool_calls" in choice["delta"] and choice["delta"]["tool_calls"] is not None
|
||||
for choice in chunk.get("choices") or ()
|
||||
)
|
||||
]
|
||||
|
||||
if len(tool_call_chunks) > 0:
|
||||
tool_calls_list: Final = processor.get_combined_tool_content(tool_call_chunks)
|
||||
_choice = response.choices[0]
|
||||
_choice.message.content = None
|
||||
_choice.message.tool_calls = tool_calls_list
|
||||
|
||||
function_call_chunks: Final = [
|
||||
chunk
|
||||
for chunk in chunks
|
||||
if chunk.get("choices")
|
||||
and "function_call" in chunk["choices"][0]["delta"]
|
||||
and chunk["choices"][0]["delta"]["function_call"] is not None
|
||||
]
|
||||
|
||||
if len(function_call_chunks) > 0:
|
||||
_choice = response.choices[0]
|
||||
_choice.message.content = None
|
||||
_choice.message.function_call = processor.get_combined_function_call_content(function_call_chunks)
|
||||
|
||||
content_chunks: Final = [
|
||||
chunk
|
||||
for chunk in chunks
|
||||
if chunk.get("choices")
|
||||
and "content" in chunk["choices"][0]["delta"]
|
||||
and chunk["choices"][0]["delta"]["content"] is not None
|
||||
]
|
||||
|
||||
if len(content_chunks) > 0:
|
||||
response["choices"][0]["message"]["content"] = processor.get_combined_content(content_chunks)
|
||||
|
||||
thinking_blocks: Final = [
|
||||
chunk
|
||||
for chunk in chunks
|
||||
if chunk.get("choices")
|
||||
and "thinking_blocks" in chunk["choices"][0]["delta"]
|
||||
and chunk["choices"][0]["delta"]["thinking_blocks"] is not None
|
||||
]
|
||||
|
||||
if len(thinking_blocks) > 0:
|
||||
response["choices"][0]["message"]["thinking_blocks"] = processor.get_combined_thinking_content(
|
||||
thinking_blocks
|
||||
)
|
||||
|
||||
reasoning_chunks: Final = [
|
||||
chunk
|
||||
for chunk in chunks
|
||||
if chunk.get("choices")
|
||||
and "reasoning_content" in chunk["choices"][0]["delta"]
|
||||
and chunk["choices"][0]["delta"]["reasoning_content"] is not None
|
||||
]
|
||||
|
||||
if len(reasoning_chunks) > 0:
|
||||
response["choices"][0]["message"]["reasoning_content"] = processor.get_combined_reasoning_content(
|
||||
reasoning_chunks
|
||||
)
|
||||
|
||||
typed_chunks: Final = cast(Sequence[object], chunks) # cast-ok: chunks are untyped upstream
|
||||
reasoning_items: Final = tuple(chain.from_iterable(_chunk_reasoning_items(chunk) for chunk in typed_chunks))
|
||||
if reasoning_items:
|
||||
response["choices"][0]["message"]["reasoning_items"] = list(reasoning_items)
|
||||
|
||||
annotation_chunks: Final = [
|
||||
chunk
|
||||
for chunk in chunks
|
||||
if chunk.get("choices")
|
||||
and "annotations" in chunk["choices"][0]["delta"]
|
||||
and chunk["choices"][0]["delta"]["annotations"] is not None
|
||||
]
|
||||
|
||||
if len(annotation_chunks) > 0:
|
||||
# Merge annotations from ALL chunks — providers may spread
|
||||
# them across multiple streaming chunks or send them only in
|
||||
# the final chunk.
|
||||
all_annotations: Final[list[object]] = []
|
||||
for ac in annotation_chunks:
|
||||
all_annotations.extend(ac["choices"][0]["delta"]["annotations"])
|
||||
message_annotations, provider_annotations = partition_annotations(all_annotations)
|
||||
if message_annotations is not None:
|
||||
response["choices"][0]["message"]["annotations"] = message_annotations
|
||||
if provider_annotations:
|
||||
response.choices[0].message.provider_specific_fields = {
|
||||
**(response.choices[0].message.provider_specific_fields or {}),
|
||||
"annotations": list(provider_annotations),
|
||||
}
|
||||
|
||||
audio_chunks: Final = [
|
||||
chunk
|
||||
for chunk in chunks
|
||||
if chunk.get("choices")
|
||||
and "audio" in chunk["choices"][0]["delta"]
|
||||
and chunk["choices"][0]["delta"]["audio"] is not None
|
||||
]
|
||||
|
||||
if len(audio_chunks) > 0:
|
||||
_choice = response.choices[0]
|
||||
_choice.message.audio = processor.get_combined_audio_content(audio_chunks)
|
||||
|
||||
# Handle image chunks from models like gemini-2.5-flash-image
|
||||
# See: https://github.com/BerriAI/litellm/issues/19478
|
||||
image_chunks: Final = [
|
||||
chunk
|
||||
for chunk in chunks
|
||||
if chunk.get("choices")
|
||||
and "images" in chunk["choices"][0]["delta"]
|
||||
and chunk["choices"][0]["delta"]["images"] is not None
|
||||
]
|
||||
|
||||
if len(image_chunks) > 0:
|
||||
# Images come complete in a single chunk, collect all images from all chunks
|
||||
all_images: Final = []
|
||||
for chunk in image_chunks:
|
||||
all_images.extend(chunk["choices"][0]["delta"]["images"])
|
||||
response["choices"][0]["message"]["images"] = all_images
|
||||
|
||||
# Combine provider_specific_fields from streaming chunks (e.g., web_search_results, citations)
|
||||
# See: https://github.com/BerriAI/litellm/issues/17737
|
||||
provider_specific_chunks: Final = [
|
||||
chunk
|
||||
for chunk in chunks
|
||||
if chunk.get("choices")
|
||||
and "provider_specific_fields" in chunk["choices"][0]["delta"]
|
||||
and chunk["choices"][0]["delta"]["provider_specific_fields"] is not None
|
||||
]
|
||||
|
||||
if len(provider_specific_chunks) > 0:
|
||||
provider_field_dicts: Final = tuple(
|
||||
fields
|
||||
for chunk in provider_specific_chunks
|
||||
for fields in (chunk["choices"][0]["delta"]["provider_specific_fields"],)
|
||||
if isinstance(fields, dict)
|
||||
)
|
||||
streamed_citations: Final = tuple(
|
||||
fields["citation"] for fields in provider_field_dicts if fields.get("citation") is not None
|
||||
)
|
||||
citation_fields: Final = (
|
||||
{"citations": _joined_streamed_citations(streamed_citations)} if streamed_citations else {}
|
||||
)
|
||||
combined_provider_fields: Final = {
|
||||
key: value
|
||||
for fields in (citation_fields, *provider_field_dicts)
|
||||
for key, value in fields.items()
|
||||
if key != "citation"
|
||||
}
|
||||
|
||||
if combined_provider_fields:
|
||||
_choice = response.choices[0]
|
||||
_choice.message.provider_specific_fields = {
|
||||
**(_choice.message.provider_specific_fields or {}),
|
||||
**combined_provider_fields,
|
||||
}
|
||||
for choice, choice_chunks in rebuilt_choices:
|
||||
for field, value in _assembled_message_fields(processor, choice_chunks):
|
||||
choice.message[field] = value
|
||||
|
||||
completion_output = get_content_from_model_response(response)
|
||||
|
||||
|
|
|
|||
|
|
@ -7,9 +7,11 @@ import re
|
|||
import struct
|
||||
from bisect import bisect_right
|
||||
from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence
|
||||
from itertools import accumulate
|
||||
from dataclasses import dataclass
|
||||
from functools import cached_property
|
||||
from itertools import accumulate, chain, islice
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Literal, cast
|
||||
from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, cast
|
||||
|
||||
import anyio
|
||||
import anyio.lowlevel
|
||||
|
|
@ -33,7 +35,12 @@ from litellm.constants import (
|
|||
MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES,
|
||||
MAX_TILE_HEIGHT,
|
||||
MAX_TILE_WIDTH,
|
||||
OPENAI_CHAT_MODES,
|
||||
OPENAI_MODEL_NAME_MAX_DASHES,
|
||||
OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS,
|
||||
PDF_DATA_URL_PREFIX,
|
||||
PDF_MAX_FORM_XOBJECT_DEPTH,
|
||||
PDF_MAX_XOBJECT_DRAWS_PER_PAGE,
|
||||
TIKTOKEN_ENCODE_CHUNK_SIZE_CHARS,
|
||||
TOKEN_COUNTER_MAX_CONCURRENT_COUNTS,
|
||||
TOKEN_COUNTER_MAX_EXACT_CHARS,
|
||||
|
|
@ -606,6 +613,35 @@ TokenCounterFunction = Callable[[str], int]
|
|||
Type for a function that counts tokens in a string.
|
||||
"""
|
||||
|
||||
PdfPagePricing: TypeAlias = Literal["openai", "claude"]
|
||||
|
||||
|
||||
class _PdfBox(Protocol):
|
||||
@property
|
||||
def width(self) -> float: ...
|
||||
|
||||
@property
|
||||
def height(self) -> float: ...
|
||||
|
||||
|
||||
class _PdfContent(Protocol):
|
||||
def get_data(self) -> bytes: ...
|
||||
|
||||
|
||||
class _PdfPage(Protocol):
|
||||
@property
|
||||
def mediabox(self) -> _PdfBox: ...
|
||||
|
||||
def extract_text(self) -> str: ...
|
||||
|
||||
def get_contents(self) -> _PdfContent | None: ...
|
||||
|
||||
|
||||
class _PdfReader(Protocol):
|
||||
@property
|
||||
def pages(self) -> Sequence[_PdfPage]: ...
|
||||
|
||||
|
||||
EXTRAPOLATION_SAMPLES: Final = 16
|
||||
T_ParamSpec: Final = ParamSpec("T_ParamSpec")
|
||||
T_Retval = TypeVar("T_Retval")
|
||||
|
|
@ -662,7 +698,7 @@ class _MessageCountParams:
|
|||
if uses_legacy_message_accounting(model):
|
||||
self.tokens_per_message = 4 # every message follows <|start|>{role/name}\n{content}<|end|>\n
|
||||
self.tokens_per_name = -1 # if there's a name, the role is omitted
|
||||
elif actual_model in litellm.open_ai_chat_completion_models or actual_model in litellm.azure_llms:
|
||||
elif _is_openai_chat_model(actual_model):
|
||||
self.tokens_per_message = 3
|
||||
self.tokens_per_name = 1
|
||||
else:
|
||||
|
|
@ -671,6 +707,63 @@ class _MessageCountParams:
|
|||
self.tokens_per_name = 1
|
||||
self.count_function = _get_count_function(model, custom_tokenizer)
|
||||
self.tool_section = chat_template_tool_section(model)
|
||||
self._model = model
|
||||
|
||||
@cached_property
|
||||
def pdf_pricing(self) -> PdfPagePricing:
|
||||
return pdf_page_pricing(self._model)
|
||||
|
||||
|
||||
def _is_openai_chat_model(model: str) -> bool:
|
||||
return model in litellm.open_ai_chat_completion_models or model in litellm.azure_llms
|
||||
|
||||
|
||||
def _openai_deployment_name(model: str) -> str:
|
||||
name: Final = model.rpartition("/")[2]
|
||||
return name.split(":")[1] if name.startswith("ft:") else name
|
||||
|
||||
|
||||
def _openai_model_family(model: str) -> str | None:
|
||||
words: Final = _openai_deployment_name(model).split("-", OPENAI_MODEL_NAME_MAX_DASHES)
|
||||
prefixes: Final = ("-".join(words[:end]) for end in range(len(words), 0, -1))
|
||||
listed: Final = next((prefix for prefix in prefixes if _is_openai_chat_model(prefix)), None)
|
||||
return None if listed is None or _is_listed_non_chat_model(listed) else listed
|
||||
|
||||
|
||||
def _untyped_cost_map() -> object:
|
||||
return litellm.model_cost # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # loaded untyped
|
||||
|
||||
|
||||
def _cost_map_entry(model: str) -> Mapping[str, object]:
|
||||
cost_map: Final = _pdf_dictionary(_untyped_cost_map()) or {}
|
||||
return _pdf_dictionary(cost_map.get(model)) or {}
|
||||
|
||||
|
||||
def _is_azure_openai_chat_model(model: str) -> bool:
|
||||
name: Final = _openai_deployment_name(model)
|
||||
entry: Final = _cost_map_entry(f"azure/{name}")
|
||||
return (
|
||||
entry.get("litellm_provider") == "azure"
|
||||
and entry.get("mode") in OPENAI_CHAT_MODES
|
||||
and (entry.get("supports_pdf_input") is True or name.startswith("gpt-"))
|
||||
)
|
||||
|
||||
|
||||
def _is_openai_or_azure_model(model: str) -> bool:
|
||||
if _is_listed_non_chat_model(_openai_deployment_name(model)):
|
||||
return False
|
||||
return _is_openai_chat_model(model) or _openai_model_family(model) is not None or _is_azure_openai_chat_model(model)
|
||||
|
||||
|
||||
def pdf_page_pricing(model: str | None) -> PdfPagePricing:
|
||||
return "openai" if _is_openai_or_azure_model(model or "") else "claude"
|
||||
|
||||
|
||||
def reads_pdf_pages(model: str) -> bool:
|
||||
segments: Final = model.split("/")
|
||||
names: Final = ("/".join(segments[start:]) for start in range(len(segments)))
|
||||
stated: Final = (_cost_map_entry(name).get("supports_pdf_input") for name in names)
|
||||
return next((support for support in stated if support is not None), False) is True
|
||||
|
||||
|
||||
def token_counter(
|
||||
|
|
@ -842,6 +935,7 @@ def _count_messages(
|
|||
value,
|
||||
use_default_image_token_count,
|
||||
default_token_count,
|
||||
params.pdf_pricing,
|
||||
)
|
||||
elif key == "search_results" and isinstance(value, list):
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
|
|
@ -986,7 +1080,7 @@ def _get_exact_count_function(
|
|||
|
||||
return count_tokens
|
||||
if tokenizer_json["type"] == "openai_tokenizer":
|
||||
encoding: Final = openai_tokenizer_encoding(model)
|
||||
encoding: Final = openai_tokenizer_encoding(model or "")
|
||||
|
||||
def encode_length(text: str) -> int:
|
||||
return _encoding_count(encoding, text)
|
||||
|
|
@ -1006,6 +1100,11 @@ def openai_tokenizer_encoding(model: str) -> "Encoding":
|
|||
return get_encoding(openai_tokenizer_encoding_name(model))
|
||||
|
||||
|
||||
def _is_listed_non_chat_model(model: str) -> bool:
|
||||
modes: Final = (_cost_map_entry(name).get("mode") for name in (model, f"azure/{model}"))
|
||||
return any(mode is not None and mode not in OPENAI_CHAT_MODES for mode in modes)
|
||||
|
||||
|
||||
def openai_tokenizer_encoding_name(model: str) -> str:
|
||||
"""The tiktoken encoding name for `model`, without loading the encoding.
|
||||
|
||||
|
|
@ -1021,7 +1120,9 @@ def openai_tokenizer_encoding_name(model: str) -> str:
|
|||
try:
|
||||
return tiktoken.encoding_name_for_model(model_to_use)
|
||||
except KeyError:
|
||||
return _suffixed_variant_encoding_name(model_to_use) or "o200k_base"
|
||||
return _suffixed_variant_encoding_name(model_to_use) or (
|
||||
"cl100k_base" if _is_listed_non_chat_model(model_to_use) else "o200k_base"
|
||||
)
|
||||
|
||||
|
||||
def unresolved_tokenizer_model(model: str) -> str:
|
||||
|
|
@ -1079,7 +1180,16 @@ def single_encoding_openai_model(served: Sequence[str | None]) -> str | None:
|
|||
|
||||
|
||||
def _openai_model_name(model: str) -> str | None:
|
||||
return _listed_openai_model_name(model) or _tiktoken_o200k_model_name(model) or _versioned_openai_model_name(model)
|
||||
return (
|
||||
_listed_openai_model_name(model)
|
||||
or _tiktoken_o200k_model_name(model)
|
||||
or _versioned_openai_model_name(model)
|
||||
or _azure_openai_chat_model_name(model)
|
||||
)
|
||||
|
||||
|
||||
def _azure_openai_chat_model_name(model: str) -> str | None:
|
||||
return _openai_deployment_name(model) if _is_azure_openai_chat_model(model) else None
|
||||
|
||||
|
||||
_VERSIONED_OPENAI_MODEL: Final = re.compile(r"(?:gpt-(?:[5-9]|[1-9]\d+)|o[1-9]\d*)(?:[.-]|$)")
|
||||
|
|
@ -1224,10 +1334,6 @@ def _validate_anthropic_content(content: Mapping[str, object]) -> type:
|
|||
if expected_cls is None:
|
||||
raise ValueError(f"Unknown Anthropic content type: '{content_type}'")
|
||||
|
||||
missing: Final = [k for k in getattr(expected_cls, "__required_keys__", set()) if k not in content]
|
||||
if missing:
|
||||
raise ValueError(f"Missing required fields in {content_type} block: {', '.join(missing)}")
|
||||
|
||||
return expected_cls
|
||||
|
||||
|
||||
|
|
@ -1253,7 +1359,165 @@ def _anthropic_rendered_page_image_tokens(width: float, height: float) -> int:
|
|||
return math.ceil(min(float(ANTHROPIC_IMAGE_MAX_PIXELS), area_at_max_edge) / ANTHROPIC_IMAGE_PIXELS_PER_TOKEN)
|
||||
|
||||
|
||||
def _count_inline_pdf_tokens(data_url: str, count_function: TokenCounterFunction) -> int | None:
|
||||
def _pdf_dictionary(node: object) -> Mapping[str, object] | None:
|
||||
return node if isinstance(node, Mapping) else None # pyright: ignore[reportUnknownVariableType] # read per key
|
||||
|
||||
|
||||
def _pdf_value(container: Mapping[str, object], key: str) -> object:
|
||||
return container[key] if key in container else None # noqa: SIM401 # pypdf resolves references only in []
|
||||
|
||||
|
||||
_PdfXObjects: TypeAlias = Mapping[str, object] | None
|
||||
_PdfXObjectUse: TypeAlias = tuple[Mapping[str, object], _PdfXObjects]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _PdfContentSummary:
|
||||
drawn_names: tuple[str, ...]
|
||||
draws_inline_image: bool
|
||||
|
||||
|
||||
_PdfDrawing: TypeAlias = tuple[_PdfContentSummary, _PdfXObjects]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _PdfWalk:
|
||||
frontier: tuple[_PdfDrawing, ...]
|
||||
seen: frozenset[tuple[int, int]]
|
||||
summaries: Mapping[int, _PdfContentSummary | None]
|
||||
draws: int
|
||||
|
||||
|
||||
def _pdf_xobjects(container: Mapping[str, object], inherited: _PdfXObjects) -> _PdfXObjects:
|
||||
resources: Final = _pdf_dictionary(_pdf_value(container, "/Resources"))
|
||||
if resources is None:
|
||||
return inherited
|
||||
return _pdf_dictionary(_pdf_value(resources, "/XObject"))
|
||||
|
||||
|
||||
_PDF_WHITESPACE: Final = rb"\x00\t\n\x0c\r "
|
||||
_PDF_TOKEN_END: Final = _PDF_WHITESPACE + rb"()<>\[\]{}/%"
|
||||
_PDF_XOBJECT_DRAW: Final = re.compile(
|
||||
rb"/([^" + _PDF_TOKEN_END + rb"]*)(?:[" + _PDF_WHITESPACE + rb"]|%[^\r\n]*[\r\n])+Do(?![^" + _PDF_TOKEN_END + rb"])"
|
||||
)
|
||||
_PDF_INLINE_IMAGE: Final = re.compile(rb"(?<![^" + _PDF_WHITESPACE + rb"])BI(?![^" + _PDF_WHITESPACE + rb"/])")
|
||||
_PDF_NAME_ESCAPE: Final = re.compile(rb"#([0-9A-Fa-f]{2})")
|
||||
_PDF_NAME_CHARSETS: Final = ("utf-8", "gbk")
|
||||
|
||||
|
||||
def _decoded(raw: bytes, charset: str) -> str | None:
|
||||
try:
|
||||
return raw.decode(charset)
|
||||
except UnicodeDecodeError:
|
||||
return None
|
||||
|
||||
|
||||
def _pdf_name(escaped: bytes) -> str:
|
||||
raw: Final = _PDF_NAME_ESCAPE.sub(lambda escape: bytes.fromhex(escape[1].decode("ascii")), escaped)
|
||||
decoded: Final = (_decoded(raw, charset) for charset in _PDF_NAME_CHARSETS)
|
||||
return "/" + next((name for name in decoded if name is not None), raw.decode("latin-1"))
|
||||
|
||||
|
||||
def _pdf_content_summary(content: _PdfContent) -> _PdfContentSummary:
|
||||
data: Final = content.get_data()
|
||||
draws: Final = islice(_PDF_XOBJECT_DRAW.finditer(data), PDF_MAX_XOBJECT_DRAWS_PER_PAGE + 1)
|
||||
return _PdfContentSummary(
|
||||
drawn_names=tuple(_pdf_name(draw[1]) for draw in draws),
|
||||
draws_inline_image=_PDF_INLINE_IMAGE.search(data) is not None,
|
||||
)
|
||||
|
||||
|
||||
def _pdf_form_summary(form: Mapping[str, object]) -> _PdfContentSummary | None:
|
||||
from pypdf.generic import StreamObject
|
||||
|
||||
if not isinstance(form, StreamObject):
|
||||
return None
|
||||
return _pdf_content_summary(cast(_PdfContent, form)) # cast-ok: pypdf is untyped
|
||||
|
||||
|
||||
def _pdf_drawn_xobjects(drawing: _PdfDrawing) -> tuple[_PdfXObjectUse, ...]:
|
||||
summary, xobjects = drawing
|
||||
if xobjects is None:
|
||||
return ()
|
||||
resolved: Final = (_pdf_dictionary(_pdf_value(xobjects, name)) for name in summary.drawn_names)
|
||||
return tuple((xobject, xobjects) for xobject in resolved if xobject is not None)
|
||||
|
||||
|
||||
def _pdf_drawing_key(use: _PdfXObjectUse) -> tuple[int, int]:
|
||||
xobject, inherited = use
|
||||
return id(xobject), id(_pdf_xobjects(xobject, inherited))
|
||||
|
||||
|
||||
def _pdf_next_walk(walk: _PdfWalk) -> _PdfWalk | None:
|
||||
draws: Final = walk.draws + sum(len(summary.drawn_names) for summary, _ in walk.frontier)
|
||||
if draws > PDF_MAX_XOBJECT_DRAWS_PER_PAGE:
|
||||
return None
|
||||
drawn: Final = tuple(chain.from_iterable(map(_pdf_drawn_xobjects, walk.frontier)))
|
||||
if any(_pdf_value(xobject, "/Subtype") == "/Image" for xobject, _ in drawn):
|
||||
return None
|
||||
forms: Final = {_pdf_drawing_key(use): use for use in drawn if _pdf_value(use[0], "/Subtype") == "/Form"}
|
||||
unseen: Final = tuple(forms[key] for key in forms.keys() - walk.seen)
|
||||
unparsed: Final = {id(form): form for form, _ in unseen if id(form) not in walk.summaries}
|
||||
summaries: Final = MappingProxyType(
|
||||
{**walk.summaries, **{key: _pdf_form_summary(form) for key, form in unparsed.items()}}
|
||||
)
|
||||
drawings: Final = ((summaries[id(form)], _pdf_xobjects(form, inherited)) for form, inherited in unseen)
|
||||
return _PdfWalk(
|
||||
frontier=tuple((summary, xobjects) for summary, xobjects in drawings if summary is not None),
|
||||
seen=walk.seen | frozenset(forms),
|
||||
summaries=summaries,
|
||||
draws=draws,
|
||||
)
|
||||
|
||||
|
||||
def _pdf_page_draws_raster_image(page: _PdfPage) -> bool:
|
||||
contents: Final = page.get_contents()
|
||||
page_dictionary: Final = _pdf_dictionary(page)
|
||||
if contents is None or page_dictionary is None:
|
||||
return False
|
||||
walk: _PdfWalk = _PdfWalk( # rebind-ok: breadth-first walk, recursion is banned
|
||||
frontier=((_pdf_content_summary(contents), _pdf_xobjects(page_dictionary, None)),),
|
||||
seen=frozenset(),
|
||||
summaries=MappingProxyType({}),
|
||||
draws=0,
|
||||
)
|
||||
for _ in range(PDF_MAX_FORM_XOBJECT_DEPTH):
|
||||
if not walk.frontier:
|
||||
return False
|
||||
if any(summary.draws_inline_image for summary, _ in walk.frontier):
|
||||
return True
|
||||
if (next_walk := _pdf_next_walk(walk)) is None:
|
||||
return True
|
||||
walk = next_walk
|
||||
return bool(walk.frontier)
|
||||
|
||||
|
||||
def _openai_pdf_page_tokens(page: _PdfPage, count_function: TokenCounterFunction) -> int:
|
||||
if _pdf_page_draws_raster_image(page):
|
||||
return _claude_pdf_page_tokens(page, count_function)
|
||||
text: Final = page.extract_text() or ""
|
||||
return count_function(text) + OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS if text.strip() else 0
|
||||
|
||||
|
||||
def _claude_pdf_page_tokens(page: _PdfPage, count_function: TokenCounterFunction) -> int:
|
||||
return count_function(page.extract_text() or "") + _anthropic_rendered_page_image_tokens(
|
||||
float(page.mediabox.width), float(page.mediabox.height)
|
||||
)
|
||||
|
||||
|
||||
def _pdf_page_tokens(page: _PdfPage, count_function: TokenCounterFunction, pdf_pricing: PdfPagePricing) -> int:
|
||||
match pdf_pricing:
|
||||
case "openai":
|
||||
return _openai_pdf_page_tokens(page, count_function)
|
||||
case "claude":
|
||||
return _claude_pdf_page_tokens(page, count_function)
|
||||
|
||||
|
||||
def _count_inline_pdf_tokens(
|
||||
data_url: str,
|
||||
count_function: TokenCounterFunction,
|
||||
pdf_pricing: PdfPagePricing = "claude",
|
||||
) -> int | None:
|
||||
if not data_url.startswith(PDF_DATA_URL_PREFIX):
|
||||
return None
|
||||
try:
|
||||
|
|
@ -1262,12 +1526,10 @@ def _count_inline_pdf_tokens(data_url: str, count_function: TokenCounterFunction
|
|||
verbose_logger.debug("pypdf is not installed, so the PDF document is priced like one image")
|
||||
return None
|
||||
try:
|
||||
reader: Final = PdfReader(io.BytesIO(base64.b64decode(data_url[len(PDF_DATA_URL_PREFIX) :])))
|
||||
return sum(
|
||||
count_function(page.extract_text() or "")
|
||||
+ _anthropic_rendered_page_image_tokens(float(page.mediabox.width), float(page.mediabox.height))
|
||||
for page in reader.pages
|
||||
reader: Final = cast( # cast-ok: pypdf's PdfReader satisfies _PdfReader, typed only where pypdf is installed
|
||||
_PdfReader, PdfReader(io.BytesIO(base64.b64decode(data_url[len(PDF_DATA_URL_PREFIX) :])))
|
||||
)
|
||||
return sum(_pdf_page_tokens(page, count_function, pdf_pricing) for page in reader.pages)
|
||||
except Exception as e:
|
||||
verbose_logger.debug("Could not read the PDF document's pages (%s), so it is priced like one image", e)
|
||||
return None
|
||||
|
|
@ -1277,8 +1539,9 @@ def _count_opaque_document_tokens(
|
|||
data_url: str,
|
||||
count_function: TokenCounterFunction,
|
||||
use_default_image_token_count: bool,
|
||||
pdf_pricing: PdfPagePricing = "claude",
|
||||
) -> int:
|
||||
pdf_tokens: Final = _count_inline_pdf_tokens(data_url, count_function)
|
||||
pdf_tokens: Final = _count_inline_pdf_tokens(data_url, count_function, pdf_pricing)
|
||||
if pdf_tokens is not None:
|
||||
return pdf_tokens
|
||||
return calculate_img_tokens(
|
||||
|
|
@ -1293,6 +1556,7 @@ def _count_document_tokens(
|
|||
count_function: TokenCounterFunction,
|
||||
use_default_image_token_count: bool,
|
||||
default_token_count: int | None,
|
||||
pdf_pricing: PdfPagePricing = "claude",
|
||||
) -> int:
|
||||
source: Final = document["source"]
|
||||
metadata_tokens: Final = sum(
|
||||
|
|
@ -1305,10 +1569,10 @@ def _count_document_tokens(
|
|||
if isinstance(content, str):
|
||||
return metadata_tokens + count_function(content)
|
||||
return metadata_tokens + _count_content_list(
|
||||
count_function, content, use_default_image_token_count, default_token_count
|
||||
count_function, content, use_default_image_token_count, default_token_count, pdf_pricing
|
||||
)
|
||||
return metadata_tokens + _count_opaque_document_tokens(
|
||||
_anthropic_image_source_data(source), count_function, use_default_image_token_count
|
||||
_anthropic_image_source_data(source), count_function, use_default_image_token_count, pdf_pricing
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -1316,6 +1580,7 @@ def _count_file_tokens(
|
|||
file_value: object,
|
||||
count_function: TokenCounterFunction,
|
||||
use_default_image_token_count: bool,
|
||||
pdf_pricing: PdfPagePricing = "claude",
|
||||
) -> int:
|
||||
"""An OpenAI `file` block is the chat-completions spelling of a document, so it prices like one."""
|
||||
if not isinstance(file_value, Mapping):
|
||||
|
|
@ -1325,7 +1590,9 @@ def _count_file_tokens(
|
|||
name_tokens: Final = count_function(filename) if isinstance(filename, str) and filename else 0
|
||||
if not isinstance(file_data, str) or not file_data:
|
||||
return name_tokens
|
||||
return name_tokens + _count_opaque_document_tokens(file_data, count_function, use_default_image_token_count)
|
||||
return name_tokens + _count_opaque_document_tokens(
|
||||
file_data, count_function, use_default_image_token_count, pdf_pricing
|
||||
)
|
||||
|
||||
|
||||
def _count_anthropic_content(
|
||||
|
|
@ -1333,6 +1600,7 @@ def _count_anthropic_content(
|
|||
count_function: TokenCounterFunction,
|
||||
use_default_image_token_count: bool,
|
||||
default_token_count: int | None,
|
||||
pdf_pricing: PdfPagePricing = "claude",
|
||||
) -> int:
|
||||
"""
|
||||
Count tokens in Anthropic-specific content blocks (tool_use, tool_result, etc.).
|
||||
|
|
@ -1367,6 +1635,7 @@ def _count_anthropic_content(
|
|||
field_value,
|
||||
use_default_image_token_count,
|
||||
default_token_count,
|
||||
pdf_pricing,
|
||||
)
|
||||
elif isinstance(field_value, dict):
|
||||
tokens += count_function(str(field_value))
|
||||
|
|
@ -1393,6 +1662,7 @@ def _count_content_list(
|
|||
],
|
||||
use_default_image_token_count: bool,
|
||||
default_token_count: int | None,
|
||||
pdf_pricing: PdfPagePricing = "claude",
|
||||
) -> int:
|
||||
"""Recursively count tokens from a list of content blocks."""
|
||||
try:
|
||||
|
|
@ -1417,12 +1687,14 @@ def _count_content_list(
|
|||
count_function,
|
||||
use_default_image_token_count,
|
||||
default_token_count,
|
||||
pdf_pricing,
|
||||
)
|
||||
elif c["type"] == "file":
|
||||
num_tokens += _count_file_tokens(
|
||||
c.get("file"),
|
||||
count_function,
|
||||
use_default_image_token_count,
|
||||
pdf_pricing,
|
||||
)
|
||||
elif c["type"] in ("tool_use", "tool_result", "search_result"):
|
||||
num_tokens += _count_anthropic_content(
|
||||
|
|
@ -1430,6 +1702,7 @@ def _count_content_list(
|
|||
count_function,
|
||||
use_default_image_token_count,
|
||||
default_token_count,
|
||||
pdf_pricing,
|
||||
)
|
||||
elif c["type"] in ("thinking", "redacted_thinking"):
|
||||
# Claude extended thinking content block
|
||||
|
|
@ -1552,12 +1825,13 @@ def _format_single_function_definition(function: Mapping[str, object]) -> tuple[
|
|||
|
||||
def _format_object_parameters(parameters, indent):
|
||||
properties: Final = parameters.get("properties")
|
||||
if not properties:
|
||||
if not isinstance(properties, Mapping) or not properties:
|
||||
return ""
|
||||
required_params: Final = parameters.get("required", [])
|
||||
required: Final = parameters.get("required")
|
||||
required_params: Final = tuple(required) if isinstance(required, list) else ()
|
||||
lines: Final = []
|
||||
for key, props in properties.items():
|
||||
description = props.get("description")
|
||||
description = props.get("description") if isinstance(props, Mapping) else None
|
||||
if description:
|
||||
lines.append(f"// {description}")
|
||||
question = "?"
|
||||
|
|
@ -1568,19 +1842,21 @@ def _format_object_parameters(parameters, indent):
|
|||
|
||||
|
||||
def _format_type(props, indent):
|
||||
if not isinstance(props, Mapping):
|
||||
return "any"
|
||||
type: Final = props.get("type")
|
||||
enum: Final = props.get("enum")
|
||||
if type == "string":
|
||||
if "enum" in props:
|
||||
return " | ".join([f'"{item}"' for item in props["enum"]])
|
||||
if isinstance(enum, list):
|
||||
return " | ".join([f'"{item}"' for item in enum])
|
||||
return "string"
|
||||
elif type == "array":
|
||||
# items is required, OpenAI throws an error if it's missing
|
||||
return f"{_format_type(props['items'], indent)}[]"
|
||||
return f"{_format_type(props.get('items'), indent)}[]"
|
||||
elif type == "object":
|
||||
return f"{{\n{_format_object_parameters(props, indent + 2)}\n}}"
|
||||
elif type in ["integer", "number"]:
|
||||
if "enum" in props:
|
||||
return " | ".join([f'"{item}"' for item in props["enum"]])
|
||||
if isinstance(enum, list):
|
||||
return " | ".join([f'"{item}"' for item in enum])
|
||||
return "number"
|
||||
elif type == "boolean":
|
||||
return "boolean"
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import json
|
|||
from collections.abc import Sequence
|
||||
from typing import TYPE_CHECKING, Any, Final, Optional
|
||||
|
||||
from pydantic import ConfigDict, TypeAdapter, ValidationError
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
|
|
@ -28,6 +29,8 @@ if TYPE_CHECKING:
|
|||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth
|
||||
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True))
|
||||
|
||||
|
||||
class _A2ATextPart(TypedDict, total=False):
|
||||
"""The subset of an A2A message part this handler reads text from."""
|
||||
|
|
@ -333,8 +336,8 @@ class A2AGuardrailHandler(BaseTranslation):
|
|||
obj = item
|
||||
elif isinstance(item, str):
|
||||
try:
|
||||
obj = json.loads(item.strip())
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
obj = _JSON_OBJECT.validate_python(json.loads(item.strip()))
|
||||
except (json.JSONDecodeError, TypeError, ValidationError):
|
||||
continue
|
||||
else:
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -18,9 +18,9 @@ from copy import deepcopy
|
|||
from dataclasses import dataclass
|
||||
from itertools import chain, repeat
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Protocol, cast, overload, runtime_checkable
|
||||
from typing import TYPE_CHECKING, Any, Final, Protocol, TypeVar, cast, overload, runtime_checkable
|
||||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config
|
||||
from typing_extensions import ReadOnly, TypedDict, assert_never
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
|
|
@ -349,14 +349,40 @@ def _tool_call_shapes(tool_calls: Sequence[object]) -> tuple[_ToolCallShape, ...
|
|||
)
|
||||
|
||||
|
||||
class _AnthropicSSEDelta(TypedDict, total=False):
|
||||
type: ReadOnly[str]
|
||||
@with_config(ConfigDict(extra="ignore", strict=True))
|
||||
class _SSETextDelta(TypedDict, total=False):
|
||||
type: ReadOnly[object]
|
||||
text: ReadOnly[str]
|
||||
stop_reason: ReadOnly[str | None]
|
||||
|
||||
|
||||
class _AnthropicSSEEvent(TypedDict, total=False):
|
||||
delta: ReadOnly[_AnthropicSSEDelta]
|
||||
@with_config(ConfigDict(extra="ignore", strict=True))
|
||||
class _SSETextDeltaEvent(TypedDict, total=False):
|
||||
delta: ReadOnly[_SSETextDelta]
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="ignore", strict=True))
|
||||
class _SSEStopDelta(TypedDict, total=False):
|
||||
stop_reason: ReadOnly[object]
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="ignore", strict=True))
|
||||
class _SSEStopDeltaEvent(TypedDict, total=False):
|
||||
delta: ReadOnly[_SSEStopDelta]
|
||||
|
||||
|
||||
_SSE_TEXT_DELTA_EVENT: Final = TypeAdapter(_SSETextDeltaEvent)
|
||||
_SSE_STOP_DELTA_EVENT: Final = TypeAdapter(_SSEStopDeltaEvent)
|
||||
_SSEEventT = TypeVar("_SSEEventT")
|
||||
|
||||
|
||||
def _parse_sse_event(adapter: TypeAdapter[_SSEEventT], event_type: str, data_line: str) -> _SSEEventT | None:
|
||||
try:
|
||||
return adapter.validate_python(json.loads(data_line))
|
||||
except json.JSONDecodeError:
|
||||
verbose_proxy_logger.warning("Skipping a %s SSE event that is not valid JSON", event_type)
|
||||
except ValidationError:
|
||||
verbose_proxy_logger.warning("Skipping a %s SSE event with an unexpected shape", event_type)
|
||||
return None
|
||||
|
||||
|
||||
class AnthropicMessagesHandler(BaseTranslation):
|
||||
|
|
@ -1806,13 +1832,10 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
|
||||
# Only process content_block_delta events
|
||||
if event_type == "content_block_delta" and data_line:
|
||||
try:
|
||||
data: _AnthropicSSEEvent = json.loads(data_line)
|
||||
delta: _AnthropicSSEDelta = data.get("delta", {})
|
||||
if delta.get("type") == "text_delta":
|
||||
text += delta.get("text", "")
|
||||
except json.JSONDecodeError:
|
||||
verbose_proxy_logger.warning("Failed to parse JSON from SSE data: %s", data_line)
|
||||
data = _parse_sse_event(_SSE_TEXT_DELTA_EVENT, event_type, data_line)
|
||||
delta: _SSETextDelta = {} if data is None else data.get("delta", {})
|
||||
if delta.get("type") == "text_delta":
|
||||
text += delta.get("text", "")
|
||||
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.error("Error extracting text from SSE: %s", e)
|
||||
|
|
@ -1869,14 +1892,11 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
|
||||
# Check for message_delta event with stop_reason
|
||||
if event_type == "message_delta" and data_line:
|
||||
try:
|
||||
data: _AnthropicSSEEvent = json.loads(data_line)
|
||||
delta: _AnthropicSSEDelta = data.get("delta", {})
|
||||
stop_reason: str | None = delta.get("stop_reason")
|
||||
if stop_reason is not None:
|
||||
return True
|
||||
except json.JSONDecodeError:
|
||||
verbose_proxy_logger.warning("Failed to parse JSON from SSE data: %s", data_line)
|
||||
data = _parse_sse_event(_SSE_STOP_DELTA_EVENT, event_type, data_line)
|
||||
delta: _SSEStopDelta = {} if data is None else data.get("delta", {})
|
||||
stop_reason: object = delta.get("stop_reason")
|
||||
if stop_reason is not None:
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.error("Error checking streaming end in SSE: %s", e)
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import copy
|
|||
import json
|
||||
import re
|
||||
from collections.abc import Container, Mapping, MutableMapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from types import MappingProxyType
|
||||
from typing import Annotated, Any, ClassVar, Final, Literal, TypeVar
|
||||
|
|
@ -1774,6 +1775,48 @@ def anthropic_request_errors_on_thinking_binding_mismatch(request_body: object)
|
|||
return binding is not None and binding.get("prefix_mismatch_behavior") == "error"
|
||||
|
||||
|
||||
_CLAUDE_STRICT_TOOLS_REJECTION: Final = re.compile(
|
||||
r"Too many strict tools \(\d+\)"
|
||||
r"|Schemas contains too many (?:optional parameters|parameters with union types)"
|
||||
r"|[Cc]ompiled grammar (?:is too large|size)"
|
||||
r"|tools\.\d+\.custom(?:\.strict)?: "
|
||||
r"|Grammar compilation timed out"
|
||||
)
|
||||
_CLAUDE_TOOLS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...])
|
||||
|
||||
|
||||
def is_claude_strict_tools_rejection(status_code: int, error_text: str) -> bool:
|
||||
return status_code == 400 and _CLAUDE_STRICT_TOOLS_REJECTION.search(error_text) is not None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ToolsWithoutStrict:
|
||||
tools: tuple[Mapping[str, object], ...]
|
||||
dropped_count: int
|
||||
|
||||
|
||||
def tools_without_strict(tools: object) -> ToolsWithoutStrict | None:
|
||||
try:
|
||||
validated: Final = _CLAUDE_TOOLS_ADAPTER.validate_python(tools)
|
||||
except ValidationError:
|
||||
return None
|
||||
dropped_count: Final = sum(tool.get("strict") is True for tool in validated)
|
||||
if dropped_count == 0:
|
||||
return None
|
||||
return ToolsWithoutStrict(
|
||||
tools=tuple({key: value for key, value in tool.items() if key != "strict"} for tool in validated),
|
||||
dropped_count=dropped_count,
|
||||
)
|
||||
|
||||
|
||||
def warn_claude_strict_tools_dropped(dropped_count: int) -> None:
|
||||
litellm.verbose_logger.warning(
|
||||
"Claude rejected the request's strict schemas, so litellm removed `strict` from every tool (%d had "
|
||||
"strict true) and sent the request again because drop_params is set.",
|
||||
dropped_count,
|
||||
)
|
||||
|
||||
|
||||
def strip_thinking_blocks_from_anthropic_messages(messages: Sequence[object]) -> list[object]:
|
||||
"""
|
||||
Return a new message list with thinking / redacted_thinking content blocks removed
|
||||
|
|
|
|||
|
|
@ -160,6 +160,7 @@ async def prepare_context_managed_request(
|
|||
"""Apply client compaction history, then optional context_management polyfill."""
|
||||
from litellm.llms.anthropic.messages.context_management.editors.compact import (
|
||||
apply_client_compaction_block_history,
|
||||
slice_around_compaction_block,
|
||||
)
|
||||
|
||||
# Skip the client-history pre-processing when a ``compact_20260112``
|
||||
|
|
@ -175,7 +176,7 @@ async def prepare_context_managed_request(
|
|||
|
||||
if polyfill_will_run:
|
||||
history_result: PolyfillResult | None = None
|
||||
working_messages: _AnthropicMessages = messages
|
||||
working_messages: _AnthropicMessages = list(slice_around_compaction_block(messages)[0])
|
||||
working_system: _AnthropicSystem = system
|
||||
else:
|
||||
history_result = apply_client_compaction_block_history(
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ from litellm._logging import verbose_logger
|
|||
from litellm.litellm_core_utils.asyncify import asyncify
|
||||
from litellm.types.llms.anthropic import AppliedEdit
|
||||
|
||||
from ..convertible_blocks import drop_non_string_tool_result_texts
|
||||
from .constants import CLEAR_TOOL_USES_EDIT_TYPE, COMPACT_EDIT_TYPE
|
||||
from .editors import (
|
||||
CheckedEdit,
|
||||
|
|
@ -103,7 +104,7 @@ async def apply_context_management(
|
|||
|
||||
Every ``clear_tool_uses_20250919`` edit is validated before any editor runs, so a malformed one raises its 400
|
||||
before a ``compact_20260112`` summary call is paid for, or runs without its malformed fields under
|
||||
``drop_invalid_edits``.
|
||||
``drop_invalid_edits``, which also drops non-string ``tool_result`` text first, as the provider never receives it.
|
||||
|
||||
The dispatcher is async so async editors (``compact_20260112``) can
|
||||
``await`` the configured summarization model. Sync editors run in a
|
||||
|
|
@ -119,7 +120,7 @@ async def apply_context_management(
|
|||
raise first_error
|
||||
well_formed_edits: Final = tuple(check.edit for check in checked)
|
||||
|
||||
current_messages = messages
|
||||
current_messages = list(drop_non_string_tool_result_texts(messages, drop_invalid_edits))
|
||||
current_system = system
|
||||
aggregated_applied: Final[list[AppliedEdit]] = []
|
||||
aggregated_compaction_block = None
|
||||
|
|
|
|||
|
|
@ -39,6 +39,21 @@ if TYPE_CHECKING:
|
|||
T = TypeVar("T")
|
||||
# api.anthropic.com, measured 2026-10-19: text results of 1 to 46 bytes are never cleared or counted toward keep
|
||||
_TINY_RESULT_BYTES: Final = 46
|
||||
_TOKEN_COUNTER_BLOCK_TYPES: Final = frozenset(
|
||||
{
|
||||
"text",
|
||||
"image_url",
|
||||
"image",
|
||||
"document",
|
||||
"file",
|
||||
"tool_use",
|
||||
"tool_result",
|
||||
"thinking",
|
||||
"redacted_thinking",
|
||||
"tool_reference",
|
||||
"search_result",
|
||||
}
|
||||
)
|
||||
ToolNames = Literal["all"] | frozenset[str]
|
||||
|
||||
|
||||
|
|
@ -52,16 +67,19 @@ class _ClearAtLeast(BaseModel):
|
|||
class _ClearToolUsesFields(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
trigger: Mapping[str, object] | None = None
|
||||
keep: Mapping[str, object] | None = None
|
||||
clear_at_least: _ClearAtLeast | None = None
|
||||
exclude_tools: tuple[str, ...] | None = None
|
||||
clear_tool_inputs: bool | tuple[str, ...] | None = None
|
||||
|
||||
|
||||
_BLOCKS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
|
||||
# api.anthropic.com, measured 2026-10-19: its wording for the two shapes pydantic words differently
|
||||
# api.anthropic.com, measured 2026-10-19 and 2026-10-28: its wording for the shapes pydantic words differently
|
||||
_API_MESSAGES: Final = MappingProxyType(
|
||||
{
|
||||
"tuple_type": "Input should be a valid array",
|
||||
"dict_type": "Input should be an object",
|
||||
"model_type": "Input does not match the expected shape.",
|
||||
}
|
||||
)
|
||||
|
|
@ -178,10 +196,12 @@ def _cleared_result(block: Mapping[str, object]) -> dict[str, object]:
|
|||
|
||||
def _freed_tokens(model: str, result: Mapping[str, object], use: Mapping[str, object], clears_input: bool) -> int:
|
||||
"""Tokens that clearing this use frees: its result, and its input when ``clear_tool_inputs`` covers it."""
|
||||
before: Final = [result, use] if clears_input else [result]
|
||||
after: Final = [_cleared_result(result), {**use, "input": {}}] if clears_input else [_cleared_result(result)]
|
||||
before: Final[Sequence[Mapping[str, object]]] = (result, use) if clears_input else (result,)
|
||||
after: Final[Sequence[Mapping[str, object]]] = (
|
||||
(_cleared_result(result), {**use, "input": {}}) if clears_input else (_cleared_result(result),)
|
||||
)
|
||||
counts: Final = tuple(
|
||||
litellm.token_counter(model=model, messages=[{"role": "user", "content": blocks}]) for blocks in (before, after)
|
||||
_count_tokens(model, [{"role": "user", "content": blocks}], None) for blocks in (before, after)
|
||||
)
|
||||
return max(counts[0] - counts[1], 0)
|
||||
|
||||
|
|
@ -199,6 +219,35 @@ def _select_in_batches(candidates: Sequence[str], tokens: Mapping[str, int], cle
|
|||
return tuple(candidates[:batch_end])
|
||||
|
||||
|
||||
def _without_uncountable_blocks(block: Mapping[str, object]) -> Mapping[str, object]:
|
||||
nested: Final = _parse(_BLOCKS, block.get("content"))
|
||||
if nested is None:
|
||||
return block
|
||||
return {**block, "content": [inner for inner in nested if inner.get("type") in _TOKEN_COUNTER_BLOCK_TYPES]}
|
||||
|
||||
|
||||
def _countable_message(message: Mapping[str, object]) -> Mapping[str, object]:
|
||||
blocks: Final = _parse(_BLOCKS, message.get("content"))
|
||||
if blocks is None:
|
||||
return message
|
||||
return {
|
||||
**message,
|
||||
"content": [
|
||||
_without_uncountable_blocks(block) for block in blocks if block.get("type") in _TOKEN_COUNTER_BLOCK_TYPES
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _count_tokens(
|
||||
model: str, messages: Sequence[Mapping[str, object]], tools: Sequence[Mapping[str, object]] | None
|
||||
) -> int:
|
||||
return litellm.token_counter(
|
||||
model=model,
|
||||
messages=[_countable_message(message) for message in messages],
|
||||
tools=cast("list[ChatCompletionToolParam] | None", tools),
|
||||
)
|
||||
|
||||
|
||||
def _trigger_met(
|
||||
trigger: dict[str, object],
|
||||
model: str,
|
||||
|
|
@ -218,11 +267,7 @@ def _trigger_met(
|
|||
|
||||
if not isinstance(threshold, int):
|
||||
threshold = DEFAULT_INPUT_TOKENS_TRIGGER
|
||||
message_tokens: Final = litellm.token_counter(
|
||||
model=model,
|
||||
messages=messages,
|
||||
tools=cast("list[ChatCompletionToolParam] | None", tools),
|
||||
)
|
||||
message_tokens: Final = _count_tokens(model, messages, tools)
|
||||
if message_tokens > threshold:
|
||||
return True, message_tokens
|
||||
current_tokens: Final = message_tokens + count_system_tokens(model, system)
|
||||
|
|
@ -375,17 +420,9 @@ def apply_clear_tool_uses_20250919(
|
|||
[_message_without_inputs(message, ids_to_clear, input_tools) for message in edited] if input_tools else edited
|
||||
)
|
||||
|
||||
tokens_before: Final = (
|
||||
counted_tokens
|
||||
if counted_tokens is not None
|
||||
else litellm.token_counter(
|
||||
model=model, messages=kept, tools=cast("list[ChatCompletionToolParam] | None", tools)
|
||||
)
|
||||
)
|
||||
tokens_before: Final = counted_tokens if counted_tokens is not None else _count_tokens(model, kept, tools)
|
||||
kept_after, _ = history_after_client_compaction(final_messages, system)
|
||||
tokens_after: Final = litellm.token_counter(
|
||||
model=model, messages=kept_after, tools=cast("list[ChatCompletionToolParam] | None", tools)
|
||||
)
|
||||
tokens_after: Final = _count_tokens(model, kept_after, tools)
|
||||
cleared_input_tokens: Final = max(tokens_before - tokens_after, 0)
|
||||
|
||||
return final_messages, {
|
||||
|
|
|
|||
|
|
@ -137,7 +137,7 @@ def _find_latest_compaction_index(
|
|||
return None, None
|
||||
|
||||
|
||||
def _slice_around_compaction_block(
|
||||
def slice_around_compaction_block(
|
||||
messages: Sequence[_MsgT],
|
||||
) -> tuple[Sequence[_MsgT | dict[str, object]], dict[str, object] | None]:
|
||||
"""Apply Anthropic's "drop everything before the compaction block" rule.
|
||||
|
|
@ -194,7 +194,7 @@ def history_after_client_compaction(
|
|||
system: str | Sequence[Mapping[str, object]] | None,
|
||||
) -> tuple[list[dict[str, object]], str | Sequence[Mapping[str, object]] | None]:
|
||||
"""Return the history the API keeps after the latest compaction block, without it, and the system with its summary."""
|
||||
sliced, block = _slice_around_compaction_block(messages)
|
||||
sliced, block = slice_around_compaction_block(messages)
|
||||
if block is None:
|
||||
return list(messages), system
|
||||
tail: Final = _strip_compaction_blocks(sliced)
|
||||
|
|
@ -615,7 +615,7 @@ def apply_client_compaction_block_history(
|
|||
tail is forwarded unchanged (with compaction blocks stripped) so recent
|
||||
turns the summary does not cover are preserved.
|
||||
"""
|
||||
effective_messages, prior_compaction_block = _slice_around_compaction_block(messages)
|
||||
effective_messages, prior_compaction_block = slice_around_compaction_block(messages)
|
||||
if prior_compaction_block is None:
|
||||
return None
|
||||
|
||||
|
|
@ -686,7 +686,7 @@ async def apply_compact_20260112(
|
|||
# opt-in gate below so that even when summarization is disabled we still
|
||||
# strip Anthropic-only ``compaction`` blocks from messages going to
|
||||
# non-Anthropic backends (which would reject them).
|
||||
effective_messages, prior_compaction_block = _slice_around_compaction_block(messages)
|
||||
effective_messages, prior_compaction_block = slice_around_compaction_block(messages)
|
||||
prior_summary_text: Final = prior_compaction_block.get("content") if prior_compaction_block else None
|
||||
augmented_system: str | list[dict[str, object]] | None = system
|
||||
if isinstance(prior_summary_text, str) and prior_summary_text:
|
||||
|
|
|
|||
|
|
@ -10,11 +10,15 @@ from typing import Final, TypeAlias
|
|||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.integrations.anthropic_cache_control_hook import (
|
||||
AnthropicCacheControlHook,
|
||||
supports_openai_prompt_cache_breakpoint,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.litellm_core_utils.asyncify import (
|
||||
run_async_function, # pyright: ignore[reportUnknownVariableType] # untyped helper
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
cache_control_as_prompt_cache_breakpoint,
|
||||
prompt_cache_breakpoint_for_wire,
|
||||
|
|
@ -28,14 +32,19 @@ from litellm.types.llms.anthropic import (
|
|||
AnthropicMessagesRequest,
|
||||
AnthropicOutputConfig,
|
||||
AnthropicOutputSchema,
|
||||
ContextManagementResponse,
|
||||
)
|
||||
from litellm.types.llms.anthropic_messages.anthropic_response import (
|
||||
AnthropicMessagesResponse,
|
||||
)
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
from litellm.types.utils import CountContextManagement
|
||||
from litellm.utils import ProviderConfigManager
|
||||
|
||||
from ..adapters.handler import request_safeguards_evaluator
|
||||
from ..adapters.handler import context_management_explicitly_dropped, request_safeguards_evaluator
|
||||
from ..context_management import AnthropicContextManagementError, PolyfillResult, apply_context_management
|
||||
from ..context_management.constants import COMPACT_EDIT_TYPE
|
||||
from ..context_management.dispatcher import normalize_spec
|
||||
from ..convertible_blocks import reject_malformed_tool_result_text
|
||||
from ..safeguards import with_safeguard_results
|
||||
from ..utils import litellm_logging_obj_from_kwargs, local_model_name
|
||||
|
|
@ -45,10 +54,111 @@ from .transformation import LiteLLMAnthropicToResponsesAPIAdapter
|
|||
AnthropicRequestMessages: TypeAlias = list[AllAnthropicMessageValues] | list[dict[str, object]]
|
||||
BlockRewrite: TypeAlias = Callable[[Mapping[str, object]], Mapping[str, object]]
|
||||
_BLOCK_LIST: Final = TypeAdapter(list[dict[str, object]])
|
||||
_SYSTEM_ITEMS: Final = TypeAdapter(list[object])
|
||||
_SPEC: Final = TypeAdapter(dict[str, object])
|
||||
_SplitEdits: TypeAlias = tuple[dict[str, object] | None, Sequence[Mapping[str, object]]]
|
||||
AnthropicContextManagementSpec: TypeAlias = dict[str, object] | list[dict[str, object]] | None
|
||||
|
||||
_ADAPTER: Final = LiteLLMAnthropicToResponsesAPIAdapter()
|
||||
|
||||
|
||||
def _split_context_management(
|
||||
spec: AnthropicContextManagementSpec,
|
||||
kwargs: Mapping[str, object],
|
||||
) -> _SplitEdits:
|
||||
"""Return (the edits the Responses API runs natively, the edits the gateway polyfill runs).
|
||||
|
||||
The Responses API compacts natively (``compact_20260112`` becomes its ``compaction``); any other
|
||||
edit, such as ``clear_tool_uses_20250919``, has no Responses API form and runs in the polyfill, as
|
||||
it does on the chat completions path. An OpenAI-style list spec is not Anthropic's shape and stays
|
||||
unsent, as it was before the polyfill ran here.
|
||||
"""
|
||||
if not isinstance(spec, dict):
|
||||
return None, ()
|
||||
if not spec or context_management_explicitly_dropped(kwargs.get("additional_drop_params")):
|
||||
return spec, ()
|
||||
edits: Final = normalize_spec(spec) or []
|
||||
native: Final = [edit for edit in edits if edit.get("type") == COMPACT_EDIT_TYPE]
|
||||
return ({"edits": native} if native else None), tuple(
|
||||
edit for edit in edits if edit.get("type") != COMPACT_EDIT_TYPE
|
||||
)
|
||||
|
||||
|
||||
def _system_blocks(system: object) -> str | Sequence[Mapping[str, object]] | None:
|
||||
if system is None or isinstance(system, str):
|
||||
return system
|
||||
try:
|
||||
items: Final = _SYSTEM_ITEMS.validate_python(system)
|
||||
except ValidationError:
|
||||
return None
|
||||
return tuple(block for block in map(as_str_mapping, items) if block is not None)
|
||||
|
||||
|
||||
async def _polyfilled(
|
||||
*,
|
||||
model: str,
|
||||
messages: AnthropicRequestMessages,
|
||||
tools: Sequence[AllAnthropicToolsValues | Mapping[str, object]] | None,
|
||||
system: object,
|
||||
edits: Sequence[Mapping[str, object]],
|
||||
kwargs: Mapping[str, object],
|
||||
) -> PolyfillResult | None:
|
||||
"""Run the edits the Responses API can't run natively; raise like the chat completions path does."""
|
||||
if not edits:
|
||||
return None
|
||||
reject_malformed_tool_result_text(messages, model, kwargs)
|
||||
drop_invalid_edits: Final = drops_unsupported_params(kwargs.get("drop_params"))
|
||||
blocks: Final = _system_blocks(system)
|
||||
try:
|
||||
return await apply_context_management(
|
||||
model=model,
|
||||
messages=_BLOCK_LIST.validate_python(messages),
|
||||
tools=_BLOCK_LIST.validate_python(tools) if tools else None,
|
||||
system=blocks if blocks is None or isinstance(blocks, str) else _BLOCK_LIST.validate_python(blocks),
|
||||
context_management_spec={"edits": [dict(edit) for edit in edits]},
|
||||
drop_invalid_edits=drop_invalid_edits,
|
||||
)
|
||||
except AnthropicContextManagementError:
|
||||
raise
|
||||
except Exception as e:
|
||||
verbose_logger.exception("context_management polyfill failed on the Responses API path: %s", e)
|
||||
raise AnthropicContextManagementError(
|
||||
status_code=500, message=f"context_management polyfill failed: {e}"
|
||||
) from e
|
||||
|
||||
|
||||
async def count_view_messages(
|
||||
*,
|
||||
model: str,
|
||||
messages: Sequence[Mapping[str, object]],
|
||||
tools: Sequence[Mapping[str, object]] | None,
|
||||
system: object,
|
||||
context_management: CountContextManagement | None,
|
||||
) -> Sequence[Mapping[str, object]]:
|
||||
"""The messages /v1/messages sends the Responses API once the polyfill has run the edits it can't run natively."""
|
||||
if context_management is None or not isinstance(context_management.spec, dict):
|
||||
return messages
|
||||
_, edits = _split_context_management(_SPEC.validate_python(context_management.spec), {})
|
||||
result: Final = await _polyfilled(
|
||||
model=model,
|
||||
messages=_BLOCK_LIST.validate_python(messages),
|
||||
tools=tools,
|
||||
system=system,
|
||||
edits=edits,
|
||||
kwargs={"drop_params": context_management.drop_invalid_edits},
|
||||
)
|
||||
return messages if result is None else result.messages
|
||||
|
||||
|
||||
def _with_applied_edits(
|
||||
response: AnthropicMessagesResponse, polyfill_result: PolyfillResult | None
|
||||
) -> AnthropicMessagesResponse:
|
||||
applied_edits: Final = polyfill_result.applied_edits_for_response() if polyfill_result else None
|
||||
if not applied_edits:
|
||||
return response
|
||||
return {**response, "context_management": ContextManagementResponse(applied_edits=list(applied_edits))}
|
||||
|
||||
|
||||
def _record_bridged_response(responses_kwargs: Mapping[str, object], result: ResponsesAPIResponse) -> None:
|
||||
logging_obj: Final = litellm_logging_obj_from_kwargs(responses_kwargs)
|
||||
if logging_obj is not None:
|
||||
|
|
@ -281,7 +391,7 @@ class LiteLLMMessagesToResponsesAPIHandler:
|
|||
max_tokens: int,
|
||||
messages: AnthropicRequestMessages,
|
||||
model: str,
|
||||
context_management: dict[str, object] | None = None,
|
||||
context_management: AnthropicContextManagementSpec = None,
|
||||
metadata: dict[str, object] | None = None,
|
||||
output_config: AnthropicOutputConfig | None = None,
|
||||
stop_sequences: list[str] | None = None,
|
||||
|
|
@ -296,11 +406,15 @@ class LiteLLMMessagesToResponsesAPIHandler:
|
|||
output_format: AnthropicOutputSchema | None = None,
|
||||
**kwargs: object,
|
||||
) -> AnthropicMessagesResponse | AsyncIterator[bytes]:
|
||||
native_context_management, polyfilled_edits = _split_context_management(context_management, kwargs)
|
||||
polyfill_result: Final = await _polyfilled(
|
||||
model=model, messages=messages, tools=tools, system=system, edits=polyfilled_edits, kwargs=kwargs
|
||||
)
|
||||
responses_kwargs: Final = _build_responses_kwargs(
|
||||
max_tokens=max_tokens,
|
||||
messages=messages,
|
||||
messages=polyfill_result.messages if polyfill_result is not None else messages,
|
||||
model=model,
|
||||
context_management=context_management,
|
||||
context_management=native_context_management,
|
||||
metadata=metadata,
|
||||
output_config=output_config,
|
||||
stop_sequences=stop_sequences,
|
||||
|
|
@ -325,13 +439,16 @@ class LiteLLMMessagesToResponsesAPIHandler:
|
|||
model=local_model_name(model, kwargs.get("custom_llm_provider")),
|
||||
litellm_logging_obj=litellm_logging_obj_from_kwargs(responses_kwargs),
|
||||
safeguards_evaluator=safeguards_evaluator,
|
||||
applied_edits=polyfill_result.applied_edits_for_response() if polyfill_result else None,
|
||||
)
|
||||
return wrapper.async_anthropic_sse_wrapper()
|
||||
|
||||
if not isinstance(result, ResponsesAPIResponse):
|
||||
raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}")
|
||||
|
||||
served: Final = await with_safeguard_results(_ADAPTER.translate_response(result), safeguards_evaluator)
|
||||
served: Final = await with_safeguard_results(
|
||||
_with_applied_edits(_ADAPTER.translate_response(result), polyfill_result), safeguards_evaluator
|
||||
)
|
||||
_record_bridged_response(responses_kwargs, result)
|
||||
return served
|
||||
|
||||
|
|
@ -340,7 +457,7 @@ class LiteLLMMessagesToResponsesAPIHandler:
|
|||
max_tokens: int,
|
||||
messages: AnthropicRequestMessages,
|
||||
model: str,
|
||||
context_management: dict[str, object] | None = None,
|
||||
context_management: AnthropicContextManagementSpec = None,
|
||||
metadata: dict[str, object] | None = None,
|
||||
output_config: AnthropicOutputConfig | None = None,
|
||||
stop_sequences: list[str] | None = None,
|
||||
|
|
@ -382,11 +499,26 @@ class LiteLLMMessagesToResponsesAPIHandler:
|
|||
)
|
||||
|
||||
# Sync path
|
||||
native_context_management, polyfilled_edits = _split_context_management(context_management, kwargs)
|
||||
ran: Final[object] = ( # pyright: ignore[reportUnknownVariableType] # run_async_function is untyped
|
||||
run_async_function(
|
||||
_polyfilled,
|
||||
model=model,
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
system=system,
|
||||
edits=polyfilled_edits,
|
||||
kwargs=kwargs,
|
||||
)
|
||||
if polyfilled_edits
|
||||
else None
|
||||
)
|
||||
polyfill_result: Final = ran if isinstance(ran, PolyfillResult) else None
|
||||
responses_kwargs: Final = _build_responses_kwargs(
|
||||
max_tokens=max_tokens,
|
||||
messages=messages,
|
||||
messages=polyfill_result.messages if polyfill_result is not None else messages,
|
||||
model=model,
|
||||
context_management=context_management,
|
||||
context_management=native_context_management,
|
||||
metadata=metadata,
|
||||
output_config=output_config,
|
||||
stop_sequences=stop_sequences,
|
||||
|
|
@ -409,10 +541,11 @@ class LiteLLMMessagesToResponsesAPIHandler:
|
|||
responses_stream=result,
|
||||
model=local_model_name(model, kwargs.get("custom_llm_provider")),
|
||||
litellm_logging_obj=litellm_logging_obj_from_kwargs(responses_kwargs),
|
||||
applied_edits=polyfill_result.applied_edits_for_response() if polyfill_result else None,
|
||||
)
|
||||
return wrapper.async_anthropic_sse_wrapper()
|
||||
|
||||
if not isinstance(result, ResponsesAPIResponse):
|
||||
raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}")
|
||||
|
||||
return _ADAPTER.translate_response(result)
|
||||
return _with_applied_edits(_ADAPTER.translate_response(result), polyfill_result)
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
## Translates OpenAI call to Anthropic `/v1/messages` format
|
||||
import asyncio
|
||||
from collections import deque
|
||||
from collections.abc import AsyncGenerator, Awaitable, Callable, Iterator, Mapping
|
||||
from collections.abc import AsyncGenerator, Awaitable, Callable, Iterator, Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
import anyio
|
||||
|
|
@ -27,6 +27,7 @@ from litellm.responses.streaming_iterator import (
|
|||
partial_responses_incomplete_event_on_disconnect,
|
||||
stream_error_status_and_message,
|
||||
)
|
||||
from litellm.types.llms.anthropic import AppliedEdit, ContextManagementResponse
|
||||
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
|
||||
|
|
@ -129,8 +130,10 @@ class AnthropicResponsesStreamWrapper:
|
|||
model: str,
|
||||
litellm_logging_obj: "LiteLLMLoggingObject | None" = None,
|
||||
safeguards_evaluator: "SafeguardsEvaluator | None" = None,
|
||||
applied_edits: Sequence[AppliedEdit] | None = None,
|
||||
) -> None:
|
||||
self.responses_stream = responses_stream
|
||||
self._applied_edits: Final = tuple(applied_edits or ())
|
||||
self.model = model
|
||||
self.litellm_logging_obj = litellm_logging_obj
|
||||
self.safeguards_evaluator: Final = safeguards_evaluator
|
||||
|
|
@ -445,6 +448,11 @@ class AnthropicResponsesStreamWrapper:
|
|||
"type": "message_delta",
|
||||
"delta": message_delta_payload,
|
||||
"usage": dict(anthropic_usage),
|
||||
**(
|
||||
{"context_management": ContextManagementResponse(applied_edits=list(self._applied_edits))}
|
||||
if self._applied_edits
|
||||
else {}
|
||||
),
|
||||
}
|
||||
)
|
||||
self._chunk_queue.append({"type": "message_stop"})
|
||||
|
|
|
|||
|
|
@ -1,7 +1,12 @@
|
|||
from typing import TYPE_CHECKING, Any, Final
|
||||
import math
|
||||
from collections.abc import Mapping
|
||||
from typing import TYPE_CHECKING, Annotated, Final, TypeAlias
|
||||
|
||||
import httpx
|
||||
from pydantic import BeforeValidator, ConfigDict, TypeAdapter, ValidationError, with_config
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.exceptions import UnsupportedParamsError
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.base_llm.image_generation.transformation import (
|
||||
|
|
@ -18,6 +23,67 @@ if TYPE_CHECKING:
|
|||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
|
||||
_TOKEN_COUNT_ADAPTER: Final[TypeAdapter[int | float]] = TypeAdapter(int | float)
|
||||
|
||||
|
||||
def _token_count_or_none(value: object) -> int | float | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
count: Final = _TOKEN_COUNT_ADAPTER.validate_python(value)
|
||||
except ValidationError:
|
||||
verbose_logger.warning("MAI image usage has a token count that is not a number, counting it as absent")
|
||||
return None
|
||||
if not math.isfinite(count):
|
||||
verbose_logger.warning("MAI image usage has a token count that is not finite, counting it as absent")
|
||||
return None
|
||||
return count
|
||||
|
||||
|
||||
_TokenCount: TypeAlias = Annotated[int | float | None, BeforeValidator(_token_count_or_none)]
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow"))
|
||||
class _MAIImageUsageDetails(TypedDict, total=False):
|
||||
text_tokens: ReadOnly[NotRequired[_TokenCount]]
|
||||
image_tokens: ReadOnly[NotRequired[_TokenCount]]
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow"))
|
||||
class _MAIImageUsage(TypedDict, total=False):
|
||||
input_tokens_details: ReadOnly[NotRequired[object]]
|
||||
num_input_text_tokens: ReadOnly[NotRequired[_TokenCount]]
|
||||
num_input_image_tokens: ReadOnly[NotRequired[_TokenCount]]
|
||||
input_tokens: ReadOnly[NotRequired[_TokenCount]]
|
||||
output_tokens: ReadOnly[NotRequired[_TokenCount]]
|
||||
num_output_tokens: ReadOnly[NotRequired[_TokenCount]]
|
||||
output_image_tokens: ReadOnly[NotRequired[_TokenCount]]
|
||||
total_tokens: ReadOnly[NotRequired[_TokenCount]]
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow"))
|
||||
class _MAIImageResponse(TypedDict, total=False):
|
||||
usage: ReadOnly[NotRequired[object]]
|
||||
|
||||
|
||||
_MAI_IMAGE_USAGE_DETAILS_ADAPTER: Final[TypeAdapter[_MAIImageUsageDetails]] = TypeAdapter(_MAIImageUsageDetails)
|
||||
_MAI_IMAGE_USAGE_ADAPTER: Final[TypeAdapter[_MAIImageUsage]] = TypeAdapter(_MAIImageUsage)
|
||||
_MAI_RAW_USAGE_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
def _raw_usage_or_none(usage: object) -> Mapping[str, object] | None:
|
||||
if usage is None:
|
||||
return None
|
||||
try:
|
||||
return _MAI_RAW_USAGE_ADAPTER.validate_python(usage)
|
||||
except ValidationError:
|
||||
verbose_logger.warning("MAI image usage is not an object, counting it as absent")
|
||||
return None
|
||||
|
||||
|
||||
_MAI_IMAGE_RESPONSE_ADAPTER: Final[TypeAdapter[_MAIImageResponse]] = TypeAdapter(_MAIImageResponse)
|
||||
|
||||
|
||||
class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig):
|
||||
"""Azure AI Foundry MAI image generation (e.g. MAI-Image-2.5)."""
|
||||
|
||||
|
|
@ -78,9 +144,10 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig):
|
|||
return "maiimage" in model_normalized
|
||||
|
||||
@staticmethod
|
||||
def normalize_mai_image_usage(usage: dict[str, Any] | None) -> dict[str, Any]:
|
||||
def normalize_mai_image_usage(usage: object) -> Mapping[str, object]:
|
||||
"""Map Azure MAI usage fields to OpenAI ImageUsage schema."""
|
||||
if usage is None:
|
||||
raw_usage: Final = _raw_usage_or_none(usage)
|
||||
if raw_usage is None:
|
||||
return {
|
||||
"input_tokens": 0,
|
||||
"input_tokens_details": {"image_tokens": 0, "text_tokens": 0},
|
||||
|
|
@ -88,10 +155,13 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig):
|
|||
"total_tokens": 0,
|
||||
}
|
||||
|
||||
normalized_usage: Final = dict(usage)
|
||||
input_tokens_details = normalized_usage.get("input_tokens_details")
|
||||
if not isinstance(input_tokens_details, dict):
|
||||
input_tokens_details = {}
|
||||
normalized_usage: Final[_MAIImageUsage] = _MAI_IMAGE_USAGE_ADAPTER.validate_python(raw_usage)
|
||||
input_tokens_details_value: Final = normalized_usage.get("input_tokens_details")
|
||||
input_tokens_details: Final[_MAIImageUsageDetails] = (
|
||||
_MAI_IMAGE_USAGE_DETAILS_ADAPTER.validate_python(input_tokens_details_value)
|
||||
if isinstance(input_tokens_details_value, dict)
|
||||
else {}
|
||||
)
|
||||
|
||||
text_tokens = normalized_usage.get("num_input_text_tokens")
|
||||
if text_tokens is None:
|
||||
|
|
@ -121,18 +191,16 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig):
|
|||
if total_tokens is None:
|
||||
total_tokens = input_tokens + output_tokens
|
||||
|
||||
normalized_usage.update(
|
||||
{
|
||||
"input_tokens": input_tokens,
|
||||
"input_tokens_details": {
|
||||
"image_tokens": image_tokens,
|
||||
"text_tokens": text_tokens,
|
||||
},
|
||||
"output_tokens": output_tokens,
|
||||
"total_tokens": total_tokens,
|
||||
}
|
||||
)
|
||||
return normalized_usage
|
||||
return {
|
||||
**raw_usage,
|
||||
"input_tokens": input_tokens,
|
||||
"input_tokens_details": {
|
||||
"image_tokens": image_tokens,
|
||||
"text_tokens": text_tokens,
|
||||
},
|
||||
"output_tokens": output_tokens,
|
||||
"total_tokens": total_tokens,
|
||||
}
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list[OpenAIImageGenerationOptionalParams]:
|
||||
return ["n", "size"]
|
||||
|
|
@ -252,12 +320,18 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig):
|
|||
json_mode: bool | None = None,
|
||||
) -> ImageResponse:
|
||||
try:
|
||||
response: Final = raw_response.json()
|
||||
response_json: Final = _MAI_IMAGE_RESPONSE_ADAPTER.validate_python(raw_response.json())
|
||||
except Exception:
|
||||
raise OpenAIError(message=raw_response.text, status_code=raw_response.status_code)
|
||||
|
||||
if "usage" in response:
|
||||
response["usage"] = self.normalize_mai_image_usage(response.get("usage"))
|
||||
response: Final = (
|
||||
{
|
||||
**response_json,
|
||||
"usage": self.normalize_mai_image_usage(response_json["usage"]),
|
||||
}
|
||||
if "usage" in response_json
|
||||
else dict(response_json)
|
||||
)
|
||||
|
||||
logging_obj.post_call(
|
||||
input=request_data.get("prompt", ""),
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Final
|
|||
|
||||
import httpx
|
||||
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.types.llms.anthropic_messages.anthropic_response import (
|
||||
AnthropicMessagesResponse,
|
||||
)
|
||||
|
|
@ -201,11 +202,23 @@ class BaseAnthropicMessagesConfig(ABC):
|
|||
anthropic_request_errors_on_thinking_binding_mismatch,
|
||||
is_anthropic_invalid_thinking_block_error,
|
||||
is_anthropic_thinking_binding_mismatch_error,
|
||||
is_claude_strict_tools_rejection,
|
||||
tools_without_strict,
|
||||
validated_object_mapping,
|
||||
)
|
||||
|
||||
if e.response.status_code != 400:
|
||||
return False
|
||||
error_text: Final[str] = e.response.text
|
||||
body: Final = validated_object_mapping(request_body)
|
||||
drops_rejected_strict_tools: Final = (
|
||||
drops_unsupported_params(litellm_params.get("drop_params"))
|
||||
and is_claude_strict_tools_rejection(e.response.status_code, error_text)
|
||||
and body is not None
|
||||
and tools_without_strict(body.get("tools")) is not None
|
||||
)
|
||||
if drops_rejected_strict_tools:
|
||||
return True
|
||||
caller_asked_for_this_error: Final = anthropic_request_errors_on_thinking_binding_mismatch(
|
||||
request_body
|
||||
) and is_anthropic_thinking_binding_mismatch_error(error_text)
|
||||
|
|
@ -219,11 +232,22 @@ class BaseAnthropicMessagesConfig(ABC):
|
|||
"""
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
is_anthropic_invalid_thinking_block_error,
|
||||
is_claude_strict_tools_rejection,
|
||||
strip_thinking_blocks_from_anthropic_messages_request_dict,
|
||||
tools_without_strict,
|
||||
warn_claude_strict_tools_dropped,
|
||||
)
|
||||
|
||||
if e.response.status_code == 400 and is_anthropic_invalid_thinking_block_error(e.response.text):
|
||||
strip_thinking_blocks_from_anthropic_messages_request_dict(request_data)
|
||||
without_strict: Final = (
|
||||
tools_without_strict(request_data.get("tools"))
|
||||
if is_claude_strict_tools_rejection(e.response.status_code, e.response.text)
|
||||
else None
|
||||
)
|
||||
if without_strict is not None:
|
||||
warn_claude_strict_tools_dropped(without_strict.dropped_count)
|
||||
request_data["tools"] = list(without_strict.tools) # rebind-ok: the retry loop re-signs this same dict
|
||||
return request_data
|
||||
|
||||
def native_output_fallback_litellm_params(
|
||||
|
|
|
|||
|
|
@ -135,6 +135,18 @@ class BaseTextToSpeechConfig(ABC):
|
|||
Transform provider response to standard format
|
||||
"""
|
||||
|
||||
async def async_transform_text_to_speech_response(
|
||||
self,
|
||||
model: str,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> "HttpxBinaryResponseContent":
|
||||
return self.transform_text_to_speech_response(
|
||||
model=model,
|
||||
raw_response=raw_response,
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
|
||||
def get_error_class(self, error_message: str, status_code: int, headers: dict) -> BaseLLMException:
|
||||
from ..chat.transformation import BaseLLMException
|
||||
|
||||
|
|
|
|||
|
|
@ -2,11 +2,11 @@ from collections.abc import Callable, Coroutine
|
|||
from functools import partial
|
||||
from typing import Final, TypeAlias
|
||||
|
||||
from litellm.llms.anthropic.common_utils import is_claude_strict_tools_rejection
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.bedrock.common_utils import (
|
||||
NativeOutputFallback,
|
||||
is_bedrock_native_output_rejection,
|
||||
is_bedrock_strict_tools_rejection,
|
||||
warn_bedrock_native_output_rejected,
|
||||
warn_bedrock_strict_tools_dropped,
|
||||
)
|
||||
|
|
@ -34,7 +34,7 @@ def _native_output_retry(
|
|||
def _strict_tools_retry(
|
||||
error: BaseLLMException, fallback: NativeOutputFallbackFactory | None, model: str
|
||||
) -> NativeOutputFallback | None:
|
||||
if fallback is None or not is_bedrock_strict_tools_rejection(error.status_code, error.message):
|
||||
if fallback is None or not is_claude_strict_tools_rejection(error.status_code, error.message):
|
||||
return None
|
||||
retry: Final = fallback()
|
||||
if retry is not None:
|
||||
|
|
|
|||
|
|
@ -366,21 +366,10 @@ class NativeOutputFallback:
|
|||
litellm_params: Mapping[str, object]
|
||||
|
||||
|
||||
_BEDROCK_STRICT_TOOLS_REJECTION: Final = re.compile(
|
||||
r"Too many strict tools \(\d+\)"
|
||||
r"|Schemas contains too many (?:optional parameters|parameters with union types)"
|
||||
r"|[Cc]ompiled grammar (?:is too large|size)"
|
||||
r"|tools\.\d+\.custom(?:\.strict)?: "
|
||||
r"|Grammar compilation timed out"
|
||||
)
|
||||
_TOOLS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...])
|
||||
_TOOL_SPEC_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
def is_bedrock_strict_tools_rejection(status_code: int, error_text: str) -> bool:
|
||||
return status_code == 400 and _BEDROCK_STRICT_TOOLS_REJECTION.search(error_text) is not None
|
||||
|
||||
|
||||
def warn_bedrock_strict_tools_dropped(model: str) -> None:
|
||||
verbose_logger.warning(
|
||||
"Bedrock rejected the strict tool schemas for model=%s, so litellm sent the request again without `strict` "
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ from collections.abc import Callable, Mapping
|
|||
from typing import TYPE_CHECKING, Final, get_args, overload
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter
|
||||
from pydantic import ConfigDict, TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.constants import BEDROCK_EMBEDDING_PROVIDERS_LITERAL
|
||||
|
|
@ -48,6 +48,8 @@ from .amazon_titan_v2_transformation import AmazonTitanV2Config
|
|||
from .cohere_transformation import BedrockCohereEmbeddingConfig
|
||||
from .twelvelabs_marengo_transformation import TwelveLabsMarengoEmbeddingConfig, drop_params_enabled
|
||||
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True))
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
|
||||
|
|
@ -155,7 +157,7 @@ class BedrockEmbedding(BaseAWSLLM):
|
|||
except httpx.TimeoutException:
|
||||
raise BedrockError(status_code=408, message="Timeout error occurred.")
|
||||
|
||||
return response.json()
|
||||
return _JSON_OBJECT.validate_python(response.json())
|
||||
|
||||
async def _make_async_call(
|
||||
self,
|
||||
|
|
@ -189,7 +191,7 @@ class BedrockEmbedding(BaseAWSLLM):
|
|||
except httpx.TimeoutException:
|
||||
raise BedrockError(status_code=408, message="Timeout error occurred.")
|
||||
|
||||
return response.json()
|
||||
return _JSON_OBJECT.validate_python(response.json())
|
||||
|
||||
def _transform_response(
|
||||
self,
|
||||
|
|
@ -689,7 +691,7 @@ class BedrockEmbedding(BaseAWSLLM):
|
|||
|
||||
# Parse response
|
||||
if response.status_code == 200:
|
||||
return response.json()
|
||||
return _JSON_OBJECT.validate_python(response.json())
|
||||
else:
|
||||
raise Exception(f"Failed to get async invoke status: {response.status_code} - {response.text}")
|
||||
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ import httpx
|
|||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.base_llm.responses.codex_compat import drop_unsupported_tools, normalize_codex_input_items
|
||||
|
|
@ -196,7 +197,9 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
|
|||
params: Final = cast( # cast-ok: the base signature leaves the params dict untyped
|
||||
"ResponsesAPIOptionalRequestParams", response_api_optional_request_params
|
||||
)
|
||||
hoisted: Final = hoist_additional_tools(input, params.get("tools"))
|
||||
hoisted: Final = hoist_additional_tools(
|
||||
input, params.get("tools"), drop_params=drops_unsupported_params(litellm_params.drop_params)
|
||||
)
|
||||
normalized_input, rewritten_types = normalize_codex_input_items(hoisted.input)
|
||||
if rewritten_types:
|
||||
verbose_logger.warning(
|
||||
|
|
|
|||
95
litellm/llms/clf_ai_gateway/chat/transformation.py
Normal file
95
litellm/llms/clf_ai_gateway/chat/transformation.py
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
from typing import Final
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.exceptions import UnsupportedParamsError
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
from litellm.router_utils.reasoning_effort_capability import declared_reasoning_efforts_for_model
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
|
||||
SUPPORTED_OPENAI_PARAMS: Final = (
|
||||
"stream",
|
||||
"stream_options",
|
||||
"frequency_penalty",
|
||||
"presence_penalty",
|
||||
"logit_bias",
|
||||
"logprobs",
|
||||
"top_logprobs",
|
||||
"max_tokens",
|
||||
"max_completion_tokens",
|
||||
"n",
|
||||
"stop",
|
||||
"temperature",
|
||||
"top_p",
|
||||
"seed",
|
||||
"response_format",
|
||||
"tools",
|
||||
"tool_choice",
|
||||
"parallel_tool_calls",
|
||||
"user",
|
||||
"store",
|
||||
"service_tier",
|
||||
)
|
||||
|
||||
|
||||
class ClfAiGatewayConfig(OpenAIGPTConfig):
|
||||
@property
|
||||
def custom_llm_provider(self) -> str | None:
|
||||
return "clf_ai_gateway"
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: matches OpenAIGPTConfig's list return
|
||||
supports_reasoning: Final = litellm.supports_reasoning(
|
||||
model=model,
|
||||
custom_llm_provider=self.custom_llm_provider,
|
||||
)
|
||||
extra: Final = ("reasoning_effort",) if supports_reasoning else ()
|
||||
return [*SUPPORTED_OPENAI_PARAMS, *extra]
|
||||
|
||||
def map_openai_params(
|
||||
self,
|
||||
non_default_params: dict[str, object], # mutable-ok: OpenAIConfig.map_openai_params signature
|
||||
optional_params: dict[str, object], # mutable-ok: OpenAIConfig.map_openai_params signature
|
||||
model: str,
|
||||
drop_params: bool,
|
||||
) -> dict[str, object]: # mutable-ok: OpenAIConfig.map_openai_params signature
|
||||
requested: Final = non_default_params.get("reasoning_effort")
|
||||
requested_mapping: Final = as_str_mapping(requested)
|
||||
effort: Final = requested_mapping.get("effort") if requested_mapping is not None else requested
|
||||
declared: Final = declared_reasoning_efforts_for_model(model, "clf_ai_gateway")
|
||||
drop_enabled: Final = drop_params or litellm.drop_params is True
|
||||
if requested_mapping is not None and effort is None and not drop_enabled:
|
||||
raise UnsupportedParamsError(
|
||||
message=f"clf_ai_gateway needs an effort level in reasoning_effort, got {requested!r}. To drop "
|
||||
"unsupported openai params from the call, set `litellm.drop_params = True`",
|
||||
llm_provider="clf_ai_gateway",
|
||||
model=model,
|
||||
status_code=400,
|
||||
)
|
||||
drops_effort: Final = drop_enabled and effort is not None and declared is not None and effort not in declared
|
||||
if drops_effort:
|
||||
verbose_logger.warning(
|
||||
"Dropping reasoning_effort=%r for model=%r (drop_params=True) because clf_ai_gateway takes only %s for it.",
|
||||
effort,
|
||||
model,
|
||||
declared,
|
||||
)
|
||||
others: Final = {param: value for param, value in non_default_params.items() if param != "reasoning_effort"}
|
||||
normalized: Final = {**others, "reasoning_effort": effort} if effort is not None else others
|
||||
forwarded: Final = others if drops_effort else normalized
|
||||
mapped: Final[dict[str, object]] = super().map_openai_params( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # OpenAIGPTConfig types its params and result as bare dicts # mutable-ok: parent returns a dict
|
||||
non_default_params=forwarded,
|
||||
optional_params=optional_params,
|
||||
model=model,
|
||||
drop_params=drop_params,
|
||||
)
|
||||
return mapped
|
||||
|
||||
def get_openai_compatible_provider_info(
|
||||
self, api_base: str | None, api_key: str | None
|
||||
) -> tuple[str | None, str | None]:
|
||||
resolved_api_base: Final = (
|
||||
api_base or get_secret_str("CLF_AI_GATEWAY_API_BASE") or "https://api.clfaigateway.dev/v1"
|
||||
)
|
||||
dynamic_api_key: Final = api_key or get_secret_str("CLF_AI_GATEWAY_API_KEY")
|
||||
return resolved_api_base, dynamic_api_key
|
||||
|
|
@ -1205,6 +1205,17 @@ class AsyncHTTPHandler:
|
|||
# Mirrors LiteLLMAiohttpTransport._background_close_tasks.
|
||||
_finalizer_close_tasks: ClassVar[set["asyncio.Task[None]"]] = set() # mutable-ok: strong refs for pending closes
|
||||
|
||||
@classmethod
|
||||
async def _close_client_at_finalization(cls, client: httpx.AsyncClient) -> None:
|
||||
try:
|
||||
await client.aclose()
|
||||
except (OSError, RuntimeError) as exc:
|
||||
verbose_logger.debug("Error closing client at finalization: %s", exc)
|
||||
finally:
|
||||
task: Final = asyncio.current_task()
|
||||
if task is not None:
|
||||
cls._finalizer_close_tasks.discard(task)
|
||||
|
||||
@classmethod
|
||||
def _on_finalizer_close_done(cls, task: "asyncio.Task[None]") -> None:
|
||||
cls._finalizer_close_tasks.discard(task)
|
||||
|
|
@ -1272,8 +1283,8 @@ class AsyncHTTPHandler:
|
|||
# here is the cross-loop path the transport refuses.
|
||||
self._dispose_wrapped_aiohttp_session()
|
||||
return
|
||||
task: Final = loop.create_task(self._client.aclose())
|
||||
cls: Final = type(self)
|
||||
task: Final = loop.create_task(cls._close_client_at_finalization(self._client))
|
||||
cls._finalizer_close_tasks.add(task)
|
||||
task.add_done_callback(cls._on_finalizer_close_done)
|
||||
except Exception:
|
||||
|
|
@ -1973,7 +1984,7 @@ def get_httpx_client(params: dict | None = None) -> HTTPHandler:
|
|||
cache = LLMClientCache()
|
||||
setattr(litellm, "in_memory_llm_clients_cache", cache)
|
||||
|
||||
_cached_client: Final = cache.get_cache(_cache_key_name)
|
||||
_cached_client: Final = cache.get_cache(_cache_key_name, loop_bound=False)
|
||||
if _cached_client:
|
||||
return _cached_client
|
||||
|
||||
|
|
@ -1989,6 +2000,7 @@ def get_httpx_client(params: dict | None = None) -> HTTPHandler:
|
|||
value=_new_client,
|
||||
ttl=_DEFAULT_TTL_FOR_HTTPX_CLIENTS,
|
||||
litellm_owned_client=True,
|
||||
loop_bound=False,
|
||||
)
|
||||
return _new_client
|
||||
|
||||
|
|
|
|||
|
|
@ -2467,8 +2467,7 @@ class BaseLLMHTTPHandler:
|
|||
)
|
||||
else:
|
||||
verbose_logger.debug(
|
||||
"Anthropic /v1/messages: invalid thinking signature; "
|
||||
"stripping thinking blocks and retrying (attempt %s/%s).",
|
||||
"Anthropic /v1/messages: retrying after a recoverable HTTP error (attempt %s/%s).",
|
||||
len(recovered_errors) + 2,
|
||||
max_attempts,
|
||||
)
|
||||
|
|
@ -12850,7 +12849,7 @@ class BaseLLMHTTPHandler:
|
|||
provider_config=text_to_speech_provider_config,
|
||||
)
|
||||
|
||||
speech_response: Final = text_to_speech_provider_config.transform_text_to_speech_response(
|
||||
speech_response: Final = await text_to_speech_provider_config.async_transform_text_to_speech_response(
|
||||
model=model,
|
||||
raw_response=response,
|
||||
logging_obj=logging_obj,
|
||||
|
|
|
|||
|
|
@ -24,7 +24,8 @@ DASHSCOPE_ANTHROPIC_PATH: Final = "/apps/anthropic"
|
|||
DASHSCOPE_NATIVE_V1_PATH: Final = "/api/v1"
|
||||
DASHSCOPE_COMPATIBLE_API_PATH: Final = "/compatible-api/v1"
|
||||
DASHSCOPE_RERANK_PATH: Final = f"{DASHSCOPE_COMPATIBLE_API_PATH}/reranks"
|
||||
DASHSCOPE_IMAGE_PATH: Final = "/api/v1/services/aigc/multimodal-generation/generation"
|
||||
DASHSCOPE_MULTIMODAL_GENERATION_PATH: Final = "/api/v1/services/aigc/multimodal-generation/generation"
|
||||
DASHSCOPE_IMAGE_PATH: Final = DASHSCOPE_MULTIMODAL_GENERATION_PATH
|
||||
DASHSCOPE_DEFAULT_ORIGIN: Final = "https://dashscope.aliyuncs.com"
|
||||
|
||||
_STANDARD_DASHSCOPE_HOSTS: Final = frozenset(
|
||||
|
|
@ -100,6 +101,21 @@ def is_dashscope_native_endpoint_url(api_base: str) -> bool:
|
|||
return urlparse(api_base).path.rstrip("/").startswith(f"{DASHSCOPE_NATIVE_V1_PATH}/")
|
||||
|
||||
|
||||
def dashscope_api_root(api_base: str) -> str:
|
||||
endpoints: Final = resolve_dashscope_endpoints(api_base)
|
||||
if endpoints is not None:
|
||||
return endpoints.origin
|
||||
trimmed: Final = api_base.rstrip("/")
|
||||
if trimmed.endswith(DASHSCOPE_CHAT_COMPATIBLE_PATH):
|
||||
return trimmed.removesuffix(DASHSCOPE_CHAT_COMPATIBLE_PATH)
|
||||
root, separator, _ = trimmed.partition(DASHSCOPE_NATIVE_V1_PATH)
|
||||
return root if separator else trimmed
|
||||
|
||||
|
||||
def is_aliyun_host(host: str | None) -> bool:
|
||||
return host is not None and (host == "aliyuncs.com" or host.endswith(".aliyuncs.com"))
|
||||
|
||||
|
||||
def resolve_dashscope_openai_api_base(
|
||||
api_base: str | None,
|
||||
env_var: str = "DASHSCOPE_API_BASE",
|
||||
|
|
@ -140,9 +156,7 @@ def _chat_shaped_aliyun_origin(api_base: str | None) -> str | None:
|
|||
if api_base is None:
|
||||
return None
|
||||
parsed: Final = urlparse(api_base)
|
||||
host: Final = parsed.hostname or ""
|
||||
on_aliyun_host: Final = host == "aliyuncs.com" or host.endswith(".aliyuncs.com")
|
||||
if not on_aliyun_host or parsed.path.rstrip("/") != DASHSCOPE_CHAT_COMPATIBLE_PATH:
|
||||
if not is_aliyun_host(parsed.hostname) or parsed.path.rstrip("/") != DASHSCOPE_CHAT_COMPATIBLE_PATH:
|
||||
return None
|
||||
return f"{parsed.scheme}://{parsed.netloc}"
|
||||
|
||||
|
|
|
|||
0
litellm/llms/dashscope/image_edit/__init__.py
Normal file
0
litellm/llms/dashscope/image_edit/__init__.py
Normal file
160
litellm/llms/dashscope/image_edit/transformation.py
Normal file
160
litellm/llms/dashscope/image_edit/transformation.py
Normal file
|
|
@ -0,0 +1,160 @@
|
|||
from collections.abc import Mapping
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
import httpx
|
||||
from httpx._types import RequestFiles
|
||||
from pydantic import TypeAdapter
|
||||
from typing_extensions import override
|
||||
|
||||
import litellm
|
||||
from litellm.images.utils import image_to_data_url
|
||||
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
|
||||
from litellm.llms.dashscope.common_utils import (
|
||||
DASHSCOPE_CHAT_COMPATIBLE_PATH,
|
||||
DASHSCOPE_MULTIMODAL_GENERATION_PATH,
|
||||
dashscope_api_root,
|
||||
is_dashscope_native_endpoint_url,
|
||||
resolve_dashscope_endpoints,
|
||||
)
|
||||
from litellm.llms.dashscope.image_generation.transformation import (
|
||||
DEFAULT_API_BASE,
|
||||
OPENAI_TO_DASHSCOPE_SIZE,
|
||||
DashScopeImageGenerationConfig,
|
||||
)
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.images.main import ImageEditOptionalRequestParams
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import ImageResponse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
|
||||
MAX_INPUT_IMAGES: Final = 3
|
||||
DASHSCOPE_IMAGE_EDIT_PARAMS: Final = ("n", "size", "negative_prompt", "prompt_extend", "watermark", "seed")
|
||||
|
||||
_IMAGES: Final = TypeAdapter(tuple[object, ...])
|
||||
|
||||
|
||||
def _dashscope_size(size: object) -> object:
|
||||
if not isinstance(size, str):
|
||||
return size
|
||||
return OPENAI_TO_DASHSCOPE_SIZE.get(size, size.replace("x", "*"))
|
||||
|
||||
|
||||
def _deployment_edit_url(api_base: str) -> str:
|
||||
derives_from_root: Final = not is_dashscope_native_endpoint_url(api_base) and (
|
||||
resolve_dashscope_endpoints(api_base) is not None
|
||||
or api_base.rstrip("/").endswith(DASHSCOPE_CHAT_COMPATIBLE_PATH)
|
||||
)
|
||||
return f"{dashscope_api_root(api_base)}{DASHSCOPE_MULTIMODAL_GENERATION_PATH}" if derives_from_root else api_base
|
||||
|
||||
|
||||
class DashScopeImageEditConfig(BaseImageEditConfig):
|
||||
def __init__(self, generation_config: DashScopeImageGenerationConfig | None = None) -> None:
|
||||
super().__init__()
|
||||
self._generation_config: Final = generation_config or DashScopeImageGenerationConfig()
|
||||
|
||||
@override
|
||||
def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: mirrors override contract
|
||||
return list(DASHSCOPE_IMAGE_EDIT_PARAMS)
|
||||
|
||||
@override
|
||||
def map_openai_params(
|
||||
self,
|
||||
image_edit_optional_params: ImageEditOptionalRequestParams,
|
||||
model: str,
|
||||
drop_params: bool,
|
||||
) -> dict[str, object]: # mutable-ok: mirrors override contract
|
||||
return {
|
||||
key: _dashscope_size(value) if key == "size" else value
|
||||
for key, value in image_edit_optional_params.items()
|
||||
if key in DASHSCOPE_IMAGE_EDIT_PARAMS
|
||||
}
|
||||
|
||||
@override
|
||||
def validate_environment(
|
||||
self,
|
||||
headers: dict[str, str], # mutable-ok: mirrors override contract
|
||||
model: str,
|
||||
api_key: str | None = None,
|
||||
litellm_params: dict[str, object] | None = None, # mutable-ok: mirrors override contract
|
||||
api_base: str | None = None,
|
||||
) -> dict[str, str]: # mutable-ok: mirrors override contract
|
||||
return self._generation_config.validate_environment(
|
||||
headers=dict(headers),
|
||||
model=model,
|
||||
messages=[],
|
||||
optional_params={},
|
||||
litellm_params=dict(litellm_params or {}),
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
)
|
||||
|
||||
@override
|
||||
def get_complete_url(
|
||||
self,
|
||||
model: str,
|
||||
api_base: str | None,
|
||||
litellm_params: dict[str, object], # mutable-ok: mirrors override contract
|
||||
) -> str:
|
||||
if api_base is not None:
|
||||
return _deployment_edit_url(api_base)
|
||||
env_api_base: Final = get_secret_str("DASHSCOPE_API_BASE")
|
||||
derived: Final = (
|
||||
f"{dashscope_api_root(env_api_base)}{DASHSCOPE_MULTIMODAL_GENERATION_PATH}" if env_api_base else None
|
||||
)
|
||||
return get_secret_str("DASHSCOPE_API_BASE_IMAGE") or derived or DEFAULT_API_BASE
|
||||
|
||||
@override
|
||||
def use_multipart_form_data(self) -> bool:
|
||||
return False
|
||||
|
||||
@override
|
||||
def transform_image_edit_request(
|
||||
self,
|
||||
model: str,
|
||||
prompt: str | None,
|
||||
image: object,
|
||||
image_edit_optional_request_params: Mapping[str, object],
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: Mapping[str, str],
|
||||
) -> tuple[dict[str, object], RequestFiles]: # mutable-ok: mirrors override contract
|
||||
candidates: Final = _IMAGES.validate_python(image) if isinstance(image, list) else (image,)
|
||||
images: Final = tuple(candidate for candidate in candidates if candidate is not None)
|
||||
if len(images) > MAX_INPUT_IMAGES:
|
||||
raise litellm.BadRequestError(
|
||||
message=f"DashScope image edit accepts at most {MAX_INPUT_IMAGES} input images, got {len(images)}",
|
||||
model=model,
|
||||
llm_provider="dashscope",
|
||||
)
|
||||
prompt_content: Final = ({"text": prompt},) if prompt is not None else ()
|
||||
return {
|
||||
"model": model,
|
||||
"input": {
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [*({"image": image_to_data_url(item)} for item in images), *prompt_content],
|
||||
}
|
||||
]
|
||||
},
|
||||
"parameters": dict(image_edit_optional_request_params),
|
||||
}, ()
|
||||
|
||||
@override
|
||||
def transform_image_edit_response(
|
||||
self,
|
||||
model: str,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: "LiteLLMLoggingObj",
|
||||
) -> ImageResponse:
|
||||
return self._generation_config.transform_image_generation_response(
|
||||
model=model,
|
||||
raw_response=raw_response,
|
||||
model_response=ImageResponse(),
|
||||
logging_obj=logging_obj,
|
||||
request_data={},
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
encoding=None,
|
||||
)
|
||||
19
litellm/llms/dashscope/image_generation/cost_calculator.py
Normal file
19
litellm/llms/dashscope/image_generation/cost_calculator.py
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
from typing import Final
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import resolve_image_model_info
|
||||
from litellm.types.utils import ImageResponse, ModelInfo
|
||||
|
||||
|
||||
def cost_calculator(model: str, image_response: ImageResponse, model_info: ModelInfo | None = None) -> float:
|
||||
prices: Final = resolve_image_model_info(
|
||||
model=model, custom_llm_provider=litellm.LlmProviders.DASHSCOPE.value, model_info=model_info
|
||||
)
|
||||
output_cost_per_image: Final = prices.get("output_cost_per_image")
|
||||
if output_cost_per_image is None:
|
||||
raise ValueError(f"No output_cost_per_image found for dashscope/{model}")
|
||||
input_images: Final = image_response.hidden_params.get("billable_input_image_count")
|
||||
input_image_cost: Final = (prices.get("input_cost_per_image") or 0.0) * (
|
||||
input_images if isinstance(input_images, int) else 0
|
||||
)
|
||||
return output_cost_per_image * len(image_response.data or ()) + input_image_cost
|
||||
|
|
@ -23,7 +23,7 @@ Response format:
|
|||
}
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable, Mapping
|
||||
from collections.abc import Iterable, Iterator, Mapping
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
import httpx
|
||||
|
|
@ -61,7 +61,7 @@ _DECODED_JSON: Final = TypeAdapter(object)
|
|||
DEFAULT_API_BASE: Final = "https://dashscope-intl.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation"
|
||||
|
||||
# Maps OpenAI size strings (WxH) to DashScope size strings (W*H)
|
||||
OPENAI_TO_DASHSCOPE_SIZE: Final[dict] = {
|
||||
OPENAI_TO_DASHSCOPE_SIZE: Final[Mapping[str, str]] = {
|
||||
"256x256": "256*256",
|
||||
"512x512": "512*512",
|
||||
"1024x1024": "1024*1024",
|
||||
|
|
@ -71,6 +71,24 @@ OPENAI_TO_DASHSCOPE_SIZE: Final[dict] = {
|
|||
}
|
||||
|
||||
|
||||
def _message_image_urls(message: Mapping[str, object]) -> Iterator[str]:
|
||||
contents: Final = _JSON_OBJECTS.validate_python(message.get("content", []))
|
||||
return (url for url in (item.get("image") for item in contents) if isinstance(url, str) and url)
|
||||
|
||||
|
||||
def _image_urls(response_object: Mapping[str, object]) -> Iterator[str]:
|
||||
output: Final = _JSON_OBJECT.validate_python(response_object.get("output", {}))
|
||||
choices: Final = _JSON_OBJECTS.validate_python(output.get("choices", []))
|
||||
for choice in choices:
|
||||
yield from _message_image_urls(_JSON_OBJECT.validate_python(choice.get("message", {})))
|
||||
|
||||
|
||||
def _dashscope_size(size: object) -> object:
|
||||
if not isinstance(size, str):
|
||||
return size
|
||||
return OPENAI_TO_DASHSCOPE_SIZE.get(size, size.replace("x", "*"))
|
||||
|
||||
|
||||
class DashScopeImageGenerationConfig(BaseImageGenerationConfig):
|
||||
"""
|
||||
Configuration for DashScope image generation (qwen-image-2.0, qwen-image-2.0-pro,
|
||||
|
|
@ -82,24 +100,16 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig):
|
|||
|
||||
def map_openai_params(
|
||||
self,
|
||||
non_default_params: dict,
|
||||
optional_params: dict,
|
||||
non_default_params: dict[str, object],
|
||||
optional_params: dict[str, object],
|
||||
model: str,
|
||||
drop_params: bool,
|
||||
) -> dict:
|
||||
) -> dict[str, object]:
|
||||
supported_params: Final = self.get_supported_openai_params(model)
|
||||
mapped: Final[dict] = {}
|
||||
for k, v in non_default_params.items():
|
||||
if k in optional_params:
|
||||
continue
|
||||
if k not in supported_params:
|
||||
continue
|
||||
if k == "size":
|
||||
# Convert "WxH" → "W*H"
|
||||
mapped["size"] = OPENAI_TO_DASHSCOPE_SIZE.get(v, v.replace("x", "*"))
|
||||
else:
|
||||
mapped[k] = v
|
||||
return mapped
|
||||
requested: Final = {
|
||||
k: v for k, v in non_default_params.items() if k in supported_params and k not in optional_params
|
||||
}
|
||||
return {k: _dashscope_size(v) if k == "size" else v for k, v in requested.items()}
|
||||
|
||||
def _resolve_api_key(self, api_key: str | None) -> str:
|
||||
resolved_api_key: Final = api_key or get_secret_str("DASHSCOPE_API_KEY")
|
||||
|
|
@ -121,8 +131,8 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig):
|
|||
api_base: str | None,
|
||||
api_key: str | None,
|
||||
model: str,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
optional_params: dict[str, object],
|
||||
litellm_params: dict[str, object],
|
||||
stream: bool | None = None,
|
||||
) -> str:
|
||||
image_env_api_base: Final = self._image_api_base_env()
|
||||
|
|
@ -140,14 +150,14 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig):
|
|||
|
||||
def validate_environment(
|
||||
self,
|
||||
headers: dict,
|
||||
headers: dict[str, str],
|
||||
model: str,
|
||||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
optional_params: dict[str, object],
|
||||
litellm_params: dict[str, object],
|
||||
api_key: str | None = None,
|
||||
api_base: str | None = None,
|
||||
) -> dict:
|
||||
) -> dict[str, str]:
|
||||
headers["Authorization"] = f"Bearer {self._resolve_api_key(api_key)}"
|
||||
headers["Content-Type"] = "application/json"
|
||||
return headers
|
||||
|
|
@ -186,9 +196,9 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig):
|
|||
raw_response: httpx.Response,
|
||||
model_response: ImageResponse,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
request_data: dict,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
request_data: dict[str, object],
|
||||
optional_params: dict[str, object],
|
||||
litellm_params: dict[str, object],
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
|
|
@ -225,17 +235,21 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig):
|
|||
headers=raw_response.headers,
|
||||
)
|
||||
|
||||
if not model_response.data:
|
||||
model_response.data = []
|
||||
|
||||
output: Final = _JSON_OBJECT.validate_python(response_object.get("output", {}))
|
||||
choices: Final = _JSON_OBJECTS.validate_python(output.get("choices", []))
|
||||
for choice in choices:
|
||||
message = _JSON_OBJECT.validate_python(choice.get("message", {}))
|
||||
content_list = _JSON_OBJECTS.validate_python(message.get("content", []))
|
||||
for content_item in content_list:
|
||||
image_url = content_item.get("image")
|
||||
if image_url:
|
||||
model_response.data.append(ImageObject(url=image_url))
|
||||
|
||||
return model_response
|
||||
images: Final = [*(model_response.data or ()), *(ImageObject(url=url) for url in _image_urls(response_object))]
|
||||
usage: Final = response_object.get("usage")
|
||||
usage_fields: Final[Mapping[str, object]] = usage if isinstance(usage, Mapping) else {}
|
||||
width: Final = usage_fields.get("output_width", usage_fields.get("width"))
|
||||
height: Final = usage_fields.get("output_height", usage_fields.get("height"))
|
||||
image_response: Final = model_response.model_copy(
|
||||
update={
|
||||
"data": images,
|
||||
**({"size": f"{width}x{height}"} if isinstance(width, int) and isinstance(height, int) else {}),
|
||||
}
|
||||
)
|
||||
input_image_count: Final = usage_fields.get("input_image_count")
|
||||
if isinstance(input_image_count, int) and input_image_count > 0:
|
||||
image_response.hidden_params = {
|
||||
**model_response.hidden_params,
|
||||
"billable_input_image_count": input_image_count,
|
||||
}
|
||||
return image_response
|
||||
|
|
|
|||
0
litellm/llms/dashscope/text_to_speech/__init__.py
Normal file
0
litellm/llms/dashscope/text_to_speech/__init__.py
Normal file
297
litellm/llms/dashscope/text_to_speech/transformation.py
Normal file
297
litellm/llms/dashscope/text_to_speech/transformation.py
Normal file
|
|
@ -0,0 +1,297 @@
|
|||
from collections.abc import Callable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from itertools import chain
|
||||
from typing import TYPE_CHECKING, Final
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
|
||||
from typing_extensions import override
|
||||
|
||||
import litellm
|
||||
from litellm.exceptions import UnsupportedParamsError
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.litellm_core_utils.url_utils import SSRFError, async_safe_get, safe_get
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.base_llm.text_to_speech.transformation import (
|
||||
BaseTextToSpeechConfig,
|
||||
TextToSpeechRequestData,
|
||||
)
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
AsyncHTTPHandler,
|
||||
HTTPHandler,
|
||||
get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # factory stub leaves params untyped
|
||||
get_httpx_client, # pyright: ignore[reportUnknownVariableType] # factory stub leaves params untyped
|
||||
)
|
||||
from litellm.llms.dashscope.common_utils import (
|
||||
DASHSCOPE_MULTIMODAL_GENERATION_PATH,
|
||||
DashScopeError,
|
||||
dashscope_api_root,
|
||||
is_aliyun_host,
|
||||
missing_dashscope_family_key_message,
|
||||
resolve_dashscope_family_api_key,
|
||||
)
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.openai import HttpxBinaryResponseContent
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
|
||||
DASHSCOPE_TTS_DEFAULT_API_BASE: Final = "https://dashscope-intl.aliyuncs.com"
|
||||
_SPEECH_SYNTHESIZER_MODEL_PREFIX: Final = "qwen-audio-"
|
||||
_TIMEOUT_PHASES: Final = TypeAdapter(dict[str, float | None])
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _TextToSpeechFamily:
|
||||
path: str
|
||||
response_formats: frozenset[str]
|
||||
sends_format: bool
|
||||
takes_instructions: bool
|
||||
|
||||
|
||||
MULTIMODAL_GENERATION_FAMILY: Final = _TextToSpeechFamily(
|
||||
path=DASHSCOPE_MULTIMODAL_GENERATION_PATH,
|
||||
response_formats=frozenset({"wav"}),
|
||||
sends_format=False,
|
||||
takes_instructions=False,
|
||||
)
|
||||
SPEECH_SYNTHESIZER_FAMILY: Final = _TextToSpeechFamily(
|
||||
path="/api/v1/services/audio/tts/SpeechSynthesizer",
|
||||
response_formats=frozenset({"mp3", "wav", "opus", "pcm"}),
|
||||
sends_format=True,
|
||||
takes_instructions=True,
|
||||
)
|
||||
|
||||
|
||||
class _Audio(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
url: str
|
||||
|
||||
|
||||
class _Output(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
audio: _Audio
|
||||
|
||||
|
||||
class _Usage(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
input_tokens: int | None = None
|
||||
output_tokens: int | None = None
|
||||
characters: int | None = None
|
||||
|
||||
|
||||
class _SpeechResponse(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
output: _Output | None = None
|
||||
usage: _Usage | None = None
|
||||
code: str | None = None
|
||||
message: str | None = None
|
||||
|
||||
|
||||
def text_to_speech_family(model: str) -> _TextToSpeechFamily:
|
||||
if model.startswith(_SPEECH_SYNTHESIZER_MODEL_PREFIX):
|
||||
return SPEECH_SYNTHESIZER_FAMILY
|
||||
return MULTIMODAL_GENERATION_FAMILY
|
||||
|
||||
|
||||
def _mapped_field(family: _TextToSpeechFamily, key: str, value: object) -> Mapping[str, object] | None:
|
||||
match key:
|
||||
case "response_format" if value in family.response_formats:
|
||||
return {"format": value} if family.sends_format else {}
|
||||
case "instructions" if family.takes_instructions:
|
||||
return {"instruction": value}
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
def _unsupported(model: str, unsupported: tuple[str, ...]) -> UnsupportedParamsError:
|
||||
return UnsupportedParamsError(
|
||||
message=f"DashScope {model} does not support {', '.join(unsupported)}, set drop_params=True to ignore it",
|
||||
model=model,
|
||||
llm_provider="dashscope",
|
||||
)
|
||||
|
||||
|
||||
def _speech_response(raw_response: httpx.Response) -> _SpeechResponse:
|
||||
try:
|
||||
return _SpeechResponse.model_validate_json(raw_response.content)
|
||||
except ValidationError as e:
|
||||
raise DashScopeError(
|
||||
status_code=502,
|
||||
message=f"Unexpected DashScope text-to-speech response: {e}",
|
||||
headers=raw_response.headers,
|
||||
) from e
|
||||
|
||||
|
||||
def _https_on_aliyun(url: str) -> str:
|
||||
parsed: Final = urlparse(url)
|
||||
return (
|
||||
parsed._replace(scheme="https").geturl() if parsed.scheme == "http" and is_aliyun_host(parsed.hostname) else url
|
||||
)
|
||||
|
||||
|
||||
def _audio_url(raw_response: httpx.Response) -> tuple[str, _Usage | None]:
|
||||
parsed: Final = _speech_response(raw_response)
|
||||
if parsed.output is None:
|
||||
raise DashScopeError(
|
||||
status_code=max(raw_response.status_code, 400),
|
||||
message=parsed.message or parsed.code or raw_response.text,
|
||||
headers=raw_response.headers,
|
||||
)
|
||||
return _https_on_aliyun(parsed.output.audio.url), parsed.usage
|
||||
|
||||
|
||||
def _request_timeout(raw_response: httpx.Response) -> httpx.Timeout | None:
|
||||
extension: Final = raw_response.request.extensions.get("timeout")
|
||||
if extension is None:
|
||||
return None
|
||||
phases: Final = _TIMEOUT_PHASES.validate_python(extension)
|
||||
return httpx.Timeout(
|
||||
connect=phases.get("connect"), read=phases.get("read"), write=phases.get("write"), pool=phases.get("pool")
|
||||
)
|
||||
|
||||
|
||||
def _token_usage(usage: _Usage | None) -> Usage | None:
|
||||
if usage is None or usage.input_tokens is None or usage.output_tokens is None:
|
||||
return None
|
||||
return Usage(
|
||||
prompt_tokens=usage.input_tokens,
|
||||
completion_tokens=usage.output_tokens,
|
||||
total_tokens=usage.input_tokens + usage.output_tokens,
|
||||
)
|
||||
|
||||
|
||||
def _refused_audio_url(error: SSRFError) -> DashScopeError:
|
||||
return DashScopeError(status_code=502, message=f"DashScope returned an audio URL that was refused: {error}")
|
||||
|
||||
|
||||
def _binary_audio(audio_response: httpx.Response, usage: _Usage | None) -> HttpxBinaryResponseContent:
|
||||
if audio_response.status_code != 200:
|
||||
raise DashScopeError(
|
||||
status_code=502,
|
||||
message=f"DashScope audio download failed with HTTP {audio_response.status_code}",
|
||||
headers=audio_response.headers,
|
||||
)
|
||||
binary: Final = HttpxBinaryResponseContent(audio_response)
|
||||
binary.usage = _token_usage(usage)
|
||||
if usage is not None and usage.characters is not None:
|
||||
binary.hidden_params = {**binary.hidden_params, "billed_characters": usage.characters}
|
||||
return binary
|
||||
|
||||
|
||||
def _shared_async_client() -> AsyncHTTPHandler:
|
||||
return get_async_httpx_client(llm_provider=litellm.LlmProviders.DASHSCOPE)
|
||||
|
||||
|
||||
class DashScopeTextToSpeechConfig(BaseTextToSpeechConfig):
|
||||
def __init__(
|
||||
self,
|
||||
sync_client: Callable[[], HTTPHandler] = get_httpx_client,
|
||||
async_client: Callable[[], AsyncHTTPHandler] = _shared_async_client,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self._sync_client: Final = sync_client
|
||||
self._async_client: Final = async_client
|
||||
|
||||
@override
|
||||
def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: mirrors override contract
|
||||
family: Final = text_to_speech_family(model)
|
||||
return ["voice", "response_format", *(("instructions",) if family.takes_instructions else ())]
|
||||
|
||||
@override
|
||||
def map_openai_params(
|
||||
self,
|
||||
model: str,
|
||||
optional_params: dict[str, object], # mutable-ok: mirrors override contract
|
||||
voice: str | dict[str, object] | None = None, # mutable-ok: mirrors override contract
|
||||
drop_params: bool = False,
|
||||
kwargs: Mapping[str, object] | None = None,
|
||||
) -> tuple[str | None, dict[str, object]]: # mutable-ok: mirrors override contract
|
||||
family: Final = text_to_speech_family(model)
|
||||
fields: Final = tuple((key, value, _mapped_field(family, key, value)) for key, value in optional_params.items())
|
||||
unsupported: Final = (
|
||||
*(f"{key}={value!r}" for key, value, field in fields if field is None),
|
||||
*(("voice as an object",) if isinstance(voice, dict) else ()),
|
||||
)
|
||||
if unsupported and not (drop_params or drops_unsupported_params((kwargs or {}).get("drop_params"))):
|
||||
raise _unsupported(model, unsupported)
|
||||
supported: Final = tuple(field for _, _, field in fields if field is not None)
|
||||
return voice if isinstance(voice, str) else None, dict(chain.from_iterable(f.items() for f in supported))
|
||||
|
||||
@override
|
||||
def validate_environment(
|
||||
self,
|
||||
headers: dict[str, str], # mutable-ok: mirrors override contract
|
||||
model: str,
|
||||
api_key: str | None = None,
|
||||
api_base: str | None = None,
|
||||
) -> dict[str, str]: # mutable-ok: mirrors override contract
|
||||
resolved_api_key: Final = resolve_dashscope_family_api_key("dashscope", api_key)
|
||||
if resolved_api_key is None:
|
||||
raise ValueError(missing_dashscope_family_key_message("dashscope"))
|
||||
return {**headers, "Authorization": f"Bearer {resolved_api_key}", "Content-Type": "application/json"}
|
||||
|
||||
@override
|
||||
def get_complete_url(
|
||||
self,
|
||||
model: str,
|
||||
api_base: str | None,
|
||||
litellm_params: dict[str, object], # mutable-ok: mirrors override contract
|
||||
) -> str:
|
||||
configured: Final = api_base or get_secret_str("DASHSCOPE_API_BASE")
|
||||
root: Final = dashscope_api_root(configured) if configured else DASHSCOPE_TTS_DEFAULT_API_BASE
|
||||
return f"{root}{text_to_speech_family(model).path}"
|
||||
|
||||
@override
|
||||
def transform_text_to_speech_request(
|
||||
self,
|
||||
model: str,
|
||||
input: str,
|
||||
voice: str | None,
|
||||
optional_params: dict[str, object], # mutable-ok: mirrors override contract
|
||||
litellm_params: dict[str, object], # mutable-ok: mirrors override contract
|
||||
headers: dict[str, str], # mutable-ok: mirrors override contract
|
||||
) -> TextToSpeechRequestData:
|
||||
voice_field: Final = {"voice": voice} if voice is not None else {}
|
||||
return {"dict_body": {"model": model, "input": {"text": input, **voice_field, **optional_params}}}
|
||||
|
||||
@override
|
||||
def transform_text_to_speech_response(
|
||||
self,
|
||||
model: str,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: "LiteLLMLoggingObj",
|
||||
) -> HttpxBinaryResponseContent:
|
||||
audio_url, usage = _audio_url(raw_response)
|
||||
try:
|
||||
audio_response: Final = safe_get(self._sync_client(), audio_url, timeout=_request_timeout(raw_response))
|
||||
except SSRFError as e:
|
||||
raise _refused_audio_url(e) from e
|
||||
return _binary_audio(audio_response, usage)
|
||||
|
||||
@override
|
||||
async def async_transform_text_to_speech_response(
|
||||
self,
|
||||
model: str,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: "LiteLLMLoggingObj",
|
||||
) -> HttpxBinaryResponseContent:
|
||||
audio_url, usage = _audio_url(raw_response)
|
||||
try:
|
||||
audio_response: Final = await async_safe_get(
|
||||
self._async_client(), audio_url, timeout=_request_timeout(raw_response)
|
||||
)
|
||||
except SSRFError as e:
|
||||
raise _refused_audio_url(e) from e
|
||||
return _binary_audio(audio_response, usage)
|
||||
|
||||
@override
|
||||
def get_error_class(
|
||||
self,
|
||||
error_message: str,
|
||||
status_code: int,
|
||||
headers: dict[str, str] | httpx.Headers, # mutable-ok: mirrors override contract
|
||||
) -> BaseLLMException:
|
||||
return DashScopeError(status_code=status_code, message=error_message, headers=httpx.Headers(headers))
|
||||
|
|
@ -9,6 +9,7 @@ from collections.abc import Mapping
|
|||
from typing import Final
|
||||
|
||||
import httpx
|
||||
from pydantic import ConfigDict, TypeAdapter
|
||||
|
||||
from litellm.litellm_core_utils.audio_utils.utils import process_audio_file
|
||||
from litellm.litellm_core_utils.core_helpers import set_response_cost_in_hidden_params
|
||||
|
|
@ -21,6 +22,8 @@ from litellm.utils import convert_to_model_response_object
|
|||
|
||||
from ..common_utils import EdenAIException, authorized_headers, endpoint_url, reported_cost
|
||||
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True))
|
||||
|
||||
|
||||
def _form_fields(model: str, optional_params: Mapping[str, object]) -> dict[str, object]: # mutable-ok: httpx form data
|
||||
"""LiteLLM parks non-OpenAI params, `model` included, under `extra_body` for the OpenAI SDK; a
|
||||
|
|
@ -75,7 +78,7 @@ class EdenAIAudioTranscriptionConfig(OpenAIWhisperAudioTranscriptionConfig):
|
|||
def transform_audio_transcription_response(self, raw_response: httpx.Response) -> TranscriptionResponse:
|
||||
if "application/json" not in raw_response.headers.get("content-type", ""):
|
||||
return TranscriptionResponse(text=raw_response.text)
|
||||
body: Final = raw_response.json()
|
||||
body: Final = _JSON_OBJECT.validate_python(raw_response.json())
|
||||
response: Final[TranscriptionResponse] = convert_to_model_response_object(
|
||||
response_object=body, model_response_object=TranscriptionResponse(), response_type="audio_transcription"
|
||||
)
|
||||
|
|
|
|||
0
litellm/llms/eu_jev/__init__.py
Normal file
0
litellm/llms/eu_jev/__init__.py
Normal file
34
litellm/llms/eu_jev/common_utils.py
Normal file
34
litellm/llms/eu_jev/common_utils.py
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
EU_JEV_LATEST_MODEL: Final = "eu-jeff-latest"
|
||||
_EU_JEV_CURRENT_RELEASE_NAMES: Final = frozenset((EU_JEV_LATEST_MODEL, "jeff-latest", "jeff-1.0.0"))
|
||||
_EU_JEV_PREFIX: Final = "eu_jev/"
|
||||
|
||||
|
||||
def eu_jev_registry_model(requested_model: str | None) -> str:
|
||||
release: Final = requested_model.removeprefix(_EU_JEV_PREFIX) if requested_model is not None else None
|
||||
if release is None or release in _EU_JEV_CURRENT_RELEASE_NAMES:
|
||||
return EU_JEV_LATEST_MODEL
|
||||
return release
|
||||
|
||||
|
||||
def eu_jev_scoped_model(decision_body: Mapping[str, object]) -> str:
|
||||
requested_model: Final = decision_body.get("model")
|
||||
return f"{_EU_JEV_PREFIX}{eu_jev_registry_model(requested_model if isinstance(requested_model, str) else None)}"
|
||||
|
||||
|
||||
def eu_jev_upstream_body(hook_body: Mapping[str, object], client_body: Mapping[str, object]) -> Mapping[str, object]:
|
||||
hook_model: Final = hook_body.get("model")
|
||||
without_model: Final = {key: value for key, value in hook_body.items() if key != "model"}
|
||||
if hook_model == eu_jev_scoped_model(client_body):
|
||||
client_model: Final = client_body.get("model")
|
||||
if client_model is None:
|
||||
return MappingProxyType(without_model)
|
||||
if isinstance(client_model, str):
|
||||
return MappingProxyType({**without_model, "model": client_model.removeprefix(_EU_JEV_PREFIX)})
|
||||
return MappingProxyType({**without_model, "model": client_model})
|
||||
if isinstance(hook_model, str):
|
||||
return MappingProxyType({**without_model, "model": hook_model.removeprefix(_EU_JEV_PREFIX)})
|
||||
return hook_body
|
||||
9
litellm/llms/eu_jev/decisions/transformation.py
Normal file
9
litellm/llms/eu_jev/decisions/transformation.py
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
|
||||
|
||||
|
||||
class EuJevDecisionsConfig(BaseDecisionsConfig):
|
||||
api_key_env = ("EU_JEV_API_KEY",)
|
||||
api_base_env = ("EU_JEV_API_BASE",)
|
||||
|
||||
def get_default_api_base(self) -> str | None:
|
||||
return "https://eu-jev.com"
|
||||
|
|
@ -5,11 +5,30 @@ LiteLLM Proxy supports the OpenAI Responses API natively when the underlying mod
|
|||
This config enables pass-through behavior to the proxy's /v1/responses endpoint.
|
||||
"""
|
||||
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
_ERROR_BODY: Final = TypeAdapter(dict[str, object])
|
||||
|
||||
|
||||
def _openai_sdk_error_body(
|
||||
error_message: str,
|
||||
) -> dict[str, object] | None: # mutable-ok: BaseLLMException.body is a dict
|
||||
try:
|
||||
decoded: Final = _ERROR_BODY.validate_json(error_message)
|
||||
except ValidationError:
|
||||
return None
|
||||
nested: Final = decoded.get("error")
|
||||
return _ERROR_BODY.validate_python(nested) if isinstance(nested, dict) else decoded
|
||||
|
||||
|
||||
class LiteLLMProxyResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
||||
"""
|
||||
|
|
@ -43,6 +62,19 @@ class LiteLLMProxyResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
|||
|
||||
return join_api_base(api_base, "responses")
|
||||
|
||||
def get_error_class(
|
||||
self,
|
||||
error_message: str,
|
||||
status_code: int,
|
||||
headers: dict[str, object] | httpx.Headers, # mutable-ok: inherited contract
|
||||
) -> BaseLLMException:
|
||||
return BaseLLMException(
|
||||
status_code=status_code,
|
||||
message=error_message,
|
||||
headers=headers,
|
||||
body=_openai_sdk_error_body(error_message),
|
||||
)
|
||||
|
||||
def supports_native_websocket(self) -> bool:
|
||||
"""LiteLLM Proxy does not support native WebSocket for Responses API"""
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -687,11 +687,6 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
responses_so_far: Sequence["ModelResponseStream"],
|
||||
litellm_logging_obj: "LiteLLMLoggingObj | None",
|
||||
) -> "ModelResponse":
|
||||
"""``stream_chunk_builder`` folds every choice of a stream into one index-0
|
||||
choice, so the stream is rebuilt one choice index at a time (every chunk
|
||||
kept, its choices narrowed to that index, so usage-only chunks still
|
||||
count) and the rebuilt choices are stitched into one response, each
|
||||
carrying the index the stream gave it."""
|
||||
choice_indices: Final = tuple(
|
||||
sorted(frozenset(choice.index for response in responses_so_far for choice in response.choices))
|
||||
)
|
||||
|
|
|
|||
|
|
@ -275,6 +275,10 @@ def build_output_token_limit_response(
|
|||
return e.response.headers, (_aiter_once(chunk) if is_async else _iter_once(chunk))
|
||||
|
||||
|
||||
def _is_loop_bound(client_initialization_params: Mapping[str, object]) -> bool:
|
||||
return client_initialization_params.get("is_async") is not False
|
||||
|
||||
|
||||
class BaseOpenAILLM:
|
||||
"""
|
||||
Base class for OpenAI LLMs for getting their httpx clients and SSL verification settings
|
||||
|
|
@ -289,7 +293,9 @@ class BaseOpenAILLM:
|
|||
client_initialization_params=client_initialization_params,
|
||||
client_type=client_type,
|
||||
)
|
||||
_cached_client: Final = litellm.in_memory_llm_clients_cache.get_cache(_cache_key)
|
||||
_cached_client: Final = litellm.in_memory_llm_clients_cache.get_cache(
|
||||
_cache_key, loop_bound=_is_loop_bound(client_initialization_params)
|
||||
)
|
||||
return _cached_client
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -330,6 +336,7 @@ class BaseOpenAILLM:
|
|||
value=openai_client,
|
||||
ttl=_DEFAULT_TTL_FOR_HTTPX_CLIENTS,
|
||||
litellm_owned_client=litellm_owned_client,
|
||||
loop_bound=_is_loop_bound(client_initialization_params),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
import httpx
|
||||
from pydantic import ConfigDict, TypeAdapter
|
||||
|
||||
from litellm.llms.base_llm.image_generation.transformation import (
|
||||
BaseImageGenerationConfig,
|
||||
|
|
@ -9,6 +10,8 @@ from litellm.types.llms.openai import OpenAIImageGenerationOptionalParams
|
|||
from litellm.types.utils import ImageResponse
|
||||
from litellm.utils import convert_to_model_response_object
|
||||
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True))
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj
|
||||
|
||||
|
|
@ -66,7 +69,7 @@ class GPTImageGenerationConfig(BaseImageGenerationConfig):
|
|||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ImageResponse:
|
||||
response: Final = raw_response.json()
|
||||
response: Final = _JSON_OBJECT.validate_python(raw_response.json())
|
||||
|
||||
stringified_response: Final = response
|
||||
## LOGGING
|
||||
|
|
|
|||
|
|
@ -15,7 +15,9 @@ from litellm._logging import verbose_logger
|
|||
from litellm.litellm_core_utils.asyncify import asyncify
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.tool_fields import DropPolicy
|
||||
from litellm.llms.anthropic.messages.context_management import AnthropicContextManagementError
|
||||
from litellm.llms.anthropic.messages.convertible_blocks import deployment_drops_params, unconvertible_blocks_error
|
||||
from litellm.llms.anthropic.messages.responses_adapters.handler import count_view_messages
|
||||
from litellm.llms.anthropic.messages.responses_adapters.transformation import (
|
||||
LiteLLMAnthropicToResponsesAPIAdapter,
|
||||
)
|
||||
|
|
@ -117,6 +119,32 @@ def _with_deployment_reasoning(
|
|||
return without_reasoning if resolved is None else {**without_reasoning, "reasoning": resolved}
|
||||
|
||||
|
||||
async def _sent_messages(
|
||||
model: str,
|
||||
messages: Sequence[Mapping[str, object]],
|
||||
tools: Sequence[Mapping[str, object]] | None,
|
||||
system: object,
|
||||
messages_format: TokenCountMessagesFormat,
|
||||
context_management: CountContextManagement | None,
|
||||
) -> Sequence[Mapping[str, object]] | AnthropicContextManagementError:
|
||||
match messages_format:
|
||||
case "openai":
|
||||
return messages
|
||||
case "anthropic":
|
||||
try:
|
||||
return await count_view_messages(
|
||||
model=model,
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
system=system,
|
||||
context_management=context_management,
|
||||
)
|
||||
except AnthropicContextManagementError as error:
|
||||
return error
|
||||
case _:
|
||||
assert_never(messages_format)
|
||||
|
||||
|
||||
class OpenAITokenCounter(BaseTokenCounter):
|
||||
"""Token counter implementation for OpenAI provider using the Responses API."""
|
||||
|
||||
|
|
@ -197,13 +225,28 @@ class OpenAITokenCounter(BaseTokenCounter):
|
|||
rejected_before_provider_call=True,
|
||||
)
|
||||
|
||||
sent_messages: Final = await _sent_messages(
|
||||
model_to_use, messages, tools, system, messages_format, context_management
|
||||
)
|
||||
if isinstance(sent_messages, AnthropicContextManagementError):
|
||||
return TokenCountResponse(
|
||||
total_tokens=0,
|
||||
request_model=request_model,
|
||||
model_used=model_to_use,
|
||||
tokenizer_type="openai_api",
|
||||
error=True,
|
||||
error_message=sent_messages.message,
|
||||
status_code=sent_messages.status_code,
|
||||
rejected_before_provider_call=True,
|
||||
)
|
||||
|
||||
resolved: Final = _count_credential(deployment, request_model or model_to_use)
|
||||
if isinstance(resolved, OpenAIError):
|
||||
return _count_error(model_to_use, request_model, resolved.message, resolved.status_code)
|
||||
|
||||
input_items, instructions, counted_tools, text = self._responses_request_parts(
|
||||
model_to_use=model_to_use,
|
||||
messages=messages,
|
||||
messages=sent_messages,
|
||||
tools=tools,
|
||||
system=system,
|
||||
messages_format=messages_format,
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
|||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
get_standard_logging_object_payload, # pyright: ignore[reportUnknownVariableType] # legacy helper has an untyped signature
|
||||
)
|
||||
from litellm.llms.eu_jev.common_utils import EU_JEV_LATEST_MODEL, eu_jev_registry_model
|
||||
from litellm.llms.laya.common_utils import laya_response_model
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.utils import ModelResponse, StandardPassThroughResponseObject, Usage
|
||||
|
|
@ -82,13 +83,21 @@ class TypeSafePassthroughLoggingHandler:
|
|||
response_model: Final = (
|
||||
laya_response_model(response_body, request_model) if custom_llm_provider == "laya" else response.model
|
||||
)
|
||||
logged_model: Final = response_model or request_model or "unknown"
|
||||
registry_request_model: Final = (
|
||||
eu_jev_registry_model(request_model) if custom_llm_provider == "eu_jev" else request_model
|
||||
)
|
||||
logged_model: Final = response_model or registry_request_model or "unknown"
|
||||
model_name: Final = f"{custom_llm_provider}/{logged_model}"
|
||||
usage: Final = response.usage or _TypeSafeUsage()
|
||||
input_tokens: Final = usage.input_tokens
|
||||
output_tokens: Final = usage.output_tokens
|
||||
candidate_models: Final = (
|
||||
(response_model, registry_request_model, EU_JEV_LATEST_MODEL)
|
||||
if custom_llm_provider == "eu_jev"
|
||||
else (response_model, registry_request_model)
|
||||
)
|
||||
candidate_model_keys: Final = tuple(
|
||||
f"{custom_llm_provider}/{model}" for model in (response_model, request_model) if model is not None
|
||||
f"{custom_llm_provider}/{model}" for model in candidate_models if model is not None
|
||||
)
|
||||
pricing: Final = _pricing_for(candidate_model_keys)
|
||||
response_cost: Final = (
|
||||
|
|
|
|||
|
|
@ -15,8 +15,10 @@ from litellm.litellm_core_utils.url_utils import (
|
|||
)
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
HTTPHandler,
|
||||
MaskedHTTPStatusError,
|
||||
get_async_httpx_client,
|
||||
get_httpx_client,
|
||||
mask_sensitive_info,
|
||||
)
|
||||
from litellm.llms.vertex_ai.common_utils import VertexAIError, get_vertex_base_url
|
||||
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM
|
||||
|
|
@ -81,6 +83,15 @@ def _vertex_batch_list_payload(response: _VertexBatchListJsonSource) -> dict[str
|
|||
return response.json()
|
||||
|
||||
|
||||
def _vertex_batch_create_error(error: httpx.HTTPStatusError) -> MaskedHTTPStatusError:
|
||||
body: Final = mask_sensitive_info(error.response.text)
|
||||
return MaskedHTTPStatusError(
|
||||
httpx.HTTPStatusError(f"{error}\n{body}", request=error.request, response=error.response),
|
||||
message=body,
|
||||
text=body,
|
||||
)
|
||||
|
||||
|
||||
class VertexAIBatchPrediction(VertexLLM):
|
||||
def __init__(self, gcs_bucket_name: str, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
|
|
@ -166,11 +177,19 @@ class VertexAIBatchPrediction(VertexLLM):
|
|||
headers=headers,
|
||||
)
|
||||
|
||||
response: Final = sync_handler.post(
|
||||
url=api_base,
|
||||
headers=headers,
|
||||
data=json.dumps(vertex_batch_request),
|
||||
)
|
||||
try:
|
||||
response: Final = sync_handler.post(
|
||||
url=api_base,
|
||||
headers=headers,
|
||||
data=json.dumps(vertex_batch_request),
|
||||
)
|
||||
except httpx.HTTPStatusError as e:
|
||||
litellm.verbose_logger.error(
|
||||
"Vertex AI batch create failed: status=%s, body=%s",
|
||||
e.response.status_code,
|
||||
e.response.text[:1000],
|
||||
)
|
||||
raise _vertex_batch_create_error(e) from e
|
||||
|
||||
payload_view: Final[_VertexBatchPayloadView] = {"payload": response.json()}
|
||||
_json_response: Final = payload_view["payload"]
|
||||
|
|
@ -266,13 +285,12 @@ class VertexAIBatchPrediction(VertexLLM):
|
|||
data=json.dumps(vertex_batch_request),
|
||||
)
|
||||
except httpx.HTTPStatusError as e:
|
||||
error_body: Final = e.response.text
|
||||
litellm.verbose_logger.error(
|
||||
"Vertex AI batch create failed: status=%s, body=%s",
|
||||
e.response.status_code,
|
||||
error_body[:1000],
|
||||
e.response.text[:1000],
|
||||
)
|
||||
raise
|
||||
raise _vertex_batch_create_error(e) from e
|
||||
|
||||
payload_view: Final[_VertexBatchPayloadView] = {"payload": response.json()}
|
||||
_json_response: Final = payload_view["payload"]
|
||||
|
|
|
|||
|
|
@ -9,14 +9,41 @@ from litellm._logging import verbose_logger
|
|||
from litellm._uuid import uuid
|
||||
from litellm.llms.vertex_ai.common_utils import (
|
||||
VertexAIError,
|
||||
VertexAIModelRoute,
|
||||
convert_vertex_datetime_to_openai_datetime,
|
||||
get_vertex_ai_model_route,
|
||||
)
|
||||
from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import VertexAIAnthropicConfig
|
||||
from litellm.types.llms.openai import BatchJobStatus, CreateBatchRequest
|
||||
from litellm.types.llms.vertex_ai import *
|
||||
from litellm.types.llms.vertex_ai import GenerateContentResponseBody
|
||||
from litellm.types.llms.vertex_ai import GenerateContentResponseBody, VertexPartnerProvider
|
||||
from litellm.types.utils import LiteLLMBatch, ModelInfo, Usage
|
||||
|
||||
_NATIVE_VERTEX_RESPONSE: Final = TypeAdapter(GenerateContentResponseBody)
|
||||
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object])
|
||||
VERTEX_ANTHROPIC_VERSION: Final = "vertex-2023-10-16"
|
||||
_ANTHROPIC_PUBLISHER_MODEL_PATH: Final = "publishers/anthropic/models/"
|
||||
|
||||
|
||||
def is_vertex_anthropic_batch_model(model: str) -> bool:
|
||||
"""
|
||||
Whether a Vertex batch model (bare, `vertex_ai/`-prefixed or a publisher path) is a Claude model, decided the
|
||||
way Vertex chat routes Claude. Vertex batches Claude only as `publishers/anthropic/models/<model>` with rows in
|
||||
Anthropic's Messages shape
|
||||
"""
|
||||
bare_model: Final = model.rsplit("/", 1)[-1]
|
||||
return _ANTHROPIC_PUBLISHER_MODEL_PATH in model or (
|
||||
get_vertex_ai_model_route(bare_model) is VertexAIModelRoute.PARTNER_MODELS
|
||||
and VertexPartnerProvider.claude.value in bare_model
|
||||
)
|
||||
|
||||
|
||||
def as_json_object(value: object) -> Mapping[str, object] | None:
|
||||
return _JSON_OBJECT.validate_python(value) if isinstance(value, dict) else None
|
||||
|
||||
|
||||
def is_vertex_anthropic_batch_output_response(response_body: Mapping[str, object]) -> bool:
|
||||
return response_body.get("type") == "message" and isinstance(response_body.get("usage"), dict)
|
||||
|
||||
|
||||
def _int_field(mapping: Mapping[str, object], key: str) -> int:
|
||||
|
|
@ -43,8 +70,22 @@ def is_vertex_embedding_batch_output_response(response_body: Mapping[str, object
|
|||
return isinstance(response_body.get("embedding"), dict)
|
||||
|
||||
|
||||
def is_vertex_anthropic_batch_output_row(vertex_output_row: Mapping[str, object]) -> bool:
|
||||
"""
|
||||
Whether a Vertex batch output row came from a Claude (Anthropic Messages) request. Vertex echoes only the
|
||||
request's top-level `custom_id` on these rows, with an Anthropic `message` response on success and an empty
|
||||
response plus a `status` on failure, so a Claude input file read back is never mistaken for output
|
||||
"""
|
||||
if not ("custom_id" in vertex_output_row and "processed_time" in vertex_output_row):
|
||||
return False
|
||||
response: Final = as_json_object(vertex_output_row.get("response"))
|
||||
return (response is not None and is_vertex_anthropic_batch_output_response(response)) or bool(
|
||||
vertex_output_row.get("status")
|
||||
)
|
||||
|
||||
|
||||
def is_native_vertex_batch_output_row(row: Mapping[str, object]) -> bool:
|
||||
return isinstance(row.get("request"), dict)
|
||||
return isinstance(row.get("request"), dict) or is_vertex_anthropic_batch_output_row(row)
|
||||
|
||||
|
||||
class NativeVertexBatchCostCalculator(Protocol):
|
||||
|
|
@ -72,6 +113,9 @@ def _native_vertex_row_usage(
|
|||
response_body: Mapping[str, object],
|
||||
calculate_usage: Callable[[GenerateContentResponseBody], Usage],
|
||||
) -> Usage | None:
|
||||
anthropic_usage: Final = as_json_object(response_body.get("usage"))
|
||||
if is_vertex_anthropic_batch_output_response(response_body) and anthropic_usage is not None:
|
||||
return VertexAIAnthropicConfig().calculate_usage(usage_object=anthropic_usage, reasoning_content=None)
|
||||
if "usageMetadata" not in response_body:
|
||||
if not is_vertex_embedding_batch_output_response(response_body):
|
||||
return None
|
||||
|
|
@ -97,21 +141,24 @@ def native_vertex_batch_row_stats(
|
|||
"""
|
||||
Usage and cost of one native Vertex predictions.jsonl row, a
|
||||
`{"request": ..., "response": {"candidates": [...], "usageMetadata": {...}, "modelVersion": ...}}`
|
||||
generateContent object or a `{"request": ..., "response": {"embedding": {...}, "usageMetadata": {...}}}`
|
||||
embedding object (an embedding row without `usageMetadata` is billed from its documented `tokenCount`).
|
||||
`model_name` (the deployment model) prices the row unless it is a wildcard, else its own `modelVersion`
|
||||
does, else the wildcard name so explicit deployment prices still apply; a row without a response, a
|
||||
generateContent row without `response.usageMetadata`, and a row whose response fails validation are
|
||||
generateContent object, a `{"request": ..., "response": {"embedding": {...}, "usageMetadata": {...}}}`
|
||||
embedding object (an embedding row without `usageMetadata` is billed from its documented `tokenCount`) or a
|
||||
`{"custom_id": ..., "request": ..., "response": {"type": "message", "model": ..., "usage": {...}}}` Claude object.
|
||||
`model_name` (the deployment model) prices the row unless it is a wildcard, else its own `modelVersion` (Claude
|
||||
rows: `model`) does, else the wildcard name so explicit deployment prices still apply; a row without a response,
|
||||
a generateContent row without `response.usageMetadata`, and a row whose response fails validation are
|
||||
None (failed).
|
||||
"""
|
||||
response_body: Final = row.get("response")
|
||||
if not isinstance(response_body, dict):
|
||||
response_body: Final = as_json_object(row.get("response"))
|
||||
if response_body is None:
|
||||
return None
|
||||
usage: Final = _native_vertex_row_usage(response_body, calculate_usage)
|
||||
if usage is None:
|
||||
return None
|
||||
total_tokens: Final = usage.total_tokens or (usage.prompt_tokens + usage.completion_tokens)
|
||||
model_version: Final = response_body.get("modelVersion")
|
||||
model_version: Final = response_body.get(
|
||||
"model" if is_vertex_anthropic_batch_output_response(response_body) else "modelVersion"
|
||||
)
|
||||
deployment_model: Final = model_name if model_name and "*" not in model_name else None
|
||||
model: Final = deployment_model or (model_version if isinstance(model_version, str) else model_name)
|
||||
if model is None:
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ from typing_extensions import ReadOnly, Required
|
|||
|
||||
import litellm
|
||||
from litellm._uuid import uuid
|
||||
from litellm.constants import DEFAULT_CHAT_COMPLETION_PARAM_VALUES, RESPONSE_FORMAT_TOOL_NAME
|
||||
from litellm.files.types import FileContentStreamingResult
|
||||
from litellm.files.utils import FilesAPIUtils
|
||||
from litellm.litellm_core_utils.cloud_storage_security import (
|
||||
|
|
@ -32,13 +33,14 @@ from litellm.litellm_core_utils.cloud_storage_security import (
|
|||
split_configured_cloud_bucket_name,
|
||||
validate_managed_cloud_file_id,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params, iter_items
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
extract_file_data,
|
||||
extract_file_metadata,
|
||||
)
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.anthropic.chat.transformation import ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY
|
||||
from litellm.llms.base_llm.base_utils import map_developer_role_to_system_role
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.base_llm.files.batch_records import responses_batch_body_to_chat_body
|
||||
|
|
@ -47,7 +49,14 @@ from litellm.llms.base_llm.files.transformation import (
|
|||
BaseFileUploadStream,
|
||||
LiteLLMLoggingObj,
|
||||
)
|
||||
from litellm.llms.vertex_ai.batches.transformation import vertex_embedding_prompt_token_count
|
||||
from litellm.llms.vertex_ai.batches.transformation import (
|
||||
VERTEX_ANTHROPIC_VERSION,
|
||||
as_json_object,
|
||||
is_vertex_anthropic_batch_model,
|
||||
is_vertex_anthropic_batch_output_response,
|
||||
is_vertex_anthropic_batch_output_row,
|
||||
vertex_embedding_prompt_token_count,
|
||||
)
|
||||
from litellm.llms.vertex_ai.common_utils import (
|
||||
convert_vertex_datetime_to_openai_datetime,
|
||||
get_vertex_ai_fine_tuned_endpoint_id,
|
||||
|
|
@ -59,6 +68,7 @@ from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
|
|||
from litellm.llms.vertex_ai.gemini_embeddings.batch_embed_content_transformation import (
|
||||
transform_openai_input_gemini_embed_content,
|
||||
)
|
||||
from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import VertexAIAnthropicConfig
|
||||
from litellm.types.files import StreamingMediaUploadConfig
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
|
|
@ -79,6 +89,7 @@ from litellm.types.utils import (
|
|||
ModelResponse,
|
||||
Usage,
|
||||
)
|
||||
from litellm.utils import get_non_default_completion_params, get_optional_params
|
||||
|
||||
from ..common_utils import VertexAIError
|
||||
from ..vertex_llm_base import VertexBase
|
||||
|
|
@ -87,6 +98,7 @@ _GCP_LABEL_VALUE_MAX_LEN: Final = 63
|
|||
_CUSTOM_ID_RAW_LABEL_PREFIX: Final = "b32_"
|
||||
_VERTEX_BATCH_KEY_FIELD: Final = "key"
|
||||
_MANAGED_GCS_MODEL_PATH_PATTERN: Final = re.compile(r"publishers/[^/]+/models/([^/?]+)")
|
||||
_PUBLISHER_MODEL_PATH_PATTERN: Final = re.compile(r"publishers/[^/]+/models/")
|
||||
_EMBED_REQUEST_FIELD_BY_GEMINI_PARAM: Final = (
|
||||
("outputDimensionality", "output_dimensionality"),
|
||||
("taskType", "task_type"),
|
||||
|
|
@ -225,10 +237,14 @@ def _get_litellm_batch_custom_id(vertex_output_row: Mapping[str, object]) -> str
|
|||
"""
|
||||
Resolve the OpenAI `custom_id` for a Vertex batch output row.
|
||||
|
||||
Embedding rows carry it in the top-level `key` field that Vertex echoes back;
|
||||
`generateContent` rows have no such field, so it is smuggled through request
|
||||
labels instead (see `_set_litellm_batch_custom_id_labels`).
|
||||
Claude rows carry it in their own top-level `custom_id` field and embedding rows in
|
||||
the top-level `key` field that Vertex echoes back; `generateContent` rows have no
|
||||
such field, so it is smuggled through request labels instead (see
|
||||
`_set_litellm_batch_custom_id_labels`).
|
||||
"""
|
||||
anthropic_custom_id: Final = vertex_output_row.get("custom_id")
|
||||
if anthropic_custom_id is not None:
|
||||
return str(anthropic_custom_id)
|
||||
key = vertex_output_row.get(_VERTEX_BATCH_KEY_FIELD)
|
||||
if key is not None:
|
||||
return unquote(str(key))
|
||||
|
|
@ -289,6 +305,12 @@ def _is_vertex_generate_content_batch_output_row(vertex_output_row: Mapping[str,
|
|||
)
|
||||
|
||||
|
||||
def _is_vertex_chat_batch_output_row(vertex_output_row: Mapping[str, object]) -> bool:
|
||||
return _is_vertex_generate_content_batch_output_row(vertex_output_row) or is_vertex_anthropic_batch_output_row(
|
||||
vertex_output_row
|
||||
)
|
||||
|
||||
|
||||
def _try_parse_vertex_batch_output_row(line: bytes) -> _VertexBatchRow | None:
|
||||
try:
|
||||
row: Final = _parse_vertex_batch_output_row(line.decode("utf-8"))
|
||||
|
|
@ -410,6 +432,37 @@ def _openai_batch_output_row(
|
|||
}
|
||||
|
||||
|
||||
def _vertex_anthropic_batch_output_row_to_openai(
|
||||
custom_id: str,
|
||||
response: object,
|
||||
mock_httpx_response: httpx.Response,
|
||||
) -> _OpenAIBatchOutputRow:
|
||||
"""
|
||||
One OpenAI batch output row from a Vertex Claude row's Anthropic `message` response
|
||||
"""
|
||||
anthropic_response: Final = as_json_object(response)
|
||||
if anthropic_response is None or not is_vertex_anthropic_batch_output_response(anthropic_response):
|
||||
return _openai_batch_output_row(
|
||||
custom_id=custom_id,
|
||||
error_code="vertex_ai_error",
|
||||
error_message="Claude batch row has no message response",
|
||||
)
|
||||
try:
|
||||
model_response: Final = VertexAIAnthropicConfig().transform_parsed_response(
|
||||
completion_response=dict(anthropic_response),
|
||||
raw_response=mock_httpx_response,
|
||||
model_response=ModelResponse(),
|
||||
json_mode=True,
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 # one malformed row must fail alone, not the whole output file
|
||||
return _openai_batch_output_row(
|
||||
custom_id=custom_id,
|
||||
error_code="transformation_error",
|
||||
error_message=f"Failed to transform response: {e}",
|
||||
)
|
||||
return _openai_batch_output_row(custom_id=custom_id, body=model_response.model_dump())
|
||||
|
||||
|
||||
def _split_vertex_batch_key(vertex_output_row: Mapping[str, object]) -> tuple[str, int, int]:
|
||||
"""
|
||||
Resolve `(custom_id, index within that custom_id, group size)` for a Vertex batch
|
||||
|
|
@ -684,21 +737,123 @@ def _openai_batch_jsonl_entry_to_vertex_embeddings_rows(
|
|||
)
|
||||
|
||||
|
||||
_CLAUDE_BATCH_ROW_FIXED_PARAMS: Final = frozenset(
|
||||
("model", "messages", "custom_llm_provider", "drop_params", "stream", "stream_options")
|
||||
)
|
||||
_CLAUDE_BATCH_OPENAI_PARAMS: Final = frozenset(DEFAULT_CHAT_COMPLETION_PARAM_VALUES) - {"metadata"}
|
||||
_CLAUDE_BATCH_DEPLOYMENT_PARAMS: Final = (
|
||||
_CLAUDE_BATCH_OPENAI_PARAMS - _CLAUDE_BATCH_ROW_FIXED_PARAMS - {"extra_headers"}
|
||||
) | {"drop_params"}
|
||||
|
||||
|
||||
def _claude_batch_deployment_params(litellm_params: Mapping[str, object]) -> Mapping[str, object]:
|
||||
return MappingProxyType(
|
||||
{key: value for key, value in litellm_params.items() if key in _CLAUDE_BATCH_DEPLOYMENT_PARAMS}
|
||||
)
|
||||
|
||||
|
||||
def _names_the_response_format_tool(tool: object) -> bool:
|
||||
match tool:
|
||||
case {"function": {"name": str(name)}}:
|
||||
return name == RESPONSE_FORMAT_TOOL_NAME
|
||||
case _:
|
||||
return False
|
||||
|
||||
|
||||
def _openai_batch_jsonl_entry_to_vertex_anthropic_row(
|
||||
custom_id: object,
|
||||
messages: list[AllMessageValues],
|
||||
request_params: Mapping[str, object],
|
||||
model: str,
|
||||
deployment_params: Mapping[str, object],
|
||||
) -> Mapping[str, object]:
|
||||
"""
|
||||
Transforms an OpenAI chat batch entry into the Anthropic Messages row Vertex batches Claude with,
|
||||
`{"custom_id": ..., "request": {"messages": [...], "anthropic_version": "vertex-2023-10-16", "max_tokens": ...}}`
|
||||
"""
|
||||
if custom_id is None:
|
||||
raise ValueError("custom_id is required on Claude lines, since Claude output rows are matched back only by it")
|
||||
config: Final = VertexAIAnthropicConfig()
|
||||
bare_model: Final = model.rsplit("/", 1)[-1]
|
||||
params: Final = {**deployment_params, **request_params}
|
||||
drop_params: Final = drops_unsupported_params(params.get("drop_params"))
|
||||
chat_params: Final = {
|
||||
**get_non_default_completion_params(params),
|
||||
**{key: value for key, value in params.items() if key in _CLAUDE_BATCH_OPENAI_PARAMS},
|
||||
}
|
||||
if any(map(_names_the_response_format_tool, iter_items(chat_params.get("tools") or ()))):
|
||||
raise ValueError(
|
||||
f"tool name {RESPONSE_FORMAT_TOOL_NAME!r} is reserved for response_format on Claude, "
|
||||
"and batch output cannot tell a user tool by that name apart, so rename it"
|
||||
)
|
||||
optional_params: Final[Mapping[str, object]] = get_optional_params(
|
||||
model=bare_model,
|
||||
custom_llm_provider="vertex_ai",
|
||||
drop_params=drop_params,
|
||||
messages=messages,
|
||||
**{ # pyright: ignore[reportArgumentType] # batch JSONL bodies are untyped; get_optional_params checks each param as chat does
|
||||
key: value for key, value in chat_params.items() if key not in _CLAUDE_BATCH_ROW_FIXED_PARAMS
|
||||
},
|
||||
)
|
||||
request_state: Final[dict[str, object]] = {"drop_params": drop_params}
|
||||
anthropic_request: Final[Mapping[str, object]] = config.transform_request(
|
||||
model=bare_model,
|
||||
messages=messages,
|
||||
optional_params={
|
||||
**{key: value for key, value in optional_params.items() if key not in ("json_mode", "extra_body")},
|
||||
"anthropic_version": VERTEX_ANTHROPIC_VERSION,
|
||||
},
|
||||
litellm_params=request_state,
|
||||
headers={},
|
||||
)
|
||||
renamed_tools: Final = as_str_mapping(request_state.get(ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY))
|
||||
if renamed_tools:
|
||||
raise ValueError(
|
||||
f"tool names {sorted(map(str, renamed_tools.values()))} must match ^[a-zA-Z0-9_-]{{1,128}}$ for Claude on Vertex, "
|
||||
"and batch output cannot map a renamed tool back, so rename these tools"
|
||||
)
|
||||
extra_body: Final = as_json_object(optional_params.get("extra_body"))
|
||||
request: Final[Mapping[str, object]] = (
|
||||
anthropic_request
|
||||
if extra_body is None
|
||||
else {
|
||||
**anthropic_request,
|
||||
**config.transform_extra_body(
|
||||
extra_body=extra_body, request=anthropic_request, model=bare_model, litellm_params={}
|
||||
),
|
||||
}
|
||||
)
|
||||
return {"custom_id": str(custom_id), "request": request}
|
||||
|
||||
|
||||
def _openai_batch_jsonl_entry_to_vertex_rows(
|
||||
openai_entry: dict[str, Any],
|
||||
map_openai_to_vertex_params: Callable[[dict[str, object]], dict[str, object]],
|
||||
batch_model: str | None = None,
|
||||
deployment_params: Mapping[str, object] = MappingProxyType({}),
|
||||
) -> tuple[Mapping[str, object], ...]:
|
||||
"""
|
||||
Transforms a single OpenAI JSONL batch entry into the Vertex rows it maps to.
|
||||
|
||||
`batch_model` is the deployment model the batch runs against and wins over the entry's own `body.model`,
|
||||
which picks the row shape: Anthropic Messages for Claude, `GenerateContentRequest` otherwise.
|
||||
|
||||
jsonl body for vertex is {"request": <request_body>}
|
||||
Example Vertex jsonl
|
||||
{"request":{"contents": [{"role": "user", "parts": [{"text": "What is the relation between the following video and image samples?"}, {"fileData": {"fileUri": "gs://cloud-samples-data/generative-ai/video/animals.mp4", "mimeType": "video/mp4"}}, {"fileData": {"fileUri": "gs://cloud-samples-data/generative-ai/image/cricket.jpeg", "mimeType": "image/jpeg"}}]}]}}
|
||||
"""
|
||||
entry_body: Final = openai_entry.get("body")
|
||||
model: Final = batch_model or (str(entry_body.get("model") or "") if isinstance(entry_body, dict) else "")
|
||||
is_anthropic_model: Final = is_vertex_anthropic_batch_model(model)
|
||||
if _is_embeddings_batch_entry(openai_entry):
|
||||
if is_anthropic_model:
|
||||
raise VertexAIError(
|
||||
status_code=400,
|
||||
message=f"Vertex AI batch embeddings are not supported for Claude models ('{model}')",
|
||||
)
|
||||
return _openai_batch_jsonl_entry_to_vertex_embeddings_rows(openai_entry)
|
||||
|
||||
if not isinstance(openai_entry.get("body"), dict):
|
||||
if not isinstance(entry_body, dict):
|
||||
raise TypeError("`body` on batch requests must be a JSON object, but was missing or not an object")
|
||||
openai_request_body: Final = openai_entry["body"]
|
||||
chat_request_body: Final = (
|
||||
|
|
@ -706,8 +861,21 @@ def _openai_batch_jsonl_entry_to_vertex_rows(
|
|||
if _is_responses_batch_entry(openai_entry)
|
||||
else openai_request_body
|
||||
)
|
||||
chat_messages: Final = map_developer_role_to_system_role(chat_request_body.get("messages", []))
|
||||
if is_anthropic_model:
|
||||
return (
|
||||
_openai_batch_jsonl_entry_to_vertex_anthropic_row(
|
||||
custom_id=openai_entry.get("custom_id"),
|
||||
messages=chat_messages,
|
||||
request_params={
|
||||
key: value for key, value in chat_request_body.items() if key not in ("model", "messages")
|
||||
},
|
||||
model=model,
|
||||
deployment_params=deployment_params,
|
||||
),
|
||||
)
|
||||
vertex_request_body: Final = transform_request_body(
|
||||
messages=map_developer_role_to_system_role(chat_request_body.get("messages", [])),
|
||||
messages=chat_messages,
|
||||
model=chat_request_body.get("model", ""),
|
||||
optional_params=map_openai_to_vertex_params(chat_request_body),
|
||||
custom_llm_provider="vertex_ai",
|
||||
|
|
@ -859,15 +1027,24 @@ class _OpenAIToVertexBatchUploadStream(BaseFileUploadStream):
|
|||
self,
|
||||
openai_file_content: FileTypes,
|
||||
map_openai_to_vertex_params: Callable[[dict[str, object]], dict[str, object]],
|
||||
batch_model: str | None = None,
|
||||
deployment_params: Mapping[str, object] = MappingProxyType({}),
|
||||
) -> None:
|
||||
self._openai_file_content = openai_file_content
|
||||
self._map_openai_to_vertex_params = map_openai_to_vertex_params
|
||||
self._batch_model = batch_model
|
||||
self._deployment_params = deployment_params
|
||||
|
||||
def _iter_vertex_jsonl_chunks(self) -> Iterator[bytes]:
|
||||
first = True
|
||||
for lineno, entry in _iter_openai_jsonl_entries(self._openai_file_content):
|
||||
try:
|
||||
vertex_rows = _openai_batch_jsonl_entry_to_vertex_rows(entry, self._map_openai_to_vertex_params)
|
||||
vertex_rows = _openai_batch_jsonl_entry_to_vertex_rows(
|
||||
entry,
|
||||
self._map_openai_to_vertex_params,
|
||||
batch_model=self._batch_model,
|
||||
deployment_params=self._deployment_params,
|
||||
)
|
||||
except Exception as e:
|
||||
raise VertexAIError(
|
||||
status_code=400,
|
||||
|
|
@ -923,16 +1100,31 @@ class _RawFileUploadStream(BaseFileUploadStream):
|
|||
return _iter_raw_file_chunks(self._file_content)
|
||||
|
||||
|
||||
def _deployment_batch_model(litellm_params: Mapping[str, object]) -> str | None:
|
||||
model: Final = litellm_params.get("model")
|
||||
return model.removeprefix("vertex_ai/") if isinstance(model, str) and model else None
|
||||
|
||||
|
||||
def _publisher_model_path(raw_model: str) -> str:
|
||||
if _PUBLISHER_MODEL_PATH_PATTERN.search(raw_model):
|
||||
return raw_model
|
||||
publisher: Final = "anthropic" if is_vertex_anthropic_batch_model(raw_model) else "google"
|
||||
return f"publishers/{publisher}/models/{raw_model}"
|
||||
|
||||
|
||||
def _managed_batch_object_name(raw_model: str, *, passthrough: bool) -> str:
|
||||
endpoint_id: Final = get_vertex_ai_fine_tuned_endpoint_id(raw_model)
|
||||
model_path: Final = (
|
||||
f"endpoints/{endpoint_id}"
|
||||
if endpoint_id is not None
|
||||
else (raw_model if "publishers/google/models" in raw_model else f"publishers/google/models/{raw_model}")
|
||||
)
|
||||
safe_model_path: Final = sanitize_cloud_object_path(model_path, fallback="model")
|
||||
model_path: Final = f"endpoints/{endpoint_id}" if endpoint_id is not None else _publisher_model_path(raw_model)
|
||||
prefix: Final = _PASSTHROUGH_MANAGED_GCS_PREFIX if passthrough else VERTEX_AI_MANAGED_GCS_PREFIX
|
||||
return f"{prefix}{safe_model_path}/{uuid.uuid4()}"
|
||||
return f"{prefix}{_safe_managed_model_path(model_path)}/{uuid.uuid4()}"
|
||||
|
||||
|
||||
def _safe_managed_model_path(model_path: str) -> str:
|
||||
if not is_vertex_anthropic_batch_model(model_path):
|
||||
return sanitize_cloud_object_path(model_path, fallback="model")
|
||||
return "@".join(
|
||||
sanitize_cloud_object_path(version_part, fallback="model") for version_part in model_path.split("@")
|
||||
)
|
||||
|
||||
|
||||
class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
|
||||
|
|
@ -1036,10 +1228,10 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
|
|||
fallback_filename="file",
|
||||
)
|
||||
|
||||
def _get_configured_bucket_name(self, litellm_params: dict) -> str:
|
||||
def _get_configured_bucket_name(self, litellm_params: Mapping[str, object]) -> str:
|
||||
configured_bucket_name: Final = litellm_params.get("gcs_bucket_name") or litellm_params.get("bucket_name")
|
||||
bucket_name: Final = (
|
||||
litellm_params.get("gcs_bucket_name")
|
||||
or litellm_params.get("bucket_name")
|
||||
(configured_bucket_name if isinstance(configured_bucket_name, str) else None)
|
||||
or os.getenv("GCS_BATCH_BUCKET_NAME")
|
||||
or os.getenv("GCS_BUCKET_NAME")
|
||||
)
|
||||
|
|
@ -1053,7 +1245,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
|
|||
api_key: str | None,
|
||||
model: str,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
litellm_params: Mapping[str, object],
|
||||
data: CreateFileRequest,
|
||||
) -> str:
|
||||
"""
|
||||
|
|
@ -1077,11 +1269,10 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
|
|||
raise ValueError("file is required")
|
||||
if purpose is None:
|
||||
raise ValueError("purpose is required")
|
||||
configured_model: Final = litellm_params.get("model")
|
||||
object_name = self.get_object_name(
|
||||
file_data,
|
||||
purpose,
|
||||
deployment_model=configured_model if isinstance(configured_model, str) else None,
|
||||
deployment_model=_deployment_batch_model(litellm_params),
|
||||
passthrough=is_passthrough_batch_upload(data, litellm_params),
|
||||
)
|
||||
if object_prefix:
|
||||
|
|
@ -1138,7 +1329,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
|
|||
model: str,
|
||||
create_file_data: CreateFileRequest,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
litellm_params: Mapping[str, object],
|
||||
) -> bytes | str | dict:
|
||||
"""
|
||||
2 Cases:
|
||||
|
|
@ -1170,6 +1361,8 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
|
|||
body_stream=_OpenAIToVertexBatchUploadStream(
|
||||
file_data,
|
||||
partial(self._map_openai_to_vertex_params, deployment_drop_params=deployment_drop_params),
|
||||
batch_model=_deployment_batch_model(litellm_params),
|
||||
deployment_params=_claude_batch_deployment_params(litellm_params),
|
||||
),
|
||||
content_type="application/json",
|
||||
)
|
||||
|
|
@ -1220,7 +1413,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
|
|||
def get_error_class(self, error_message: str, status_code: int, headers: dict | Headers) -> BaseLLMException:
|
||||
return VertexAIError(status_code=status_code, message=error_message, headers=headers)
|
||||
|
||||
def _parse_gcs_uri(self, file_id: str, litellm_params: dict | None = None) -> tuple[str, str]:
|
||||
def _parse_gcs_uri(self, file_id: str, litellm_params: Mapping[str, object] | None = None) -> tuple[str, str]:
|
||||
"""
|
||||
Validate a managed GCS file_id and return (bucket, url-encoded-object-path).
|
||||
"""
|
||||
|
|
@ -1238,7 +1431,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
|
|||
self,
|
||||
file_id: str,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
litellm_params: Mapping[str, object],
|
||||
) -> tuple[str, dict]:
|
||||
bucket, encoded_object = self._parse_gcs_uri(file_id, litellm_params)
|
||||
url: Final = f"https://storage.googleapis.com/storage/v1/b/{bucket}/o/{encoded_object}"
|
||||
|
|
@ -1268,7 +1461,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
|
|||
self,
|
||||
file_id: str,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
litellm_params: Mapping[str, object],
|
||||
) -> tuple[str, dict]:
|
||||
bucket, encoded_object = self._parse_gcs_uri(file_id, litellm_params)
|
||||
url: Final = f"https://storage.googleapis.com/storage/v1/b/{bucket}/o/{encoded_object}"
|
||||
|
|
@ -1311,7 +1504,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
|
|||
self,
|
||||
file_content_request,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
litellm_params: Mapping[str, object],
|
||||
) -> tuple[str, dict]:
|
||||
file_id: Final = file_content_request.get("file_id", "")
|
||||
bucket, encoded_object = self._parse_gcs_uri(file_id, litellm_params)
|
||||
|
|
@ -1408,7 +1601,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
|
|||
headers=MappingProxyType({**headers, "content-length": str(len(transformed_content))}),
|
||||
)
|
||||
|
||||
if not _is_vertex_generate_content_batch_output_row(first_row):
|
||||
if not _is_vertex_chat_batch_output_row(first_row):
|
||||
return FileContentStreamingResult(stream_iterator=replayed_stream, headers=headers)
|
||||
|
||||
return FileContentStreamingResult(
|
||||
|
|
@ -1492,10 +1685,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
|
|||
# first line is not valid UTF-8/JSON) raises and falls through to the
|
||||
# passthrough below, leaving the content untouched.
|
||||
first_row: Final = _parse_vertex_batch_output_row(first_line)
|
||||
if not (
|
||||
_is_vertex_embeddings_batch_output_row(first_row)
|
||||
or _is_vertex_generate_content_batch_output_row(first_row)
|
||||
):
|
||||
if not (_is_vertex_embeddings_batch_output_row(first_row) or _is_vertex_chat_batch_output_row(first_row)):
|
||||
return content
|
||||
|
||||
context: Final = _new_vertex_batch_output_row_transform_context()
|
||||
|
|
@ -1560,6 +1750,13 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
|
|||
error_message=status,
|
||||
)
|
||||
|
||||
if is_vertex_anthropic_batch_output_row(vertex_output):
|
||||
return _vertex_anthropic_batch_output_row_to_openai(
|
||||
custom_id=custom_id,
|
||||
response=vertex_output.get("response"),
|
||||
mock_httpx_response=mock_httpx_response,
|
||||
)
|
||||
|
||||
# Transform successful response using existing transformation
|
||||
vertex_response: Final = vertex_output.get("response", {})
|
||||
|
||||
|
|
|
|||
|
|
@ -7975,6 +7975,8 @@ async def atext_completion(*args, **kwargs) -> TextCompletionResponse | TextComp
|
|||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
stream_options=kwargs.get("stream_options"),
|
||||
usage_injected_by_proxy="_litellm_strip_stream_usage" in kwargs
|
||||
and kwargs["_litellm_strip_stream_usage"] is True,
|
||||
)
|
||||
else:
|
||||
## OpenAI / Azure Text Completion Returns here
|
||||
|
|
@ -8824,8 +8826,9 @@ def speech(
|
|||
extra_headers: Final = kwargs.get("extra_headers", None)
|
||||
model_info: Final = kwargs.get("model_info", None)
|
||||
shared_session: Final = kwargs.get("shared_session", None)
|
||||
caller_api_base: Final = api_base or base_url
|
||||
model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider(
|
||||
model=model, custom_llm_provider=custom_llm_provider, api_base=api_base or base_url
|
||||
model=model, custom_llm_provider=custom_llm_provider, api_base=caller_api_base
|
||||
)
|
||||
kwargs.pop("tags", [])
|
||||
|
||||
|
|
@ -8880,13 +8883,13 @@ def speech(
|
|||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
response: HttpxBinaryResponseContent | Coroutine[object, object, HttpxBinaryResponseContent] | None = None
|
||||
if custom_llm_provider == "edenai":
|
||||
litellm_params_dict["api_base"] = api_base
|
||||
if custom_llm_provider in ("edenai", "dashscope") and text_to_speech_provider_config is not None:
|
||||
litellm_params_dict["api_base"] = caller_api_base if custom_llm_provider == "dashscope" else api_base
|
||||
response = base_llm_http_handler.text_to_speech_handler(
|
||||
model=model,
|
||||
input=input,
|
||||
voice=voice if isinstance(voice, str) else None,
|
||||
text_to_speech_provider_config=text_to_speech_provider_config or litellm.EdenAITextToSpeechConfig(),
|
||||
text_to_speech_provider_config=text_to_speech_provider_config,
|
||||
text_to_speech_optional_params=optional_params,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
litellm_params=litellm_params_dict,
|
||||
|
|
|
|||
|
|
@ -121,6 +121,7 @@ _ASYNC_FEATURES: Final[detectors.Detectors] = MappingProxyType(
|
|||
Feature.PRE_REQUEST_HOOK: route_detectors.pre_request_hook,
|
||||
Feature.WEBSEARCH_INTERCEPTION: route_detectors.websearch_interception,
|
||||
Feature.MESSAGES_INTERCEPTOR: route_detectors.intercepted,
|
||||
Feature.STRICT_TOOLS_RETRY: route_detectors.strict_tools_retry,
|
||||
}
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -18346,25 +18346,31 @@
|
|||
"dashscope/qwen-image-2.0": {
|
||||
"litellm_provider": "dashscope",
|
||||
"mode": "image_generation",
|
||||
"source": "https://www.alibabacloud.com/help/en/model-studio/models",
|
||||
"output_cost_per_image": 0.035,
|
||||
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/images/generations"
|
||||
"/v1/images/generations",
|
||||
"/v1/images/edits"
|
||||
]
|
||||
},
|
||||
"dashscope/qwen-image-2.0-pro": {
|
||||
"litellm_provider": "dashscope",
|
||||
"mode": "image_generation",
|
||||
"source": "https://www.alibabacloud.com/help/en/model-studio/models",
|
||||
"output_cost_per_image": 0.075,
|
||||
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/images/generations"
|
||||
"/v1/images/generations",
|
||||
"/v1/images/edits"
|
||||
]
|
||||
},
|
||||
"dashscope/qwen-image-3.0": {
|
||||
"litellm_provider": "dashscope",
|
||||
"mode": "image_generation",
|
||||
"source": "https://www.alibabacloud.com/help/en/model-studio/models",
|
||||
"output_cost_per_image": 0.03,
|
||||
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/images/generations"
|
||||
"/v1/images/generations",
|
||||
"/v1/images/edits"
|
||||
]
|
||||
},
|
||||
"dashscope/qwen-image-3.0-pro": {
|
||||
|
|
@ -18372,7 +18378,54 @@
|
|||
"mode": "image_generation",
|
||||
"source": "https://www.alibabacloud.com/help/en/model-studio/models",
|
||||
"supported_endpoints": [
|
||||
"/v1/images/generations"
|
||||
"/v1/images/generations",
|
||||
"/v1/images/edits"
|
||||
]
|
||||
},
|
||||
"dashscope/qwen3-tts-flash": {
|
||||
"input_cost_per_character": 1e-05,
|
||||
"litellm_provider": "dashscope",
|
||||
"mode": "audio_speech",
|
||||
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"dashscope/qwen-audio-3.1-tts-flash": {
|
||||
"input_cost_per_token": 2.3e-07,
|
||||
"output_cost_per_token": 1.87e-06,
|
||||
"litellm_provider": "dashscope",
|
||||
"mode": "audio_speech",
|
||||
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"dashscope/qwen-audio-3.1-tts-plus": {
|
||||
"input_cost_per_character": 3e-05,
|
||||
"litellm_provider": "dashscope",
|
||||
"mode": "audio_speech",
|
||||
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"dashscope/qwen-audio-3.0-tts-plus": {
|
||||
"input_cost_per_character": 2e-05,
|
||||
"litellm_provider": "dashscope",
|
||||
"mode": "audio_speech",
|
||||
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"dashscope/qwen-audio-3.0-tts-flash": {
|
||||
"input_cost_per_character": 1.5e-05,
|
||||
"litellm_provider": "dashscope",
|
||||
"mode": "audio_speech",
|
||||
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"dashscope/wan3.0-video": {
|
||||
|
|
@ -21382,7 +21435,7 @@
|
|||
"cache_creation_input_token_cost": 3.74997e-06,
|
||||
"cache_read_input_token_cost": 3.0002e-07,
|
||||
"deprecation_date": "2026-10-09",
|
||||
"input_cost_per_token": 2.9999900000000002e-06,
|
||||
"input_cost_per_token": 2.99999e-06,
|
||||
"input_dbu_cost_per_token": 4.2857e-05,
|
||||
"litellm_provider": "databricks",
|
||||
"max_input_tokens": 200000,
|
||||
|
|
@ -21391,7 +21444,7 @@
|
|||
"notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
|
||||
},
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.5000020000000002e-05,
|
||||
"output_cost_per_token": 1.500002e-05,
|
||||
"output_dbu_cost_per_token": 0.000214286,
|
||||
"prompt_cache_min_tokens": 1024,
|
||||
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
|
||||
|
|
@ -21504,6 +21557,7 @@
|
|||
"databricks/databricks-deepseek-v4-flash-0731": {
|
||||
"cache_creation_input_token_cost": 1.4e-07,
|
||||
"cache_read_input_token_cost": 2.8e-08,
|
||||
"deprecation_date": "2026-11-05",
|
||||
"input_cost_per_token": 1.4e-07,
|
||||
"input_dbu_cost_per_token": 2e-06,
|
||||
"litellm_provider": "databricks",
|
||||
|
|
@ -21516,7 +21570,7 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 2.8e-07,
|
||||
"output_dbu_cost_per_token": 4e-06,
|
||||
"source": "https://www.databricks.com/product/pricing/foundation-model-serving",
|
||||
"source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models",
|
||||
"supported_modalities": [
|
||||
"text"
|
||||
],
|
||||
|
|
@ -21537,6 +21591,7 @@
|
|||
"databricks/databricks-deepseek-v4-pro-0813": {
|
||||
"cache_creation_input_token_cost": 1.31999e-06,
|
||||
"cache_read_input_token_cost": 1.3202e-07,
|
||||
"deprecation_date": "2026-10-30",
|
||||
"input_cost_per_token": 1.31999e-06,
|
||||
"input_dbu_cost_per_token": 1.8857e-05,
|
||||
"litellm_provider": "databricks",
|
||||
|
|
@ -21548,7 +21603,7 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 3.95997e-06,
|
||||
"output_dbu_cost_per_token": 5.6571e-05,
|
||||
"source": "https://www.databricks.com/product/pricing/foundation-model-serving",
|
||||
"source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models",
|
||||
"supported_modalities": [
|
||||
"text"
|
||||
],
|
||||
|
|
@ -21735,7 +21790,7 @@
|
|||
"notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
|
||||
},
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.5000020000000002e-05,
|
||||
"output_cost_per_token": 1.500002e-05,
|
||||
"output_cost_per_token_above_200k_tokens": 2.25e-05,
|
||||
"output_dbu_cost_per_token": 0.000214286,
|
||||
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
|
||||
|
|
@ -21994,7 +22049,7 @@
|
|||
"databricks/databricks-gemma-3-12b": {
|
||||
"cache_creation_input_token_cost": 1.5001e-07,
|
||||
"cache_read_input_token_cost": 1.5001e-07,
|
||||
"input_cost_per_token": 1.5000999999999998e-07,
|
||||
"input_cost_per_token": 1.5001e-07,
|
||||
"input_dbu_cost_per_token": 2.1429999999999996e-06,
|
||||
"litellm_provider": "databricks",
|
||||
"max_input_tokens": 128000,
|
||||
|
|
@ -22117,7 +22172,7 @@
|
|||
"notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
|
||||
},
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 9.999990000000002e-06,
|
||||
"output_cost_per_token": 9.99999e-06,
|
||||
"output_dbu_cost_per_token": 0.000142857,
|
||||
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
|
||||
"supports_function_calling": true,
|
||||
|
|
@ -22140,7 +22195,7 @@
|
|||
"notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
|
||||
},
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 9.999990000000002e-06,
|
||||
"output_cost_per_token": 9.99999e-06,
|
||||
"output_dbu_cost_per_token": 0.000142857,
|
||||
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
|
||||
"supports_function_calling": true,
|
||||
|
|
@ -22211,7 +22266,7 @@
|
|||
"notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
|
||||
},
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.5000020000000002e-05,
|
||||
"output_cost_per_token": 1.500002e-05,
|
||||
"output_cost_per_token_above_272k_tokens": 2.25e-05,
|
||||
"output_dbu_cost_per_token": 0.000214286,
|
||||
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
|
||||
|
|
@ -22486,7 +22541,7 @@
|
|||
"databricks/databricks-gpt-5-mini": {
|
||||
"cache_creation_input_token_cost": 2.4997e-07,
|
||||
"cache_read_input_token_cost": 2.499e-08,
|
||||
"input_cost_per_token": 2.4997000000000006e-07,
|
||||
"input_cost_per_token": 2.4997e-07,
|
||||
"input_dbu_cost_per_token": 3.571e-06,
|
||||
"litellm_provider": "databricks",
|
||||
"max_input_tokens": 272000,
|
||||
|
|
@ -22495,7 +22550,7 @@
|
|||
"notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
|
||||
},
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.9999700000000004e-06,
|
||||
"output_cost_per_token": 1.99997e-06,
|
||||
"output_dbu_cost_per_token": 2.8571e-05,
|
||||
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
|
||||
"supports_function_calling": true,
|
||||
|
|
@ -22518,7 +22573,7 @@
|
|||
"notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
|
||||
},
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3.9998000000000007e-07,
|
||||
"output_cost_per_token": 3.9998e-07,
|
||||
"output_dbu_cost_per_token": 5.714000000000001e-06,
|
||||
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
|
||||
"supports_function_calling": true,
|
||||
|
|
@ -22528,7 +22583,7 @@
|
|||
"databricks/databricks-gpt-oss-120b": {
|
||||
"cache_creation_input_token_cost": 1.5001e-07,
|
||||
"cache_read_input_token_cost": 1.5001e-07,
|
||||
"input_cost_per_token": 1.5000999999999998e-07,
|
||||
"input_cost_per_token": 1.5001e-07,
|
||||
"input_dbu_cost_per_token": 2.1429999999999996e-06,
|
||||
"litellm_provider": "databricks",
|
||||
"max_input_tokens": 131072,
|
||||
|
|
@ -22553,7 +22608,7 @@
|
|||
"notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
|
||||
},
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3.0001999999999996e-07,
|
||||
"output_cost_per_token": 3.0002e-07,
|
||||
"output_dbu_cost_per_token": 4.285999999999999e-06,
|
||||
"source": "https://www.databricks.com/product/pricing/foundation-model-serving"
|
||||
},
|
||||
|
|
@ -22586,7 +22641,7 @@
|
|||
"databricks/databricks-gte-large-en": {
|
||||
"cache_creation_input_token_cost": 1.2999e-07,
|
||||
"cache_read_input_token_cost": 1.2999e-07,
|
||||
"input_cost_per_token": 1.2999000000000001e-07,
|
||||
"input_cost_per_token": 1.2999e-07,
|
||||
"input_dbu_cost_per_token": 1.857e-06,
|
||||
"litellm_provider": "databricks",
|
||||
"max_input_tokens": 8192,
|
||||
|
|
@ -22602,6 +22657,7 @@
|
|||
"databricks/databricks-inkling": {
|
||||
"cache_creation_input_token_cost": 1.00002e-06,
|
||||
"cache_read_input_token_cost": 1.7003e-07,
|
||||
"deprecation_date": "2026-10-30",
|
||||
"input_cost_per_token": 1.00002e-06,
|
||||
"input_dbu_cost_per_token": 1.4286e-05,
|
||||
"litellm_provider": "databricks",
|
||||
|
|
@ -22612,7 +22668,7 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 4.04999e-06,
|
||||
"output_dbu_cost_per_token": 5.7857e-05,
|
||||
"source": "https://www.databricks.com/product/pricing/foundation-model-serving",
|
||||
"source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models",
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image"
|
||||
|
|
@ -22677,7 +22733,7 @@
|
|||
"notes": "Databricks documentation now provides both DBU costs (_dbu_cost_per_token) and dollar costs(_cost_per_token)."
|
||||
},
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.5000300000000002e-06,
|
||||
"output_cost_per_token": 1.50003e-06,
|
||||
"output_dbu_cost_per_token": 2.1429e-05,
|
||||
"source": "https://www.databricks.com/product/pricing/foundation-model-serving",
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -22685,7 +22741,7 @@
|
|||
"databricks/databricks-meta-llama-3-1-8b-instruct": {
|
||||
"cache_creation_input_token_cost": 1.5001e-07,
|
||||
"cache_read_input_token_cost": 1.5001e-07,
|
||||
"input_cost_per_token": 1.5000999999999998e-07,
|
||||
"input_cost_per_token": 1.5001e-07,
|
||||
"input_dbu_cost_per_token": 2.1429999999999996e-06,
|
||||
"litellm_provider": "databricks",
|
||||
"max_input_tokens": 200000,
|
||||
|
|
@ -22694,7 +22750,7 @@
|
|||
"notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
|
||||
},
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4.5003000000000007e-07,
|
||||
"output_cost_per_token": 4.5003e-07,
|
||||
"output_dbu_cost_per_token": 6.429000000000001e-06,
|
||||
"source": "https://www.databricks.com/product/pricing/foundation-model-serving"
|
||||
},
|
||||
|
|
@ -22710,7 +22766,7 @@
|
|||
"notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
|
||||
},
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.5000300000000002e-06,
|
||||
"output_cost_per_token": 1.50003e-06,
|
||||
"output_dbu_cost_per_token": 2.1429e-05,
|
||||
"source": "https://www.databricks.com/product/pricing/foundation-model-serving",
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -27741,6 +27797,7 @@
|
|||
},
|
||||
"fireworks_ai/accounts/fireworks/models/deepseek-v4-flash": {
|
||||
"cache_read_input_token_cost": 2.8e-08,
|
||||
"deprecation_date": "2026-08-14",
|
||||
"input_cost_per_token": 1.4e-07,
|
||||
"litellm_provider": "fireworks_ai",
|
||||
"max_input_tokens": 1048576,
|
||||
|
|
@ -28189,6 +28246,7 @@
|
|||
},
|
||||
"fireworks_ai/deepseek-v4-flash": {
|
||||
"cache_read_input_token_cost": 2.8e-08,
|
||||
"deprecation_date": "2026-08-14",
|
||||
"input_cost_per_token": 1.4e-07,
|
||||
"litellm_provider": "fireworks_ai",
|
||||
"max_input_tokens": 1048576,
|
||||
|
|
@ -48560,13 +48618,13 @@
|
|||
"supports_reasoning": true
|
||||
},
|
||||
"together_ai/Qwen/Qwen3.7-Max": {
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
"input_cost_per_token": 2.5e-06,
|
||||
"cache_read_input_token_cost": 3e-07,
|
||||
"input_cost_per_token": 1.5e-06,
|
||||
"litellm_provider": "together_ai",
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 1000000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 7.5e-06,
|
||||
"output_cost_per_token": 4.5e-06,
|
||||
"source": "https://www.together.ai/pricing",
|
||||
"supports_prompt_caching": true
|
||||
},
|
||||
|
|
@ -65073,7 +65131,7 @@
|
|||
"wandb/zai-org/GLM-5.2": {
|
||||
"supports_reasoning": true,
|
||||
"max_output_tokens": 262144,
|
||||
"max_input_tokens": 1049000,
|
||||
"max_input_tokens": 262000,
|
||||
"input_cost_per_token": 7.6e-07,
|
||||
"output_cost_per_token": 2.42e-06,
|
||||
"cache_read_input_token_cost": 1.4e-07,
|
||||
|
|
@ -65081,7 +65139,7 @@
|
|||
"litellm_provider": "wandb",
|
||||
"mode": "chat",
|
||||
"supports_vision": false,
|
||||
"source": "https://wandb.ai/site/pricing/tokens/"
|
||||
"source": "https://www.coreweave.com/products/coreweave-forge/serverless-inference"
|
||||
},
|
||||
"deepinfra/openai/gpt-oss-120b-Turbo": {
|
||||
"max_output_tokens": 131072,
|
||||
|
|
@ -72330,6 +72388,28 @@
|
|||
"source": "https://docs.typesafe.ai/models",
|
||||
"supports_box_questions": true
|
||||
},
|
||||
"eu_jev/eu-jeff-1.0.0": {
|
||||
"input_cost_per_token": 4.15e-08,
|
||||
"litellm_provider": "eu_jev",
|
||||
"max_input_tokens": 32768,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://eu-jev.com/docs",
|
||||
"supported_endpoints": [
|
||||
"/v1/systemone"
|
||||
]
|
||||
},
|
||||
"eu_jev/eu-jeff-latest": {
|
||||
"input_cost_per_token": 4.15e-08,
|
||||
"litellm_provider": "eu_jev",
|
||||
"max_input_tokens": 32768,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://eu-jev.com/docs",
|
||||
"supported_endpoints": [
|
||||
"/v1/systemone"
|
||||
]
|
||||
},
|
||||
"wandb/deepseek-ai/DeepSeek-V4.1-Flash": {
|
||||
"cache_read_input_token_cost": 3e-08,
|
||||
"input_cost_per_token": 2e-07,
|
||||
|
|
@ -78558,6 +78638,272 @@
|
|||
"supports_xhigh_reasoning_effort": true,
|
||||
"use_openai_responses_path": true
|
||||
},
|
||||
"clf_ai_gateway/glm-5.3": {
|
||||
"metadata": {
|
||||
"comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends."
|
||||
},
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 5.6e-07,
|
||||
"output_cost_per_token": 1.76e-06,
|
||||
"cache_read_input_token_cost": 1.04e-07,
|
||||
"litellm_provider": "clf_ai_gateway",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true,
|
||||
"reasoning_effort_levels": [
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"max"
|
||||
],
|
||||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://clfaigateway.dev/models"
|
||||
},
|
||||
"clf_ai_gateway/glm-5.3-flash": {
|
||||
"metadata": {
|
||||
"comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends."
|
||||
},
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 6e-08,
|
||||
"output_cost_per_token": 2e-07,
|
||||
"cache_read_input_token_cost": 1.2e-08,
|
||||
"litellm_provider": "clf_ai_gateway",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true,
|
||||
"reasoning_effort_levels": [
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh",
|
||||
"max"
|
||||
],
|
||||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_vision": true,
|
||||
"source": "https://clfaigateway.dev/models"
|
||||
},
|
||||
"clf_ai_gateway/glm-5.2": {
|
||||
"metadata": {
|
||||
"comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends."
|
||||
},
|
||||
"max_input_tokens": 262144,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 5.6e-07,
|
||||
"output_cost_per_token": 1.76e-06,
|
||||
"cache_read_input_token_cost": 1.04e-07,
|
||||
"litellm_provider": "clf_ai_gateway",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true,
|
||||
"reasoning_effort_levels": [
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh",
|
||||
"max"
|
||||
],
|
||||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://clfaigateway.dev/models"
|
||||
},
|
||||
"clf_ai_gateway/glm-4.7-flash": {
|
||||
"metadata": {
|
||||
"comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends."
|
||||
},
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 2.4e-08,
|
||||
"output_cost_per_token": 1.6e-07,
|
||||
"litellm_provider": "clf_ai_gateway",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true,
|
||||
"reasoning_effort_levels": [
|
||||
"low",
|
||||
"medium",
|
||||
"high"
|
||||
],
|
||||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"source": "https://clfaigateway.dev/models"
|
||||
},
|
||||
"clf_ai_gateway/deepseek-v4-pro": {
|
||||
"metadata": {
|
||||
"comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends."
|
||||
},
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 5.28e-07,
|
||||
"output_cost_per_token": 1.584e-06,
|
||||
"cache_read_input_token_cost": 1.7e-08,
|
||||
"litellm_provider": "clf_ai_gateway",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true,
|
||||
"reasoning_effort_levels": [
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh",
|
||||
"max"
|
||||
],
|
||||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://clfaigateway.dev/models"
|
||||
},
|
||||
"clf_ai_gateway/deepseek-v4-flash": {
|
||||
"metadata": {
|
||||
"comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends."
|
||||
},
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 1.76e-07,
|
||||
"output_cost_per_token": 5.28e-07,
|
||||
"cache_read_input_token_cost": 5e-09,
|
||||
"litellm_provider": "clf_ai_gateway",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true,
|
||||
"reasoning_effort_levels": [
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh",
|
||||
"max"
|
||||
],
|
||||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_prompt_caching": true,
|
||||
"source": "https://clfaigateway.dev/models"
|
||||
},
|
||||
"clf_ai_gateway/deepseek-v4.1-flash": {
|
||||
"metadata": {
|
||||
"comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends."
|
||||
},
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"output_cost_per_token": 4.8e-07,
|
||||
"cache_read_input_token_cost": 2e-09,
|
||||
"litellm_provider": "clf_ai_gateway",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true,
|
||||
"reasoning_effort_levels": [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh",
|
||||
"max"
|
||||
],
|
||||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_vision": true,
|
||||
"source": "https://clfaigateway.dev/models"
|
||||
},
|
||||
"clf_ai_gateway/kimi-k2.6": {
|
||||
"metadata": {
|
||||
"comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends."
|
||||
},
|
||||
"max_input_tokens": 262144,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 3.8e-07,
|
||||
"output_cost_per_token": 1.6e-06,
|
||||
"cache_read_input_token_cost": 6.4e-08,
|
||||
"litellm_provider": "clf_ai_gateway",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true,
|
||||
"reasoning_effort_levels": [
|
||||
"none",
|
||||
"low",
|
||||
"medium",
|
||||
"high"
|
||||
],
|
||||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_vision": true,
|
||||
"source": "https://clfaigateway.dev/models"
|
||||
},
|
||||
"clf_ai_gateway/kimi-k2.7-code": {
|
||||
"metadata": {
|
||||
"comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends."
|
||||
},
|
||||
"max_input_tokens": 262144,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 3.8e-07,
|
||||
"output_cost_per_token": 1.6e-06,
|
||||
"cache_read_input_token_cost": 7.6e-08,
|
||||
"litellm_provider": "clf_ai_gateway",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true,
|
||||
"reasoning_effort_levels": [
|
||||
"low",
|
||||
"medium",
|
||||
"high"
|
||||
],
|
||||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_vision": true,
|
||||
"source": "https://clfaigateway.dev/models"
|
||||
},
|
||||
"clf_ai_gateway/qwen3.8-27b": {
|
||||
"metadata": {
|
||||
"comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends."
|
||||
},
|
||||
"max_input_tokens": 262144,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 1.8e-07,
|
||||
"output_cost_per_token": 1.28e-06,
|
||||
"cache_read_input_token_cost": 2e-08,
|
||||
"litellm_provider": "clf_ai_gateway",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true,
|
||||
"reasoning_effort_levels": [
|
||||
"low",
|
||||
"medium",
|
||||
"xhigh"
|
||||
],
|
||||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_vision": true,
|
||||
"source": "https://clfaigateway.dev/models"
|
||||
},
|
||||
"kyma/claude-fable-5": {
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
|
|
@ -80538,5 +80884,22 @@
|
|||
"/v1/messages"
|
||||
],
|
||||
"supports_native_streaming": true
|
||||
},
|
||||
"fireworks_ai/accounts/fireworks/models/ultramarine-1-flash-us": {
|
||||
"cache_read_input_token_cost": 4.5e-08,
|
||||
"cache_read_input_token_cost_priority": 5.625e-08,
|
||||
"input_cost_per_token": 2.25e-07,
|
||||
"input_cost_per_token_priority": 2.8125e-07,
|
||||
"litellm_provider": "fireworks_ai",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 1048576,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 7.5e-07,
|
||||
"output_cost_per_token_priority": 9.375e-07,
|
||||
"source": "https://docs.fireworks.ai/serverless/pricing",
|
||||
"supports_function_calling": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": false,
|
||||
"supports_vision": true
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -827,6 +827,32 @@
|
|||
"generateContent": false
|
||||
}
|
||||
},
|
||||
"clf_ai_gateway": {
|
||||
"display_name": "CLF AI Gateway (`clf_ai_gateway`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/clf_ai_gateway",
|
||||
"endpoints": {
|
||||
"chat_completions": true,
|
||||
"responses": true,
|
||||
"text_completion": true,
|
||||
"embeddings": false,
|
||||
"image_generations": false,
|
||||
"image_edits": false,
|
||||
"audio_transcriptions": false,
|
||||
"audio_speech": false,
|
||||
"rerank": false,
|
||||
"ocr": false,
|
||||
"search": false,
|
||||
"moderations": false,
|
||||
"realtime": false,
|
||||
"video_generations": false,
|
||||
"batches": false,
|
||||
"messages": true,
|
||||
"interactions": true,
|
||||
"systemone": false,
|
||||
"a2a": true,
|
||||
"generateContent": true
|
||||
}
|
||||
},
|
||||
"cloudflare": {
|
||||
"display_name": "Cloudflare AI Workers (`cloudflare`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/cloudflare_workers",
|
||||
|
|
@ -1148,9 +1174,9 @@
|
|||
"text_completion": true,
|
||||
"embeddings": false,
|
||||
"image_generations": true,
|
||||
"image_edits": false,
|
||||
"image_edits": true,
|
||||
"audio_transcriptions": false,
|
||||
"audio_speech": false,
|
||||
"audio_speech": true,
|
||||
"rerank": false,
|
||||
"ocr": false,
|
||||
"search": false,
|
||||
|
|
@ -2241,6 +2267,32 @@
|
|||
"generateContent": false
|
||||
}
|
||||
},
|
||||
"eu_jev": {
|
||||
"display_name": "eu/jev (`eu_jev`)",
|
||||
"url": "https://docs.litellm.ai/docs/pass_through/eu_jev",
|
||||
"endpoints": {
|
||||
"chat_completions": false,
|
||||
"responses": false,
|
||||
"text_completion": false,
|
||||
"embeddings": false,
|
||||
"image_generations": false,
|
||||
"image_edits": false,
|
||||
"audio_transcriptions": false,
|
||||
"audio_speech": false,
|
||||
"rerank": false,
|
||||
"ocr": false,
|
||||
"search": false,
|
||||
"moderations": false,
|
||||
"realtime": false,
|
||||
"video_generations": false,
|
||||
"batches": false,
|
||||
"messages": false,
|
||||
"interactions": false,
|
||||
"systemone": true,
|
||||
"a2a": false,
|
||||
"generateContent": false
|
||||
}
|
||||
},
|
||||
"laya": {
|
||||
"display_name": "Laya (`laya`)",
|
||||
"url": "https://docs.litellm.ai/docs/auto_router/decision_classifiers",
|
||||
|
|
|
|||
|
|
@ -238,6 +238,7 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = (
|
|||
"/tinyfish/",
|
||||
"/transcribe",
|
||||
"/typesafe/",
|
||||
"/eu_jev/",
|
||||
"/laya/",
|
||||
"/bespoke/",
|
||||
"/openrouter/",
|
||||
|
|
|
|||
|
|
@ -18332,6 +18332,7 @@
|
|||
{
|
||||
"enum": [
|
||||
"block",
|
||||
"mask",
|
||||
"monitor",
|
||||
"inject_system_message"
|
||||
],
|
||||
|
|
@ -18342,7 +18343,7 @@
|
|||
}
|
||||
],
|
||||
"default": "block",
|
||||
"description": "Action to take when content is flagged: 'block' (raise exception), 'monitor' (log only), or 'inject_system_message' (append an advisory system message and let the LLM decide)",
|
||||
"description": "Action to take when content is flagged: 'block' (raise exception), 'mask' (mask PII-only detections and allow, block anything else), 'monitor' (log only), or 'inject_system_message' (append an advisory system message and let the LLM decide)",
|
||||
"title": "On Flagged"
|
||||
},
|
||||
"on_flagged_action": {
|
||||
|
|
@ -23169,6 +23170,39 @@
|
|||
"title": "CachedTokensDetails",
|
||||
"type": "object"
|
||||
},
|
||||
"ChoiceAnswer": {
|
||||
"additionalProperties": true,
|
||||
"properties": {
|
||||
"choice": {
|
||||
"title": "Choice",
|
||||
"type": "string"
|
||||
},
|
||||
"confidence": {
|
||||
"title": "Confidence",
|
||||
"type": "number"
|
||||
},
|
||||
"probabilities": {
|
||||
"additionalProperties": {
|
||||
"type": "number"
|
||||
},
|
||||
"title": "Probabilities",
|
||||
"type": "object"
|
||||
},
|
||||
"type": {
|
||||
"const": "choice",
|
||||
"title": "Type",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"choice",
|
||||
"confidence",
|
||||
"probabilities"
|
||||
],
|
||||
"title": "ChoiceAnswer",
|
||||
"type": "object"
|
||||
},
|
||||
"Click": {
|
||||
"additionalProperties": true,
|
||||
"description": "A click action.",
|
||||
|
|
@ -23933,6 +23967,112 @@
|
|||
"title": "ErrorResponse",
|
||||
"type": "object"
|
||||
},
|
||||
"EuJevDecisionResponse": {
|
||||
"properties": {
|
||||
"answers": {
|
||||
"additionalProperties": {
|
||||
"discriminator": {
|
||||
"mapping": {
|
||||
"choice": "#/components/schemas/ChoiceAnswer",
|
||||
"noul": "#/components/schemas/NoulAnswer",
|
||||
"score": "#/components/schemas/ScoreAnswer"
|
||||
},
|
||||
"propertyName": "type"
|
||||
},
|
||||
"oneOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/ChoiceAnswer"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/NoulAnswer"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/ScoreAnswer"
|
||||
}
|
||||
]
|
||||
},
|
||||
"description": "One answer per question, under the same names",
|
||||
"title": "Answers",
|
||||
"type": "object"
|
||||
},
|
||||
"meta": {
|
||||
"$ref": "#/components/schemas/EuJevMeta"
|
||||
},
|
||||
"model": {
|
||||
"description": "The release that answered, such as eu-jeff-1.0.0",
|
||||
"title": "Model",
|
||||
"type": "string"
|
||||
},
|
||||
"usage": {
|
||||
"$ref": "#/components/schemas/EuJevUsage"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"model",
|
||||
"answers",
|
||||
"usage",
|
||||
"meta"
|
||||
],
|
||||
"title": "EuJevDecisionResponse",
|
||||
"type": "object"
|
||||
},
|
||||
"EuJevMeta": {
|
||||
"properties": {
|
||||
"cost_eur": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "The exact charge in euros, on authenticated requests",
|
||||
"title": "Cost Eur"
|
||||
},
|
||||
"latency_ms": {
|
||||
"description": "Server-side time for token quoting, credit reservation and inference",
|
||||
"title": "Latency Ms",
|
||||
"type": "integer"
|
||||
},
|
||||
"mode": {
|
||||
"const": "live",
|
||||
"title": "Mode",
|
||||
"type": "string"
|
||||
},
|
||||
"request_id": {
|
||||
"title": "Request Id",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"request_id",
|
||||
"mode",
|
||||
"latency_ms"
|
||||
],
|
||||
"title": "EuJevMeta",
|
||||
"type": "object"
|
||||
},
|
||||
"EuJevUsage": {
|
||||
"properties": {
|
||||
"input_tokens": {
|
||||
"minimum": 0.0,
|
||||
"title": "Input Tokens",
|
||||
"type": "integer"
|
||||
},
|
||||
"output_tokens": {
|
||||
"minimum": 0.0,
|
||||
"title": "Output Tokens",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"input_tokens",
|
||||
"output_tokens"
|
||||
],
|
||||
"title": "EuJevUsage",
|
||||
"type": "object"
|
||||
},
|
||||
"FileSearchTool": {
|
||||
"additionalProperties": true,
|
||||
"description": "A tool that searches for relevant content from uploaded files.\n\nLearn more about the [file search tool](https://platform.openai.com/docs/guides/tools-file-search).",
|
||||
|
|
@ -25593,6 +25733,26 @@
|
|||
"title": "NamespaceTool",
|
||||
"type": "object"
|
||||
},
|
||||
"NoulAnswer": {
|
||||
"additionalProperties": true,
|
||||
"properties": {
|
||||
"noul": {
|
||||
"title": "Noul",
|
||||
"type": "number"
|
||||
},
|
||||
"type": {
|
||||
"const": "noul",
|
||||
"title": "Type",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"noul"
|
||||
],
|
||||
"title": "NoulAnswer",
|
||||
"type": "object"
|
||||
},
|
||||
"OperationCreateFile": {
|
||||
"additionalProperties": true,
|
||||
"description": "Instruction describing how to create a file via the apply_patch tool.",
|
||||
|
|
@ -28865,6 +29025,59 @@
|
|||
"title": "Result",
|
||||
"type": "object"
|
||||
},
|
||||
"ScoreAnswer": {
|
||||
"additionalProperties": true,
|
||||
"properties": {
|
||||
"confidence": {
|
||||
"title": "Confidence",
|
||||
"type": "number"
|
||||
},
|
||||
"legend": {
|
||||
"additionalProperties": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"items": {},
|
||||
"type": "array"
|
||||
}
|
||||
]
|
||||
},
|
||||
"title": "Legend",
|
||||
"type": "object"
|
||||
},
|
||||
"probabilities": {
|
||||
"additionalProperties": {
|
||||
"type": "number"
|
||||
},
|
||||
"title": "Probabilities",
|
||||
"type": "object"
|
||||
},
|
||||
"score": {
|
||||
"title": "Score",
|
||||
"type": "number"
|
||||
},
|
||||
"type": {
|
||||
"const": "score",
|
||||
"title": "Type",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"score",
|
||||
"confidence",
|
||||
"legend",
|
||||
"probabilities"
|
||||
],
|
||||
"title": "ScoreAnswer",
|
||||
"type": "object"
|
||||
},
|
||||
"Screenshot": {
|
||||
"additionalProperties": true,
|
||||
"description": "A screenshot action.",
|
||||
|
|
@ -31757,6 +31970,315 @@
|
|||
]
|
||||
}
|
||||
},
|
||||
"/eu_jev/v1/systemone": {
|
||||
"post": {
|
||||
"description": "Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`.\n\n`POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`)\nand returns one answer per question name, with spend booked from the reported `usage`. `custom_body` and\n`stream` are rejected with a 400. A pass-through endpoint configured at this path keeps serving it.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)",
|
||||
"operationId": "eu_jev_proxy_route_eu_jev_v1_systemone_post",
|
||||
"requestBody": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"example": {
|
||||
"model": "eu-jeff-latest",
|
||||
"questions": {
|
||||
"frustration": {
|
||||
"criteria": [
|
||||
"calm",
|
||||
"mildly annoyed",
|
||||
"angry"
|
||||
],
|
||||
"instructions": "How frustrated is the customer?",
|
||||
"type": "score"
|
||||
},
|
||||
"is_refund_request": {
|
||||
"instructions": "Is the customer asking for a refund?",
|
||||
"type": "noul"
|
||||
},
|
||||
"team": {
|
||||
"criteria": {
|
||||
"billing": "Payments, invoices, and refunds",
|
||||
"support": "Technical issues and bugs"
|
||||
},
|
||||
"instructions": "Which team should handle this?",
|
||||
"type": "choice"
|
||||
}
|
||||
},
|
||||
"state": "I was charged twice. Can I get a refund?"
|
||||
},
|
||||
"schema": {
|
||||
"properties": {
|
||||
"model": {
|
||||
"default": "eu-jeff-latest",
|
||||
"description": "eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it",
|
||||
"title": "Model",
|
||||
"type": "string"
|
||||
},
|
||||
"questions": {
|
||||
"additionalProperties": {
|
||||
"discriminator": {
|
||||
"propertyName": "type"
|
||||
},
|
||||
"oneOf": [
|
||||
{
|
||||
"properties": {
|
||||
"criteria": {
|
||||
"additionalProperties": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"items": {},
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference"
|
||||
},
|
||||
"description": "Two to 64 options, option name to description. A null description uses the option name",
|
||||
"maxProperties": 64,
|
||||
"minProperties": 2,
|
||||
"title": "Criteria",
|
||||
"type": "object"
|
||||
},
|
||||
"instructions": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"items": {},
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference",
|
||||
"title": "Instructions"
|
||||
},
|
||||
"type": {
|
||||
"const": "choice",
|
||||
"title": "Type",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"instructions",
|
||||
"criteria"
|
||||
],
|
||||
"title": "EuJevChoiceQuestion",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"criteria": {
|
||||
"anyOf": [
|
||||
{
|
||||
"additionalProperties": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"items": {},
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"propertyNames": {
|
||||
"enum": [
|
||||
"true",
|
||||
"false"
|
||||
]
|
||||
},
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "Optional descriptions of what true and false mean",
|
||||
"title": "Criteria"
|
||||
},
|
||||
"instructions": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"items": {},
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference",
|
||||
"title": "Instructions"
|
||||
},
|
||||
"type": {
|
||||
"const": "noul",
|
||||
"title": "Type",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"instructions"
|
||||
],
|
||||
"title": "EuJevNoulQuestion",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"criteria": {
|
||||
"description": "Two to 10 level descriptions, ordered from low to high",
|
||||
"items": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"items": {},
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference"
|
||||
},
|
||||
"maxItems": 10,
|
||||
"minItems": 2,
|
||||
"title": "Criteria",
|
||||
"type": "array"
|
||||
},
|
||||
"instructions": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"items": {},
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference",
|
||||
"title": "Instructions"
|
||||
},
|
||||
"type": {
|
||||
"const": "score",
|
||||
"title": "Type",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"instructions",
|
||||
"criteria"
|
||||
],
|
||||
"title": "EuJevScoreQuestion",
|
||||
"type": "object"
|
||||
}
|
||||
]
|
||||
},
|
||||
"description": "One to eight questions, keyed by names of up to 80 characters",
|
||||
"maxProperties": 8,
|
||||
"minProperties": 1,
|
||||
"propertyNames": {
|
||||
"maxLength": 80,
|
||||
"minLength": 1
|
||||
},
|
||||
"title": "Questions",
|
||||
"type": "object"
|
||||
},
|
||||
"state": {
|
||||
"anyOf": [
|
||||
{
|
||||
"maxLength": 100000,
|
||||
"minLength": 1,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"items": {},
|
||||
"type": "array"
|
||||
}
|
||||
],
|
||||
"description": "The content to evaluate, a string of up to 100,000 characters or a JSON object or array",
|
||||
"title": "State"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"state",
|
||||
"questions"
|
||||
],
|
||||
"title": "EuJevDecisionRequest",
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": true
|
||||
},
|
||||
"responses": {
|
||||
"200": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/EuJevDecisionResponse"
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "Successful Response"
|
||||
}
|
||||
},
|
||||
"security": [
|
||||
{
|
||||
"APIKeyHeader": []
|
||||
}
|
||||
],
|
||||
"summary": "Eu Jev Proxy Route",
|
||||
"tags": [
|
||||
"llm_passthrough"
|
||||
]
|
||||
}
|
||||
},
|
||||
"/fal_ai/{endpoint}": {
|
||||
"delete": {
|
||||
"operationId": "fal_ai_proxy_route_fal_ai__endpoint__delete",
|
||||
|
|
|
|||
|
|
@ -3177,11 +3177,18 @@ from litellm.types.proxy.auth.user_api_key_auth import (
|
|||
)
|
||||
|
||||
|
||||
class AdminMCPPermissions(LiteLLMPydanticObjectBase):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
tools: tuple[str, ...]
|
||||
action_allowed: bool | None = None
|
||||
|
||||
|
||||
class UserInfoResponse(LiteLLMPydanticObjectBase):
|
||||
user_id: str | None
|
||||
user_info: dict | BaseModel | None
|
||||
keys: list
|
||||
teams: list
|
||||
admin_mcp_permissions: AdminMCPPermissions | None = None
|
||||
|
||||
|
||||
class UserInfoV2Response(LiteLLMPydanticObjectBase):
|
||||
|
|
|
|||
|
|
@ -1,20 +1,21 @@
|
|||
import os
|
||||
import re
|
||||
from collections.abc import AsyncGenerator, Mapping
|
||||
from collections.abc import AsyncGenerator, Iterator, Mapping
|
||||
from contextlib import asynccontextmanager
|
||||
from contextvars import ContextVar
|
||||
from typing import Final
|
||||
from urllib.parse import urlsplit
|
||||
from urllib.parse import unquote, urlsplit
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi import FastAPI, HTTPException, Request
|
||||
from fastapi.routing import APIRoute
|
||||
from pydantic import TypeAdapter
|
||||
from starlette.datastructures import Headers
|
||||
from starlette.requests import Request
|
||||
from starlette.routing import Mount
|
||||
from starlette.types import ASGIApp, Receive, Scope, Send
|
||||
|
||||
from litellm.constants import STANDARD_CUSTOMER_ID_HEADERS
|
||||
from litellm.proxy._types import SpecialHeaders
|
||||
from litellm.proxy._types import AdminMCPPermissions, SpecialHeaders, UserAPIKeyAuth
|
||||
from litellm.proxy.auth.route_checks import RouteChecks
|
||||
from litellm.proxy.middleware.admission_control_middleware import ADMISSION_LEASE_SCOPE_KEY
|
||||
|
||||
_REQUEST_HEADERS: Final = frozenset(
|
||||
|
|
@ -58,6 +59,91 @@ _RESERVED_KEY_HEADERS: Final = (
|
|||
_SETTINGS: Final = TypeAdapter(Mapping[str, object])
|
||||
_IDENTITY_MAPPINGS: Final = TypeAdapter(tuple[dict[str, object], ...] | dict[str, object] | None)
|
||||
_OAUTH_MAPPINGS: Final = TypeAdapter(dict[str, str])
|
||||
_ROUTE_GRANTS: Final = TypeAdapter(tuple[str, ...])
|
||||
|
||||
|
||||
def _admin_route_allowed(auth: UserAPIKeyAuth, request: Request, method: str, path: str) -> bool:
|
||||
target: Final = Request({**request.scope, "method": method, "path": path, "raw_path": path.encode()})
|
||||
try:
|
||||
RouteChecks.should_call_route(route=path, valid_token=auth, request=target)
|
||||
except HTTPException:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _admin_routes(request: Request) -> Iterator[tuple[tuple[str, str], APIRoute]]:
|
||||
app: Final = request.app
|
||||
if not isinstance(app, FastAPI):
|
||||
return
|
||||
for route in app.routes:
|
||||
if not isinstance(route, APIRoute):
|
||||
continue
|
||||
for method in route.methods:
|
||||
yield (method, route.path_format), route
|
||||
|
||||
|
||||
def _admin_grant_paths(route: APIRoute, grant: str) -> Iterator[str]:
|
||||
if route.path_regex.fullmatch(grant) is not None:
|
||||
yield grant
|
||||
prefix: Final = grant[:-1] if grant.endswith("*") else grant + "/"
|
||||
representative: Final = re.sub(r"\{[^}]+\}", "_", route.path_format)
|
||||
# A matching suffix supplies the rest of a concrete route for prefix grants,
|
||||
# including grants ending inside a static segment or a path parameter.
|
||||
for candidate in (prefix + representative[offset:] for offset in range(len(representative) + 1)):
|
||||
if route.path_regex.fullmatch(candidate) is not None:
|
||||
yield candidate
|
||||
|
||||
|
||||
def _admin_grant_allowed(auth: UserAPIKeyAuth, request: Request, method: str, route: APIRoute, grant: str) -> bool:
|
||||
return any(_admin_route_allowed(auth, request, method, path) for path in _admin_grant_paths(route, grant))
|
||||
|
||||
|
||||
def _admin_tool_visible(auth: UserAPIKeyAuth, request: Request, method: str, route: APIRoute) -> bool:
|
||||
try:
|
||||
RouteChecks.check_route_enabled(route.path)
|
||||
except HTTPException:
|
||||
return False
|
||||
if _admin_route_allowed(auth, request, method, route.path):
|
||||
return True
|
||||
grants: Final = _ROUTE_GRANTS.validate_python(getattr(auth, "allowed_routes", None) or ())
|
||||
return bool(route.param_convertors) and any(
|
||||
_admin_grant_allowed(auth, request, method, route, grant) for grant in grants if grant.startswith("/")
|
||||
)
|
||||
|
||||
|
||||
def get_admin_mcp_permissions(
|
||||
auth: UserAPIKeyAuth, request: Request, tool: str | None = None, path: str | None = None
|
||||
) -> AdminMCPPermissions:
|
||||
try:
|
||||
from litellm_admin_mcp.catalog import ( # pyright: ignore[reportMissingTypeStubs] # connector has no py.typed marker
|
||||
BY_NAME,
|
||||
OPERATIONS,
|
||||
)
|
||||
except ImportError as exc:
|
||||
raise HTTPException(
|
||||
status_code=503, detail="Admin MCP permissions require the admin-mcp dependency group (Python 3.12+)"
|
||||
) from exc
|
||||
|
||||
routes: Final = dict(_admin_routes(request))
|
||||
allowed: Final = tuple(
|
||||
operation.name
|
||||
for operation in OPERATIONS
|
||||
if (route := routes.get((operation.method, operation.path))) is not None
|
||||
and _admin_tool_visible(auth, request, operation.method, route)
|
||||
)
|
||||
if tool is None:
|
||||
return AdminMCPPermissions(tools=allowed)
|
||||
operation: Final = BY_NAME.get(tool)
|
||||
if operation is None or tool not in allowed:
|
||||
return AdminMCPPermissions(tools=allowed, action_allowed=False)
|
||||
if path is None:
|
||||
return AdminMCPPermissions(tools=allowed, action_allowed=True)
|
||||
registered: Final = routes[(operation.method, operation.path)]
|
||||
decoded_path: Final = unquote(path)
|
||||
action_allowed: Final = registered.path_regex.fullmatch(decoded_path) is not None and _admin_route_allowed(
|
||||
auth, request, operation.method, decoded_path
|
||||
)
|
||||
return AdminMCPPermissions(tools=allowed, action_allowed=action_allowed)
|
||||
|
||||
|
||||
def _configured_key_header() -> bytes | None:
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ from litellm.proxy.auth.auth_utils import ( # noqa: F401 # legacy module expor
|
|||
)
|
||||
from litellm.proxy.auth.model_access_denied import ModelAccessDeniedHTTPException
|
||||
from litellm.proxy.auth.resolvers.exceptions import KeyNotFoundError
|
||||
from litellm.proxy.common_utils.openai_error_payload import openai_error_param
|
||||
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
|
||||
from litellm.types.services import ServiceTypes
|
||||
|
||||
|
|
@ -61,14 +62,14 @@ def _as_proxy_exception(e: Exception) -> ProxyException:
|
|||
message=str(e.detail),
|
||||
internal_message=e.internal_message,
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param="None",
|
||||
param=None,
|
||||
code=e.status_code,
|
||||
)
|
||||
if isinstance(e, HTTPException):
|
||||
return ProxyException(
|
||||
message=getattr(e, "detail", f"Authentication Error({e})"),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
param=openai_error_param(e),
|
||||
code=getattr(e, "status_code", status.HTTP_401_UNAUTHORIZED),
|
||||
)
|
||||
if isinstance(e, ProxyException):
|
||||
|
|
@ -78,7 +79,7 @@ def _as_proxy_exception(e: Exception) -> ProxyException:
|
|||
return ProxyException(
|
||||
message="Authentication Error, " + str(e),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
param=openai_error_param(e),
|
||||
code=status.HTTP_401_UNAUTHORIZED,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ from litellm.litellm_core_utils.url_utils import (
|
|||
from litellm.llms.azure.passthrough.transformation import azure_router_model_in_endpoint
|
||||
from litellm.llms.nvidia_nim.passthrough.transformation import nvidia_nim_model_group_in_path
|
||||
from litellm.proxy._types import *
|
||||
from litellm.proxy.auth.route_checks import RouteChecks
|
||||
from litellm.proxy.common_utils.http_parsing_utils import extract_nested_form_metadata
|
||||
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
|
||||
LITELLM_PASS_THROUGH_ENDPOINT_MARKER,
|
||||
|
|
@ -884,7 +885,6 @@ def route_in_additonal_public_routes(current_route: str):
|
|||
public_routes: ["LiteLLMRoutes.public_routes", "/spend/calculate", "/api/*"]
|
||||
```
|
||||
"""
|
||||
from litellm.proxy.auth.route_checks import RouteChecks
|
||||
from litellm.proxy.proxy_server import general_settings, premium_user
|
||||
|
||||
try:
|
||||
|
|
@ -2175,6 +2175,13 @@ def _extract_model_candidates_from_request(
|
|||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
return _dedupe_model_candidates((f"{provider}/{decision_model}",))
|
||||
if RouteChecks.is_builtin_eu_jev_decision_route(route=route):
|
||||
from litellm.llms.eu_jev.common_utils import eu_jev_registry_model
|
||||
|
||||
requested_model: Final = TypeAdapter(Mapping[str, object]).validate_python(request_data).get("model")
|
||||
if requested_model is not None and not isinstance(requested_model, str):
|
||||
raise HTTPException(status_code=400, detail="eu_jev model must be a string")
|
||||
return [f"eu_jev/{eu_jev_registry_model(requested_model)}"]
|
||||
if route == "/cost/predict-cache":
|
||||
prediction_models: Final = _cache_prediction_model_candidates(request_data, llm_router, team_id) # pyright: ignore[reportUnknownArgumentType] # the typed reader validates each deployment ID from this legacy payload
|
||||
return _dedupe_model_candidates(prediction_models)
|
||||
|
|
|
|||
|
|
@ -225,7 +225,7 @@ def jwks_unavailable_exception(error: JWKSUnreachableError) -> ProxyException:
|
|||
f"unreachable, so the JWT signature could not be verified. Please retry shortly. Error: {error}"
|
||||
),
|
||||
type=ProxyErrorTypes.auth_provider_unavailable,
|
||||
param="None",
|
||||
param=None,
|
||||
code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
)
|
||||
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue