scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-08 22:21:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 78

Merge remote-tracking branch 'upstream/litellm_internal_staging' into litellm_bedrock_mantle_native_web_search

Browse source
This commit is contained in:
longwind48 2026-08-25 13:26:14 +08:00
commit 19d043b268
370 changed files with 18351 additions and 1704 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

195
.circleci/config.yml
View file

@ -430,7 +430,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=./litellm \
--cov=./litellm --cov=./enterprise/litellm_enterprise \
--cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=20 \
@ -504,7 +504,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=./litellm \
--cov=./litellm --cov=./enterprise/litellm_enterprise \
--cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=20 \
@ -631,7 +631,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 2"
@ -651,126 +651,6 @@ jobs:
- auth_ui_unit_tests_coverage.xml
- auth_ui_unit_tests_coverage
proxy_behavior_tests:
docker:
- *python312_image
- image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
environment:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: litellm_test
working_directory: ~/project
environment:
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- install_rust
- run:
name: Install Dependencies
command: |
uv sync --frozen --all-groups --all-extras --python 3.12
- wait_for_service:
url: tcp://localhost:5432
timeout: "60"
- run:
name: Seed DB schema via prisma db push
command: |
uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
- run:
name: Generate Prisma Client
command: uv run --no-sync python -m prisma generate
- run:
name: Run proxy management behavior tests
command: |
mkdir -p test-results
uv run --no-sync python -m pytest tests/proxy_behavior \
-v --junitxml=test-results/junit.xml --durations=10
no_output_timeout: 15m
- store_test_results:
path: test-results
proxy_security_tests:
docker:
- *python312_image
- image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
environment:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: litellm_test
working_directory: ~/project
environment:
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- install_rust
- run:
name: Install Dependencies
command: |
uv sync --frozen --all-groups --all-extras --python 3.12
- wait_for_service:
url: tcp://localhost:5432
timeout: "60"
- run:
name: Seed DB schema via prisma db push
command: |
uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
- run:
name: Generate Prisma Client
command: uv run --no-sync python -m prisma generate
- run:
name: Run proxy security tests
command: |
mkdir -p test-results
uv run --no-sync python -m pytest tests/proxy_security_tests \
-v --junitxml=test-results/junit.xml --durations=10
no_output_timeout: 15m
- store_test_results:
path: test-results
schema_migration_check:
docker:
- *python312_image
- image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
environment:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: litellm_test
working_directory: ~/project
environment:
# An empty database; the test applies every committed migration itself.
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- install_rust
- run:
name: Install Dependencies
command: |
uv sync --frozen --all-groups --all-extras --python 3.12
- wait_for_service:
url: tcp://localhost:5432
timeout: "60"
- run:
name: Generate Prisma Client
command: uv run --no-sync python -m prisma generate
- run:
name: Check schema.prisma is in sync with committed migrations
command: |
mkdir -p test-results
uv run --no-sync python -m pytest tests/proxy_migration_tests \
-v --junitxml=test-results/junit.xml --durations=10
no_output_timeout: 15m
- store_test_results:
path: test-results
litellm_router_testing: # Runs all tests with the "router" keyword
docker:
- *python312_image
@ -858,7 +738,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
@ -985,7 +865,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=20 \
-n 4 \
@ -1030,7 +910,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
@ -1074,7 +954,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 2 \
@ -1120,7 +1000,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
--retries 3 --retry-delay 5"
@ -1211,7 +1091,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
@ -1255,7 +1135,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
@ -1274,40 +1154,6 @@ jobs:
paths:
- search_coverage.xml
- search_coverage
litellm_mapped_enterprise_tests:
docker:
- *python312_image
working_directory: ~/project
resource_class: large
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- install_rust
- run:
name: Install Dependencies
command: |
uv sync --frozen --all-groups --all-extras --python 3.12
- setup_litellm_enterprise_pip
- run:
name: Run enterprise tests
command: |
uv run --no-sync python -m prisma generate
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/enterprise/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--junitxml=test-results/junit-enterprise.xml \
--durations=10 \
-n 4"
no_output_timeout: 15m
# Store test results
- store_test_results:
path: test-results
batches_testing:
docker:
- *python312_image
@ -1333,7 +1179,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 2"
@ -1377,7 +1223,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 2"
@ -1422,7 +1268,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
@ -1501,7 +1347,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
-n 4 \
--junitxml=test-results/junit.xml \
--durations=5 \
@ -1546,7 +1392,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
@ -1599,7 +1445,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 -n 2 \
--reruns 2 --reruns-delay 1"
@ -3105,12 +2951,6 @@ workflows:
filters: *main_branches
- auth_ui_unit_tests:
filters: *main_branches
- proxy_behavior_tests:
filters: *main_branches
- proxy_security_tests:
filters: *main_branches
- schema_migration_check:
filters: *main_branches
- build_docker_database_image:
filters: *main_branches
- e2e_ui_testing:
@ -3167,8 +3007,6 @@ workflows:
filters: *main_branches
- search_testing:
filters: *main_branches
- litellm_mapped_enterprise_tests:
filters: *main_branches
- batches_testing:
filters: *main_branches
- litellm_utils_testing:
@ -3191,7 +3029,6 @@ workflows:
- guardrails_testing
- ocr_testing
- search_testing
- litellm_mapped_enterprise_tests
- batches_testing
- litellm_utils_testing
- pass_through_unit_testing

3
.github/CODEOWNERS vendored
View file

@ -1,6 +1,9 @@
/ui/ @yuneng-berri @ryan-crabbe-berri
/litellm/proxy/_experimental/out/ @yuneng-berri @ryan-crabbe-berri
/ui/Dockerfile
/ui/nginx.conf
/ui/litellm-dashboard/src/lib/http/schema.d.ts
/ui/litellm-dashboard/tsconfig.tsbuildinfo
/model_prices_and_context_window.json @mateo-berri
/litellm/model_prices_and_context_window_backup.json @mateo-berri
/litellm-proxy-extras/litellm_proxy_extras/migrations/ @yuneng-berri @ryan-crabbe-berri

36
.github/ci-coverage-allowlist.yml vendored
View file

@ -5,24 +5,21 @@ description: >-
test_paths:
- reason: >-
The caching suite in tests/local_testing, which runs nowhere. Every job that globs that
directory either deselects it (local_testing_part1 and part2 carry `-k "... and not caching
and not cache"`) or keeps only another keyword (langfuse, router, assistants), and no job
names these files the way redis_caching_unit_tests names test_dual_cache.py. Measured
2026-08-20 by collecting the directory under each job's own selector: 118 tests across
these eight files are selected by none of them. Listed so the gap is a decision rather
than an accident, and so the --slices guard has a baseline to ratchet down from. Revisit
when tests/local_testing is ported off CircleCI, where the keyless part of this suite
belongs in a real job
What is left of the caching suite in tests/local_testing that runs nowhere. Every job that
globs that directory either deselects it (local_testing_part1 and part2 carry `-k "... and
not caching and not cache"`) or keeps only another keyword (langfuse, router, assistants),
and no job names these files the way redis_caching_unit_tests names test_dual_cache.py.
The gap was eight files and 118 tests when measured 2026-08-20; the five keyless ones now
run in the caching-local shard, leaving these three. Measured 2026-08-21 with no provider
credentials and no Redis: test_caching.py needs both (37 of 65 fail without them),
test_disk_cache_unit_tests.py needs OPENAI_API_KEY for 2 of its 4, and
test_gcs_cache_unit_tests.py needs GCS credentials for all 4. They want the keyless/live
split that porting tests/local_testing off CircleCI will force, not a job that is red by
construction
paths:
- tests/local_testing/test_cache_preset_key.py
- tests/local_testing/test_caching.py
- tests/local_testing/test_caching_handler.py
- tests/local_testing/test_disk_cache_unit_tests.py
- tests/local_testing/test_gcs_cache_unit_tests.py
- tests/local_testing/test_prompt_caching.py
- tests/local_testing/test_responses_stream_cache_keys.py
- tests/local_testing/test_unit_test_caching.py
- reason: >-
The end-to-end suite runs against a deployed proxy from its own in-cluster rig rather than
from a pull request; it needs a live gateway and provider credentials no PR job holds
@ -92,17 +89,6 @@ test_paths:
- tests/integration/sandbox/test_e2b_sandbox.py
- tests/integration/test_oci_integration.py
- tests/integration/test_oci_proxy_integration.py
- reason: >-
A unit test for the proxy-extras package that no job invokes, while the package's other
tests live under tests/proxy_migration_tests. Measured 2026-08-20: 24 of its 28 tests pass
and the 4 in TestMigrationSQLIdempotency fail, because 13 migrations from 2026-03 onward use
bare CREATE TABLE, ADD COLUMN, CREATE INDEX and ADD CONSTRAINT rather than the guarded forms
this file requires. It also matches those keywords inside SQL comments, so two further
migrations are reported that are in fact fine. Wiring it up means deciding what to do about
the 13 first, and they cannot simply be edited: Prisma checksums an applied migration, so a
changed one breaks migrate deploy for existing installs
paths:
- tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py
dockerfiles:
- reason: >-

16
.github/scripts/assert_ci_coverage.py vendored
View file

@ -312,8 +312,23 @@ def _matchable_names(relative_path: str) -> frozenset[str]:
)
def _workflow_named_tokens() -> frozenset[str]:
"""Test tokens a GitHub Actions job names directly.
A CircleCI `-k` that deselects a file no longer means the file runs nowhere once a
workflow names it, so the slice check has to credit those the same way the census does.
"""
return _invoked_test_tokens(
scalar
for path in _config_files()
if path != CIRCLECI_CONFIG
for scalar in _scalars(yaml.safe_load(path.read_text(encoding="utf-8")), path.name)
)
def _deselected_everywhere(allowlist: Allowlist) -> tuple[Finding, ...]:
slices: Final = _slices()
named_by_workflow: Final = _workflow_named_tokens()
globbed: Final = tuple(
path
for path in _test_files()
@ -326,6 +341,7 @@ def _deselected_everywhere(allowlist: Allowlist) -> tuple[Finding, ...]:
)
for path in globbed
if not allowlist.covers_test(path)
and not any(_token_covers(token, path) for token in named_by_workflow)
and not any(slice_.claims(path, _matchable_names(path)) for slice_ in slices)
)

198
.github/scripts/e2e_egress_sentinel.py vendored Executable file
View file

@ -0,0 +1,198 @@
"""Prove an e2e replay run makes zero outbound provider calls, by counting them.
`serve` pins each provider host (`--host`) to a local sink address in the hosts
file and binds a counting listener on that address, so any connection the proxy
or the record/replay edge opens to a real provider is redirected to the sink,
recorded as one line in `--hits-file`, and never leaves the box. The record and
replay edge only ever dials `127.0.0.1:<edge-port>` (a different host than the
pinned provider names), so in a clean replay the sink sees nothing; a single hit
means a provider call escaped the bundle. `assert-empty` turns that hit file into
the pass/fail check.
Stdlib only, so CI runs it under the system interpreter as root (binding :443 and
editing the hosts file both need root); `--sink-address`, `--port`, and
`--hosts-file` are injectable so it runs unprivileged against a temp hosts file on
a high port under test.
"""
# ruff: noqa: T201 # CLI script: its stdout/stderr progress and results are the interface
from __future__ import annotations
import argparse
import json
import os
import signal
import socket
import sys
import threading
import time
from dataclasses import dataclass
from pathlib import Path
from types import FrameType
from typing import Final
_BLOCK_BEGIN: Final = "# BEGIN e2e-egress-sentinel"
_BLOCK_END: Final = "# END e2e-egress-sentinel"
@dataclass(frozen=True, slots=True)
class ServeConfig:
hosts: tuple[str, ...]
sink_address: str
ports: tuple[int, ...]
hits_file: Path
hosts_file: Path
ready_file: Path | None
pid_file: Path | None
def _pin_block(sink_address: str, hosts: tuple[str, ...]) -> str:
lines = "\n".join(f"{sink_address}\t{host}" for host in hosts)
return f"\n{_BLOCK_BEGIN}\n{lines}\n{_BLOCK_END}\n"
def _install_pins(hosts_file: Path, sink_address: str, hosts: tuple[str, ...]) -> bytes:
original = hosts_file.read_bytes() if hosts_file.exists() else b""
hosts_file.write_bytes(original + _pin_block(sink_address, hosts).encode())
return original
def _restore_pins(hosts_file: Path, original: bytes) -> None:
hosts_file.write_bytes(original)
def _bind(sink_address: str, port: int) -> socket.socket:
listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
listener.bind((sink_address, port))
listener.listen(128)
return listener
@dataclass(frozen=True, slots=True)
class _HitLog:
path: Path
_lock: threading.Lock
def record(self, *, port: int, peer: tuple[str, int]) -> None:
entry = json.dumps({"ts": time.time(), "port": port, "peer": list(peer)})
with self._lock:
with self.path.open("a", encoding="utf-8") as handle:
handle.write(entry + "\n")
def _serve_socket(listener: socket.socket, port: int, hits: _HitLog, stop: threading.Event) -> None:
while not stop.is_set():
try:
conn, peer = listener.accept()
except OSError:
return
hits.record(port=port, peer=(peer[0], peer[1]))
try:
conn.close()
except OSError:
pass
def serve(config: ServeConfig) -> int:
config.hits_file.write_text("", encoding="utf-8")
original_hosts = _install_pins(config.hosts_file, config.sink_address, config.hosts)
try:
listeners = tuple(_bind(config.sink_address, port) for port in config.ports)
except OSError as exc:
_restore_pins(config.hosts_file, original_hosts)
print(f"egress sentinel could not bind a sink: {exc}", file=sys.stderr)
return 1
stop = threading.Event()
hits = _HitLog(path=config.hits_file, _lock=threading.Lock())
threads = tuple(
threading.Thread(target=_serve_socket, args=(listener, port, hits, stop), daemon=True)
for listener, port in zip(listeners, config.ports)
)
for thread in threads:
thread.start()
def _handle(_signum: int, _frame: FrameType | None) -> None:
stop.set()
for listener in listeners:
try:
listener.close()
except OSError:
pass
signal.signal(signal.SIGTERM, _handle)
signal.signal(signal.SIGINT, _handle)
if config.pid_file is not None:
config.pid_file.write_text(str(os.getpid()), encoding="utf-8")
if config.ready_file is not None:
config.ready_file.write_text("ready", encoding="utf-8")
print(
f"egress sentinel up: pinned {', '.join(config.hosts)} to {config.sink_address} "
f"on port(s) {', '.join(str(p) for p in config.ports)}",
flush=True,
)
stop.wait()
_restore_pins(config.hosts_file, original_hosts)
if config.ready_file is not None and config.ready_file.exists():
config.ready_file.unlink()
if config.pid_file is not None and config.pid_file.exists():
config.pid_file.unlink()
return 0
def assert_empty(hits_file: Path) -> int:
if not hits_file.exists():
print(f"egress sentinel recorded no provider calls ({hits_file} absent): zero egress")
return 0
hits = [line for line in hits_file.read_text(encoding="utf-8").splitlines() if line.strip()]
if not hits:
print("egress sentinel recorded no provider calls: zero egress")
return 0
print(f"egress sentinel recorded {len(hits)} provider call(s); replay was not hermetic:", file=sys.stderr)
for line in hits:
print(f" {line}", file=sys.stderr)
return 1
def _serve_from_args(args: argparse.Namespace) -> int:
config = ServeConfig(
hosts=tuple(args.host),
sink_address=args.sink_address,
ports=tuple(args.port),
hits_file=Path(args.hits_file),
hosts_file=Path(args.hosts_file),
ready_file=Path(args.ready_file) if args.ready_file else None,
pid_file=Path(args.pid_file) if args.pid_file else None,
)
return serve(config)
def main(argv: tuple[str, ...]) -> int:
parser = argparse.ArgumentParser(description="count outbound provider calls during an e2e replay")
sub = parser.add_subparsers(dest="command", required=True)
serve_parser = sub.add_parser("serve", help="pin provider hosts and count connection attempts")
serve_parser.add_argument("--host", action="append", required=True, help="provider host to pin and watch")
serve_parser.add_argument("--sink-address", default="127.0.0.1")
serve_parser.add_argument("--port", action="append", type=int, default=None)
serve_parser.add_argument("--hits-file", required=True)
serve_parser.add_argument("--hosts-file", default="/etc/hosts")
serve_parser.add_argument("--ready-file", default=None)
serve_parser.add_argument("--pid-file", default=None)
assert_parser = sub.add_parser("assert-empty", help="exit non-zero if any provider call was recorded")
assert_parser.add_argument("--hits-file", required=True)
args = parser.parse_args(argv)
if args.command == "serve":
if args.port is None:
args.port = [443]
return _serve_from_args(args)
return assert_empty(Path(args.hits_file))
if __name__ == "__main__":
raise SystemExit(main(tuple(sys.argv[1:])))

55
.github/scripts/e2e_fetch_fixture_bundle.sh vendored Executable file
View file

@ -0,0 +1,55 @@
#!/usr/bin/env bash
set -euo pipefail
REPO="${1:-${GITHUB_REPOSITORY:?REPO required}}"
ARTIFACT_NAME="${2:-e2e-fixtures-bundle}"
BASE_BRANCH="${3:?base branch required}"
DEST_DIR="${4:?destination bundle dir required}"
: "${GH_TOKEN:?GH_TOKEN required to query and download artifacts}"
WORKDIR="$(mktemp -d)"
trap 'rm -rf "${WORKDIR}"' EXIT
echo "resolving newest non-expired '${ARTIFACT_NAME}' artifact on ${REPO}@${BASE_BRANCH}"
SELECTED="$(
gh api "repos/${REPO}/actions/artifacts" -X GET -f per_page=100 --paginate \
--jq ".artifacts[] | select(.name == \"${ARTIFACT_NAME}\" and .expired == false and .workflow_run.head_branch == \"${BASE_BRANCH}\") | {id, digest, created_at, run_id: .workflow_run.id, run_number: .workflow_run.run_number}" \
| jq -s 'sort_by(.created_at) | reverse | .[0] // empty'
)"
if [[ -z "${SELECTED}" ]]; then
echo "no usable '${ARTIFACT_NAME}' artifact on ${BASE_BRANCH}: the last record run produced none (a red Saturday), so there is nothing fresh to replay; failing loudly instead of replaying a stale bundle" >&2
exit 1
fi
RUN_ID="$(echo "${SELECTED}" | jq -r '.run_id')"
RUN_NUMBER="$(echo "${SELECTED}" | jq -r '.run_number')"
ARTIFACT_ID="$(echo "${SELECTED}" | jq -r '.id')"
GH_DIGEST="$(echo "${SELECTED}" | jq -r '.digest // "unknown"')"
CREATED_AT="$(echo "${SELECTED}" | jq -r '.created_at')"
echo "pinned bundle: run #${RUN_NUMBER} (run_id=${RUN_ID}, artifact_id=${ARTIFACT_ID}), recorded ${CREATED_AT}, github digest ${GH_DIGEST}"
gh run download "${RUN_ID}" --repo "${REPO}" -n "${ARTIFACT_NAME}" -D "${WORKDIR}"
TARBALL="$(find "${WORKDIR}" -name '*.tar.gz' -type f | head -n 1)"
if [[ -z "${TARBALL}" ]]; then
echo "downloaded artifact contained no tarball" >&2
exit 1
fi
SIDECAR="${TARBALL}.sha256"
if [[ ! -f "${SIDECAR}" ]]; then
echo "downloaded artifact has no ${SIDECAR}: cannot verify the bundle digest" >&2
exit 1
fi
echo "verifying bundle against its recorded sha256 digest"
( cd "$(dirname "${TARBALL}")" && sha256sum -c "$(basename "${SIDECAR}")" )
mkdir -p "${DEST_DIR}"
tar xzf "${TARBALL}" -C "${DEST_DIR}"
echo "extracted bundle into ${DEST_DIR}"
python3 -c "import json,sys; m=json.load(open(sys.argv[1])); print(' recorded_at', m['recorded_at'], 'harness', m['harness_version'], 'format_version', m['format_version'])" "${DEST_DIR}/manifest.json"

36
.github/scripts/e2e_pack_fixture_bundle.sh vendored Executable file
View file

@ -0,0 +1,36 @@
#!/usr/bin/env bash
set -euo pipefail
if [[ $# -ne 2 ]]; then
echo "usage: $0 <bundle-dir> <out-tarball>" >&2
exit 2
fi
BUNDLE_DIR="$1"
OUT_TARBALL="$2"
MANIFEST="${BUNDLE_DIR}/manifest.json"
if [[ ! -f "${MANIFEST}" ]]; then
echo "no ${MANIFEST}: refusing to publish a bundle with no manifest (record produced nothing)" >&2
exit 1
fi
echo "packing fixture bundle from ${BUNDLE_DIR}"
python3 -c "import json,sys; m=json.load(open(sys.argv[1])); print(' format_version', m['format_version'], 'recorded_at', m['recorded_at'], 'harness', m['harness_version'])" "${MANIFEST}"
TEST_DIRS=$(find "${BUNDLE_DIR}" -mindepth 1 -maxdepth 1 -type d | wc -l | tr -d ' ')
if [[ "${TEST_DIRS}" -eq 0 ]]; then
echo "bundle at ${BUNDLE_DIR} has a manifest but no recorded interactions; refusing to publish an empty bundle" >&2
exit 1
fi
echo " ${TEST_DIRS} recorded test director(ies)"
mkdir -p "$(dirname "${OUT_TARBALL}")"
tar czf "${OUT_TARBALL}" -C "${BUNDLE_DIR}" .
OUT_DIR="$(cd "$(dirname "${OUT_TARBALL}")" && pwd)"
OUT_BASE="$(basename "${OUT_TARBALL}")"
( cd "${OUT_DIR}" && sha256sum "${OUT_BASE}" > "${OUT_BASE}.sha256" )
echo "wrote ${OUT_TARBALL} ($(du -h "${OUT_TARBALL}" | cut -f1)) and ${OUT_BASE}.sha256"
cat "${OUT_DIR}/${OUT_BASE}.sha256"

4
.github/workflows/_test-unit-base.yml vendored
View file

@ -149,7 +149,7 @@ jobs:
--reruns "${RERUNS}" \
--reruns-delay 1 \
--durations=20 \
--cov=./litellm \
--cov=./litellm --cov=./enterprise/litellm_enterprise \
--cov-report=xml:coverage.xml \
--cov-config=pyproject.toml
else
@ -161,7 +161,7 @@ jobs:
--reruns-delay 1 \
--dist="${DIST}" \
--durations=20 \
--cov=./litellm \
--cov=./litellm --cov=./enterprise/litellm_enterprise \
--cov-report=xml:coverage.xml \
--cov-config=pyproject.toml
fi

237
.github/workflows/e2e_record_replay.yml vendored Normal file
View file

@ -0,0 +1,237 @@
name: "E2E Record and Replay"
on:
schedule:
- cron: "0 8 * * 6"
- cron: "0 8 * * 1-5"
workflow_dispatch:
inputs:
mode:
description: "record (hits real providers and publishes a fresh bundle) or replay (bundle only, zero provider egress)"
type: choice
options:
- record
- replay
default: record
permissions:
contents: read
jobs:
record:
name: "Record the e2e suite against real providers"
if: >-
(github.event_name != 'schedule' || github.repository == 'BerriAI/litellm') &&
(github.event.schedule == '0 8 * * 6' ||
(github.event_name == 'workflow_dispatch' && github.event.inputs.mode == 'record'))
runs-on: ubuntu-latest
timeout-minutes: 45
services:
postgres:
image: postgres:16.6
env:
POSTGRES_USER: llmproxy
POSTGRES_PASSWORD: dbpassword9090
POSTGRES_DB: litellm
ports:
- 5432:5432
options: >-
--health-cmd "pg_isready -U llmproxy"
--health-interval 5s
--health-timeout 5s
--health-retries 10
env:
DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
LITELLM_MASTER_KEY: sk-e2e-record-replay
LITELLM_LOCAL_MODEL_COST_MAP: "True"
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Cache the Rust build
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy
- name: Cache Prisma binaries
uses: ./.github/actions/cache-prisma-binaries
- name: Generate Prisma client
run: |
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
- name: Start the proxy
run: |
nohup uv run --no-sync litellm --config tests/e2e/gateway/record_replay_ci_config.yml --port 4000 > proxy.log 2>&1 &
for _ in $(seq 1 90); do
if curl -fs http://localhost:4000/health/liveliness > /dev/null; then
exit 0
fi
sleep 2
done
echo "proxy never became live"
tail -n 100 proxy.log
exit 1
- name: Record the replayable e2e lane
env:
E2E_FIXTURE_MODE: record
run: |
uv run --no-sync pytest tests/e2e -m replayable --reruns 0 -v --tb=short -rA
- name: Pack the fixture bundle
run: |
.github/scripts/e2e_pack_fixture_bundle.sh tests/e2e/.fixtures "${RUNNER_TEMP}/bundle/e2e-fixtures.tar.gz"
- name: Publish the fixture bundle
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
with:
name: e2e-fixtures-bundle
path: |
${{ runner.temp }}/bundle/e2e-fixtures.tar.gz
${{ runner.temp }}/bundle/e2e-fixtures.tar.gz.sha256
if-no-files-found: error
retention-days: 30
- name: Show proxy log on failure
if: failure()
run: tail -n 300 proxy.log
replay:
name: "Replay the e2e suite from the pinned bundle with zero egress"
if: >-
(github.event_name != 'schedule' || github.repository == 'BerriAI/litellm') &&
(github.event.schedule == '0 8 * * 1-5' ||
(github.event_name == 'workflow_dispatch' && github.event.inputs.mode == 'replay'))
runs-on: ubuntu-latest
timeout-minutes: 45
permissions:
contents: read
actions: read
services:
postgres:
image: postgres:16.6
env:
POSTGRES_USER: llmproxy
POSTGRES_PASSWORD: dbpassword9090
POSTGRES_DB: litellm
ports:
- 5432:5432
options: >-
--health-cmd "pg_isready -U llmproxy"
--health-interval 5s
--health-timeout 5s
--health-retries 10
env:
DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
LITELLM_MASTER_KEY: sk-e2e-record-replay
LITELLM_LOCAL_MODEL_COST_MAP: "True"
GH_TOKEN: ${{ github.token }}
OPENAI_API_KEY: sk-replay-must-never-reach-a-provider
ANTHROPIC_API_KEY: sk-ant-replay-must-never-reach-a-provider
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Cache the Rust build
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy
- name: Cache Prisma binaries
uses: ./.github/actions/cache-prisma-binaries
- name: Generate Prisma client
run: |
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
- name: Fetch the pinned fixture bundle by digest
env:
BASE_BRANCH: ${{ github.ref_name }}
run: |
.github/scripts/e2e_fetch_fixture_bundle.sh \
"${GITHUB_REPOSITORY}" \
e2e-fixtures-bundle \
"${BASE_BRANCH}" \
tests/e2e/.fixtures
- name: Start the proxy
run: |
nohup uv run --no-sync litellm --config tests/e2e/gateway/record_replay_ci_config.yml --port 4000 > proxy.log 2>&1 &
for _ in $(seq 1 90); do
if curl -fs http://localhost:4000/health/liveliness > /dev/null; then
exit 0
fi
sleep 2
done
echo "proxy never became live"
tail -n 100 proxy.log
exit 1
- name: Start the egress sentinel
run: |
# shellcheck disable=SC2024 # the log redirect is deliberately the runner user's, so a later non-sudo cat can read it
sudo python3 .github/scripts/e2e_egress_sentinel.py serve \
--host api.openai.com \
--host api.anthropic.com \
--hits-file "${RUNNER_TEMP}/egress-hits.jsonl" \
--ready-file "${RUNNER_TEMP}/egress-ready" \
--pid-file "${RUNNER_TEMP}/egress.pid" \
> "${RUNNER_TEMP}/egress-sentinel.log" 2>&1 &
for _ in $(seq 1 30); do
if [[ -f "${RUNNER_TEMP}/egress-ready" ]]; then
cat "${RUNNER_TEMP}/egress-sentinel.log"
exit 0
fi
sleep 1
done
echo "egress sentinel never became ready"
cat "${RUNNER_TEMP}/egress-sentinel.log"
exit 1
- name: Replay the replayable e2e lane
env:
E2E_FIXTURE_MODE: replay
run: |
uv run --no-sync pytest tests/e2e -m replayable --reruns 0 -v --tb=short -rA
- name: Stop the egress sentinel and assert zero provider egress
if: always()
run: |
if [[ -f "${RUNNER_TEMP}/egress.pid" ]]; then
sudo kill -TERM "$(cat "${RUNNER_TEMP}/egress.pid")" 2>/dev/null || true
sleep 2
fi
python3 .github/scripts/e2e_egress_sentinel.py assert-empty --hits-file "${RUNNER_TEMP}/egress-hits.jsonl"
- name: Show proxy log on failure
if: failure()
run: tail -n 300 proxy.log

31
.github/workflows/image-scan.yml vendored
View file

@ -23,6 +23,8 @@ on:
- tests/proxy_migration_tests/**
- uv.lock
- ui/litellm-dashboard/package-lock.json
- ui/Dockerfile
- ui/nginx.conf
- .github/workflows/image-scan.yml
schedule:
- cron: "41 6 * * *"
@ -185,6 +187,35 @@ jobs:
python -m pip install "pytest==9.0.3"
python -m pytest tests/proxy_migration_tests/test_component_image_serves_offline.py -v
ui-image:
name: ui-image
runs-on: ubuntu-latest
if: >-
github.event_name != 'pull_request' ||
github.event.pull_request.head.repo.full_name == github.repository
timeout-minutes: 30
permissions:
contents: read
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Build UI image
run: docker build -f ui/Dockerfile -t litellm-ui-scan:${{ github.sha }} .
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Verify the UI serves offline as an arbitrary uid with a read-only root fs
env:
LITELLM_IMAGE: litellm-ui-scan:${{ github.sha }}
run: |
python -m pip install "pytest==9.0.3"
python -m pytest tests/proxy_migration_tests/test_ui_image_serves_offline.py -v
backend-image:
name: backend-image
runs-on: ubuntu-latest

3
.github/workflows/test-code-quality.yml vendored
View file

@ -131,6 +131,9 @@ jobs:
- name: check_e2e_no_raw_requests
run: uv run --no-sync python ./tests/code_coverage_tests/check_e2e_no_raw_requests.py
- name: check_migrations_no_data_rewrites
run: uv run --no-sync python ./tests/code_coverage_tests/check_migrations_no_data_rewrites.py
- name: memory_test
run: uv run --no-sync python ./tests/code_coverage_tests/memory_test.py

11
.github/workflows/test-linting.yml vendored
View file

@ -67,6 +67,17 @@ jobs:
with:
version: "0.10.9"
- name: Cache uv dependencies
if: steps.changes.outputs.decision != 'skip'
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
~/.cache/uv
.venv
key: ${{ runner.os }}-uv-lint-${{ hashFiles('uv.lock') }}
restore-keys: |
${{ runner.os }}-uv-lint-
- name: Clean Python cache
if: steps.changes.outputs.decision != 'skip'
run: |

2
.github/workflows/test-mcp.yml vendored
View file

@ -60,4 +60,4 @@ jobs:
- name: Run MCP tests
if: steps.changes.outputs.decision != 'skip'
run: |
uv run --no-sync pytest tests/mcp_tests -x -vv -n 4 --cov=./litellm --cov-report=xml --durations=5
uv run --no-sync pytest tests/mcp_tests -x -vv -n 4 --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml --durations=5

145
.github/workflows/test-postgres.yml vendored Normal file
View file

@ -0,0 +1,145 @@
name: "Postgres Tests"
on:
pull_request:
branches:
- main
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
push:
branches:
- main
- litellm_internal_staging
workflow_dispatch:
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
postgres:
name: ${{ matrix.shard }}
runs-on: ubuntu-latest
timeout-minutes: ${{ matrix.job-timeout-minutes }}
permissions:
contents: read
services:
postgres:
image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: litellm_test
ports:
- 5432:5432
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 10
strategy:
fail-fast: false
matrix:
include:
- shard: proxy-behavior
test-path: "tests/proxy_behavior"
seed: db-push
workers: 0
timeout-minutes: 25
job-timeout-minutes: 50
- shard: proxy-security
test-path: "tests/proxy_security_tests"
seed: db-push
workers: 0
timeout-minutes: 15
job-timeout-minutes: 40
- shard: schema-migration
test-path: "tests/proxy_migration_tests"
seed: none
workers: 0
timeout-minutes: 20
job-timeout-minutes: 45
env:
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
timeout-minutes: 3
with:
persist-credentials: false
- name: Detect relevant changes
id: changes
timeout-minutes: 2
uses: ./.github/actions/detect-changes
- name: Set up Python
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 3
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 3
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Cache uv dependencies
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 5
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
~/.cache/uv
.venv
key: ${{ runner.os }}-uv-postgres-${{ hashFiles('uv.lock') }}
restore-keys: |
${{ runner.os }}-uv-postgres-
- name: Install dependencies
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 12
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras
- name: Cache Prisma binaries
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 3
uses: ./.github/actions/cache-prisma-binaries
- name: Generate Prisma client
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 5
run: |
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
- name: Seed database schema
if: steps.changes.outputs.decision != 'skip' && matrix.seed != 'none'
timeout-minutes: 10
run: |
uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
- name: Run tests
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: ${{ matrix.timeout-minutes }}
env:
TEST_PATH: ${{ matrix.test-path }}
WORKERS: ${{ matrix.workers }}
run: |
if [ "${WORKERS}" = "0" ]; then
uv run --no-sync pytest ${TEST_PATH:?} -vv --tb=short --durations=10
else
uv run --no-sync pytest ${TEST_PATH:?} -vv --tb=short --durations=10 -n "${WORKERS}"
fi

31
.github/workflows/test-unit.yml vendored
View file

@ -168,7 +168,7 @@ jobs:
tests/test_litellm/proxy/ui_crud_endpoints
tests/test_litellm/proxy/config_resolvers
tests/test_litellm/proxy/utils
workers: 2
workers: 4
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 60
@ -195,11 +195,40 @@ jobs:
tests/test_litellm/proxy/types_utils
tests/test_litellm/proxy/logging_endpoints
tests/test_litellm/proxy/test_*.py
workers: 4
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 60
- shard: caching-local
artifact-name: caching-local
test-path: >-
tests/local_testing/test_cache_preset_key.py
tests/local_testing/test_caching_handler.py
tests/local_testing/test_prompt_caching.py
tests/local_testing/test_responses_stream_cache_keys.py
tests/local_testing/test_unit_test_caching.py
workers: 2
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 60
- shard: proxy-extras
artifact-name: proxy-extras
test-path: "tests/litellm-proxy-extras"
workers: 2
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 60
- shard: enterprise-package
artifact-name: enterprise-package
test-path: "tests/enterprise"
workers: 4
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 60
- shard: responses-caching-types
artifact-name: responses-caching-types
test-path: >-

2
CLAUDE.md
View file

@ -79,6 +79,8 @@ Do not put names of customers or customer company names in code, PR descriptions
CI supply-chain safety: Never pipe a remote script into a shell (`curl ... | bash`, `wget ... | sh`); download the artifact to a file, verify its SHA-256 checksum, then install. Pin every external tool to a specific version with a full URL (not `latest` or `stable`). Verify checksums for all downloaded binaries, using the provider's official `.sha256` / `.sha256sum` sidecar when available. These rules apply to every download in CI
Prisma migrations apply synchronously at proxy boot, before it serves traffic, so a migration must only change schema, never rewrite rows. No `UPDATE`, `DELETE` or `MERGE`, and no `INSERT ... SELECT`: on a spend-log-sized table any of those is minutes of downtime plus a doubled heap that plain autovacuum won't give back. `tests/code_coverage_tests/check_migrations_no_data_rewrites.py` enforces this. When a rewrite is genuinely bounded and has to ship inside the migration, mark the statement `-- data-migration-ok: <what bounds it>`
Follow these coding conventions for new/updated code (a three-line fix in a legacy file shouldn't trigger huge drive-by refactors):
- Composition over inheritance

8
helm/litellm/values.yaml
View file

@ -428,9 +428,11 @@ ui:
maxUnavailable: ""
podAnnotations: {}
# Same shape as the gateway blocks of the same name. The nginx runtime
# writes its pid, cache, and proxy temp files under the image's root
# filesystem, so `securityContext.readOnlyRootFilesystem: true` here needs
# emptyDir volumes mounted over those paths.
# writes its pid, cache, and proxy temp files under /tmp, so it boots as
# any (arbitrary, non-root) uid; `securityContext.readOnlyRootFilesystem:
# true` here needs an emptyDir volume mounted over /tmp. Images before
# the /tmp move instead need emptyDirs over /var/cache/nginx and /run to
# run as a non-root uid at all.
podLabels: {}
podSecurityContext: {}
securityContext: {}

38
litellm/a2a_protocol/card_resolver.py
View file

@ -4,6 +4,7 @@ Custom A2A Card Resolver for LiteLLM.
Extends the A2A SDK's card resolver to support multiple well-known paths.
"""
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final
from litellm._logging import verbose_logger
@ -48,6 +49,43 @@ def is_localhost_or_internal_url(url: str | None) -> bool:
return any(pattern in url_lower for pattern in LOCALHOST_URL_PATTERNS)
_CANONICAL_PROTOCOL_BINDINGS: Final = MappingProxyType(
{
"jsonrpc": "JSONRPC",
"http+json": "HTTP+JSON",
"grpc": "GRPC",
}
)
_LEGACY_PROTOCOL_VERSION: Final = "0.3"
def normalize_agent_card_interfaces(agent_card: "AgentCard") -> "AgentCard":
"""
Canonicalize the supported interfaces of spec-adjacent agent cards.
Some A2A servers (e.g. LangGraph Platform) serve agent cards with lowercase
bindings like "jsonrpc", but a2a-sdk's ClientFactory matches bindings
case-sensitively against its uppercase TransportProtocol constants and fails
with "no compatible transports found." for spec-adjacent casings.
The same servers also speak the A2A 0.3 JSON dialect ("kind"-discriminated
payloads) while declaring protocolVersion "1.0", which a2a-sdk's strict v1
proto parsing rejects. A mis-cased binding fingerprints such a server, so its
declared version is downgraded to 0.3 to route the SDK's ClientFactory onto
its v0.3 compat transport, which speaks that dialect.
"""
normalized: Final = type(agent_card)()
normalized.CopyFrom(agent_card)
for interface in normalized.supported_interfaces:
canonical: str | None = _CANONICAL_PROTOCOL_BINDINGS.get(interface.protocol_binding.lower())
if canonical is None or canonical == interface.protocol_binding:
continue
interface.protocol_binding = canonical
interface.protocol_version = _LEGACY_PROTOCOL_VERSION
return normalized
def get_agent_card_url(agent_card: "AgentCard") -> str | None:
"""Return the agent endpoint URL from the resolved SDK card."""
url: Final = getattr(agent_card, "url", None)

13
litellm/a2a_protocol/main.py
View file

@ -73,6 +73,7 @@ except ImportError:
from litellm.a2a_protocol.card_resolver import (
LiteLLMA2ACardResolver,
get_agent_card_url,
normalize_agent_card_interfaces,
)
from litellm.a2a_protocol.exception_mapping_utils import (
handle_a2a_localhost_retry,
@ -782,13 +783,17 @@ async def create_a2a_client(
if extra_headers:
verbose_proxy_logger.debug("A2A client created with extra_headers=%s", list(extra_headers.keys()))
resolver: Final = A2ACardResolver(httpx_client=httpx_client, base_url=base_url)
agent_card: Final = normalize_agent_card_interfaces(
await resolver.get_agent_card(http_kwargs={"headers": extra_headers} if extra_headers else None)
)
a2a_client: Final = await create_client( # pyright: ignore[reportOptionalCall]
base_url,
agent_card,
client_config=ClientConfig( # pyright: ignore[reportOptionalCall]
httpx_client=httpx_client,
streaming=streaming,
),
resolver_http_kwargs={"headers": extra_headers} if extra_headers else None,
)
# Stash LiteLLM-owned handles on the client so the localhost-retry path can reuse
# the configured httpx client and this agent's headers without excavating
@ -799,9 +804,7 @@ async def create_a2a_client(
if extra_headers
else None
)
agent_card: Final = getattr(a2a_client, "_card", None)
if agent_card is not None:
a2a_client._litellm_agent_card = agent_card
a2a_client._litellm_agent_card = agent_card
verbose_logger.info("A2A client created for %s", base_url)

37
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -21,6 +21,9 @@ from pydantic import BaseModel
import litellm
from litellm import ModelResponse
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.prompt_templates.common_utils import (
responses_reasoning_item_from_thinking_blocks,
)
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.llms.base_llm.bridges.completion_transformation import (
CompletionTransformationBridge,
@ -85,6 +88,22 @@ def _get_reasoning_items(
return []
def _reasoning_input_items(msg: "AllMessageValues") -> list[dict[str, object]]: # mutable-ok: API message payload
"""Reasoning input items for an assistant message.
Stored reasoning items win because they carry an id the Responses API minted; thinking
blocks are the fallback for turns that arrived over another API surface.
"""
items: Final = _get_reasoning_items(msg)
stored: Final = [_reasoning_item_to_response_input(item) for item in items] # mutable-ok: API message payload
if stored:
return stored
raw_blocks: Final = msg.get("thinking_blocks") or ()
blocks: Final = cast("Iterable[ChatCompletionThinkingBlock]", raw_blocks) # cast-ok: untyped client json
from_thinking: Final = responses_reasoning_item_from_thinking_blocks(blocks)
return [] if from_thinking is None else [dict(from_thinking)] # mutable-ok: API message payload
def _build_reasoning_item(
item_id: str,
encrypted_content: str | None,
@ -372,8 +391,15 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
)
)
elif role == "assistant" and tool_calls and isinstance(tool_calls, list):
for r_item in _get_reasoning_items(msg):
input_items.append(_reasoning_item_to_response_input(r_item))
input_items.extend(_reasoning_input_items(msg))
if content:
input_items.append(
{ # mutable-ok: API message payload
"type": "message",
"role": "assistant",
"content": self._convert_content_to_responses_format(content, "assistant"),
}
)
for tool_call in tool_calls:
function = tool_call.get("function")
custom = tool_call.get("custom")
@ -400,15 +426,16 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
raise ValueError(f"tool call not supported: {tool_call}")
elif content is not None:
if role == "assistant":
for r_item in _get_reasoning_items(msg):
input_items.append(_reasoning_item_to_response_input(r_item))
input_items.extend(_reasoning_input_items(msg))
input_items.append(
{
{ # mutable-ok: API message payload
"type": "message",
"role": role,
"content": self._convert_content_to_responses_format(content, cast(str, role)),
}
)
elif role == "assistant":
input_items.extend(_reasoning_input_items(msg))
return input_items, instructions

14
litellm/constants.py
View file

@ -48,6 +48,7 @@ LITELLM_MAX_STREAMING_DURATION_SECONDS: Final = (
# Data URIs exceeding this are replaced with a size placeholder.
# Set to 0 to disable truncation.
MAX_BASE64_LENGTH_FOR_LOGGING: Final = int(os.getenv("MAX_BASE64_LENGTH_FOR_LOGGING", 64))
REDACTED_BY_LITELLM: Final = "redacted-by-litellm"
MAX_STRING_LENGTH_STDOUT_LOG: Final = get_env_int("MAX_STRING_LENGTH_STDOUT_LOG", 4096)
@ -1563,6 +1564,19 @@ STALE_OBJECT_CLEANUP_BATCH_SIZE: Final = max(1, int(os.getenv("STALE_OBJECT_CLEA
# installations with large numbers of stale managed objects).
_batch_polling_env: Final = os.getenv("PROXY_BATCH_POLLING_ENABLED", "true").lower()
PROXY_BATCH_POLLING_ENABLED: Final = _batch_polling_env == "true"
BACKGROUND_INTERACTION_COST_POLL_INITIAL_INTERVAL_SECONDS: Final = float(
os.getenv("BACKGROUND_INTERACTION_COST_POLL_INITIAL_INTERVAL_SECONDS", "5")
)
BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS: Final = float(
os.getenv("BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS", "60")
)
BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS: Final = float(
os.getenv("BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS", "3600")
)
_background_interaction_cost_polling_env: Final = os.getenv(
"BACKGROUND_INTERACTION_COST_POLLING_ENABLED", "true"
).lower()
BACKGROUND_INTERACTION_COST_POLLING_ENABLED: Final = _background_interaction_cost_polling_env == "true"
PROXY_BUDGET_RESCHEDULER_MAX_TIME: Final = int(os.getenv("PROXY_BUDGET_RESCHEDULER_MAX_TIME", 605))
PROXY_BATCH_WRITE_AT: Final = int(os.getenv("PROXY_BATCH_WRITE_AT", 10)) # in seconds, increased from 10
PROXY_CONFIG_RELOAD_INTERVAL_SECONDS: Final = get_env_int("PROXY_CONFIG_RELOAD_INTERVAL_SECONDS", 30)

25
litellm/cost_calculator.py
View file

@ -19,6 +19,7 @@ from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
StandardBuiltInToolCostTracking,
)
from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import (
InteractionsUsageObjectTransformation,
TranscriptionUsageObjectTransformation,
)
from litellm.litellm_core_utils.llm_cost_calc.utils import (
@ -150,6 +151,7 @@ _VIDEO_CALL_TYPES: Final = frozenset(
}
)
_SPEECH_CALL_TYPES: Final = frozenset(
{
CallTypes.speech.value,
@ -912,6 +914,8 @@ def _get_usage_object(
usage_obj,
)
)
elif isinstance(usage_obj, dict) and InteractionsUsageObjectTransformation.is_interactions_usage_object(usage_obj):
return InteractionsUsageObjectTransformation.transform_interactions_usage_object(usage_obj)
elif isinstance(usage_obj, dict):
return Usage(**usage_obj)
elif isinstance(usage_obj, BaseModel):
@ -1288,6 +1292,10 @@ def completion_cost(
)
if tr_usage is not None:
_usage = tr_usage.model_dump()
elif InteractionsUsageObjectTransformation.is_interactions_usage_object(_usage):
_usage = InteractionsUsageObjectTransformation.transform_interactions_usage_object(
_usage
).model_dump()
else:
_usage = _usage
@ -1372,23 +1380,36 @@ def completion_cost(
if custom_pricing and litellm_logging_obj is not None:
_litellm_params = getattr(litellm_logging_obj, "litellm_params", None)
if _litellm_params is not None:
_metadata = _litellm_params.get("metadata", {}) or {}
_video_model_info = _metadata.get("model_info", None)
_video_model_info = next(
(
model_info
for _metadata_key in ("metadata", "litellm_metadata")
if (model_info := (_litellm_params.get(_metadata_key) or {}).get("model_info"))
is not None
),
None,
)
usage_obj = getattr(completion_response, "usage", None)
duration_seconds: float | None = None
video_resolution: str | None = None
provider_reported_cost: float | None = None
if completion_response is not None and usage_obj:
# Handle both dict and Pydantic Usage object
if isinstance(usage_obj, dict):
duration_seconds = usage_obj.get("duration_seconds", None)
_vr = usage_obj.get("video_resolution", None)
provider_reported_cost = usage_obj.get("provider_reported_cost_usd", None)
else:
duration_seconds = getattr(usage_obj, "duration_seconds", None)
_vr = getattr(usage_obj, "video_resolution", None)
provider_reported_cost = getattr(usage_obj, "provider_reported_cost_usd", None)
if _vr is not None:
video_resolution = str(_vr).strip().lower()
if _video_model_info is None and provider_reported_cost is not None:
return float(provider_reported_cost)
if duration_seconds is not None:
# Calculate cost based on video duration using video-specific cost calculation
from litellm.llms.openai.cost_calculation import (

46
litellm/images/main.py
View file

@ -19,6 +19,7 @@ from litellm.constants import request_timeout as DEFAULT_REQUEST_TIMEOUT
from litellm.exceptions import LiteLLMUnknownProvider
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.llm_request_utils import flatten_form_field_values
from litellm.litellm_core_utils.mock_functions import mock_image_generation
from litellm.llms.base_llm import BaseImageEditConfig, BaseImageGenerationConfig
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
@ -422,24 +423,32 @@ def image_generation(
aimg_generation=aimg_generation,
)
elif custom_llm_provider == "azure_ai":
from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
from litellm.llms.azure_ai.common_utils import (
AzureFoundryModelInfo,
get_azure_ai_auth_headers,
)
api_base = AzureFoundryModelInfo.get_api_base(api_base)
api_key = AzureFoundryModelInfo.get_api_key(api_key)
if extra_headers is not None:
optional_params["extra_headers"] = extra_headers
default_headers = {
caller_header_names = frozenset(name.lower() for name in headers)
caller_set_auth = "api-key" in caller_header_names or "authorization" in caller_header_names
auth_headers = (
headers
if caller_set_auth
else get_azure_ai_auth_headers(
api_key=api_key,
litellm_params=litellm_params_dict,
api_key_header="api-key",
)
)
request_headers: Final = {
"Content-Type": "application/json",
**auth_headers,
**headers,
}
# Only add api-key header if api_key is not None
# Azure AD authentication will use Authorization header instead
if api_key is not None:
default_headers["api-key"] = api_key
for k, v in default_headers.items():
if k not in headers:
headers[k] = v
model_response = azure_chat_completions.image_generation(
model=model,
@ -455,7 +464,7 @@ def image_generation(
api_version=api_version,
aimg_generation=aimg_generation,
client=client,
headers=headers,
headers=request_headers,
litellm_params=litellm_params_dict,
)
elif (
@ -846,6 +855,18 @@ def image_edit(
additional_drop_params=kwargs.get("additional_drop_params"),
)
if (
custom_llm_provider == "openai"
or custom_llm_provider == "azure"
or custom_llm_provider in litellm.openai_compatible_providers
):
image_edit_request_params.update(
flatten_form_field_values(
non_default_params,
extra_body if isinstance(extra_body, dict) else None,
)
)
# Pre Call logging
litellm_logging_obj.update_from_kwargs(
kwargs=kwargs,
@ -995,6 +1016,9 @@ async def aimage_edit(
response_format=response_format,
size=size,
user=user,
extra_headers=extra_headers,
extra_query=extra_query,
extra_body=extra_body,
timeout=timeout,
custom_llm_provider=custom_llm_provider,
**kwargs,

14
litellm/integrations/callback_configs.json
View file

@ -294,12 +294,18 @@
"id": "newrelic",
"displayName": "New Relic",
"logo": "newrelic.png",
"supports_key_team_logging": false,
"supports_key_team_logging": true,
"dynamic_params": {
"NEW_RELIC_AI_MONITORING_RECORD_CONTENT_ENABLED": {
"newrelic_api_key": {
"type": "password",
"ui_name": "New Relic Ingest License Key",
"description": "Per-team ingest (license) key. Team traces export to this key's New Relic account over OTLP.",
"required": false
},
"newrelic_region": {
"type": "text",
"ui_name": "Record AI Content (default: true)",
"description": "Whether to record AI message content. Set to false to disable.",
"ui_name": "New Relic Region (us or eu)",
"description": "Data center region for this team's account. Defaults to us.",
"required": false
}
},

22
litellm/integrations/langsmith.py
View file

@ -168,17 +168,20 @@ class LangsmithLogger(CustomBatchLogger):
return outputs
def _ensure_required_ids(self, data: dict, run_id: str | None):
resolved_id: Final = run_id or str(uuid.uuid4())
if "id" not in data or data["id"] is None:
run_id = str(uuid.uuid4())
data["id"] = run_id
data["id"] = resolved_id
if "trace_id" not in data or data["trace_id"] is None:
if run_id is not None and isinstance(run_id, str):
data["trace_id"] = run_id
# LangSmith rejects the whole ingest batch unless a root run's trace_id
# equals the run id embedded in the first segment of dotted_order
posts_as_root: Final = ("parent_run_id" not in data or data["parent_run_id"] is None) and (
"dotted_order" not in data or data["dotted_order"] is None
)
if posts_as_root or "trace_id" not in data or data["trace_id"] is None:
data["trace_id"] = resolved_id
if "dotted_order" not in data or data["dotted_order"] is None:
if run_id is not None and isinstance(run_id, str):
data["dotted_order"] = self.make_dot_order(run_id=run_id)
data["dotted_order"] = self.make_dot_order(run_id=resolved_id)
def _prepare_log_data(
self,
@ -193,6 +196,11 @@ class LangsmithLogger(CustomBatchLogger):
metadata = _litellm_params.get("metadata", {}) or {}
fields: Final = self._extract_metadata_fields(metadata, credentials)
# the proxy header fan-out mirrors one value into both keys, and LangSmith
# rejects the whole ingest batch when run-body session_id is not an
# existing tracer-session uuid
if fields["session_id"] == fields["trace_id"]:
fields["session_id"] = None
verbose_logger.debug(
"Langsmith Logging - project_name: %s, run_name %s", fields["project_name"], fields["run_name"]
)

6
litellm/integrations/otel/emitter.py
View file

@ -156,6 +156,12 @@ class SpanEmitter:
links=list(links) if links else None,
)
def mark_emitted(self, dedup_key: str | None, role: SpanRole) -> None:
"""Register a span emitted outside :meth:`emit` (the boundary-opened
LLM-call span closed via :meth:`finish_span`) so a later :meth:`emit`
for the same ``(dedup_key, role)`` deduplicates against it."""
self._seen(dedup_key, role)
def _seen(self, dedup_key: str | None, role: SpanRole) -> bool:
"""Return True once a ``(dedup_key, role)`` pair has been emitted.

36
litellm/integrations/otel/logger.py
View file

@ -484,10 +484,15 @@ class OpenTelemetryV2(CustomLogger):
# ``pop`` is the dedup: this method runs from both the success and failure
# paths, and whichever fires first removes the carrier and closes the span.
carrier: Final = self._open_llm_calls.pop(call_id, None) if call_id else None
if carrier is None:
# A missing carrier does not always mean nothing happened: a team/key-scoped
# logger is a success/failure callback only, so ``pre_call`` never reaches it
# and no carrier exists. The payload plus the request-level provider-handoff
# stamp (``upstream_started``) is the affirmative signal of a real call; a
# gate rejection carries ``is_no_upstream_call`` and gets no span.
if carrier is None and (call.is_no_upstream_call or not call.upstream_started or call.payload is None):
return None
try:
return self._finish_carrier(carrier, call, end_time)
return self._finish_carrier(carrier, call, start_time, end_time)
finally:
# After the span has ended, so a release-triggered provider shutdown
# force-flushes it out rather than racing its enqueue.
@ -497,8 +502,11 @@ class OpenTelemetryV2(CustomLogger):
"""Remember an in-flight LLM call, evicting the oldest if over budget.
A call that opens but never closes (a stream that only fires stream
events) would linger otherwise; the evicted span is simply dropped
(never exported).
events) would linger otherwise. Eviction only drops the boundary carrier,
not the call: if that call later closes as a real completed call, it still
emits through the deferred branch in ``_close_llm_call`` (the same path a
team/key-scoped logger uses, since it never opens a carrier), deduplicated
by call id. Only a call that is evicted and never closes goes unexported.
"""
self._open_llm_calls[call_id] = carrier
if len(self._open_llm_calls) > _OPEN_CALLS_MAX:
@ -512,15 +520,20 @@ class OpenTelemetryV2(CustomLogger):
def _finish_carrier(
self,
carrier: _LLMCallSpan,
carrier: "_LLMCallSpan | None",
call: LLMCallEvent,
start_time: datetime | float | None,
end_time: datetime | float | None,
) -> Span | None:
payload: Final = call.payload
call_id: Final = call.call_id
if payload is None:
if carrier.span is not None:
if carrier is not None and carrier.span is not None:
# Opened at the boundary but the payload never materialized — end
# it (named provisionally) so it isn't leaked as an open span.
# it (named provisionally) so it isn't leaked as an open span, and
# register the dedup marker so a later payload-carrying close for
# the same call id cannot re-emit through the deferred branch.
self._emitter.mark_emitted(call_id, SpanRole.LLM_CALL)
carrier.span.end(end_time=to_ns(end_time))
return None
data: Final = LLMCallSpanData.from_standard_logging_payload(
@ -529,10 +542,13 @@ class OpenTelemetryV2(CustomLogger):
time_to_first_chunk_seconds=call.time_to_first_chunk_seconds,
)
end_time_ns: Final = to_ns(end_time)
if carrier.span is not None:
if carrier is not None and carrier.span is not None:
# Born at the boundary: stamp attributes from the typed payload, set
# status, and end it. Its parent (the server span) was captured at
# creation from real ambient context.
# creation from real ambient context. Register the dedup marker so a
# second close for the same call id (success then failure on one
# logging object) cannot re-emit through the deferred branch.
self._emitter.mark_emitted(call_id, SpanRole.LLM_CALL)
self._emitter.finish_span(SpanRole.LLM_CALL, carrier.span, data, end_time_ns=end_time_ns)
return carrier.span
# Deferred: ``pre_call`` saw no recordable parent, so create the span now.
@ -549,7 +565,7 @@ class OpenTelemetryV2(CustomLogger):
SpanRole.LLM_CALL,
data,
parent_context=(set_span_in_context(INVALID_SPAN, parent_ctx) if route.detached else parent_ctx),
start_time_ns=carrier.start_time_ns,
start_time_ns=(carrier.start_time_ns if carrier is not None else to_ns(start_time)),
end_time_ns=end_time_ns,
tracer=route.tracer,
links=_request_trace_links(parent_ctx) if route.detached else None,

10
litellm/integrations/otel/model/config.py
View file

@ -39,6 +39,7 @@ class ExporterOwner(str, Enum):
WEAVE_OTEL = "weave_otel"
LEVO = "levo"
AGENTOPS = "agentops"
NEWRELIC = "newrelic"
class _OTelV2Flag(BaseSettings):
@ -97,6 +98,15 @@ class ExporterSpec(BaseModel):
"auto (Simple for console/in_memory, Batch otherwise)."
),
)
requires_headers: bool = Field(
default=False,
description=(
"Skip this exporter when no headers are resolved. For destinations "
"that reject unauthenticated exports (e.g. New Relic), a spec kept "
"only as the per-request credential-stamping target would otherwise "
"export keyless traffic and produce a 4xx for every span batch."
),
)
class OpenTelemetryV2Config(BaseSettings):

6
litellm/integrations/otel/model/metadata.py
View file

@ -203,6 +203,11 @@ class LLMCallEvent:
# True for synthetic proxy-gate logs (auth / rate-limit rejections): they fire
# the ``pre_call`` hook but never made an upstream call, so they get no span.
is_no_upstream_call: bool
# True once the request handed off to a provider (``pre_call`` stamped
# ``api_call_start_time``). The affirmative signal that an LLM call was
# actually attempted — router pre-call rejections, SDK failures before the
# provider handoff, and standalone guardrail runs all lack it.
upstream_started: bool
# A best-effort ``"{operation} {model}"`` name known at ``pre_call`` time. The
# span is renamed from the typed payload at close (``finish_span``); this only
# needs to be reasonable for a span that never gets closed (a leak).
@ -221,6 +226,7 @@ class LLMCallEvent:
dynamic_params=kwargs.get("standard_callback_dynamic_params"),
auth_metadata=auth_metadata(payload, kwargs),
is_no_upstream_call=bool(kwargs.get(LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL)),
upstream_started=kwargs.get("api_call_start_time") is not None,
provisional_span_name=f"{operation.value} {model}".strip(),
time_to_first_chunk_seconds=time_to_first_chunk_seconds(kwargs),
)

2
litellm/integrations/otel/plumbing/providers.py
View file

@ -436,6 +436,8 @@ def build_tracer_provider(
# ``config._normalize`` guarantees at least one spec (it folds the top-level
# ``exporter``/``endpoint``/``headers`` fields in when ``exporters`` is empty).
for spec in config.exporters:
if spec.requires_headers and not spec.headers:
continue
exp = _exporter_from_spec(spec)
provider.add_span_processor(
_processor_for(

29
litellm/integrations/otel/plumbing/routing.py
View file

@ -28,6 +28,7 @@ from litellm.integrations.otel.plumbing.providers import (
get_tracer,
)
from litellm.integrations.otel.presets import (
dynamic_otlp_endpoint,
dynamic_otlp_headers,
project_routing_headers,
)
@ -129,7 +130,9 @@ class TenantTracerCache:
# thread-pool workers concurrently with the event loop, so cache
# updates, span counts, and retirement must be atomic.
self._lock: Final = threading.Lock()
self._providers: OrderedDict[tuple[_HeaderItems, _HeaderItems], TracerProvider] = OrderedDict()
self._providers: OrderedDict[tuple[_HeaderItems, _HeaderItems, str | None], TracerProvider] = (
OrderedDict() # mutable-ok: bounded LRU; eviction needs in-place ordered mutation
)
self._open_span_counts: dict[TracerProvider, int] = {} # mutable-ok: live refcount state
# Oldest-first so an overflow of draining providers sheds the stalest.
self._retired: OrderedDict[TracerProvider, None] = OrderedDict() # mutable-ok: draining evicted providers
@ -182,12 +185,16 @@ class TenantTracerCache:
project_headers: Final = self._project_headers(auth_metadata)
if not credential_headers and not project_headers:
return TenantRoute(tracer=default, detached=False)
# A fixed per-integration region endpoint (New Relic us/eu), never a
# caller-supplied host; ``None`` keeps the preset's own endpoint.
endpoint: Final = dynamic_otlp_endpoint(self._callback_name, dynamic_params)
cache_key: Final = (
tuple(sorted(credential_headers.items())),
tuple(sorted(project_headers.items())),
endpoint,
)
with self._lock:
provider: Final = self._cached_provider_locked(cache_key, credential_headers, project_headers)
provider: Final = self._cached_provider_locked(cache_key, credential_headers, project_headers, endpoint)
self._open_span_counts[provider] = self._open_span_counts.get(provider, 0) + 1
evicted: Final = self._evicted_on_overflow_locked()
if evicted is not None:
@ -200,15 +207,16 @@ class TenantTracerCache:
def _cached_provider_locked(
self,
cache_key: tuple[_HeaderItems, _HeaderItems],
cache_key: tuple[_HeaderItems, _HeaderItems, str | None],
credential_headers: Mapping[str, str],
project_headers: Mapping[str, str],
endpoint: str | None,
) -> TracerProvider:
cached: Final = self._providers.get(cache_key)
if cached is not None:
self._providers.move_to_end(cache_key)
return cached
built: Final = build_tracer_provider(self._routed_config(credential_headers, project_headers))
built: Final = build_tracer_provider(self._routed_config(credential_headers, project_headers, endpoint))
self._providers[cache_key] = built
return built
@ -257,6 +265,7 @@ class TenantTracerCache:
self,
credential_headers: Mapping[str, str],
project_headers: Mapping[str, str],
endpoint: str | None = None,
) -> OpenTelemetryV2Config:
"""Clone the config, rewriting headers on the callback's own exporter.
@ -272,7 +281,8 @@ class TenantTracerCache:
``Authorization``), which must survive routing to a project.
"""
exporters: Final = [
self._routed_exporter(spec, credential_headers, project_headers) for spec in self._config.exporters
self._routed_exporter(spec, credential_headers, project_headers, endpoint)
for spec in self._config.exporters
]
return self._config.model_copy(update={"exporters": exporters})
@ -281,6 +291,7 @@ class TenantTracerCache:
spec: ExporterSpec,
credential_headers: Mapping[str, str],
project_headers: Mapping[str, str],
endpoint: str | None = None,
) -> ExporterSpec:
kind: Final = spec.kind.lower()
if spec.owner != self._callback_name or kind in _NON_OTLP_KINDS:
@ -291,4 +302,10 @@ class TenantTracerCache:
if project_headers and kind not in _GRPC_KINDS
else base
)
return spec if routed == spec.headers else spec.model_copy(update={"headers": routed})
update: Final = { # mutable-ok: model_copy(update=...) requires a plain dict
field: value
for field, value in (("headers", routed), ("endpoint", endpoint))
if (field == "headers" and routed != spec.headers)
or (field == "endpoint" and endpoint is not None and endpoint != spec.endpoint)
}
return spec if not update else spec.model_copy(update=update)

70
litellm/integrations/otel/presets/__init__.py
View file

@ -21,6 +21,11 @@ from litellm.integrations.otel.presets.langfuse import (
)
from litellm.integrations.otel.presets.langtrace import langtrace_preset
from litellm.integrations.otel.presets.levo import levo_preset
from litellm.integrations.otel.presets.newrelic import (
newrelic_dynamic_endpoint,
newrelic_dynamic_headers,
newrelic_preset,
)
from litellm.integrations.otel.presets.phoenix import (
phoenix_preset,
phoenix_project_headers,
@ -30,25 +35,45 @@ from litellm.types.utils import StandardCallbackDynamicParams
#: Callback name → preset. The ``Preset`` annotation makes mypy verify every
#: registered value matches the preset interface.
PRESET_BY_CALLBACK: Final[dict[str, Preset]] = {
"agentops": agentops_preset,
"arize": arize_preset,
"arize_phoenix": phoenix_preset,
"langfuse_otel": langfuse_preset,
"langtrace": langtrace_preset,
"levo": levo_preset,
"weave_otel": weave_preset,
}
PRESET_BY_CALLBACK: Final[Mapping[str, Preset]] = MappingProxyType(
{
"agentops": agentops_preset,
"arize": arize_preset,
"arize_phoenix": phoenix_preset,
"langfuse_otel": langfuse_preset,
"langtrace": langtrace_preset,
"levo": levo_preset,
"newrelic": newrelic_preset,
"weave_otel": weave_preset,
}
)
#: Callback name → per-request OTLP header builder (team/key multi-tenant
#: routing). Only integrations that support dynamic credentials appear here —
#: Arize-Phoenix/Langtrace/Levo/AgentOps don't, so they use the logger's
#: default tracer.
DYNAMIC_HEADERS_BY_CALLBACK: Final[dict[str, Callable[[StandardCallbackDynamicParams], dict[str, str]]]] = {
"arize": arize_dynamic_headers,
"langfuse_otel": langfuse_dynamic_headers,
"weave_otel": weave_dynamic_headers,
}
DYNAMIC_HEADERS_BY_CALLBACK: Final[Mapping[str, Callable[[StandardCallbackDynamicParams], dict[str, str]]]] = (
MappingProxyType(
{
"arize": arize_dynamic_headers,
"langfuse_otel": langfuse_dynamic_headers,
"newrelic": newrelic_dynamic_headers,
"weave_otel": weave_dynamic_headers,
}
)
)
#: Callback name → per-request OTLP endpoint resolver. Only integrations whose
#: destination host varies per tenant (from a fixed region table, never a
#: caller-supplied URL) appear here; for everyone else the preset's endpoint is
#: authoritative.
DYNAMIC_ENDPOINT_BY_CALLBACK: Final[Mapping[str, Callable[[StandardCallbackDynamicParams], str | None]]] = (
MappingProxyType(
{
"newrelic": newrelic_dynamic_endpoint,
}
)
)
#: Callback name → per-request *routing* header builder, sourced from the key/team
@ -98,17 +123,34 @@ def project_routing_headers(
return builder(auth_metadata)
def dynamic_otlp_endpoint(
callback_name: str | None,
dynamic_params: StandardCallbackDynamicParams | None,
) -> str | None:
"""Per-request OTLP endpoint for ``callback_name``, or ``None`` if N/A.
``None`` means "keep the preset's own endpoint".
"""
resolver: Final = DYNAMIC_ENDPOINT_BY_CALLBACK.get(callback_name or "")
if resolver is None or not dynamic_params:
return None
return resolver(dynamic_params)
__all__ = [
"DYNAMIC_ENDPOINT_BY_CALLBACK",
"DYNAMIC_HEADERS_BY_CALLBACK",
"PRESET_BY_CALLBACK",
"PROJECT_HEADERS_BY_CALLBACK",
"Preset",
"agentops_preset",
"arize_preset",
"dynamic_otlp_endpoint",
"dynamic_otlp_headers",
"langfuse_preset",
"langtrace_preset",
"levo_preset",
"newrelic_preset",
"phoenix_preset",
"project_routing_headers",
"weave_preset",

104
litellm/integrations/otel/presets/newrelic.py Normal file
View file

@ -0,0 +1,104 @@
"""New Relic preset — OTLP/HTTP exporter to New Relic + GenAI vocabulary."""
from collections.abc import Mapping
from types import MappingProxyType
from typing import Final
from pydantic import Field
from pydantic_settings import BaseSettings, SettingsConfigDict
from litellm._logging import verbose_logger
from litellm.integrations.otel.model.config import (
ExporterOwner,
ExporterSpec,
OpenTelemetryV2Config,
)
from litellm.integrations.otel.presets.utils import ensure_mappers
from litellm.types.utils import StandardCallbackDynamicParams
#: Region -> OTLP base endpoint. A fixed table by design: team config picks a
#: region enum rather than a free-form endpoint, so callback vars can never
#: redirect telemetry to an arbitrary host.
NEWRELIC_OTLP_ENDPOINT_BY_REGION: Final[Mapping[str, str]] = MappingProxyType(
{
"us": "https://otlp.nr-data.net",
"eu": "https://otlp.eu01.nr-data.net",
}
)
_DEFAULT_REGION: Final = "us"
class _NewRelicSettings(BaseSettings):
model_config = SettingsConfigDict(case_sensitive=False, extra="ignore")
# The same env vars the agent-based integration documents; the key is the
# operator-level fallback for traffic without team credentials, the region
# picks that fallback's data center, and the record-content flag keeps its
# documented meaning when the OTel path replaces the agent.
license_key: str | None = Field(default=None, validation_alias="NEW_RELIC_LICENSE_KEY")
region: str | None = Field(default=None, validation_alias="NEW_RELIC_REGION")
record_content: bool | None = Field(default=None, validation_alias="NEW_RELIC_AI_MONITORING_RECORD_CONTENT_ENABLED")
def newrelic_preset(
*,
config_overrides: OpenTelemetryV2Config | None = None,
) -> OpenTelemetryV2Config:
settings: Final = _NewRelicSettings()
base: Final = config_overrides or OpenTelemetryV2Config()
endpoint: Final = NEWRELIC_OTLP_ENDPOINT_BY_REGION.get(
(settings.region or _DEFAULT_REGION).lower(), NEWRELIC_OTLP_ENDPOINT_BY_REGION[_DEFAULT_REGION]
)
return base.model_copy(
update={
"exporters": [
*base.exporters,
ExporterSpec(
kind="otlp_http",
endpoint=endpoint,
headers=(f"api-key={settings.license_key}" if settings.license_key else None),
owner=ExporterOwner.NEWRELIC,
requires_headers=True,
),
],
# New Relic ingests the OTLP GenAI semantic conventions natively.
"mapper_names": ensure_mappers(base.mapper_names, "genai"),
**(
{"capture_message_content": ("span_only" if settings.record_content else "no_content")}
if settings.record_content is not None
else {}
),
}
)
def newrelic_dynamic_headers(params: StandardCallbackDynamicParams) -> dict[str, str]:
"""Per-request New Relic OTLP headers from team/key dynamic params."""
api_key: Final = params.get("newrelic_api_key")
return {header: value for header, value in (("api-key", api_key),) if value}
def newrelic_dynamic_endpoint(params: StandardCallbackDynamicParams) -> str:
"""Per-request OTLP endpoint for the team's ``newrelic_region``.
Always the team's own region endpoint, defaulting to US when the team left
the region unset. It never falls through to the preset's endpoint, which
follows the operator's ``NEW_RELIC_REGION`` env; a team that saved only its
ingest key must not inherit the operator's region and have its US-account
spans rejected by an EU-configured default (or vice versa). An unknown
region likewise resolves to the documented US default rather than a guess.
"""
region: Final = params.get("newrelic_region")
default_endpoint: Final = NEWRELIC_OTLP_ENDPOINT_BY_REGION[_DEFAULT_REGION]
if not region:
return default_endpoint
endpoint: Final = NEWRELIC_OTLP_ENDPOINT_BY_REGION.get(region.lower())
if endpoint is None:
verbose_logger.warning(
"New Relic: unknown newrelic_region %r; supported regions: %s. Using the default (US) endpoint.",
region,
", ".join(sorted(NEWRELIC_OTLP_ENDPOINT_BY_REGION)),
)
return default_endpoint
return endpoint

99
litellm/integrations/s3_v2.py
View file

@ -11,6 +11,7 @@ import time
from collections.abc import Mapping
from datetime import datetime
from typing import Final, cast
from urllib.parse import quote
import litellm
from litellm._logging import print_verbose, verbose_logger
@ -206,6 +207,23 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
params.get("s3_sse_kms_key_id") or s3_sse_kms_key_id,
)
def _build_object_url(self, s3_object_key: str) -> str:
"""
Build the exact URL that is both signed and sent, with the key percent-encoded once.
S3SigV4Auth signs the path verbatim while S3 canonicalizes the received path with reserved
characters encoded, so an unencoded `=`, `+`, `&`, `#`, `?`, `%` or space in the key makes
the two signatures disagree (403 SignatureDoesNotMatch).
"""
encoded_key: Final = quote(s3_object_key, safe="/")
if self.s3_endpoint_url and self.s3_bucket_name:
if self.s3_use_virtual_hosted_style:
endpoint_host: Final = self.s3_endpoint_url.replace("https://", "").replace("http://", "")
protocol: Final = "https://" if self.s3_endpoint_url.startswith("https://") else "http://"
return f"{protocol}{self.s3_bucket_name}.{endpoint_host}/{encoded_key}"
return f"{self.s3_endpoint_url}/{self.s3_bucket_name}/{encoded_key}"
return f"https://{self.s3_bucket_name}.s3.{self.s3_region_name}.amazonaws.com/{encoded_key}"
def _sse_headers(self) -> Mapping[str, str]:
candidates: Final = {
"x-amz-server-side-encryption": self.s3_server_side_encryption,
@ -292,7 +310,6 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
import base64
import hashlib
import requests
from botocore.auth import S3SigV4Auth
from botocore.awsrequest import AWSRequest
except ImportError:
@ -316,18 +333,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
verbose_logger.debug("s3_v2 logger - uploading data to s3 - %s", batch_logging_element.s3_object_key)
verbose_logger.debug("s3_v2 logger - s3_verify setting: %s", self.s3_verify)
# Prepare the URL
url = f"https://{self.s3_bucket_name}.s3.{self.s3_region_name}.amazonaws.com/{batch_logging_element.s3_object_key}"
if self.s3_endpoint_url and self.s3_bucket_name:
if self.s3_use_virtual_hosted_style:
# Virtual-hosted-style: bucket.endpoint/key
endpoint_host: Final = self.s3_endpoint_url.replace("https://", "").replace("http://", "")
protocol: Final = "https://" if self.s3_endpoint_url.startswith("https://") else "http://"
url = f"{protocol}{self.s3_bucket_name}.{endpoint_host}/{batch_logging_element.s3_object_key}"
else:
# Path-style: endpoint/bucket/key
url = self.s3_endpoint_url + "/" + self.s3_bucket_name + "/" + batch_logging_element.s3_object_key
url: Final = self._build_object_url(batch_logging_element.s3_object_key)
# Convert JSON to string
json_string: Final = safe_dumps(batch_logging_element.payload)
@ -348,29 +354,19 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
"Cache-Control": "private, immutable, max-age=31536000, s-maxage=0",
**self._sse_headers(),
}
req: Final = requests.Request("PUT", url, data=json_string, headers=headers)
prepped: Final = req.prepare()
# Sign the request
aws_request: Final = AWSRequest(
method=prepped.method,
url=prepped.url,
data=prepped.body,
headers=prepped.headers,
)
aws_request: Final = AWSRequest(method="PUT", url=url, data=json_string, headers=headers)
aws_region_name: Final = self.get_aws_region_name_for_non_llm_api_calls(aws_region_name=self.s3_region_name)
S3SigV4Auth(credentials, "s3", aws_region_name).add_auth(aws_request)
# Prepare the signed headers
signed_headers: Final = dict(aws_request.headers.items())
# Use prepared URL so path segments match SigV4 canonical request (e.g. %20 for spaces).
request_url: Final = prepped.url or url
# Make the request with retry for transient S3 errors (500/503)
max_retries: Final = 3
for attempt in range(max_retries):
response = await self.async_httpx_client.put(request_url, data=json_string, headers=signed_headers)
response = await self.async_httpx_client.put(url, data=json_string, headers=signed_headers)
if response.status_code in (500, 503) and attempt < max_retries - 1:
wait_time = 2**attempt # 1s, 2s
verbose_logger.warning(
@ -478,7 +474,6 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
import base64
import hashlib
import requests
from botocore.auth import S3SigV4Auth
from botocore.awsrequest import AWSRequest
from botocore.credentials import Credentials
@ -493,18 +488,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
aws_region_name=self.s3_region_name,
)
# Prepare the URL
url = f"https://{self.s3_bucket_name}.s3.{self.s3_region_name}.amazonaws.com/{batch_logging_element.s3_object_key}"
if self.s3_endpoint_url and self.s3_bucket_name:
if self.s3_use_virtual_hosted_style:
# Virtual-hosted-style: bucket.endpoint/key
endpoint_host: Final = self.s3_endpoint_url.replace("https://", "").replace("http://", "")
protocol: Final = "https://" if self.s3_endpoint_url.startswith("https://") else "http://"
url = f"{protocol}{self.s3_bucket_name}.{endpoint_host}/{batch_logging_element.s3_object_key}"
else:
# Path-style: endpoint/bucket/key
url = self.s3_endpoint_url + "/" + self.s3_bucket_name + "/" + batch_logging_element.s3_object_key
url: Final = self._build_object_url(batch_logging_element.s3_object_key)
# Convert JSON to string
json_string: Final = safe_dumps(batch_logging_element.payload)
@ -525,32 +509,22 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
"Cache-Control": "private, immutable, max-age=31536000, s-maxage=0",
**self._sse_headers(),
}
req: Final = requests.Request("PUT", url, data=json_string, headers=headers)
prepped: Final = req.prepare()
# Sign the request
aws_request: Final = AWSRequest(
method=prepped.method,
url=prepped.url,
data=prepped.body,
headers=prepped.headers,
)
aws_request: Final = AWSRequest(method="PUT", url=url, data=json_string, headers=headers)
aws_region_name: Final = self.get_aws_region_name_for_non_llm_api_calls(aws_region_name=self.s3_region_name)
S3SigV4Auth(credentials, "s3", aws_region_name).add_auth(aws_request)
# Prepare the signed headers
signed_headers: Final = dict(aws_request.headers.items())
# Use prepared URL so path segments match SigV4 canonical request (e.g. %20 for spaces).
request_url: Final = prepped.url or url
httpx_client: Final = _get_httpx_client(
params=({"ssl_verify": self.s3_verify} if self.s3_verify is not None else None)
)
# Make the request with retry for transient S3 errors (500/503)
max_retries: Final = 3
for attempt in range(max_retries):
response = httpx_client.put(request_url, data=json_string, headers=signed_headers)
response = httpx_client.put(url, data=json_string, headers=signed_headers)
if response.status_code in (500, 503) and attempt < max_retries - 1:
wait_time = 2**attempt # 1s, 2s
verbose_logger.warning(
@ -582,7 +556,6 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
try:
import hashlib
import requests
from botocore.auth import S3SigV4Auth
from botocore.awsrequest import AWSRequest
except ImportError:
@ -607,18 +580,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
verbose_logger.debug("s3_v2 logger - downloading data from s3 - %s", s3_object_key)
# Prepare the URL
url = f"https://{self.s3_bucket_name}.s3.{self.s3_region_name}.amazonaws.com/{s3_object_key}"
if self.s3_endpoint_url and self.s3_bucket_name:
if self.s3_use_virtual_hosted_style:
# Virtual-hosted-style: bucket.endpoint/key
endpoint_host: Final = self.s3_endpoint_url.replace("https://", "").replace("http://", "")
protocol: Final = "https://" if self.s3_endpoint_url.startswith("https://") else "http://"
url = f"{protocol}{self.s3_bucket_name}.{endpoint_host}/{s3_object_key}"
else:
# Path-style: endpoint/bucket/key
url = self.s3_endpoint_url + "/" + self.s3_bucket_name + "/" + s3_object_key
url: Final = self._build_object_url(s3_object_key)
# Prepare the request for GET operation
# For GET requests, we need x-amz-content-sha256 with hash of empty string
@ -626,22 +588,15 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
headers: Final = {
"x-amz-content-sha256": empty_string_hash,
}
req: Final = requests.Request("GET", url, headers=headers)
prepped: Final = req.prepare()
# Sign the request
aws_request: Final = AWSRequest(
method=prepped.method,
url=prepped.url,
headers=prepped.headers,
)
aws_request: Final = AWSRequest(method="GET", url=url, headers=headers)
S3SigV4Auth(credentials, "s3", self.s3_region_name).add_auth(aws_request)
# Prepare the signed headers
signed_headers: Final = dict(aws_request.headers.items())
request_url: Final = prepped.url or url
response: Final = await self.async_httpx_client.get(request_url, headers=signed_headers)
response: Final = await self.async_httpx_client.get(url, headers=signed_headers)
if response.status_code != 200:
verbose_logger.exception("S3 object not found, saw response=", response.text)

313
litellm/interactions/background_cost_polling.py Normal file
View file

@ -0,0 +1,313 @@
"""
Cost tracking for background interactions.
A create request with ``background=true`` returns ``in_progress`` with no
usage block, and GET polls are deliberately never billed (billing them would
double-charge every poll; the GET response also does not echo ``background``,
so a poll cannot be told apart from a re-fetch of an already-billed
interaction). The create call is therefore the only place that can own
billing: it schedules a poll task that fetches the interaction until it
reaches a terminal status and logs the final usage as a single success event
attributed to the original request.
``requires_action`` is terminal for the interaction it names. The API has no
operation that resumes one: a caller answers a tool request by creating a new
interaction whose ``previous_interaction_id`` points at it, and that new
interaction bills itself. The paused interaction keeps the tokens it already
spent producing the tool request, so it is billed and settled where it stops
rather than polled until the timeout, which would both lose that usage and
hold its budget reservation open for the whole timeout window.
Deleting an interaction makes every subsequent poll fail, which would let a
caller retrieve the completed output themselves and then delete it before the
poll task settles, leaving the work unbilled and the budget reservation
refunded at the poll timeout. ``adelete`` therefore settles any pending poll
for the interaction before dispatching the delete: it fetches the current
state with the create's credentials, bills it if it is terminal with usage,
and releases the reservation otherwise. A settlement gate on the create's
logging object makes the poll task and the delete path mutually exclusive, so
the interaction is billed exactly once no matter who settles first.
"""
import asyncio
from collections.abc import Awaitable, Callable, Iterator, Mapping
from dataclasses import dataclass
from typing import TYPE_CHECKING, Final, TypeAlias
from litellm._logging import verbose_logger
from litellm.constants import (
BACKGROUND_INTERACTION_COST_POLL_INITIAL_INTERVAL_SECONDS,
BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS,
BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS,
BACKGROUND_INTERACTION_COST_POLLING_ENABLED,
)
from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs
from litellm.types.interactions import InteractionsAPIResponse
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
_TERMINAL_STATUSES: Final = frozenset(
{"completed", "failed", "cancelled", "incomplete", "budget_exceeded", "requires_action"}
)
_POLLABLE_STATUSES: Final = frozenset({"in_progress", "queued"})
_STATUSES_THAT_PRODUCED_OUTPUT: Final = frozenset({"completed", "requires_action"})
@dataclass(frozen=True, slots=True)
class BackgroundInteractionPollContext:
interaction_id: str
custom_llm_provider: str
logging_obj: "LiteLLMLoggingObj"
api_key: str | None = None
api_base: str | None = None
initial_interval_seconds: float = BACKGROUND_INTERACTION_COST_POLL_INITIAL_INTERVAL_SECONDS
max_interval_seconds: float = BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS
timeout_seconds: float = BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS
FetchInteraction: TypeAlias = Callable[[BackgroundInteractionPollContext], Awaitable[InteractionsAPIResponse]]
async def _fetch_interaction(context: BackgroundInteractionPollContext) -> InteractionsAPIResponse:
from litellm.interactions import aget
return await aget(
interaction_id=context.interaction_id,
custom_llm_provider=context.custom_llm_provider,
api_key=context.api_key,
api_base=context.api_base,
**{
"no-log": True
}, # mutable-ok: "no-log" is not a valid identifier, so it can only be passed through a mapping
)
def _poll_intervals(initial: float, maximum: float, timeout: float) -> Iterator[float]:
elapsed = 0.0
interval = initial
while interval > 0 and elapsed + interval <= timeout:
yield interval
elapsed += interval
interval = min(interval * 2, maximum)
_SETTLED_KEY = "background_interaction_settled"
def _is_settled(logging_obj: "LiteLLMLoggingObj") -> bool:
return logging_obj.model_call_details.get(_SETTLED_KEY) is True
def _claim_settlement(logging_obj: "LiteLLMLoggingObj") -> bool:
"""
Exactly-once gate between the poll task and the delete-time settlement:
both run on the same event loop and neither awaits between reading and
setting the flag, so whichever claims first owns billing or release.
"""
if _is_settled(logging_obj):
return False
logging_obj.model_call_details[_SETTLED_KEY] = True # rebind-ok: both settlers must see the same settlement flag
return True
async def poll_and_log_background_interaction_cost(
context: BackgroundInteractionPollContext,
fetch_interaction: FetchInteraction = _fetch_interaction,
) -> None:
last_seen_status: str | None = None
for interval in _poll_intervals(
initial=context.initial_interval_seconds,
maximum=context.max_interval_seconds,
timeout=context.timeout_seconds,
):
await asyncio.sleep(interval)
if _is_settled(context.logging_obj):
return
try:
response = await fetch_interaction(context)
except Exception as e: # noqa: BLE001 # any fetch error must not kill the billing poll loop
verbose_logger.debug(
"Background interaction cost poll for %s failed, will retry: %s",
context.interaction_id,
e,
)
continue
last_seen_status = response.status
if response.status not in _TERMINAL_STATUSES:
continue
if not _claim_settlement(context.logging_obj):
return
if response.usage is not None:
await _bill_settled_interaction(logging_obj=context.logging_obj, response=response)
else:
await _release_open_budget_reservation(logging_obj=context.logging_obj)
return
if not _claim_settlement(context.logging_obj):
return
if last_seen_status is not None and last_seen_status not in _POLLABLE_STATUSES:
verbose_logger.error(
"Gave up cost polling for background interaction %s after %ss: its last status %r is in neither "
"the pollable nor the terminal set, so this proxy never learned how to settle it and its usage "
"will not be tracked",
context.interaction_id,
context.timeout_seconds,
last_seen_status,
)
else:
verbose_logger.warning(
"Gave up cost polling for background interaction %s after %ss; its usage will not be tracked",
context.interaction_id,
context.timeout_seconds,
)
await _release_open_budget_reservation(logging_obj=context.logging_obj)
async def _release_open_budget_reservation(logging_obj: "LiteLLMLoggingObj") -> None:
"""
The proxy keeps the pre-call budget reservation open for an in-progress
background interaction so concurrent creates cannot stack past the budget.
The completion success event reconciles it to the actual cost; when the
interaction terminates without billable usage (or polling gives up, or it
is deleted before settling), no such event fires, so whoever claims the
settlement must release the reservation here or the spend counters stay
pinned at the estimated cost.
"""
metadata = get_litellm_metadata_from_kwargs(kwargs=logging_obj.model_call_details)
budget_reservation = metadata.get("user_api_key_budget_reservation")
if not isinstance(budget_reservation, dict):
return
from litellm.proxy.spend_tracking.budget_reservation import release_budget_reservation
try:
await release_budget_reservation(budget_reservation=budget_reservation)
except Exception: # noqa: BLE001 # a failed release must not crash the poll task; counters expire via TTL
verbose_logger.exception("Failed to release budget reservation for an unbilled background interaction")
async def _bill_settled_interaction(logging_obj: "LiteLLMLoggingObj", response: InteractionsAPIResponse) -> None:
"""
Claiming the settlement makes the claimer solely responsible for the
reservation, and no one retries a claim that is already set. A billing
failure here must therefore release the reservation on its way out, or it
stays pinned at the estimated cost until the whole poll times out.
"""
try:
await logging_obj.async_log_background_interaction_completion(result=response)
except Exception:
await _release_open_budget_reservation(logging_obj=logging_obj)
raise
def is_pollable_background_interaction(response: InteractionsAPIResponse) -> bool:
"""
The single gate deciding whether a create's response gets a poll task.
The proxy's success callback defers releasing the budget reservation for
exactly these responses, on the promise that a poll task will settle them,
so a response one site accepts and the other refuses strands its
reservation on the spend counters with nothing left to reconcile it.
``queued`` belongs here alongside ``in_progress``. It is the API's
not-started-yet state, so it reaches a terminal status the same way and
needs polling for the same reason: nothing else in the proxy ever bills a
create that came back without usage, so a status missing from both this
set and ``_TERMINAL_STATUSES`` is billed nowhere and alerts nobody.
"""
return response.status in _POLLABLE_STATUSES and bool(response.id)
def missing_usage_is_expected(response: InteractionsAPIResponse) -> bool:
"""
Whether a response arriving with no usage block is a normal outcome rather
than lost billing data. An interaction that is still running, or that
stopped at ``failed``, ``cancelled``, ``incomplete`` or ``budget_exceeded``,
has nothing to charge for and should not raise a cost-tracking alarm.
``completed`` and ``requires_action`` both mean the model produced output,
so a usage block is always expected with them. If one arrives without it
the charge for real work has been lost, which is precisely what the
proxy's cost-tracking alert exists to surface.
"""
return response.status not in _STATUSES_THAT_PRODUCED_OUTPUT
@dataclass(frozen=True, slots=True)
class _ActiveBackgroundPoll:
task: "asyncio.Task[None]"
context: BackgroundInteractionPollContext
_ACTIVE_POLLS: dict[str, _ActiveBackgroundPoll] = {} # mutable-ok: asyncio needs strong refs to running poll tasks
def _discard_poll(interaction_id: str, task: "asyncio.Task[None]") -> None:
entry = _ACTIVE_POLLS.get(interaction_id)
if entry is not None and entry.task is task:
del _ACTIVE_POLLS[interaction_id]
def maybe_schedule_background_interaction_cost_polling(
response: object,
create_kwargs: Mapping[str, object],
custom_llm_provider: str,
) -> "asyncio.Task[None] | None":
from litellm.litellm_core_utils.litellm_logging import Logging
if not BACKGROUND_INTERACTION_COST_POLLING_ENABLED:
return None
if not isinstance(response, InteractionsAPIResponse):
return None
if not is_pollable_background_interaction(response):
return None
logging_obj = create_kwargs.get("litellm_logging_obj")
if not isinstance(logging_obj, Logging):
return None
try:
asyncio.get_running_loop()
except RuntimeError:
return None
api_key = create_kwargs.get("api_key")
api_base = create_kwargs.get("api_base")
context = BackgroundInteractionPollContext(
interaction_id=response.id,
custom_llm_provider=custom_llm_provider,
logging_obj=logging_obj,
api_key=api_key if isinstance(api_key, str) else None,
api_base=api_base if isinstance(api_base, str) else None,
)
task = asyncio.create_task(poll_and_log_background_interaction_cost(context))
_ACTIVE_POLLS[context.interaction_id] = _ActiveBackgroundPoll(task=task, context=context)
task.add_done_callback(
lambda finished, interaction_id=context.interaction_id: _discard_poll(interaction_id, finished)
)
return task
async def maybe_settle_background_interaction_before_delete(
interaction_id: str,
fetch_interaction: FetchInteraction = _fetch_interaction,
) -> None:
entry = _ACTIVE_POLLS.get(interaction_id)
if entry is None:
return
context = entry.context
try:
response = await fetch_interaction(context)
except Exception as e: # noqa: BLE001 # unfetchable pre-delete state settles by releasing the reservation
verbose_logger.debug(
"Could not fetch background interaction %s before delete, releasing its reservation: %s",
interaction_id,
e,
)
if _claim_settlement(context.logging_obj):
await _release_open_budget_reservation(logging_obj=context.logging_obj)
return
if not _claim_settlement(context.logging_obj):
return
if response.status in _TERMINAL_STATUSES and response.usage is not None:
await _bill_settled_interaction(logging_obj=context.logging_obj, response=response)
return
await _release_open_budget_reservation(logging_obj=context.logging_obj)

12
litellm/interactions/main.py
View file

@ -40,6 +40,10 @@ from typing import Any, Final
import httpx
import litellm
from litellm.interactions.background_cost_polling import (
maybe_schedule_background_interaction_cost_polling,
maybe_settle_background_interaction_before_delete,
)
from litellm.interactions.http_handler import interactions_http_handler
from litellm.interactions.utils import (
InteractionsAPIRequestUtils,
@ -171,6 +175,12 @@ async def acreate(
else:
response = init_response
maybe_schedule_background_interaction_cost_polling(
response=response,
create_kwargs=kwargs,
custom_llm_provider=custom_llm_provider,
)
return response
except Exception as e:
raise litellm.exception_type(
@ -462,6 +472,8 @@ async def adelete(
loop: Final = asyncio.get_event_loop()
kwargs["adelete_interaction"] = True
await maybe_settle_background_interaction_before_delete(interaction_id=interaction_id)
func: Final = partial(
delete,
interaction_id=interaction_id,

1
litellm/litellm_core_utils/exception_mapping_utils.py
View file

@ -2301,6 +2301,7 @@ def exception_type(
or custom_llm_provider == "custom_openai"
or custom_llm_provider in litellm.openai_compatible_providers
or custom_llm_provider == "mistral"
or custom_llm_provider == "runwayml"
):
_map_openai_exception(
model=model,

32
litellm/litellm_core_utils/initialize_dynamic_callback_params.py
View file

@ -46,7 +46,7 @@ def validate_no_callback_env_reference(param: str, value: object, *, source: str
# Hardcoded list of supported callback params to avoid runtime inspection issues with TypedDict
_supported_callback_params: Final = [
_supported_callback_params: Final[tuple[str, ...]] = (
"langfuse_public_key",
"langfuse_secret",
"langfuse_secret_key",
@ -72,8 +72,10 @@ _supported_callback_params: Final = [
"dd_site",
"dd_agent_host",
"dd_agent_port",
"newrelic_api_key",
"newrelic_region",
"turn_off_message_logging",
]
)
_request_blocked_callback_params: Final = frozenset(
{
@ -83,6 +85,20 @@ _request_blocked_callback_params: Final = frozenset(
"dd_site",
"dd_agent_host",
"dd_agent_port",
"newrelic_api_key",
"newrelic_region",
}
)
# Request-blocked params that must still reach ``standard_callback_dynamic_params``
# when the proxy itself stamped them from admin-configured team/key callback
# settings (the trusted-vars channel). The OTel per-tenant tracer routing reads
# ``standard_callback_dynamic_params``, so without this overlay a blocked param
# could never drive routing at all.
_trusted_overlay_callback_params: Final = frozenset(
{
"newrelic_api_key",
"newrelic_region",
}
)
@ -121,7 +137,9 @@ def initialize_standard_callback_dynamic_params(
if param in kwargs:
_param_value = kwargs.get(param)
validate_no_callback_env_reference(param, _param_value, source="request body")
standard_callback_dynamic_params[param] = _param_value
standard_callback_dynamic_params[param] = ( # pyright: ignore[reportGeneralTypeIssues] # several supported params predate their StandardCallbackDynamicParams fields
_param_value
)
for slot_label, metadata in iter_client_callback_metadata_dicts(kwargs):
for param in _supported_callback_params:
@ -130,6 +148,12 @@ def initialize_standard_callback_dynamic_params(
if param not in standard_callback_dynamic_params and param in metadata:
_param_value = metadata.get(param)
validate_no_callback_env_reference(param, _param_value, source=slot_label)
standard_callback_dynamic_params[param] = _param_value
standard_callback_dynamic_params[param] = ( # pyright: ignore[reportGeneralTypeIssues] # several supported params predate their StandardCallbackDynamicParams fields
_param_value
)
for param, trusted_value in get_trusted_callback_params(kwargs):
if param in _trusted_overlay_callback_params:
standard_callback_dynamic_params[param] = trusted_value
return standard_callback_dynamic_params

157
litellm/litellm_core_utils/litellm_logging.py
View file

@ -71,6 +71,9 @@ from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import (
from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
StandardBuiltInToolCostTracking,
)
from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import (
InteractionsUsageObjectTransformation,
)
from litellm.litellm_core_utils.logging_utils import truncate_base64_in_messages
from litellm.litellm_core_utils.model_param_helper import ModelParamHelper
from litellm.litellm_core_utils.redact_messages import (
@ -83,6 +86,10 @@ from litellm.llms.base_llm.search.transformation import SearchResponse
from litellm.responses.utils import ResponseAPILoggingUtils
from litellm.types.agents import LiteLLMSendMessageResponse
from litellm.types.containers.main import ContainerObject
from litellm.types.interactions import (
InteractionsAPIResponse,
InteractionsAPIStreamingResponse,
)
from litellm.types.llms.openai import (
AllMessageValues,
Batch,
@ -2145,6 +2152,11 @@ class Logging(LiteLLMLoggingBaseClass):
or isinstance(logging_result, OpenAIModerationResponse)
or isinstance(logging_result, OCRResponse) # OCR
or isinstance(logging_result, SearchResponse) # Search API
or (
isinstance(logging_result, InteractionsAPIResponse)
and logging_result.usage is not None
and self._is_interactions_create_call_type()
)
or isinstance(logging_result, dict)
and logging_result.get("object") == "vector_store.search_results.page"
or isinstance(logging_result, dict)
@ -2157,6 +2169,87 @@ class Logging(LiteLLMLoggingBaseClass):
return True
return False
def _is_interactions_create_call_type(self) -> bool:
"""
Only interaction creation is billable. GET polls, deletes, and cancels
also return an ``InteractionsAPIResponse`` (with usage once completed),
so recognizing those would write spend on every poll of a background
interaction. The proxy sets ``call_type`` from its route_type
(``create_interaction``/``acreate_interaction``); the SDK sets it from
the decorated function name (``create``/``acreate``).
Recognition additionally requires a usage block (checked at the call
site): a ``background=true`` create returns ``in_progress`` without
usage, and billing it would write a $0 spend log under the interaction
id that collides with the row the background poll task writes once the
interaction completes (see
``litellm.interactions.background_cost_polling``).
"""
return self.call_type in (
CallTypes.create_interaction.value,
CallTypes.acreate_interaction.value,
"create",
"acreate",
)
async def async_log_background_interaction_completion(
self,
result: InteractionsAPIResponse,
) -> None:
"""
Log the terminal result of a background interaction as a fresh success
event. The create request already ran success logging for its
``in_progress`` response (no usage, so no cost was tracked); clearing
the dedup flags lets the completed result flow through cost calculation
and spend tracking exactly once, spanning create to completion.
The poll fetched this body through its own client call, which priced it
against a throwaway logging object holding none of this request's
deployment context: no ``model_info``, no router ``model_id``, no
deployment ``litellm_params``. Keeping that price would bill a
custom-priced deployment at the wrong rate, and it would also satisfy
the "already calculated" shortcut and skip repricing here, leaving the
cost breakdown at the zeros the usage-less create stamped and writing
those zeros to the spend log. Dropping it makes this event price the
settled body itself, against the deployment that served the create.
The same throwaway call stamped the deployment identity that travels
with the price, so ``model_id`` and ``litellm_model_name`` go with it.
Left in place they overwrite the create's real deployment with the
poll's empty one in the payload every logging integration reads.
"""
settled_hidden_params: Final = getattr(result, "_hidden_params", None)
if isinstance(settled_hidden_params, dict):
for poll_scoped_key in ("response_cost", "model_id", "litellm_model_name"):
settled_hidden_params.pop(poll_scoped_key, None)
self._reset_success_emission_dedupe()
await self.async_success_handler(result=result)
def _reset_success_emission_dedupe(self) -> None:
"""
Success callbacks dedupe per request, because the sync and async
handlers both fire on some paths and would otherwise report one call
twice. A settled background interaction is a genuinely second success
event on the same request, so every such marker has to be cleared or
the completion, the only event that carries usage and cost, is
discarded as a duplicate of the in-progress create.
"""
self.model_call_details.pop("has_logged_async_success", None)
litellm_params = self.model_call_details.get("litellm_params")
if not isinstance(litellm_params, dict):
return
metadata = litellm_params.get("metadata")
if not isinstance(metadata, dict):
return
otel_internal = metadata.get("_otel_internal")
if not isinstance(otel_internal, dict):
return
spans_logged = otel_internal.get("spans_logged")
if not isinstance(spans_logged, dict):
return
for scope in [key for key in spans_logged if isinstance(key, tuple) and key[-1:] == ("success",)]:
del spans_logged[scope]
def _flush_passthrough_collected_chunks_helper(
self,
raw_bytes: list[bytes],
@ -2282,7 +2375,9 @@ class Logging(LiteLLMLoggingBaseClass):
is_sync_request: Final = self._is_sync_litellm_request(litellm_params)
try:
## BUILD COMPLETE STREAMED RESPONSE
complete_streaming_response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | None = None
complete_streaming_response: (
ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None
) = None
if "complete_streaming_response" in self.model_call_details:
return # break out of this.
complete_streaming_response = self._get_assembled_streaming_response(
@ -2768,14 +2863,14 @@ class Logging(LiteLLMLoggingBaseClass):
## BUILD COMPLETE STREAMED RESPONSE
if "async_complete_streaming_response" in self.model_call_details:
return # break out of this.
complete_streaming_response: Final[ModelResponse | TextCompletionResponse | ResponsesAPIResponse | None] = (
self._get_assembled_streaming_response(
result=result,
start_time=start_time,
end_time=end_time,
is_async=True,
streaming_chunks=self.streaming_chunks,
)
complete_streaming_response: Final[
ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None
] = self._get_assembled_streaming_response(
result=result,
start_time=start_time,
end_time=end_time,
is_async=True,
streaming_chunks=self.streaming_chunks,
)
if complete_streaming_response is not None:
@ -3558,7 +3653,7 @@ class Logging(LiteLLMLoggingBaseClass):
end_time: datetime.datetime,
is_async: bool,
streaming_chunks: list[object],
) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | None:
) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None:
if self.stream is not True:
return None
if isinstance(result, ModelResponse) or isinstance(result, TextCompletionResponse):
@ -3583,9 +3678,40 @@ class Logging(LiteLLMLoggingBaseClass):
),
)
return result.response
elif isinstance(result, InteractionsAPIStreamingResponse):
return self._assemble_completed_interaction_response(result)
else:
return None
@staticmethod
def _assemble_completed_interaction_response(
result: InteractionsAPIStreamingResponse,
) -> InteractionsAPIResponse | None:
"""
The Interactions API streaming iterator hands the terminal event to the
success handlers: the new schema (Api-Revision: 2026-05-20) emits
``interaction.completed`` carrying the full interaction object, the
legacy schema (2026-05-07) emits a chunk with ``status="completed"``
and usage on the chunk itself. Build the equivalent non-streaming
response so cost calculation and spend tracking see one shape.
"""
if result.event_type == "interaction.completed" and result.interaction is not None:
return InteractionsAPIResponse(**result.interaction)
if result.status == "completed":
return InteractionsAPIResponse(
**result.model_dump(
exclude={ # mutable-ok: pydantic types exclude as set[str], which a frozenset does not satisfy
"event_type",
"delta",
"index",
"step",
"interaction_id",
"interaction",
}
)
)
return None
def _handle_anthropic_messages_response_logging(self, result: Any) -> ModelResponse:
"""
Handles logging for Anthropic messages responses.
@ -4503,6 +4629,9 @@ def _init_custom_logger_compatible_class(
_in_memory_loggers.append(gitlab_logger)
return gitlab_logger
elif logging_integration == "newrelic":
_v2 = _maybe_construct_otel_v2("newrelic", _in_memory_loggers)
if _v2 is not None:
return _v2
for callback in _in_memory_loggers:
if isinstance(callback, NewRelicLogger):
return callback
@ -4789,7 +4918,11 @@ def get_custom_logger_compatible_class(
if isinstance(callback, SMTPEmailLogger):
return callback
elif logging_integration == "newrelic":
from litellm.integrations.otel.logger import OpenTelemetryV2
for callback in _in_memory_loggers:
if isinstance(callback, OpenTelemetryV2) and callback.callback_name == "newrelic":
return callback
if isinstance(callback, NewRelicLogger):
return callback
return None
@ -5085,6 +5218,8 @@ class StandardLoggingPayloadSetup:
elif isinstance(usage, dict):
if ResponseAPILoggingUtils._is_response_api_usage(usage):
return ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(usage)
if InteractionsUsageObjectTransformation.is_interactions_usage_object(usage):
return InteractionsUsageObjectTransformation.transform_interactions_usage_object(usage)
return Usage(**usage)
raise ValueError(f"usage is required, got={usage} of type {type(usage)}")
@ -5111,6 +5246,8 @@ class StandardLoggingPayloadSetup:
if isinstance(_raw, dict):
if ResponseAPILoggingUtils._is_response_api_usage(_raw):
return ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(_raw).model_dump()
if InteractionsUsageObjectTransformation.is_interactions_usage_object(_raw):
return InteractionsUsageObjectTransformation.transform_interactions_usage_object(_raw).model_dump()
return _raw
if isinstance(_raw, Usage):
return _raw.model_dump()

127
litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py
View file

@ -1,6 +1,9 @@
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Any
from litellm.types.utils import (
CompletionTokensDetailsWrapper,
PromptTokensDetailsWrapper,
TranscriptionUsageDurationObject,
TranscriptionUsageTokensObject,
@ -34,3 +37,127 @@ class TranscriptionUsageObjectTransformation:
),
)
return None
_INTERACTIONS_MODALITY_FIELDS: Mapping[str, str] = MappingProxyType(
{
"text": "text_tokens",
"audio": "audio_tokens",
"image": "image_tokens",
"video": "video_tokens",
"document": "text_tokens",
}
)
def _modality_field(entry: Mapping[str, Any]) -> str | None:
return _INTERACTIONS_MODALITY_FIELDS.get(str(entry.get("modality", "")).lower())
def _token_count(value: object) -> int:
return value if isinstance(value, int) else 0
def _modality_token_sums(entries: Sequence[Mapping[str, Any]]) -> Mapping[str, int]:
fields = frozenset(field for entry in entries if (field := _modality_field(entry)) is not None)
return MappingProxyType(
{
field: sum(_token_count(entry.get("tokens")) for entry in entries if _modality_field(entry) == field)
for field in fields
}
)
def _google_search_query_count(usage_object: Mapping[str, Any]) -> int:
return sum(
_token_count(entry.get("count"))
for entry in tuple(usage_object.get("grounding_tool_count") or ())
if isinstance(entry, Mapping) and entry.get("type") == "google_search" # pyright: ignore[reportUnnecessaryIsInstance] # provider JSON, not the empty tuple inferred from `or ()`
)
def _subtract_cached_from_input(
input_sums: Mapping[str, int],
cached_sums: Mapping[str, int],
total_cached_tokens: int,
) -> Mapping[str, int]:
if cached_sums:
return MappingProxyType(
{field: max(0, tokens - cached_sums.get(field, 0)) for field, tokens in input_sums.items()}
)
if total_cached_tokens and "text_tokens" in input_sums:
return MappingProxyType(
{
**input_sums,
"text_tokens": max(0, input_sums["text_tokens"] - total_cached_tokens),
}
)
return input_sums
class InteractionsUsageObjectTransformation:
"""
Maps the Google Interactions API usage block (total_input_tokens,
output_tokens_by_modality, ...) into LiteLLM's chat-format ``Usage`` so the
generic cost calculator and spend tracking can bill it.
"""
@staticmethod
def is_interactions_usage_object(usage_object: object) -> bool:
if not isinstance(usage_object, dict):
return False
if "prompt_tokens" in usage_object or "input_tokens" in usage_object:
return False
return "total_input_tokens" in usage_object or "total_output_tokens" in usage_object
@staticmethod
def transform_interactions_usage_object(usage_object: Mapping[str, Any]) -> Usage:
input_entries = tuple(usage_object.get("input_tokens_by_modality") or ()) + tuple(
usage_object.get("tool_use_tokens_by_modality") or ()
)
cached_sums = _modality_token_sums(tuple(usage_object.get("cached_tokens_by_modality") or ()))
output_sums = _modality_token_sums(tuple(usage_object.get("output_tokens_by_modality") or ()))
total_cached_tokens = _token_count(usage_object.get("total_cached_tokens"))
input_sums = _subtract_cached_from_input(
input_sums=_modality_token_sums(input_entries),
cached_sums=cached_sums,
total_cached_tokens=total_cached_tokens,
)
reasoning_tokens = _token_count(usage_object.get("total_reasoning_tokens")) or _token_count(
usage_object.get("total_thought_tokens")
)
prompt_tokens = _token_count(usage_object.get("total_input_tokens")) + _token_count(
usage_object.get("total_tool_use_tokens")
)
completion_tokens = _token_count(usage_object.get("total_output_tokens")) + reasoning_tokens
total_tokens = _token_count(usage_object.get("total_tokens")) or (prompt_tokens + completion_tokens)
web_search_requests = _google_search_query_count(usage_object)
prompt_tokens_details = (
PromptTokensDetailsWrapper(
cached_tokens=total_cached_tokens or None,
web_search_requests=web_search_requests or None,
**input_sums,
)
if input_sums or total_cached_tokens or web_search_requests
else None
)
completion_tokens_details = (
CompletionTokensDetailsWrapper(
reasoning_tokens=reasoning_tokens or None,
**output_sums,
)
if output_sums or reasoning_tokens
else None
)
return Usage(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=total_tokens,
prompt_tokens_details=prompt_tokens_details,
completion_tokens_details=completion_tokens_details,
cache_read_input_tokens=total_cached_tokens or None,
)

80
litellm/litellm_core_utils/llm_request_utils.py
View file

@ -1,8 +1,88 @@
from collections.abc import Mapping
from typing import Final
import litellm
def _form_field_value(value: object) -> str:
if value is True:
return "true"
if value is False:
return "false"
return str(value)
def _flatten_form_field(key: str, value: object) -> tuple[tuple[str, str], ...]:
if isinstance(value, Mapping):
return tuple(
item for subkey, subvalue in value.items() for item in _flatten_form_field(f"{key}[{subkey}]", subvalue)
)
if isinstance(value, (list, tuple)):
return tuple(item for entry in value for item in _flatten_form_field(f"{key}[]", entry))
if value is None:
return ()
serialized: Final = _form_field_value(value)
if not serialized:
return ()
return ((key, serialized),)
def _is_form_scalar(value: object) -> bool:
return value is not None and not isinstance(value, (Mapping, list, tuple))
def _flatten_form_data_field(key: str, value: object) -> tuple[tuple[str, str | tuple[str, ...]], ...]:
if isinstance(value, Mapping):
return tuple(
item
for subkey, subvalue in value.items()
for item in _flatten_form_data_field(f"{key}[{subkey}]", subvalue)
)
if isinstance(value, (list, tuple)):
if all(_is_form_scalar(entry) for entry in value):
serialized_fields: Final = tuple(field for entry in value if (field := _form_field_value(entry)))
return ((key, serialized_fields),) if serialized_fields else ()
return tuple(item for entry in value for item in _flatten_form_data_field(f"{key}[]", entry))
if value is None:
return ()
serialized: Final = _form_field_value(value)
if not serialized:
return ()
return ((key, serialized),)
def flatten_form_field_values(*sources: Mapping[str, object] | None) -> tuple[tuple[str, str | tuple[str, ...]], ...]:
"""
Flatten JSON-shaped bodies into ``(name, value)`` form fields for a ``dict``-backed
multipart body, applying ``sources`` in order so a later source wins on a key collision
under ``dict.update``. Nested objects become ``key[subkey]`` fields the way the OpenAI SDK
serializes them, so provider params reach a multipart request without handing the httpx
encoder a nested value it rejects with ``Invalid type for value``. A scalar list becomes a
single field carrying a tuple value, which httpx emits as one repeated part per element, so
every element survives instead of collapsing to the last under ``dict.update``.
"""
return tuple(
pair
for source in sources
if source is not None
for top_key, top_value in source.items()
for pair in _flatten_form_data_field(top_key, top_value)
)
def serialize_multipart_form_fields(data: Mapping[str, object]) -> tuple[tuple[str, tuple[None, str]], ...]:
"""
Encode a JSON-shaped body as OpenAI-SDK-style multipart file-tuples so a file-less
request is still sent as multipart/form-data, working around httpx downgrading a
file-less ``data=`` payload to application/x-www-form-urlencoded.
"""
return tuple(
(key, (None, serialized))
for top_key, top_value in data.items()
for key, serialized in _flatten_form_field(top_key, top_value)
)
def _ensure_extra_body_is_safe(extra_body: dict | None) -> dict | None:
"""
Ensure that the extra_body sent in the request is safe, otherwise users will see this error

36
litellm/litellm_core_utils/logging_worker.py
View file

@ -5,7 +5,7 @@ import asyncio
import atexit
import contextvars
import logging
from collections.abc import Coroutine
from collections.abc import Coroutine, Iterator
from typing import Final
from typing_extensions import TypedDict
@ -61,6 +61,19 @@ class LoggingWorker:
# Register cleanup handler to flush remaining events on exit
atexit.register(self._flush_on_exit)
@staticmethod
def _drain_pending(queue: "asyncio.Queue[LoggingTask]") -> tuple[LoggingTask, ...]:
"""Pop every task still queued, without awaiting them, so they can be moved to another queue."""
def _pop_until_empty() -> Iterator[LoggingTask]:
while True:
try:
yield queue.get_nowait()
except asyncio.QueueEmpty:
return
return tuple(_pop_until_empty())
def _ensure_queue(self) -> None:
"""Initialize the queue if it doesn't exist or if event loop has changed."""
try:
@ -69,14 +82,27 @@ class LoggingWorker:
# No running loop, can't initialize
return
# Check if we need to reinitialize due to event loop change
# The queue, semaphore and worker task are all bound to the loop that created them. On a
# loop change we hand the still-pending tasks to a fresh queue instead of dropping them,
# so queued spend-logging coroutines are not silently discarded (and never left un-awaited).
if self._queue is not None and self._bound_loop is not current_loop:
verbose_logger.debug("LoggingWorker: Event loop changed, reinitializing queue and worker")
# Clear old state - these are bound to the old loop
self._queue = None
carried_over: Final = self._drain_pending(self._queue)
new_queue: Final[asyncio.Queue[LoggingTask]] = asyncio.Queue(maxsize=self.max_queue_size)
for carried_task in carried_over:
new_queue.put_nowait(carried_task)
if carried_over:
verbose_logger.warning(
"LoggingWorker: event loop changed; carried %d pending logging task(s) onto the new loop",
len(carried_over),
)
else:
verbose_logger.debug("LoggingWorker: Event loop changed, reinitializing queue and worker")
self._sem = None
self._worker_task = None
self._running_tasks.clear()
self._queue = new_queue
self._bound_loop = current_loop
return
if self._queue is None:
self._queue = asyncio.Queue(maxsize=self.max_queue_size)

58
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -28,8 +28,12 @@ from litellm.types.llms.openai import (
ChatCompletionAssistantMessage,
ChatCompletionFileObject,
ChatCompletionImageObject,
ChatCompletionReasoningItem,
ChatCompletionReasoningSummaryTextBlock,
ChatCompletionRedactedThinkingBlock,
ChatCompletionResponseMessage,
ChatCompletionTextObject,
ChatCompletionThinkingBlock,
ChatCompletionToolParam,
ChatCompletionUserMessage,
)
@ -466,6 +470,8 @@ def update_messages_with_model_file_ids(
from litellm.proxy.openai_files_endpoints.common_utils import (
_is_base64_encoded_unified_file_id,
convert_b64_uid_to_unified_uid,
get_original_file_id,
is_model_embedded_id,
)
for message in messages:
@ -504,6 +510,11 @@ def update_messages_with_model_file_ids(
unified_file_id = convert_b64_uid_to_unified_uid(file_id)
if "llm_output_file_id," in unified_file_id:
provider_file_id = unified_file_id.split("llm_output_file_id,")[1].split(";")[0]
if not provider_file_id and is_model_embedded_id(file_id):
# `litellm:<raw_id>;model,<m>` encoding from the
# x-litellm-model upload path. Strip the wrapper
# so the provider sees its own ID.
provider_file_id = get_original_file_id(file_id)
file_object_file_field["file_id"] = provider_file_id or file_id
if format:
file_object_file_field["format"] = format
@ -531,6 +542,8 @@ def update_responses_input_with_model_file_ids(
from litellm.proxy.openai_files_endpoints.common_utils import (
_is_base64_encoded_unified_file_id,
convert_b64_uid_to_unified_uid,
get_original_file_id,
is_model_embedded_id,
)
if isinstance(input, str):
@ -574,6 +587,13 @@ def update_responses_input_with_model_file_ids(
updated_content_item = content_item.copy()
updated_content_item["file_id"] = provider_file_id
updated_content.append(updated_content_item)
elif is_model_embedded_id(file_id):
# `litellm:<raw_id>;model,<m>` encoding from the
# x-litellm-model upload path. Strip the wrapper
# so the provider sees its own ID.
updated_content_item = content_item.copy()
updated_content_item["file_id"] = get_original_file_id(file_id)
updated_content.append(updated_content_item)
else:
# Not a managed file, keep as-is
updated_content.append(content_item)
@ -1549,6 +1569,44 @@ def _extract_reasoning_content(message: dict) -> tuple[str | None, str | None]:
return None, message_content
def _readable_thinking_text(
block: ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock,
) -> str:
"""The text a chat model can read back, empty for redacted blocks and malformed ones."""
if block.get("type") != "thinking":
return ""
thinking: Final = cast(ChatCompletionThinkingBlock, block).get("thinking") # cast-ok: narrowed by the type tag
return str(thinking or "")
def reasoning_content_from_thinking_blocks(
thinking_blocks: Iterable[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock],
) -> str:
"""Flatten Anthropic thinking blocks into the `reasoning_content` string chat models expect.
Redacted blocks carry no readable text, so they contribute nothing.
"""
return "\n".join(text for block in thinking_blocks if (text := _readable_thinking_text(block)))
def responses_reasoning_item_from_thinking_blocks(
thinking_blocks: Iterable[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock],
) -> ChatCompletionReasoningItem | None:
"""Build a Responses API `reasoning` input item from Anthropic thinking blocks.
The item carries no `id`: the Responses API rejects an empty one and 404s on any id it
did not mint itself, while an item without an id is always accepted.
"""
summary: Final[list[ChatCompletionReasoningSummaryTextBlock]] = [ # mutable-ok: API message payload
ChatCompletionReasoningSummaryTextBlock(type="summary_text", text=text)
for block in thinking_blocks
if (text := _readable_thinking_text(block))
]
if not summary:
return None
return ChatCompletionReasoningItem(type="reasoning", summary=summary)
def _parse_content_for_reasoning(
message_text: str | None,
) -> tuple[str | None, str | None]:

4
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -16,6 +16,7 @@ import litellm.types
import litellm.types.llms
from litellm import verbose_logger
from litellm._uuid import uuid
from litellm.constants import REDACTED_BY_LITELLM
from litellm.litellm_core_utils.url_utils import async_safe_get, safe_get
from litellm.llms.custom_httpx.http_handler import HTTPHandler, get_async_httpx_client
from litellm.types.files import get_file_extension_from_mime_type
@ -5383,12 +5384,13 @@ def _parse_tool_call_arguments(raw: Any, tool_name: str | None, context: str) ->
return raw
if not isinstance(raw, str):
return {}
normalized_raw: Final = "{}" if raw == REDACTED_BY_LITELLM else raw
from litellm.litellm_core_utils.prompt_templates.common_utils import (
parse_tool_call_arguments,
)
try:
parsed: Final = parse_tool_call_arguments(raw, tool_name=tool_name, context=context)
parsed: Final = parse_tool_call_arguments(normalized_raw, tool_name=tool_name, context=context)
except ValueError as e:
verbose_logger.warning("Failed to parse tool call arguments: %s", e)
return {}

33
litellm/litellm_core_utils/redact_messages.py
View file

@ -13,6 +13,7 @@ import inspect
from typing import TYPE_CHECKING, Any, Final
import litellm
from litellm.constants import REDACTED_BY_LITELLM
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import (
get_metadata_variable_name_from_kwargs,
@ -84,29 +85,29 @@ def _redact_tool_calls(tool_calls) -> None:
for tool_call in tool_calls:
function = getattr(tool_call, "function", None)
if function is not None and hasattr(function, "arguments"):
function.arguments = "redacted-by-litellm"
function.arguments = REDACTED_BY_LITELLM
def _redact_function_call(function_call) -> None:
"""Redact legacy assistant function_call arguments."""
if function_call is not None and hasattr(function_call, "arguments"):
function_call.arguments = "redacted-by-litellm"
function_call.arguments = REDACTED_BY_LITELLM
def _redact_choice_content(choice):
"""Helper to redact content in a choice (message or delta)."""
if isinstance(choice, litellm.Choices):
choice.message.content = "redacted-by-litellm"
choice.message.content = REDACTED_BY_LITELLM
if hasattr(choice.message, "reasoning_content"):
choice.message.reasoning_content = "redacted-by-litellm"
choice.message.reasoning_content = REDACTED_BY_LITELLM
if hasattr(choice.message, "thinking_blocks"):
choice.message.thinking_blocks = None
_redact_tool_calls(getattr(choice.message, "tool_calls", None))
_redact_function_call(getattr(choice.message, "function_call", None))
elif isinstance(choice, litellm.utils.StreamingChoices):
choice.delta.content = "redacted-by-litellm"
choice.delta.content = REDACTED_BY_LITELLM
if hasattr(choice.delta, "reasoning_content"):
choice.delta.reasoning_content = "redacted-by-litellm"
choice.delta.reasoning_content = REDACTED_BY_LITELLM
if hasattr(choice.delta, "thinking_blocks"):
choice.delta.thinking_blocks = None
_redact_tool_calls(getattr(choice.delta, "tool_calls", None))
@ -117,22 +118,22 @@ def _redact_responses_api_output(output_items):
"""Helper to redact ResponsesAPIResponse output items."""
for output_item in output_items:
if hasattr(output_item, "text"):
output_item.text = "redacted-by-litellm"
output_item.text = REDACTED_BY_LITELLM
if hasattr(output_item, "content") and isinstance(output_item.content, list):
for content_part in output_item.content:
if hasattr(content_part, "text"):
content_part.text = "redacted-by-litellm"
content_part.text = REDACTED_BY_LITELLM
# Redact reasoning items in output array
if hasattr(output_item, "type") and output_item.type == "reasoning":
if hasattr(output_item, "summary") and isinstance(output_item.summary, list):
for summary_item in output_item.summary:
if hasattr(summary_item, "text"):
summary_item.text = "redacted-by-litellm"
summary_item.text = REDACTED_BY_LITELLM
if hasattr(output_item, "type") and output_item.type == "function_call" and hasattr(output_item, "arguments"):
output_item.arguments = "redacted-by-litellm"
output_item.arguments = REDACTED_BY_LITELLM
def _redact_responses_api_output_dict(output_items, redacted_str: str):
@ -164,7 +165,7 @@ def _redact_standard_logging_object(model_call_details: dict):
if standard_logging_object is None:
return
redacted_str: Final = "redacted-by-litellm"
redacted_str: Final = REDACTED_BY_LITELLM
if standard_logging_object.get("messages") is not None:
standard_logging_object["messages"] = [{"role": "user", "content": redacted_str}]
@ -235,7 +236,7 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
copy via redact_streaming_responses_for_custom_logger instead.
"""
# Redact model_call_details
model_call_details["messages"] = [{"role": "user", "content": "redacted-by-litellm"}]
model_call_details["messages"] = [{"role": "user", "content": REDACTED_BY_LITELLM}]
model_call_details["prompt"] = ""
model_call_details["input"] = ""
_redact_standard_logging_object(model_call_details)
@ -256,7 +257,7 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
or hasattr(result, "__anext__") # async generator
): # async iterator
# For async objects, return a simple redacted response without deepcopy
return {"text": "redacted-by-litellm"}
return {"text": REDACTED_BY_LITELLM}
if not (
isinstance(result, (litellm.ModelResponse, litellm.ResponsesAPIResponse, litellm.EmbeddingResponse))
@ -273,11 +274,11 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
elif isinstance(_result, dict) and "choices" in _result:
# Handle dict representation of ModelResponse (e.g., from model_dump())
if _result.get("choices") is not None:
_redact_model_response_dict_choices(_result["choices"], "redacted-by-litellm")
_redact_model_response_dict_choices(_result["choices"], REDACTED_BY_LITELLM)
redact_vertex_ai_metadata_from_logged_object(_result)
elif isinstance(_result, dict) and "output" in _result:
if isinstance(_result.get("output"), list):
_redact_responses_api_output_dict(_result["output"], "redacted-by-litellm")
_redact_responses_api_output_dict(_result["output"], REDACTED_BY_LITELLM)
elif isinstance(_result, litellm.ResponsesAPIResponse):
if hasattr(_result, "output"):
_redact_responses_api_output(_result.output)
@ -288,7 +289,7 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
if hasattr(_result, "data") and _result.data is not None:
_result.data = []
else:
return {"text": "redacted-by-litellm"}
return {"text": REDACTED_BY_LITELLM}
return _result

33
litellm/llms/anthropic/chat/transformation.py
View file

@ -1,7 +1,8 @@
import json
import re
import time
from collections.abc import Mapping, Sequence
from collections.abc import Callable, Mapping, Sequence
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, NoReturn, cast
import httpx
@ -121,6 +122,32 @@ else:
# response side.
_ANTHROPIC_TOOL_NAME_INVALID_CHARS: Final = re.compile(r"[^a-zA-Z0-9_-]")
_ANTHROPIC_TOOL_NAME_MAX_LEN: Final = 128
_ENUM_TYPE_CHECKS: Final[Mapping[str, Callable[[Any], bool]]] = MappingProxyType(
{
"null": lambda v: v is None,
"boolean": lambda v: isinstance(v, bool),
"integer": lambda v: isinstance(v, int) and not isinstance(v, bool),
"number": lambda v: isinstance(v, (int, float)) and not isinstance(v, bool),
"string": lambda v: isinstance(v, str),
"array": lambda v: isinstance(v, list),
"object": lambda v: isinstance(v, dict),
}
)
def _enum_conflicts_with_declared_type(schema: Mapping[str, Any]) -> bool:
"""Whether ``schema``'s ``enum`` cannot match its declared ``type``."""
enum_values: Final = schema.get("enum")
declared_type: Final = schema.get("type")
if not isinstance(enum_values, list) or declared_type is None:
return False
if isinstance(declared_type, list):
return True
check: Final = _ENUM_TYPE_CHECKS.get(declared_type)
return check is not None and not all(check(value) for value in enum_values)
# Single, internal-only key on ``litellm_params`` used to thread the per-
# request reverse map (sanitized -> original) from request build to response
# parsing. ``litellm_params`` is never serialized to a provider; ``optional_
@ -565,9 +592,13 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
else:
result["description"] = constraint_note
drops_conflicting_type: Final = _enum_conflicts_with_declared_type(schema)
for key, value in schema.items():
if key in unsupported_fields:
continue
if key == "type" and drops_conflicting_type:
continue
if key == "description" and "description" in result:
# Already handled above
continue

10
litellm/llms/anthropic/common_utils.py
View file

@ -440,6 +440,16 @@ class AnthropicModelInfo(BaseLLMModelInfo):
"""
return AnthropicModelInfo._supports_model_capability(model, "thinking_always_on", custom_llm_provider)
@staticmethod
def _supports_legacy_thinking(model: str, custom_llm_provider: str) -> bool:
"""Whether ``model`` is an adaptive-thinking model that still accepts legacy
``thinking.type=enabled`` with ``budget_tokens`` (the Claude 4.6 family).
The model cost map is authoritative: an explicit ``supports_legacy_thinking``
entry resolved under ``custom_llm_provider``, or a ``fallback_generalizations``
rule for unmapped 4.6 ids. Absent flag means the model rejects the legacy shape.
"""
return AnthropicModelInfo._supports_model_capability(model, "supports_legacy_thinking", custom_llm_provider)
@staticmethod
def maybe_drop_disabled_thinking(
model: str,

4
litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py
View file

@ -64,6 +64,7 @@ from openai.types.chat.chat_completion_chunk import Choice as OpenAIStreamingCho
from litellm.litellm_core_utils.prompt_templates.common_utils import (
parse_tool_call_arguments,
reasoning_content_from_thinking_blocks,
with_prompt_cache_breakpoint,
)
from litellm.litellm_core_utils.prompt_templates.factory import (
@ -592,6 +593,9 @@ class LiteLLMAnthropicMessagesAdapter:
assistant_message["tool_calls"] = tool_calls
if len(thinking_blocks) > 0:
assistant_message["thinking_blocks"] = thinking_blocks
reasoning_content = reasoning_content_from_thinking_blocks(thinking_blocks)
if reasoning_content:
assistant_message["reasoning_content"] = reasoning_content
new_messages.append(assistant_message)
return new_messages

10
litellm/llms/anthropic/experimental_pass_through/messages/transformation.py
View file

@ -379,13 +379,19 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
def _translate_legacy_thinking_for_adaptive_model(
model: str, optional_params: dict, custom_llm_provider: str
) -> None:
"""Translate legacy ``thinking.type=enabled`` to adaptive for 4.6/4.7.
Caller-provided ``output_config.effort`` is never overridden.
"""Translate legacy ``thinking.type=enabled`` to adaptive for the
adaptive-thinking models that reject it (4.7+ and the 5 families).
Models flagged ``supports_legacy_thinking`` (the 4.6 family) accept the
legacy shape natively, so it is forwarded verbatim and the caller's
``budget_tokens`` cap keeps applying. Caller-provided
``output_config.effort`` is never overridden.
"""
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
if not AnthropicModelInfo._is_adaptive_thinking_model(model, custom_llm_provider):
return
if AnthropicModelInfo._supports_legacy_thinking(model, custom_llm_provider):
return
thinking: Final = optional_params.get("thinking")
if not isinstance(thinking, dict) or thinking.get("type") != "enabled":
return

5
litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py
View file

@ -152,7 +152,10 @@ class AnthropicResponsesStreamWrapper:
if block_idx < 0:
if not delta:
return
block_idx = self._open_block(item_id, {"type": "thinking", "thinking": ""})
block_idx = self._open_block(
item_id,
{"type": "thinking", "thinking": "", "signature": ""}, # mutable-ok: API message payload
)
self._chunk_queue.append(
{
"type": "content_block_delta",

112
litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py
View file

@ -6,12 +6,14 @@ path used for OpenAI and Azure models.
"""
import json
from collections.abc import Iterable
from collections.abc import Iterable, Mapping
from itertools import groupby
from typing import Any, Final, cast
from litellm.litellm_core_utils.prompt_templates.common_utils import (
TOOL_RESULT_IMAGE_BOUNDARY,
TOOL_RESULT_IMAGE_PLACEHOLDER,
responses_reasoning_item_from_thinking_blocks,
with_prompt_cache_breakpoint,
)
from litellm.litellm_core_utils.reasoning_effort_utils import (
@ -36,7 +38,11 @@ from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,
AnthropicUsage,
)
from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse
from litellm.types.llms.openai import (
ChatCompletionThinkingBlock,
ResponseAPIUsage,
ResponsesAPIResponse,
)
class LiteLLMAnthropicToResponsesAPIAdapter:
@ -100,6 +106,58 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
if isinstance(block, dict) and block.get("type") == "text" and (text := block.get("text")) # pyright: ignore[reportUnnecessaryIsInstance] # untrusted client payload
]
@staticmethod
def _summary_part_text(part: object) -> str:
if isinstance(part, Mapping):
mapping: Final = cast(Mapping[str, Any], part) # cast-ok: summary parts are untyped provider json
return str(mapping.get("text") or "")
return str(getattr(part, "text", None) or "")
@classmethod
def _thinking_blocks_from_reasoning_item(
cls,
summary: Iterable[object],
) -> tuple[dict[str, Any], ...]: # mutable-ok: API message payload
"""Anthropic thinking blocks for one Responses reasoning item.
The signature stays empty: only Anthropic can sign a thinking block, and a stand-in
value would be replayed as a real one and rejected by every backend that verifies it.
"""
return tuple(
AnthropicResponseContentBlockThinking(
type="thinking",
thinking=text,
signature=None,
).model_dump()
for part in summary
if (text := cls._summary_part_text(part))
)
@staticmethod
def _assistant_block_group_key(indexed_block: tuple[int, Mapping[str, Any]]) -> str:
"""Group a run of consecutive thinking blocks together; keep every other block alone."""
index, block = indexed_block
return "thinking" if block.get("type") == "thinking" else f"block:{index}"
@classmethod
def _assistant_group_to_input_item(
cls, group: tuple[Mapping[str, Any], ...]
) -> dict[str, Any] | None: # mutable-ok: API message payload
first: Final = group[0]
btype: Final = first.get("type")
if btype == "thinking":
blocks: Final = cast(tuple[ChatCompletionThinkingBlock, ...], group) # cast-ok: untrusted client payload
reasoning_item: Final = responses_reasoning_item_from_thinking_blocks(blocks)
return None if reasoning_item is None else dict(reasoning_item) # mutable-ok: API message payload
if btype == "tool_use":
return { # mutable-ok: API message payload
"type": "function_call",
"call_id": first.get("id", ""),
"name": first.get("name", ""),
"arguments": json.dumps(first.get("input", {})), # mutable-ok: API message payload
}
return None
def translate_messages_to_responses_input(
self,
messages: list[AllAnthropicPassThroughMessageValues],
@ -113,6 +171,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
user image -> message(role=user, input_image)
user tool_result -> function_call_output
assistant text -> message(role=assistant, output_text)
assistant thinking -> reasoning
assistant tool_use -> function_call
"""
input_items: Final[list[dict[str, Any]]] = []
@ -233,27 +292,17 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
}
)
elif isinstance(content, list):
asst_parts: list[dict[str, Any]] = []
for block in content:
if not isinstance(block, dict):
continue
btype = block.get("type")
if btype == "text":
asst_parts.append({"type": "output_text", "text": block.get("text", "")})
elif btype == "tool_use":
# tool_use becomes a top-level function_call item
input_items.append(
{
"type": "function_call",
"call_id": block.get("id", ""),
"name": block.get("name", ""),
"arguments": json.dumps(block.get("input", {})),
}
)
elif btype == "thinking":
thinking_text = block.get("thinking", "")
if thinking_text:
asst_parts.append({"type": "output_text", "text": thinking_text})
blocks = tuple(block for block in content if isinstance(block, dict))
input_items.extend(
item
for _, group in groupby(enumerate(blocks), key=self._assistant_block_group_key)
if (item := self._assistant_group_to_input_item(tuple(block for _, block in group))) is not None
)
asst_parts: list[dict[str, Any]] = [ # mutable-ok: API message payload
{"type": "output_text", "text": block.get("text", "")} # mutable-ok: API message payload
for block in blocks
if block.get("type") == "text"
]
if asst_parts:
input_items.append(
{
@ -514,16 +563,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
for item in response.output:
if isinstance(item, ResponseReasoningItem):
for summary in item.summary:
text = getattr(summary, "text", "")
if text:
content.append(
AnthropicResponseContentBlockThinking(
type="thinking",
thinking=text,
signature=None,
).model_dump()
)
content.extend(self._thinking_blocks_from_reasoning_item(item.summary))
elif isinstance(item, ResponseOutputMessage):
for part in item.content:
@ -555,6 +595,12 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
content.append(
AnthropicResponseContentBlockText(type="text", text=part.get("text", "")).model_dump()
)
elif item_type == "reasoning":
content.extend(
self._thinking_blocks_from_reasoning_item(
cast(Iterable[object], item.get("summary") or ()), # cast-ok: untyped provider json
)
)
elif item_type == "function_call":
try:
input_data = json.loads(item.get("arguments", "{}"))

29
litellm/llms/azure/common_utils.py
View file

@ -3,6 +3,7 @@ import hashlib
import json
import os
from collections.abc import Callable, Mapping
from functools import lru_cache
from typing import Any, Final, Literal, NamedTuple, cast
import httpx
@ -75,6 +76,24 @@ def process_azure_headers(headers: httpx.Headers | dict) -> dict:
return {**llm_response_headers, **openai_headers}
@lru_cache(maxsize=128)
def _cached_entra_id_token_provider(
tenant_id: str,
client_id: str,
client_secret: str,
scope: str,
) -> Callable[[], str]:
"""Build (once per credential set) a bearer token provider backed by a `ClientSecretCredential`.
The credential caches the access token internally and only talks to Entra ID when it is close
to expiry, so reusing the provider keeps one AAD round trip per token lifetime instead of one
per request.
"""
from azure.identity import ClientSecretCredential, get_bearer_token_provider
return get_bearer_token_provider(ClientSecretCredential(tenant_id, client_id, client_secret), scope)
def get_azure_ad_token_from_entra_id(
tenant_id: str,
client_id: str,
@ -93,8 +112,6 @@ def get_azure_ad_token_from_entra_id(
Returns:
callable that returns a bearer token.
"""
from azure.identity import ClientSecretCredential, get_bearer_token_provider
verbose_logger.debug("Getting Azure AD Token from Entra ID")
if tenant_id.startswith("os.environ/"):
@ -120,9 +137,13 @@ def get_azure_ad_token_from_entra_id(
)
if _tenant_id is None or _client_id is None or _client_secret is None:
raise ValueError("tenant_id, client_id, and client_secret must be provided")
credential: Final = ClientSecretCredential(_tenant_id, _client_id, _client_secret)
token_provider: Final = get_bearer_token_provider(credential, scope)
token_provider: Final = _cached_entra_id_token_provider(
tenant_id=_tenant_id,
client_id=_client_id,
client_secret=_client_secret,
scope=scope,
)
verbose_logger.debug("token_provider %s", token_provider)

3
litellm/llms/azure/search/__init__.py Normal file
View file

@ -0,0 +1,3 @@
from litellm.llms.azure.search.transformation import BingGroundingSearchConfig
__all__ = ("BingGroundingSearchConfig",)

442
litellm/llms/azure/search/transformation.py Normal file
View file

@ -0,0 +1,442 @@
"""
Calls the Microsoft Foundry Responses API with the `bing_grounding` or `web_search`
tool to search the web (Grounding with Bing Search).
Microsoft docs: https://learn.microsoft.com/en-us/azure/ai-foundry/agents/how-to/tools/bing-grounding
Setup:
1. Set BING_GROUNDING_PROJECT_ENDPOINT to the Foundry project endpoint, e.g.
https://<account>.services.ai.azure.com/api/projects/<project>
2. Set BING_GROUNDING_MODEL to a model deployment in that project (e.g. gpt-4.1);
it runs the grounded search and its tokens are billed on that deployment
3. Optional: set BING_GROUNDING_CONNECTION_ID to a Grounding with Bing Search
project connection id to use the `bing_grounding` tool; without it the
project's built-in `web_search` tool is used
4. Auth: pass api_key (an Azure API key, sent in the api-key header), or set
BING_GROUNDING_TOKEN to an Entra bearer token for scope
https://ai.azure.com/.default, or configure azure-identity (AZURE_CLIENT_ID /
AZURE_CLIENT_SECRET / AZURE_TENANT_ID, managed identity, or any
DefaultAzureCredential source) and the token is minted automatically
Usage:
response = litellm.search(
query="latest AI developments",
search_provider="bing_grounding",
max_results=5,
)
"""
from __future__ import annotations
from collections.abc import Callable, Mapping
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal
import httpx
from pydantic import BaseModel, ConfigDict, ValidationError
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.search.transformation import (
BaseSearchConfig,
SearchResponse,
SearchResult,
)
from litellm.secret_managers.main import get_secret_str
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
_DOCS_URL: Final = "https://learn.microsoft.com/en-us/azure/ai-foundry/agents/how-to/tools/bing-grounding"
PROJECT_ENDPOINT_ENV: Final = "BING_GROUNDING_PROJECT_ENDPOINT"
MODEL_ENV: Final = "BING_GROUNDING_MODEL"
CONNECTION_ID_ENV: Final = "BING_GROUNDING_CONNECTION_ID"
TOKEN_ENV: Final = "BING_GROUNDING_TOKEN"
ENTRA_SCOPE: Final = "https://ai.azure.com/.default"
_RESPONSES_PATH: Final = "/openai/v1/responses"
_SNIPPET_FALLBACK_LENGTH: Final = 300
_UPSTREAM_ERROR_STATUS: Final = 502
_RESPONSE_COST_HEADER: Final = "llm_provider-x-litellm-response-cost"
class _Annotation(BaseModel):
model_config = ConfigDict(extra="ignore", frozen=True)
type: str = ""
url: str | None = None
title: str | None = None
start_index: int | None = None
end_index: int | None = None
class _ContentPart(BaseModel):
model_config = ConfigDict(extra="ignore", frozen=True)
type: str = ""
text: str = ""
annotations: tuple[_Annotation, ...] = ()
class _OutputItem(BaseModel):
model_config = ConfigDict(extra="ignore", frozen=True)
type: str = ""
content: tuple[_ContentPart, ...] = ()
class _ErrorBody(BaseModel):
model_config = ConfigDict(extra="ignore", frozen=True)
message: str | None = None
class _IncompleteDetails(BaseModel):
model_config = ConfigDict(extra="ignore", frozen=True)
reason: str | None = None
class _ResponsesEnvelope(BaseModel):
"""A Foundry Responses API body. `output` is required: a body without it is not a
Responses API response and must not be reported as a successful empty search.
A 200 body can still carry `status` `failed` or `incomplete`; those are surfaced as
errors rather than reported as a successful empty search."""
model_config = ConfigDict(extra="ignore", frozen=True)
output: tuple[_OutputItem, ...]
status: str | None = None
error: _ErrorBody | None = None
incomplete_details: _IncompleteDetails | None = None
class _ErrorEnvelope(BaseModel):
model_config = ConfigDict(extra="ignore", frozen=True)
error: _ErrorBody | None = None
def _unwrap_error_detail(error_message: str) -> str:
"""
Surface the human-readable message inside Foundry's error envelope.
Tool failures nest a second JSON document as a string inside `error.message`
(observed live for `bing_grounding` connection errors), so the unwrap runs twice.
Falls back to the raw body for anything else.
"""
try:
envelope: Final = _ErrorEnvelope.model_validate_json(error_message)
except ValidationError:
return error_message
message: Final = envelope.error.message if envelope.error else None
if message is None:
return error_message
try:
nested: Final = _ErrorBody.model_validate_json(message)
except ValidationError:
return message
return nested.message or message
def _snippet(text: str, annotation: _Annotation) -> str:
"""
The text a citation supports, not the citation marker itself.
A url_citation's start/end indices span the inline marker ("([host](url))"),
which follows the claim it backs, so the snippet is the marker's own line up
to where the marker starts.
"""
start: Final = annotation.start_index
marker_start: Final = start if start is not None and 0 <= start <= len(text) else len(text)
claim: Final = text[:marker_start].rsplit("\n", 1)[-1].strip()
if claim:
return claim[-_SNIPPET_FALLBACK_LENGTH:]
return text[:_SNIPPET_FALLBACK_LENGTH]
def _citation_results(envelope: _ResponsesEnvelope) -> tuple[SearchResult, ...]:
"""One result per cited URL: first occurrence wins, order preserved as answered."""
cited: Final = tuple(
SearchResult(
title=annotation.title or "",
url=annotation.url or "",
snippet=_snippet(part.text, annotation),
date=None,
last_updated=None,
)
for item in envelope.output
if item.type == "message"
for part in item.content
if part.type == "output_text"
for annotation in part.annotations
if annotation.type == "url_citation" and annotation.url
)
first_by_url: Final = MappingProxyType({result.url: result for result in reversed(cited)})
return tuple(first_by_url[url] for url in dict.fromkeys(result.url for result in cited))
def _valid_max_results(max_results: object) -> int | None:
"""A positive-int `max_results`, else None. Rejects bools, an `int` subclass, and
non-positive values so neither the request-side `count` nor the response-side cap
forwards a value the other would silently ignore.
"""
if isinstance(max_results, bool) or not isinstance(max_results, int):
return None
return max_results if max_results > 0 else None
def _requested_max_results(response_kwargs: Mapping[str, object]) -> int | None:
"""The unified `max_results` cap the caller asked for, if any.
The built-in web_search tool has no server-side result-count knob, so the cap is
enforced here after the fact; connection mode also honors it as a hard ceiling on
top of the tool's `count` hint.
"""
optional_params: Final = response_kwargs.get("optional_params")
if not isinstance(optional_params, Mapping):
return None
return _valid_max_results(optional_params.get("max_results"))
def _capped(results: tuple[SearchResult, ...], max_results: int | None) -> tuple[SearchResult, ...]:
return results[:max_results] if max_results is not None else results
class _SearchConfiguration(BaseModel):
model_config = ConfigDict(frozen=True)
project_connection_id: str
count: int | None = None
class _BingGroundingParams(BaseModel):
model_config = ConfigDict(frozen=True)
search_configurations: tuple[_SearchConfiguration, ...]
class _BingGroundingTool(BaseModel):
model_config = ConfigDict(frozen=True)
type: Literal["bing_grounding"] = "bing_grounding"
bing_grounding: _BingGroundingParams
class _UserLocation(BaseModel):
model_config = ConfigDict(frozen=True)
type: Literal["approximate"] = "approximate"
country: str
class _WebSearchTool(BaseModel):
model_config = ConfigDict(frozen=True)
type: Literal["web_search"] = "web_search"
user_location: _UserLocation | None = None
class _ResponsesRequest(BaseModel):
model_config = ConfigDict(frozen=True)
model: str
input: str
tools: tuple[_BingGroundingTool | _WebSearchTool, ...]
def _search_tool(optional_params: Mapping[str, object]) -> _BingGroundingTool | _WebSearchTool:
connection_id: Final = get_secret_str(CONNECTION_ID_ENV)
max_results: Final = optional_params.get("max_results")
country: Final = optional_params.get("country")
if connection_id:
configuration: Final = _SearchConfiguration(
project_connection_id=connection_id,
count=_valid_max_results(max_results),
)
return _BingGroundingTool(bing_grounding=_BingGroundingParams(search_configurations=(configuration,)))
location: Final = _UserLocation(country=country.upper()) if isinstance(country, str) else None
return _WebSearchTool(user_location=location)
def _default_entra_token_minter() -> str:
from litellm.secret_managers.get_azure_ad_token_provider import get_azure_ad_token_provider
return get_azure_ad_token_provider(azure_scope=ENTRA_SCOPE)()
class BingGroundingSearchConfig(BaseSearchConfig):
def __init__(self, entra_token_minter: Callable[[], str] | None = None) -> None:
super().__init__()
self._entra_token_minter = entra_token_minter
@staticmethod
def ui_friendly_name() -> str:
return "Grounding with Bing Search"
def validate_environment(
self,
headers: dict[str, str], # mutable-ok: BaseSearchConfig.validate_environment signature
api_key: str | None = None,
api_base: str | None = None,
**kwargs: object, # kwargs-ok: BaseSearchConfig.validate_environment signature
) -> dict[str, str]: # mutable-ok: the http handler passes this straight to httpx as headers
"""
Validate environment and return headers.
Returns a new dict rather than mutating ``headers``: the http handler calls this
a second time after ``litellm/search/main.py`` already did, so it has to be idempotent.
"""
return { # mutable-ok: httpx requires a plain dict of headers
**headers,
**self._auth_header(api_key, api_base),
"Content-Type": "application/json",
}
def _auth_header(self, api_key: str | None, api_base: str | None) -> Mapping[str, str]:
"""
A caller-supplied ``api_key`` is an Azure API key and rides the ``api-key`` header;
an Entra bearer token (``BING_GROUNDING_TOKEN`` or one minted via azure-identity)
rides ``Authorization: Bearer``. Foundry rejects the wrong scheme for each.
"""
if api_key:
return MappingProxyType({"api-key": api_key})
token: Final = self.resolve_server_api_key(
caller_api_key=None,
caller_api_base=api_base,
key_env_vars=(TOKEN_ENV,),
base_env_var=PROJECT_ENDPOINT_ENV,
default_api_base=None,
) or self._mint_entra_token(api_base)
return MappingProxyType({"Authorization": f"Bearer {token}"})
def _mint_entra_token(self, caller_api_base: str | None) -> str:
self._assert_trusted_api_base_for_server_credential(
caller_api_base, None, PROJECT_ENDPOINT_ENV, "Azure AD token"
)
minter: Final = self._entra_token_minter or _default_entra_token_minter
try:
return minter()
except Exception as e:
raise ValueError(
f"Grounding with Bing Search: no credential available. Pass api_key, set {TOKEN_ENV} "
f"to an Entra bearer token, or configure azure-identity (AZURE_CLIENT_ID / "
f"AZURE_CLIENT_SECRET / AZURE_TENANT_ID or any DefaultAzureCredential source) "
f"for scope {ENTRA_SCOPE}. Underlying error: {e}"
) from e
def get_complete_url(
self,
api_base: str | None,
optional_params: dict[str, object], # mutable-ok: BaseSearchConfig.get_complete_url signature
data: dict[str, object] | list[dict[str, object]] | None = None, # mutable-ok: base signature
**kwargs: object, # kwargs-ok: BaseSearchConfig.get_complete_url signature
) -> str:
resolved_base: Final = api_base or get_secret_str(PROJECT_ENDPOINT_ENV)
if not resolved_base:
raise ValueError(
f"{PROJECT_ENDPOINT_ENV} is not set. Set it to your Microsoft Foundry project "
f"endpoint, e.g. https://<account>.services.ai.azure.com/api/projects/<project>."
)
trimmed: Final = resolved_base.rstrip("/")
if trimmed.endswith(_RESPONSES_PATH):
return trimmed
return f"{trimmed}{_RESPONSES_PATH}"
def transform_search_request(
self,
query: str | list[str], # mutable-ok: BaseSearchConfig.transform_search_request signature
optional_params: dict[str, object], # mutable-ok: base signature
**kwargs: object, # kwargs-ok: BaseSearchConfig.transform_search_request signature
) -> dict[str, object]: # mutable-ok: the http handler passes this straight to httpx as the JSON body
"""
Transform Search request to the Foundry Responses API format.
The unified params map as far as the API allows:
- max_results -> the bing_grounding search configuration's `count`; the built-in
web_search tool has no result-count knob, so that mode instead caps the returned
results after the fact (see transform_search_response)
- country -> web_search's approximate `user_location` (bing_grounding's `market`
wants a full locale like en-US, which a bare country code cannot fill)
- search_domain_filter, max_tokens_per_page -> no API equivalent, dropped
"""
model: Final = get_secret_str(MODEL_ENV)
if not model:
raise ValueError(
f"{MODEL_ENV} is not set. Set it to a model deployment in the Foundry project "
f"that runs the grounded search, e.g. gpt-4.1."
)
request: Final = _ResponsesRequest(
model=model,
input=" ".join(query) if isinstance(query, list) else query,
tools=(_search_tool(optional_params),),
)
return request.model_dump(mode="json", exclude_none=True)
def transform_search_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
**kwargs: object, # kwargs-ok: BaseSearchConfig.transform_search_response signature
) -> SearchResponse:
try:
parsed: Final = _ResponsesEnvelope.model_validate_json(raw_response.content)
except ValidationError as e:
raise self.get_error_class(
error_message=f"response does not match the Foundry Responses API schema: {e}",
status_code=raw_response.status_code,
headers=dict(raw_response.headers), # mutable-ok: BaseSearchConfig.get_error_class signature
)
if parsed.status == "failed":
detail: Final = (
parsed.error.message if parsed.error and parsed.error.message else "the grounded search failed"
)
raise self._upstream_error(detail, raw_response)
results: Final = _capped(_citation_results(parsed), _requested_max_results(kwargs))
if not results and parsed.status == "incomplete":
reason: Final = (
parsed.incomplete_details.reason
if parsed.incomplete_details and parsed.incomplete_details.reason
else "unknown reason"
)
raise self._upstream_error(f"the grounded search was incomplete: {reason}", raw_response)
return self._priced(results)
def _upstream_error(self, detail: str, raw_response: httpx.Response) -> Exception:
return self.get_error_class(
error_message=detail,
status_code=_UPSTREAM_ERROR_STATUS,
headers=dict(raw_response.headers), # mutable-ok: BaseSearchConfig.get_error_class signature
)
def _priced(self, results: tuple[SearchResult, ...]) -> SearchResponse:
"""web_search mode runs no paid Grounding with Bing transaction, so it must not
inherit the connection-mode ``bing_grounding/search`` price; zero its per-query
cost while leaving connection mode to the cost map."""
response: Final = SearchResponse(
results=list(results), # mutable-ok: SearchResponse.results is list[SearchResult]
object="search",
)
if get_secret_str(CONNECTION_ID_ENV):
return response
response._hidden_params[
"additional_headers"
] = { # mutable-ok: response_cost_calculator writes into _hidden_params
_RESPONSE_COST_HEADER: 0.0
}
return response
def get_error_class(
self,
error_message: str,
status_code: int,
headers: dict[str, str], # mutable-ok: BaseSearchConfig.get_error_class signature
) -> Exception:
detail: Final = _unwrap_error_detail(error_message).rstrip(". ")
return BaseLLMException(
status_code=status_code,
message=f"Grounding with Bing Search: {detail}. See {_DOCS_URL} for details.",
headers=headers,
)

10
litellm/llms/azure_ai/chat/transformation.py
View file

@ -30,7 +30,12 @@ class AzureFoundryErrorStrings(str, enum.Enum):
SET_EXTRA_PARAMETERS_TO_PASS_THROUGH = "Set extra-parameters to 'pass-through'"
NON_OPENAI_SPEC_MESSAGE_FIELDS: Final = ("thinking_blocks", "provider_specific_fields", "cache_control")
NON_OPENAI_SPEC_MESSAGE_FIELDS: Final = (
"thinking_blocks",
"reasoning_content",
"provider_specific_fields",
"cache_control",
)
class AzureAIStudioConfig(OpenAIConfig):
@ -173,7 +178,8 @@ class AzureAIStudioConfig(OpenAIConfig):
"""
- Azure AI Studio doesn't support content as a list. This handles:
1. Strips message fields that are not part of the OpenAI chat-completions
schema (thinking_blocks, provider_specific_fields, cache_control).
schema (thinking_blocks, reasoning_content, provider_specific_fields,
cache_control).
Azure AI Foundry backends set additionalProperties=false and reject
these with "Extra inputs are not permitted", which breaks multi-turn
Anthropic-format clients that echo thinking blocks back as history.

47
litellm/llms/azure_ai/common_utils.py
View file

@ -1,9 +1,54 @@
from collections.abc import Mapping
from typing import Final, Literal
import litellm
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues
from litellm.types.router import GenericLiteLLMParams
AzureAIApiKeyHeader = Literal["Authorization", "api-key", "Api-Key", "Ocp-Apim-Subscription-Key"]
def get_azure_ai_entra_token(litellm_params: Mapping[str, object] | None = None) -> str | None:
"""
Resolve an Entra ID / OAuth access token for an Azure AI Foundry deployment.
Accepts the same credential set as the `azure` provider: service principal
(`tenant_id` / `client_id` / `client_secret`), a pre-fetched `azure_ad_token`, an OIDC
federated token, username/password, or `DefaultAzureCredential` / managed identity.
"""
from litellm.llms.azure.common_utils import get_azure_ad_token
params = GenericLiteLLMParams.model_validate(litellm_params) if litellm_params else GenericLiteLLMParams()
return get_azure_ad_token(params)
def get_azure_ai_auth_headers(
api_key: str | None,
litellm_params: Mapping[str, object] | None = None,
api_key_header: AzureAIApiKeyHeader = "Authorization",
api_key_env_var: str = "AZURE_AI_API_KEY",
) -> Mapping[str, str]:
"""
Build the auth headers for an Azure AI Foundry route.
Prefers the API key when one is configured, and otherwise falls back to Entra ID / OAuth,
sending the access token as a bearer token.
"""
if api_key:
return {api_key_header: f"Bearer {api_key}" if api_key_header == "Authorization" else api_key}
azure_ad_token = get_azure_ai_entra_token(litellm_params=litellm_params)
if azure_ad_token:
return {"Authorization": f"Bearer {azure_ad_token}"}
raise ValueError(
f"Missing Azure AI credentials - set an API key (`api_key` or {api_key_env_var}), or Entra ID / OAuth "
"credentials (`tenant_id` + `client_id` + `client_secret`, `azure_ad_token`, an OIDC token, or a managed "
"identity with `litellm.enable_azure_ad_token_refresh = True`)"
)
class AzureFoundryModelInfo(BaseLLMModelInfo):
@ -43,7 +88,7 @@ class AzureFoundryModelInfo(BaseLLMModelInfo):
@staticmethod
def get_api_key(api_key: str | None = None) -> str | None:
return api_key or litellm.api_key or litellm.openai_key or get_secret_str("AZURE_AI_API_KEY")
return api_key or litellm.api_key or get_secret_str("AZURE_AI_API_KEY")
@property
def api_version(self, api_version: str | None = None) -> str | None:

18
litellm/llms/azure_ai/image_edit/flux2_transformation.py
View file

@ -5,7 +5,10 @@ from typing import Any, Final
from httpx._types import RequestFiles
import litellm
from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
from litellm.llms.azure_ai.common_utils import (
AzureFoundryModelInfo,
get_azure_ai_auth_headers,
)
from litellm.llms.azure_ai.image_generation.flux_transformation import (
AzureFoundryFluxImageGenerationConfig,
)
@ -71,16 +74,13 @@ class AzureFoundryFlux2ImageEditConfig(OpenAIImageEditConfig):
"""
Validate Azure AI Foundry environment and set up authentication
"""
api_key = AzureFoundryModelInfo.get_api_key(api_key)
if not api_key:
raise ValueError(
f"Azure AI API key is required for model {model}. Set AZURE_AI_API_KEY environment variable or pass api_key parameter."
)
headers.update(
{
"Api-Key": api_key,
**get_azure_ai_auth_headers(
api_key=AzureFoundryModelInfo.get_api_key(api_key),
litellm_params=litellm_params,
api_key_header="Api-Key",
),
"Content-Type": "application/json",
}
)

19
litellm/llms/azure_ai/image_edit/mai_transformation.py
View file

@ -3,7 +3,10 @@ from typing import TYPE_CHECKING, Any, Final, cast
import httpx
from httpx._types import RequestFiles
from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
from litellm.llms.azure_ai.common_utils import (
AzureFoundryModelInfo,
get_azure_ai_auth_headers,
)
from litellm.llms.azure_ai.image_generation.mai_transformation import (
AzureFoundryMAIImageGenerationConfig,
)
@ -91,15 +94,13 @@ class AzureFoundryMAIImageEditConfig(OpenAIImageEditConfig):
litellm_params: dict | None = None,
api_base: str | None = None,
) -> dict:
api_key = AzureFoundryModelInfo.get_api_key(api_key)
if not api_key:
raise ValueError(
f"Azure AI API key is required for model {model}. "
"Set AZURE_AI_API_KEY environment variable or pass api_key parameter."
headers.update(
get_azure_ai_auth_headers(
api_key=AzureFoundryModelInfo.get_api_key(api_key),
litellm_params=litellm_params,
api_key_header="api-key",
)
headers.update({"api-key": api_key})
)
return headers
def get_complete_url(

22
litellm/llms/azure_ai/image_edit/transformation.py
View file

@ -3,7 +3,10 @@ from typing import Final
import httpx
import litellm
from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
from litellm.llms.azure_ai.common_utils import (
AzureFoundryModelInfo,
get_azure_ai_auth_headers,
)
from litellm.llms.openai.image_edit.transformation import OpenAIImageEditConfig
from litellm.secret_managers.main import get_secret_str
from litellm.utils import _add_path_to_api_base
@ -30,19 +33,14 @@ class AzureFoundryFluxImageEditConfig(OpenAIImageEditConfig):
) -> dict:
"""
Validate Azure AI Foundry environment and set up authentication
Uses Api-Key header format
Uses the Api-Key header format, or an Entra ID / OAuth bearer token when no key is set
"""
api_key = AzureFoundryModelInfo.get_api_key(api_key)
if not api_key:
raise ValueError(
f"Azure AI API key is required for model {model}. Set AZURE_AI_API_KEY environment variable or pass api_key parameter."
)
headers.update(
{
"Api-Key": api_key, # Azure AI Foundry uses Api-Key header format
}
get_azure_ai_auth_headers(
api_key=AzureFoundryModelInfo.get_api_key(api_key),
litellm_params=litellm_params,
api_key_header="Api-Key",
)
)
return headers

16
litellm/llms/azure_ai/ocr/document_intelligence/transformation.py
View file

@ -26,6 +26,7 @@ from litellm.constants import (
)
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.url_utils import SSRFError, assert_same_origin, encode_url_path_segment
from litellm.llms.azure_ai.common_utils import get_azure_ai_auth_headers
from litellm.llms.base_llm.ocr.transformation import (
OCR_REQUEST_FORMAT_PARAM,
BaseOCRConfig,
@ -236,17 +237,13 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
"""
Validate environment and return headers for Azure Document Intelligence.
Authentication uses Ocp-Apim-Subscription-Key header.
Authentication uses the Ocp-Apim-Subscription-Key header, or an Entra ID / OAuth bearer
token when no subscription key is set.
"""
# Get API key from environment if not provided
if api_key is None:
api_key = get_secret_str(AZURE_DOCUMENT_INTELLIGENCE_API_KEY_ENV_VAR)
if api_key is None:
raise ValueError(
"Missing Azure Document Intelligence API Key - Set AZURE_DOCUMENT_INTELLIGENCE_API_KEY environment variable or pass api_key parameter"
)
# Validate API base/endpoint is provided
if api_base is None:
api_base = get_secret_str("AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT")
@ -257,7 +254,12 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
)
headers = {
"Ocp-Apim-Subscription-Key": api_key,
**get_azure_ai_auth_headers(
api_key=api_key,
litellm_params=litellm_params,
api_key_header="Ocp-Apim-Subscription-Key",
api_key_env_var=AZURE_DOCUMENT_INTELLIGENCE_API_KEY_ENV_VAR,
),
"Content-Type": "application/json",
**headers,
}

10
litellm/llms/azure_ai/ocr/transformation.py
View file

@ -9,6 +9,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import (
async_convert_url_to_base64,
convert_url_to_base64,
)
from litellm.llms.azure_ai.common_utils import get_azure_ai_auth_headers
from litellm.llms.base_llm.ocr.transformation import DocumentType, OCRRequestData
from litellm.llms.mistral.ocr.transformation import MistralOCRConfig
from litellm.secret_managers.main import get_secret_str
@ -47,17 +48,12 @@ class AzureAIOCRConfig(MistralOCRConfig):
"""
Validate environment and return headers for Azure AI OCR.
Azure AI uses Bearer token authentication with AZURE_AI_API_KEY.
Authenticates with AZURE_AI_API_KEY, or with an Entra ID / OAuth token when no key is set.
"""
# Get API key from environment if not provided
if api_key is None:
api_key = get_secret_str(AZURE_AI_OCR_API_KEY_ENV_VAR)
if api_key is None:
raise ValueError(
"Missing Azure AI API Key - A call is being made to Azure AI but no key is set either in the environment variables or via params"
)
# Validate API base is provided
if api_base is None:
api_base = get_secret_str("AZURE_AI_API_BASE")
@ -68,7 +64,7 @@ class AzureAIOCRConfig(MistralOCRConfig):
)
headers = {
"Authorization": f"Bearer {api_key}",
**get_azure_ai_auth_headers(api_key=api_key, litellm_params=litellm_params),
"Content-Type": "application/json",
**headers,
}

8
litellm/llms/azure_ai/rerank/transformation.py
View file

@ -2,12 +2,14 @@
Translate between Cohere's `/rerank` format and Azure AI's `/rerank` format.
"""
from collections.abc import Mapping
from typing import Final
import httpx
import litellm
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.azure_ai.common_utils import get_azure_ai_auth_headers
from litellm.llms.cohere.rerank.transformation import CohereRerankConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.utils import RerankResponse
@ -64,15 +66,13 @@ class AzureAIRerankConfig(CohereRerankConfig):
model: str,
api_key: str | None = None,
optional_params: dict | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> dict:
if api_key is None:
api_key = get_secret_str("AZURE_AI_API_KEY") or litellm.azure_key
if api_key is None:
raise ValueError("Azure AI API key is required. Please set 'AZURE_AI_API_KEY' or 'litellm.azure_key'")
default_headers: Final = {
"Authorization": f"Bearer {api_key}",
**get_azure_ai_auth_headers(api_key=api_key, litellm_params=litellm_params),
"accept": "application/json",
"content-type": "application/json",
}

2
litellm/llms/base_llm/rerank/transformation.py
View file

@ -1,4 +1,5 @@
from abc import ABC, abstractmethod
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final
import httpx
@ -24,6 +25,7 @@ class BaseRerankConfig(ABC):
model: str,
api_key: str | None = None,
optional_params: dict | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> dict:
pass

21
litellm/llms/base_llm/videos/transformation.py
View file

@ -1,9 +1,10 @@
import types
from abc import ABC, abstractmethod
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any
import httpx
from httpx._types import RequestFiles
from httpx._types import FileContent, RequestFiles
from litellm.types.responses.main import *
from litellm.types.router import GenericLiteLLMParams
@ -91,6 +92,14 @@ class BaseVideoConfig(ABC):
raise ValueError("api_base is required")
return api_base
def use_multipart_form_data(self) -> bool:
"""
Whether video create requests without files must still be sent as
multipart/form-data (the encoding the OpenAI SDK always uses for
/videos), instead of falling back to JSON.
"""
return False
@abstractmethod
def transform_video_create_request(
self,
@ -332,14 +341,18 @@ class BaseVideoConfig(ABC):
api_base: str,
litellm_params: GenericLiteLLMParams,
headers: dict,
video_file: FileContent | None = None,
extra_body: dict[str, Any] | None = None,
prefetched_source_data: dict[str, Any] | None = None,
) -> tuple[str, dict]:
) -> tuple[str, Mapping[str, object], RequestFiles | None]:
"""
Transform the video edit request into a URL and JSON data.
Transform the video edit request into a URL plus either JSON data or
multipart form fields and files.
Returns:
Tuple[str, Dict]: (url, data) for the POST request
tuple[str, Mapping[str, object], RequestFiles | None]: (url, data,
files). When files is None the handler sends data as JSON; otherwise
data holds the form fields and files holds the uploaded source video.
"""
raise NotImplementedError("video edit is not supported for this provider")

8
litellm/llms/bedrock/chat/invoke_handler.py
View file

@ -567,6 +567,10 @@ class AWSEventStreamDecoder:
if thinking_blocks:
self._thinking_ran = True
carries_message_content: Final = any(
key in chunk_data for key in ("start", "delta", "contentBlockIndex", "stopReason", "trace")
)
model_response_provider_specific_fields: Final = {}
if "trace" in chunk_data:
trace: Final = chunk_data.get("trace")
@ -577,8 +581,8 @@ class AWSEventStreamDecoder:
finish_reason=finish_reason,
index=0, # Always 0 - Bedrock never returns multiple choices
delta=Delta(
content=text,
role="assistant",
content=text if carries_message_content else None,
role="assistant" if carries_message_content else None,
tool_calls=[tool_use] if tool_use else None,
provider_specific_fields=(provider_specific_fields if provider_specific_fields else None),
thinking_blocks=thinking_blocks,

2
litellm/llms/cohere/rerank/transformation.py
View file

@ -1,3 +1,4 @@
from collections.abc import Mapping
from typing import Any, Final
import httpx
@ -81,6 +82,7 @@ class CohereRerankConfig(BaseRerankConfig):
model: str,
api_key: str | None = None,
optional_params: dict | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> dict:
if api_key is None:
api_key = get_secret_str("COHERE_API_KEY") or get_secret_str("CO_API_KEY") or litellm.cohere_key

59
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, Optional, TypedDict, Type
from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
import httpx
from httpx._types import FileContent
from openai.types.file_deleted import FileDeleted
import litellm
@ -24,6 +25,7 @@ from litellm.litellm_core_utils.agentic_loop_settings import (
validated_max_agentic_loops,
)
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.llm_request_utils import serialize_multipart_form_fields
from litellm.litellm_core_utils.realtime_errors import realtime_error_event, websocket_close_reason
from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
@ -1108,6 +1110,7 @@ class BaseLLMHTTPHandler:
headers=headers or {},
model=model,
optional_params=optional_rerank_params,
litellm_params=litellm_params,
)
api_base = provider_config.get_complete_url(
@ -1844,6 +1847,7 @@ class BaseLLMHTTPHandler:
return provider_config.transform_search_response(
raw_response=response,
logging_obj=logging_obj,
optional_params=optional_params,
)
async def async_search(
@ -1942,6 +1946,7 @@ class BaseLLMHTTPHandler:
return provider_config.transform_search_response(
raw_response=response,
logging_obj=logging_obj,
optional_params=optional_params,
)
async def _async_post_anthropic_messages_with_http_error_retry(
@ -7050,9 +7055,7 @@ class BaseLLMHTTPHandler:
)
try:
# Use JSON when no files, otherwise use form data with files
if files and len(files) > 0:
# Use multipart/form-data when files are present
response = sync_httpx_client.post(
url=api_base,
headers=headers,
@ -7060,9 +7063,14 @@ class BaseLLMHTTPHandler:
files=files,
timeout=timeout,
)
elif video_generation_provider_config.use_multipart_form_data():
response = sync_httpx_client.post( # rebind-ok: one of three mutually-exclusive branches
url=api_base,
headers=headers,
files=serialize_multipart_form_fields(data),
timeout=timeout,
)
else:
# Use JSON content type for POST requests without files
response = sync_httpx_client.post(
url=api_base,
headers=headers,
@ -7154,20 +7162,26 @@ class BaseLLMHTTPHandler:
)
try:
# Use JSON when no files, otherwise use form data with files
if files is None or len(files) == 0:
if files and len(files) > 0:
response = await async_httpx_client.post(
url=api_base,
headers=headers,
json=data,
data=data,
files=files,
timeout=timeout,
)
elif video_generation_provider_config.use_multipart_form_data():
response = await async_httpx_client.post( # rebind-ok: one of three mutually-exclusive branches
url=api_base,
headers=headers,
files=serialize_multipart_form_fields(data),
timeout=timeout,
)
else:
response = await async_httpx_client.post(
url=api_base,
headers=headers,
data=data,
files=files,
json=data,
timeout=timeout,
)
@ -7827,6 +7841,7 @@ class BaseLLMHTTPHandler:
custom_llm_provider: str,
litellm_params,
logging_obj,
video_file: FileContent | None = None,
extra_headers: dict[str, object] | None = None,
extra_body: dict[str, object] | None = None,
timeout: float | None = None,
@ -7838,6 +7853,7 @@ class BaseLLMHTTPHandler:
return self.async_video_edit_handler(
prompt=prompt,
video_id=video_id,
video_file=video_file,
video_provider_config=video_provider_config,
custom_llm_provider=custom_llm_provider,
litellm_params=litellm_params,
@ -7891,9 +7907,10 @@ class BaseLLMHTTPHandler:
prefetched_source_data = prefetch_resp.json()
try:
url, data = video_provider_config.transform_video_edit_request(
url, data, files = video_provider_config.transform_video_edit_request(
prompt=prompt,
video_id=video_id,
video_file=video_file,
api_base=api_base,
litellm_params=litellm_params,
headers=headers,
@ -7912,11 +7929,10 @@ class BaseLLMHTTPHandler:
},
)
response: Final = sync_httpx_client.post(
url=url,
headers=headers,
json=data,
timeout=timeout,
response: Final = (
sync_httpx_client.post(url=url, headers=headers, data=data, files=files, timeout=timeout)
if files
else sync_httpx_client.post(url=url, headers=headers, json=data, timeout=timeout)
)
response.raise_for_status()
return video_provider_config.transform_video_edit_response(
@ -7936,6 +7952,7 @@ class BaseLLMHTTPHandler:
custom_llm_provider: str,
litellm_params,
logging_obj,
video_file: FileContent | None = None,
extra_headers: dict[str, object] | None = None,
extra_body: dict[str, object] | None = None,
timeout: float | None = None,
@ -7987,9 +8004,10 @@ class BaseLLMHTTPHandler:
prefetched_source_data = prefetch_resp.json()
try:
url, data = video_provider_config.transform_video_edit_request(
url, data, files = video_provider_config.transform_video_edit_request(
prompt=prompt,
video_id=video_id,
video_file=video_file,
api_base=api_base,
litellm_params=litellm_params,
headers=headers,
@ -8008,11 +8026,10 @@ class BaseLLMHTTPHandler:
},
)
response: Final = await async_httpx_client.post(
url=url,
headers=headers,
json=data,
timeout=timeout,
response: Final = await (
async_httpx_client.post(url=url, headers=headers, data=data, files=files, timeout=timeout)
if files
else async_httpx_client.post(url=url, headers=headers, json=data, timeout=timeout)
)
response.raise_for_status()
return video_provider_config.transform_video_edit_response(

2
litellm/llms/dashscope/rerank/transformation.py
View file

@ -22,6 +22,7 @@ as supported only for gte-rerank-v2 / qwen3-vl-rerank.
Docs - https://help.aliyun.com/zh/model-studio/text-rerank-api
"""
from collections.abc import Mapping
from typing import Any, Final
import httpx
@ -85,6 +86,7 @@ class DashScopeRerankConfig(BaseRerankConfig):
model: str,
api_key: str | None = None,
optional_params: dict | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> dict:
if api_key is None:
api_key = get_secret_str("DASHSCOPE_API_KEY")

61
litellm/llms/databricks/cost_calculator.py
View file

@ -3,10 +3,31 @@ Helper util for handling databricks-specific cost calculation
- e.g.: handling 'dbrx-instruct-*'
"""
from types import MappingProxyType
from typing import Final
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
from litellm.types.utils import Usage
from litellm.utils import get_model_info
_LEGACY_ENDPOINT_NAMES: Final = MappingProxyType(
{
"dbrx-instruct": "databricks-dbrx-instruct",
"meta-llama-3.1-70b-instruct": "databricks-meta-llama-3-1-70b-instruct",
"meta-llama-3.1-405b-instruct": "databricks-meta-llama-3-1-405b-instruct",
"mixtral-8x7b-instruct-v0.1": "databricks-mixtral-8x7b-instruct",
"bge-large-en": "databricks-bge-large-en",
"gte-large-en": "databricks-gte-large-en",
"llama-2-70b-chat": "databricks-llama-2-70b-chat",
}
)
def _registry_key(model: str) -> str:
name: Final = model.removeprefix("databricks/")
return next(
(key for prefix, key in _LEGACY_ENDPOINT_NAMES.items() if name.startswith(prefix)),
name,
)
def cost_per_token(model: str, usage: Usage) -> tuple[float, float]:
@ -20,36 +41,8 @@ def cost_per_token(model: str, usage: Usage) -> tuple[float, float]:
Returns:
Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd
"""
base_model = model
if model.startswith("databricks/dbrx-instruct") or model.startswith("dbrx-instruct"):
base_model = "databricks-dbrx-instruct"
elif model.startswith("databricks/meta-llama-3.1-70b-instruct") or model.startswith("meta-llama-3.1-70b-instruct"):
base_model = "databricks-meta-llama-3-1-70b-instruct"
elif model.startswith("databricks/meta-llama-3.1-405b-instruct") or model.startswith(
"meta-llama-3.1-405b-instruct"
):
base_model = "databricks-meta-llama-3-1-405b-instruct"
elif (
model.startswith("databricks/mixtral-8x7b-instruct-v0.1")
or model.startswith("mixtral-8x7b-instruct-v0.1")
or model.startswith("databricks/mixtral-8x7b-instruct-v0.1")
or model.startswith("mixtral-8x7b-instruct-v0.1")
):
base_model = "databricks-mixtral-8x7b-instruct"
elif model.startswith("databricks/bge-large-en") or model.startswith("bge-large-en"):
base_model = "databricks-bge-large-en"
elif model.startswith("databricks/gte-large-en") or model.startswith("gte-large-en"):
base_model = "databricks-gte-large-en"
elif model.startswith("databricks/llama-2-70b-chat") or model.startswith("llama-2-70b-chat"):
base_model = "databricks-llama-2-70b-chat"
## GET MODEL INFO
model_info: Final = get_model_info(model=base_model, custom_llm_provider="databricks")
## CALCULATE INPUT COST
prompt_cost: Final[float] = usage["prompt_tokens"] * model_info["input_cost_per_token"]
## CALCULATE OUTPUT COST
completion_cost: Final = usage["completion_tokens"] * model_info["output_cost_per_token"]
return prompt_cost, completion_cost
return generic_cost_per_token(
model=_registry_key(model),
usage=usage,
custom_llm_provider="databricks",
)

2
litellm/llms/deepinfra/rerank/transformation.py
View file

@ -2,6 +2,7 @@
Translate between Cohere's `/rerank` format and Deepinfra's `/rerank` format.
"""
from collections.abc import Mapping
from typing import Any, Final
import httpx
@ -67,6 +68,7 @@ class DeepinfraRerankConfig(BaseRerankConfig):
model: str,
api_key: str | None = None,
optional_params: dict | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> dict:
if api_key is None:
api_key = get_secret_str("DEEPINFRA_API_KEY")

1
litellm/llms/fireworks_ai/chat/transformation.py
View file

@ -504,6 +504,7 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
m = cast(dict, message)
m.pop("provider_specific_fields", None)
m.pop("thinking_blocks", None)
m.pop("reasoning_content", None)
return messages

2
litellm/llms/fireworks_ai/rerank/transformation.py
View file

@ -4,6 +4,7 @@ Fireworks AI Rerank API transformation
Reference: https://docs.fireworks.ai/inference-api-reference/rerank
"""
from collections.abc import Mapping
from typing import Any, Final
import httpx
@ -102,6 +103,7 @@ class FireworksAIRerankConfig(FireworksAIMixin, BaseRerankConfig):
model: str,
api_key: str | None = None,
optional_params: dict | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> dict:
api_key = self._get_api_key(api_key)
if api_key is None:

1
litellm/llms/gemini/videos/transformation.py
View file

@ -566,6 +566,7 @@ class GeminiVideoConfig(BaseVideoConfig):
api_base,
litellm_params,
headers,
video_file=None,
extra_body=None,
prefetched_source_data=None,
):

5
litellm/llms/hosted_vllm/chat/transformation.py
View file

@ -164,12 +164,13 @@ class HostedVLLMChatConfig(OpenAIGPTConfig):
"""
Support translating:
- video files from file_id or file_data to video_url
- thinking_blocks on assistant messages are removed, and content lists
are converted to strings for vLLM compatibility
- thinking_blocks and reasoning_content on assistant messages are removed,
and content lists are converted to strings for vLLM compatibility
"""
for message in messages:
if message["role"] == "assistant":
message.pop("thinking_blocks", None)
message.pop("reasoning_content", None)
existing_content = message.get("content")
if isinstance(existing_content, list):
text_parts = []

2
litellm/llms/hosted_vllm/rerank/transformation.py
View file

@ -2,6 +2,7 @@
Transformation logic for Hosted VLLM rerank
"""
from collections.abc import Mapping
from typing import Any, Final
import httpx
@ -107,6 +108,7 @@ class HostedVLLMRerankConfig(BaseRerankConfig):
model: str,
api_key: str | None = None,
optional_params: dict | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> dict:
if api_key is None:
api_key = get_secret_str("HOSTED_VLLM_API_KEY") or "fake-api-key"

2
litellm/llms/huggingface/rerank/transformation.py
View file

@ -1,4 +1,5 @@
import os
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final
import httpx
@ -123,6 +124,7 @@ class HuggingFaceRerankConfig(BaseRerankConfig):
model: str,
api_key: str | None = None,
optional_params: dict | None = None,
litellm_params: Mapping[str, object] | None = None,
api_base: str | None = None,
) -> dict:
# Get API credentials

2
litellm/llms/infinity/rerank/transformation.py
View file

@ -4,6 +4,7 @@ Transformation logic from Cohere's /v1/rerank format to Infinity's `/v1/rerank`
Why separate file? Make it easy to see how transformation works
"""
from collections.abc import Mapping
from typing import Final
import httpx
@ -46,6 +47,7 @@ class InfinityRerankConfig(CohereRerankConfig):
model: str,
api_key: str | None = None,
optional_params: dict | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> dict:
if api_key is None:
api_key = get_secret_str("INFINITY_API_KEY") or get_secret_str("INFINITY_API_KEY") or litellm.infinity_key

2
litellm/llms/jina_ai/rerank/transformation.py
View file

@ -6,6 +6,7 @@ Why separate file? Make it easy to see how transformation works
Docs - https://jina.ai/reranker
"""
from collections.abc import Mapping
from typing import Any, Final
from httpx import URL, Response
@ -139,6 +140,7 @@ class JinaAIRerankConfig(BaseRerankConfig):
model: str,
api_key: str | None = None,
optional_params: dict | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> dict:
if api_key is None:
raise ValueError("api_key is required. Set via `api_key` parameter or `JINA_API_KEY` environment variable.")

2
litellm/llms/nvidia_nim/rerank/transformation.py
View file

@ -1,3 +1,4 @@
from collections.abc import Mapping
from typing import Any, Final, Literal
import httpx
@ -152,6 +153,7 @@ class NvidiaNimRerankConfig(BaseRerankConfig):
model: str,
api_key: str | None = None,
optional_params: dict | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> dict:
"""
Validate that the Nvidia NIM API key is present.

9
litellm/llms/openai/cost_calculation.py
View file

@ -134,14 +134,7 @@ def cost_per_second(model: str, custom_llm_provider: str | None, duration: float
def _video_resolution_to_cost_field_suffix(resolution: str) -> str | None:
"""
Map usage resolution to a safe suffix for ``output_cost_per_second_<suffix>`` keys.
Note: Currently only ``output_cost_per_second_1080p`` is explicitly declared in
ModelInfo (types/utils.py). Other resolution tiers (e.g., 720p, 4k) can be added
to model_prices_and_context_window.json but are not exposed via get_model_info()
until added to the ModelInfo TypedDict.
"""
"""Map usage resolution to a safe suffix for ``output_cost_per_second_<suffix>`` keys."""
r: Final = resolution.strip().lower()
if not r:
return None

41
litellm/llms/openai/videos/transformation.py
View file

@ -1,5 +1,7 @@
import mimetypes
from collections.abc import Mapping
from io import BufferedReader, BytesIO
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, cast
from urllib.parse import quote
@ -101,6 +103,9 @@ class OpenAIVideoConfig(BaseVideoConfig):
return f"{api_base.rstrip('/')}/videos"
def use_multipart_form_data(self) -> bool:
return True
def transform_video_create_request(
self,
model: str,
@ -499,15 +504,26 @@ class OpenAIVideoConfig(BaseVideoConfig):
api_base: str,
litellm_params: GenericLiteLLMParams,
headers: dict,
video_file: FileContent | None = None,
extra_body: dict[str, object] | None = None,
prefetched_source_data: dict[str, object] | None = None,
) -> tuple[str, dict]:
original_video_id: Final = extract_original_video_id(video_id)
) -> tuple[str, Mapping[str, object], RequestFiles | None]:
url: Final = f"{api_base.rstrip('/')}/edits"
if video_file is not None:
files: Final[RequestFiles] = (self._video_file_tuple(video_file, "video"),)
form_data: Final = (
MappingProxyType({"prompt": prompt, **extra_body})
if extra_body
else MappingProxyType({"prompt": prompt})
)
return url, form_data, files
original_video_id: Final = extract_original_video_id(video_id)
data: Final[dict[str, object]] = {"prompt": prompt, "video": {"id": original_video_id}}
if extra_body:
data.update(extra_body)
return url, data
return url, data, None
def transform_video_edit_response(
self,
@ -567,21 +583,22 @@ class OpenAIVideoConfig(BaseVideoConfig):
else:
files_list.append((field_name, ("input_reference.png", image, image_content_type)))
def _video_file_tuple(self, video: FileContent, field_name: str) -> tuple[str, FileTypes]:
"""
Build a multipart field tuple for a video upload with proper video MIME
type detection: these paths must send video/mp4, not image/* content types.
"""
filename: Final = getattr(video, "name", None) or "input_video.mp4"
content_type: Final = self._get_video_content_type(video=video, filename=filename)
return (field_name, (filename, video, content_type))
def _add_video_to_files(
self,
files_list: list[tuple[str, FileTypes]],
video: FileContent,
field_name: str,
) -> None:
"""
Add a video to files with proper video MIME type detection.
This path is used by POST /videos/characters and must send video/mp4,
not image/* content types.
"""
filename: Final = getattr(video, "name", None) or "input_video.mp4"
content_type: Final = self._get_video_content_type(video=video, filename=filename)
files_list.append((field_name, (filename, video, content_type)))
files_list.append(self._video_file_tuple(video, field_name))
def _get_video_content_type(self, video: FileContent, filename: str) -> str:
guessed_content_type, _ = mimetypes.guess_type(filename)

119
litellm/llms/runwayml/videos/transformation.py
View file

@ -1,5 +1,6 @@
from collections.abc import Mapping, Sequence
from datetime import datetime
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal
import httpx
@ -33,6 +34,10 @@ else:
LiteLLMLoggingObj = Any
class RunwayMLError(BaseLLMException):
pass
class _RunwayTaskResponse(TypedDict, total=False):
id: ReadOnly[str]
status: ReadOnly[str]
@ -41,7 +46,8 @@ class _RunwayTaskResponse(TypedDict, total=False):
output: ReadOnly[Sequence[str] | str]
failureCode: ReadOnly[str]
failure: ReadOnly[str]
progress: ReadOnly[int]
progress: ReadOnly[float]
estimatedCost: ReadOnly[Mapping[str, float]]
class _VideoObjectData(TypedDict, extra_items=object):
@ -56,12 +62,54 @@ def _parse_runway_task_response(raw_response: httpx.Response) -> _RunwayTaskResp
return response_data
_USD_PER_CREDIT: Final = 0.01
_RESOLUTION_AREA_TIERS: Final[tuple[tuple[int, str], ...]] = (
(600_000, "480p"),
(1_500_000, "720p"),
(4_000_000, "1080p"),
)
def _ratio_to_resolution(ratio: object) -> str | None:
if not isinstance(ratio, str) or ":" not in ratio:
return None
width_str, _, height_str = ratio.partition(":")
if not (width_str.isdigit() and height_str.isdigit()):
return None
area: Final = int(width_str) * int(height_str)
return next((label for threshold, label in _RESOLUTION_AREA_TIERS if area < threshold), "4k")
def _duration_seconds(seconds: str | None) -> float | None:
if not seconds:
return None
try:
return float(seconds)
except ValueError:
return None
def _estimated_cost_usd(response_data: _RunwayTaskResponse) -> float | None:
estimated_cost: Final = response_data.get("estimatedCost")
if not isinstance(estimated_cost, Mapping):
return None
credits: Final = estimated_cost.get("credits")
if not isinstance(credits, (int, float)):
return None
return float(credits) * _USD_PER_CREDIT
def _progress_percent(progress: float) -> int:
return min(100, max(0, round(float(progress) * 100)))
class RunwayMLVideoConfig(BaseVideoConfig):
"""
Configuration class for RunwayML video generation.
RunwayML uses a task-based API where:
1. POST /v1/image_to_video creates a task
1. POST /v1/text_to_video, /v1/image_to_video, or /v1/video_to_video creates a task
2. The task returns immediately with a task ID
3. Client must poll or wait for task completion
"""
@ -195,31 +243,36 @@ class RunwayMLVideoConfig(BaseVideoConfig):
"""
Transform the video creation request for RunwayML API.
RunwayML expects:
{
"model": "gen4_turbo",
"promptImage": "https://... or data:image/...",
"promptText": "description",
"ratio": "1280:720",
"duration": 5
}
RunwayML has three generation endpoints discriminated by which input is
present, and each request body rejects unknown fields:
- /text_to_video: promptText only (rejects promptImage)
- /image_to_video: promptImage (+ optional promptText)
- /video_to_video: promptVideo or videoUri (rejects promptImage)
"""
# Build the request data
merged_params: Final = MappingProxyType(
{
"model": model,
"promptText": prompt,
**video_create_optional_request_params,
}
)
endpoint: Final = self._select_generation_endpoint(merged_params)
request_data: Final[dict[str, object]] = {
"model": model,
"promptText": prompt,
key: value for key, value in merged_params.items() if endpoint == "image_to_video" or key != "promptImage"
}
# Add mapped parameters
request_data.update(video_create_optional_request_params)
# RunwayML uses JSON body, no files multipart
files_list: Final[RequestFiles] = []
# Append the specific endpoint for video generation
full_api_base: Final = f"{api_base}/image_to_video"
return request_data, files_list, f"{api_base}/{endpoint}"
return request_data, files_list, full_api_base
def _select_generation_endpoint(self, request_data: Mapping[str, object]) -> str:
if request_data.get("promptVideo") is not None or request_data.get("videoUri") is not None:
return "video_to_video"
if request_data.get("promptImage") is not None:
return "image_to_video"
return "text_to_video"
def transform_video_create_response(
self,
@ -285,13 +338,15 @@ class RunwayMLVideoConfig(BaseVideoConfig):
video_obj.id = encode_video_id_with_provider(video_obj.id, custom_llm_provider, model)
# Add usage data for cost tracking
usage_data: Final = {}
if video_obj and hasattr(video_obj, "seconds") and video_obj.seconds:
try:
usage_data["duration_seconds"] = float(video_obj.seconds)
except (ValueError, TypeError):
pass
video_obj.usage = usage_data
video_obj.usage = {
key: value
for key, value in (
("duration_seconds", _duration_seconds(video_obj.seconds)),
("video_resolution", _ratio_to_resolution(request_data.get("ratio") if request_data else None)),
("provider_reported_cost_usd", _estimated_cost_usd(response_data)),
)
if value is not None
}
return video_obj
@ -581,8 +636,9 @@ class RunwayMLVideoConfig(BaseVideoConfig):
if "completedAt" in response_data:
video_data["completed_at"] = self._parse_runway_timestamp(response_data.get("completedAt"))
if "progress" in response_data:
video_data["progress"] = response_data["progress"]
progress_value: Final = response_data.get("progress")
if progress_value is not None:
video_data["progress"] = _progress_percent(progress_value)
if "failureCode" in response_data or "failure" in response_data:
video_data["error"] = {
@ -616,6 +672,7 @@ class RunwayMLVideoConfig(BaseVideoConfig):
api_base,
litellm_params,
headers,
video_file=None,
extra_body=None,
prefetched_source_data=None,
):
@ -646,9 +703,7 @@ class RunwayMLVideoConfig(BaseVideoConfig):
raise NotImplementedError("video extension is not supported for RunwayML")
def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException:
from ...base_llm.chat.transformation import BaseLLMException
raise BaseLLMException(
return RunwayMLError(
status_code=status_code,
message=error_message,
headers=headers,

8
litellm/llms/vertex_ai/rerank/transformation.py
View file

@ -4,6 +4,7 @@ Translates from Cohere's `/v1/rerank` input format to Vertex AI Discovery Engine
Why separate file? Make it easy to see how transformation works
"""
from collections.abc import Mapping
from typing import Any, Final
import httpx
@ -74,14 +75,15 @@ class VertexAIRerankConfig(BaseRerankConfig, VertexBase):
model: str,
api_key: str | None = None,
optional_params: dict | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> dict:
"""
Validate and set up authentication for Vertex AI Discovery Engine API
"""
# Get credentials and project info from optional_params (which contains vertex_credentials, etc.)
litellm_params: Final = optional_params.copy() if optional_params else {}
vertex_credentials: Final = self.safe_get_vertex_ai_credentials(litellm_params)
vertex_project: Final = self.safe_get_vertex_ai_project(litellm_params)
vertex_params: Final = optional_params.copy() if optional_params else {}
vertex_credentials: Final = self.safe_get_vertex_ai_credentials(vertex_params)
vertex_project: Final = self.safe_get_vertex_ai_project(vertex_params)
# Get access token using the base class method
access_token, project_id = self._ensure_access_token(

9
litellm/llms/vertex_ai/videos/transformation.py
View file

@ -7,11 +7,11 @@ Based on: https://docs.cloud.google.com/vertex-ai/generative-ai/docs/model-refer
import base64
import time
from collections.abc import Sequence
from collections.abc import Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final, TypedDict, cast
import httpx
from httpx._types import RequestFiles
from httpx._types import FileContent, RequestFiles
from typing_extensions import ReadOnly
from litellm.constants import DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS
@ -677,9 +677,10 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase):
api_base: str,
litellm_params: GenericLiteLLMParams,
headers: dict,
video_file: FileContent | None = None,
extra_body: dict[str, object] | None = None,
prefetched_source_data: dict[str, Any] | None = None,
) -> tuple[str, dict]:
) -> tuple[str, Mapping[str, object], RequestFiles | None]:
"""
Build a predictLongRunning edit request from the pre-fetched source video.
@ -727,7 +728,7 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase):
request_data["parameters"] = vertex_params
edit_url: Final = f"{api_base.rstrip('/')}/{model}:predictLongRunning"
return edit_url, request_data
return edit_url, request_data, None
def transform_video_edit_response(
self,

2
litellm/llms/voyage/rerank/transformation.py
View file

@ -4,6 +4,7 @@ Transformation logic for Voyage AI's /v1/rerank endpoint.
Docs - https://docs.voyageai.com/docs/reranker
"""
from collections.abc import Mapping
from typing import Any, Final
import httpx
@ -137,6 +138,7 @@ class VoyageRerankConfig(BaseRerankConfig):
model: str,
api_key: str | None = None,
optional_params: dict | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> dict:
if api_key is None:
api_key = get_secret_str("VOYAGE_API_KEY") or get_secret_str("VOYAGE_AI_API_KEY")

2
litellm/llms/watsonx/rerank/transformation.py
View file

@ -5,6 +5,7 @@ Docs - https://cloud.ibm.com/apidocs/watsonx-ai#text-rerank
"""
import uuid
from collections.abc import Mapping
from typing import Any, Final, cast
import httpx
@ -60,6 +61,7 @@ class IBMWatsonXRerankConfig(IBMWatsonXMixin, BaseRerankConfig):
model: str,
api_key: str | None = None,
optional_params: dict | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> dict:
optional_params = optional_params or {}

26
litellm/main.py
View file

@ -6828,6 +6828,8 @@ def embedding(
aembedding=aembedding,
)
elif custom_llm_provider == "azure_ai":
from litellm.llms.azure_ai.common_utils import get_azure_ai_entra_token
api_base = (
api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there
or litellm.api_base
@ -6837,8 +6839,8 @@ def embedding(
api_key = (
api_key
or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there
or litellm.openai_key
or get_secret_str("AZURE_AI_API_KEY")
or get_azure_ai_entra_token(litellm_params=litellm_params_dict)
)
## EMBEDDING CALL
@ -8566,7 +8568,7 @@ def stream_chunk_builder(
if len(chunks) == 0:
return None
## Route to the text completion logic
first_chunk_with_choices: Final = next((c for c in chunks if c["choices"]), None)
first_chunk_with_choices: Final = next((c for c in chunks if c.get("choices")), None)
if first_chunk_with_choices is not None and isinstance(
first_chunk_with_choices["choices"][0], litellm.utils.TextChoices
): # route to the text completion logic
@ -8581,7 +8583,7 @@ def stream_chunk_builder(
simple_content_parts: Final[list[str]] = []
is_simple_text_stream = True
for chunk in chunks:
if len(chunk["choices"]) == 0:
if not chunk.get("choices"):
continue
choice = chunk["choices"][0]
@ -8647,7 +8649,7 @@ def stream_chunk_builder(
tool_call_chunks: Final = [
chunk
for chunk in chunks
if len(chunk["choices"]) > 0
if chunk.get("choices")
and "tool_calls" in chunk["choices"][0]["delta"]
and chunk["choices"][0]["delta"]["tool_calls"] is not None
]
@ -8661,7 +8663,7 @@ def stream_chunk_builder(
function_call_chunks: Final = [
chunk
for chunk in chunks
if len(chunk["choices"]) > 0
if chunk.get("choices")
and "function_call" in chunk["choices"][0]["delta"]
and chunk["choices"][0]["delta"]["function_call"] is not None
]
@ -8674,7 +8676,7 @@ def stream_chunk_builder(
content_chunks: Final = [
chunk
for chunk in chunks
if len(chunk["choices"]) > 0
if chunk.get("choices")
and "content" in chunk["choices"][0]["delta"]
and chunk["choices"][0]["delta"]["content"] is not None
]
@ -8685,7 +8687,7 @@ def stream_chunk_builder(
thinking_blocks: Final = [
chunk
for chunk in chunks
if len(chunk["choices"]) > 0
if chunk.get("choices")
and "thinking_blocks" in chunk["choices"][0]["delta"]
and chunk["choices"][0]["delta"]["thinking_blocks"] is not None
]
@ -8698,7 +8700,7 @@ def stream_chunk_builder(
reasoning_chunks: Final = [
chunk
for chunk in chunks
if len(chunk["choices"]) > 0
if chunk.get("choices")
and "reasoning_content" in chunk["choices"][0]["delta"]
and chunk["choices"][0]["delta"]["reasoning_content"] is not None
]
@ -8711,7 +8713,7 @@ def stream_chunk_builder(
annotation_chunks: Final = [
chunk
for chunk in chunks
if len(chunk["choices"]) > 0
if chunk.get("choices")
and "annotations" in chunk["choices"][0]["delta"]
and chunk["choices"][0]["delta"]["annotations"] is not None
]
@ -8728,7 +8730,7 @@ def stream_chunk_builder(
audio_chunks: Final = [
chunk
for chunk in chunks
if len(chunk["choices"]) > 0
if chunk.get("choices")
and "audio" in chunk["choices"][0]["delta"]
and chunk["choices"][0]["delta"]["audio"] is not None
]
@ -8742,7 +8744,7 @@ def stream_chunk_builder(
image_chunks: Final = [
chunk
for chunk in chunks
if len(chunk["choices"]) > 0
if chunk.get("choices")
and "images" in chunk["choices"][0]["delta"]
and chunk["choices"][0]["delta"]["images"] is not None
]
@ -8759,7 +8761,7 @@ def stream_chunk_builder(
provider_specific_chunks: Final = [
chunk
for chunk in chunks
if len(chunk["choices"]) > 0
if chunk.get("choices")
and "provider_specific_fields" in chunk["choices"][0]["delta"]
and chunk["choices"][0]["delta"]["provider_specific_fields"] is not None
]

466
litellm/model_prices_and_context_window_backup.json
View file

@ -1019,6 +1019,7 @@
},
"anthropic.claude-opus-4-6-v1": {
"supports_adaptive_thinking": true,
"supports_legacy_thinking": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
"cache_read_input_token_cost": 5e-07,
@ -1053,6 +1054,7 @@
},
"global.anthropic.claude-opus-4-6-v1": {
"supports_adaptive_thinking": true,
"supports_legacy_thinking": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
"cache_read_input_token_cost": 5e-07,
@ -1087,6 +1089,7 @@
},
"us.anthropic.claude-opus-4-6-v1": {
"supports_adaptive_thinking": true,
"supports_legacy_thinking": true,
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_1hr": 1.1e-05,
"cache_read_input_token_cost": 5.5e-07,
@ -1121,6 +1124,7 @@
},
"eu.anthropic.claude-opus-4-6-v1": {
"supports_adaptive_thinking": true,
"supports_legacy_thinking": true,
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_1hr": 1.1e-05,
"cache_read_input_token_cost": 5.5e-07,
@ -1155,6 +1159,7 @@
},
"au.anthropic.claude-opus-4-6-v1": {
"supports_adaptive_thinking": true,
"supports_legacy_thinking": true,
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_1hr": 1.1e-05,
"cache_read_input_token_cost": 5.5e-07,
@ -2233,6 +2238,7 @@
},
"anthropic.claude-sonnet-4-6": {
"supports_adaptive_thinking": true,
"supports_legacy_thinking": true,
"cache_creation_input_token_cost": 3.75e-06,
"cache_creation_input_token_cost_above_1hr": 6e-06,
"cache_read_input_token_cost": 3e-07,
@ -2266,6 +2272,7 @@
},
"global.anthropic.claude-sonnet-4-6": {
"supports_adaptive_thinking": true,
"supports_legacy_thinking": true,
"cache_creation_input_token_cost": 3.75e-06,
"cache_creation_input_token_cost_above_1hr": 6e-06,
"cache_read_input_token_cost": 3e-07,
@ -2299,6 +2306,7 @@
},
"us.anthropic.claude-sonnet-4-6": {
"supports_adaptive_thinking": true,
"supports_legacy_thinking": true,
"cache_creation_input_token_cost": 4.125e-06,
"cache_creation_input_token_cost_above_1hr": 6.6e-06,
"cache_read_input_token_cost": 3.3e-07,
@ -2332,6 +2340,7 @@
},
"eu.anthropic.claude-sonnet-4-6": {
"supports_adaptive_thinking": true,
"supports_legacy_thinking": true,
"cache_creation_input_token_cost": 4.125e-06,
"cache_creation_input_token_cost_above_1hr": 6.6e-06,
"cache_read_input_token_cost": 3.3e-07,
@ -2365,6 +2374,7 @@
},
"au.anthropic.claude-sonnet-4-6": {
"supports_adaptive_thinking": true,
"supports_legacy_thinking": true,
"cache_creation_input_token_cost": 4.125e-06,
"cache_creation_input_token_cost_above_1hr": 6.6e-06,
"cache_read_input_token_cost": 3.3e-07,
@ -2398,6 +2408,7 @@
},
"jp.anthropic.claude-sonnet-4-6": {
"supports_adaptive_thinking": true,
"supports_legacy_thinking": true,
"cache_creation_input_token_cost": 4.125e-06,
"cache_creation_input_token_cost_above_1hr": 6.6e-06,
"cache_read_input_token_cost": 3.3e-07,
@ -2950,6 +2961,7 @@
"azure_ai/claude-opus-4-6": {
"deprecation_date": "2027-02-02",
"supports_adaptive_thinking": true,
"supports_legacy_thinking": true,
"input_cost_per_token": 5e-06,
"output_cost_per_token": 2.5e-05,
"litellm_provider": "azure_ai",
@ -3181,6 +3193,7 @@
"azure_ai/claude-sonnet-4-6": {
"deprecation_date": "2027-02-10",
"supports_adaptive_thinking": true,
"supports_legacy_thinking": true,
"cache_creation_input_token_cost": 3.75e-06,
"cache_creation_input_token_cost_above_1hr": 6e-06,
"cache_read_input_token_cost": 3e-07,
@ -12489,6 +12502,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_legacy_thinking": true,
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
@ -12698,6 +12712,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_legacy_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -12735,6 +12750,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_legacy_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -14551,6 +14567,8 @@
]
},
"databricks/databricks-bge-large-en": {
"cache_creation_input_token_cost": 1.0003e-07,
"cache_read_input_token_cost": 1.0003e-07,
"input_cost_per_token": 1.0003e-07,
"input_dbu_cost_per_token": 1.429e-06,
"litellm_provider": "databricks",
@ -14566,6 +14584,8 @@
"source": "https://www.databricks.com/product/pricing/foundation-model-serving"
},
"databricks/databricks-claude-3-7-sonnet": {
"cache_creation_input_token_cost": 3.74997e-06,
"cache_read_input_token_cost": 3.0002e-07,
"input_cost_per_token": 2.9999900000000002e-06,
"input_dbu_cost_per_token": 4.2857e-05,
"litellm_provider": "databricks",
@ -14581,10 +14601,41 @@
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
},
"databricks/databricks-claude-fable-5": {
"cache_creation_input_token_cost": 1.250004e-05,
"cache_read_input_token_cost": 1.00002e-06,
"input_cost_per_token": 1.000006e-05,
"input_dbu_cost_per_token": 0.000142858,
"litellm_provider": "databricks",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"metadata": {
"notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields."
},
"mode": "chat",
"output_cost_per_token": 5.000002e-05,
"output_dbu_cost_per_token": 0.000714286,
"prompt_cache_min_tokens": 512,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_function_calling": true,
"supports_mid_conversation_system": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": false,
"thinking_always_on": true
},
"databricks/databricks-claude-haiku-4-5": {
"cache_creation_input_token_cost": 1.24999e-06,
"cache_read_input_token_cost": 1.0003e-07,
"input_cost_per_token": 1.00002e-06,
"input_dbu_cost_per_token": 1.4286e-05,
"litellm_provider": "databricks",
@ -14600,10 +14651,13 @@
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
},
"databricks/databricks-claude-opus-4": {
"cache_creation_input_token_cost": 1.874999e-05,
"cache_read_input_token_cost": 1.50003e-06,
"input_cost_per_token": 1.5000020000000002e-05,
"input_dbu_cost_per_token": 0.000214286,
"litellm_provider": "databricks",
@ -14619,10 +14673,13 @@
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
},
"databricks/databricks-claude-opus-4-1": {
"cache_creation_input_token_cost": 1.874999e-05,
"cache_read_input_token_cost": 1.50003e-06,
"input_cost_per_token": 1.5000020000000002e-05,
"input_dbu_cost_per_token": 0.000214286,
"litellm_provider": "databricks",
@ -14638,10 +14695,13 @@
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
},
"databricks/databricks-claude-opus-4-5": {
"cache_creation_input_token_cost": 6.25002e-06,
"cache_read_input_token_cost": 5.0001e-07,
"input_cost_per_token": 5.00003e-06,
"input_dbu_cost_per_token": 7.1429e-05,
"litellm_provider": "databricks",
@ -14657,11 +14717,14 @@
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_output_config": true
},
"databricks/databricks-claude-opus-4-6": {
"cache_creation_input_token_cost": 6.25002e-06,
"cache_read_input_token_cost": 5.0001e-07,
"input_cost_per_token": 5.00003e-06,
"input_dbu_cost_per_token": 7.1429e-05,
"litellm_provider": "databricks",
@ -14677,10 +14740,94 @@
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_legacy_thinking": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
},
"databricks/databricks-claude-opus-4-7": {
"cache_creation_input_token_cost": 6.25002e-06,
"cache_read_input_token_cost": 5.0001e-07,
"input_cost_per_token": 5.00003e-06,
"input_dbu_cost_per_token": 7.1429e-05,
"litellm_provider": "databricks",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"metadata": {
"notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields."
},
"mode": "chat",
"output_cost_per_token": 2.500001e-05,
"output_dbu_cost_per_token": 0.000357143,
"prompt_cache_min_tokens": 2048,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true
},
"databricks/databricks-claude-opus-4-8": {
"cache_creation_input_token_cost": 6.25002e-06,
"cache_read_input_token_cost": 5.0001e-07,
"input_cost_per_token": 5.00003e-06,
"input_dbu_cost_per_token": 7.1429e-05,
"litellm_provider": "databricks",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"metadata": {
"notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields."
},
"mode": "chat",
"output_cost_per_token": 2.500001e-05,
"output_dbu_cost_per_token": 0.000357143,
"prompt_cache_min_tokens": 1024,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_function_calling": true,
"supports_mid_conversation_system": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true
},
"databricks/databricks-claude-opus-5": {
"cache_creation_input_token_cost": 6.25002e-06,
"cache_read_input_token_cost": 5.0001e-07,
"input_cost_per_token": 5.00003e-06,
"input_dbu_cost_per_token": 7.1429e-05,
"litellm_provider": "databricks",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"metadata": {
"notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields."
},
"mode": "chat",
"output_cost_per_token": 2.500001e-05,
"output_dbu_cost_per_token": 0.000357143,
"prompt_cache_min_tokens": 512,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_function_calling": true,
"supports_mid_conversation_system": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true
},
"databricks/databricks-claude-sonnet-4": {
"cache_creation_input_token_cost": 3.74997e-06,
"cache_read_input_token_cost": 3.0002e-07,
"input_cost_per_token": 2.9999900000000002e-06,
"input_dbu_cost_per_token": 4.2857e-05,
"litellm_provider": "databricks",
@ -14696,10 +14843,13 @@
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
},
"databricks/databricks-claude-sonnet-4-1": {
"cache_creation_input_token_cost": 3.74997e-06,
"cache_read_input_token_cost": 3.0002e-07,
"input_cost_per_token": 2.9999900000000002e-06,
"input_dbu_cost_per_token": 4.2857e-05,
"litellm_provider": "databricks",
@ -14715,10 +14865,13 @@
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
},
"databricks/databricks-claude-sonnet-4-5": {
"cache_creation_input_token_cost": 3.74997e-06,
"cache_read_input_token_cost": 3.0002e-07,
"input_cost_per_token": 2.9999900000000002e-06,
"input_dbu_cost_per_token": 4.2857e-05,
"litellm_provider": "databricks",
@ -14734,10 +14887,13 @@
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
},
"databricks/databricks-claude-sonnet-4-6": {
"cache_creation_input_token_cost": 3.74997e-06,
"cache_read_input_token_cost": 3.0002e-07,
"input_cost_per_token": 2.9999900000000002e-06,
"input_dbu_cost_per_token": 4.2857e-05,
"litellm_provider": "databricks",
@ -14753,10 +14909,41 @@
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_legacy_thinking": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
},
"databricks/databricks-claude-sonnet-5": {
"cache_creation_input_token_cost": 3.74997e-06,
"cache_read_input_token_cost": 3.0002e-07,
"input_cost_per_token": 2.99999e-06,
"input_dbu_cost_per_token": 4.2857e-05,
"litellm_provider": "databricks",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"metadata": {
"notes": "Costs per token are the published Global DBU rates times $0.070 per DBU. The '*_dbu_cost_per_token' fields are provided for reference; cost calculation reads the dollar '*_cost_per_token' fields. Introductory launch rates of 28.571 input / 142.857 output / 35.714 cache write / 2.857 cache read DBU run through 2026-08-31; the standard rates are listed here because entries carry no expiry date."
},
"mode": "chat",
"output_cost_per_token": 1.500002e-05,
"output_dbu_cost_per_token": 0.000214286,
"prompt_cache_min_tokens": 1024,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_function_calling": true,
"supports_mid_conversation_system": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true
},
"databricks/databricks-gemini-2-5-flash": {
"cache_creation_input_token_cost": 3.0002e-07,
"cache_read_input_token_cost": 3.0002e-08,
"input_cost_per_token": 3.0001999999999996e-07,
"input_dbu_cost_per_token": 4.285999999999999e-06,
"litellm_provider": "databricks",
@ -14771,9 +14958,12 @@
"output_dbu_cost_per_token": 3.5714e-05,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_tool_choice": true
},
"databricks/databricks-gemini-2-5-pro": {
"cache_creation_input_token_cost": 1.24999e-06,
"cache_read_input_token_cost": 1.24999e-07,
"input_cost_per_token": 1.24999e-06,
"input_dbu_cost_per_token": 1.7857e-05,
"litellm_provider": "databricks",
@ -14788,9 +14978,12 @@
"output_dbu_cost_per_token": 0.000142857,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_tool_choice": true
},
"databricks/databricks-gemini-3-1-flash-lite": {
"cache_creation_input_token_cost": 3.1248e-07,
"cache_read_input_token_cost": 3.122e-08,
"input_cost_per_token": 3.1248e-07,
"input_dbu_cost_per_token": 4.464e-06,
"litellm_provider": "databricks",
@ -14805,9 +14998,12 @@
"output_dbu_cost_per_token": 2.6786e-05,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_tool_choice": true
},
"databricks/databricks-gemini-3-1-pro": {
"cache_creation_input_token_cost": 2.49998e-06,
"cache_read_input_token_cost": 2.4997e-07,
"input_cost_per_token": 2.49998e-06,
"input_dbu_cost_per_token": 3.5714e-05,
"litellm_provider": "databricks",
@ -14822,9 +15018,12 @@
"output_dbu_cost_per_token": 0.000214286,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_tool_choice": true
},
"databricks/databricks-gemini-3-flash": {
"cache_creation_input_token_cost": 6.2503e-07,
"cache_read_input_token_cost": 6.251e-08,
"input_cost_per_token": 6.2503e-07,
"input_dbu_cost_per_token": 8.929e-06,
"litellm_provider": "databricks",
@ -14839,9 +15038,12 @@
"output_dbu_cost_per_token": 5.3571e-05,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_tool_choice": true
},
"databricks/databricks-gemini-3-pro": {
"cache_creation_input_token_cost": 2.49998e-06,
"cache_read_input_token_cost": 2.4997e-07,
"input_cost_per_token": 2.49998e-06,
"input_dbu_cost_per_token": 3.5714e-05,
"litellm_provider": "databricks",
@ -14856,9 +15058,12 @@
"output_dbu_cost_per_token": 0.000214286,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_tool_choice": true
},
"databricks/databricks-gemma-3-12b": {
"cache_creation_input_token_cost": 1.5001e-07,
"cache_read_input_token_cost": 1.5001e-07,
"input_cost_per_token": 1.5000999999999998e-07,
"input_dbu_cost_per_token": 2.1429999999999996e-06,
"litellm_provider": "databricks",
@ -14874,6 +15079,8 @@
"source": "https://www.databricks.com/product/pricing/foundation-model-serving"
},
"databricks/databricks-gpt-5": {
"cache_creation_input_token_cost": 1.24999e-06,
"cache_read_input_token_cost": 1.2502e-07,
"input_cost_per_token": 1.24999e-06,
"input_dbu_cost_per_token": 1.7857e-05,
"litellm_provider": "databricks",
@ -14886,9 +15093,12 @@
"mode": "chat",
"output_cost_per_token": 9.999990000000002e-06,
"output_dbu_cost_per_token": 0.000142857,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"supports_prompt_caching": true
},
"databricks/databricks-gpt-5-1": {
"cache_creation_input_token_cost": 1.24999e-06,
"cache_read_input_token_cost": 1.2502e-07,
"input_cost_per_token": 1.24999e-06,
"input_dbu_cost_per_token": 1.7857e-05,
"litellm_provider": "databricks",
@ -14901,9 +15111,12 @@
"mode": "chat",
"output_cost_per_token": 9.999990000000002e-06,
"output_dbu_cost_per_token": 0.000142857,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"supports_prompt_caching": true
},
"databricks/databricks-gpt-5-1-codex-max": {
"cache_creation_input_token_cost": 1.24999e-06,
"cache_read_input_token_cost": 1.2502e-07,
"input_cost_per_token": 1.24999e-06,
"input_dbu_cost_per_token": 1.7857e-05,
"litellm_provider": "databricks",
@ -14916,9 +15129,12 @@
"mode": "chat",
"output_cost_per_token": 9.999990000000002e-06,
"output_dbu_cost_per_token": 0.000142857,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"supports_prompt_caching": true
},
"databricks/databricks-gpt-5-1-codex-mini": {
"cache_creation_input_token_cost": 2.4997e-07,
"cache_read_input_token_cost": 2.499e-08,
"input_cost_per_token": 2.4997e-07,
"input_dbu_cost_per_token": 3.571e-06,
"litellm_provider": "databricks",
@ -14931,9 +15147,12 @@
"mode": "chat",
"output_cost_per_token": 1.99997e-06,
"output_dbu_cost_per_token": 2.8571e-05,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"supports_prompt_caching": true
},
"databricks/databricks-gpt-5-2": {
"cache_creation_input_token_cost": 1.75e-06,
"cache_read_input_token_cost": 1.75e-07,
"input_cost_per_token": 1.75e-06,
"input_dbu_cost_per_token": 2.5e-05,
"litellm_provider": "databricks",
@ -14946,9 +15165,12 @@
"mode": "chat",
"output_cost_per_token": 1.4e-05,
"output_dbu_cost_per_token": 0.0002,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"supports_prompt_caching": true
},
"databricks/databricks-gpt-5-2-codex": {
"cache_creation_input_token_cost": 1.75e-06,
"cache_read_input_token_cost": 1.75e-07,
"input_cost_per_token": 1.75e-06,
"input_dbu_cost_per_token": 2.5e-05,
"litellm_provider": "databricks",
@ -14961,9 +15183,12 @@
"mode": "chat",
"output_cost_per_token": 1.4e-05,
"output_dbu_cost_per_token": 0.0002,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"supports_prompt_caching": true
},
"databricks/databricks-gpt-5-3-codex": {
"cache_creation_input_token_cost": 1.75e-06,
"cache_read_input_token_cost": 1.75e-07,
"input_cost_per_token": 1.75e-06,
"input_dbu_cost_per_token": 2.5e-05,
"litellm_provider": "databricks",
@ -14976,9 +15201,12 @@
"mode": "chat",
"output_cost_per_token": 1.4e-05,
"output_dbu_cost_per_token": 0.0002,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"supports_prompt_caching": true
},
"databricks/databricks-gpt-5-4": {
"cache_creation_input_token_cost": 2.49998e-06,
"cache_read_input_token_cost": 2.4997e-07,
"input_cost_per_token": 2.49998e-06,
"input_dbu_cost_per_token": 3.5714e-05,
"litellm_provider": "databricks",
@ -14991,9 +15219,12 @@
"mode": "chat",
"output_cost_per_token": 1.5000020000000002e-05,
"output_dbu_cost_per_token": 0.000214286,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"supports_prompt_caching": true
},
"databricks/databricks-gpt-5-4-mini": {
"cache_creation_input_token_cost": 7.4998e-07,
"cache_read_input_token_cost": 7.497e-08,
"input_cost_per_token": 7.4998e-07,
"input_dbu_cost_per_token": 1.0714e-05,
"litellm_provider": "databricks",
@ -15006,9 +15237,12 @@
"mode": "chat",
"output_cost_per_token": 4.50002e-06,
"output_dbu_cost_per_token": 6.4286e-05,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"supports_prompt_caching": true
},
"databricks/databricks-gpt-5-4-nano": {
"cache_creation_input_token_cost": 1.9999e-07,
"cache_read_input_token_cost": 2.002e-08,
"input_cost_per_token": 1.9999e-07,
"input_dbu_cost_per_token": 2.857e-06,
"litellm_provider": "databricks",
@ -15021,9 +15255,12 @@
"mode": "chat",
"output_cost_per_token": 1.24999e-06,
"output_dbu_cost_per_token": 1.7857e-05,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"supports_prompt_caching": true
},
"databricks/databricks-gpt-5-mini": {
"cache_creation_input_token_cost": 2.4997e-07,
"cache_read_input_token_cost": 2.499e-08,
"input_cost_per_token": 2.4997000000000006e-07,
"input_dbu_cost_per_token": 3.571e-06,
"litellm_provider": "databricks",
@ -15036,9 +15273,12 @@
"mode": "chat",
"output_cost_per_token": 1.9999700000000004e-06,
"output_dbu_cost_per_token": 2.8571e-05,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"supports_prompt_caching": true
},
"databricks/databricks-gpt-5-nano": {
"cache_creation_input_token_cost": 4.998e-08,
"cache_read_input_token_cost": 4.97e-09,
"input_cost_per_token": 4.998e-08,
"input_dbu_cost_per_token": 7.14e-07,
"litellm_provider": "databricks",
@ -15051,9 +15291,12 @@
"mode": "chat",
"output_cost_per_token": 3.9998000000000007e-07,
"output_dbu_cost_per_token": 5.714000000000001e-06,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"supports_prompt_caching": true
},
"databricks/databricks-gpt-oss-120b": {
"cache_creation_input_token_cost": 1.5001e-07,
"cache_read_input_token_cost": 1.5001e-07,
"input_cost_per_token": 1.5000999999999998e-07,
"input_dbu_cost_per_token": 2.1429999999999996e-06,
"litellm_provider": "databricks",
@ -15069,6 +15312,8 @@
"source": "https://www.databricks.com/product/pricing/foundation-model-serving"
},
"databricks/databricks-gpt-oss-20b": {
"cache_creation_input_token_cost": 7e-08,
"cache_read_input_token_cost": 7e-08,
"input_cost_per_token": 7e-08,
"input_dbu_cost_per_token": 1e-06,
"litellm_provider": "databricks",
@ -15084,6 +15329,8 @@
"source": "https://www.databricks.com/product/pricing/foundation-model-serving"
},
"databricks/databricks-gte-large-en": {
"cache_creation_input_token_cost": 1.2999e-07,
"cache_read_input_token_cost": 1.2999e-07,
"input_cost_per_token": 1.2999000000000001e-07,
"input_dbu_cost_per_token": 1.857e-06,
"litellm_provider": "databricks",
@ -15099,6 +15346,8 @@
"source": "https://www.databricks.com/product/pricing/foundation-model-serving"
},
"databricks/databricks-llama-2-70b-chat": {
"cache_creation_input_token_cost": 5.0001e-07,
"cache_read_input_token_cost": 5.0001e-07,
"input_cost_per_token": 5.0001e-07,
"input_dbu_cost_per_token": 7.143e-06,
"litellm_provider": "databricks",
@ -15115,6 +15364,8 @@
"supports_tool_choice": true
},
"databricks/databricks-llama-4-maverick": {
"cache_creation_input_token_cost": 5.0001e-07,
"cache_read_input_token_cost": 5.0001e-07,
"input_cost_per_token": 5.0001e-07,
"input_dbu_cost_per_token": 7.143e-06,
"litellm_provider": "databricks",
@ -15131,6 +15382,8 @@
"supports_tool_choice": true
},
"databricks/databricks-meta-llama-3-1-405b-instruct": {
"cache_creation_input_token_cost": 5.00003e-06,
"cache_read_input_token_cost": 5.00003e-06,
"input_cost_per_token": 5.00003e-06,
"input_dbu_cost_per_token": 7.1429e-05,
"litellm_provider": "databricks",
@ -15147,6 +15400,8 @@
"supports_tool_choice": true
},
"databricks/databricks-meta-llama-3-1-8b-instruct": {
"cache_creation_input_token_cost": 1.5001e-07,
"cache_read_input_token_cost": 1.5001e-07,
"input_cost_per_token": 1.5000999999999998e-07,
"input_dbu_cost_per_token": 2.1429999999999996e-06,
"litellm_provider": "databricks",
@ -15162,6 +15417,8 @@
"source": "https://www.databricks.com/product/pricing/foundation-model-serving"
},
"databricks/databricks-meta-llama-3-3-70b-instruct": {
"cache_creation_input_token_cost": 5.0001e-07,
"cache_read_input_token_cost": 5.0001e-07,
"input_cost_per_token": 5.0001e-07,
"input_dbu_cost_per_token": 7.143e-06,
"litellm_provider": "databricks",
@ -15178,6 +15435,8 @@
"supports_tool_choice": true
},
"databricks/databricks-meta-llama-3-70b-instruct": {
"cache_creation_input_token_cost": 1.00002e-06,
"cache_read_input_token_cost": 1.00002e-06,
"input_cost_per_token": 1.00002e-06,
"input_dbu_cost_per_token": 1.4286e-05,
"litellm_provider": "databricks",
@ -15194,6 +15453,8 @@
"supports_tool_choice": true
},
"databricks/databricks-mixtral-8x7b-instruct": {
"cache_creation_input_token_cost": 5.0001e-07,
"cache_read_input_token_cost": 5.0001e-07,
"input_cost_per_token": 5.0001e-07,
"input_dbu_cost_per_token": 7.143e-06,
"litellm_provider": "databricks",
@ -15210,6 +15471,8 @@
"supports_tool_choice": true
},
"databricks/databricks-mpt-30b-instruct": {
"cache_creation_input_token_cost": 1.00002e-06,
"cache_read_input_token_cost": 1.00002e-06,
"input_cost_per_token": 1.00002e-06,
"input_dbu_cost_per_token": 1.4286e-05,
"litellm_provider": "databricks",
@ -15226,6 +15489,8 @@
"supports_tool_choice": true
},
"databricks/databricks-mpt-7b-instruct": {
"cache_creation_input_token_cost": 5.0001e-07,
"cache_read_input_token_cost": 5.0001e-07,
"input_cost_per_token": 5.0001e-07,
"input_dbu_cost_per_token": 7.143e-06,
"litellm_provider": "databricks",
@ -16890,6 +17155,14 @@
"notes": "Web Search on Amazon Bedrock AgentCore, billed by AWS on the gateway"
}
},
"bing_grounding/search": {
"input_cost_per_query": 0.035,
"litellm_provider": "bing_grounding",
"mode": "search",
"metadata": {
"notes": "Grounding with Bing Search (G1 SKU): $35 per 1,000 transactions. Tokens for the Foundry model deployment that runs the grounded search are billed separately on that deployment."
}
},
"tinyfish/search": {
"input_cost_per_query": 0.0,
"litellm_provider": "tinyfish",
@ -23180,6 +23453,7 @@
},
"github_copilot/claude-opus-4.6-fast": {
"supports_adaptive_thinking": true,
"supports_legacy_thinking": true,
"litellm_provider": "github_copilot",
"max_input_tokens": 128000,
"max_output_tokens": 16000,
@ -33563,6 +33837,7 @@
},
"openrouter/anthropic/claude-sonnet-4.6": {
"supports_adaptive_thinking": true,
"supports_legacy_thinking": true,
"cache_creation_input_token_cost": 3.75e-06,
"cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
"cache_read_input_token_cost": 3e-07,
@ -33607,6 +33882,7 @@
},
"openrouter/anthropic/claude-opus-4.6": {
"supports_adaptive_thinking": true,
"supports_legacy_thinking": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_read_input_token_cost": 5e-07,
"input_cost_per_token": 5e-06,
@ -35681,6 +35957,7 @@
},
"perplexity/anthropic/claude-opus-4-6": {
"supports_adaptive_thinking": true,
"supports_legacy_thinking": true,
"litellm_provider": "perplexity",
"mode": "responses",
"supports_web_search": true,
@ -39056,6 +39333,7 @@
},
"vercel_ai_gateway/anthropic/claude-opus-4.6": {
"supports_adaptive_thinking": true,
"supports_legacy_thinking": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_read_input_token_cost": 5e-07,
"input_cost_per_token": 5e-06,
@ -40315,6 +40593,7 @@
"deprecation_date": "2027-02-05",
"regional_endpoint_uplift_multiplier": 1.1,
"supports_adaptive_thinking": true,
"supports_legacy_thinking": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
"cache_read_input_token_cost": 5e-07,
@ -40347,6 +40626,7 @@
"deprecation_date": "2027-02-05",
"regional_endpoint_uplift_multiplier": 1.1,
"supports_adaptive_thinking": true,
"supports_legacy_thinking": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
"cache_read_input_token_cost": 5e-07,
@ -40712,6 +40992,7 @@
"vertex_ai/claude-sonnet-4-6": {
"regional_endpoint_uplift_multiplier": 1.1,
"supports_adaptive_thinking": true,
"supports_legacy_thinking": true,
"cache_creation_input_token_cost": 3.75e-06,
"cache_creation_input_token_cost_above_1hr": 6e-06,
"cache_read_input_token_cost": 3e-07,
@ -43795,10 +44076,10 @@
"comment": "5 credits per second @ $0.01 per credit = $0.05 per second"
}
},
"runwayml/gen4_aleph": {
"runwayml/gen4.5": {
"litellm_provider": "runwayml",
"mode": "video_generation",
"output_cost_per_video_per_second": 0.15,
"output_cost_per_second": 0.12,
"source": "https://docs.dev.runwayml.com/guides/pricing/",
"supported_modalities": [
"text",
@ -43808,13 +44089,136 @@
"video"
],
"metadata": {
"comment": "15 credits per second @ $0.01 per credit = $0.15 per second"
"comment": "12 credits per second @ $0.01 per credit = $0.12 per second"
}
},
"runwayml/gen3a_turbo": {
"runwayml/aleph2": {
"litellm_provider": "runwayml",
"mode": "video_generation",
"output_cost_per_video_per_second": 0.05,
"output_cost_per_second": 0.28,
"source": "https://docs.dev.runwayml.com/guides/pricing/",
"supported_modalities": [
"text",
"video"
],
"supported_output_modalities": [
"video"
],
"metadata": {
"comment": "28 credits per second @ $0.01 per credit = $0.28 per second; 56 credit minimum per task not modeled"
}
},
"runwayml/seedance2": {
"litellm_provider": "runwayml",
"mode": "video_generation",
"output_cost_per_second": 0.36,
"output_cost_per_second_1080p": 0.4,
"output_cost_per_second_4k": 1.5,
"source": "https://docs.dev.runwayml.com/guides/pricing/",
"supported_modalities": [
"text",
"image",
"video"
],
"supported_output_modalities": [
"video"
],
"metadata": {
"comment": "36 credits per second at 480p/720p, 40 at 1080p, 150 at 4K @ $0.01 per credit"
}
},
"runwayml/seedance2_fast": {
"litellm_provider": "runwayml",
"mode": "video_generation",
"output_cost_per_second": 0.29,
"source": "https://docs.dev.runwayml.com/guides/pricing/",
"supported_modalities": [
"text",
"image",
"video"
],
"supported_output_modalities": [
"video"
],
"metadata": {
"comment": "29 credits per second at 480p/720p @ $0.01 per credit = $0.29 per second"
}
},
"runwayml/seedance2_mini": {
"litellm_provider": "runwayml",
"mode": "video_generation",
"output_cost_per_second": 0.16,
"source": "https://docs.dev.runwayml.com/guides/pricing/",
"supported_modalities": [
"text",
"image",
"video"
],
"supported_output_modalities": [
"video"
],
"metadata": {
"comment": "16 credits per second @ $0.01 per credit = $0.16 per second; 64 credit minimum per task not modeled"
}
},
"runwayml/seedance2_5": {
"litellm_provider": "runwayml",
"mode": "video_generation",
"output_cost_per_second": 0.3,
"output_cost_per_second_480p": 0.2,
"output_cost_per_second_1080p": 0.68,
"source": "https://docs.dev.runwayml.com/guides/pricing/",
"supported_modalities": [
"text",
"image",
"video"
],
"supported_output_modalities": [
"video"
],
"metadata": {
"comment": "Output: 20/30/68 credits per second at 480p/720p/1080p @ $0.01 per credit; input video billed additionally at 10/15/34 credits per input second and the 80 credit minimum per task are not modeled"
}
},
"runwayml/hailuo3": {
"litellm_provider": "runwayml",
"mode": "video_generation",
"output_cost_per_second": 0.1,
"output_cost_per_second_1080p": 0.15,
"source": "https://docs.dev.runwayml.com/guides/pricing/",
"supported_modalities": [
"text",
"image",
"video"
],
"supported_output_modalities": [
"video"
],
"metadata": {
"comment": "10 credits per second at 768P, 15 at 2K (mapped to the 1080p tier) @ $0.01 per credit; 2 credits per reference image not modeled"
}
},
"runwayml/gemini_omni_flash": {
"litellm_provider": "runwayml",
"mode": "video_generation",
"output_cost_per_second": 0.1,
"source": "https://docs.dev.runwayml.com/guides/pricing/",
"supported_modalities": [
"text",
"image",
"video"
],
"supported_output_modalities": [
"video"
],
"metadata": {
"comment": "10 credits per second @ $0.01 per credit = $0.10 per second"
}
},
"runwayml/veo3.1": {
"litellm_provider": "runwayml",
"mode": "video_generation",
"output_cost_per_second": 0.4,
"source": "https://docs.dev.runwayml.com/guides/pricing/",
"supported_modalities": [
"text",
@ -43824,7 +44228,23 @@
"video"
],
"metadata": {
"comment": "5 credits per second @ $0.01 per credit = $0.05 per second"
"comment": "40 credits per second with audio, 20 without @ $0.01 per credit; priced at the with-audio rate"
}
},
"runwayml/veo3.1_fast": {
"litellm_provider": "runwayml",
"mode": "video_generation",
"output_cost_per_second": 0.15,
"source": "https://docs.dev.runwayml.com/guides/pricing/",
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"video"
],
"metadata": {
"comment": "15 credits per second with audio, 10 without @ $0.01 per credit; priced at the with-audio rate"
}
},
"runwayml/gen4_image": {
@ -48481,6 +48901,7 @@
"vertex_ai/claude-sonnet-4-6@default": {
"regional_endpoint_uplift_multiplier": 1.1,
"supports_adaptive_thinking": true,
"supports_legacy_thinking": true,
"cache_creation_input_token_cost": 3.75e-06,
"cache_creation_input_token_cost_above_1hr": 6e-06,
"cache_read_input_token_cost": 3e-07,
@ -49295,6 +49716,7 @@
},
"snowflake/claude-sonnet-4-6": {
"supports_adaptive_thinking": true,
"supports_legacy_thinking": true,
"max_tokens": 16384,
"max_input_tokens": 200000,
"max_output_tokens": 16384,
@ -50285,6 +50707,14 @@
"supports_adaptive_thinking": true
}
},
{
"name": "claude-legacy-thinking",
"pattern": "claude-[a-z]+-4[-._]6(?!\\d)",
"description": "Claude at version 4.6 exactly, in any id shape that contains claude-<family>-4-6 (dotted and underscored minors included, dated releases such as claude-sonnet-4-6-20260219 too). The 4.6 family is adaptive-thinking yet still accepts legacy thinking.type=enabled with budget_tokens, so the caller's hard budget cap is forwarded verbatim instead of being rewritten to an uncapped output_config.effort. The lookahead keeps two-digit minors such as 4-60 from matching. 4.7+ and 5+ majors reject the legacy shape and stay on the adaptive translation.",
"model_info": {
"supports_legacy_thinking": true
}
},
{
"name": "claude-always-on-thinking",
"pattern": "claude-(?:fable|mythos)-",

39
litellm/proxy/analytics_endpoints/cache_activity.py
View file

@ -34,10 +34,18 @@ class CacheActivityFilterOptions(BaseModel):
models: list[str]
class CacheActivityErrorBucket(BaseModel):
call_type: str
error_code: str
error_class: str
count: int
class CacheActivityResponse(BaseModel):
groups: list[CacheActivityGroup]
totals: CacheActivityTotals
filter_options: CacheActivityFilterOptions
error_breakdown: tuple[CacheActivityErrorBucket, ...]
GROUPS_SQL: Final = """
@ -65,6 +73,26 @@ GROUPS_SQL: Final = """
ORDER BY (COUNT(*)) DESC
"""
ERROR_BREAKDOWN_SQL: Final = """
SELECT
CASE WHEN sl."call_type" = '' THEN 'Unknown' ELSE sl."call_type" END AS call_type,
COALESCE(NULLIF(sl."metadata"->'error_information'->>'error_code', ''), 'Unknown') AS error_code,
COALESCE(NULLIF(sl."metadata"->'error_information'->>'error_class', ''), 'Unknown') AS error_class,
COUNT(*)::int AS count
FROM "LiteLLM_SpendLogs" sl
LEFT JOIN "LiteLLM_VerificationToken" vt ON sl."api_key" = vt."token"
WHERE
sl."status" = 'failure'
AND sl."startTime" >= ($1::timestamptz AT TIME ZONE 'UTC')
AND sl."startTime" < (($2::timestamptz + INTERVAL '1 day') AT TIME ZONE 'UTC')
AND ($3::jsonb = '[]'::jsonb
OR COALESCE(vt."key_alias", 'Unnamed Key') IN (SELECT jsonb_array_elements_text($3::jsonb)))
AND ($4::jsonb = '[]'::jsonb
OR sl."model" IN (SELECT jsonb_array_elements_text($4::jsonb)))
GROUP BY 1, 2, 3
ORDER BY (COUNT(*)) DESC
"""
KEY_ALIAS_OPTIONS_SQL: Final = """
SELECT DISTINCT COALESCE(vt."key_alias", 'Unnamed Key') AS key_alias
FROM "LiteLLM_SpendLogs" sl
@ -95,6 +123,7 @@ class _ModelRow(BaseModel):
_groups_adapter: Final = TypeAdapter(list[CacheActivityGroup])
_error_buckets_adapter: Final = TypeAdapter(tuple[CacheActivityErrorBucket, ...])
_key_alias_rows_adapter: Final = TypeAdapter(list[_KeyAliasRow])
_model_rows_adapter: Final = TypeAdapter(list[_ModelRow])
@ -120,10 +149,11 @@ async def get_cache_activity(
key_aliases: Sequence[str],
models: Sequence[str],
) -> CacheActivityResponse:
group_rows, key_alias_rows, model_rows = await asyncio.gather(
prisma_client.db.query_raw(
GROUPS_SQL, start_date, end_date, json.dumps(list(key_aliases)), json.dumps(list(models))
),
key_aliases_json: Final = json.dumps(list(key_aliases))
models_json: Final = json.dumps(list(models))
group_rows, error_rows, key_alias_rows, model_rows = await asyncio.gather(
prisma_client.db.query_raw(GROUPS_SQL, start_date, end_date, key_aliases_json, models_json),
prisma_client.db.query_raw(ERROR_BREAKDOWN_SQL, start_date, end_date, key_aliases_json, models_json),
prisma_client.db.query_raw(KEY_ALIAS_OPTIONS_SQL, start_date, end_date),
prisma_client.db.query_raw(MODEL_OPTIONS_SQL, start_date, end_date),
)
@ -135,4 +165,5 @@ async def get_cache_activity(
key_aliases=[row.key_alias for row in _key_alias_rows_adapter.validate_python(key_alias_rows or [])],
models=[row.model for row in _model_rows_adapter.validate_python(model_rows or [])],
),
error_breakdown=_error_buckets_adapter.validate_python(error_rows or []),
)

52
litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py
View file

@ -6,21 +6,21 @@ Plugins are stored as metadata + git source references in LiteLLM database.
Actual plugin files are hosted on GitHub/GitLab/Bitbucket.
Endpoints:
/claude-code/marketplace.json - GET - List plugins for Claude Code discovery
/claude-code/plugins - POST - Register a new plugin (create-only)
/claude-code/plugins - GET - List plugins (admin)
/claude-code/plugins/{name} - GET - Get plugin details
/claude-code/plugins/{name} - PUT - Update an existing plugin
/claude-code/plugins/{name}/enable - POST - Enable a plugin
/claude-code/plugins/{name}/disable - POST - Disable a plugin
/claude-code/plugins/{name} - DELETE - Delete a plugin
/claude-code/marketplace.json - GET - List plugins for Claude Code discovery (unauthenticated)
/claude-code/plugins - POST - Register a new plugin (create-only, proxy admin only)
/claude-code/plugins - GET - List plugins (any authenticated key)
/claude-code/plugins/{name} - GET - Get plugin details (any authenticated key)
/claude-code/plugins/{name} - PUT - Update an existing plugin (proxy admin only)
/claude-code/plugins/{name}/enable - POST - Enable a plugin (proxy admin only)
/claude-code/plugins/{name}/disable - POST - Disable a plugin (proxy admin only)
/claude-code/plugins/{name} - DELETE - Delete a plugin (proxy admin only)
"""
import json
import re
from collections.abc import Mapping, Sequence
from datetime import datetime, timezone
from typing import Final, Protocol, TypedDict
from typing import Annotated, Final, Protocol, TypedDict
from fastapi import APIRouter, Depends, HTTPException
from fastapi.responses import JSONResponse
@ -28,6 +28,7 @@ from fastapi.responses import JSONResponse
from litellm._logging import verbose_proxy_logger
from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils.resource_ownership import is_proxy_admin
from litellm.repositories.table_repositories import ClaudeCodePluginRepository
from litellm.types.proxy.claude_code_endpoints import (
ListPluginsResponse,
@ -221,6 +222,18 @@ def _name_conflict_error(name: str) -> HTTPException:
)
def _require_proxy_admin(user_api_key_dict: UserAPIKeyAuth) -> None:
"""Catalog mutations are restricted to proxy admins: marketplace.json is served
unauthenticated and any registered/updated entry is immediately installable by
every user, so a non-admin key must never be able to add or overwrite one.
"""
if not is_proxy_admin(user_api_key_dict):
raise HTTPException(
status_code=403,
detail={"error": "Only proxy admins may modify the Claude Code plugin marketplace."},
)
@router.post(
"/claude-code/plugins",
tags=["Claude Code Marketplace"],
@ -242,6 +255,8 @@ async def register_plugin(
the same name already exists it returns 409 Conflict; use
PUT /claude-code/plugins/{plugin_name} to update an existing plugin.
Requires a proxy admin API key.
Parameters:
- name: Plugin name (kebab-case)
- source: Git source reference (github, url, or git-subdir format)
@ -271,6 +286,8 @@ async def register_plugin(
from prisma.errors import UniqueViolationError
try:
_require_proxy_admin(user_api_key_dict)
prisma_client: Final = await _get_prisma_client()
if not re.match(r"^[a-z0-9-]+$", request.name):
@ -468,6 +485,7 @@ async def get_plugin(
async def update_plugin(
plugin_name: str,
request: UpdatePluginRequest,
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
):
"""
Update an existing plugin in the LiteLLM marketplace.
@ -481,6 +499,8 @@ async def update_plugin(
Returns 404 if no plugin with the given name exists; use
POST /claude-code/plugins to create a new plugin.
Requires a proxy admin API key.
Parameters:
- plugin_name: Name of the plugin to update (path parameter)
- source: Git source reference (github, url, or git-subdir format)
@ -509,6 +529,8 @@ async def update_plugin(
from prisma.errors import PrismaError
try:
_require_proxy_admin(user_api_key_dict)
prisma_client: Final = await _get_prisma_client()
_validate_plugin_source(request.source)
@ -566,10 +588,14 @@ async def enable_plugin(
"""
Enable a disabled plugin.
Requires a proxy admin API key.
Parameters:
- plugin_name: The name of the plugin to enable
"""
try:
_require_proxy_admin(user_api_key_dict)
prisma_client: Final = await _get_prisma_client()
plugin: Final[_PluginRecord | None] = await ClaudeCodePluginRepository(prisma_client).table.find_unique(
@ -611,10 +637,14 @@ async def disable_plugin(
"""
Disable a plugin without deleting it.
Requires a proxy admin API key.
Parameters:
- plugin_name: The name of the plugin to disable
"""
try:
_require_proxy_admin(user_api_key_dict)
prisma_client: Final = await _get_prisma_client()
plugin: Final[_PluginRecord | None] = await ClaudeCodePluginRepository(prisma_client).table.find_unique(
@ -656,10 +686,14 @@ async def delete_plugin(
"""
Delete a plugin from the marketplace.
Requires a proxy admin API key.
Parameters:
- plugin_name: The name of the plugin to delete
"""
try:
_require_proxy_admin(user_api_key_dict)
prisma_client: Final = await _get_prisma_client()
plugin: Final[_PluginRecord | None] = await ClaudeCodePluginRepository(prisma_client).table.find_unique(

77
litellm/proxy/common_utils/callback_config_validation.py Normal file
View file

@ -0,0 +1,77 @@
"""Save-time validation of team/key logging configs the runtime cannot honor.
Team callbacks arrive as a single ``AddTeamCallback``, key callbacks arrive as a
``logging`` list inside the key metadata, so both shapes funnel into the same
per-integration checks here.
"""
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Final
_NEWRELIC_CALLBACK: Final = "newrelic"
_NEWRELIC_VAR_PREFIX: Final = "newrelic_"
def callback_config_error(callback_name: str | None, callback_vars: Mapping[str, str] | None) -> str | None:
if callback_name != _NEWRELIC_CALLBACK or not callback_vars:
return None
return _newrelic_config_error(callback_vars)
def logging_metadata_config_error(metadata: Mapping[str, object] | None) -> str | None:
"""Validate every ``logging`` entry of a team/key metadata payload."""
if not metadata:
return None
entries: Final = metadata.get("logging")
if not isinstance(entries, Sequence) or isinstance(entries, (str, bytes)):
return None
return next(
(error for error in (_logging_entry_error(entry) for entry in entries) if error is not None),
None,
)
def _logging_entry_error(entry: object) -> str | None:
if not isinstance(entry, Mapping):
return None
callback_name: Final = entry.get("callback_name")
callback_vars: Final = entry.get("callback_vars")
if not isinstance(callback_name, str) or not isinstance(callback_vars, Mapping):
return None
return callback_config_error(
callback_name,
MappingProxyType({str(key): str(value) for key, value in callback_vars.items()}),
)
def _newrelic_config_error(callback_vars: Mapping[str, str]) -> str | None:
"""Per-team New Relic routing runs on the OTel v2 path only.
Accepting the config with the flag off would silently ship the team's traffic
through the operator's env-configured agent instead of the team's account. A
region outside the fixed table, or a region without a key, would likewise be
accepted and then silently ignored or misrouted at request time.
"""
if not any(key.startswith(_NEWRELIC_VAR_PREFIX) for key in callback_vars):
return None
from litellm.integrations.otel.model.config import is_otel_v2_enabled
from litellm.integrations.otel.presets.newrelic import NEWRELIC_OTLP_ENDPOINT_BY_REGION
if not is_otel_v2_enabled():
return "Per-team New Relic routing requires the proxy to run with LITELLM_OTEL_V2=true."
region: Final = callback_vars.get("newrelic_region")
if region is not None and region.lower() not in NEWRELIC_OTLP_ENDPOINT_BY_REGION:
return (
f"Unknown newrelic_region {region!r}. "
f"Supported regions: {', '.join(sorted(NEWRELIC_OTLP_ENDPOINT_BY_REGION))}."
)
# ``callback_vars`` values are str()-coerced upstream, so a JSON ``null`` key
# arrives as the literal ``"None"``; treat that and the empty string as absent.
api_key: Final = callback_vars.get("newrelic_api_key")
if region is not None and (not api_key or api_key == "None"):
return "newrelic_region requires newrelic_api_key; the region rides the team's own key."
return None

4
litellm/proxy/common_utils/http_parsing_utils.py
View file

@ -274,10 +274,8 @@ async def get_form_data(request: Request) -> dict[str, Any]:
Handles when OpenAI SDKs pass form keys as `timestamp_granularities[]="word"` instead of `timestamp_granularities=["word", "sentence"]`
"""
form: Final = await request.form()
form_data: Final = dict(form)
parsed_form_data: Final[dict[str, Any]] = {}
for key, value in form_data.items():
# OpenAI SDKs pass form keys as `timestamp_granularities[]="word"` instead of `timestamp_granularities=["word", "sentence"]`
for key, value in form.multi_items(): # not dict(form), which keeps only the last repeat
if key.endswith("[]"):
clean_key = key[:-2]
parsed_form_data.setdefault(clean_key, []).append(value)

30
litellm/proxy/example_config_yaml/bing_grounding_websearch_config.yaml Normal file
View file

@ -0,0 +1,30 @@
# Web search via Microsoft Foundry (Grounding with Bing Search / the built-in
# web_search tool), called through the Foundry Responses API.
#
# Configure the provider with env vars (setup and pricing are in the LiteLLM docs;
# the code lives in litellm/llms/azure/search/transformation.py):
# BING_GROUNDING_PROJECT_ENDPOINT (required) the Foundry project endpoint
# BING_GROUNDING_MODEL (required) a model deployment in that project
# BING_GROUNDING_CONNECTION_ID (optional) a Grounding with Bing connection id;
# without it the built-in web_search tool is used
# BING_GROUNDING_TOKEN (optional) an Entra bearer token; without it (and
# without api_key) azure-identity mints one
model_list:
- model_name: claude-sonnet
litellm_params:
model: bedrock/us.anthropic.claude-sonnet-5
aws_region_name: us-east-1
search_tools:
- search_tool_name: bing-grounding-search
litellm_params:
search_provider: bing_grounding
# Optional: an Azure API key instead of BING_GROUNDING_TOKEN / azure-identity
# api_key: os.environ/AZURE_AI_API_KEY
litellm_settings:
callbacks: ["websearch_interception"]
websearch_interception_params:
enabled_providers: ["bedrock"]
search_tool_name: bing-grounding-search

35
litellm/proxy/health_check.py
View file

@ -18,6 +18,7 @@ from litellm.constants import (
DEFAULT_HEALTH_CHECK_PROMPT,
HEALTH_CHECK_TIMEOUT_SECONDS,
)
from litellm.router_utils.auto_router_model_naming import classify_strategy_router_model
ILLEGAL_DISPLAY_PARAMS: Final = [
"messages",
@ -182,30 +183,17 @@ async def run_with_timeout(task, timeout):
return {"error": "Timeout exceeded", "exception": timeout_exception}
def _is_semantic_auto_router_deployment(litellm_params: dict) -> bool:
"""
True for semantic auto_router deployments (auto_router/<name>) that are not
sub-strategies (complexity_router, adaptive_router, quality_router).
These are meta-routers that select among real LLM deployments at request time;
they have no LLM endpoint to health-check.
"""
def _is_strategy_router_deployment(litellm_params: dict) -> bool:
"""True for strategy-router deployments."""
model: Final[object] = litellm_params.get("model", "")
if not isinstance(model, str):
return False
if not model.startswith("auto_router/"):
return False
for sub_strategy in ("complexity_router", "adaptive_router", "quality_router"):
if model.startswith(f"auto_router/{sub_strategy}"):
return False
return True
return isinstance(model, str) and classify_strategy_router_model(model) is not None
async def _run_model_health_check(model: dict):
litellm_params = model["litellm_params"]
model_info: Final = model.get("model_info", {})
if _is_semantic_auto_router_deployment(litellm_params):
if _is_strategy_router_deployment(litellm_params):
return {}
mode: Final = _resolve_health_check_mode(
@ -445,6 +433,9 @@ def _update_litellm_params_for_health_check(model_info: dict, litellm_params: di
"""
Update the litellm params for health check.
- merges `model_info.health_check_params` into the probe request, so a deployment whose provider
requires a payload field litellm does not synthesize (e.g. `mediaSource` for Bedrock TwelveLabs
Pegasus) can supply it. The dedicated knobs below are applied afterwards and win on conflict.
- gets a short `messages` param for health check
- adds a bounded `max_tokens` when the deployment is a chat-style mode
(`chat`, `completion`, `responses`) or the operator explicitly opts in
@ -459,6 +450,16 @@ def _update_litellm_params_for_health_check(model_info: dict, litellm_params: di
model_info,
litellm_params, # any-ok: untyped router config dict
)
_health_check_params: Final = model_info.get("health_check_params", None)
if isinstance(_health_check_params, dict):
litellm_params.update(_health_check_params)
elif _health_check_params is not None:
logger.warning(
"health_check_params for model %s is a %s, expected a dict. Ignoring it.",
litellm_params.get("model"),
type(_health_check_params).__name__,
)
litellm_params["messages"] = _get_random_llm_message()
if _should_inject_health_check_max_tokens(
model_info,

18
litellm/proxy/health_endpoints/_health_endpoints.py
View file

@ -1888,6 +1888,8 @@ async def test_model_connection(
# already resolved before reaching this endpoint; any remaining
# reference must have come from the request body.
_reject_os_environ_references(request_litellm_params)
if model_info:
_reject_os_environ_references(model_info)
model_name: Final = request_litellm_params.get("model")
# Look up model configuration from router if model name is provided
@ -1950,23 +1952,23 @@ async def test_model_connection(
**request_litellm_params,
}
## Auth check
auth_model_info: Final = loaded_model_info if loaded_model_info is not None else model_info
resolved_model_info: Final = loaded_model_info if loaded_model_info is not None else model_info
litellm_params = _update_litellm_params_for_health_check(
model_info=resolved_model_info or {},
litellm_params=litellm_params,
)
## Auth check, on the final probe params so health_check_params cannot retarget it afterwards
await ModelManagementAuthChecks.can_user_make_model_call(
model_params=Deployment(
model_name="test_model",
litellm_params=LiteLLM_Params(**litellm_params),
model_info=auth_model_info,
model_info=resolved_model_info,
),
user_api_key_dict=user_api_key_dict,
prisma_client=prisma_client,
premium_user=premium_user,
)
# Include health_check_params if provided
litellm_params = _update_litellm_params_for_health_check(
model_info={},
litellm_params=litellm_params,
)
mode = mode or litellm_params.pop("mode", None)
result: Final = await run_with_timeout(

34
litellm/proxy/hooks/proxy_track_cost_callback.py
View file

@ -5,6 +5,7 @@ from typing import Any, Final, cast
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import BACKGROUND_INTERACTION_COST_POLLING_ENABLED
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import (
_get_parent_otel_span_from_kwargs,
@ -318,6 +319,21 @@ class _ProxyDBLogger(CustomLogger):
elif budget_reservation is not None:
await _release_budget_reservation(budget_reservation=budget_reservation)
else:
if _is_unbilled_interaction_response(completion_response):
if BACKGROUND_INTERACTION_COST_POLLING_ENABLED and _is_unbilled_in_progress_interaction(
completion_response
):
verbose_proxy_logger.debug(
"Cost tracking deferred for in-progress background interaction; "
"the budget reservation stays open until the poll task logs the final usage"
)
return
await _release_budget_reservation(budget_reservation=budget_reservation)
verbose_proxy_logger.debug(
"Released the budget reservation for an interaction create with no usage "
"that no poll task will settle"
)
return
await _release_budget_reservation(budget_reservation=budget_reservation)
# Non-model call types (health checks, afile_delete) have no model or standard_logging_object.
# Use .get() for "stream" to avoid KeyError on health checks.
@ -463,6 +479,24 @@ def _write_spend_metadata_to_kwargs(kwargs: dict, metadata: dict) -> None:
bucket[key] = value
def _is_unbilled_interaction_response(completion_response: object) -> bool:
from litellm.interactions.background_cost_polling import missing_usage_is_expected
from litellm.types.interactions import InteractionsAPIResponse
if not isinstance(completion_response, InteractionsAPIResponse):
return False
return completion_response.usage is None and missing_usage_is_expected(completion_response)
def _is_unbilled_in_progress_interaction(completion_response: object) -> bool:
from litellm.interactions.background_cost_polling import is_pollable_background_interaction
from litellm.types.interactions import InteractionsAPIResponse
if not isinstance(completion_response, InteractionsAPIResponse):
return False
return completion_response.usage is None and is_pollable_background_interaction(completion_response)
def _should_track_cost_callback(
user_api_key: str | None,
user_id: str | None,

5
litellm/proxy/image_endpoints/endpoints.py
View file

@ -1,6 +1,7 @@
import asyncio
import io
import traceback
from collections.abc import Sequence
from typing import Final
import orjson
@ -33,10 +34,10 @@ async def uploadfile_to_bytesio(upload: UploadFile) -> io.BytesIO:
async def batch_to_bytesio(
uploads: list[UploadFile] | None,
uploads: Sequence[UploadFile] | None,
) -> list[io.BytesIO] | None:
"""
Convert a list of UploadFiles to a list of BytesIO buffers, or None.
Convert a sequence of UploadFiles to a list of BytesIO buffers, or None.
"""
if not uploads:
return None

6
litellm/proxy/litellm_pre_call_utils.py
View file

@ -754,6 +754,12 @@ def convert_key_logging_metadata_to_callback(
team_callback_settings_obj.callbacks.append(data.callback_name)
for var, value in data.callback_vars.items():
# New Relic routing reads these from the trusted-vars overlay with no
# callback-name check, so scope them to the newrelic entry: a team that
# put newrelic_* under a different callback never asked for New Relic and
# must not export to it.
if var.startswith("newrelic_") and data.callback_name != "newrelic":
continue
if team_callback_settings_obj.callback_vars is None:
team_callback_settings_obj.callback_vars = {}
team_callback_settings_obj.callback_vars[var] = str(value)

62
litellm/proxy/management_endpoints/auto_router_endpoints.py
View file

@ -36,6 +36,7 @@ from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
from litellm.repositories.base_repository import SupportsModelDump
from litellm.repositories.team_repository import TeamRepository
from litellm.router_strategy.complexity_router import ComplexityRouter
from litellm.router_utils.auto_router_model_naming import classify_strategy_router_model
from litellm.types.management_endpoints.auto_router_endpoints import (
SHADOW_EVAL_TURN_VALVE,
AutoRouterBenchmarkGroup,
@ -510,6 +511,53 @@ def _summed_agg_row(rows: Sequence[_SessionAggRow]) -> _SessionAggRow:
)
def _strategy_router_key(deployment: object) -> tuple[str, str] | None:
"""``(model_name, kind)`` for a deployment whose routing the session rollup records.
Kinds come from ``classify_strategy_router_model``, the same rule the Router registers a
deployment by, so this arm cannot disagree with the arm that stamped ``router_type`` onto
the session rows. Semantic auto-routers return None: they record no routing decision, so
they can never own a session row, and ``AutoRouterBenchmarkGroup.router_type`` has no
value for them. A permanent zero would read as "no traffic" rather than "not instrumented".
"""
if not isinstance(deployment, Mapping):
return None
litellm_params: Final = deployment.get("litellm_params")
router_name: Final = deployment.get("model_name")
if not (isinstance(litellm_params, Mapping) and isinstance(router_name, str) and router_name):
return None
model: Final = litellm_params.get("model")
if not isinstance(model, str):
return None
kind: Final = classify_strategy_router_model(model)
return None if kind is None or kind == "semantic" else (router_name, kind)
def _idle_router_groups(
llm_router: "Router | None", covered: frozenset[tuple[str, str]]
) -> tuple[AutoRouterBenchmarkGroup, ...]:
"""Zeroed groups for configured strategy routers the window's traffic did not cover.
The dashboard's router picker has to list a router the moment it is created rather than
once it has spent something, so the registry drives the list and the rollup only supplies
the measures. ``_summed_agg_row`` over no sessions is already the zero element of the
fold, so a group with every measure at zero costs one relabel rather than a literal that
would go stale the next time the response grows a field.
"""
if llm_router is None:
return ()
zero: Final = _summed_agg_row(())
idle: Final = frozenset(
key
for key in (_strategy_router_key(deployment) for deployment in llm_router.model_list or ())
if key is not None and key not in covered
)
return tuple(
_benchmark_group(zero.model_copy(update=MappingProxyType({"router_name": name, "router_type": kind})))
for name, kind in sorted(idle)
)
@router.get(
"/auto_router/benchmarks",
tags=("auto router",),
@ -532,8 +580,13 @@ async def get_auto_router_benchmarks(
overlaps it: its last turn is on or after start_date and its first turn is on or before
end_date. Overall hit rate is over telemetry-bearing turns; each bucket's hit rate is
over that bucket's turns.
The rollup supplies the measures, never the list. Which routers appear comes from the
model registry, so one shows up as soon as it is configured and reads zero until it
serves traffic, and `routers_in_scope` counts those too rather than only the routers the
window recorded.
"""
from litellm.proxy.proxy_server import prisma_client
from litellm.proxy.proxy_server import llm_router, prisma_client
_require_admin_viewer(user_api_key_dict, "view auto-router benchmarks across the deployment")
if prisma_client is None:
@ -555,11 +608,14 @@ async def get_auto_router_benchmarks(
(end_day + timedelta(days=1)).isoformat(),
)
rows: Final = _SESSION_AGG_ROWS.validate_python(raw_rows or ())
groups: Final = tuple(_benchmark_group(row) for row in rows)
groups: Final = (
*(_benchmark_group(row) for row in rows),
*_idle_router_groups(llm_router, frozenset((row.router_name, row.router_type) for row in rows)),
)
return AutoRouterBenchmarksResponse(
start_date=start_day.strftime("%Y-%m-%d"),
end_date=end_day.strftime("%Y-%m-%d"),
routers_in_scope=len(rows),
routers_in_scope=len(groups),
totals=_benchmark_totals(_summed_agg_row(rows)),
groups=groups,
)

15
litellm/proxy/management_endpoints/key_management_endpoints.py
View file

@ -62,6 +62,7 @@ from litellm.proxy.auth.auth_utils import (
enforce_output_token_estimates_are_admin_only,
)
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils.callback_config_validation import logging_metadata_config_error
from litellm.proxy.common_utils.callback_utils import (
decrypt_callback_vars,
encrypt_callback_vars,
@ -553,6 +554,17 @@ def key_generation_check(
return _personal_key_generation_check(user_api_key_dict=user_api_key_dict, data=data)
def raise_on_invalid_key_logging_config(metadata: Mapping[str, object] | None) -> None:
"""Key-level logging writes go through key metadata, not /team/callback.
Without this the same New Relic config the team endpoint rejects would be
accepted here and then silently ignored or misrouted at request time.
"""
error: Final = logging_metadata_config_error(metadata)
if error is not None:
raise HTTPException(status_code=400, detail={"error": error}) # mutable-ok: FastAPI detail contract
def common_key_access_checks(
user_api_key_dict: UserAPIKeyAuth,
data: GenerateKeyRequest | UpdateKeyRequest,
@ -891,6 +903,7 @@ async def _common_key_generation_helper(
)
validate_budget_duration(data.budget_duration)
raise_on_invalid_key_logging_config(data.metadata)
if data.throttle_on_budget_exceeded is True and user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN.value:
raise HTTPException(
@ -1992,6 +2005,8 @@ def prepare_metadata_fields(data: BaseModel, non_default_values: dict, existing_
"""
Check LiteLLM_ManagementEndpoint_MetadataFields (proxy/_types.py) for fields that are allowed to be updated
"""
raise_on_invalid_key_logging_config(non_default_values.get("metadata"))
if "metadata" not in non_default_values: # allow user to set metadata to none
non_default_values["metadata"] = existing_metadata.copy()

Some files were not shown because too many files have changed in this diff Show more