scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-08-28 05:25:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 36

Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_compat_matrix_stack

Browse source 
# Conflicts:
#	.gitignore
This commit is contained in:
mateo-berri 2026-05-15 23:20:54 +00:00
commit 82e3e73bfd
1007 changed files with 50363 additions and 12523 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

345
.circleci/config.yml
View file

@ -228,7 +228,7 @@ jobs:
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=litellm \
--cov=./litellm \
--cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=20 \
@ -293,7 +293,7 @@ jobs:
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=litellm \
--cov=./litellm \
--cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=20 \
@ -350,7 +350,15 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -v tests/local_testing -x --junitxml=test-results/junit.xml --durations=5 -k "langfuse"
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--junitxml=test-results/junit.xml \
--durations=5 \
-k \"langfuse\""
no_output_timeout: 15m
# Store test results
- store_test_results:
@ -395,7 +403,15 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -v tests/proxy_admin_ui_tests -x --junitxml=test-results/junit.xml --durations=5 -n 2
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/proxy_admin_ui_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 2"
no_output_timeout: 15m
# Store test results
@ -471,7 +487,15 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -v tests/router_unit_tests -x --junitxml=test-results/junit.xml --durations=5 -n 4
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/router_unit_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
no_output_timeout: 15m
# Store test results
- store_test_results:
@ -495,7 +519,15 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest tests/local_testing/ -v -k "assistants" -x --junitxml=test-results/junit.xml --durations=5
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--junitxml=test-results/junit.xml \
--durations=5 \
-k \"assistants\""
no_output_timeout: 15m
# Store test results
- store_test_results:
@ -528,14 +560,19 @@ jobs:
# Add --timeout to kill hanging tests after 120s (2 min)
# Add --durations=20 to show 20 slowest tests for debugging
# Subdirectories with dedicated jobs (maintain this list as new jobs are added)
IGNORE_DIRS=(
"tests/llm_translation/realtime"
)
IGNORE_ARGS=""
for dir in "${IGNORE_DIRS[@]}"; do
IGNORE_ARGS="$IGNORE_ARGS --ignore=$dir"
done
uv run --no-sync python -m pytest -v tests/llm_translation $IGNORE_ARGS --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread --retries 2 --retry-delay 5 --max-worker-restart=5
mkdir -p test-results
# Glob excludes the realtime/ subdirectory since it has its own job
TEST_FILES=$(circleci tests glob "tests/llm_translation/**/test_*.py" | grep -v "^tests/llm_translation/realtime/")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v \
--junitxml=test-results/junit.xml \
--durations=20 \
-n 4 \
--timeout=120 --timeout_method=thread \
--retries 2 --retry-delay 5 \
--max-worker-restart=5"
no_output_timeout: 15m
# Store test results
@ -560,7 +597,17 @@ jobs:
command: |
# Add --timeout to kill hanging tests after 120s (2 min)
# Add --durations=20 to show 20 slowest tests for debugging
uv run --no-sync python -m pytest -vv tests/llm_translation/realtime --cov=litellm --cov-report=xml -v --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/llm_translation/realtime/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=20 \
-n 4 \
--timeout=120 --timeout_method=thread"
no_output_timeout: 15m
- run:
name: Rename the coverage files
@ -593,7 +640,15 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -vv tests/agent_tests --ignore=tests/agent_tests/local_only_agent_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/agent_tests/**/test_*.py" | grep -v "^tests/agent_tests/local_only_agent_tests/")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
- run:
name: Rename the coverage files
@ -626,7 +681,18 @@ jobs:
- run:
name: Run tests
command: |
LITELLM_LOG=WARNING uv run --no-sync python -m pytest tests/guardrails_tests -vv --cov=litellm --cov-report=xml --junitxml=test-results/junit.xml --durations=5 -n 2 --timeout=120 --timeout_method=thread
mkdir -p test-results
export LITELLM_LOG=WARNING
TEST_FILES=$(circleci tests glob "tests/guardrails_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 2 \
--timeout=120 --timeout_method=thread"
no_output_timeout: 15m
- run:
name: Rename the coverage files
@ -660,7 +726,16 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -vv tests/unified_google_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 --retries 3 --retry-delay 5
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/unified_google_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
--retries 3 --retry-delay 5"
no_output_timeout: 15m
- run:
name: Rename the coverage files
@ -702,7 +777,15 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -v tests/llm_responses_api_testing -x --junitxml=test-results/junit.xml --durations=5 -n 8
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/llm_responses_api_testing/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 8"
no_output_timeout: 15m
# Store test results
@ -725,7 +808,16 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -vv tests/ocr_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/ocr_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
no_output_timeout: 15m
- run:
name: Rename the coverage files
@ -758,7 +850,16 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -vv tests/search_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/search_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
no_output_timeout: 15m
- run:
name: Rename the coverage files
@ -793,7 +894,15 @@ jobs:
name: Run enterprise tests
command: |
uv run --no-sync python -m prisma generate
uv run --no-sync python -m pytest -v tests/enterprise -x --junitxml=test-results/junit-enterprise.xml --durations=10 -n 4
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/enterprise/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--junitxml=test-results/junit-enterprise.xml \
--durations=10 \
-n 4"
no_output_timeout: 15m
# Store test results
- store_test_results:
@ -815,7 +924,16 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -vv tests/batches_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 -n 2
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/batches_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 2"
no_output_timeout: 15m
- run:
name: Rename the coverage files
@ -848,7 +966,16 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -vv tests/litellm_utils_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 -n 2
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/litellm_utils_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 2"
no_output_timeout: 15m
- run:
name: Rename the coverage files
@ -882,7 +1009,16 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -vv tests/pass_through_unit_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/pass_through_unit_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
no_output_timeout: 15m
- run:
name: Rename the coverage files
@ -916,7 +1052,15 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -v tests/image_gen_tests -n 4 -x --junitxml=test-results/junit.xml --durations=5
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/image_gen_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
no_output_timeout: 15m
# Store test results
- store_test_results:
@ -939,7 +1083,18 @@ jobs:
- run:
name: Run tests
command: |
LITELLM_LOG=WARNING uv run --no-sync python -m pytest tests/logging_callback_tests -vv --cov=litellm --cov-report=xml -n 4 --junitxml=test-results/junit.xml --durations=5 --timeout=120 --timeout_method=thread
mkdir -p test-results
export LITELLM_LOG=WARNING
TEST_FILES=$(circleci tests glob "tests/logging_callback_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=./litellm --cov-report=xml \
-n 4 \
--junitxml=test-results/junit.xml \
--durations=5 \
--timeout=120 --timeout_method=thread"
no_output_timeout: 15m
- run:
name: Rename the coverage files
@ -972,7 +1127,15 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -vv tests/audio_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/audio_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
- run:
name: Rename the coverage files
@ -1012,14 +1175,19 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -vv \
mkdir -p test-results
TEST_FILES=$(printf "%s\n" \
tests/local_testing/test_dual_cache.py \
tests/local_testing/test_redis_batch_optimizations.py \
tests/local_testing/test_router_utils.py \
--cov=litellm --cov-report=xml \
-x -s -v --junitxml=test-results/junit.xml \
--durations=5 -n 2 \
--reruns 2 --reruns-delay 1
tests/local_testing/test_router_utils.py)
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 -n 2 \
--reruns 2 --reruns-delay 1"
no_output_timeout: 20m
- run:
name: Rename the coverage files
@ -1260,8 +1428,17 @@ jobs:
- run:
name: Run Basic Proxy Startup Tests (Health Readiness and Chat Completion)
command: |
uv run --no-sync python -m pytest -v tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--junitxml=test-results/junit-2.xml \
--durations=5"
no_output_timeout: 15m
- store_test_results:
path: test-results
build_and_test:
machine:
@ -1331,7 +1508,18 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -s -v tests/*.py -x --junitxml=test-results/junit.xml -n 4 --durations=5 --ignore=tests/otel_tests --ignore=tests/spend_tracking_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation --ignore=tests/llm_responses_api_testing --ignore=tests/mcp_tests --ignore=tests/guardrails_tests --ignore=tests/image_gen_tests --ignore=tests/pass_through_unit_tests
mkdir -p test-results
# Original used `tests/*.py` (top-level only); the `--ignore=...`
# flags were vestigial since shell globbing did not descend into
# subdirectories. Replicate by globbing only top-level test files.
TEST_FILES=$(circleci tests glob "tests/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-s -v -x \
--junitxml=test-results/junit.xml \
-n 4 \
--durations=5"
no_output_timeout: 15m
# Store test results
@ -1406,7 +1594,14 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -s -vv tests/openai_endpoints_tests --junitxml=test-results/junit.xml --durations=5
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/openai_endpoints_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-s -vv \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
# Store test results
@ -1475,7 +1670,14 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -v tests/otel_tests --junitxml=test-results/junit.xml --durations=5
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/otel_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
# Clean up first container
- run:
@ -1518,7 +1720,14 @@ jobs:
- run:
name: Run second round of tests
command: |
uv run --no-sync python -m pytest -v tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--junitxml=test-results/junit-2.xml \
--durations=5"
no_output_timeout: 15m
# Store test results
@ -1587,8 +1796,17 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -vv tests/spend_tracking_tests -x --junitxml=test-results/junit.xml --durations=5
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/spend_tracking_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
- store_test_results:
path: test-results
- run:
name: Stop and remove first container
when: always
@ -1676,7 +1894,14 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -vv tests/multi_instance_e2e_tests -x --junitxml=test-results/junit.xml --durations=5
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/multi_instance_e2e_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
# Clean up first container
# Store test results
@ -1732,7 +1957,14 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -vv tests/store_model_in_db_tests -x --junitxml=test-results/junit.xml --durations=5
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/store_model_in_db_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
- run:
name: Stop and remove containers
@ -1805,9 +2037,18 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -vv tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--junitxml=test-results/junit-2.xml \
--durations=5"
no_output_timeout: 15m
# Clean up first container
- store_test_results:
path: test-results
- run:
name: Stop and remove first container
command: |
@ -1940,7 +2181,14 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -v tests/pass_through_tests/ -x --junitxml=test-results/junit.xml --durations=5
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/pass_through_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
# Store test results
@ -1997,9 +2245,16 @@ jobs:
- run:
name: Run Claude Agent SDK E2E Tests
command: |
mkdir -p test-results
export LITELLM_PROXY_URL="http://localhost:4000"
export LITELLM_API_KEY="sk-1234"
uv run --no-sync python -m pytest -vv tests/proxy_e2e_anthropic_messages_tests/ -x -s --junitxml=test-results/junit.xml --durations=5
TEST_FILES=$(circleci tests glob "tests/proxy_e2e_anthropic_messages_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
# Store test results

94
.github/actions/helm-oci-chart-releaser/action.yml vendored
View file

@ -1,94 +0,0 @@
name: Helm OCI Chart Releaser
description: Push Helm charts to OCI-based (Docker) registries
author: sergeyshaykhullin
branding:
color: yellow
icon: upload-cloud
inputs:
name:
required: true
description: Chart name
repository:
required: true
description: Chart repository name
tag:
required: true
description: Chart version
app_version:
required: true
description: App version
path:
required: false
description: Chart path (Default 'charts/{name}')
registry:
required: true
description: OCI registry
registry_username:
required: true
description: OCI registry username
registry_password:
required: true
description: OCI registry password
update_dependencies:
required: false
default: 'false'
description: Update chart dependencies before packaging (Default 'false')
outputs:
image:
value: ${{ steps.output.outputs.image }}
description: Chart image (Default '{registry}/{repository}/{image}:{tag}')
runs:
using: composite
steps:
- name: Helm | Setup
uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1
with:
version: v3.20.0
- name: Helm | Login
shell: bash
env:
REGISTRY_PASSWORD: ${{ inputs.registry_password }}
REGISTRY_USERNAME: ${{ inputs.registry_username }}
REGISTRY: ${{ inputs.registry }}
run: echo "$REGISTRY_PASSWORD" | helm registry login -u "$REGISTRY_USERNAME" --password-stdin "$REGISTRY"
- name: Helm | Dependency
if: inputs.update_dependencies == 'true'
shell: bash
env:
CHART_PATH: ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }}
run: helm dependency update "$CHART_PATH"
- name: Helm | Package
shell: bash
env:
CHART_PATH: ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }}
TAG: ${{ inputs.tag }}
APP_VERSION: ${{ inputs.app_version }}
run: helm package "$CHART_PATH" --version "$TAG" --app-version "$APP_VERSION"
- name: Helm | Push
shell: bash
env:
NAME: ${{ inputs.name }}
TAG: ${{ inputs.tag }}
REGISTRY: ${{ inputs.registry }}
REPOSITORY: ${{ inputs.repository }}
run: helm push "${NAME}-${TAG}.tgz" "oci://${REGISTRY}/${REPOSITORY}"
- name: Helm | Logout
shell: bash
env:
REGISTRY: ${{ inputs.registry }}
run: helm registry logout "$REGISTRY"
- name: Helm | Output
id: output
shell: bash
env:
REGISTRY: ${{ inputs.registry }}
REPOSITORY: ${{ inputs.repository }}
NAME: ${{ inputs.name }}
TAG: ${{ inputs.tag }}
run: echo "image=${REGISTRY}/${REPOSITORY}/${NAME}:${TAG}" >> $GITHUB_OUTPUT

35
.github/workflows/README.md vendored
View file

@ -1,35 +0,0 @@
# Simple PyPI Publishing
A GitHub workflow to manually publish LiteLLM packages to PyPI with a specified version.
## How to Use
1. Go to the **Actions** tab in the GitHub repository
2. Select **Simple PyPI Publish** from the workflow list
3. Click **Run workflow**
4. Enter the version to publish (e.g., `1.74.10`)
## What the Workflow Does
1. **Updates** the version in `pyproject.toml`
2. **Copies** the model prices backup file
3. **Builds** the Python package
4. **Publishes** to PyPI
## Prerequisites
Make sure the following secret is configured in the repository:
- `PYPI_PUBLISH_PASSWORD`: PyPI API token for authentication
## Example Usage
- Version: `1.74.11` → Publishes as v1.74.11
- Version: `1.74.10-hotfix1` → Publishes as v1.74.10-hotfix1
## Features
- ✅ Manual trigger with version input
- ✅ Automatic version updates in `pyproject.toml`
- ✅ Repository safety check (only runs on official repo)
- ✅ Clean package building and publishing
- ✅ Success confirmation with PyPI package link

2
.github/workflows/_test-unit-base.yml vendored
View file

@ -91,7 +91,7 @@ jobs:
--reruns-delay 1 \
--dist=loadscope \
--durations=20 \
--cov=litellm \
--cov=./litellm \
--cov-report=xml:coverage.xml \
--cov-config=pyproject.toml

4
.github/workflows/_test-unit-services-base.yml vendored
View file

@ -132,7 +132,7 @@ jobs:
--reruns "${RERUNS}" \
--reruns-delay 1 \
--durations=20 \
--cov=litellm \
--cov=./litellm \
--cov-report=xml:coverage.xml \
--cov-config=pyproject.toml
else
@ -144,7 +144,7 @@ jobs:
--reruns-delay 1 \
--dist="${DIST}" \
--durations=20 \
--cov=litellm \
--cov=./litellm \
--cov-report=xml:coverage.xml \
--cov-config=pyproject.toml
fi

92
.github/workflows/llm-translation-testing.yml vendored
View file

@ -1,92 +0,0 @@
name: LLM Translation Tests
on:
workflow_dispatch:
inputs:
release_candidate_tag:
description: "Release candidate tag/version"
required: true
type: string
push:
tags:
- "v*-rc*" # Triggers on release candidate tags like v1.0.0-rc1
permissions:
contents: read
jobs:
run-llm-translation-tests:
runs-on: ubuntu-latest
timeout-minutes: 90
steps:
- name: Checkout code
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
ref: ${{ github.event.inputs.release_candidate_tag || github.ref }}
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
with:
version: "0.10.9"
enable-cache: false
- name: Restore uv dependencies cache
uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
~/.cache/uv
.venv
key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
restore-keys: |
${{ runner.os }}-uv-
- name: Install dependencies
run: |
uv sync --frozen
- name: Create test results directory
run: mkdir -p test-results
- name: Run LLM Translation Tests
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }}
AZURE_API_VERSION: ${{ secrets.AZURE_API_VERSION }}
RC_TAG: ${{ github.event.inputs.release_candidate_tag || github.ref_name }}
COMMIT_SHA: ${{ github.sha }}
run: |
python .github/workflows/run_llm_translation_tests.py \
--tag "$RC_TAG" \
--commit "$COMMIT_SHA" \
|| true # Continue even if tests fail
- name: Display test summary
if: always()
run: |
if [ -f "test-results/llm_translation_report.md" ]; then
echo "Test report generated successfully!"
echo "Artifact will contain:"
echo "- test-results/junit.xml (JUnit XML results)"
echo "- test-results/llm_translation_report.md (Beautiful markdown report)"
else
echo "Warning: Test report was not generated"
fi
- name: Upload test artifacts
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
if: always()
with:
name: LLM-Translation-Artifact-${{ github.event.inputs.release_candidate_tag || github.ref_name }}
path: test-results/
retention-days: 30

131
.github/workflows/mutation-test.yml vendored Normal file
View file

@ -0,0 +1,131 @@
name: "Mutation Test (manual)"
# Manually-triggered mutation testing. Runs mutmut against the scope
# configured in [tool.mutmut] in pyproject.toml (currently the
# litellm/proxy/management_endpoints/ folder). Intended cadence is roughly
# weekly — clicked from the Actions tab when someone wants a fresh report.
#
# Uploads a structured `mutation-report.md` (Meta ACH-style: original +
# mutated function with `# MUTANT START`/`# MUTANT END` delimiters + the
# existing tests + a task instruction) as a workflow artifact. Failures
# do not block anything because nothing depends on this workflow.
on:
workflow_dispatch:
permissions:
contents: read
concurrency:
group: mutation-test-${{ github.ref }}
cancel-in-progress: true
jobs:
mutation:
name: Run mutmut
runs-on: ubuntu-latest
# Whole-folder mutation against ~15 files / ~7.5k LOC can take hours.
# 350 minutes is just under the GitHub-hosted job cap of 360 minutes.
timeout-minutes: 350
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
with:
version: "0.10.9"
- name: Cache uv dependencies
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
~/.cache/uv
.venv
key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
restore-keys: |
${{ runner.os }}-uv-
- name: Install dependencies
run: |
uv sync --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
- name: Generate Prisma client
env:
PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
run: |
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
# mutmut 3.x runs tests inside a `mutants/` sandbox where it injects
# mutation trampolines. uv installs the project as editable by default,
# which puts the original source dir on sys.path via a .pth file and
# shadows the sandbox copy — so tests would never exercise the mutated
# code. Reinstalling non-editable removes the .pth shadow.
- name: Reinstall litellm non-editable (so mutants/ is not shadowed)
run: |
uv pip uninstall litellm
uv pip install . --no-deps
# pytest-retry's pytest_configure hook crashes with
# `INTERNALERROR: no option named 'filtered_exceptions'` when invoked
# via mutmut's in-process pytest.main() call. The entry-point name
# doesn't normalize cleanly with `-p no:<name>`, so just remove the
# package outright. Reruns are wrong for mutation testing anyway —
# rerunning a "failed" mutant test would mask which mutants are killed.
- name: Remove pytest plugins that conflict with mutmut
run: |
uv pip uninstall pytest-retry || true
- name: Run mutmut
env:
# Make the mutants/ sandbox win over site-packages on sys.path so the
# trampolined files are imported instead of the installed copy.
PYTHONPATH: ${{ github.workspace }}/mutants
run: |
set -o pipefail
mkdir -p mutants
uv run --no-sync --with mutmut==3.5.0 mutmut run 2>&1 | tee mutmut-run.log
# Generate the structured report. The script embeds the enclosing
# function source for each survivor (via Python AST) and includes the
# existing test files, so an LLM agent has enough context to write
# killing tests without further file lookups. Modeled on Meta's ACH
# prompt template (arXiv 2501.12862).
- name: Generate detailed mutation report
if: always()
run: |
set +e
uv run --no-sync --with mutmut==3.5.0 mutmut export-cicd-stats > /dev/null 2>&1
uv run --no-sync --with mutmut==3.5.0 mutmut results > mutmut-results.txt 2>&1
uv run --no-sync python scripts/mutation_report.py
# The full report can be very long for big test files; the run-page
# summary cuts off at 1 MB. Append the head of the report (summary
# + survivor list) and link out to the artifact for the full body.
{
head -c 900000 mutation-report.md
echo ""
echo ""
echo "_Full report (with embedded function bodies and test files) is in the workflow artifact._"
} >> "$GITHUB_STEP_SUMMARY"
- name: Upload mutmut artifacts
if: always()
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
with:
name: mutmut-${{ github.run_id }}-${{ github.run_attempt }}
path: |
mutation-report.md
mutmut-results.txt
mutmut-run.log
mutants/mutmut-stats.json
mutants/mutmut-cicd-stats.json
mutants/litellm/proxy/management_endpoints/**/*.py
if-no-files-found: warn
retention-days: 14

153
.github/workflows/publish_to_pypi.yml vendored
View file

@ -1,153 +0,0 @@
name: Publish to PyPI
on:
workflow_dispatch:
jobs:
preflight-checks:
name: Preflight Checks
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: read
# No environment — read-only checks, no approval needed
outputs:
needs_publish: ${{ steps.check-litellm.outputs.needs_publish }}
version: ${{ steps.check-litellm.outputs.version }}
steps:
- name: Checkout repo
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
with:
version: "0.10.9"
enable-cache: false
- name: Check litellm version on PyPI
id: check-litellm
run: |
VERSION=$(python - <<'PY'
import tomllib
with open("pyproject.toml", "rb") as f:
print(tomllib.load(f)["project"]["version"])
PY
)
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
echo "Checking if litellm $VERSION exists on PyPI..."
HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "https://pypi.org/pypi/litellm/$VERSION/json")
if [ "$HTTP_STATUS" = "200" ]; then
echo "litellm $VERSION already exists on PyPI. Skipping publish."
echo "needs_publish=false" >> "$GITHUB_OUTPUT"
else
echo "litellm $VERSION not found on PyPI. Publish needed."
echo "needs_publish=true" >> "$GITHUB_OUTPUT"
fi
- name: Sanity check proxy-extras version
run: |
# Read pinned version from project optional dependencies
PYPROJECT_VERSION=$(python3 - <<'PY'
import sys
import tomllib
with open("pyproject.toml", "rb") as f:
proxy_requirements = tomllib.load(f)["project"]["optional-dependencies"]["proxy"]
version = None
for requirement in proxy_requirements:
normalized = requirement.split(";", 1)[0].strip()
if not normalized.startswith("litellm-proxy-extras"):
continue
parts = normalized.split("==", 1)
if len(parts) == 2 and parts[0].strip() == "litellm-proxy-extras":
candidate = parts[1].strip()
if candidate:
version = candidate
break
if version is None:
print(
"::error::Could not find an exact litellm-proxy-extras pin in project.optional-dependencies.proxy",
file=sys.stderr,
)
sys.exit(1)
print(version)
PY
)
echo "pyproject.toml pins litellm-proxy-extras version: $PYPROJECT_VERSION"
# Check that the pinned version exists on PyPI
echo "Checking if litellm-proxy-extras $PYPROJECT_VERSION exists on PyPI..."
HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "https://pypi.org/pypi/litellm-proxy-extras/$PYPROJECT_VERSION/json")
if [ "$HTTP_STATUS" != "200" ]; then
echo "::error::litellm-proxy-extras $PYPROJECT_VERSION is not published on PyPI yet. Publish it before releasing litellm."
exit 1
fi
echo "litellm-proxy-extras $PYPROJECT_VERSION exists on PyPI. Sanity check passed."
publish-litellm:
name: Publish litellm to PyPI
needs: preflight-checks
if: needs.preflight-checks.outputs.needs_publish == 'true'
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
id-token: write
contents: read
environment: pypi-publish
steps:
- name: Checkout repo
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
with:
version: "0.10.9"
enable-cache: false
- name: Copy model prices backup
run: cp model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
- name: Build package
run: |
rm -rf build dist
uv build
- name: Verify build artifacts
env:
EXPECTED_VERSION: ${{ needs.preflight-checks.outputs.version }}
run: |
echo "Contents of dist/:"
ls -la dist/
# Ensure we have both sdist and wheel
ls dist/*.tar.gz
ls dist/*.whl
# Verify built version matches expected
ls dist/ | grep -q "litellm-${EXPECTED_VERSION}" || {
echo "::error::Built artifacts do not match expected version $EXPECTED_VERSION"
ls dist/
exit 1
}
- name: Validate package metadata
run: |
uv tool run --from 'twine==6.2.0' twine check dist/*
- name: Publish to PyPI
uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0

28
.github/workflows/read_pyproject_version.yml vendored
View file

@ -1,28 +0,0 @@
name: Read Version from pyproject.toml
on:
push:
branches:
- main # Change this to the default branch of your repository
permissions:
contents: read
jobs:
read-version:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Read version from pyproject.toml
id: read-version
run: |
version=$(grep -m1 '^version' pyproject.toml | sed 's/version = "\(.*\)"/\1/')
printf "LITELLM_VERSION=%s" "$version" >> $GITHUB_ENV
- name: Display version
run: echo "Current version is $LITELLM_VERSION"

27
.github/workflows/results_stats.csv vendored
View file

@ -1,27 +0,0 @@
Date,"Ben
Ashley",Tom Brooks,Jimmy Cooney,"Sue
Daniels",Berlinda Fong,Terry Jones,Angelina Little,Linda Smith
10/1,FALSE,TRUE,TRUE,TRUE,TRUE,TRUE,FALSE,TRUE
10/2,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
10/3,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
10/4,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
10/5,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
10/6,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
10/7,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
10/8,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
10/9,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
10/10,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
10/11,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
10/12,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
10/13,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
10/14,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
10/15,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
10/16,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
10/17,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
10/18,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
10/19,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
10/20,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
10/21,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
10/22,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
10/23,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
Total,0,1,1,1,1,1,0,1
1 Date Ben Ashley Tom Brooks Jimmy Cooney Sue Daniels Berlinda Fong Terry Jones Angelina Little Linda Smith
2 10/1 FALSE TRUE TRUE TRUE TRUE TRUE FALSE TRUE
3 10/2 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
4 10/3 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
5 10/4 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
6 10/5 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
7 10/6 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
8 10/7 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
9 10/8 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
10 10/9 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
11 10/10 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
12 10/11 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
13 10/12 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
14 10/13 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
15 10/14 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
16 10/15 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
17 10/16 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
18 10/17 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
19 10/18 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
20 10/19 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
21 10/20 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
22 10/21 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
23 10/22 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
24 10/23 FALSE FALSE FALSE FALSE FALSE FALSE FALSE FALSE
25 Total 0 1 1 1 1 1 0 1

229
.github/workflows/run_observatory_tests.yml vendored
View file

@ -1,229 +0,0 @@
name: Run Observatory Tests
on:
workflow_dispatch:
inputs:
tag:
description: "Docker image tag to test (e.g. v1.61.0.rc1)"
required: true
type: string
commit_hash:
description: "Commit hash (defaults to HEAD of current branch)"
required: false
type: string
workflow_call:
inputs:
tag:
description: "Docker image tag to test"
required: true
type: string
commit_hash:
description: "Commit hash of the release"
required: true
type: string
permissions:
contents: read
env:
LITELLM_MASTER_KEY: ${{ secrets.LITELLM_MASTER_KEY_STAGING }}
jobs:
observatory-tests:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- name: Checkout repository
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Validate tag input
env:
TAG: ${{ inputs.tag }}
run: |
if [[ ! "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+ ]]; then
echo "Invalid tag format: $TAG (expected vX.Y.Z...)"
exit 1
fi
- name: Start LiteLLM container
env:
TAG: ${{ inputs.tag }}
AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }}
WORKSPACE: ${{ github.workspace }}
run: |
docker run -d \
--name litellm-rc \
-p 4000:4000 \
-v "${WORKSPACE}/.github/observatory/litellm_config.yaml:/app/config.yaml" \
-e LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY}" \
-e AZURE_API_KEY="${AZURE_API_KEY}" \
-e AZURE_API_BASE="${AZURE_API_BASE}" \
"litellm/litellm:${TAG}" \
--config /app/config.yaml --port 4000
- name: Wait for LiteLLM health check
run: |
echo "Waiting for LiteLLM to be ready..."
for i in $(seq 1 30); do
if curl -s -f http://localhost:4000/health/liveliness > /dev/null 2>&1; then
echo "LiteLLM is healthy"
exit 0
fi
echo "Attempt $i/30 - not ready yet, waiting 10s..."
sleep 10
done
echo "LiteLLM failed to start within 5 minutes"
docker logs litellm-rc
exit 1
- name: Start cloudflared tunnel
run: |
# Install cloudflared (pinned version + checksum)
curl -sL https://github.com/cloudflare/cloudflared/releases/download/2025.2.1/cloudflared-linux-amd64 -o /usr/local/bin/cloudflared
echo "afdfadd1ef552e66bffc35246fe30a9bd578356d2d386de95585ccfc432472b8 /usr/local/bin/cloudflared" | sha256sum -c -
chmod +x /usr/local/bin/cloudflared
# Start a quick tunnel (no account needed) and capture the URL
cloudflared tunnel --url http://localhost:4000 --no-autoupdate > /tmp/cloudflared.log 2>&1 &
CLOUDFLARED_PID=$!
echo "CLOUDFLARED_PID=$CLOUDFLARED_PID" >> $GITHUB_ENV
# Wait for tunnel URL to appear in logs
echo "Waiting for tunnel URL..."
for i in $(seq 1 30); do
TUNNEL_URL=$(grep -oP 'https://[a-z0-9-]+\.trycloudflare\.com' /tmp/cloudflared.log | head -1 || true)
if [ -n "$TUNNEL_URL" ]; then
echo "Tunnel URL: $TUNNEL_URL"
echo "TUNNEL_URL=$TUNNEL_URL" >> $GITHUB_ENV
exit 0
fi
sleep 2
done
echo "Failed to get tunnel URL"
cat /tmp/cloudflared.log
exit 1
- name: Verify tunnel connectivity
run: |
echo "Testing tunnel at ${TUNNEL_URL}..."
# Quick tunnels need time for DNS propagation; retry to avoid
# transient NXDOMAIN (curl exit code 6) on first attempt.
for i in $(seq 1 10); do
if curl -sf "${TUNNEL_URL}/health/liveliness" > /dev/null 2>&1; then
echo "Tunnel is working (attempt $i)"
exit 0
fi
echo "Attempt $i/10 - tunnel not routable yet, waiting 5s..."
sleep 5
done
echo "Tunnel failed to become reachable after 50s"
cat /tmp/cloudflared.log
exit 1
- name: Trigger observatory test run
id: trigger
env:
OBSERVATORY_URL: ${{ secrets.OBSERVATORY_URL }}
OBSERVATORY_API_KEY: ${{ secrets.OBSERVATORY_API_KEY }}
run: |
PAYLOAD=$(jq -n \
--arg url "${TUNNEL_URL}" \
--arg key "${LITELLM_MASTER_KEY}" \
'{
deployment_url: $url,
api_key: $key,
test_suite: "TestOAIAzureRelease",
models: ["gpt-4o-mini", "gpt-4o"]
}')
RESPONSE=$(curl -s -w "\n%{http_code}" -X POST "${OBSERVATORY_URL}/run-test" \
-H "Content-Type: application/json" \
-H "X-LiteLLM-Observatory-API-Key: ${OBSERVATORY_API_KEY}" \
-d "$PAYLOAD")
HTTP_CODE=$(echo "$RESPONSE" | tail -1)
BODY=$(echo "$RESPONSE" | head -n -1)
echo "Response ($HTTP_CODE): $BODY"
if [ "$HTTP_CODE" -ge 400 ]; then
echo "Failed to trigger test run"
exit 1
fi
# Extract request_id for polling this specific run
REQUEST_ID=$(echo "$BODY" | jq -r '.results.request_id')
if [ -z "$REQUEST_ID" ] || [ "$REQUEST_ID" = "null" ]; then
echo "Failed to extract request_id from response"
exit 1
fi
echo "Request ID: $REQUEST_ID"
echo "request_id=$REQUEST_ID" >> $GITHUB_OUTPUT
- name: Poll for test completion
id: poll
env:
OBSERVATORY_URL: ${{ secrets.OBSERVATORY_URL }}
OBSERVATORY_API_KEY: ${{ secrets.OBSERVATORY_API_KEY }}
REQUEST_ID: ${{ steps.trigger.outputs.request_id }}
run: |
TIMEOUT=900 # 15 minutes
INTERVAL=30
ELAPSED=0
while [ $ELAPSED -lt $TIMEOUT ]; do
STATUS=$(curl -s "${OBSERVATORY_URL}/run-status/${REQUEST_ID}" \
-H "X-LiteLLM-Observatory-API-Key: ${OBSERVATORY_API_KEY}")
RUN_STATUS=$(echo "$STATUS" | jq -r '.status')
echo "Run status (${ELAPSED}s elapsed): $RUN_STATUS"
if [ "$RUN_STATUS" = "completed" ] || [ "$RUN_STATUS" = "failed" ]; then
echo "Test finished with status: $RUN_STATUS"
echo "$STATUS" > /tmp/observatory_result.json
exit 0
fi
sleep $INTERVAL
ELAPSED=$((ELAPSED + INTERVAL))
done
echo "Timed out waiting for test to complete after ${TIMEOUT}s"
exit 1
- name: Verify test results
run: |
RESULT=$(cat /tmp/observatory_result.json)
echo "Full result: $RESULT"
STATUS=$(echo "$RESULT" | jq -r '.status')
TEST_PASSED=$(echo "$RESULT" | jq -r '.result.test_passed // false')
FAILURE_RATE=$(echo "$RESULT" | jq -r '.result.failure_rate // "N/A"')
ERROR=$(echo "$RESULT" | jq -r '.error // empty')
echo "Status: $STATUS"
echo "Test passed: $TEST_PASSED"
echo "Failure rate: $FAILURE_RATE"
if [ -n "$ERROR" ]; then
echo "Error: $ERROR"
fi
if [ "$STATUS" = "failed" ]; then
echo "Test run failed"
exit 1
fi
if [ "$TEST_PASSED" != "true" ]; then
echo "Tests did not pass (failure rate: $FAILURE_RATE)"
exit 1
fi
echo "All tests passed!"
- name: Print LiteLLM logs on failure
if: failure()
run: |
docker logs litellm-rc 2>/dev/null || true
cat /tmp/cloudflared.log 2>/dev/null || true
- name: Cleanup
if: always()
run: |
kill "$CLOUDFLARED_PID" 2>/dev/null || true
docker rm -f litellm-rc 2>/dev/null || true

48
.github/workflows/scan_duplicate_issues.yml vendored
View file

@ -1,48 +0,0 @@
name: Scan Duplicate Issues (One-Time)
on:
workflow_dispatch:
inputs:
threshold:
description: "Similarity threshold (0-1)"
required: false
default: "0.85"
close:
description: "Actually close duplicates (false = dry run)"
required: false
type: boolean
default: false
jobs:
scan:
runs-on: ubuntu-latest
permissions:
issues: write
contents: read
steps:
- name: Checkout scripts
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
sparse-checkout: .github/scripts
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Scan for duplicate issues
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
INPUT_THRESHOLD: ${{ inputs.threshold }}
INPUT_CLOSE: ${{ inputs.close }}
run: |
CLOSE_FLAG=""
if [ "$INPUT_CLOSE" = "true" ]; then
CLOSE_FLAG="--close"
fi
python3 .github/scripts/close_duplicate_issues.py \
--scan \
--repo ${{ github.repository }} \
--threshold "$INPUT_THRESHOLD" \
$CLOSE_FLAG

45
.github/workflows/test-litellm.yml vendored
View file

@ -1,45 +0,0 @@
name: LiteLLM Mock Tests (folder - tests/test_litellm)
# DEPRECATED: This workflow is replaced by test-litellm-matrix.yml which runs
# the same tests in parallel across 10 jobs for faster CI times.
# Kept for manual debugging only.
on:
workflow_dispatch: # Manual trigger only
# pull_request:
# branches: [ main ]
permissions:
contents: read
jobs:
test:
runs-on: ubuntu-latest
timeout-minutes: 25
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Thank You Message
run: |
echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY
echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
with:
version: "0.10.9"
- name: Install dependencies
run: |
uv lock --check
uv sync --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
- name: Run tests
run: |
uv run --no-sync pytest tests/test_litellm --tb=short -vv --maxfail=10 -n 4 --durations=50

2
.github/workflows/test-mcp.yml vendored
View file

@ -43,4 +43,4 @@ jobs:
- name: Run MCP tests
run: |
uv run --no-sync pytest tests/mcp_tests -x -vv -n 4 --cov=litellm --cov-report=xml --durations=5
uv run --no-sync pytest tests/mcp_tests -x -vv -n 4 --cov=./litellm --cov-report=xml --durations=5

3
.github/workflows/test-unit-proxy-db.yml vendored
View file

@ -100,6 +100,7 @@ jobs:
test-path: >-
tests/proxy_unit_tests/test_auth_checks.py
tests/proxy_unit_tests/test_user_api_key_auth.py
tests/proxy_unit_tests/test_deprecated_key_grace_period.py
workers: 4
dist: loadscope
timeout: 15
@ -141,6 +142,8 @@ jobs:
tests/proxy_unit_tests/test_server_root_path.py
tests/proxy_unit_tests/test_proxy_pass_user_config.py
tests/proxy_unit_tests/test_proxy_token_counter.py
tests/proxy_unit_tests/test_request_size_limit_middleware.py
tests/proxy_unit_tests/test_multipart_bypass_repro.py
workers: 4
dist: loadscope
timeout: 15

54
.github/workflows/update_release.py vendored
View file

@ -1,54 +0,0 @@
import os
import requests
from datetime import datetime
# GitHub API endpoints
GITHUB_API_URL = "https://api.github.com"
REPO_OWNER = "BerriAI"
REPO_NAME = "litellm"
# GitHub personal access token (required for uploading release assets)
GITHUB_ACCESS_TOKEN = os.environ.get("GITHUB_ACCESS_TOKEN")
# Headers for GitHub API requests
headers = {
"Accept": "application/vnd.github+json",
"Authorization": f"Bearer {GITHUB_ACCESS_TOKEN}",
"X-GitHub-Api-Version": "2022-11-28",
}
# Get the latest release
releases_url = f"{GITHUB_API_URL}/repos/{REPO_OWNER}/{REPO_NAME}/releases/latest"
response = requests.get(releases_url, headers=headers)
latest_release = response.json()
print("Latest release:", latest_release)
# Upload an asset to the latest release
upload_url = latest_release["upload_url"].split("{?")[0]
asset_name = "results_stats.csv"
asset_path = os.path.join(os.getcwd(), asset_name)
print("upload_url:", upload_url)
with open(asset_path, "rb") as asset_file:
asset_data = asset_file.read()
upload_payload = {
"name": asset_name,
"label": "Load test results",
"created_at": datetime.utcnow().isoformat() + "Z",
}
upload_headers = headers.copy()
upload_headers["Content-Type"] = "application/octet-stream"
upload_response = requests.post(
upload_url,
headers=upload_headers,
data=asset_data,
params=upload_payload,
)
if upload_response.status_code == 201:
print(f"Asset '{asset_name}' uploaded successfully to the latest release.")
else:
print(f"Failed to upload asset. Response: {upload_response.text}")

2
.gitignore vendored
View file

@ -108,3 +108,5 @@ compat-results.json.shards/
compat-rate-limit-summary.json
# Matrix JSON produced by the daily-cron publisher (pushed to litellm-docs).
compatibility-matrix.json
.vscode

21
AGENTS.md
View file

@ -241,10 +241,27 @@ When opening issues or pull requests, follow these templates:
### Running the proxy server
Start the proxy with a config file:
Create a minimal config file and start the proxy:
```yaml
# config.yaml
model_list:
- model_name: fake-openai-endpoint
litellm_params:
model: openai/fake-model
api_key: fake-key
api_base: https://fake-api.example.com
general_settings:
master_key: sk-1234
litellm_settings:
drop_params: True
telemetry: False
```
```bash
uv run litellm --config dev_config.yaml --port 4000
uv run litellm --config config.yaml --port 4000
```
The proxy takes ~15-20 seconds to fully start (it runs Prisma migrations on boot). Wait for `/health` to return before sending requests. Without a PostgreSQL `DATABASE_URL`, the proxy connects to a default Neon dev database embedded in the `litellm-proxy-extras` package.

3
CLAUDE.md
View file

@ -117,6 +117,7 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components:
- **Always use `antd` for new UI components** — we are migrating off of `@tremor/react`. Do not introduce new `Badge`, `Text`, `Card`, `Grid`, `Title`, or other imports from `@tremor/react` in any new or modified file. Use `antd` equivalents: `Tag` for labels, `Typography.Text` / `Typography.Title` / `Typography.Paragraph` for textual content (avoid plain text-only `<span>`, `<p>`, `<h*>` when Typography fits), and `Card` from `antd`. Note that `antd` has no `"yellow"` Tag color — use `"gold"` for amber/yellow.
### MCP OAuth / OpenAPI Transport Mapping
- **`available_on_public_internet: false` with `delegate_auth_to_upstream: true` (oauth2, interactive — not `client_credentials`)** — LiteLLM still allows the anonymous upstream PKCE path (no proxy API key for `/authorize` and matching MCP routes). The internal-only flag mainly affects other surfaces (e.g. IP-based discovery). Rely on the upstream IdP and network policy; the dashboard shows a warning when both are set, and the proxy logs a warning when the server is loaded from config or the database.
- `TRANSPORT.OPENAPI` is a UI-only concept. The backend only accepts `"http"`, `"sse"`, or `"stdio"`. Always map it to `"http"` before any API call (including pre-OAuth temp-session calls).
- FastAPI validation errors return `detail` as an array of `{loc, msg, type}` objects. Error extractors must handle: array (map `.msg`), string, nested `{error: string}`, and fallback.
- When an MCP server already has `authorization_url` stored, skip OAuth discovery (`_discovery_metadata`) — the server URL for OpenAPI MCPs is the spec file, not the API base, and fetching it causes timeouts.
@ -146,7 +147,7 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components:
- **Bound large result sets.** Prisma materializes full results in memory. For results over ~10 MB, paginate with `take`/`skip` or `cursor`/`take`, always with an explicit `order`. Prefer cursor-based pagination (`skip` is O(n)). Don't paginate naturally small result sets.
- **Limit fetched columns on wide tables.** Use `select` to fetch only needed fields — returns a partial object, so downstream code must not access unselected fields.
- **Check index coverage.** For new or modified queries, check `schema.prisma` for a supporting index. Prefer extending an existing index (e.g. `@@index([a])` → `@@index([a, b])`) over adding a new one, unless it's a `@@unique`. Only add indexes for large/frequent queries.
- **Keep schema files in sync.** Apply schema changes to all `schema.prisma` copies (`schema.prisma`, `litellm/proxy/`, `litellm-proxy-extras/`, `litellm-js/spend-logs/` for SpendLogs) with a migration under `litellm-proxy-extras/litellm_proxy_extras/migrations/`.
- **Keep schema files in sync.** Apply schema changes to all `schema.prisma` copies (`schema.prisma`, `litellm/proxy/`, `litellm-proxy-extras/`) with a migration under `litellm-proxy-extras/litellm_proxy_extras/migrations/`.
### Setup Wizard (`litellm/setup_wizard.py`)
- The wizard is implemented as a single `SetupWizard` class with `@staticmethod` methods — keep it that way. No module-level functions except `run_setup_wizard()` (the public entrypoint) and pure helpers (color, ANSI).

20
Dockerfile
View file

@ -1,8 +1,8 @@
# Base image for building
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:3258be472764337fd13095bcbb3182da170243b5819fd67ad4c0754590588b31
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:3258be472764337fd13095bcbb3182da170243b5819fd67ad4c0754590588b31
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin
@ -68,8 +68,8 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
USER root
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 libsndfile supervisor && \
npm install -g npm@11.12.1 tar@7.5.11 glob@13.0.6 @isaacs/brace-expansion@5.0.1 brace-expansion@5.0.5 minimatch@10.2.4 diff@8.0.3 picomatch@4.0.4 && \
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 libsndfile && \
npm install -g npm@11.14.0 tar@7.5.11 glob@13.0.6 @isaacs/brace-expansion@5.0.1 brace-expansion@5.0.5 minimatch@10.2.4 diff@8.0.3 picomatch@4.0.4 && \
GLOBAL="$(npm root -g)" && \
for pkg in tar glob @isaacs/brace-expansion brace-expansion minimatch diff picomatch; do \
name="${pkg##*/}"; \
@ -85,17 +85,17 @@ ENV PATH="/app/.venv/bin:${PATH}"
COPY --from=builder /app /app
# Prisma binaries live in $HOME/.cache (default prisma-python location),
# which is /root/.cache here. Copy them from the builder so they survive
# deployments that volume-mount /app/.cache (e.g. readOnlyRootFilesystem
# + emptyDir) — otherwise the mount would shadow the baked-in query engine.
COPY --from=builder /root/.cache /root/.cache
# which is /root/.cache here. Copy only the Prisma subdirs — copying the
# whole /root/.cache drags in the uv build cache (~660 MB, includes a
# setuptools wheel that surfaces as a CVE finding even though it's not
# on the runtime sys.path).
COPY --from=builder /root/.cache/prisma /root/.cache/prisma
COPY --from=builder /root/.cache/prisma-python /root/.cache/prisma-python
RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
find /app/.venv -type d -path "*/tornado/test" -delete
EXPOSE 4000/tcp
COPY docker/supervisord.conf /etc/supervisord.conf
ENTRYPOINT ["docker/prod_entrypoint.sh"]
CMD ["--port", "4000"]

7
codecov.yaml
View file

@ -1,3 +1,8 @@
codecov:
require_ci_to_pass: false # post coverage status even if CI has unrelated failures
notify:
wait_for_ci: false # post as soon as expected uploads arrive, don't wait on CI
component_management:
individual_components:
- component_id: "Router"
@ -28,7 +33,7 @@ coverage:
project:
default:
target: auto
threshold: 1% # at maximum allow project coverage to drop by 1%
threshold: 0% # do not allow project coverage to drop
patch:
default:
target: auto

18
deploy/Dockerfile.ghcr_base
View file

@ -1,18 +0,0 @@
# Use the provided base image
FROM ghcr.io/berriai/litellm:main-latest@sha256:7c311546c25e7bb6e8cafede9fcd3d0d622ac636b5c9418befaa32e85dfb0186
# Set the working directory to /app
WORKDIR /app
# Copy the configuration file into the container at /app
COPY config.yaml .
# Make sure your docker/entrypoint.sh is executable
# Convert Windows line endings to Unix
RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
# Expose the necessary port
EXPOSE 4000/tcp
# Override the CMD instruction with your desired command and arguments
CMD ["--port", "4000", "--config", "config.yaml", "--detailed_debug", "--run_gunicorn"]

44
deploy/charts/litellm-helm/templates/deployment.yaml
View file

@ -100,6 +100,16 @@ spec:
- name: DATABASE_URL
value: {{ .Values.db.url | quote }}
{{- end }}
{{- if and .Values.db.useExisting .Values.db.secret.readReplicaUrlKey }}
- name: DATABASE_URL_READ_REPLICA
valueFrom:
secretKeyRef:
name: {{ .Values.db.secret.name }}
key: {{ .Values.db.secret.readReplicaUrlKey }}
{{- else if .Values.db.readReplicaUrl }}
- name: DATABASE_URL_READ_REPLICA
value: {{ .Values.db.readReplicaUrl | quote }}
{{- end }}
- name: PROXY_MASTER_KEY
valueFrom:
secretKeyRef:
@ -116,21 +126,32 @@ spec:
name: {{ include "redis.secretName" .Subcharts.redis }}
key: {{include "redis.secretPasswordKey" .Subcharts.redis }}
{{- end }}
{{- /*
Inject LITELLM_LOG only when envVars does not already define it.
*/}}
{{- if and .Values.logLevel (not (hasKey (default dict .Values.envVars) "LITELLM_LOG")) }}
- name: LITELLM_LOG
value: {{ .Values.logLevel | quote }}
{{- end }}
{{- if .Values.envVars }}
{{- range $key, $val := .Values.envVars }}
- name: {{ $key }}
value: {{ $val | quote }}
{{- end }}
{{- end }}
{{- if .Values.separateHealthApp }}
- name: SEPARATE_HEALTH_APP
value: "1"
- name: SEPARATE_HEALTH_PORT
value: {{ .Values.separateHealthPort | default "8081" | quote }}
{{- end }}
{{- with .Values.extraEnvVars }}
{{- toYaml . | nindent 12 }}
{{- end }}
{{- if .Values.migrationJob.enabled }}
# Schema updates are owned by the dedicated migrations Job; skip
# the proxy's startup `prisma db push` so N replicas don't race
# one DB on every rollout. Placed last (after envVars and
# extraEnvVars) so this override can't be silently shadowed by a
# user-supplied DISABLE_SCHEMA_UPDATE under last-wins duplicate-env
# semantics — same pattern the migrations Job uses.
- name: DISABLE_SCHEMA_UPDATE
value: "true"
{{- end }}
envFrom:
{{- range .Values.environmentSecrets }}
- secretRef:
@ -158,15 +179,10 @@ spec:
- name: http
containerPort: {{ .Values.service.port }}
protocol: TCP
{{- if .Values.separateHealthApp }}
- name: health
containerPort: {{ .Values.separateHealthPort | default 8081 }}
protocol: TCP
{{- end }}
livenessProbe:
httpGet:
path: {{ .Values.livenessProbe.path | quote }}
port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
port: "http"
initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.livenessProbe.periodSeconds }}
timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }}
@ -175,7 +191,7 @@ spec:
readinessProbe:
httpGet:
path: {{ .Values.readinessProbe.path | quote }}
port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
port: "http"
initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }}
@ -184,7 +200,7 @@ spec:
startupProbe:
httpGet:
path: {{ .Values.startupProbe.path | quote }}
port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
port: "http"
initialDelaySeconds: {{ .Values.startupProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.startupProbe.periodSeconds }}
timeoutSeconds: {{ .Values.startupProbe.timeoutSeconds }}

10
deploy/charts/litellm-helm/tests/deployment_tests.yaml
View file

@ -257,16 +257,16 @@ tests:
value: 0
- equal:
path: spec.template.spec.containers[0].livenessProbe.periodSeconds
value: 10
value: 15
- equal:
path: spec.template.spec.containers[0].livenessProbe.timeoutSeconds
value: 1
value: 5
- equal:
path: spec.template.spec.containers[0].livenessProbe.successThreshold
value: 1
- equal:
path: spec.template.spec.containers[0].livenessProbe.failureThreshold
value: 3
value: 5
- equal:
path: spec.template.spec.containers[0].readinessProbe.httpGet.path
value: /health/readiness
@ -278,7 +278,7 @@ tests:
value: 10
- equal:
path: spec.template.spec.containers[0].readinessProbe.timeoutSeconds
value: 1
value: 5
- equal:
path: spec.template.spec.containers[0].readinessProbe.successThreshold
value: 1
@ -296,7 +296,7 @@ tests:
value: 10
- equal:
path: spec.template.spec.containers[0].startupProbe.timeoutSeconds
value: 1
value: 5
- equal:
path: spec.template.spec.containers[0].startupProbe.successThreshold
value: 1

49
deploy/charts/litellm-helm/values.yaml
View file

@ -88,26 +88,20 @@ service:
# optionally specify loadBalancerClass
# loadBalancerClass: tailscale
# Separate health app configuration
# When enabled, health checks will use a separate port and the application
# will receive SEPARATE_HEALTH_APP=1 and SEPARATE_HEALTH_PORT from environment variables
separateHealthApp: false
separateHealthPort: 8081
# Probe tuning for proxy container
# Probes for LiteLLM gateway container
livenessProbe:
path: /health/liveliness
initialDelaySeconds: 0
periodSeconds: 10
timeoutSeconds: 1
periodSeconds: 15
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 3
failureThreshold: 5
readinessProbe:
path: /health/readiness
initialDelaySeconds: 0
periodSeconds: 10
timeoutSeconds: 1
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 3
@ -115,7 +109,7 @@ startupProbe:
path: /health/readiness
initialDelaySeconds: 0
periodSeconds: 10
timeoutSeconds: 1
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 30
@ -258,6 +252,26 @@ db:
passwordKey: password
# Optional: when set, DATABASE_HOST will be sourced from this secret key instead of db.endpoint
endpointKey: ""
# Optional: when set, DATABASE_URL_READ_REPLICA will be sourced from this
# secret key instead of db.readReplicaUrl. Prefer this over the plain
# value: read-replica URLs typically embed credentials, and a value
# written to db.readReplicaUrl ends up visible in the rendered pod spec
# and the Helm release secret.
readReplicaUrlKey: ""
# Optional read-replica routing. When set, the proxy sends read-only
# queries (find_*, count, group_by, query_raw/_first) to this URL while
# writes continue to go to db.url. Useful for Aurora-style clusters with
# separate reader/writer endpoints. Leave empty to keep single-DB behavior.
# When IAM_TOKEN_DB_AUTH is enabled, the reader URL is auto-refreshed
# alongside the writer (host/port/user/db are parsed from this URL once
# at startup; only the IAM token rotates).
#
# If the URL embeds credentials, prefer db.secret.readReplicaUrlKey over
# this field — the plain value is rendered into the pod spec and the
# Helm release secret. This field is intended for credential-less URLs
# only (e.g. when IAM_TOKEN_DB_AUTH supplies the token at runtime).
readReplicaUrl: ""
# Use the Stackgres Helm chart to deploy an instance of a Stackgres cluster.
# The Stackgres Operator must already be installed within the target
@ -329,6 +343,17 @@ migrationJob:
helm:
enabled: false
# Log level for the litellm proxy (sets LITELLM_LOG in the deployment env).
# Rendered as a direct `env:` entry, which in Kubernetes takes precedence over
# any `envFrom:` source. If you currently source LITELLM_LOG from an
# environmentSecret or environmentConfigMap, set `logLevel: ""` here to
# disable injection — otherwise this value silently overrides your secret /
# configmap entry.
#
# Setting LITELLM_LOG inside `envVars:` below also wins: the template skips
# this injection entirely when envVars already defines LITELLM_LOG.
logLevel: INFO
# Additional environment variables to be added to the deployment as a map of key-value pairs
envVars: {}

56
deploy/kubernetes/kub.yaml
View file

@ -1,56 +0,0 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: litellm-deployment
spec:
replicas: 3
selector:
matchLabels:
app: litellm
template:
metadata:
labels:
app: litellm
spec:
containers:
- name: litellm-container
image: ghcr.io/berriai/litellm:main-latest
imagePullPolicy: Always
env:
- name: AZURE_API_KEY
value: "d6f****"
- name: AZURE_API_BASE
value: "https://openai"
- name: LITELLM_MASTER_KEY
value: "sk-1234"
- name: DATABASE_URL
value: "postgresql://ishaan*********"
args:
- "--config"
- "/app/proxy_config.yaml" # Update the path to mount the config file
volumeMounts: # Define volume mount for proxy_config.yaml
- name: config-volume
mountPath: /app
readOnly: true
livenessProbe:
httpGet:
path: /health/liveliness
port: 4000
initialDelaySeconds: 120
periodSeconds: 15
successThreshold: 1
failureThreshold: 3
timeoutSeconds: 10
readinessProbe:
httpGet:
path: /health/readiness
port: 4000
initialDelaySeconds: 120
periodSeconds: 15
successThreshold: 1
failureThreshold: 3
timeoutSeconds: 10
volumes: # Define volume to mount proxy_config.yaml
- name: config-volume
configMap:
name: litellm-config

12
deploy/kubernetes/service.yaml
View file

@ -1,12 +0,0 @@
apiVersion: v1
kind: Service
metadata:
name: litellm-service
spec:
selector:
app: litellm
ports:
- protocol: TCP
port: 4000
targetPort: 4000
type: LoadBalancer

13
dev_config.yaml
View file

@ -1,13 +0,0 @@
model_list:
- model_name: fake-openai-endpoint
litellm_params:
model: openai/fake-model
api_key: fake-key
api_base: https://exampleopenaiendpoint-production.up.railway.app/
general_settings:
master_key: sk-1234
litellm_settings:
drop_params: True
telemetry: False

5
docker-compose.yml
View file

@ -16,6 +16,11 @@ services:
- "4000:4000" # Map the container port to the host, change the host port if necessary
environment:
DATABASE_URL: "postgresql://llmproxy:dbpassword9090@db:5432/litellm"
# Optional: route read-only queries (find_*, count, group_by, query_raw/_first)
# to a separate reader endpoint, e.g. an Aurora reader. Leave unset for
# single-DB deployments. With IAM_TOKEN_DB_AUTH enabled, the reader URL
# is auto-refreshed alongside the writer.
# DATABASE_URL_READ_REPLICA: "postgresql://llmproxy:dbpassword9090@db-reader:5432/litellm"
STORE_MODEL_IN_DB: "True" # allows adding models to proxy via UI
env_file:
- .env # Load local .env file

68
docker/Dockerfile.alpine
View file

@ -1,68 +0,0 @@
# Base image for building
ARG LITELLM_BUILD_IMAGE=python:3.11-alpine@sha256:f07e2ace46f560f09a6eeec7b4913b80ee99546e749ef82342a419a326620856
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=python:3.11-alpine@sha256:f07e2ace46f560f09a6eeec7b4913b80ee99546e749ef82342a419a326620856
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin
FROM $LITELLM_BUILD_IMAGE AS builder
WORKDIR /app
COPY --from=uvbin /uv /usr/local/bin/uv
COPY --from=uvbin /uvx /usr/local/bin/uvx
RUN apk add --no-cache gcc python3-dev musl-dev nodejs npm libsndfile
ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
UV_PROJECT_ENVIRONMENT=/app/.venv \
UV_LINK_MODE=copy \
XDG_CACHE_HOME=/app/.cache \
PATH="/app/.venv/bin:${PATH}"
# Copy dependency metadata first for layer caching
COPY pyproject.toml uv.lock ./
COPY enterprise/pyproject.toml enterprise/
COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/
# Install third-party dependencies (cached unless pyproject.toml/uv.lock change)
RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--extra extra_proxy \
--extra semantic-router \
--python python3
# Copy full source tree
COPY . .
# Install project and workspace packages (fast - deps already cached)
RUN uv sync --frozen --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--extra extra_proxy \
--extra semantic-router \
--python python3
RUN prisma generate --schema=./schema.prisma
RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
FROM $LITELLM_RUNTIME_IMAGE AS runtime
RUN apk upgrade --no-cache && apk add --no-cache libsndfile nodejs npm
WORKDIR /app
ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
XDG_CACHE_HOME=/app/.cache \
PATH="/app/.venv/bin:${PATH}"
COPY --from=builder /app /app
EXPOSE 4000/tcp
ENTRYPOINT ["docker/prod_entrypoint.sh"]
CMD ["--port", "4000"]

86
docker/Dockerfile.custom_ui
View file

@ -1,86 +0,0 @@
# Use the provided base image
# NOTE: This is a dev/branch-specific tag. Update digest when the base image is rebuilt.
FROM ghcr.io/berriai/litellm:litellm_fwd_server_root_path-dev
# Set the working directory to /app
WORKDIR /app
# Install Node.js and npm (adjust version as needed)
RUN apt-get update && apt-get upgrade -y \
libxml2 \
libexpat1 \
openssl \
libssl3 \
git \
libkrb5-3 \
libglib2.0-0 \
wget \
libaom3 \
libxslt1.1 \
libgnutls30 \
libc6 && \
apt-get install -y --no-install-recommends nodejs npm && \
npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
GLOBAL="$(npm root -g)" && \
find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
done && \
find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
done && \
find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
done && \
find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
done && \
find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
done && \
find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \
npm cache clean --force && \
apt-get purge -y npm
# Copy the UI source into the container
COPY ./ui/litellm-dashboard /app/ui/litellm-dashboard
# Set an environment variable for UI_BASE_PATH
# This can be overridden at build time
# set UI_BASE_PATH to "<your server root path>/ui"
ENV UI_BASE_PATH="/prod/ui"
# Build the UI with the specified UI_BASE_PATH
WORKDIR /app/ui/litellm-dashboard
RUN npm ci
RUN UI_BASE_PATH=$UI_BASE_PATH npm run build
# Create the destination directory
RUN mkdir -p /app/litellm/proxy/_experimental/out
# Move the built files to the appropriate location
# Assuming the build output is in ./out directory
RUN rm -rf /app/litellm/proxy/_experimental/out/* && \
mv ./out/* /app/litellm/proxy/_experimental/out/
# Switch back to the main app directory
WORKDIR /app
# Make sure your docker/entrypoint.sh is executable
# Convert Windows line endings to Unix for entrypoint scripts
RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
# Run as non-root user
RUN groupadd --gid 1000 appuser && useradd --uid 1000 --gid 1000 --no-create-home appuser \
&& chown -R appuser:appuser /app
USER appuser
# Expose the necessary port
EXPOSE 4000/tcp
HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:4000/health')"]
# Override the CMD instruction with your desired command and arguments
CMD ["--port", "4000", "--config", "config.yaml", "--detailed_debug"]

4
docker/Dockerfile.database
View file

@ -66,7 +66,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
USER root
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 libsndfile supervisor && \
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 libsndfile && \
npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
GLOBAL="$(npm root -g)" && \
find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
@ -102,7 +102,5 @@ RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
EXPOSE 4000/tcp
COPY docker/supervisord.conf /etc/supervisord.conf
ENTRYPOINT ["docker/prod_entrypoint.sh"]
CMD ["--port", "4000"]

121
docker/Dockerfile.dev
View file

@ -1,121 +0,0 @@
# Base image for building
ARG LITELLM_BUILD_IMAGE=python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin
FROM $LITELLM_BUILD_IMAGE AS builder
WORKDIR /app
USER root
COPY --from=uvbin /uv /usr/local/bin/uv
COPY --from=uvbin /uvx /usr/local/bin/uvx
RUN apt-get update && apt-get install -y --no-install-recommends \
gcc \
g++ \
python3-dev \
libssl-dev \
pkg-config \
nodejs \
npm \
&& rm -rf /var/lib/apt/lists/*
ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
UV_PROJECT_ENVIRONMENT=/app/.venv \
UV_LINK_MODE=copy \
XDG_CACHE_HOME=/app/.cache \
PATH="/app/.venv/bin:${PATH}"
# Copy dependency metadata first for layer caching
COPY pyproject.toml uv.lock ./
COPY enterprise/pyproject.toml enterprise/
COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/
# Install third-party dependencies (cached unless pyproject.toml/uv.lock change)
RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--extra extra_proxy \
--extra semantic-router \
--python python
# Copy full source tree
COPY . .
# Build Admin UI before final sync
RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
# Install project and workspace packages (fast - deps already cached)
RUN uv sync --frozen --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--extra extra_proxy \
--extra semantic-router \
--python python
RUN prisma generate --schema=./schema.prisma
RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
FROM $LITELLM_RUNTIME_IMAGE AS runtime
USER root
RUN apt-get update && apt-get upgrade -y \
libxml2 \
libexpat1 \
openssl \
libssl3 \
git \
libkrb5-3 \
libglib2.0-0 \
wget \
libaom3 \
libxslt1.1 \
libgnutls30 \
libc6 \
&& apt-get install -y --no-install-recommends \
libssl3 \
libatomic1 \
nodejs \
npm \
&& rm -rf /var/lib/apt/lists/* \
&& npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \
&& GLOBAL="$(npm root -g)" \
&& find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
done \
&& find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
done \
&& find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
done \
&& find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
done \
&& find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
done \
&& find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null \
&& npm cache clean --force \
&& apt-get purge -y npm
WORKDIR /app
ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
XDG_CACHE_HOME=/app/.cache \
PATH="/app/.venv/bin:${PATH}"
COPY --from=builder /app /app
EXPOSE 4000/tcp
ENTRYPOINT ["docker/prod_entrypoint.sh"]
CMD ["--port", "4000"]

30
docker/Dockerfile.health_check
View file

@ -1,30 +0,0 @@
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin
FROM python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d
WORKDIR /app
# Copy the uv binary and the health check script.
COPY --from=uvbin /uv /usr/local/bin/uv
COPY pyproject.toml uv.lock /app/
COPY scripts/health_check/health_check_client.py /app/health_check_client.py
# Resolve and install the health-check dependencies from the project lockfile
# so the runtime image stays self-contained and reproducible.
RUN uv export --frozen --no-default-groups --only-group healthcheck --no-emit-project --no-hashes --output-file /tmp/health-check-requirements.txt \
&& uv pip install --system -r /tmp/health-check-requirements.txt \
&& rm /tmp/health-check-requirements.txt \
&& rm /app/pyproject.toml /app/uv.lock \
&& chmod +x /app/health_check_client.py
# Run as non-root user
RUN groupadd --gid 1000 appuser && useradd --uid 1000 --gid 1000 --no-create-home appuser
USER appuser
# Health check
HEALTHCHECK --interval=30s --timeout=5s --start-period=5s --retries=3 \
CMD ["python", "/app/health_check_client.py", "--help"]
# Set entrypoint
ENTRYPOINT ["python", "/app/health_check_client.py"]

3
docker/Dockerfile.non_root
View file

@ -103,13 +103,12 @@ RUN for i in 1 2 3; do \
apk upgrade --no-cache && break || sleep 5; \
done && \
for i in 1 2 3; do \
apk add --no-cache python3 bash openssl tzdata supervisor libsndfile nodejs && break || sleep 5; \
apk add --no-cache python3 bash openssl tzdata libsndfile nodejs && break || sleep 5; \
done
COPY --from=builder /app /app
COPY --from=builder /var/lib/litellm/ui /var/lib/litellm/ui
COPY --from=builder /var/lib/litellm/assets /var/lib/litellm/assets
COPY --from=builder /app/docker/supervisord.conf /etc/supervisord.conf
ENV PATH="/app/.venv/bin:${PATH}" \
PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \

8
docker/prod_entrypoint.sh
View file

@ -1,14 +1,8 @@
#!/bin/sh
if [ "$SEPARATE_HEALTH_APP" = "1" ]; then
export LITELLM_ARGS="$@"
export SUPERVISORD_STOPWAITSECS="${SUPERVISORD_STOPWAITSECS:-3600}"
exec supervisord -c /etc/supervisord.conf
fi
if [ "$USE_DDTRACE" = "true" ]; then
export DD_TRACE_OPENAI_ENABLED="False"
exec ddtrace-run litellm "$@"
else
exec litellm "$@"
fi
fi

46
docker/supervisord.conf
View file

@ -1,46 +0,0 @@
[supervisord]
nodaemon=true
loglevel=info
logfile=/tmp/supervisord.log
pidfile=/tmp/supervisord.pid
[group:litellm]
programs=main,health
[program:main]
command=sh -c 'if [ "$USE_DDTRACE" = "true" ]; then export DD_TRACE_OPENAI_ENABLED="False"; exec ddtrace-run python -m litellm.proxy.proxy_cli --host 0.0.0.0 --port=4000 $LITELLM_ARGS; else exec python -m litellm.proxy.proxy_cli --host 0.0.0.0 --port=4000 $LITELLM_ARGS; fi'
autostart=true
autorestart=true
startretries=3
priority=1
exitcodes=0
stopasgroup=true
killasgroup=true
stopwaitsecs=%(ENV_SUPERVISORD_STOPWAITSECS)s
stdout_logfile=/dev/stdout
stderr_logfile=/dev/stderr
stdout_logfile_maxbytes = 0
stderr_logfile_maxbytes = 0
environment=PYTHONUNBUFFERED=true
[program:health]
command=sh -c '[ "$SEPARATE_HEALTH_APP" = "1" ] && exec uvicorn litellm.proxy.health_endpoints.health_app_factory:build_health_app --factory --host 0.0.0.0 --port=${SEPARATE_HEALTH_PORT:-4001} || exit 0'
autostart=true
autorestart=true
startretries=3
priority=2
exitcodes=0
stopasgroup=true
killasgroup=true
stopwaitsecs=%(ENV_SUPERVISORD_STOPWAITSECS)s
stdout_logfile=/dev/stdout
stderr_logfile=/dev/stderr
stdout_logfile_maxbytes = 0
stderr_logfile_maxbytes = 0
environment=PYTHONUNBUFFERED=true
[eventlistener:process_monitor]
command=python -c "from supervisor import childutils; import os, signal; [os.kill(os.getppid(), signal.SIGTERM) for h,p in iter(lambda: childutils.listener.wait(), None) if h['eventname'] in ['PROCESS_STATE_FATAL', 'PROCESS_STATE_EXITED'] and dict([x.split(':') for x in p.split(' ')])['processname'] in ['main', 'health'] or childutils.listener.ok()]"
events=PROCESS_STATE_EXITED,PROCESS_STATE_FATAL
autostart=true
autorestart=true

BIN
docs/images/local-testing/hosted-vllm-custom-tool-local-test.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 82 KiB

36
enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
View file

@ -300,6 +300,42 @@ class CheckBatchCost:
custom_llm_provider=custom_llm_provider,
)
# CheckBatchCost bypasses async_post_call_success_hook, so convert raw
# output/error file IDs to managed base64 IDs before the DB write here.
managed_files_hook = self.proxy_logging_obj.get_proxy_hook("managed_files")
if managed_files_hook is not None:
from litellm.proxy._types import UserAPIKeyAuth
_minimal_auth = UserAPIKeyAuth(
user_id=job.created_by or "default-user-id",
team_id=getattr(job, "team_id", None),
)
for _file_attr in ["output_file_id", "error_file_id"]:
_raw_file_id = getattr(response, _file_attr, None)
if _raw_file_id and not _is_base64_encoded_unified_file_id(_raw_file_id):
try:
_unified_file_id = managed_files_hook.get_unified_output_file_id(
output_file_id=_raw_file_id,
model_id=model_id,
model_name=str(model_name) if model_name else deployment_info.model_name or None,
)
await managed_files_hook.store_unified_file_id(
file_id=_unified_file_id,
file_object=None,
litellm_parent_otel_span=None,
model_mappings={model_id: _raw_file_id},
user_api_key_dict=_minimal_auth,
)
setattr(response, _file_attr, _unified_file_id)
verbose_proxy_logger.info(
f"CheckBatchCost: converted {_file_attr} "
f"{_raw_file_id!r} -> managed ID for batch {batch_id}"
)
except Exception as _e:
verbose_proxy_logger.warning(
f"CheckBatchCost: failed to create managed file ID for "
f"{_file_attr}={_raw_file_id!r}: {_e}"
)
# Pass deployment model_info so custom batch pricing
# (input_cost_per_token_batches etc.) is used for cost calc
deployment_model_info = deployment_info.model_info.model_dump() if deployment_info.model_info else {}

108
index.yaml
View file

@ -1,108 +0,0 @@
apiVersion: v1
entries:
litellm-helm:
- apiVersion: v2
appVersion: v1.43.18
created: "2024-08-19T23:58:25.331689+08:00"
dependencies:
- condition: db.deployStandalone
name: postgresql
repository: oci://registry-1.docker.io/bitnamicharts
version: '>=13.3.0'
- condition: redis.enabled
name: redis
repository: oci://registry-1.docker.io/bitnamicharts
version: '>=18.0.0'
description: Call all LLM APIs using the OpenAI format
digest: 0411df3dc42868be8af3ad3e00cb252790e6bd7ad15f5b77f1ca5214573a8531
name: litellm-helm
type: application
urls:
- https://berriai.github.io/litellm/litellm-helm-0.2.3.tgz
version: 0.2.3
postgresql:
- annotations:
category: Database
images: |
- name: os-shell
image: docker.io/bitnami/os-shell:12-debian-12-r16
- name: postgres-exporter
image: docker.io/bitnami/postgres-exporter:0.15.0-debian-12-r14
- name: postgresql
image: docker.io/bitnami/postgresql:16.2.0-debian-12-r6
licenses: Apache-2.0
apiVersion: v2
appVersion: 16.2.0
created: "2024-08-19T23:58:25.335716+08:00"
dependencies:
- name: common
repository: oci://registry-1.docker.io/bitnamicharts
tags:
- bitnami-common
version: 2.x.x
description: PostgreSQL (Postgres) is an open source object-relational database
known for reliability and data integrity. ACID-compliant, it supports foreign
keys, joins, views, triggers and stored procedures.
digest: 3c8125526b06833df32e2f626db34aeaedb29d38f03d15349db6604027d4a167
home: https://bitnami.com
icon: https://bitnami.com/assets/stacks/postgresql/img/postgresql-stack-220x234.png
keywords:
- postgresql
- postgres
- database
- sql
- replication
- cluster
maintainers:
- name: VMware, Inc.
url: https://github.com/bitnami/charts
name: postgresql
sources:
- https://github.com/bitnami/charts/tree/main/bitnami/postgresql
urls:
- https://berriai.github.io/litellm/charts/postgresql-14.3.1.tgz
version: 14.3.1
redis:
- annotations:
category: Database
images: |
- name: kubectl
image: docker.io/bitnami/kubectl:1.29.2-debian-12-r3
- name: os-shell
image: docker.io/bitnami/os-shell:12-debian-12-r16
- name: redis
image: docker.io/bitnami/redis:7.2.4-debian-12-r9
- name: redis-exporter
image: docker.io/bitnami/redis-exporter:1.58.0-debian-12-r4
- name: redis-sentinel
image: docker.io/bitnami/redis-sentinel:7.2.4-debian-12-r7
licenses: Apache-2.0
apiVersion: v2
appVersion: 7.2.4
created: "2024-08-19T23:58:25.339392+08:00"
dependencies:
- name: common
repository: oci://registry-1.docker.io/bitnamicharts
tags:
- bitnami-common
version: 2.x.x
description: Redis(R) is an open source, advanced key-value store. It is often
referred to as a data structure server since keys can contain strings, hashes,
lists, sets and sorted sets.
digest: b2fa1835f673a18002ca864c54fadac3c33789b26f6c5e58e2851b0b14a8f984
home: https://bitnami.com
icon: https://bitnami.com/assets/stacks/redis/img/redis-stack-220x234.png
keywords:
- redis
- keyvalue
- database
maintainers:
- name: VMware, Inc.
url: https://github.com/bitnami/charts
name: redis
sources:
- https://github.com/bitnami/charts/tree/main/bitnami/redis
urls:
- https://berriai.github.io/litellm/charts/redis-18.19.1.tgz
version: 18.19.1
generated: "2024-08-19T23:58:25.322532+08:00"

5
litellm-js/proxy/.npmrc
View file

@ -1,5 +0,0 @@
# Supply-chain hardening
# Packages needing lifecycle scripts: npm rebuild <pkg>
ignore-scripts=true
# Protects local npm install only — npm ci (used in CI) ignores this
min-release-age=3

8
litellm-js/proxy/README.md
View file

@ -1,8 +0,0 @@
```
npm install
npm run dev
```
```
npm run deploy
```

2054
litellm-js/proxy/package-lock.json generated
View file

File diff suppressed because it is too large Load diff

14
litellm-js/proxy/package.json
View file

@ -1,14 +0,0 @@
{
"scripts": {
"dev": "wrangler dev src/index.ts",
"deploy": "wrangler deploy --minify src/index.ts"
},
"dependencies": {
"hono": "4.12.16",
"openai": "4.29.2"
},
"devDependencies": {
"@cloudflare/workers-types": "4.20260501.1",
"wrangler": "4.87.0"
}
}

59
litellm-js/proxy/src/index.ts
View file

@ -1,59 +0,0 @@
import { Hono } from 'hono'
import { Context } from 'hono';
import { bearerAuth } from 'hono/bearer-auth'
import OpenAI from "openai";
const openai = new OpenAI({
apiKey: "sk-1234",
baseURL: "https://openai-endpoint.ishaanjaffer0324.workers.dev"
});
async function call_proxy() {
const completion = await openai.chat.completions.create({
messages: [{ role: "system", content: "You are a helpful assistant." }],
model: "gpt-3.5-turbo",
});
return completion
}
const app = new Hono()
// Middleware for API Key Authentication
const apiKeyAuth = async (c: Context, next: Function) => {
const apiKey = c.req.header('Authorization');
if (!apiKey || apiKey !== 'Bearer sk-1234') {
return c.text('Unauthorized', 401);
}
await next();
};
app.use('/*', apiKeyAuth)
app.get('/', (c) => {
return c.text('Hello Hono!')
})
// Handler for chat completions
const chatCompletionHandler = async (c: Context) => {
// Assuming your logic for handling chat completion goes here
// For demonstration, just returning a simple JSON response
const response = await call_proxy()
return c.json(response);
};
// Register the above handler for different POST routes with the apiKeyAuth middleware
app.post('/v1/chat/completions', chatCompletionHandler);
app.post('/chat/completions', chatCompletionHandler);
// Example showing how you might handle dynamic segments within the URL
// Here, using ':model*' to capture the rest of the path as a parameter 'model'
app.post('/openai/deployments/:model*/chat/completions', chatCompletionHandler);
export default app

17
litellm-js/proxy/tsconfig.json
View file

@ -1,17 +0,0 @@
{
"compilerOptions": {
"target": "ESNext",
"module": "ESNext",
"moduleResolution": "Bundler",
"strict": true,
"lib": [
"ESNext"
],
"types": [
"@cloudflare/workers-types"
],
"jsx": "react-jsx",
"jsxImportSource": "hono/jsx",
"skipLibCheck": true
},
}

18
litellm-js/proxy/wrangler.toml
View file

@ -1,18 +0,0 @@
name = "my-app"
compatibility_date = "2023-12-01"
# [vars]
# MY_VAR = "my-variable"
# [[kv_namespaces]]
# binding = "MY_KV_NAMESPACE"
# id = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
# [[r2_buckets]]
# binding = "MY_BUCKET"
# bucket_name = "my-bucket"
# [[d1_databases]]
# binding = "DB"
# database_name = "my-database"
# database_id = ""

5
litellm-js/spend-logs/.npmrc
View file

@ -1,5 +0,0 @@
# Supply-chain hardening
# Packages needing lifecycle scripts: npm rebuild <pkg>
ignore-scripts=true
# Protects local npm install only — npm ci (used in CI) ignores this
min-release-age=3

26
litellm-js/spend-logs/Dockerfile
View file

@ -1,26 +0,0 @@
# Use the specific Node.js v20.11.0 image
FROM node:20.18.1-alpine3.20
# Set the working directory inside the container
WORKDIR /app
# Copy package.json and package-lock.json to the working directory
COPY ./litellm-js/spend-logs/package*.json ./
# Install dependencies
RUN npm ci
# Install Prisma globally
RUN npm install -g prisma
# Copy the rest of the application code
COPY ./litellm-js/spend-logs .
# Generate Prisma client
RUN npx prisma generate
# Expose the port that the Node.js server will run on
EXPOSE 3000
# Command to run the Node.js app with npm run dev
CMD ["npm", "run", "dev"]

8
litellm-js/spend-logs/README.md
View file

@ -1,8 +0,0 @@
```
npm install
npm run dev
```
```
open http://localhost:3000
```

597
litellm-js/spend-logs/package-lock.json generated
View file

@ -1,597 +0,0 @@
{
"name": "spend-logs",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"dependencies": {
"@hono/node-server": "1.19.13",
"hono": "4.12.16"
},
"devDependencies": {
"@types/node": "20.19.25",
"tsx": "4.20.6"
}
},
"node_modules/@esbuild/aix-ppc64": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz",
"integrity": "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==",
"cpu": [
"ppc64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"aix"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/android-arm": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.12.tgz",
"integrity": "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==",
"cpu": [
"arm"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"android"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/android-arm64": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.12.tgz",
"integrity": "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==",
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"android"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/android-x64": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.12.tgz",
"integrity": "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==",
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"android"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/darwin-arm64": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.12.tgz",
"integrity": "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==",
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/darwin-x64": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.12.tgz",
"integrity": "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==",
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/freebsd-arm64": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.12.tgz",
"integrity": "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==",
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"freebsd"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/freebsd-x64": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.12.tgz",
"integrity": "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==",
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"freebsd"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/linux-arm": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.12.tgz",
"integrity": "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==",
"cpu": [
"arm"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/linux-arm64": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.12.tgz",
"integrity": "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==",
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/linux-ia32": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.12.tgz",
"integrity": "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==",
"cpu": [
"ia32"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/linux-loong64": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.12.tgz",
"integrity": "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==",
"cpu": [
"loong64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/linux-mips64el": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.12.tgz",
"integrity": "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==",
"cpu": [
"mips64el"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/linux-ppc64": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.12.tgz",
"integrity": "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==",
"cpu": [
"ppc64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/linux-riscv64": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.12.tgz",
"integrity": "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==",
"cpu": [
"riscv64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/linux-s390x": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.12.tgz",
"integrity": "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==",
"cpu": [
"s390x"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/linux-x64": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.12.tgz",
"integrity": "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==",
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/netbsd-arm64": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.12.tgz",
"integrity": "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==",
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"netbsd"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/netbsd-x64": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.12.tgz",
"integrity": "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==",
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"netbsd"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/openbsd-arm64": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.12.tgz",
"integrity": "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==",
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"openbsd"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/openbsd-x64": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.12.tgz",
"integrity": "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==",
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"openbsd"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/openharmony-arm64": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.12.tgz",
"integrity": "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==",
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"openharmony"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/sunos-x64": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.12.tgz",
"integrity": "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==",
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"sunos"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/win32-arm64": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.12.tgz",
"integrity": "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==",
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"win32"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/win32-ia32": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.12.tgz",
"integrity": "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==",
"cpu": [
"ia32"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"win32"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/win32-x64": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.12.tgz",
"integrity": "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==",
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"win32"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@hono/node-server": {
"version": "1.19.13",
"resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-1.19.13.tgz",
"integrity": "sha512-TsQLe4i2gvoTtrHje625ngThGBySOgSK3Xo2XRYOdqGN1teR8+I7vchQC46uLJi8OF62YTYA3AhSpumtkhsaKQ==",
"license": "MIT",
"engines": {
"node": ">=18.14.1"
},
"peerDependencies": {
"hono": "^4"
}
},
"node_modules/@types/node": {
"version": "20.19.25",
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.25.tgz",
"integrity": "sha512-ZsJzA5thDQMSQO788d7IocwwQbI8B5OPzmqNvpf3NY/+MHDAS759Wo0gd2WQeXYt5AAAQjzcrTVC6SKCuYgoCQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"undici-types": "~6.21.0"
}
},
"node_modules/esbuild": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.12.tgz",
"integrity": "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==",
"dev": true,
"hasInstallScript": true,
"license": "MIT",
"bin": {
"esbuild": "bin/esbuild"
},
"engines": {
"node": ">=18"
},
"optionalDependencies": {
"@esbuild/aix-ppc64": "0.25.12",
"@esbuild/android-arm": "0.25.12",
"@esbuild/android-arm64": "0.25.12",
"@esbuild/android-x64": "0.25.12",
"@esbuild/darwin-arm64": "0.25.12",
"@esbuild/darwin-x64": "0.25.12",
"@esbuild/freebsd-arm64": "0.25.12",
"@esbuild/freebsd-x64": "0.25.12",
"@esbuild/linux-arm": "0.25.12",
"@esbuild/linux-arm64": "0.25.12",
"@esbuild/linux-ia32": "0.25.12",
"@esbuild/linux-loong64": "0.25.12",
"@esbuild/linux-mips64el": "0.25.12",
"@esbuild/linux-ppc64": "0.25.12",
"@esbuild/linux-riscv64": "0.25.12",
"@esbuild/linux-s390x": "0.25.12",
"@esbuild/linux-x64": "0.25.12",
"@esbuild/netbsd-arm64": "0.25.12",
"@esbuild/netbsd-x64": "0.25.12",
"@esbuild/openbsd-arm64": "0.25.12",
"@esbuild/openbsd-x64": "0.25.12",
"@esbuild/openharmony-arm64": "0.25.12",
"@esbuild/sunos-x64": "0.25.12",
"@esbuild/win32-arm64": "0.25.12",
"@esbuild/win32-ia32": "0.25.12",
"@esbuild/win32-x64": "0.25.12"
}
},
"node_modules/fsevents": {
"version": "2.3.3",
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz",
"integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==",
"dev": true,
"hasInstallScript": true,
"license": "MIT",
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
}
},
"node_modules/get-tsconfig": {
"version": "4.13.0",
"resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.13.0.tgz",
"integrity": "sha512-1VKTZJCwBrvbd+Wn3AOgQP/2Av+TfTCOlE4AcRJE72W1ksZXbAx8PPBR9RzgTeSPzlPMHrbANMH3LbltH73wxQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"resolve-pkg-maps": "^1.0.0"
},
"funding": {
"url": "https://github.com/privatenumber/get-tsconfig?sponsor=1"
}
},
"node_modules/hono": {
"version": "4.12.16",
"resolved": "https://registry.npmjs.org/hono/-/hono-4.12.16.tgz",
"integrity": "sha512-jN0ZewiNAWSe5khM3EyCmBb250+b40wWbwNILNfEvq84VREWwOIkuUsFONk/3i3nqkz7Oe1PcpM2mwQEK2L9Kg==",
"license": "MIT",
"engines": {
"node": ">=16.9.0"
}
},
"node_modules/resolve-pkg-maps": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz",
"integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==",
"dev": true,
"license": "MIT",
"funding": {
"url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1"
}
},
"node_modules/tsx": {
"version": "4.20.6",
"resolved": "https://registry.npmjs.org/tsx/-/tsx-4.20.6.tgz",
"integrity": "sha512-ytQKuwgmrrkDTFP4LjR0ToE2nqgy886GpvRSpU0JAnrdBYppuY5rLkRUYPU1yCryb24SsKBTL/hlDQAEFVwtZg==",
"dev": true,
"license": "MIT",
"dependencies": {
"esbuild": "~0.25.0",
"get-tsconfig": "^4.7.5"
},
"bin": {
"tsx": "dist/cli.mjs"
},
"engines": {
"node": ">=18.0.0"
},
"optionalDependencies": {
"fsevents": "~2.3.3"
}
},
"node_modules/undici-types": {
"version": "6.21.0",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz",
"integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==",
"dev": true,
"license": "MIT"
}
}
}

13
litellm-js/spend-logs/package.json
View file

@ -1,13 +0,0 @@
{
"scripts": {
"dev": "tsx watch src/index.ts"
},
"dependencies": {
"@hono/node-server": "1.19.13",
"hono": "4.12.16"
},
"devDependencies": {
"@types/node": "20.19.25",
"tsx": "4.20.6"
}
}

29
litellm-js/spend-logs/schema.prisma
View file

@ -1,29 +0,0 @@
generator client {
provider = "prisma-client-js"
}
datasource client {
provider = "postgresql"
url = env("DATABASE_URL")
}
model LiteLLM_SpendLogs {
request_id String @id
call_type String
api_key String @default("")
spend Float @default(0.0)
total_tokens Int @default(0)
prompt_tokens Int @default(0)
completion_tokens Int @default(0)
startTime DateTime
endTime DateTime
model String @default("")
api_base String @default("")
user String @default("")
metadata Json @default("{}")
cache_hit String @default("")
cache_key String @default("")
request_tags Json @default("[]")
team_id String?
end_user String?
}

32
litellm-js/spend-logs/src/_types.ts
View file

@ -1,32 +0,0 @@
export type LiteLLM_IncrementSpend = {
key_transactions: Array<LiteLLM_IncrementObject>, // [{"key": spend},..]
user_transactions: Array<LiteLLM_IncrementObject>,
team_transactions: Array<LiteLLM_IncrementObject>,
spend_logs_transactions: Array<LiteLLM_SpendLogs>
}
export type LiteLLM_IncrementObject = {
key: string,
spend: number
}
export type LiteLLM_SpendLogs = {
request_id: string; // @id means it's a unique identifier
call_type: string;
api_key: string; // @default("") means it defaults to an empty string if not provided
spend: number; // Float in Prisma corresponds to number in TypeScript
total_tokens: number; // Int in Prisma corresponds to number in TypeScript
prompt_tokens: number;
completion_tokens: number;
startTime: Date; // DateTime in Prisma corresponds to Date in TypeScript
endTime: Date;
model: string; // @default("") means it defaults to an empty string if not provided
api_base: string;
user: string;
metadata: any; // Json type in Prisma is represented by any in TypeScript; could also use a more specific type if the structure of JSON is known
cache_hit: string;
cache_key: string;
request_tags: any; // Similarly, this could be an array or a more specific type depending on the expected structure
team_id?: string | null; // ? indicates it's optional and can be undefined, but could also be null if not provided
end_user?: string | null;
};

84
litellm-js/spend-logs/src/index.ts
View file

@ -1,84 +0,0 @@
import { serve } from '@hono/node-server'
import { Hono } from 'hono'
import { PrismaClient } from '@prisma/client'
import {LiteLLM_SpendLogs, LiteLLM_IncrementSpend, LiteLLM_IncrementObject} from './_types'
const app = new Hono()
const prisma = new PrismaClient()
// In-memory storage for logs
let spend_logs: LiteLLM_SpendLogs[] = [];
const key_logs: LiteLLM_IncrementObject[] = [];
const user_logs: LiteLLM_IncrementObject[] = [];
const transaction_logs: LiteLLM_IncrementObject[] = [];
app.get('/', (c) => {
return c.text('Hello Hono!')
})
const MIN_LOGS = 1; // Minimum number of logs needed to initiate a flush
const FLUSH_INTERVAL = 5000; // Time in ms to wait before trying to flush again
const BATCH_SIZE = 100; // Preferred size of each batch to write to the database
const MAX_LOGS_PER_INTERVAL = 1000; // Maximum number of logs to flush in a single interval
const flushLogsToDb = async () => {
if (spend_logs.length >= MIN_LOGS) {
// Limit the logs to process in this interval to MAX_LOGS_PER_INTERVAL or less
const logsToProcess = spend_logs.slice(0, MAX_LOGS_PER_INTERVAL);
for (let i = 0; i < logsToProcess.length; i += BATCH_SIZE) {
// Create subarray for current batch, ensuring it doesn't exceed the BATCH_SIZE
const batch = logsToProcess.slice(i, i + BATCH_SIZE);
// Convert datetime strings to Date objects
const batchWithDates = batch.map(entry => ({
...entry,
startTime: new Date(entry.startTime),
endTime: new Date(entry.endTime),
// Repeat for any other DateTime fields you may have
}));
await prisma.liteLLM_SpendLogs.createMany({
data: batchWithDates,
});
console.log(`Flushed ${batch.length} logs to the DB.`);
}
// Remove the processed logs from spend_logs
spend_logs = spend_logs.slice(logsToProcess.length);
console.log(`${logsToProcess.length} logs processed. Remaining in queue: ${spend_logs.length}`);
} else {
// This will ensure it doesn't falsely claim "No logs to flush." when it's merely below the MIN_LOGS threshold.
if(spend_logs.length > 0) {
console.log(`Accumulating logs. Currently at ${spend_logs.length}, waiting for at least ${MIN_LOGS}.`);
} else {
console.log("No logs to flush.");
}
}
};
// Setup interval for attempting to flush the logs
setInterval(flushLogsToDb, FLUSH_INTERVAL);
// Route to receive log messages
app.post('/spend/update', async (c) => {
const incomingLogs = await c.req.json<LiteLLM_SpendLogs[]>();
spend_logs.push(...incomingLogs);
console.log(`Received and stored ${incomingLogs.length} logs. Total logs in memory: ${spend_logs.length}`);
return c.json({ message: `Successfully stored ${incomingLogs.length} logs` });
});
const port = 3000
console.log(`Server is running on port ${port}`)
serve({
fetch: app.fetch,
port
})

13
litellm-js/spend-logs/tsconfig.json
View file

@ -1,13 +0,0 @@
{
"compilerOptions": {
"target": "ESNext",
"module": "ESNext",
"moduleResolution": "Bundler",
"strict": true,
"types": [
"node"
],
"jsx": "react-jsx",
"jsxImportSource": "hono/jsx",
}
}

2
litellm-proxy-extras/litellm_proxy_extras/migrations/20260421135425_add_team_membership_total_spend/migration.sql
View file

@ -1,3 +1,3 @@
-- AlterTable
ALTER TABLE "LiteLLM_TeamMembership" ADD COLUMN "total_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
ALTER TABLE "LiteLLM_TeamMembership" ADD COLUMN IF NOT EXISTS "total_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;

2
litellm-proxy-extras/litellm_proxy_extras/migrations/20260513120000_add_delegate_auth_to_upstream_to_mcp_servers/migration.sql Normal file
View file

@ -0,0 +1,2 @@
-- AlterTable
ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN IF NOT EXISTS "delegate_auth_to_upstream" BOOLEAN NOT NULL DEFAULT false;

1
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -323,6 +323,7 @@ model LiteLLM_MCPServerTable {
registration_url String?
allow_all_keys Boolean @default(false)
available_on_public_internet Boolean @default(true)
delegate_auth_to_upstream Boolean @default(false)
is_byok Boolean @default(false)
byok_description String[] @default([])
byok_api_key_help_url String?

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
version = "0.4.71"
version = "0.4.72"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.4.71"
version = "0.4.72"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-proxy-extras==",

19
litellm/__init__.py
View file

@ -206,6 +206,7 @@ add_user_information_to_llm_headers: Optional[bool] = (
)
store_audit_logs = False # Enterprise feature, allow users to see audit logs
skip_system_message_in_guardrail: bool = False
skip_tool_message_in_guardrail: bool = False
### end of callbacks #############
email: Optional[str] = (
@ -388,6 +389,7 @@ anthropic_beta_headers_url: str = os.getenv(
suppress_debug_info = False
dynamodb_table_name: Optional[str] = None
s3_callback_params: Optional[Dict] = None
s3_audit_callback_params: Optional[Dict] = None
datadog_llm_observability_params: Optional[Union[DatadogLLMObsInitParams, Dict]] = None
datadog_params: Optional[Union[DatadogInitParams, Dict]] = None
aws_sqs_callback_params: Optional[Dict] = None
@ -414,6 +416,9 @@ custom_prometheus_metadata_labels: List[str] = []
custom_prometheus_tags: List[str] = []
prometheus_metrics_config: Optional[List] = None
prometheus_emit_stream_label: bool = False
prometheus_end_user_metrics_max_series_per_metric: Optional[int] = 10000
prometheus_end_user_metrics_ttl_seconds: Optional[float] = 3600.0
prometheus_end_user_metrics_cleanup_interval_seconds: Optional[float] = 60.0
disable_add_prefix_to_prompt: bool = (
False # used by anthropic, to disable adding prefix to prompt
)
@ -586,6 +591,7 @@ anyscale_models: Set = set()
cerebras_models: Set = set()
galadriel_models: Set = set()
nvidia_nim_models: Set = set()
nvidia_riva_models: Set = set()
sambanova_models: Set = set()
sambanova_embedding_models: Set = set()
novita_models: Set = set()
@ -812,6 +818,8 @@ def add_known_models(model_cost_map: Optional[Dict] = None):
galadriel_models.add(key)
elif value.get("litellm_provider") == "nvidia_nim":
nvidia_nim_models.add(key)
elif value.get("litellm_provider") == "nvidia_riva":
nvidia_riva_models.add(key)
elif value.get("litellm_provider") == "sambanova":
sambanova_models.add(key)
elif value.get("litellm_provider") == "sambanova-embedding-models":
@ -971,6 +979,7 @@ model_list = list(
| cerebras_models
| galadriel_models
| nvidia_nim_models
| nvidia_riva_models
| sambanova_models
| azure_text_models
| novita_models
@ -1067,6 +1076,7 @@ models_by_provider: dict = {
"cerebras": cerebras_models,
"galadriel": galadriel_models,
"nvidia_nim": nvidia_nim_models,
"nvidia_riva": nvidia_riva_models,
"sambanova": sambanova_models | sambanova_embedding_models,
"novita": novita_models,
"nebius": nebius_models | nebius_embedding_models,
@ -1416,6 +1426,12 @@ if TYPE_CHECKING:
)
from .llms.datarobot.chat.transformation import DataRobotConfig as DataRobotConfig
from .llms.anthropic.chat.transformation import AnthropicConfig as AnthropicConfig
from .llms.bedrock.claude_platform.transformation import (
BedrockClaudePlatformConfig as BedrockClaudePlatformConfig,
)
from .llms.bedrock.claude_platform.messages_transformation import (
BedrockClaudePlatformMessagesConfig as BedrockClaudePlatformMessagesConfig,
)
from .llms.anthropic.completion.transformation import (
AnthropicTextConfig as AnthropicTextConfig,
)
@ -1618,6 +1634,9 @@ if TYPE_CHECKING:
from .llms.deepgram.audio_transcription.transformation import (
DeepgramAudioTranscriptionConfig as DeepgramAudioTranscriptionConfig,
)
from .llms.nvidia_riva.audio_transcription.transformation import (
NvidiaRivaAudioTranscriptionConfig as NvidiaRivaAudioTranscriptionConfig,
)
from .llms.topaz.image_variations.transformation import (
TopazImageVariationConfig as TopazImageVariationConfig,
)

12
litellm/_lazy_imports_registry.py
View file

@ -131,6 +131,7 @@ LLM_CONFIG_NAMES = (
"OpenrouterConfig",
"DataRobotConfig",
"AnthropicConfig",
"BedrockClaudePlatformConfig",
"AnthropicTextConfig",
"GroqSTTConfig",
"TritonConfig",
@ -170,6 +171,7 @@ LLM_CONFIG_NAMES = (
"SagemakerNovaConfig",
"CohereChatConfig",
"AnthropicMessagesConfig",
"BedrockClaudePlatformMessagesConfig",
"AmazonAnthropicClaudeMessagesConfig",
"AmazonMantleMessagesConfig",
"TogetherAIConfig",
@ -374,7 +376,6 @@ UTILS_MODULE_NAMES = (
"HTTPHandler",
"get_num_retries_from_retry_policy",
"reset_retry_policy",
"get_secret",
"get_coroutine_checker",
"get_litellm_logging_class",
"get_set_callbacks",
@ -610,6 +611,10 @@ _LLM_CONFIGS_IMPORT_MAP = {
"OpenrouterConfig": (".llms.openrouter.chat.transformation", "OpenrouterConfig"),
"DataRobotConfig": (".llms.datarobot.chat.transformation", "DataRobotConfig"),
"AnthropicConfig": (".llms.anthropic.chat.transformation", "AnthropicConfig"),
"BedrockClaudePlatformConfig": (
".llms.bedrock.claude_platform.transformation",
"BedrockClaudePlatformConfig",
),
"AnthropicTextConfig": (
".llms.anthropic.completion.transformation",
"AnthropicTextConfig",
@ -712,6 +717,10 @@ _LLM_CONFIGS_IMPORT_MAP = {
".llms.anthropic.experimental_pass_through.messages.transformation",
"AnthropicMessagesConfig",
),
"BedrockClaudePlatformMessagesConfig": (
".llms.bedrock.claude_platform.messages_transformation",
"BedrockClaudePlatformMessagesConfig",
),
"AmazonAnthropicClaudeMessagesConfig": (
".llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation",
"AmazonAnthropicClaudeMessagesConfig",
@ -1274,7 +1283,6 @@ _UTILS_MODULE_IMPORT_MAP = {
"litellm.router_utils.get_retry_from_policy",
"reset_retry_policy",
),
"get_secret": ("litellm.secret_managers.main", "get_secret"),
"get_coroutine_checker": (
"litellm.litellm_core_utils.cached_imports",
"get_coroutine_checker",

1
litellm/_logging.py
View file

@ -404,6 +404,7 @@ def _turn_on_debug():
def _disable_debugging():
"""Disable the package, router, and proxy verbose loggers."""
verbose_logger.disabled = True
verbose_router_logger.disabled = True
verbose_proxy_logger.disabled = True

279
litellm/_redis.py
View file

@ -19,6 +19,7 @@ import redis.asyncio as async_redis # type: ignore
from litellm import get_secret, get_secret_str
from litellm._redis_credential_provider import (
AzureADCredentialProvider,
GCPIAMCredentialProvider,
_generate_gcp_iam_access_token,
)
@ -27,6 +28,8 @@ from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker
from ._logging import verbose_logger
AZURE_REDIS_SCOPE = "https://redis.azure.com/.default"
def _get_redis_kwargs():
arg_spec = inspect.getfullargspec(redis.Redis)
@ -38,14 +41,18 @@ def _get_redis_kwargs():
"retry",
}
include_args = [
include_args = {
"url",
"redis_connect_func",
"gcp_service_account",
"gcp_ssl_ca_certs",
]
"azure_redis_ad_token",
"azure_client_id",
"azure_tenant_id",
"azure_client_secret",
}
available_args = [x for x in arg_spec.args if x not in exclude_args] + include_args
available_args = {x for x in arg_spec.args if x not in exclude_args} | include_args
return available_args
@ -77,19 +84,23 @@ def _get_redis_cluster_kwargs(client=None):
# Only allow primitive arguments
exclude_args = {"self", "connection_pool", "retry", "host", "port", "startup_nodes"}
available_args = [x for x in arg_spec.args if x not in exclude_args]
available_args.append("password")
available_args.append("username")
available_args.append("ssl")
available_args.append("ssl_cert_reqs")
available_args.append("ssl_check_hostname")
available_args.append("ssl_ca_certs")
available_args.append(
"redis_connect_func"
) # Needed for sync clusters and IAM detection
available_args.append("gcp_service_account")
available_args.append("gcp_ssl_ca_certs")
available_args.append("max_connections")
available_args = {x for x in arg_spec.args if x not in exclude_args}
available_args |= {
"password",
"username",
"ssl",
"ssl_cert_reqs",
"ssl_check_hostname",
"ssl_ca_certs",
"redis_connect_func", # Needed for sync clusters and IAM detection
"gcp_service_account",
"gcp_ssl_ca_certs",
"azure_redis_ad_token",
"azure_client_id",
"azure_tenant_id",
"azure_client_secret",
"max_connections",
}
return available_args
@ -155,6 +166,125 @@ def create_gcp_iam_redis_connect_func(
return iam_connect
def _build_azure_credential(
azure_client_id: Optional[str] = None,
azure_tenant_id: Optional[str] = None,
azure_client_secret: Optional[str] = None,
):
"""
Build a long-lived Azure credential object.
Azure SDK credentials cache tokens internally and handle expiry/refresh
transparently, so this should be called once and the result reused.
"""
try:
from azure.identity import (
ClientSecretCredential,
DefaultAzureCredential,
ManagedIdentityCredential,
)
except ImportError:
raise ImportError(
"azure-identity is required for Azure AD Redis authentication. "
"Install it with: pip install azure-identity"
)
_client_id = azure_client_id or os.environ.get("AZURE_CLIENT_ID")
_tenant_id = azure_tenant_id or os.environ.get("AZURE_TENANT_ID")
_client_secret = azure_client_secret or os.environ.get("AZURE_CLIENT_SECRET")
if _client_id and _tenant_id and _client_secret:
return ClientSecretCredential(
client_id=_client_id,
tenant_id=_tenant_id,
client_secret=_client_secret,
)
elif _client_id:
return ManagedIdentityCredential(client_id=_client_id)
else:
return DefaultAzureCredential()
def _generate_azure_ad_redis_token(
azure_client_id: Optional[str] = None,
azure_tenant_id: Optional[str] = None,
azure_client_secret: Optional[str] = None,
) -> str:
"""
One-shot helper that builds a credential and fetches a single Azure AD
access token for Redis. Each call rebuilds the credential and performs a
network round-trip, so it should not be used in steady-state Redis flows
— the sync (``create_azure_ad_redis_connect_func``) and async paths
(``AzureADCredentialProvider``) keep the credential alive across
connections so the Azure SDK's internal cache + silent refresh apply.
"""
credential = _build_azure_credential(
azure_client_id=azure_client_id,
azure_tenant_id=azure_tenant_id,
azure_client_secret=azure_client_secret,
)
token = credential.get_token(AZURE_REDIS_SCOPE)
return token.token
def create_azure_ad_redis_connect_func(
azure_client_id: Optional[str] = None,
azure_tenant_id: Optional[str] = None,
azure_client_secret: Optional[str] = None,
) -> Callable:
"""
Creates a custom Redis connection function for Azure AD authentication.
Used for sync Redis clients. The credential is created once (captured by the
closure) and reused across connections — the Azure SDK handles token caching
and silent renewal internally. Only ``get_token`` is called per connection.
"""
credential = _build_azure_credential(
azure_client_id=azure_client_id,
azure_tenant_id=azure_tenant_id,
azure_client_secret=azure_client_secret,
)
def ad_connect(self):
"""Initialize the connection and authenticate using Azure AD"""
from redis.exceptions import (
AuthenticationError,
AuthenticationWrongNumberOfArgsError,
)
from redis.utils import str_if_bytes
self._parser.on_connect(self)
access_token = credential.get_token(AZURE_REDIS_SCOPE).token
# Only include username when explicitly set — sending AUTH "" <token>
# is invalid for most ACL-configured Azure Redis instances.
username = os.environ.get("REDIS_USERNAME", "")
if username:
auth_args = (username, access_token)
else:
auth_args = (access_token,)
self.send_command("AUTH", *auth_args, check_health=False)
try:
auth_response = self.read_response()
except AuthenticationWrongNumberOfArgsError:
# Fallback: try with just the token (Redis < 6 / no ACL)
self.send_command("AUTH", access_token, check_health=False)
auth_response = self.read_response()
if str_if_bytes(auth_response) != "OK":
raise AuthenticationError("Azure AD authentication failed for Redis")
# Attach the live credential object so async paths can wrap it in
# AzureADCredentialProvider for refresh-aware token retrieval. The raw
# client_id/tenant_id/secret are intentionally NOT exposed here — the
# credential closure already holds them.
ad_connect._azure_credential = credential # type: ignore[attr-defined]
return ad_connect
def get_redis_url_from_environment():
if "REDIS_URL" in os.environ:
return os.environ["REDIS_URL"]
@ -179,7 +309,7 @@ def get_redis_url_from_environment():
return f"{redis_protocol}://{auth_part}{os.environ['REDIS_HOST']}:{os.environ['REDIS_PORT']}"
def _get_redis_client_logic(**env_overrides):
def _get_redis_client_logic(**env_overrides): # noqa: PLR0915
"""
Common functionality across sync + async redis client implementations
"""
@ -253,6 +383,52 @@ def _get_redis_client_logic(**env_overrides):
if _gcp_ssl_ca_certs and redis_kwargs.get("ssl", False):
redis_kwargs["ssl_ca_certs"] = _gcp_ssl_ca_certs
# Handle Azure AD authentication (after GCP IAM block)
_azure_redis_ad_token = redis_kwargs.get("azure_redis_ad_token") or get_secret(
"REDIS_AZURE_AD_TOKEN"
)
_azure_ad_enabled = (
_azure_redis_ad_token is not None
and str(_azure_redis_ad_token).lower() == "true"
)
if _azure_ad_enabled and _gcp_service_account is not None:
verbose_logger.warning(
"Both GCP IAM (gcp_service_account) and Azure AD (azure_redis_ad_token) are configured for Redis. "
"Using GCP IAM. Remove one to avoid misconfiguration."
)
if _azure_ad_enabled and _gcp_service_account is None:
_azure_client_id = redis_kwargs.get("azure_client_id") or get_secret_str(
"AZURE_CLIENT_ID"
)
_azure_tenant_id = redis_kwargs.get("azure_tenant_id") or get_secret_str(
"AZURE_TENANT_ID"
)
_azure_client_secret = redis_kwargs.get(
"azure_client_secret"
) or get_secret_str("AZURE_CLIENT_SECRET")
verbose_logger.debug("Setting up Azure AD authentication for Redis.")
redis_kwargs["redis_connect_func"] = create_azure_ad_redis_connect_func(
azure_client_id=_azure_client_id,
azure_tenant_id=_azure_tenant_id,
azure_client_secret=_azure_client_secret,
)
# Marker for async paths to detect Azure AD auth. The live credential
# object is attached separately as `_azure_credential` by
# `create_azure_ad_redis_connect_func`; the raw client_id/tenant_id/secret
# are intentionally NOT exposed on the function to avoid leaking
# credentials via inspection or logging.
redis_kwargs["redis_connect_func"]._azure_redis_ad_token = True # type: ignore[attr-defined]
# Always remove Azure-specific kwargs that shouldn't be passed to Redis client
redis_kwargs.pop("azure_redis_ad_token", None)
redis_kwargs.pop("azure_client_id", None)
redis_kwargs.pop("azure_tenant_id", None)
redis_kwargs.pop("azure_client_secret", None)
if "url" in redis_kwargs and redis_kwargs["url"] is not None:
# Only strip host/port/db/password when not routing to a cluster.
# When startup_nodes is also present the cluster path takes priority and
@ -303,10 +479,24 @@ def init_redis_cluster(redis_kwargs) -> redis.RedisCluster:
return redis.RedisCluster(startup_nodes=new_startup_nodes, **cluster_kwargs) # type: ignore
def _get_redis_sentinel_connection_kwargs(redis_kwargs: dict) -> dict:
connection_kwargs = {}
args = _get_redis_kwargs()
for arg in redis_kwargs:
if arg in args:
connection_kwargs[arg] = redis_kwargs[arg]
return connection_kwargs
def _init_redis_sentinel(redis_kwargs) -> redis.Redis:
sentinel_nodes = redis_kwargs.get("sentinel_nodes")
sentinel_password = redis_kwargs.get("sentinel_password")
service_name = redis_kwargs.get("service_name")
connection_kwargs = _get_redis_sentinel_connection_kwargs(redis_kwargs)
connection_kwargs.setdefault("socket_timeout", REDIS_SOCKET_TIMEOUT)
sentinel_kwargs = dict(connection_kwargs)
sentinel_kwargs["password"] = sentinel_password
if not sentinel_nodes or not service_name:
raise ValueError(
@ -318,19 +508,22 @@ def _init_redis_sentinel(redis_kwargs) -> redis.Redis:
# Set up the Sentinel client
sentinel = redis.Sentinel(
sentinel_nodes,
socket_timeout=REDIS_SOCKET_TIMEOUT,
password=sentinel_password,
sentinel_kwargs=sentinel_kwargs,
)
# Return the master instance for the given service
return sentinel.master_for(service_name)
return sentinel.master_for(service_name, **connection_kwargs)
def _init_async_redis_sentinel(redis_kwargs) -> async_redis.Redis:
sentinel_nodes = redis_kwargs.get("sentinel_nodes")
sentinel_password = redis_kwargs.get("sentinel_password")
service_name = redis_kwargs.get("service_name")
connection_kwargs = _get_redis_sentinel_connection_kwargs(redis_kwargs)
connection_kwargs.setdefault("socket_timeout", REDIS_SOCKET_TIMEOUT)
sentinel_kwargs = dict(connection_kwargs)
sentinel_kwargs["password"] = sentinel_password
if not sentinel_nodes or not service_name:
raise ValueError(
@ -342,13 +535,12 @@ def _init_async_redis_sentinel(redis_kwargs) -> async_redis.Redis:
# Set up the Sentinel client
sentinel = async_redis.Sentinel(
sentinel_nodes,
socket_timeout=REDIS_SOCKET_TIMEOUT,
password=sentinel_password,
sentinel_kwargs=sentinel_kwargs,
)
# Return the master instance for the given service
return sentinel.master_for(service_name)
return sentinel.master_for(service_name, **connection_kwargs)
def get_redis_client(**env_overrides):
@ -373,7 +565,7 @@ def get_redis_client(**env_overrides):
return redis.Redis(**redis_kwargs)
def get_redis_async_client(
def get_redis_async_client( # noqa: PLR0915
connection_pool: Optional[async_redis.BlockingConnectionPool] = None,
**env_overrides,
) -> Union[async_redis.Redis, async_redis.RedisCluster]:
@ -398,6 +590,14 @@ def get_redis_async_client(
cluster_kwargs["credential_provider"] = GCPIAMCredentialProvider(
redis_connect_func._gcp_service_account
)
# Handle Azure AD authentication for async clusters via CredentialProvider
# so the credential's internal cache + silent refresh runs per connection
# (mirrors GCP IAM above; avoids static-token-baked-in-pool expiry).
elif redis_connect_func and hasattr(redis_connect_func, "_azure_credential"):
cluster_kwargs["credential_provider"] = AzureADCredentialProvider(
redis_connect_func._azure_credential,
username=os.environ.get("REDIS_USERNAME") or None,
)
new_startup_nodes: List[ClusterNode] = []
@ -431,6 +631,22 @@ def get_redis_async_client(
# Check for Redis Sentinel
if "sentinel_nodes" in redis_kwargs and "service_name" in redis_kwargs:
return _init_async_redis_sentinel(redis_kwargs)
# Wrap GCP / Azure AD auth in a CredentialProvider for the standard async
# Redis client. The async client doesn't support redis_connect_func, but it
# does honour credential_provider — which is called per connection, so the
# underlying SDK can refresh tokens silently before they expire.
redis_connect_func = redis_kwargs.pop("redis_connect_func", None)
if redis_connect_func and hasattr(redis_connect_func, "_azure_credential"):
redis_kwargs["credential_provider"] = AzureADCredentialProvider(
redis_connect_func._azure_credential,
username=os.environ.get("REDIS_USERNAME") or None,
)
elif redis_connect_func and hasattr(redis_connect_func, "_gcp_service_account"):
redis_kwargs["credential_provider"] = GCPIAMCredentialProvider(
redis_connect_func._gcp_service_account
)
_pretty_print_redis_config(redis_kwargs=redis_kwargs)
if connection_pool is not None:
@ -464,6 +680,21 @@ def get_redis_connection_pool(
redis_kwargs["max_connections"],
)
return async_redis.BlockingConnectionPool.from_url(**pool_kwargs)
# Wrap GCP / Azure AD auth in a CredentialProvider so pool-managed
# connections re-fetch tokens via the SDK's internal cache + silent refresh
# rather than reusing a single token captured at pool creation.
redis_connect_func = redis_kwargs.pop("redis_connect_func", None)
if redis_connect_func and hasattr(redis_connect_func, "_azure_credential"):
redis_kwargs["credential_provider"] = AzureADCredentialProvider(
redis_connect_func._azure_credential,
username=os.environ.get("REDIS_USERNAME") or None,
)
elif redis_connect_func and hasattr(redis_connect_func, "_gcp_service_account"):
redis_kwargs["credential_provider"] = GCPIAMCredentialProvider(
redis_connect_func._gcp_service_account
)
connection_class = async_redis.Connection
if "ssl" in redis_kwargs:
connection_class = async_redis.SSLConnection

35
litellm/_redis_credential_provider.py
View file

@ -1,10 +1,13 @@
import asyncio
import threading
import time
from typing import Dict, Tuple
from typing import Any, Dict, Optional, Tuple, Union
from redis.credentials import CredentialProvider # type: ignore[attr-defined]
# Azure AD scope for Redis Cache for Azure.
AZURE_REDIS_SCOPE = "https://redis.azure.com/.default"
# GCP IAM tokens are valid for 1 hour. Cache for 55 minutes to refresh before expiry.
_GCP_IAM_TOKEN_TTL_SECONDS = 3300
@ -101,3 +104,33 @@ class GCPIAMCredentialProvider(CredentialProvider):
_get_cached_gcp_iam_token, self._gcp_service_account
)
return (token,)
class AzureADCredentialProvider(CredentialProvider):
"""
redis.credentials.CredentialProvider implementation that supplies Azure AD
tokens for Redis authentication.
Wraps an azure-identity credential object so the Azure SDK's internal token
cache and silent refresh are honoured on every Redis connection. This avoids
the static-token-baked-in-pool issue where pool-managed connections would
fail authentication after the initial token expired (~1 hour TTL).
"""
def __init__(self, credential: Any, username: Optional[str] = None) -> None:
self._credential = credential
self._username = username
def get_credentials(self) -> Union[Tuple[str], Tuple[str, str]]:
token = self._credential.get_token(AZURE_REDIS_SCOPE).token
if self._username:
return (self._username, token)
return (token,)
async def get_credentials_async(self) -> Union[Tuple[str], Tuple[str, str]]:
token_obj = await asyncio.to_thread(
self._credential.get_token, AZURE_REDIS_SCOPE
)
if self._username:
return (self._username, token_obj.token)
return (token_obj.token,)

23
litellm/batches/batch_utils.py
View file

@ -113,8 +113,11 @@ def _batch_cost_calculator(
"""
Calculate the cost of a batch based on the output file id
"""
# Handle Vertex AI with specialized method
if custom_llm_provider == "vertex_ai" and model_name:
if (
custom_llm_provider == "vertex_ai"
and model_name
and getattr(litellm, "disable_vertex_batch_output_transformation", False)
):
batch_cost, _ = calculate_vertex_ai_batch_cost_and_usage(
file_content_dictionary, model_name
)
@ -136,10 +139,13 @@ def calculate_vertex_ai_batch_cost_and_usage(
model_name: Optional[str] = None,
) -> Tuple[float, Usage]:
"""
Calculate both cost and usage from Vertex AI batch responses.
Calculate both cost and usage from raw Vertex AI batch responses.
Vertex AI batch output lines have format:
{"request": ..., "status": "", "response": {"candidates": [...], "usageMetadata": {...}}}
Used only when ``litellm.disable_vertex_batch_output_transformation = True``.
In that case the GCS predictions.jsonl is returned as-is, with each line in
the native Vertex format:
{"request": ..., "response": {"candidates": [...], "usageMetadata": {...}}}
usageMetadata contains promptTokenCount, candidatesTokenCount, totalTokenCount.
"""
@ -362,8 +368,11 @@ def _get_batch_job_total_usage_from_file_content(
"""
Get the tokens of a batch job from the file content
"""
# Handle Vertex AI with specialized method
if custom_llm_provider == "vertex_ai" and model_name:
if (
custom_llm_provider == "vertex_ai"
and model_name
and getattr(litellm, "disable_vertex_batch_output_transformation", False)
):
_, batch_usage = calculate_vertex_ai_batch_cost_and_usage(
file_content_dictionary, model_name
)

45
litellm/batches/main.py
View file

@ -617,24 +617,35 @@ def retrieve_batch(
_is_async = kwargs.pop("aretrieve_batch", False) is True
client = kwargs.get("client", None)
# Check if this is an async invoke ARN (different from regular batch ARN)
# Async invoke ARNs have format: arn:aws(-[^:]+)?:bedrock:[a-z0-9-]{1,20}:[0-9]{12}:async-invoke/[a-z0-9]{12}
if (
batch_id.startswith("arn:aws")
and ":bedrock:" in batch_id
and ":async-invoke/" in batch_id
):
# Handle async invoke status check
# Remove aws_region_name from kwargs to avoid duplicate parameter
async_kwargs = kwargs.copy()
async_kwargs.pop("aws_region_name", None)
# Bedrock has two distinct ARN families that need different APIs:
# * async-invoke ARNs (Twelve Labs Marengo embeddings) -> bedrock-runtime data plane
# * model-invocation-job ARNs (CreateModelInvocationJob batch) -> bedrock control plane
# They live on different AWS service endpoints and can't share a handler.
# ARN shapes:
# arn:aws(-[^:]+)?:bedrock:<region>:<account>:async-invoke/<id>
# arn:aws(-[^:]+)?:bedrock:<region>:<account>:model-invocation-job/<id>
if batch_id.startswith("arn:aws") and ":bedrock:" in batch_id:
if ":async-invoke/" in batch_id:
# Remove aws_region_name from kwargs to avoid duplicate parameter
async_kwargs = kwargs.copy()
async_kwargs.pop("aws_region_name", None)
return BedrockBatchesHandler._handle_async_invoke_status(
batch_id=batch_id,
aws_region_name=kwargs.get("aws_region_name", "us-east-1"),
logging_obj=litellm_logging_obj,
**async_kwargs,
)
return BedrockBatchesHandler._handle_async_invoke_status(
batch_id=batch_id,
aws_region_name=kwargs.get("aws_region_name", "us-east-1"),
logging_obj=litellm_logging_obj,
**async_kwargs,
)
if ":model-invocation-job/" in batch_id:
mij_kwargs = kwargs.copy()
mij_kwargs.pop("aws_region_name", None)
return BedrockBatchesHandler._handle_model_invocation_job_status(
batch_id=batch_id,
aws_region_name=kwargs.get("aws_region_name"),
logging_obj=litellm_logging_obj,
**mij_kwargs,
)
# Try to use provider config first (for providers like bedrock)
model: Optional[str] = kwargs.get("model", None)

12
litellm/budget_manager.py
View file

@ -178,6 +178,18 @@ class BudgetManager:
return list(self.user_dict.keys())
def reset_cost(self, user):
"""
Reset the tracked spend for a user back to zero.
Clears both the aggregate ``current_cost`` and the per-model
``model_cost`` breakdown stored for the given user.
Args:
user: The user identifier whose cost should be reset.
Returns:
dict: ``{"user": <updated user record>}`` reflecting the reset state.
"""
self.user_dict[user]["current_cost"] = 0
self.user_dict[user]["model_cost"] = {}
return {"user": self.user_dict[user]}

18
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -119,6 +119,20 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
def __init__(self):
pass
def _normalize_tool_choice_for_responses_api(self, tool_choice: Any) -> Any:
"""Chat tool_choice uses function.name; Responses API expects top-level name."""
if not isinstance(tool_choice, dict) or tool_choice.get("type") != "function":
return tool_choice
if isinstance(tool_choice.get("name"), str) and tool_choice.get("name"):
# Return only Responses shape so stray chat ``function`` key is not sent upstream.
return {"type": "function", "name": tool_choice["name"]}
fn = tool_choice.get("function")
if isinstance(fn, dict):
fn_name = fn.get("name")
if isinstance(fn_name, str) and fn_name:
return {"type": "function", "name": fn_name}
return tool_choice
def _handle_raw_dict_response_item(
self, item: Dict[str, Any], index: int
) -> Tuple[Optional[Any], int]:
@ -309,6 +323,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
text_format = self._transform_response_format_to_text_format(value)
if text_format:
responses_api_request["text"] = text_format # type: ignore
elif key == "tool_choice":
responses_api_request["tool_choice"] = ( # type: ignore[assignment]
self._normalize_tool_choice_for_responses_api(value)
)
elif key in ResponsesAPIOptionalRequestParams.__annotations__.keys():
responses_api_request[key] = value # type: ignore
elif key == "previous_response_id":

20
litellm/constants.py
View file

@ -161,6 +161,11 @@ MCP_STDIO_ALLOWED_COMMANDS: frozenset = frozenset(
| (set(_MCP_STDIO_EXTRA_COMMANDS.split(",")) - {""})
)
# MCP OAuth2 Token Exchange (OBO) Defaults
MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE = int(
os.getenv("MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE", "500")
)
LITELLM_UI_ALLOW_HEADERS = [
"x-litellm-semantic-filter",
"x-litellm-semantic-filter-tools",
@ -1457,6 +1462,12 @@ KEY_ROTATION_JOB_NAME = "litellm_key_rotation_job"
EXPIRED_UI_SESSION_KEY_CLEANUP_JOB_NAME = "litellm_expired_ui_session_key_cleanup_job"
SPEND_LOG_RUN_LOOPS = int(os.getenv("SPEND_LOG_RUN_LOOPS", 500))
SPEND_LOG_CLEANUP_BATCH_SIZE = int(os.getenv("SPEND_LOG_CLEANUP_BATCH_SIZE", 1000))
SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES = int(
os.getenv("SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES", 3)
)
SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS = float(
os.getenv("SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS", 0.5)
)
SPEND_LOG_QUEUE_SIZE_THRESHOLD = int(os.getenv("SPEND_LOG_QUEUE_SIZE_THRESHOLD", 100))
SPEND_LOG_QUEUE_POLL_INTERVAL = float(os.getenv("SPEND_LOG_QUEUE_POLL_INTERVAL", 2.0))
SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE = int(
@ -1558,6 +1569,15 @@ DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL = int(
os.getenv("DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL", 60)
)
DEFAULT_ACCESS_GROUP_CACHE_TTL = int(os.getenv("DEFAULT_ACCESS_GROUP_CACHE_TTL", 600))
# Short TTL for negative MCP access-group existence lookups. Keeps unauthenticated
# callers from forcing a DB query per request for unknown names, while bounding
# staleness so a transient DB error (which surfaces as an empty list) cannot
# hide a real group for long.
DEFAULT_MCP_ACCESS_GROUP_NEGATIVE_CACHE_TTL = 10
# Maximum number of comma-separated MCP server / access-group tokens accepted
# in a single ``/{name1,name2,...}/mcp`` URL. Bounds the per-request DB / cache
# fan-out an authenticated caller can trigger by stuffing the path with tokens.
DEFAULT_MCP_NAMESPACE_CSV_MAX_TOKENS = 16
# Sentry Scrubbing Configuration
SENTRY_DENYLIST = [

20
litellm/cost_calculator.py
View file

@ -2120,6 +2120,26 @@ def batch_cost_calculator(
)
except Exception:
model_info = None
elif not any(
model_info.get(k) is not None
for k in (
"input_cost_per_token_batches",
"input_cost_per_token",
"output_cost_per_token_batches",
"output_cost_per_token",
)
):
# model_info was provided (e.g. deployment metadata with only id/db_model)
# but carries no pricing fields. Fall back to the global pricing table so
# that standard model pricing is used instead of silently returning $0.
try:
global_info = litellm.get_model_info(
model=model, custom_llm_provider=custom_llm_provider
)
if global_info:
model_info = global_info
except Exception:
pass
if not model_info:
return 0.0, 0.0

2
litellm/experimental_mcp_client/client.py
View file

@ -366,6 +366,8 @@ class MCPClient:
headers["Authorization"] = f"Bearer {self._mcp_auth_value}"
elif self.auth_type == MCPAuth.token:
headers["Authorization"] = f"token {self._mcp_auth_value}"
elif self.auth_type == MCPAuth.oauth2_token_exchange:
headers["Authorization"] = f"Bearer {self._mcp_auth_value}"
elif isinstance(self._mcp_auth_value, dict):
headers.update(self._mcp_auth_value)
# Note: aws_sigv4 auth is not handled here — SigV4 requires per-request

139
litellm/integrations/azure_sentinel/azure_sentinel.py
View file

@ -14,16 +14,18 @@ For batching specific details see CustomBatchLogger class
import asyncio
import os
import time
import traceback
from typing import List, Optional
from typing import List, Optional, Union
from litellm._logging import verbose_logger
from litellm.integrations.custom_batch_logger import CustomBatchLogger
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
)
from litellm.types.utils import StandardLoggingPayload
from litellm.types.utils import StandardAuditLogPayload, StandardLoggingPayload
class AzureSentinelLogger(CustomBatchLogger):
@ -39,6 +41,7 @@ class AzureSentinelLogger(CustomBatchLogger):
tenant_id: Optional[str] = None,
client_id: Optional[str] = None,
client_secret: Optional[str] = None,
audit_stream_name: Optional[str] = None,
**kwargs,
):
"""
@ -57,57 +60,77 @@ class AzureSentinelLogger(CustomBatchLogger):
If not provided, will use AZURE_SENTINEL_CLIENT_ID or AZURE_CLIENT_ID env var.
client_secret (str, optional): Azure Client Secret for OAuth2 authentication.
If not provided, will use AZURE_SENTINEL_CLIENT_SECRET or AZURE_CLIENT_SECRET env var.
audit_stream_name (str, optional): Stream name from DCR for audit logs.
If not provided, audit logs use the standard stream name.
"""
self.async_httpx_client = get_async_httpx_client(
llm_provider=httpxSpecialProvider.LoggingCallback
)
self.dcr_immutable_id = dcr_immutable_id or os.getenv(
resolved_dcr_immutable_id = dcr_immutable_id or os.getenv(
"AZURE_SENTINEL_DCR_IMMUTABLE_ID"
)
self.stream_name = stream_name or os.getenv(
"AZURE_SENTINEL_STREAM_NAME", "Custom-LiteLLM"
resolved_stream_name = (
stream_name or os.getenv("AZURE_SENTINEL_STREAM_NAME") or "Custom-LiteLLM"
)
self.endpoint = endpoint or os.getenv("AZURE_SENTINEL_ENDPOINT")
self.tenant_id = (
resolved_audit_stream_name = audit_stream_name or resolved_stream_name
resolved_endpoint = endpoint or os.getenv("AZURE_SENTINEL_ENDPOINT")
resolved_tenant_id = (
tenant_id
or os.getenv("AZURE_SENTINEL_TENANT_ID")
or os.getenv("AZURE_TENANT_ID")
)
self.client_id = (
resolved_client_id = (
client_id
or os.getenv("AZURE_SENTINEL_CLIENT_ID")
or os.getenv("AZURE_CLIENT_ID")
)
self.client_secret = (
resolved_client_secret = (
client_secret
or os.getenv("AZURE_SENTINEL_CLIENT_SECRET")
or os.getenv("AZURE_CLIENT_SECRET")
)
if not self.dcr_immutable_id:
if not resolved_dcr_immutable_id:
raise ValueError(
"AZURE_SENTINEL_DCR_IMMUTABLE_ID is required. Set it as an environment variable or pass dcr_immutable_id parameter."
)
if not self.endpoint:
if not resolved_endpoint:
raise ValueError(
"AZURE_SENTINEL_ENDPOINT is required. Set it as an environment variable or pass endpoint parameter."
)
if not self.tenant_id:
if not resolved_tenant_id:
raise ValueError(
"AZURE_SENTINEL_TENANT_ID or AZURE_TENANT_ID is required. Set it as an environment variable or pass tenant_id parameter."
)
if not self.client_id:
if not resolved_client_id:
raise ValueError(
"AZURE_SENTINEL_CLIENT_ID or AZURE_CLIENT_ID is required. Set it as an environment variable or pass client_id parameter."
)
if not self.client_secret:
if not resolved_client_secret:
raise ValueError(
"AZURE_SENTINEL_CLIENT_SECRET or AZURE_CLIENT_SECRET is required. Set it as an environment variable or pass client_secret parameter."
)
self.dcr_immutable_id = resolved_dcr_immutable_id
self.stream_name = resolved_stream_name
self.audit_stream_name = resolved_audit_stream_name
self.endpoint = resolved_endpoint
self.tenant_id = resolved_tenant_id
self.client_id = resolved_client_id
self.client_secret = resolved_client_secret
# Build API endpoint: {Endpoint}/dataCollectionRules/{DCR Immutable ID}/streams/{Stream Name}?api-version=2023-01-01
self.api_endpoint = f"{self.endpoint.rstrip('/')}/dataCollectionRules/{self.dcr_immutable_id}/streams/{self.stream_name}?api-version=2023-01-01"
self.api_endpoint = self._build_api_endpoint(
endpoint=resolved_endpoint,
dcr_immutable_id=resolved_dcr_immutable_id,
stream_name=resolved_stream_name,
)
self.audit_api_endpoint = self._build_api_endpoint(
endpoint=resolved_endpoint,
dcr_immutable_id=resolved_dcr_immutable_id,
stream_name=resolved_audit_stream_name,
)
# OAuth2 scope for Azure Monitor
self.oauth_scope = "https://monitor.azure.com/.default"
@ -118,6 +141,13 @@ class AzureSentinelLogger(CustomBatchLogger):
super().__init__(**kwargs, flush_lock=self.flush_lock)
asyncio.create_task(self.periodic_flush())
self.log_queue: List[StandardLoggingPayload] = []
self.audit_log_queue: List[StandardAuditLogPayload] = []
@staticmethod
def _build_api_endpoint(
endpoint: str, dcr_immutable_id: str, stream_name: str
) -> str:
return f"{endpoint.rstrip('/')}/dataCollectionRules/{dcr_immutable_id}/streams/{stream_name}?api-version=2023-01-01"
async def _get_oauth_token(self) -> str:
"""
@ -126,9 +156,6 @@ class AzureSentinelLogger(CustomBatchLogger):
Returns:
Bearer token string
"""
# Check if we have a valid cached token
import time
if (
self.oauth_token
and self.oauth_token_expires_at
@ -170,9 +197,6 @@ class AzureSentinelLogger(CustomBatchLogger):
if not self.oauth_token:
raise Exception("OAuth2 token response did not contain access_token")
# Cache token expiry time
import time
self.oauth_token_expires_at = time.time() + expires_in
return self.oauth_token
@ -246,6 +270,34 @@ class AzureSentinelLogger(CustomBatchLogger):
)
pass
async def async_log_audit_log_event(
self, audit_log: StandardAuditLogPayload
) -> None:
"""
Async log LiteLLM audit log events to Azure Sentinel.
Audit logs are queued separately from standard LLM logs so mixed callback
usage never sends schema-mismatched records in the same ingestion batch.
"""
try:
verbose_logger.debug(
"Azure Sentinel: Logging audit event id=%s action=%s table=%s",
audit_log.get("id"),
audit_log.get("action"),
audit_log.get("table_name"),
)
self.audit_log_queue.append(audit_log)
if len(self.audit_log_queue) >= self.batch_size:
await self.async_send_audit_batch()
except Exception as e:
verbose_logger.exception(
f"Azure Sentinel Audit Log Layer Error - {str(e)}\n{traceback.format_exc()}"
)
pass
async def async_send_batch(self):
"""
Sends the batch of logs to Azure Monitor Logs Ingestion API
@ -253,22 +305,42 @@ class AzureSentinelLogger(CustomBatchLogger):
Raises:
Raises a NON Blocking verbose_logger.exception if an error occurs
"""
await self._async_send_batch_to_api(
log_queue=self.log_queue,
api_endpoint=self.api_endpoint,
log_type="logs",
)
async def async_send_audit_batch(self):
"""
Sends the batch of audit logs to Azure Monitor Logs Ingestion API
"""
await self._async_send_batch_to_api(
log_queue=self.audit_log_queue,
api_endpoint=self.audit_api_endpoint,
log_type="audit logs",
)
async def _async_send_batch_to_api(
self,
log_queue: List[Union[StandardLoggingPayload, StandardAuditLogPayload]],
api_endpoint: str,
log_type: str,
) -> None:
try:
if not self.log_queue:
if not log_queue:
return
verbose_logger.debug(
"Azure Sentinel - about to flush %s events", len(self.log_queue)
"Azure Sentinel - about to flush %s %s", len(log_queue), log_type
)
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
# Get OAuth2 token
bearer_token = await self._get_oauth_token()
# Convert log queue to JSON array format expected by Logs Ingestion API
# Each log entry should be a JSON object in the array
body = safe_dumps(self.log_queue)
body = safe_dumps(log_queue)
# Set headers for Logs Ingestion API
headers = {
@ -278,7 +350,7 @@ class AzureSentinelLogger(CustomBatchLogger):
# Send the request
response = await self.async_httpx_client.post(
url=self.api_endpoint, data=body.encode("utf-8"), headers=headers
url=api_endpoint, data=body.encode("utf-8"), headers=headers
)
if response.status_code not in [200, 204]:
@ -301,4 +373,15 @@ class AzureSentinelLogger(CustomBatchLogger):
f"Azure Sentinel Error sending batch API - {str(e)}\n{traceback.format_exc()}"
)
finally:
self.log_queue.clear()
log_queue.clear()
async def flush_queue(self):
if self.flush_lock is None:
return
async with self.flush_lock:
if self.log_queue:
await self.async_send_batch()
if self.audit_log_queue:
await self.async_send_audit_batch()
self.last_flush_time = time.time()

29
litellm/integrations/custom_guardrail.py
View file

@ -888,6 +888,15 @@ def log_guardrail_information(func):
- pre_call
- during_call
- post_call
Some guardrails (e.g. ``block_code_execution``) call
``add_standard_logging_guardrail_information_to_request_data`` directly
from inside the wrapped function so they can record a richer payload
(structured detections, tracing detail) than this decorator's
"allow"/"mask"/raw-response default. To avoid double-recording in that
case (which would emit two spans, two Datadog records, two spend-log
entries, etc.), snapshot the entry count before invocation: if the
wrapped function already appended its own entry, skip the auto-record.
"""
import functools
import inspect
@ -907,6 +916,16 @@ def log_guardrail_information(func):
return GuardrailEventHooks.post_call
return None
def _count_recorded_guardrail_entries(request_data: dict) -> int:
total = 0
for container_key in ("metadata", "litellm_metadata"):
container = request_data.get(container_key)
if isinstance(container, dict):
entries = container.get("standard_logging_guardrail_information")
if isinstance(entries, list):
total += len(entries)
return total
@functools.wraps(func)
async def async_wrapper(*args, **kwargs):
start_time = datetime.now() # Move start_time inside the wrapper
@ -919,8 +938,11 @@ def log_guardrail_information(func):
if func.__name__ == "apply_guardrail" and "inputs" in kwargs:
original_inputs = kwargs.get("inputs")
entries_before = _count_recorded_guardrail_entries(request_data)
try:
response = await func(*args, **kwargs)
if _count_recorded_guardrail_entries(request_data) > entries_before:
return response
return self._process_response(
response=response,
request_data=request_data,
@ -931,6 +953,8 @@ def log_guardrail_information(func):
original_inputs=original_inputs,
)
except Exception as e:
if _count_recorded_guardrail_entries(request_data) > entries_before:
raise
return self._process_error(
e=e,
request_data=request_data,
@ -952,8 +976,11 @@ def log_guardrail_information(func):
if func.__name__ == "apply_guardrail" and "inputs" in kwargs:
original_inputs = kwargs.get("inputs")
entries_before = _count_recorded_guardrail_entries(request_data)
try:
response = func(*args, **kwargs)
if _count_recorded_guardrail_entries(request_data) > entries_before:
return response
return self._process_response(
response=response,
request_data=request_data,
@ -962,6 +989,8 @@ def log_guardrail_information(func):
original_inputs=original_inputs,
)
except Exception as e:
if _count_recorded_guardrail_entries(request_data) > entries_before:
raise
return self._process_error(
e=e,
request_data=request_data,

21
litellm/integrations/custom_logger.py
View file

@ -697,6 +697,27 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
"""
return AgenticLoopPlan(run_agentic_loop=False)
async def async_post_agentic_loop_response_hook(
self,
response: Any,
plan: AgenticLoopPlan,
kwargs: Dict,
) -> Any:
"""
Post-process the response returned by the agentic-loop follow-up call.
Called after BaseLLMHTTPHandler executes ``AgenticLoopPlan.request_patch``
and receives the final response from the provider. Lets callbacks shape
what the client sees without bypassing the loop's safety / observability
machinery (depth tracking, fingerprinting, etc.).
Use ``plan.metadata`` to carry whatever the build step decided to expose
for post-processing (e.g. native tool_result blocks to inject).
Default returns ``response`` unchanged.
"""
return response
async def async_should_run_chat_completion_agentic_loop(
self,
response: Any,

397
litellm/integrations/opentelemetry.py
View file

@ -57,6 +57,17 @@ LITELLM_PROXY_REQUEST_SPAN_NAME = "Received Proxy Server Request"
RAW_REQUEST_SPAN_NAME = "raw_gen_ai_request"
LITELLM_REQUEST_SPAN_NAME = "litellm_request"
CAPTURE_MODE_NO_CONTENT = "NO_CONTENT"
CAPTURE_MODE_SPAN_ONLY = "SPAN_ONLY"
CAPTURE_MODE_EVENT_ONLY = "EVENT_ONLY"
CAPTURE_MODE_SPAN_AND_EVENT = "SPAN_AND_EVENT"
_VALID_CAPTURE_MODES = {
CAPTURE_MODE_NO_CONTENT,
CAPTURE_MODE_SPAN_ONLY,
CAPTURE_MODE_EVENT_ONLY,
CAPTURE_MODE_SPAN_AND_EVENT,
}
@dataclass
class OpenTelemetryConfig:
@ -71,6 +82,9 @@ class OpenTelemetryConfig:
ignore_context_propagation: Optional[bool] = None
# When True, create a private TracerProvider instead of reusing or setting the global one.
skip_set_global: bool = False
# Programmatic override for OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT.
# One of NO_CONTENT, SPAN_ONLY, EVENT_ONLY, SPAN_AND_EVENT (or "true" as legacy alias).
capture_message_content: Optional[str] = None
def __post_init__(self) -> None:
# If endpoint is specified but exporter is still the default "console",
@ -182,6 +196,9 @@ class OpenTelemetry(CustomLogger):
super().__init__(**kwargs)
self._init_metrics(meter_provider)
self._init_logs(logger_provider)
# Sample env-var / config / message_logging at init so subsequent
# _capture_in_span / _capture_in_event calls are deterministic.
self._capture_mode_cached = self._compute_capture_mode_from_init_state()
self._init_otel_logger_on_litellm_proxy()
@staticmethod
@ -220,7 +237,14 @@ class OpenTelemetry(CustomLogger):
not isinstance(cb, OpenTelemetry) for cb in litellm.service_callback
):
litellm.service_callback.append(self)
setattr(proxy_server, "open_telemetry_logger", self)
# avoid proxy logger ownership being overwritten by later
# handlers. Multiple integrations (default OTEL, Langfuse OTEL,
# Arize OTEL, etc.) may initialize in sequence; without this guard,
# the last one silently replaces the first and breaks expected
# routing for proxy_server.open_telemetry_logger consumers.
# Behavior: first-registered wins.
if getattr(proxy_server, "open_telemetry_logger", None) is None:
setattr(proxy_server, "open_telemetry_logger", self)
def _get_or_create_provider(
self,
@ -306,6 +330,62 @@ class OpenTelemetry(CustomLogger):
hasattr(self, "callback_name") and self.callback_name == "langfuse_otel"
)
def _compute_capture_mode_from_init_state(self) -> Optional[str]:
"""Sample explicit settings at init. Returns the resolved mode or
None if nothing explicit is set (in which case the legacy
``self.message_logging`` flag is consulted dynamically per request).
``"true"``/``"1"`` map to ``EVENT_ONLY`` per the contrib convention.
``"false"``/``"0"`` map to ``NO_CONTENT``.
Unknown values are ignored.
"""
explicit = self.config.capture_message_content or os.getenv(
"OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT"
)
if not explicit:
return None
normalized = explicit.upper()
if normalized in ("TRUE", "1"):
return CAPTURE_MODE_EVENT_ONLY
if normalized in ("FALSE", "0"):
return CAPTURE_MODE_NO_CONTENT
if normalized in _VALID_CAPTURE_MODES:
return normalized
return None
def _resolve_capture_mode(self) -> str:
"""Return the active capture mode for this request.
Precedence:
1. ``litellm.turn_off_message_logging=True`` forces ``NO_CONTENT``
(kill-switch checked dynamically).
2. Explicit setting sampled at init from
``OpenTelemetryConfig.capture_message_content`` or
``OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT``.
3. Legacy ``self.message_logging`` (checked dynamically).
"""
if litellm.turn_off_message_logging:
return CAPTURE_MODE_NO_CONTENT
if self._capture_mode_cached is not None:
return self._capture_mode_cached
return (
CAPTURE_MODE_SPAN_AND_EVENT
if self.message_logging
else CAPTURE_MODE_NO_CONTENT
)
def _capture_in_span(self) -> bool:
return self._resolve_capture_mode() in (
CAPTURE_MODE_SPAN_ONLY,
CAPTURE_MODE_SPAN_AND_EVENT,
)
def _capture_in_event(self) -> bool:
return self._resolve_capture_mode() in (
CAPTURE_MODE_EVENT_ONLY,
CAPTURE_MODE_SPAN_AND_EVENT,
)
def _init_tracing(self, tracer_provider):
from opentelemetry import trace
from opentelemetry.sdk.trace import TracerProvider
@ -721,12 +801,100 @@ class OpenTelemetry(CustomLogger):
# End of Team/Key Based Logging Control Flow
#########################################################
def _emit_once(self, kwargs: dict, *scope: object) -> bool:
"""Return True the first time this handler is asked to emit a span
for the given (handler, scope) on this kwargs; False on repeats.
Used to suppress duplicate span emission for two distinct patterns:
1. **Handler-level dual-fire**: streaming code paths trigger both
the sync and async callback for one request, so ``_handle_success``
/ ``_handle_failure`` would otherwise produce two
``litellm_request`` spans. Scope: ``("success",)`` / ``("failure",)``.
2. **Payload-driven multi-entrypoint emission**: a span loop that
reads entries from ``standard_logging_payload`` (currently only
guardrails) is invoked from multiple lifecycle points
(post-call hooks, success callback, failure callback). The list
can be re-read with mutated entries between calls, so dedupe
must be at entry granularity. Scope: the entry's stable identity.
``scope`` parts can be any hashable identity. The marker is stored
in ``kwargs["litellm_params"]["metadata"]["_otel_internal"]`` so it
is request-local (kwargs is shared across the sync/async callbacks
and lifecycle hooks for one request).
"""
litellm_params = kwargs.get("litellm_params")
if not isinstance(litellm_params, dict):
litellm_params = {}
kwargs["litellm_params"] = litellm_params
_metadata = litellm_params.get("metadata")
if not isinstance(_metadata, dict):
_metadata = {}
litellm_params["metadata"] = _metadata
_otel_internal = _metadata.get("_otel_internal")
if not isinstance(_otel_internal, dict):
_otel_internal = {}
_metadata["_otel_internal"] = _otel_internal
spans_logged = _otel_internal.get("spans_logged")
if not isinstance(spans_logged, dict):
spans_logged = {}
_otel_internal["spans_logged"] = spans_logged
dedupe_key = (self.__class__.__name__, id(self), *scope)
if spans_logged.get(dedupe_key) is True:
return False
spans_logged[dedupe_key] = True
return True
def _end_proxy_span_from_kwargs(self, kwargs: dict, end_time) -> None:
"""Close the proxy-level parent span if it is still recording.
This helper retrieves the proxy span directly from kwargs metadata
and closes it after all child spans have been recorded.
Only called from the success path. The failure path deliberately
leaves the proxy span open so ``async_post_call_failure_hook`` can
append the ``"Failed Proxy Server Request"`` child span before
closing it.
Only spans named ``LITELLM_PROXY_REQUEST_SPAN_NAME`` are closed —
externally provided spans must not be closed by LiteLLM.
"""
litellm_params = kwargs.get("litellm_params", {}) or {}
_metadata = litellm_params.get("metadata", {}) or {}
proxy_span = _metadata.get("litellm_parent_otel_span", None)
if (
proxy_span is not None
and getattr(proxy_span, "name", None) == LITELLM_PROXY_REQUEST_SPAN_NAME
and hasattr(proxy_span, "is_recording")
and proxy_span.is_recording()
):
proxy_span.end(end_time=self._to_ns(end_time))
def _handle_success(self, kwargs, response_obj, start_time, end_time):
"""Create the litellm_request span then close the proxy span."""
verbose_logger.debug(
"OpenTelemetry Logger: Logging kwargs: %s, OTEL config settings=%s",
kwargs,
self.config,
)
# sync + async success handlers can both fire for one
# request (notably in streaming code paths). Guard against duplicate
# span writes — but still close the proxy span on the skip path so
# the trace doesn't leak an open root span.
if not self._emit_once(kwargs, "success"):
verbose_logger.debug(
"OpenTelemetry: skipping duplicate success span for handler=%s",
self.__class__.__name__,
)
self._end_proxy_span_from_kwargs(kwargs, end_time)
return
ctx, parent_span = self._get_span_context(kwargs)
if self.config.ignore_context_propagation:
@ -786,7 +954,7 @@ class OpenTelemetry(CustomLogger):
# 6. Do NOT end parent span - it should be managed by its creator
# External spans (from Langfuse, user code, HTTP headers, global context) must not be closed by LiteLLM
# However, proxy-created spans should be closed here
# However, proxy-created spans should be closed here.
if (
parent_span is not None
and hasattr(parent_span, "name")
@ -794,6 +962,11 @@ class OpenTelemetry(CustomLogger):
):
parent_span.end(end_time=self._to_ns(end_time))
# close the proxy span explicitly from kwargs metadata
# after all child spans (litellm_request, guardrail, raw_request)
# have been fully recorded and exported.
self._end_proxy_span_from_kwargs(kwargs, end_time)
def _start_primary_span(
self,
kwargs,
@ -825,8 +998,7 @@ class OpenTelemetry(CustomLogger):
from opentelemetry import trace
from opentelemetry.trace import Status, StatusCode
# only log raw LLM request/response if message_logging is on and not globally turned off
if litellm.turn_off_message_logging or not self.message_logging:
if not self._capture_in_span():
return
litellm_params = kwargs.get("litellm_params", {})
@ -1117,9 +1289,14 @@ class OpenTelemetry(CustomLogger):
}
if role == "tool" and msg.get("id"):
attrs["id"] = msg["id"]
if self.message_logging and msg.get("content"):
capture_event_content = self._capture_in_event()
if capture_event_content and msg.get("content"):
attrs["gen_ai.prompt"] = msg["content"]
body = msg.copy()
if not capture_event_content:
body.pop("content", None)
log_record = SdkLogRecord(
timestamp=self._to_ns(datetime.now()),
trace_id=parent_ctx.trace_id,
@ -1127,7 +1304,7 @@ class OpenTelemetry(CustomLogger):
trace_flags=parent_ctx.trace_flags,
severity_number=SeverityNumber.INFO,
severity_text="INFO",
body=msg.copy(),
body=body,
attributes=attrs,
)
otel_logger.emit(log_record)
@ -1141,14 +1318,15 @@ class OpenTelemetry(CustomLogger):
"finish_reason": choice.get("finish_reason"),
}
body_msg = choice.get("message", {})
if self.message_logging and body_msg.get("content"):
capture_event_content = self._capture_in_event()
if capture_event_content and body_msg.get("content"):
attrs["message.content"] = body_msg["content"]
body = {
"index": idx,
"finish_reason": choice.get("finish_reason"),
"message": {"role": body_msg.get("role", "assistant")},
}
if self.message_logging and body_msg.get("content"):
if capture_event_content and body_msg.get("content"):
body["message"]["content"] = body_msg["content"]
log_record = SdkLogRecord(
@ -1218,6 +1396,21 @@ class OpenTelemetry(CustomLogger):
for guardrail_information in guardrail_information_list:
start_time_float = guardrail_information.get("start_time")
end_time_float = guardrail_information.get("end_time")
# ``_create_guardrail_span`` is called from three lifecycle
# points (``async_post_call_success_hook``, ``_handle_success``,
# ``_handle_failure``) and re-reads the (mutating) entry list
# each time. Dedupe at entry granularity so a single real
# guardrail invocation produces exactly one span per handler.
if not self._emit_once(
kwargs,
"guardrail",
guardrail_information.get("guardrail_name"),
start_time_float,
guardrail_information.get("guardrail_mode"),
):
continue
start_time_datetime = datetime.now()
if start_time_float is not None:
start_time_datetime = datetime.fromtimestamp(start_time_float)
@ -1271,6 +1464,21 @@ class OpenTelemetry(CustomLogger):
kwargs,
self.config,
)
# sync + async failure handlers can both fire for one
# request (notably in streaming code paths), producing two
# semantically identical ERROR spans. Unlike the success path, the
# proxy span is intentionally left open here so that
# ``async_post_call_failure_hook`` can append the
# "Failed Proxy Server Request" child span before closing it —
# there is no proxy-span side-effect to preserve on the skip path.
if not self._emit_once(kwargs, "failure"):
verbose_logger.debug(
"OpenTelemetry: skipping duplicate failure span for handler=%s",
self.__class__.__name__,
)
return
_parent_context, parent_otel_span = self._get_span_context(kwargs)
if self.config.ignore_context_propagation:
@ -1674,9 +1882,7 @@ class OpenTelemetry(CustomLogger):
########## LLM Request Medssages / tools / content Attributes ###########
#########################################################################
if litellm.turn_off_message_logging is True:
return
if self.message_logging is not True:
if not self._capture_in_span():
return
if optional_params.get("tools"):
@ -1695,17 +1901,41 @@ class OpenTelemetry(CustomLogger):
value=safe_dumps(transformed_messages),
)
if kwargs.get("system_instructions"):
transformed_system_instructions = (
self._transform_messages_to_otel_semantic_conventions(
kwargs.get("system_instructions")
# Coalesce the different kwarg names that carry the system
# prompt depending on the call path:
# - "system_instructions" — Vertex AI Gemini chat-completion
# - "instructions" — OpenAI Responses API
# - "system" — Anthropic Messages API
# Use `is not None` rather than truthiness to avoid falsy
# values (e.g. []) falling through to the wrong kwarg.
system_instructions = (
kwargs.get("system_instructions")
if kwargs.get("system_instructions") is not None
else (
kwargs.get("instructions")
if kwargs.get("instructions") is not None
else kwargs.get("system")
)
)
if system_instructions:
if isinstance(system_instructions, str):
# Plain text system prompt — no transformation needed
self.safe_set_attribute(
span=span,
key=SpanAttributes.GEN_AI_SYSTEM_INSTRUCTIONS.value,
value=system_instructions,
)
else:
transformed_system_instructions = (
self._transform_messages_to_otel_semantic_conventions(
system_instructions
)
)
self.safe_set_attribute(
span=span,
key=SpanAttributes.GEN_AI_SYSTEM_INSTRUCTIONS.value,
value=safe_dumps(transformed_system_instructions),
)
)
self.safe_set_attribute(
span=span,
key=SpanAttributes.GEN_AI_SYSTEM_INSTRUCTIONS.value,
value=safe_dumps(transformed_system_instructions),
)
self.safe_set_attribute(
span=span,
@ -1764,6 +1994,57 @@ class OpenTelemetry(CustomLogger):
value=value,
)
elif response_obj.get("output"):
# Responses API: ResponsesAPIResponse has an "output"
# list instead of "choices". Each item with
# type="message" contains a "content" list of
# OutputText objects (type="output_text").
output_items = response_obj.get("output")
output_messages = self._transform_responses_api_output_to_otel(
output_items
)
if output_messages:
self.safe_set_attribute(
span=span,
key=SpanAttributes.GEN_AI_OUTPUT_MESSAGES.value,
value=safe_dumps(output_messages),
)
# Emit per-tool-call span attributes (parity with
# the choices branch that calls _tool_calls_kv_pair).
# Convert Responses API function_call items to the
# ChatCompletionMessageToolCall format expected by
# _tool_calls_kv_pair.
tool_calls = []
for out_item in output_items:
item_d = self._to_dict(out_item)
if item_d and item_d.get("type") == "function_call":
tool_calls.append(
{
"function": {
"name": item_d.get("name", ""),
"arguments": item_d.get("arguments", ""),
}
}
)
if tool_calls:
kv_pairs = OpenTelemetry._tool_calls_kv_pair(tool_calls) # type: ignore
for key, value in kv_pairs.items():
self.safe_set_attribute(
span=span,
key=key,
value=value,
)
# Extract finish reason from ResponsesAPIResponse.status
status = response_obj.get("status")
if status:
self.safe_set_attribute(
span=span,
key=SpanAttributes.GEN_AI_RESPONSE_FINISH_REASONS.value,
value=safe_dumps([status]),
)
except Exception as e:
self.handle_callback_failure(
callback_name=self.callback_name or "opentelemetry"
@ -1859,6 +2140,78 @@ class OpenTelemetry(CustomLogger):
transformed.append(transformed_msg)
return transformed
@staticmethod
def _to_dict(obj) -> Optional[dict]:
"""Normalize an object to a plain dict.
Handles three forms that appear in practice:
1. Plain ``dict`` — returned as-is.
2. LiteLLM's ``BaseLiteLLMOpenAIResponseObject`` — exposes a
``.get()`` method that delegates to ``__dict__``.
3. Raw Pydantic v2 models from the ``openai`` SDK (e.g.
``ResponseOutputMessage``, ``ResponseOutputText``) — these do
**not** have ``.get()`` but do have ``.model_dump()``.
Returns ``None`` for anything else so callers can skip it.
"""
if isinstance(obj, dict):
return obj
if hasattr(obj, "get"):
# BaseLiteLLMOpenAIResponseObject duck-type
return obj # type: ignore[return-value]
if hasattr(obj, "model_dump"):
# Raw Pydantic v2 model (e.g. openai SDK types)
return obj.model_dump() # type: ignore[union-attr]
return None
def _transform_responses_api_output_to_otel(self, output: List) -> List[dict]:
"""
Transform Responses API output items into OTEL GenAI 1.38 format.
The Responses API returns output as a list of items, each with a
``type`` field. Message items (``type="message"``) contain a
``content`` list of ``OutputText`` objects with ``type="output_text"``
and ``text`` fields.
Items may be plain dicts, LiteLLM wrapper objects (with ``.get()``),
or raw Pydantic v2 models from the ``openai`` SDK (with
``.model_dump()``). We normalize each item to a dict via
``_to_dict`` before processing.
This method converts them to the same ``{"role": ..., "parts": [...]}``
format used by ``_transform_choices_to_otel_semantic_conventions``.
"""
transformed = []
for raw_item in output:
item = self._to_dict(raw_item)
if item is None:
continue
if item.get("type") == "message":
role = item.get("role", "assistant")
parts = []
for raw_content in item.get("content", []):
content = self._to_dict(raw_content)
if content is None:
continue
if content.get("type") == "output_text":
text = content.get("text", "")
if text:
parts.append({"type": "text", "content": text})
if parts:
transformed.append({"role": role, "parts": parts})
elif item.get("type") == "function_call":
# Surface tool calls from Responses API output
part: dict = {
"type": "tool_call",
"name": item.get("name", ""),
"arguments": item.get("arguments", ""),
}
if item.get("call_id"):
part["id"] = item["call_id"]
transformed.append({"role": "assistant", "parts": [part]})
return transformed
def set_raw_request_attributes(self, span: Span, kwargs, response_obj):
try:
# Only set provider-specific raw payload attributes on this span.
@ -1965,7 +2318,7 @@ class OpenTelemetry(CustomLogger):
verbose_logger.debug(
"OpenTelemetry: Using explicit parent span from metadata"
)
return trace.set_span_in_context(parent_otel_span), parent_otel_span
return trace.set_span_in_context(parent_otel_span), None
# Priority 2: HTTP traceparent header
if traceparent is not None:

294
litellm/integrations/prometheus.py
View file

@ -25,6 +25,9 @@ from typing import (
import litellm
from litellm._logging import print_verbose, verbose_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.prometheus_helpers.bounded_prometheus_series_tracker import (
BoundedPrometheusSeriesTracker,
)
from litellm.integrations.prometheus_helpers import (
PrometheusLabelFactoryContext,
_get_cached_end_user_id_for_cost_tracking,
@ -81,6 +84,7 @@ class PrometheusLogger(CustomLogger):
if _custom_buckets is not None
else LATENCY_BUCKETS
)
self._bounded_prometheus_series_tracker = BoundedPrometheusSeriesTracker()
# Create metric factory functions
self._counter_factory = self._create_metric_factory(Counter)
@ -984,6 +988,40 @@ class PrometheusLogger(CustomLogger):
return filtered_labels
def _track_end_user_metric_series(
self,
metric: Any,
metric_name: DEFINED_PROMETHEUS_METRICS,
labels: Dict[str, Optional[str]],
) -> None:
"""
Cap the cardinality of metrics that include the ``end_user`` label.
Called *after* ``metric.labels(...).inc()/observe()`` so the emission is
recorded in prometheus-client's child map before any eviction runs.
Series that get evicted before the next scrape lose updates accrued
since the last scrape — this is inherent to any cardinality cap.
"""
labelnames = self.get_labels_for_metric(metric_name)
if UserAPIKeyLabelNames.END_USER.value not in labelnames:
return
if labels.get(UserAPIKeyLabelNames.END_USER.value) is None:
return
max_series = litellm.prometheus_end_user_metrics_max_series_per_metric
ttl_seconds = litellm.prometheus_end_user_metrics_ttl_seconds
if max_series is None and ttl_seconds is None:
return
self._bounded_prometheus_series_tracker.track_series(
metric=metric,
metric_name=metric_name,
label_values=tuple(labels.get(label) for label in labelnames),
max_series=max_series,
ttl_seconds=ttl_seconds,
cleanup_interval_seconds=litellm.prometheus_end_user_metrics_cleanup_interval_seconds,
)
def _inc_labeled_counter(
self,
counter: Any,
@ -998,6 +1036,7 @@ class PrometheusLogger(CustomLogger):
label_context=label_context,
)
counter.labels(**_labels).inc(amount)
self._track_end_user_metric_series(counter, metric_name, _labels)
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
# Define prometheus client
@ -1047,21 +1086,9 @@ class PrometheusLogger(CustomLogger):
output_tokens = standard_logging_payload["completion_tokens"]
tokens_used = standard_logging_payload["total_tokens"]
response_cost = standard_logging_payload["response_cost"]
_requester_metadata: Optional[dict] = standard_logging_payload["metadata"].get(
"requester_metadata"
combined_metadata = _get_combined_custom_metadata_from_standard_logging_payload(
standard_logging_payload=standard_logging_payload
)
user_api_key_auth_metadata: Optional[dict] = standard_logging_payload[
"metadata"
].get("user_api_key_auth_metadata")
spend_logs_metadata: Optional[dict] = standard_logging_payload["metadata"].get(
"spend_logs_metadata"
)
combined_metadata: Dict[str, Any] = {
**(_requester_metadata if _requester_metadata else {}),
**(user_api_key_auth_metadata if user_api_key_auth_metadata else {}),
**(spend_logs_metadata if spend_logs_metadata else {}),
}
if standard_logging_payload is not None and isinstance(
standard_logging_payload, dict
):
@ -1199,6 +1226,17 @@ class PrometheusLogger(CustomLogger):
label_context=label_context,
)
# Provider-agnostic fallback: providers like Bedrock and Vertex don't return
# x-ratelimit-remaining-* headers, so the gauges above only fire for OpenAI /
# Anthropic / Azure. When the proxy router has tpm/rpm configured for the
# model_group, derive remaining from configured-limit minus current usage so
# the same metric is populated for any provider.
await self._async_set_router_remaining_metrics(
standard_logging_payload=standard_logging_payload, # type: ignore
enum_values=enum_values,
label_context=label_context,
)
# cache metrics
self._increment_cache_metrics(
standard_logging_payload=standard_logging_payload, # type: ignore
@ -1416,26 +1454,46 @@ class PrometheusLogger(CustomLogger):
)
remaining_tokens_variable_name = f"litellm-key-remaining-tokens-{model_group}"
remaining_requests = (
metadata.get(remaining_requests_variable_name, sys.maxsize) or sys.maxsize
remaining_requests = metadata.get(remaining_requests_variable_name)
if remaining_requests is None:
remaining_requests = sys.maxsize
remaining_tokens = metadata.get(remaining_tokens_variable_name)
if remaining_tokens is None:
remaining_tokens = sys.maxsize
enum_values = UserAPIKeyLabelValues(
hashed_api_key=user_api_key,
api_key_alias=user_api_key_alias,
model=model_group,
model_id=model_id,
custom_metadata_labels=get_custom_labels_from_metadata(
metadata=_get_combined_custom_metadata_from_standard_logging_payload(
standard_logging_payload=kwargs.get("standard_logging_object")
)
),
)
remaining_tokens = (
metadata.get(remaining_tokens_variable_name, sys.maxsize) or sys.maxsize
label_context = PrometheusLabelFactoryContext(enum_values)
requests_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
"litellm_remaining_api_key_requests_for_model"
),
enum_values=enum_values,
label_context=label_context,
)
self.litellm_remaining_api_key_requests_for_model.labels(**requests_labels).set(
remaining_requests
)
self.litellm_remaining_api_key_requests_for_model.labels(
_sanitize_prometheus_label_value(user_api_key),
_sanitize_prometheus_label_value(user_api_key_alias),
_sanitize_prometheus_label_value(model_group),
_sanitize_prometheus_label_value(model_id),
).set(remaining_requests)
self.litellm_remaining_api_key_tokens_for_model.labels(
_sanitize_prometheus_label_value(user_api_key),
_sanitize_prometheus_label_value(user_api_key_alias),
_sanitize_prometheus_label_value(model_group),
_sanitize_prometheus_label_value(model_id),
).set(remaining_tokens)
tokens_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
"litellm_remaining_api_key_tokens_for_model"
),
enum_values=enum_values,
label_context=label_context,
)
self.litellm_remaining_api_key_tokens_for_model.labels(**tokens_labels).set(
remaining_tokens
)
def _set_latency_metrics(
self,
@ -1471,6 +1529,11 @@ class PrometheusLogger(CustomLogger):
self.litellm_llm_api_time_to_first_token_metric.labels(
**_ttft_labels
).observe(time_to_first_token_seconds)
self._track_end_user_metric_series(
self.litellm_llm_api_time_to_first_token_metric,
"litellm_llm_api_time_to_first_token_metric",
_ttft_labels,
)
else:
verbose_logger.debug(
"Time to first token metric not emitted, stream option in model_parameters is not True"
@ -1491,6 +1554,11 @@ class PrometheusLogger(CustomLogger):
self.litellm_llm_api_latency_metric.labels(**_labels).observe(
api_call_total_time_seconds
)
self._track_end_user_metric_series(
self.litellm_llm_api_latency_metric,
"litellm_llm_api_latency_metric",
_labels,
)
# total request latency
total_time_seconds = self._safe_duration_seconds(
@ -1508,6 +1576,11 @@ class PrometheusLogger(CustomLogger):
self.litellm_request_total_latency_metric.labels(**_labels).observe(
total_time_seconds
)
self._track_end_user_metric_series(
self.litellm_request_total_latency_metric,
"litellm_request_total_latency_metric",
_labels,
)
# request queue time (time from arrival to processing start)
_litellm_params = kwargs.get("litellm_params", {}) or {}
@ -1525,6 +1598,11 @@ class PrometheusLogger(CustomLogger):
self.litellm_request_queue_time_metric.labels(**_labels).observe(
queue_time_seconds
)
self._track_end_user_metric_series(
self.litellm_request_queue_time_metric,
"litellm_request_queue_time_seconds",
_labels,
)
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
verbose_logger.debug(
@ -1561,18 +1639,27 @@ class PrometheusLogger(CustomLogger):
)
try:
self.litellm_llm_api_failed_requests_metric.labels(
_sanitize_prometheus_label_value(end_user_id),
_sanitize_prometheus_label_value(user_api_key),
_sanitize_prometheus_label_value(user_api_key_alias),
_sanitize_prometheus_label_value(model),
_sanitize_prometheus_label_value(user_api_team),
_sanitize_prometheus_label_value(user_api_team_alias),
_sanitize_prometheus_label_value(user_id),
_sanitize_prometheus_label_value(
standard_logging_payload.get("model_id", "")
enum_values = UserAPIKeyLabelValues(
end_user=end_user_id,
hashed_api_key=user_api_key,
api_key_alias=user_api_key_alias,
model=model,
team=user_api_team,
team_alias=user_api_team_alias,
user=user_id,
model_id=standard_logging_payload.get("model_id", ""),
custom_metadata_labels=get_custom_labels_from_metadata(
metadata=_get_combined_custom_metadata_from_standard_logging_payload(
standard_logging_payload=standard_logging_payload
)
),
).inc()
)
PrometheusLogger._inc_labeled_counter(
self,
self.litellm_llm_api_failed_requests_metric,
"litellm_llm_api_failed_requests_metric",
enum_values,
)
self.set_llm_deployment_failure_metrics(kwargs)
await self._set_org_budget_metrics_after_api_request(
org_id=user_api_key_org_id,
@ -2123,6 +2210,99 @@ class PrometheusLogger(CustomLogger):
)
self.litellm_deployment_rpm_limit.labels(**_labels).set(rpm)
async def _async_set_router_remaining_metrics(
self,
standard_logging_payload: StandardLoggingPayload,
enum_values: UserAPIKeyLabelValues,
label_context: Optional[PrometheusLabelFactoryContext] = None,
) -> None:
"""
Populate ``litellm_remaining_tokens_metric`` /
``litellm_remaining_requests_metric`` from the router's internal usage
counters when the upstream provider did not return
``x-ratelimit-remaining-*`` response headers.
OpenAI / Anthropic / Azure return remaining tokens/requests in response
headers, but Bedrock and Vertex AI do not. This fallback computes
``configured_limit - current_usage`` via
``Router.get_remaining_model_group_usage`` so the same gauges are
emitted for every provider when tpm/rpm is configured on the
deployment.
"""
try:
additional_headers = (
standard_logging_payload.get("hidden_params", {}) or {}
).get("additional_headers") or {}
already_have_tokens = (
additional_headers.get("x_ratelimit_remaining_tokens") is not None
)
already_have_requests = (
additional_headers.get("x_ratelimit_remaining_requests") is not None
)
if already_have_tokens and already_have_requests:
return
model_group = standard_logging_payload.get("model_group")
if not model_group:
return
try:
from litellm.proxy.proxy_server import llm_router
except ImportError:
llm_router = None
if llm_router is None:
return
try:
remaining_usage = await llm_router.get_remaining_model_group_usage(
model_group
)
except Exception as e:
verbose_logger.exception(
"Prometheus: get_remaining_model_group_usage failed for "
"model_group=%s: %s",
model_group,
e,
)
return
if not remaining_usage:
return
remaining_tokens = remaining_usage.get("x-ratelimit-remaining-tokens")
remaining_requests = remaining_usage.get("x-ratelimit-remaining-requests")
if not already_have_tokens and remaining_tokens is not None:
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_remaining_tokens_metric"
),
enum_values=enum_values,
label_context=label_context,
)
self.litellm_remaining_tokens_metric.labels(**_labels).set(
remaining_tokens
)
if not already_have_requests and remaining_requests is not None:
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_remaining_requests_metric"
),
enum_values=enum_values,
label_context=label_context,
)
self.litellm_remaining_requests_metric.labels(**_labels).set(
remaining_requests
)
except Exception as e:
verbose_logger.exception(
"Prometheus Error: _async_set_router_remaining_metrics. "
"Exception occured - {}".format(str(e))
)
def set_llm_deployment_success_metrics(
self,
request_kwargs: dict,
@ -3622,6 +3802,36 @@ def get_custom_labels_from_metadata(metadata: dict) -> Dict[str, str]:
return result
def _get_combined_custom_metadata_from_standard_logging_payload(
standard_logging_payload: Optional[dict],
) -> Dict[str, Any]:
"""
Combine the metadata sources that can supply custom Prometheus labels.
"""
if not isinstance(standard_logging_payload, dict):
return {}
standard_logging_metadata = standard_logging_payload.get("metadata") or {}
if not isinstance(standard_logging_metadata, dict):
return {}
requester_metadata = standard_logging_metadata.get("requester_metadata")
user_api_key_auth_metadata = standard_logging_metadata.get(
"user_api_key_auth_metadata"
)
spend_logs_metadata = standard_logging_metadata.get("spend_logs_metadata")
return {
**(requester_metadata if isinstance(requester_metadata, dict) else {}),
**(
user_api_key_auth_metadata
if isinstance(user_api_key_auth_metadata, dict)
else {}
),
**(spend_logs_metadata if isinstance(spend_logs_metadata, dict) else {}),
}
def _tag_matches_wildcard_configured_pattern(
tags: Sequence[str], configured_tag: str
) -> bool:

107
litellm/integrations/prometheus_helpers/bounded_prometheus_series_tracker.py Normal file
View file

@ -0,0 +1,107 @@
from __future__ import annotations
import time
from collections import OrderedDict
from threading import RLock
from typing import Any, Dict, Optional
class BoundedPrometheusSeriesTracker:
"""
Tracks Prometheus child series and removes stale/excess labelsets.
The tracker is label-agnostic: callers decide which series should be tracked
and pass the full label tuple used by the Prometheus metric.
"""
def __init__(self) -> None:
self._series: Dict[str, OrderedDict[tuple[Optional[str], ...], float]] = {}
self._last_ttl_cleanup: Dict[str, float] = {}
self.lock = RLock()
def track_series(
self,
metric: Any,
metric_name: str,
label_values: tuple[Optional[str], ...],
max_series: Optional[int],
ttl_seconds: Optional[float],
cleanup_interval_seconds: Optional[float],
) -> None:
if max_series is None and ttl_seconds is None:
return
now = time.monotonic()
with self.lock:
series = self._series.setdefault(metric_name, OrderedDict())
series[label_values] = now
series.move_to_end(label_values)
if ttl_seconds is not None and self._should_run_ttl_cleanup(
metric_name=metric_name,
now=now,
cleanup_interval_seconds=cleanup_interval_seconds,
):
expired_label_values = [
tracked_label_values
for tracked_label_values, last_seen in series.items()
if now - last_seen > ttl_seconds
]
for tracked_label_values in expired_label_values:
self._remove_metric_series(metric, series, tracked_label_values)
# max_series <= 0 is treated as "unlimited" so a misconfigured zero
# value cannot silently drop every emission for this metric.
if max_series is not None and max_series > 0:
while len(series) > max_series:
tracked_label_values = next(iter(series))
if not self._remove_metric_child(metric, tracked_label_values):
break
del series[tracked_label_values]
def _should_run_ttl_cleanup(
self,
metric_name: str,
now: float,
cleanup_interval_seconds: Optional[float],
) -> bool:
if cleanup_interval_seconds is None or cleanup_interval_seconds <= 0:
self._last_ttl_cleanup[metric_name] = now
return True
last_cleanup = self._last_ttl_cleanup.get(metric_name)
if last_cleanup is None or now - last_cleanup >= cleanup_interval_seconds:
self._last_ttl_cleanup[metric_name] = now
return True
return False
def _remove_metric_series(
self,
metric: Any,
series: OrderedDict[tuple[Optional[str], ...], float],
label_values: tuple[Optional[str], ...],
) -> None:
if self._remove_metric_child(metric, label_values):
series.pop(label_values, None)
@staticmethod
def _remove_metric_child(
metric: Any, label_values: tuple[Optional[str], ...]
) -> bool:
"""
Remove the Prometheus child for ``label_values`` and report whether the
tracker should commit the matching state change.
Returns ``True`` when the child is no longer present in Prometheus
(either it was just removed or it was already gone), and ``False`` when
``metric.remove()`` raised an unexpected error and the child likely
still exists.
"""
try:
metric.remove(*label_values)
return True
except KeyError:
return True
except (AttributeError, ValueError):
return False

100
litellm/integrations/s3_v2.py
View file

@ -16,6 +16,7 @@ from litellm._logging import print_verbose, verbose_logger
from litellm.constants import DEFAULT_S3_BATCH_SIZE, DEFAULT_S3_FLUSH_INTERVAL_SECONDS
from litellm.integrations.s3 import get_s3_object_key
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.custom_httpx.http_handler import (
_get_httpx_client,
@ -53,15 +54,25 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
s3_strip_base64_files: bool = False,
s3_use_key_prefix: bool = False,
s3_use_virtual_hosted_style: bool = False,
s3_callback_params_override: Optional[dict] = None,
**kwargs,
):
try:
verbose_logger.debug(
f"in init s3 logger - s3_callback_params {litellm.s3_callback_params}"
)
_masker = SensitiveDataMasker()
if s3_callback_params_override is not None:
verbose_logger.debug(
f"in init s3 logger (audit override) - "
f"{_masker.mask_dict(dict(s3_callback_params_override))}"
)
else:
verbose_logger.debug(
f"in init s3 logger - s3_callback_params "
f"{_masker.mask_dict(dict(litellm.s3_callback_params or {}))}"
)
# Initialize S3 params first to get the correct s3_verify value
self._init_s3_params(
params_source=s3_callback_params_override,
s3_bucket_name=s3_bucket_name,
s3_region_name=s3_region_name,
s3_api_version=s3_api_version,
@ -139,94 +150,85 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
s3_strip_base64_files: bool = False,
s3_use_key_prefix: bool = False,
s3_use_virtual_hosted_style: bool = False,
params_source: Optional[dict] = None,
):
"""
Initialize the s3 params for this logging callback
Initialize the s3 params for this logging callback. Reads from
`params_source` if given (e.g. `s3_audit_callback_params` for the
audit-log instance), otherwise falls back to `litellm.s3_callback_params`.
Resolves `os.environ/X` markers into a local dict; never mutates the source.
"""
litellm.s3_callback_params = litellm.s3_callback_params or {}
# read in .env variables - example os.environ/AWS_BUCKET_NAME
for key, value in litellm.s3_callback_params.items():
if isinstance(value, str) and value.startswith("os.environ/"):
litellm.s3_callback_params[key] = litellm.get_secret(value)
if params_source is None:
params_source = litellm.s3_callback_params or {}
params: dict = {
key: (
litellm.get_secret(value)
if isinstance(value, str) and value.startswith("os.environ/")
else value
)
for key, value in params_source.items()
}
self.s3_bucket_name = (
litellm.s3_callback_params.get("s3_bucket_name") or s3_bucket_name
)
self.s3_region_name = (
litellm.s3_callback_params.get("s3_region_name") or s3_region_name
)
self.s3_api_version = (
litellm.s3_callback_params.get("s3_api_version") or s3_api_version
)
self.s3_bucket_name = params.get("s3_bucket_name") or s3_bucket_name
self.s3_region_name = params.get("s3_region_name") or s3_region_name
self.s3_api_version = params.get("s3_api_version") or s3_api_version
self.s3_use_ssl = (
litellm.s3_callback_params.get("s3_use_ssl", True)
if litellm.s3_callback_params.get("s3_use_ssl") is not None
params.get("s3_use_ssl", True)
if params.get("s3_use_ssl") is not None
else s3_use_ssl
)
self.s3_verify = (
litellm.s3_callback_params.get("s3_verify")
if litellm.s3_callback_params.get("s3_verify") is not None
params.get("s3_verify")
if params.get("s3_verify") is not None
else s3_verify
)
self.s3_endpoint_url = (
litellm.s3_callback_params.get("s3_endpoint_url") or s3_endpoint_url
)
self.s3_endpoint_url = params.get("s3_endpoint_url") or s3_endpoint_url
self.s3_aws_access_key_id = (
litellm.s3_callback_params.get("s3_aws_access_key_id")
or s3_aws_access_key_id
params.get("s3_aws_access_key_id") or s3_aws_access_key_id
)
self.s3_aws_secret_access_key = (
litellm.s3_callback_params.get("s3_aws_secret_access_key")
or s3_aws_secret_access_key
params.get("s3_aws_secret_access_key") or s3_aws_secret_access_key
)
self.s3_aws_session_token = (
litellm.s3_callback_params.get("s3_aws_session_token")
or s3_aws_session_token
params.get("s3_aws_session_token") or s3_aws_session_token
)
self.s3_aws_session_name = (
litellm.s3_callback_params.get("s3_aws_session_name") or s3_aws_session_name
params.get("s3_aws_session_name") or s3_aws_session_name
)
self.s3_aws_profile_name = (
litellm.s3_callback_params.get("s3_aws_profile_name") or s3_aws_profile_name
params.get("s3_aws_profile_name") or s3_aws_profile_name
)
self.s3_aws_role_name = (
litellm.s3_callback_params.get("s3_aws_role_name") or s3_aws_role_name
)
self.s3_aws_role_name = params.get("s3_aws_role_name") or s3_aws_role_name
self.s3_aws_web_identity_token = (
litellm.s3_callback_params.get("s3_aws_web_identity_token")
or s3_aws_web_identity_token
params.get("s3_aws_web_identity_token") or s3_aws_web_identity_token
)
self.s3_aws_sts_endpoint = (
litellm.s3_callback_params.get("s3_aws_sts_endpoint") or s3_aws_sts_endpoint
params.get("s3_aws_sts_endpoint") or s3_aws_sts_endpoint
)
self.s3_config = litellm.s3_callback_params.get("s3_config") or s3_config
self.s3_path = litellm.s3_callback_params.get("s3_path") or s3_path
# done reading litellm.s3_callback_params
self.s3_config = params.get("s3_config") or s3_config
self.s3_path = params.get("s3_path") or s3_path
self.s3_use_team_prefix = (
bool(litellm.s3_callback_params.get("s3_use_team_prefix", False))
or s3_use_team_prefix
bool(params.get("s3_use_team_prefix", False)) or s3_use_team_prefix
)
self.s3_use_key_prefix = (
bool(litellm.s3_callback_params.get("s3_use_key_prefix", False))
or s3_use_key_prefix
bool(params.get("s3_use_key_prefix", False)) or s3_use_key_prefix
)
self.s3_strip_base64_files = (
bool(litellm.s3_callback_params.get("s3_strip_base64_files", False))
or s3_strip_base64_files
bool(params.get("s3_strip_base64_files", False)) or s3_strip_base64_files
)
self.s3_use_virtual_hosted_style = (
bool(litellm.s3_callback_params.get("s3_use_virtual_hosted_style", False))
bool(params.get("s3_use_virtual_hosted_style", False))
or s3_use_virtual_hosted_style
)

247
litellm/integrations/websearch_interception/handler.py
View file

@ -19,12 +19,14 @@ from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.websearch_interception.tools import (
get_litellm_web_search_tool,
get_litellm_web_search_tool_openai,
is_anthropic_native_web_search_tool,
is_web_search_tool,
is_web_search_tool_chat_completion,
)
from litellm.integrations.websearch_interception.transformation import (
WebSearchTransformation,
)
from litellm.llms.base_llm.search.transformation import SearchResponse
from litellm.types.integrations.websearch_interception import (
WebSearchInterceptionConfig,
)
@ -36,6 +38,16 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager
# Key used to flag, on per-request kwargs, that the originating client sent
# an Anthropic-native ``web_search_*`` tool — meaning the final response
# should include ``web_search_tool_result`` content blocks so the client
# (e.g. Claude Desktop's citations panel) can render sources.
WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY = "_websearch_interception_emit_native_blocks"
# Key on ``AgenticLoopPlan.metadata`` carrying the list of pre-built
# ``web_search_tool_result`` blocks to inject into the final response.
WEBSEARCH_NATIVE_BLOCKS_METADATA_KEY = "websearch_native_blocks"
class WebSearchInterceptionLogger(CustomLogger):
"""
@ -152,22 +164,55 @@ class WebSearchInterceptionLogger(CustomLogger):
f"(provider={provider_str}, query='{query}')"
)
# Execute search
# Native clients (Claude Desktop / Cowork / Anthropic SDK) make a
# standalone /v1/messages sub-request just for the search, and they
# expect the response in native shape with server_tool_use +
# web_search_tool_result content blocks so the citations panel can
# render. The agentic-loop post-hook never fires on this path because
# there is no model call — emit the native blocks here instead.
native_tool = next(
(t for t in tools if is_anthropic_native_web_search_tool(t)),
None,
)
# Execute search — keep the structured SearchResponse so the native
# block can carry per-result url/title/page_age.
try:
search_result_text = await self._execute_search(query)
search_result_text, structured = await self._execute_search(query)
except Exception as e:
verbose_logger.error(
f"WebSearchInterception: Short-circuit search failed: {e}"
)
search_result_text = f"Search failed: {e}"
search_result_text, structured = f"Search failed: {e}", None
content: List[Dict[str, Any]] = []
if native_tool is not None:
tool_use_id = f"srvtoolu_{uuid.uuid4().hex}"
tool_name = native_tool.get("name") or "web_search"
content.append(
{
"type": "server_tool_use",
"id": tool_use_id,
"name": tool_name,
"input": {"query": query},
}
)
content.append(
WebSearchTransformation.build_web_search_tool_result_block(
tool_use_id=tool_use_id,
search_response=structured,
)
)
# Keep the text block so non-native short-circuit callers (Claude Code,
# github_copilot, etc.) see the same payload they always have.
content.append({"type": "text", "text": search_result_text})
# Build synthetic Anthropic response
response: Dict[str, Any] = {
"id": f"msg_{str(uuid.uuid4())}",
"type": "message",
"role": "assistant",
"model": model,
"content": [{"type": "text", "text": search_result_text}],
"content": content,
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 0, "output_tokens": 0},
@ -175,7 +220,8 @@ class WebSearchInterceptionLogger(CustomLogger):
verbose_logger.debug(
"WebSearchInterception: Short-circuit search completed, "
f"returning synthetic response ({len(search_result_text)} chars)"
f"returning synthetic response ({len(search_result_text)} chars, "
f"native_blocks={native_tool is not None})"
)
return response
@ -219,6 +265,14 @@ class WebSearchInterceptionLogger(CustomLogger):
"WebSearchInterception: Converting native web_search tools to LiteLLM standard"
)
# If the client sent an Anthropic-native web_search_* tool, mark the
# request so the agentic loop emits native web_search_tool_result
# blocks in the final response (matches async_pre_request_hook). This
# deployment hook fires before async_pre_request_hook on some paths,
# so flagging here ensures the signal isn't lost regardless of order.
if any(is_anthropic_native_web_search_tool(t) for t in tools):
kwargs[WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY] = True
# Convert native/custom web_search tools to LiteLLM standard
converted_tools = []
for tool in tools:
@ -342,6 +396,14 @@ class WebSearchInterceptionLogger(CustomLogger):
f"WebSearchInterception: Pre-request hook triggered for provider={custom_llm_provider}"
)
# If the client sent an Anthropic-native web_search_* tool, mark the
# request so the agentic loop emits native web_search_tool_result
# blocks in the final response (for citations panels, etc.). The flag
# is read by async_build_agentic_loop_plan; the leading underscore
# prefix ensures it is stripped before the follow-up call kwargs.
if any(is_anthropic_native_web_search_tool(t) for t in tools):
kwargs[WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY] = True
# Convert native web search tools to LiteLLM standard
converted_tools = []
for tool in tools:
@ -591,7 +653,7 @@ class WebSearchInterceptionLogger(CustomLogger):
) -> AgenticLoopPlan:
tool_calls = tools["tool_calls"]
thinking_blocks = tools.get("thinking_blocks", [])
request_patch = await self._build_anthropic_request_patch(
request_patch, structured_results = await self._build_anthropic_request_patch(
model=model,
messages=messages,
tool_calls=tool_calls,
@ -600,12 +662,92 @@ class WebSearchInterceptionLogger(CustomLogger):
logging_obj=logging_obj,
kwargs=kwargs,
)
metadata: Dict[str, Any] = {
"tool_type": "websearch",
"response_format": "anthropic",
}
# If the client request originally carried a native web_search_* tool,
# pre-build the Anthropic-native ``web_search_tool_result`` blocks now
# (while we still have the structured SearchResponse list) and stash
# them on plan metadata for the post-hook to inject.
if kwargs.get(WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY):
metadata[WEBSEARCH_NATIVE_BLOCKS_METADATA_KEY] = (
self._build_native_result_blocks(
tool_calls=tool_calls,
structured_results=structured_results,
)
)
return AgenticLoopPlan(
run_agentic_loop=True,
request_patch=request_patch,
metadata={"tool_type": "websearch", "response_format": "anthropic"},
metadata=metadata,
)
async def async_post_agentic_loop_response_hook(
self,
response: Any,
plan: AgenticLoopPlan,
kwargs: Dict,
) -> Any:
"""
Inject Anthropic-native ``web_search_tool_result`` blocks into the
final response when the originating client used a native
``web_search_*`` tool.
See ``WebSearchTransformation.build_web_search_tool_result_block`` for
the block shape. The blocks are prepended to ``response.content`` so
Anthropic-native clients (Claude Desktop, the Anthropic SDK) can
render citations / sources alongside the model's textual reply.
"""
native_blocks = plan.metadata.get(WEBSEARCH_NATIVE_BLOCKS_METADATA_KEY)
if not native_blocks:
return response
return self._inject_native_blocks(response, native_blocks)
@staticmethod
def _build_native_result_blocks(
tool_calls: List[Dict],
structured_results: List[Optional[SearchResponse]],
) -> List[Dict[str, Any]]:
"""Build one ``web_search_tool_result`` block per tool_call."""
blocks: List[Dict[str, Any]] = []
for i, tool_call in enumerate(tool_calls):
tool_use_id = tool_call.get("id") or ""
structured = structured_results[i] if i < len(structured_results) else None
blocks.append(
WebSearchTransformation.build_web_search_tool_result_block(
tool_use_id=tool_use_id,
search_response=structured,
)
)
return blocks
@staticmethod
def _inject_native_blocks(
response: Any, native_blocks: List[Dict[str, Any]]
) -> Any:
"""Prepend native blocks to response content, dict or object form."""
if not native_blocks:
return response
if isinstance(response, dict):
existing = response.get("content") or []
response["content"] = list(native_blocks) + list(existing)
return response
existing = getattr(response, "content", None) or []
try:
response.content = list(native_blocks) + list(existing)
except (AttributeError, TypeError):
# Object refused write — fall through and leave the response
# untouched rather than crash the request.
verbose_logger.debug(
"WebSearchInterception: could not inject native blocks into "
f"response of type {type(response).__name__}"
)
return response
async def async_run_chat_completion_agentic_loop(
self,
tools: Dict,
@ -733,7 +875,7 @@ class WebSearchInterceptionLogger(CustomLogger):
kwargs: Dict,
) -> Any:
"""Legacy path: execute search + build patch + run follow-up call."""
request_patch = await self._build_anthropic_request_patch(
request_patch, structured_results = await self._build_anthropic_request_patch(
model=model,
messages=messages,
tool_calls=tool_calls,
@ -755,7 +897,7 @@ class WebSearchInterceptionLogger(CustomLogger):
if max_tokens is None:
max_tokens = cast(int, kwargs.get("max_tokens", 1024))
return await anthropic_messages.acreate(
response = await anthropic_messages.acreate(
max_tokens=max_tokens,
messages=request_patch.messages,
model=request_patch.model or model,
@ -763,6 +905,18 @@ class WebSearchInterceptionLogger(CustomLogger):
**request_patch.kwargs,
)
# Legacy path: the new path goes through the typed plan + core
# dispatcher which runs the post-hook automatically. Mirror the
# native-block injection here so both paths behave identically.
if kwargs.get(WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY):
native_blocks = self._build_native_result_blocks(
tool_calls=tool_calls,
structured_results=structured_results,
)
response = self._inject_native_blocks(response, native_blocks)
return response
async def _build_anthropic_request_patch(
self,
model: str,
@ -772,8 +926,16 @@ class WebSearchInterceptionLogger(CustomLogger):
anthropic_messages_optional_request_params: Dict,
logging_obj: Any,
kwargs: Dict,
) -> AgenticLoopRequestPatch:
"""Execute litellm.search() and build follow-up request patch."""
) -> Tuple[AgenticLoopRequestPatch, List[Optional[SearchResponse]]]:
"""
Execute litellm.search() and build follow-up request patch.
Returns the patch alongside the parallel list of structured
``SearchResponse`` objects (one per tool_call, ``None`` when the
search failed or the tool_call had no query). The caller uses these
to optionally build Anthropic-native ``web_search_tool_result``
content blocks for the final response.
"""
# Extract search queries from tool_use blocks
search_tasks = []
@ -797,23 +959,38 @@ class WebSearchInterceptionLogger(CustomLogger):
)
search_results = await asyncio.gather(*search_tasks, return_exceptions=True)
# Handle any exceptions in search results
# Split the gathered (text, structured) tuples into two parallel lists.
# The text list feeds the follow-up model call; the structured list
# is returned to the caller for native-block emission.
final_search_results: List[str] = []
structured_results: List[Optional[SearchResponse]] = []
for i, result in enumerate(search_results):
if isinstance(result, Exception):
verbose_logger.error(
f"WebSearchInterception: Search {i} failed with error: {str(result)}"
)
final_search_results.append(f"Search failed: {str(result)}")
elif isinstance(result, str):
# Explicitly cast to str for type checker
final_search_results.append(cast(str, result))
structured_results.append(None)
elif isinstance(result, tuple) and len(result) == 2:
text_value, structured_value = result
final_search_results.append(
cast(str, text_value)
if isinstance(text_value, str)
else str(text_value)
)
structured_results.append(
structured_value
if isinstance(structured_value, SearchResponse)
else None
)
else:
# Should never happen, but handle for type safety
# Defensive: legacy callers / unexpected shape — preserve text,
# drop structure.
verbose_logger.debug(
f"WebSearchInterception: Unexpected result type {type(result)} at index {i}"
)
final_search_results.append(str(result))
structured_results.append(None)
# Build assistant and user messages using transformation
assistant_message, user_message = WebSearchTransformation.transform_response(
@ -859,16 +1036,26 @@ class WebSearchInterceptionLogger(CustomLogger):
len(follow_up_messages),
len(final_search_results),
)
return AgenticLoopRequestPatch(
patch = AgenticLoopRequestPatch(
model=full_model_name,
messages=follow_up_messages,
max_tokens=max_tokens,
optional_params=optional_params_without_max_tokens,
kwargs=kwargs_for_followup,
)
return patch, structured_results
async def _execute_search(self, query: str) -> str:
"""Execute a single web search using router's search tools"""
async def _execute_search(self, query: str) -> Tuple[str, Optional[SearchResponse]]:
"""
Execute a single web search using router's search tools.
Returns both the formatted text (fed back to the model in the follow-up
call) and the structured ``SearchResponse`` (preserved so callers can
build Anthropic-native ``web_search_tool_result`` blocks for clients
that requested a native ``web_search_*`` tool). The structured value
is None on the failure path so callers can still emit an empty result
block rather than dropping the search entirely.
"""
try:
# Import router from proxy_server
try:
@ -934,7 +1121,7 @@ class WebSearchInterceptionLogger(CustomLogger):
verbose_logger.debug(
f"WebSearchInterception: Search completed for '{query}', got {len(search_result_text)} chars"
)
return search_result_text
return search_result_text, result
except Exception as e:
verbose_logger.error(
f"WebSearchInterception: Search failed for '{query}': {str(e)}"
@ -1015,7 +1202,8 @@ class WebSearchInterceptionLogger(CustomLogger):
)
search_results = await asyncio.gather(*search_tasks, return_exceptions=True)
# Handle any exceptions in search results
# Chat-completion path only needs text — OpenAI tool_result format
# has no equivalent of Anthropic's web_search_tool_result block.
final_search_results: List[str] = []
for i, result in enumerate(search_results):
if isinstance(result, Exception):
@ -1023,8 +1211,13 @@ class WebSearchInterceptionLogger(CustomLogger):
f"WebSearchInterception: Search {i} failed with error: {str(result)}"
)
final_search_results.append(f"Search failed: {str(result)}")
elif isinstance(result, str):
final_search_results.append(cast(str, result))
elif isinstance(result, tuple) and len(result) == 2:
text_value, _ = result
final_search_results.append(
cast(str, text_value)
if isinstance(text_value, str)
else str(text_value)
)
else:
verbose_logger.debug(
f"WebSearchInterception: Unexpected result type {type(result)} at index {i}"
@ -1112,9 +1305,11 @@ class WebSearchInterceptionLogger(CustomLogger):
kwargs=kwargs_for_followup,
)
async def _create_empty_search_result(self) -> str:
async def _create_empty_search_result(
self,
) -> Tuple[str, Optional[SearchResponse]]:
"""Create an empty search result for tool calls without queries"""
return "No search query provided"
return "No search query provided", None
@staticmethod
def initialize_from_proxy_config(

47
litellm/integrations/websearch_interception/tools.py
View file

@ -126,6 +126,27 @@ def is_web_search_tool_chat_completion(tool: Dict[str, Any]) -> bool:
return False
def is_anthropic_native_web_search_tool(tool: Dict[str, Any]) -> bool:
"""
Check if a tool is an Anthropic-native ``web_search_*`` tool.
Native clients (Anthropic SDK, Claude Desktop, Anthropic Console) send
tools like ``{"type": "web_search_20250305", "name": "web_search"}`` and
expect the response to contain ``web_search_tool_result`` content blocks
so that citations can be rendered. This helper identifies that contract
so the agentic loop can emit native-format blocks for those clients
without affecting clients that send the LiteLLM standard tool.
Returns False for the LiteLLM standard tool (``litellm_web_search``),
the OpenAI-shaped variant, the bare ``WebSearch`` legacy name, and the
bare ``web_search`` name (Claude Code style).
"""
tool_type = tool.get("type", "")
if not isinstance(tool_type, str):
return False
return tool_type.startswith("web_search_") and tool_type != "function"
def is_web_search_tool(tool: Dict[str, Any]) -> bool:
"""
Check if a tool is a web search tool (native or LiteLLM standard).
@ -135,7 +156,22 @@ def is_web_search_tool(tool: Dict[str, Any]) -> bool:
- OpenAI format: type == "function" with function.name == "litellm_web_search"
- Anthropic native: type starts with "web_search_" (e.g., "web_search_20250305")
- Claude Code: name == "web_search" with a type field
- Custom: name == "WebSearch" (legacy format)
- Custom: name == "WebSearch" (legacy interception marker — only matched
when input_schema is absent; see note below)
Note on the legacy ``WebSearch`` name:
Clients like Claude Desktop / Cowork ship a *client-side* tool called
``WebSearch`` (a fully-formed Anthropic client tool with its own
``input_schema``) that they handle themselves. Treating that as our
interception marker hijacks it server-side and the client's own tool
handler never fires — which means Cowork's separate native
``web_search_20250305`` sub-request (where citation data actually
flows) never gets made.
Real Anthropic client tools always carry an ``input_schema`` (the API
rejects them otherwise), so a bare ``{name: "WebSearch"}`` with no
schema is the only thing that could be a legacy interception marker.
Gate the match on schema absence to keep both groups working.
Args:
tool: Tool dictionary to check
@ -152,6 +188,10 @@ def is_web_search_tool(tool: Dict[str, Any]) -> bool:
True
>>> is_web_search_tool({"name": "calculator"})
False
>>> is_web_search_tool({"name": "WebSearch"}) # legacy interception marker
True
>>> is_web_search_tool({"name": "WebSearch", "input_schema": {"type": "object"}}) # Cowork client tool
False
"""
tool_name = tool.get("name", "")
tool_type = tool.get("type", "")
@ -175,8 +215,9 @@ def is_web_search_tool(tool: Dict[str, Any]) -> bool:
if tool_name == "web_search" and tool_type:
return True
# Check for legacy WebSearch format
if tool_name == "WebSearch":
# Legacy "WebSearch" interception marker — only when no schema is
# present, so real client-side WebSearch tools (Cowork) pass through.
if tool_name == "WebSearch" and "input_schema" not in tool:
return True
return False

66
litellm/integrations/websearch_interception/transformation.py
View file

@ -100,11 +100,14 @@ class WebSearchTransformation:
block_id = getattr(block, "id", None)
block_input = getattr(block, "input", {})
# Check for LiteLLM standard or legacy web search tools
# Handles: litellm_web_search, WebSearch, web_search
# Detect tool_use blocks that came from interception. After
# pre-request conversion the model always sees
# ``litellm_web_search``; the bare ``web_search`` entry handles
# callers that bypass our pre-request hooks (e.g. direct
# litellm.acompletion). "WebSearch" is intentionally omitted —
# see is_web_search_tool for the Cowork rationale.
if block_type == "tool_use" and block_name in (
LITELLM_WEB_SEARCH_TOOL_NAME,
"WebSearch",
"web_search",
):
# Convert to dict for easier handling
@ -190,10 +193,12 @@ class WebSearchTransformation:
getattr(function, "arguments", None) if function else None
)
# Check for LiteLLM standard or legacy web search tools
# Detect function-style web search tool_calls. ``WebSearch`` is
# intentionally omitted — see is_web_search_tool for the Cowork
# rationale (clients ship their own client-side ``WebSearch`` and
# we must not hijack it).
if tool_type == "function" and function_name in (
LITELLM_WEB_SEARCH_TOOL_NAME,
"WebSearch",
"web_search",
):
# Parse arguments (might be JSON string)
@ -350,6 +355,57 @@ class WebSearchTransformation:
return assistant_message, tool_messages
@staticmethod
def build_web_search_tool_result_block(
tool_use_id: str,
search_response: Optional[SearchResponse],
) -> Dict[str, Any]:
"""
Build an Anthropic-native ``web_search_tool_result`` content block.
Native Anthropic clients (Claude Desktop, the Anthropic SDK, the
Anthropic Console) expect search-tool results to be returned as
structured ``web_search_tool_result`` blocks so that citations and
source links can be rendered. The agentic loop currently feeds the
model a flat text blob in the follow-up call (which is correct — the
model needs readable evidence). This helper produces the *additional*
block that should accompany the model's text reply when the original
request used a native ``web_search_*`` tool.
Spec reference:
https://docs.anthropic.com/en/api/web-search-tool
Args:
tool_use_id: The ``tool_use_id`` the model emitted on the first
turn. Must match exactly so the client can pair the result
with its tool_use block.
search_response: Structured ``SearchResponse`` from
``litellm.asearch()``. If None or empty, the block is still
emitted with an empty result list (signals "search ran, no
results" rather than "search did not run").
"""
items: List[Dict[str, Any]] = []
if search_response is not None:
results = getattr(search_response, "results", None) or []
for r in results:
url = getattr(r, "url", "") or ""
title = getattr(r, "title", "") or ""
page_age = getattr(r, "date", None) or getattr(r, "last_updated", None)
items.append(
{
"type": "web_search_result",
"url": url,
"title": title,
"page_age": page_age,
"encrypted_content": "",
}
)
return {
"type": "web_search_tool_result",
"tool_use_id": tool_use_id,
"content": items,
}
@staticmethod
def format_search_response(result: SearchResponse) -> str:
"""

45
litellm/litellm_core_utils/audio_utils/utils.py
View file

@ -53,8 +53,19 @@ def process_audio_file(audio_file: FileTypes) -> ProcessedAudioFile:
# Raw bytes
filename = "audio.wav"
file_content = bytes(audio_file)
elif isinstance(audio_file, (str, os.PathLike)):
# File path or PathLike
elif isinstance(audio_file, str):
# Bare strings are rejected — see extract_file_data for the same
# rationale: in a proxy request handler the string is
# attacker-controlled, and opening it as a path is an arbitrary
# file read.
raise ValueError(
"process_audio_file does not accept bare str inputs. Pass bytes, "
"an open file handle, a (filename, content) tuple, or a "
"pathlib.Path."
)
elif isinstance(audio_file, os.PathLike):
# File path or PathLike — PathLike is a Python-level type that
# HTTP form values can't fabricate.
file_path = str(audio_file)
with open(file_path, "rb") as f:
file_content = f.read()
@ -66,8 +77,14 @@ def process_audio_file(audio_file: FileTypes) -> ProcessedAudioFile:
content = audio_file[1]
if isinstance(content, (bytes, bytearray)):
file_content = bytes(content)
elif isinstance(content, (str, os.PathLike)):
# File path or PathLike
elif isinstance(content, str):
raise ValueError(
"process_audio_file does not accept bare str tuple "
"contents. Pass bytes, an open file handle, or a "
"pathlib.Path."
)
elif isinstance(content, os.PathLike):
# PathLike: SDK convenience for local-file uploads.
with open(str(content), "rb") as f:
file_content = f.read()
elif hasattr(content, "read"):
@ -149,7 +166,14 @@ def get_audio_file_content_hash(file_obj: FileTypes) -> str:
try:
if isinstance(file_content_obj, (bytes, bytearray)):
file_content = bytes(file_content_obj)
elif isinstance(file_content_obj, (str, os.PathLike)):
elif isinstance(file_content_obj, str):
# Bare strings are not treated as file paths in this helper —
# the cache-key path is reached from request handlers where the
# value is attacker-controlled. Fall back to hashing the string
# itself rather than opening it.
fallback_filename = file_content_obj
file_content = None
elif isinstance(file_content_obj, os.PathLike):
try:
with open(str(file_content_obj), "rb") as f:
file_content = f.read()
@ -229,8 +253,15 @@ def calculate_request_duration(file: FileTypes) -> Optional[float]:
if isinstance(file, (bytes, bytearray)):
# Raw bytes
file_content = bytes(file)
elif isinstance(file, (str, os.PathLike)):
# File path
elif isinstance(file, str):
# Bare strings are rejected — see extract_file_data.
raise ValueError(
"calculate_request_duration does not accept bare str inputs. "
"Pass bytes, an open file handle, a (filename, content) "
"tuple, or a pathlib.Path."
)
elif isinstance(file, os.PathLike):
# File path (PathLike): SDK convenience.
with open(str(file), "rb") as f:
file_content = f.read()
elif isinstance(file, tuple):

2
litellm/litellm_core_utils/custom_logger_registry.py
View file

@ -14,6 +14,7 @@ from litellm import _custom_logger_compatible_callbacks_literal
from litellm.integrations.agentops import AgentOps
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
from litellm.integrations.argilla import ArgillaLogger
from litellm.integrations.azure_sentinel.azure_sentinel import AzureSentinelLogger
from litellm.integrations.azure_storage.azure_storage import AzureBlobStorageLogger
from litellm.integrations.bitbucket import BitBucketPromptManager
from litellm.integrations.braintrust_logging import BraintrustLogger
@ -73,6 +74,7 @@ class CustomLoggerRegistry:
"opik": OpikLogger,
"argilla": ArgillaLogger,
"opentelemetry": OpenTelemetry,
"azure_sentinel": AzureSentinelLogger,
"azure_storage": AzureBlobStorageLogger,
"humanloop": HumanloopLogger,
# OTEL compatible loggers

12
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -621,6 +621,18 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915
or "https://integrate.api.nvidia.com/v1"
) # type: ignore
dynamic_api_key = api_key or get_secret_str("NVIDIA_NIM_API_KEY")
elif custom_llm_provider == "nvidia_riva":
# NVIDIA Riva is gRPC-based; api_base must be a host:port like
# `grpc.nvcf.nvidia.com:443` or `localhost:50051`. There is no
# public-default endpoint, so we do not fill one in here.
api_base = api_base or get_secret_str("NVIDIA_RIVA_API_BASE") # type: ignore
# Fall back to NVIDIA_NIM_API_KEY because users running both NVCF
# services typically reuse the same nvapi-* key.
dynamic_api_key = (
api_key
or get_secret_str("NVIDIA_RIVA_API_KEY")
or get_secret_str("NVIDIA_NIM_API_KEY")
)
elif custom_llm_provider == "cerebras":
api_base = (
api_base or get_secret("CEREBRAS_API_BASE") or "https://api.cerebras.ai/v1"

2
litellm/litellm_core_utils/litellm_logging.py
View file

@ -1212,7 +1212,7 @@ class Logging(LiteLLMLoggingBaseClass):
# Log the exact result from the LLM API, for streaming - log the type of response received
litellm.error_logs["POST_CALL"] = locals()
if isinstance(original_response, dict):
original_response = json.dumps(original_response)
original_response = json.dumps(original_response, default=str)
try:
self.model_call_details["input"] = input
self.model_call_details["api_key"] = api_key

60
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -436,12 +436,21 @@ def update_messages_with_model_file_ids(
"""
Updates messages with model file ids.
For managed files (unified file IDs), uses model_file_id_mapping if it
resolves the id, otherwise decodes the base64-encoded unified file ID
and extracts the llm_output_file_id directly. Mirrors the Responses-API
sibling `update_responses_input_with_model_file_ids`.
model_file_id_mapping: Dict[str, Dict[str, str]] = {
"litellm_proxy/file_id": {
"model_id": "provider_file_id"
}
}
"""
from litellm.proxy.openai_files_endpoints.common_utils import (
_is_base64_encoded_unified_file_id,
convert_b64_uid_to_unified_uid,
)
for message in messages:
if message.get("role") == "user":
@ -450,7 +459,13 @@ def update_messages_with_model_file_ids(
if isinstance(content, str):
continue
for c in content:
if c["type"] == "file":
if not isinstance(c, dict):
# Content list items aren't always dicts. e.g.
# text_completion forwards a token-ids list/list-of-
# lists through this path. Skip non-dict items
# instead of indexing into them.
continue
if c.get("type") == "file":
file_object = cast(ChatCompletionFileObject, c)
file_object_file_field = file_object.get("file")
if not isinstance(file_object_file_field, dict):
@ -468,9 +483,23 @@ def update_messages_with_model_file_ids(
if file_id:
provider_file_id = (
model_file_id_mapping.get(file_id, {}).get(model_id)
or file_id
if model_file_id_mapping
else None
)
if (
not provider_file_id
and _is_base64_encoded_unified_file_id(file_id)
):
unified_file_id = convert_b64_uid_to_unified_uid(
file_id
)
if "llm_output_file_id," in unified_file_id:
provider_file_id = unified_file_id.split(
"llm_output_file_id,"
)[1].split(";")[0]
file_object_file_field["file_id"] = (
provider_file_id or file_id
)
file_object_file_field["file_id"] = provider_file_id
if format:
file_object_file_field["format"] = format
return messages
@ -726,14 +755,25 @@ def extract_file_data(file_data: FileTypes) -> ExtractedFileData:
else:
file_content = file_data
# Convert content to bytes
if isinstance(file_content, (str, PathLike)):
# If it's a path, open and read the file
# Extract filename from path if not already set
if isinstance(file_content, str):
# Bare string inputs are rejected: when this helper runs in a proxy
# request handler the string came from an attacker-controlled form
# field, and opening it as a path is an arbitrary file read on the
# proxy host. SDK callers who want to upload from a path should
# either pass a pathlib.Path (a PathLike instance — see the branch
# below) or open the file themselves and pass the handle / bytes.
raise ValueError(
"extract_file_data does not accept bare str inputs. Pass bytes, "
"an open file handle, a (filename, content) tuple, or a "
"pathlib.Path. To upload a local file from a path, call "
"open(path, 'rb') yourself."
)
if isinstance(file_content, PathLike):
# PathLike (pathlib.Path) is a Python-level type that HTTP form
# values can't fabricate. Treat as a local file path for SDK
# convenience.
if filename is None:
if isinstance(file_content, PathLike):
filename = Path(file_content).name
else:
filename = Path(str(file_content)).name
filename = Path(file_content).name
with open(file_content, "rb") as f:
content = f.read()
elif isinstance(file_content, io.IOBase):

74
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -2124,27 +2124,62 @@ def anthropic_process_openai_file_message(
)
_EMPTY_TEXT_PLACEHOLDER = (
"[System: Empty message content sanitised to satisfy protocol]"
)
def _sanitize_empty_text_content(
message: AllMessageValues,
) -> AllMessageValues:
"""
Case C: Sanitize empty text content
- Replace empty or whitespace-only text content with a placeholder message.
- Handles both string content and list-of-blocks content (rewriting only
the empty text blocks in place; non-text blocks like images are left
untouched).
Returns:
The message with sanitized content if needed, otherwise the original message
"""
if message.get("role") in ["user", "assistant"]:
content = message.get("content")
if isinstance(content, str):
if not content or not content.strip():
message = cast(AllMessageValues, dict(message)) # Make a copy
message["content"] = (
"[System: Empty message content sanitised to satisfy protocol]"
)
verbose_logger.debug(
f"_sanitize_empty_text_content: Replaced empty text content in {message.get('role')} message"
)
if message.get("role") not in ["user", "assistant"]:
return message
content = message.get("content")
if isinstance(content, str):
if not content or not content.strip():
message = cast(AllMessageValues, dict(message)) # Make a copy
message["content"] = _EMPTY_TEXT_PLACEHOLDER
verbose_logger.debug(
f"_sanitize_empty_text_content: Replaced empty text content in {message.get('role')} message"
)
return message
if isinstance(content, list):
# Walk the blocks and rewrite any empty text blocks. We rewrite (rather
# than drop) so callers don't end up with an entirely empty content
# list, which Anthropic also rejects.
new_blocks: List[Any] = []
rewrote_any = False
for block in content:
if isinstance(block, dict) and block.get("type") == "text":
text = block.get("text")
if not isinstance(text, str) or not text or not text.strip():
new_block = dict(block)
new_block["text"] = _EMPTY_TEXT_PLACEHOLDER
new_blocks.append(new_block)
rewrote_any = True
continue
new_blocks.append(block)
if rewrote_any:
message = cast(AllMessageValues, dict(message)) # Make a copy
message["content"] = new_blocks # type: ignore
verbose_logger.debug(
f"_sanitize_empty_text_content: Replaced empty text block(s) in {message.get('role')} message"
)
return message
@ -2427,6 +2462,18 @@ def anthropic_messages_pt( # noqa: PLR0915
# Sanitize messages for tool calling issues when modify_params=True
messages = sanitize_messages_for_tool_calling(messages)
# Anthropic rejects empty text content blocks with:
# "messages: text content blocks must be non-empty"
# OpenAI/other providers silently tolerate `{"role": "user", "content": ""}`,
# so callers (and upstream agent frameworks like pydantic-ai) routinely
# send empty user/assistant turns. We always rewrite these to a placeholder
# for Anthropic-shaped requests, independent of `litellm.modify_params`,
# because there is no way to "pass through" an empty text block — the
# request will always 400 otherwise. The richer tool-call sanitization
# (Cases A/B/D in `sanitize_messages_for_tool_calling`) remains gated on
# `modify_params` because it actually mutates conversation structure.
messages = [_sanitize_empty_text_content(m) for m in messages]
# add role=tool support to allow function call result/error submission
user_message_types = {"user", "tool", "function"}
# reformat messages to ensure user/assistant are alternating, if there's either 2 consecutive 'user' messages or 2 consecutive 'assistant' message, merge them.
@ -4930,8 +4977,9 @@ class BedrockConverseMessagesProcessor:
)
if reasoning_text and not reasoning_text.get("signature"):
reasoning_text_text = reasoning_text["text"]
assistants_part = BedrockContentBlock(text=reasoning_text_text)
assistant_parts.append(assistants_part)
if reasoning_text_text.strip():
assistants_part = BedrockContentBlock(text=reasoning_text_text)
assistant_parts.append(assistants_part)
else:
filtered_thinking_blocks.append(block)
if len(filtered_thinking_blocks) > 0:

308
litellm/litellm_core_utils/realtime_streaming.py
View file

@ -31,6 +31,8 @@ DefaultLoggedRealTimeEventTypes = [
"session.created",
"response.create",
"response.done",
"conversation.item.added", # GA
"conversation.item.done", # GA
]
@ -44,6 +46,7 @@ class RealTimeStreaming:
model: str = "",
user_api_key_dict: Optional[Any] = None,
request_data: Optional[Dict] = None,
backend_uses_beta_protocol: Optional[bool] = None,
):
self.websocket = websocket
self.backend_ws = backend_ws
@ -54,6 +57,14 @@ class RealTimeStreaming:
self.session_tools: List[Dict] = []
self.tool_calls: List[Dict] = []
# Detect whether the client is explicitly opting into the beta protocol.
self._client_wants_beta = self._detect_beta_header(websocket)
self._backend_uses_beta_protocol = (
self._client_wants_beta
if backend_uses_beta_protocol is None
else backend_uses_beta_protocol
)
_logged_real_time_event_types = litellm.logged_real_time_event_types
if _logged_real_time_event_types is None:
@ -76,6 +87,27 @@ class RealTimeStreaming:
# response.create can be rewritten to include the failure context.
self._pending_guardrail_message: Optional[str] = None
_SESSION_EVENT_TYPES = frozenset(["session.created", "session.updated"])
_AUDIO_FORMAT_MAP: Dict[str, Dict[str, Any]] = {
"pcm16": {"type": "audio/pcm", "rate": 24000},
"g711_ulaw": {"type": "audio/G711-ulaw", "rate": 8000},
"g711_alaw": {"type": "audio/G711-alaw", "rate": 8000},
}
# GA name → beta name (when client WebSocket includes OpenAI-Beta: realtime=v1)
_GA_TO_BETA_EVENT_TYPES: Dict[str, str] = {
"conversation.item.added": "conversation.item.created",
"response.output_text.delta": "response.text.delta",
"response.output_audio.delta": "response.audio.delta",
"response.output_audio_transcript.delta": "response.audio_transcript.delta",
"response.output_text.done": "response.text.done",
"response.output_audio.done": "response.audio.done",
"response.output_audio_transcript.done": "response.audio_transcript.done",
}
_GA_TO_BETA_CONTENT_TYPES: Dict[str, str] = {
"output_text": "text",
"output_audio": "audio",
}
def _should_store_message(
self,
message_obj: Union[dict, OpenAIRealtimeEvents],
@ -92,24 +124,27 @@ class RealTimeStreaming:
if isinstance(message, bytes):
message = message.decode("utf-8")
if isinstance(message, dict):
message_obj = message
# TypedDict union members do not narrow to plain dict for mypy.
message_obj: Dict[str, Any] = cast(Dict[str, Any], message)
else:
message_obj = json.loads(message)
message_obj = cast(Dict[str, Any], json.loads(cast(str, message)))
self._collect_tool_calls_from_response_done(cast(dict, message_obj))
try:
if (
not isinstance(message, dict)
or message_obj.get("type") == "session.created"
or message_obj.get("type") == "session.updated"
):
message_obj = OpenAIRealtimeStreamSessionEvents(**message_obj) # type: ignore
elif not isinstance(message, dict):
message_obj = OpenAIRealtimeStreamResponseBaseObject(**message_obj) # type: ignore
event_type = message_obj.get("type", "")
if event_type in self._SESSION_EVENT_TYPES:
typed_obj = OpenAIRealtimeStreamSessionEvents(**message_obj) # type: ignore
else:
# Use the base object as a safe catch-all for all other event types
# (both beta and GA), so unknown/new event names never raise here.
typed_obj = OpenAIRealtimeStreamResponseBaseObject(**message_obj) # type: ignore
except Exception as e:
verbose_logger.debug(f"Error parsing message for logging: {e}")
raise e
if self._should_store_message(message_obj):
self.messages.append(message_obj)
# Don't re-raise — a parse failure must not drop or delay the message
if self._should_store_message(message_obj):
self.messages.append(message_obj) # type: ignore[arg-type]
return
if self._should_store_message(typed_obj):
self.messages.append(typed_obj)
def _collect_user_input_from_client_event(self, message: Union[str, dict]) -> None:
"""Extract user text content from client WebSocket events for spend logging."""
@ -147,6 +182,8 @@ class RealTimeStreaming:
tools = session.get("tools")
if tools and isinstance(tools, list):
self.session_tools = tools
# GA: session.type is required; log it for traceability but no action needed
verbose_logger.debug(f"Realtime session.type: {session.get('type')}")
except (json.JSONDecodeError, AttributeError, TypeError):
pass
@ -228,6 +265,23 @@ class RealTimeStreaming:
else:
await self.backend_ws.send(message) # type: ignore[union-attr, attr-defined]
def _make_disable_auto_response_message(self) -> str:
"""Return a session.update that disables VAD auto-response."""
if self._backend_uses_beta_protocol:
session: Dict[str, Any] = {
"turn_detection": {"create_response": False},
}
else:
session = {
"type": "realtime",
"audio": {
"input": {
"turn_detection": {"create_response": False},
}
},
}
return json.dumps({"type": "session.update", "session": session})
def _has_realtime_guardrails(self) -> bool:
"""Return True if any callback is registered for realtime guardrail event types."""
from litellm.integrations.custom_guardrail import CustomGuardrail
@ -435,14 +489,7 @@ class RealTimeStreaming:
):
self.store_message(event_str)
await self.websocket.send_text(event_str)
await self._send_to_backend(
json.dumps(
{
"type": "session.update",
"session": {"turn_detection": {"create_response": False}},
}
)
)
await self._send_to_backend(self._make_disable_auto_response_message())
continue
## GUARDRAIL: run on transcription events in provider_config path too
if (
@ -484,14 +531,7 @@ class RealTimeStreaming:
):
self.store_message(raw_response)
await self.websocket.send_text(raw_response)
await self._send_to_backend(
json.dumps(
{
"type": "session.update",
"session": {"turn_detection": {"create_response": False}},
}
)
)
await self._send_to_backend(self._make_disable_auto_response_message())
return True
if (
@ -542,7 +582,20 @@ class RealTimeStreaming:
continue
## LOGGING
self.store_message(raw_response)
await self.websocket.send_text(raw_response)
# If the client opted into beta protocol, translate GA event
# names/shapes back to the beta equivalents before forwarding.
if self._client_wants_beta:
try:
event_dict = json.loads(raw_response)
translated = self._translate_event_to_beta(event_dict)
if translated is None:
continue # drop GA-only events (e.g. conversation.item.done)
await self.websocket.send_text(json.dumps(translated))
except Exception:
await self.websocket.send_text(raw_response)
else:
await self.websocket.send_text(raw_response)
except websockets.exceptions.ConnectionClosed as e: # type: ignore
verbose_logger.exception(
@ -553,6 +606,183 @@ class RealTimeStreaming:
finally:
await self.log_messages()
@staticmethod
def _detect_beta_header(websocket: Any) -> bool:
"""Return True if the client sent 'OpenAI-Beta: realtime=v1'.
Checks the raw ASGI scope headers so it works for both FastAPI WebSocket
objects and any test doubles that expose a .scope dict.
"""
try:
headers = websocket.scope.get("headers", [])
for name, value in headers:
if isinstance(name, bytes):
name = name.decode("latin-1")
if isinstance(value, bytes):
value = value.decode("latin-1")
if name.lower() == "openai-beta" and "realtime=v1" in value.lower():
return True
except Exception:
pass
return False
@staticmethod
def _remap_beta_session_to_ga(session: dict) -> dict:
"""
Convert a beta-style session.update payload to the GA nested schema.
Beta → GA field mappings
─────────────────────────────────────────────────────────────────────
session.type (inject "realtime" if absent)
session.modalities → session.output_modalities
session.voice → session.audio.output.voice
session.input_audio_format → session.audio.input.format (with type/rate)
session.output_audio_format → session.audio.output.format (with type/rate)
session.turn_detection → session.audio.input.turn_detection
session.input_audio_transcription → session.audio.input.transcription
─────────────────────────────────────────────────────────────────────
Fields not in the mapping (instructions, tools, etc.) are passed through.
GA clients that already use the nested shape are unaffected.
"""
# Work on a shallow copy so we don't mutate the caller's dict
session = dict(session)
# 1. Ensure session.type is present
if "type" not in session:
session["type"] = "realtime"
# 2. Rename modalities → output_modalities and normalise combinations.
# Beta allowed ["audio", "text"] together; GA only supports ["audio"] or
# ["text"] as single-element lists. When both are present we prefer
# ["audio"] because audio mode already delivers transcripts via events.
if "modalities" in session:
mods = session.pop("modalities")
if "output_modalities" not in session:
mods_set = {m.lower() for m in (mods or [])}
if "audio" in mods_set:
session["output_modalities"] = ["audio"]
elif "text" in mods_set:
session["output_modalities"] = ["text"]
# 3-7. Lift flat audio fields into the nested audio object
audio: Dict[str, Any] = {}
inp: Dict[str, Any] = {}
out: Dict[str, Any] = {}
# voice → audio.output.voice
if "voice" in session:
out["voice"] = session.pop("voice")
# input_audio_format → audio.input.format
if "input_audio_format" in session:
raw = session.pop("input_audio_format")
inp["format"] = (
RealTimeStreaming._AUDIO_FORMAT_MAP.get(raw, raw)
if isinstance(raw, str)
else raw
)
# output_audio_format → audio.output.format
if "output_audio_format" in session:
raw = session.pop("output_audio_format")
out["format"] = (
RealTimeStreaming._AUDIO_FORMAT_MAP.get(raw, raw)
if isinstance(raw, str)
else raw
)
# turn_detection → audio.input.turn_detection
if "turn_detection" in session:
inp["turn_detection"] = session.pop("turn_detection")
# input_audio_transcription → audio.input.transcription
if "input_audio_transcription" in session:
inp["transcription"] = session.pop("input_audio_transcription")
if inp:
audio["input"] = inp
if out:
audio["output"] = out
if audio:
# Merge with any existing GA-style `audio` block the client already set,
# letting the remapped values take precedence within each sub-key.
existing = session.get("audio") or {}
for sub_key, sub_val in audio.items():
if (
sub_key in existing
and isinstance(existing[sub_key], dict)
and isinstance(sub_val, dict)
):
existing[sub_key] = {**existing[sub_key], **sub_val}
else:
existing[sub_key] = sub_val
session["audio"] = existing
return session
@staticmethod
def _translate_event_to_beta(event: dict) -> Optional[dict]:
"""Translate a single GA event dict to its beta equivalent.
Returns None if the event should be dropped entirely (e.g. the GA-only
conversation.item.done has no beta counterpart).
Returns the (possibly mutated copy of the) event otherwise.
"""
event_type = event.get("type", "")
# conversation.item.done has no beta equivalent — the client already
# received conversation.item.created (translated from .added).
if event_type == "conversation.item.done":
return None
# Shallow-copy so we don't mutate the stored message
translated = dict(event)
# Rename the type field
if event_type in RealTimeStreaming._GA_TO_BETA_EVENT_TYPES:
translated["type"] = RealTimeStreaming._GA_TO_BETA_EVENT_TYPES[event_type]
# Fix content block types inside items (response.done output list,
# conversation.item.created item content, etc.)
if "item" in translated and isinstance(translated["item"], dict):
translated["item"] = RealTimeStreaming._translate_item_content_types(
dict(translated["item"])
)
if "response" in translated and isinstance(translated["response"], dict):
resp = dict(translated["response"])
if "output" in resp and isinstance(resp["output"], list):
resp["output"] = [
(
RealTimeStreaming._translate_item_content_types(dict(o))
if isinstance(o, dict)
else o
)
for o in resp["output"]
]
translated["response"] = resp
return translated
@staticmethod
def _translate_item_content_types(item: dict) -> dict:
"""Replace GA content type names with beta names inside a single item."""
if "content" not in item or not isinstance(item["content"], list):
return item
new_content = []
for block in item["content"]:
if (
isinstance(block, dict)
and block.get("type") in RealTimeStreaming._GA_TO_BETA_CONTENT_TYPES
):
block = dict(block)
block["type"] = RealTimeStreaming._GA_TO_BETA_CONTENT_TYPES[
block["type"]
]
new_content.append(block)
item["content"] = new_content
return item
async def client_ack_messages(self):
try:
while True:
@ -594,6 +824,19 @@ class RealTimeStreaming:
self._pending_guardrail_message = None
continue
# GA compatibility: remap beta-style session fields only when
# the upstream is in GA mode. Beta upstreams expect the flat
# session shape unchanged.
if (
msg_type == "session.update"
and not self._backend_uses_beta_protocol
):
session = msg_obj.get("session", {})
if isinstance(session, dict):
session = self._remap_beta_session_to_ga(session)
msg_obj["session"] = session
message = json.dumps(msg_obj)
except (json.JSONDecodeError, AttributeError):
pass
@ -627,3 +870,8 @@ class RealTimeStreaming:
await forward_task
except asyncio.CancelledError:
pass
def client_sent_openai_beta_realtime_header(websocket: Any) -> bool:
"""True when the client WebSocket includes ``OpenAI-Beta: realtime=v1``."""
return RealTimeStreaming._detect_beta_header(websocket)

12
litellm/llms/anthropic/chat/guardrail_translation/handler.py
View file

@ -23,7 +23,9 @@ from litellm.llms.anthropic.experimental_pass_through.adapters.transformation im
from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
from litellm.llms.base_llm.guardrail_translation.utils import (
effective_skip_system_message_for_guardrail,
effective_skip_tool_message_for_guardrail,
openai_messages_without_system,
openai_messages_without_tool,
)
from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler import (
AnthropicPassthroughLoggingHandler,
@ -108,6 +110,7 @@ class AnthropicMessagesHandler(BaseTranslation):
return data
skip_system = effective_skip_system_message_for_guardrail(guardrail_to_apply)
skip_tool = effective_skip_tool_message_for_guardrail(guardrail_to_apply)
chat_completion_compatible_request = self._translate_to_openai(data)
@ -117,6 +120,8 @@ class AnthropicMessagesHandler(BaseTranslation):
)
if skip_system:
structured_messages = openai_messages_without_system(structured_messages)
if skip_tool:
structured_messages = openai_messages_without_tool(structured_messages)
texts_to_check: List[str] = []
images_to_check: List[str] = []
@ -134,6 +139,7 @@ class AnthropicMessagesHandler(BaseTranslation):
images_to_check=images_to_check,
task_mappings=task_mappings,
skip_system_message=skip_system,
skip_tool_message=skip_tool,
)
# Step 2: Apply guardrail to all texts in batch
@ -198,13 +204,17 @@ class AnthropicMessagesHandler(BaseTranslation):
images_to_check: List[str],
task_mappings: List[Tuple[int, Optional[int]]],
skip_system_message: bool = False,
skip_tool_message: bool = False,
) -> None:
"""
Extract text content and images from a message.
Override this method to customize text/image extraction logic.
"""
if skip_system_message and str(message.get("role") or "").lower() == "system":
role = str(message.get("role") or "").lower()
if skip_system_message and role == "system":
return
if skip_tool_message and role == "tool":
return
content = message.get("content", None)

52
litellm/llms/anthropic/chat/handler.py
View file

@ -65,7 +65,7 @@ from litellm.types.utils import (
from ...base import BaseLLM
from ..common_utils import AnthropicError, process_anthropic_headers
from .transformation import AnthropicConfig
from .transformation import ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY, AnthropicConfig
if TYPE_CHECKING:
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
@ -83,6 +83,7 @@ async def make_call(
timeout: Optional[Union[float, httpx.Timeout]],
json_mode: bool,
speed: Optional[str] = None,
tool_name_reverse_map: Optional[Dict[str, str]] = None,
) -> Tuple[Any, httpx.Headers]:
if client is None:
client = litellm.module_level_aclient
@ -117,6 +118,7 @@ async def make_call(
sync_stream=False,
json_mode=json_mode,
speed=speed,
tool_name_reverse_map=tool_name_reverse_map,
)
# LOGGING
@ -141,6 +143,7 @@ def make_sync_call(
timeout: Optional[Union[float, httpx.Timeout]],
json_mode: bool,
speed: Optional[str] = None,
tool_name_reverse_map: Optional[Dict[str, str]] = None,
) -> Tuple[Any, httpx.Headers]:
if client is None:
client = litellm.module_level_client # re-use a module level client
@ -183,6 +186,7 @@ def make_sync_call(
sync_stream=True,
json_mode=json_mode,
speed=speed,
tool_name_reverse_map=tool_name_reverse_map,
)
# LOGGING
@ -237,6 +241,11 @@ class AnthropicChatCompletion(BaseLLM):
timeout=timeout,
json_mode=json_mode,
speed=optional_params.get("speed") if optional_params else None,
tool_name_reverse_map=(
litellm_params.get(ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY)
if isinstance(litellm_params, dict)
else None
),
)
streamwrapper = CustomStreamWrapper(
completion_stream=completion_stream,
@ -462,6 +471,11 @@ class AnthropicChatCompletion(BaseLLM):
timeout=timeout,
json_mode=json_mode,
speed=optional_params.get("speed") if optional_params else None,
tool_name_reverse_map=(
litellm_params.get(ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY)
if isinstance(litellm_params, dict)
else None
),
)
return CustomStreamWrapper(
completion_stream=completion_stream,
@ -526,6 +540,7 @@ class ModelResponseIterator:
sync_stream: bool,
json_mode: Optional[bool] = False,
speed: Optional[str] = None,
tool_name_reverse_map: Optional[Dict[str, str]] = None,
):
self.streaming_response = streaming_response
self.response_iterator = self.streaming_response
@ -533,6 +548,13 @@ class ModelResponseIterator:
self.tool_index = -1
self.json_mode = json_mode
self.speed = speed
# rewritten-name -> caller's original. Built per-request from the
# forward map in AnthropicConfig._build_request_tool_name_maps; only
# contains entries we actually rewrote, so a tool legitimately named
# `foo_bar` is *not* reverse-mapped just because some other tool was
# rewritten to `foo_bar` in a different request. Empty/None is the
# common case (no '/' or other invalid chars in any tool name).
self.tool_name_reverse_map: Dict[str, str] = tool_name_reverse_map or {}
# Generate response ID once per stream to match OpenAI-compatible behavior
self.response_id = _generate_id()
@ -557,6 +579,10 @@ class ModelResponseIterator:
# Accumulate compaction blocks for multi-turn reconstruction
self.compaction_blocks: List[Dict[str, Any]] = []
# Accumulate streamed thinking text so final usage can split reasoning
# tokens from regular output tokens.
self.reasoning_content_chunks: List[str] = []
# Track server tool use inputs and results for code_interpreter_results
self._server_tool_inputs: Dict[str, Any] = {}
self.tool_results: List[Dict[str, Any]] = []
@ -587,9 +613,14 @@ class ModelResponseIterator:
return False
def _handle_usage(self, anthropic_usage_chunk: Union[dict, UsageDelta]) -> Usage:
reasoning_content = (
"".join(self.reasoning_content_chunks)
if self.reasoning_content_chunks
else None
)
return AnthropicConfig().calculate_usage(
usage_object=cast(dict, anthropic_usage_chunk),
reasoning_content=None,
reasoning_content=reasoning_content,
speed=self.speed,
)
@ -636,10 +667,13 @@ class ModelResponseIterator:
"thinking" in content_block["delta"]
or "signature" in content_block["delta"]
):
thinking_content = content_block["delta"].get("thinking")
if isinstance(thinking_content, str) and thinking_content:
self.reasoning_content_chunks.append(thinking_content)
thinking_blocks = [
ChatCompletionThinkingBlock(
type="thinking",
thinking=content_block["delta"].get("thinking") or "",
thinking=thinking_content or "",
signature=str(content_block["delta"].get("signature") or ""),
)
]
@ -792,6 +826,16 @@ class ModelResponseIterator:
or content_block_start["content_block"]["type"] == "server_tool_use"
):
self.tool_index += 1
# Reverse-map the (sanitized) tool name back to the
# caller's original. No-op when the map is empty.
_stream_tool_name = content_block_start["content_block"]["name"]
if (
self.tool_name_reverse_map
and _stream_tool_name in self.tool_name_reverse_map
):
_stream_tool_name = self.tool_name_reverse_map[
_stream_tool_name
]
# Use empty string for arguments in content_block_start - actual arguments
# come in subsequent content_block_delta chunks and get accumulated.
# Using str(input) here would prepend '{}' causing invalid JSON accumulation.
@ -799,7 +843,7 @@ class ModelResponseIterator:
id=content_block_start["content_block"]["id"],
type="function",
function=ChatCompletionToolCallFunctionChunk(
name=content_block_start["content_block"]["name"],
name=_stream_tool_name,
arguments="",
),
index=self.tool_index,

381
litellm/llms/anthropic/chat/transformation.py
View file

@ -105,6 +105,115 @@ else:
LoggingClass = Any
# Anthropic requires tool names to match ^[a-zA-Z0-9_-]{1,128}$. Any other
# character (commonly '/' or '.' from OpenAPI-derived MCP tools, e.g.
# "actions/download-job-logs-for-workflow-run") must be replaced before
# the request is sent.
#
# A naive "replace [^a-zA-Z0-9_-] with _" is unsafe because it's lossy:
# `foo/bar` and `foo_bar` both collapse to `foo_bar`. Two tools with the
# same sanitized name would either 400 at Anthropic (duplicate) or, worse,
# cause the response side to mis-translate `foo_bar` (a name the caller
# really did register) back to `foo/bar`.
#
# Instead we build a *per-request* forward map (original -> sanitized)
# whose codomain is unique within the request: when two originals collapse
# to the same candidate, or when a sanitized name collides with an already-
# valid name elsewhere in the request, we append numeric suffixes
# (`_2`, `_3`, ...) until the result is free.
#
# The reverse map (sanitized -> original) only contains entries where the
# original was actually rewritten. So a tool whose name is already valid
# round-trips identically and is *never* mistakenly re-mapped on the
# response side.
_ANTHROPIC_TOOL_NAME_INVALID_CHARS = re.compile(r"[^a-zA-Z0-9_-]")
_ANTHROPIC_TOOL_NAME_MAX_LEN = 128
# Single, internal-only key on ``litellm_params`` used to thread the per-
# request reverse map (sanitized -> original) from request build to response
# parsing. ``litellm_params`` is never serialized to a provider; ``optional_
# params`` IS (it becomes the JSON body via ``data = {**optional_params}``).
# Keep these two channels strictly separate -- never stash internal
# coordination state in ``optional_params``.
ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY = "_anthropic_tool_name_map"
def _basic_sanitize_anthropic_tool_name(name: str) -> str:
"""Lossy: replace [^a-zA-Z0-9_-] with '_' and truncate to 128.
Used as a candidate generator for the per-request forward map.
Callers should NOT use this directly for translation -- always go
through the forward map so collisions are resolved.
"""
if not isinstance(name, str) or not name:
return name
return _ANTHROPIC_TOOL_NAME_INVALID_CHARS.sub("_", name)[
:_ANTHROPIC_TOOL_NAME_MAX_LEN
]
def _build_anthropic_tool_name_maps(
original_names: List[str],
) -> Tuple[Dict[str, str], Dict[str, str]]:
"""Build (forward, reverse) tool-name maps for a single request.
forward[original] = sanitized -- only present when name was rewritten
reverse[sanitized] = original -- inverse of `forward`
Properties:
- All sanitized names satisfy ^[a-zA-Z0-9_-]{1,128}$.
- Sanitized names are unique within the request (no two originals
collide on the wire).
- A name that's already valid AND doesn't collide with another tool's
sanitized form passes through untouched and is absent from the maps.
That's the key correctness property: response-side translation only
runs on entries we actually rewrote, so a tool legitimately named
`foo_bar` is never incorrectly retyped to `foo/bar` just because
some *other* request had that pair.
- Order-dependent: when two originals would clash, the *second* one
seen gets the disambiguating suffix. Callers should preserve the
caller's tool order (we do).
"""
forward: Dict[str, str] = {}
used: set = set()
# First pass: reserve slots for names that are already valid so they
# always have priority regardless of input order.
for original in original_names:
if not isinstance(original, str) or not original:
continue
candidate = _basic_sanitize_anthropic_tool_name(original)
if candidate == original:
used.add(candidate)
# Second pass: sanitize/disambiguate names that need rewriting.
for original in original_names:
if not isinstance(original, str) or not original:
continue
candidate = _basic_sanitize_anthropic_tool_name(original)
if candidate == original:
continue
# Skip duplicates of the same original name. Without this guard the
# second pass would assign a fresh suffix and overwrite the forward
# map entry, causing every reference to map to the suffixed name and
# leaving the original sanitized slot orphaned in `used` with no
# reverse mapping.
if original in forward:
continue
# Disambiguate against names already chosen this request.
unique = candidate
n = 1
while unique in used:
n += 1
suffix = f"_{n}"
# Keep within the 128-char cap.
head = candidate[: _ANTHROPIC_TOOL_NAME_MAX_LEN - len(suffix)]
unique = f"{head}{suffix}"
forward[original] = unique
used.add(unique)
reverse = {v: k for k, v in forward.items()}
return forward, reverse
REASONING_EFFORT_TO_OUTPUT_CONFIG_EFFORT: Dict[str, str] = {
"low": "low",
"minimal": "low",
@ -486,7 +595,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
}
def _map_tool_choice(
self, tool_choice: Optional[str], parallel_tool_use: Optional[bool]
self,
tool_choice: Optional[str],
parallel_tool_use: Optional[bool],
) -> Optional[AnthropicMessagesToolChoice]:
_tool_choice: Optional[AnthropicMessagesToolChoice] = None
if tool_choice == "auto":
@ -527,7 +638,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
return _tool_choice
def _map_tool_helper( # noqa: PLR0915
self, tool: ChatCompletionToolParam
self,
tool: ChatCompletionToolParam,
) -> Tuple[Optional[AllAnthropicToolsValues], Optional[AnthropicMcpServerTool]]:
returned_tool: Optional[AllAnthropicToolsValues] = None
mcp_server: Optional[AnthropicMcpServerTool] = None
@ -783,7 +895,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
return initial_tool
def _map_tools(
self, tools: List
self,
tools: List,
) -> Tuple[List[AllAnthropicToolsValues], List[AnthropicMcpServerTool]]:
anthropic_tools = []
mcp_servers = []
@ -799,6 +912,174 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
mcp_servers.append(mcp_server_tool)
return anthropic_tools, mcp_servers
@staticmethod
def _rewrite_tool_names_in_messages(
messages: List[AllMessageValues],
name_forward_map: Dict[str, str],
) -> List[AllMessageValues]:
"""Return a copy of `messages` with tool_call/function_call names
rewritten using the per-request forward map.
Only mutates messages whose tool_call/function_call name is *in* the
forward map. Names absent from the map (already valid, no collision)
round-trip untouched. We only deep-copy the entries we actually
change to keep this O(turns-with-rewritten-tools), not O(history).
"""
if not name_forward_map:
return messages
new_messages: List[AllMessageValues] = []
for msg in messages:
if not isinstance(msg, dict):
new_messages.append(msg)
continue
tool_calls = msg.get("tool_calls")
function_call = msg.get("function_call")
if not tool_calls and not function_call:
new_messages.append(msg)
continue
new_msg = dict(msg)
if isinstance(tool_calls, list):
new_calls = []
for tc in tool_calls:
if not isinstance(tc, dict):
new_calls.append(tc)
continue
fn = tc.get("function")
fn_name = fn.get("name") if isinstance(fn, dict) else None
if (
isinstance(fn, dict)
and isinstance(fn_name, str)
and fn_name in name_forward_map
):
new_fn = dict(fn)
new_fn["name"] = name_forward_map[fn_name]
new_tc = dict(tc)
new_tc["function"] = new_fn
new_calls.append(new_tc)
else:
new_calls.append(tc)
new_msg["tool_calls"] = new_calls
fc_name = (
function_call.get("name") if isinstance(function_call, dict) else None
)
if (
isinstance(function_call, dict)
and isinstance(fc_name, str)
and fc_name in name_forward_map
):
new_fc = dict(function_call)
new_fc["name"] = name_forward_map[fc_name]
new_msg["function_call"] = new_fc
new_messages.append(cast(AllMessageValues, new_msg))
return new_messages
@staticmethod
def _build_request_tool_name_maps(
tools: List,
) -> Tuple[Dict[str, str], Dict[str, str]]:
"""Build the (forward, reverse) tool-name maps for an OpenAI tools list.
Operates on **OpenAI-format** tool dicts (pre-``_map_tools``). The
production sanitization path uses ``_sanitize_tool_names_in_request``
instead, which operates on **Anthropic-format** tools (post-
``_map_tools``, where ``type == "custom"``). This helper exists for
callers that need to compute the maps from the raw OpenAI shape --
e.g. test setup or future pre-mapping consumers.
See _build_anthropic_tool_name_maps for the collision rules. Pulls
the original name out of either ``{"function": {"name": ...}}``
(legacy OpenAI shape) or ``{"name": ...}`` (rare top-level shape).
"""
original_names: List[str] = []
for tool in tools or []:
if not isinstance(tool, dict):
continue
original = (
tool.get("function", {}).get("name")
if isinstance(tool.get("function"), dict)
else None
)
if original is None:
original = tool.get("name")
if isinstance(original, str) and original:
original_names.append(original)
return _build_anthropic_tool_name_maps(original_names)
@staticmethod
def _sanitize_tool_names_in_request(
optional_params: Dict[str, Any],
) -> Tuple[Dict[str, str], Dict[str, str]]:
"""Sanitize ``optional_params['tools']`` and ``optional_params['tool_choice']``
in place so every name matches Anthropic's ``^[a-zA-Z0-9_-]{1,128}$``.
Returns ``(forward, reverse)`` for use by message-history rewriting
and response translation. ``forward[original] = sanitized`` is only
populated for names that were actually rewritten -- i.e. either
contained an invalid character or collided with another tool's
sanitized form. Names already valid AND unique pass through and are
absent from both maps.
Only ``type == "custom"`` tools (the OpenAI function-tool shape) are
considered. Hosted tools (``web_search``, ``bash``, ``code_execution``,
``computer_*``, ``mcp``, ...) own reserved names defined by Anthropic
and must not be touched.
"""
tools = optional_params.get("tools")
if not isinstance(tools, list) or not tools:
return {}, {}
# 1. Collect originals from the Anthropic-shaped custom-tool entries.
# Order matters: the first occurrence wins the canonical slot;
# later collisions get numeric suffixes (see
# ``_build_anthropic_tool_name_maps``).
original_names: List[str] = []
for t in tools:
if not isinstance(t, dict):
continue
if t.get("type") != "custom":
continue
name = t.get("name")
if isinstance(name, str) and name:
original_names.append(name)
if not original_names:
return {}, {}
forward, reverse = _build_anthropic_tool_name_maps(original_names)
if not forward:
# Every name was already valid -- nothing to do.
return forward, reverse
# 2. Apply forward map. Build a new list with copy-on-change entries
# so a caller reusing the same tool list/dicts across requests
# doesn't see its inputs permanently rewritten (which would also
# drop the original key from `forward` on the next request).
new_tools: List[Any] = []
for t in tools:
if (
isinstance(t, dict)
and t.get("type") == "custom"
and isinstance(t.get("name"), str)
and t["name"] in forward
):
new_tools.append({**t, "name": forward[t["name"]]})
else:
new_tools.append(t)
optional_params["tools"] = new_tools
# 3. Same for ``tool_choice`` when it targets a named tool. Copy
# rather than mutate for the same reason as above.
tool_choice = optional_params.get("tool_choice")
if isinstance(tool_choice, dict) and tool_choice.get("type") == "tool":
tc_name = tool_choice.get("name")
if isinstance(tc_name, str) and tc_name in forward:
optional_params["tool_choice"] = {
**tool_choice,
"name": forward[tc_name],
}
return forward, reverse
def _detect_tool_search_tools(self, tools: Optional[List]) -> bool:
"""Check if tool search tools are present in the tools list."""
if not tools:
@ -1125,6 +1406,17 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
non_default_params=non_default_params
)
# NB: ``map_openai_params`` deliberately does NOT sanitize tool names
# here. Names are the *original* OpenAI names at this stage, and must
# remain so until ``transform_request`` -- which is the single
# chokepoint where Anthropic, Bedrock-Anthropic, and Vertex-Anthropic
# all pass through. Doing it there guarantees:
# 1. one source of truth for the per-request forward/reverse maps,
# 2. the maps land on ``litellm_params`` (internal), never on
# ``optional_params`` (which is serialized into the request body
# via ``data = {**optional_params}`` and would 400 with
# ``Extra inputs are not permitted``).
for param, value in non_default_params.items():
if param == "max_tokens":
optional_params["max_tokens"] = (
@ -1135,7 +1427,6 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
value if isinstance(value, int) else max(1, int(round(value)))
)
elif param == "tools":
# check if optional params already has tools
anthropic_tools, mcp_servers = self._map_tools(value)
optional_params = self._add_tools_to_optional_params(
optional_params=optional_params, tools=anthropic_tools
@ -1518,9 +1809,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
Translate messages to anthropic format.
"""
## VALIDATE REQUEST
"""
Anthropic doesn't support tool calling without `tools=` param specified.
"""
"""Anthropic requires ``tools`` when messages include tool blocks; LiteLLM injects a dummy tool if omitted (no ``modify_params`` needed)."""
from litellm.litellm_core_utils.prompt_templates.factory import (
anthropic_messages_pt,
)
@ -1530,16 +1819,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
and messages is not None
and has_tool_call_blocks(messages)
):
if litellm.modify_params:
optional_params["tools"], _ = self._map_tools(
add_dummy_tool(custom_llm_provider="anthropic")
)
else:
raise litellm.UnsupportedParamsError(
message="Anthropic doesn't support tool calling without `tools=` param specified. Pass `tools=` param OR set `litellm.modify_params = True` // `litellm_settings::modify_params: True` to add dummy tool to the request.",
model="",
llm_provider="anthropic",
)
optional_params["tools"], _ = self._map_tools(
add_dummy_tool(custom_llm_provider="anthropic")
)
# Drop thinking param if thinking is enabled but thinking_blocks are missing
# This prevents the error: "Expected thinking or redacted_thinking, but found tool_use"
@ -1565,6 +1847,34 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
headers=headers, optional_params=optional_params
)
# === Tool-name sanitization (single chokepoint) ===
# Anthropic enforces ^[a-zA-Z0-9_-]{1,128}$ on every tool name. We
# sanitize *here* -- not in map_openai_params -- because:
#
# - This function is the single boundary shared by AnthropicConfig,
# AmazonAnthropicConfig (Bedrock invoke), VertexAIAnthropicConfig,
# and AzureAnthropicConfig (all call ``super().transform_request``
# or ``AnthropicConfig.transform_request(self, ...)``). Sanitizing
# once here covers every Anthropic-shaped request.
# - The forward/reverse maps are coordination state; they belong on
# ``litellm_params`` (internal-only), never on ``optional_params``
# (which becomes the JSON body via ``{**optional_params}``).
# - It keeps ``map_openai_params`` a pure param translator with no
# side-channel state.
#
# The reverse map only contains entries for names that were actually
# rewritten -- so a tool legitimately named ``foo_bar`` is never
# incorrectly retyped to ``foo/bar`` on the response side.
# See _build_anthropic_tool_name_maps for the collision-handling
# rules and rationale.
_name_forward_map, _name_reverse_map = self._sanitize_tool_names_in_request(
optional_params=optional_params,
)
if _name_forward_map:
messages = self._rewrite_tool_names_in_messages(messages, _name_forward_map)
if _name_reverse_map and isinstance(litellm_params, dict):
litellm_params[ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY] = _name_reverse_map
# Separate system prompt from rest of message
anthropic_system_message_list = self.translate_system_message(messages=messages)
# Handling anthropic API Prompt Caching
@ -1837,8 +2147,16 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
speed: Optional[str] = None,
) -> Usage:
# NOTE: Sometimes the usage object has None set explicitly for token counts, meaning .get() & key access returns None, and we need to account for this
prompt_tokens = usage_object.get("input_tokens", 0) or 0
completion_tokens = usage_object.get("output_tokens", 0) or 0
raw_prompt_tokens = usage_object.get("input_tokens", 0) or 0
prompt_tokens: int = (
int(raw_prompt_tokens) if isinstance(raw_prompt_tokens, (int, float)) else 0
)
raw_completion_tokens = usage_object.get("output_tokens", 0) or 0
completion_tokens: int = (
int(raw_completion_tokens)
if isinstance(raw_completion_tokens, (int, float))
else 0
)
_usage = usage_object
cache_creation_input_tokens: int = 0
cache_read_input_tokens: int = 0
@ -1907,11 +2225,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
text_tokens=raw_input_tokens,
)
# Always populate completion_token_details, not just when there's reasoning_content
reasoning_tokens = (
estimated_reasoning_tokens = (
token_counter(text=reasoning_content, count_response_tokens=True)
if reasoning_content
else 0
)
reasoning_tokens = min(estimated_reasoning_tokens, completion_tokens)
completion_token_details = CompletionTokensDetailsWrapper(
reasoning_tokens=reasoning_tokens if reasoning_tokens > 0 else 0,
text_tokens=(
@ -2041,6 +2360,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
json_mode: Optional[bool] = None,
prefix_prompt: Optional[str] = None,
speed: Optional[str] = None,
tool_name_reverse_map: Optional[Dict[str, str]] = None,
):
_hidden_params: Dict = {}
_hidden_params["additional_headers"] = process_anthropic_headers(
@ -2065,6 +2385,21 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
compaction_blocks,
) = self.extract_response_content(completion_response=completion_response)
# Reverse-map rewritten tool names back to caller's originals so a
# downstream OpenAI-style dispatcher can match on the registered name.
# See _build_anthropic_tool_name_maps for why this is keyed on the
# per-request reverse map (so a tool legitimately named `foo_bar` is
# never incorrectly retyped to `foo/bar`). No-op when the map is
# empty (the common case).
if tool_name_reverse_map and tool_calls:
for tc in tool_calls:
fn = tc.get("function") if isinstance(tc, dict) else None
if fn is None:
continue
_name = fn.get("name")
if isinstance(_name, str) and _name in tool_name_reverse_map:
fn["name"] = tool_name_reverse_map[_name]
if (
prefix_prompt is not None
and not text_content.startswith(prefix_prompt)
@ -2191,6 +2526,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
prefix_prompt = self.get_prefix_prompt(messages=messages)
speed = optional_params.get("speed")
tool_name_reverse_map: Optional[Dict[str, str]] = None
if isinstance(litellm_params, dict):
_candidate = litellm_params.get(ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY)
if isinstance(_candidate, dict):
tool_name_reverse_map = _candidate
model_response = self.transform_parsed_response(
completion_response=completion_response,
@ -2199,6 +2539,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
json_mode=json_mode,
prefix_prompt=prefix_prompt,
speed=speed,
tool_name_reverse_map=tool_name_reverse_map,
)
return model_response

43
litellm/llms/anthropic/common_utils.py
View file

@ -832,6 +832,49 @@ def strip_thinking_blocks_from_anthropic_messages_request_dict(
data.pop("thinking", None)
def strip_empty_text_blocks_from_anthropic_messages(
messages: List[Any],
) -> List[Any]:
"""
Return a new message list with empty or whitespace-only ``{"type": "text"}``
content blocks removed.
Anthropic's API rejects requests containing such blocks with
``"messages: text content blocks must be non-empty"``, but assistant
messages from Anthropic routinely arrive with ``{"type": "text", "text": ""}``
alongside ``tool_use`` blocks (see anthropics/anthropic-sdk-python#461).
Multi-turn tool-use clients (e.g. Claude Code) loop these prior responses
back as conversation history, which then causes the next request to 400
on the unified ``/v1/messages`` path. ``/v1/chat/completions`` already
handles this in ``anthropic_messages_pt``; this helper provides the
equivalent guarantee for the native Anthropic Messages path.
Messages whose content is a list and becomes empty after stripping are
omitted, matching :func:`strip_thinking_blocks_from_anthropic_messages`.
The caller's list and its content blocks are never mutated; modified
messages are returned as shallow copies with a fresh content list.
"""
out: List[Any] = []
for m in messages:
if not isinstance(m, dict) or not isinstance(m.get("content"), list):
out.append(m)
continue
content = m["content"]
filtered = [b for b in content if not _is_empty_text_block(b)]
if len(filtered) == len(content):
out.append(m)
elif filtered:
out.append({**m, "content": filtered})
return out
def _is_empty_text_block(block: Any) -> bool:
if not isinstance(block, dict) or block.get("type") != "text":
return False
text = block.get("text")
return not isinstance(text, str) or not text.strip()
def process_anthropic_headers(headers: Union[httpx.Headers, dict]) -> dict:
openai_headers = {}
if "anthropic-ratelimit-requests-limit" in headers:

11
litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py
View file

@ -1299,9 +1299,18 @@ class LiteLLMAnthropicMessagesAdapter:
else truncated_name
)
# Strip Gemini thought-signature suffix from id (mirrors streaming
# path below); base64 chars (+ / =) violate Anthropic's
# `^[a-zA-Z0-9_-]+$` tool_use.id pattern when replayed.
raw_id = tool_call.id or ""
base_id = (
raw_id.split(THOUGHT_SIGNATURE_SEPARATOR, 1)[0]
if THOUGHT_SIGNATURE_SEPARATOR in raw_id
else raw_id
)
tool_use_block = AnthropicResponseContentBlockToolUse(
type="tool_use",
id=tool_call.id,
id=base_id,
name=original_name,
input=parse_tool_call_arguments(
tool_call.function.arguments,

22
litellm/llms/anthropic/experimental_pass_through/messages/handler.py
View file

@ -12,6 +12,9 @@ from typing import Any, AsyncIterator, Coroutine, Dict, List, Optional, Union, c
import litellm
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.anthropic.common_utils import (
strip_empty_text_blocks_from_anthropic_messages,
)
from litellm.llms.base_llm.anthropic_messages.transformation import (
BaseAnthropicMessagesConfig,
)
@ -188,8 +191,20 @@ async def anthropic_messages(
**kwargs,
) -> Union[AnthropicMessagesResponse, AsyncIterator]:
"""
Async: Make llm api request in Anthropic /messages API spec
Async: Make llm api request in Anthropic /messages API spec.
Runs the empty-text-block sanitizer before any backend dispatch.
"""
# Anthropic's API rejects requests containing empty / whitespace-only
# text content blocks with "messages: text content blocks must be
# non-empty". Multi-turn tool-use clients (e.g. Claude Code) routinely
# loop assistant responses that contain {"type": "text", "text": ""}
# alongside tool_use blocks back as conversation history, which then
# causes the next /v1/messages call to 400. /v1/chat/completions
# already handles this in anthropic_messages_pt; sanitize the native
# Anthropic Messages path here for the same guarantee. See #22930.
messages = strip_empty_text_blocks_from_anthropic_messages(messages)
original_stream = stream or kwargs.get(
"_websearch_interception_converted_stream", False
)
@ -336,6 +351,11 @@ def anthropic_messages_handler(
"""
from litellm.types.utils import LlmProviders
# Sanitize empty text blocks here too so the sync entry point
# (litellm.messages.create -> anthropic_messages_handler) gets the same
# protection as the async wrapper. Idempotent when called twice.
messages = strip_empty_text_blocks_from_anthropic_messages(messages)
metadata = validate_anthropic_api_metadata(metadata)
local_vars = locals()

48
litellm/llms/azure/image_edit/transformation.py
View file

@ -3,8 +3,10 @@ from typing import Optional, cast
import httpx
import litellm
from litellm.llms.azure.common_utils import BaseAzureLLM
from litellm.llms.openai.image_edit.transformation import OpenAIImageEditConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.router import GenericLiteLLMParams
from litellm.utils import _add_path_to_api_base
@ -30,20 +32,42 @@ class AzureImageEditConfig(OpenAIImageEditConfig):
litellm_params: Optional[dict] = None,
api_base: Optional[str] = None,
) -> dict:
api_key = (
api_key
or litellm.api_key
or litellm.azure_key
or get_secret_str("AZURE_OPENAI_API_KEY")
or get_secret_str("AZURE_API_KEY")
)
"""
Validate Azure environment and set up authentication headers.
headers.update(
{
"Authorization": f"Bearer {api_key}",
}
Delegates to ``BaseAzureLLM._base_validate_azure_environment`` so the
Azure image-edit route uses the same auth resolution as every other
Azure provider (videos, vector_stores, responses, containers, ...):
- prefers the Azure-style ``api-key`` header when an API key is available
- falls back to ``Authorization: Bearer <azure_ad_token>`` only when AAD
auth is configured
The previous implementation unconditionally set
``Authorization: Bearer <api_key>``, which is correct for OpenAI direct
but not for Azure OpenAI / API Management gateways that expect the
``api-key`` header. Subscription-key-based deployments (e.g., behind
Azure APIM) responded with ``401 "Access denied due to missing
subscription key"``.
API-key precedence (matches ``AzureVideosConfig``):
- ``litellm_params["api_key"]`` is the source of truth.
- The positional ``api_key`` kwarg only fills in when
``litellm_params["api_key"]`` is empty.
- This is a deliberate change from the old ``or`` chain (where the
positional ``api_key`` argument won) so behavior matches every other
Azure ``validate_environment`` implementation. In production the only
caller (``llm_http_handler.image_edit``) sources both values from
the same ``litellm_params.api_key``, so the precedence only matters
for direct callers of this method.
"""
params = GenericLiteLLMParams(**(litellm_params or {}))
if api_key is not None and params.api_key is None:
params.api_key = api_key
return BaseAzureLLM._base_validate_azure_environment(
headers=headers, litellm_params=params
)
return headers
def get_complete_url(
self,

6
litellm/llms/azure/realtime/handler.py
View file

@ -89,9 +89,10 @@ class AzureOpenAIRealtime(AzureChatCompletion):
if api_base is None:
raise ValueError("api_base is required for Azure OpenAI calls")
if api_version is None and (
backend_uses_beta_protocol = (
realtime_protocol is None or realtime_protocol.upper() not in ("GA", "V1")
):
)
if api_version is None and backend_uses_beta_protocol:
raise ValueError("api_version is required for Azure OpenAI calls")
url = self._construct_url(
@ -114,6 +115,7 @@ class AzureOpenAIRealtime(AzureChatCompletion):
logging_obj,
user_api_key_dict=user_api_key_dict,
request_data={"litellm_metadata": litellm_metadata or {}},
backend_uses_beta_protocol=backend_uses_beta_protocol,
)
await realtime_streaming.bidirectional_forward()

15
litellm/llms/base_llm/guardrail_translation/utils.py
View file

@ -14,7 +14,22 @@ def effective_skip_system_message_for_guardrail(guardrail_to_apply: Any) -> bool
return bool(getattr(litellm, "skip_system_message_in_guardrail", False))
def effective_skip_tool_message_for_guardrail(guardrail_to_apply: Any) -> bool:
per = getattr(guardrail_to_apply, "skip_tool_message_in_guardrail", None)
if per is not None:
return bool(per)
import litellm
return bool(getattr(litellm, "skip_tool_message_in_guardrail", False))
def openai_messages_without_system(
messages: List[AllMessageValues],
) -> List[AllMessageValues]:
return [m for m in messages if str((m or {}).get("role") or "").lower() != "system"]
def openai_messages_without_tool(
messages: List[AllMessageValues],
) -> List[AllMessageValues]:
return [m for m in messages if str((m or {}).get("role") or "").lower() != "tool"]

213
litellm/llms/bedrock/base_aws_llm.py
View file

@ -7,6 +7,8 @@ from datetime import datetime
from typing import (
TYPE_CHECKING,
Any,
Callable,
ClassVar,
Dict,
List,
Literal,
@ -63,8 +65,16 @@ class AwsAuthError(Exception):
class BaseAWSLLM:
# Process-wide IAM credential cache (shared across instances — Bedrock passthrough is per-request).
# Storage is in-process memory only: default ``DualCache()`` has no Redis backend unless attached
# elsewhere. Entry TTL: static access-key + secret + region use ``_get_default_ttl_for_boto3_credentials``
# (~59 minutes); ambient env (``_auth_with_env_vars`` returns ``ttl=None``) uses ``InMemoryCache``'s
# ``default_ttl`` (600 seconds / 10 minutes). AssumeRole, web identity, profiles, and explicit
# session-token tuples are not cached — see ``get_credentials`` and ``_get_or_set_cached_credentials``.
_shared_iam_cache: ClassVar[DualCache] = DualCache()
def __init__(self) -> None:
self.iam_cache = DualCache()
self.iam_cache = BaseAWSLLM._shared_iam_cache
super().__init__()
self.aws_authentication_params = [
"aws_access_key_id",
@ -103,6 +113,79 @@ class BaseAWSLLM:
credential_str = json.dumps(credential_args, sort_keys=True)
return hashlib.sha256(credential_str.encode()).hexdigest()
def _get_or_set_cached_credentials(
self,
credential_args: Dict[str, Optional[str]],
credential_fetcher: Callable[[], Tuple[Any, Optional[int]]],
) -> Any:
"""
Read-through IAM cache on the process-wide ``DualCache``.
Only the in-memory layer is used by default (no Redis on ``_shared_iam_cache`` unless
configured globally). TTL on write: static access-key fetches pass
``_get_default_ttl_for_boto3_credentials()`` (~59 minutes); ambient env passes ``ttl=None``,
which ``InMemoryCache.set_cache`` resolves to ``default_ttl`` (600 seconds / 10 minutes by
default).
Used only for static access-key credentials and ambient credentials from
``_auth_with_env_vars`` (including when skipping AssumeRole because the runtime identity
already matches ``aws_role_name``).
AssumeRole, web identity exchange, profiles, and explicit session-token tuples are not
cached here — shared ``Credentials`` / refresh state must not span logical sessions.
"""
cache_key = self.get_cache_key(credential_args)
_cached = self.iam_cache.get_cache(cache_key)
if _cached:
return _cached
credentials, ttl = credential_fetcher()
self.iam_cache.set_cache(cache_key, credentials, ttl=ttl)
return credentials
@staticmethod
def _is_auth_with_web_identity_token(
aws_web_identity_token: Optional[str],
aws_role_name: Optional[str],
aws_session_name: Optional[str],
) -> bool:
return (
aws_web_identity_token is not None
and aws_role_name is not None
and aws_session_name is not None
)
@staticmethod
def _is_auth_with_aws_role(aws_role_name: Optional[str]) -> bool:
return aws_role_name is not None
@staticmethod
def _is_auth_with_aws_profile(aws_profile_name: Optional[str]) -> bool:
return aws_profile_name is not None
@staticmethod
def _is_auth_with_aws_session_token_tuple(
aws_access_key_id: Optional[str],
aws_secret_access_key: Optional[str],
aws_session_token: Optional[str],
) -> bool:
return (
aws_access_key_id is not None
and aws_secret_access_key is not None
and aws_session_token is not None
)
@staticmethod
def _is_auth_with_access_key_and_secret_key(
aws_access_key_id: Optional[str],
aws_secret_access_key: Optional[str],
aws_region_name: Optional[str],
) -> bool:
return (
aws_access_key_id is not None
and aws_secret_access_key is not None
and aws_region_name is not None
)
@tracer.wrap()
def get_credentials(
self,
@ -184,95 +267,97 @@ class BaseAWSLLM:
aws_external_id,
)
# create cache key for non-expiring auth flows
args = {
k: v
for k, v in locals().items()
if k.startswith("aws_") or k == "ssl_verify"
}
cache_key = self.get_cache_key(args)
_cached_credentials = self.iam_cache.get_cache(cache_key)
if _cached_credentials:
return _cached_credentials
#########################################################
# Handle diff boto3 auth flows
# for each helper
# Return:
# Credentials - boto3.Credentials
# cache ttl - Optional[int]. If None, the credentials are not cached. Some auth flows have no expiry time.
#
# iam_cache: static keys and ambient env only (including skip-AssumeRole path).
# Do not cache AssumeRole / web identity / profile / explicit session-token paths here.
#########################################################
if (
aws_web_identity_token is not None
and aws_role_name is not None
and aws_session_name is not None
if self._is_auth_with_web_identity_token(
aws_web_identity_token,
aws_role_name,
aws_session_name,
):
credentials, _cache_ttl = self._auth_with_web_identity_token(
aws_web_identity_token=aws_web_identity_token,
aws_role_name=aws_role_name,
aws_session_name=aws_session_name,
aws_web_identity_token=cast(str, aws_web_identity_token),
aws_role_name=cast(str, aws_role_name),
aws_session_name=cast(str, aws_session_name),
aws_region_name=aws_region_name,
aws_sts_endpoint=aws_sts_endpoint,
aws_external_id=aws_external_id,
)
elif aws_role_name is not None:
# Check if we're already running as the target role and can skip assumption
# This handles IRSA (EKS), ECS task roles, and EC2 instance profiles
if self._is_already_running_as_role(aws_role_name, ssl_verify=ssl_verify):
return credentials
elif self._is_auth_with_aws_role(aws_role_name):
# Same role (IRSA/ECS/EC2): ambient creds via _get_or_set_cached_credentials like the
# default env branch; never pre-read cache (must run _is_already_running_as_role first).
if self._is_already_running_as_role(
cast(str, aws_role_name), ssl_verify=ssl_verify
):
verbose_logger.debug(
"Already running as target role %s, using ambient credentials",
aws_role_name,
)
credentials, _cache_ttl = self._auth_with_env_vars()
else:
verbose_logger.debug(
"Using role assumption: calling _auth_with_aws_role"
return self._get_or_set_cached_credentials(
args, self._auth_with_env_vars
)
# If aws_session_name is not provided, generate a default one
if aws_session_name is None:
aws_session_name = (
f"litellm-session-{int(datetime.now().timestamp())}"
)
credentials, _cache_ttl = self._auth_with_aws_role(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_session_token=aws_session_token,
aws_role_name=aws_role_name,
aws_session_name=aws_session_name,
aws_region_name=aws_region_name,
aws_sts_endpoint=aws_sts_endpoint,
aws_external_id=aws_external_id,
ssl_verify=ssl_verify,
)
elif aws_profile_name is not None: ### CHECK SESSION ###
credentials, _cache_ttl = self._auth_with_aws_profile(aws_profile_name)
elif (
aws_access_key_id is not None
and aws_secret_access_key is not None
and aws_session_token is not None
):
credentials, _cache_ttl = self._auth_with_aws_session_token(
verbose_logger.debug("Using role assumption: calling _auth_with_aws_role")
# If aws_session_name is not provided, generate a default one
if aws_session_name is None:
aws_session_name = f"litellm-session-{int(datetime.now().timestamp())}"
credentials, _assume_ttl = self._auth_with_aws_role(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_session_token=aws_session_token,
)
elif (
aws_access_key_id is not None
and aws_secret_access_key is not None
and aws_region_name is not None
):
credentials, _cache_ttl = self._auth_with_access_key_and_secret_key(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_role_name=cast(str, aws_role_name),
aws_session_name=aws_session_name,
aws_region_name=aws_region_name,
aws_sts_endpoint=aws_sts_endpoint,
aws_external_id=aws_external_id,
ssl_verify=ssl_verify,
)
return credentials
elif self._is_auth_with_aws_profile(aws_profile_name):
credentials, _cache_ttl = self._auth_with_aws_profile(
cast(str, aws_profile_name)
)
return credentials
elif self._is_auth_with_aws_session_token_tuple(
aws_access_key_id,
aws_secret_access_key,
aws_session_token,
):
credentials, _cache_ttl = self._auth_with_aws_session_token(
aws_access_key_id=cast(str, aws_access_key_id),
aws_secret_access_key=cast(str, aws_secret_access_key),
aws_session_token=cast(str, aws_session_token),
)
return credentials
elif self._is_auth_with_access_key_and_secret_key(
aws_access_key_id,
aws_secret_access_key,
aws_region_name,
):
return self._get_or_set_cached_credentials(
args,
lambda: self._auth_with_access_key_and_secret_key(
aws_access_key_id=cast(str, aws_access_key_id),
aws_secret_access_key=cast(str, aws_secret_access_key),
aws_region_name=cast(str, aws_region_name),
),
)
else:
credentials, _cache_ttl = self._auth_with_env_vars()
self.iam_cache.set_cache(cache_key, credentials, ttl=_cache_ttl)
return credentials
return self._get_or_set_cached_credentials(args, self._auth_with_env_vars)
def _get_aws_region_from_model_arn(self, model: Optional[str]) -> Optional[str]:
try:
@ -1343,7 +1428,13 @@ class BaseAWSLLM:
def _sign_request(
self,
service_name: Literal["bedrock", "sagemaker", "bedrock-agentcore", "s3vectors"],
service_name: Literal[
"bedrock",
"sagemaker",
"bedrock-agentcore",
"s3vectors",
"aws-external-anthropic",
],
headers: dict,
optional_params: dict,
request_data: dict,

241
litellm/llms/bedrock/batches/handler.py
View file

@ -1,8 +1,79 @@
from datetime import datetime
from typing import Any, Optional, cast
from openai.types.batch import BatchRequestCounts
from openai.types.batch import Metadata as OpenAIBatchMetadata
from litellm.types.utils import LiteLLMBatch
# AWS Bedrock model-invocation-job statuses → OpenAI Batch statuses.
# Mirrors the mapping used by `BedrockBatchesConfig.transform_create_batch_response`
# so create / retrieve return consistent statuses.
_BEDROCK_MIJ_STATUS_TO_OPENAI = {
"Submitted": "validating",
"Validating": "validating",
"Scheduled": "validating",
"InProgress": "in_progress",
"Stopping": "cancelling",
"Stopped": "cancelled",
"Completed": "completed",
"PartiallyCompleted": "completed",
"Failed": "failed",
"Expired": "expired",
}
def _extract_region_from_bedrock_arn(arn: str) -> Optional[str]:
"""ARN shape: ``arn:aws:bedrock:<region>:<account>:<type>/<id>``"""
try:
parts = arn.split(":")
if len(parts) >= 4 and parts[2] == "bedrock":
return parts[3] or None
except Exception:
pass
return None
def _extract_job_id_from_arn(arn: str) -> Optional[str]:
"""``arn:aws:bedrock:<region>:<acct>:model-invocation-job/<job-id>`` -> ``<job-id>``."""
if ":model-invocation-job/" not in arn:
return None
return arn.rsplit("/", 1)[-1] or None
def _predict_output_file_uri(
output_prefix: str, input_uri: str, job_id: Optional[str]
) -> Optional[str]:
"""
Compute the deterministic per-job result file URI Bedrock writes to.
Bedrock lays results out as::
<output_prefix>/<job-id>/<basename(input_uri)>.out
We compute it client-side so OpenAI-style ``client.files.content(output_file_id)``
works without an extra S3 ``ListObjectsV2`` round-trip. Returns ``None`` if we
don't have enough info; callers should fall back to the bare prefix.
"""
if not output_prefix or not input_uri or not job_id:
return None
if not output_prefix.endswith("/"):
output_prefix = output_prefix + "/"
input_basename = input_uri.rsplit("/", 1)[-1]
if not input_basename:
return None
return f"{output_prefix}{job_id}/{input_basename}.out"
def _to_epoch(value: Any) -> Optional[int]:
if value is None:
return None
if isinstance(value, (int, float)):
return int(value)
if isinstance(value, datetime):
return int(value.timestamp())
return None
class BedrockBatchesHandler:
"""
@ -97,3 +168,173 @@ class BedrockBatchesHandler:
with concurrent.futures.ThreadPoolExecutor() as executor:
future = executor.submit(run_in_thread)
return future.result()
@staticmethod
def _handle_model_invocation_job_status(
batch_id: str,
aws_region_name: Optional[str] = None,
logging_obj=None,
**kwargs,
) -> "LiteLLMBatch":
"""
Handle ``GetModelInvocationJob`` status check for AWS Bedrock bulk batch
inference jobs (the ARN type returned by ``CreateModelInvocationJob``).
``CreateModelInvocationJob`` lives on the Bedrock **control plane**
(``bedrock.<region>.amazonaws.com``), distinct from the data-plane
``bedrock-runtime`` endpoint that serves Twelve Labs async-invoke ARNs.
The two ARN families therefore can't share a handler — see
``litellm/batches/main.py`` for the dispatch.
Args:
batch_id: A ``arn:aws:bedrock:<region>:<acct>:model-invocation-job/<id>``
ARN (or just the trailing job id; both are accepted by
``GetModelInvocationJob``).
aws_region_name: Region for the boto3 ``bedrock`` client. If omitted,
we fall back to parsing the region out of ``batch_id`` itself.
logging_obj: Optional litellm logging object.
**kwargs: Optional AWS credential overrides
(``aws_access_key_id``, ``aws_secret_access_key``,
``aws_session_token``, ``aws_profile_name``,
``aws_role_name``, ``aws_session_name``,
``aws_web_identity_token``, ``aws_sts_endpoint``,
``aws_external_id``). Unknown keys are ignored.
Returns:
``LiteLLMBatch`` shaped like an OpenAI Batch resource. Note that
``request_counts`` is always ``(0, 0, 0)`` because
``GetModelInvocationJob`` does not surface per-record counts;
callers that need accurate counts should parse
``manifest.json.out`` from the output S3 prefix.
"""
try:
import boto3
except ImportError as exc:
raise ImportError(
"Missing boto3 to call bedrock. Run 'pip install boto3'."
) from exc
# Resolve region: explicit > parsed-from-ARN > us-east-1 (boto3 default).
region = (
aws_region_name or _extract_region_from_bedrock_arn(batch_id) or "us-east-1"
)
# Resolve credentials through the same path the rest of the bedrock
# provider uses, so model_list / env / role-assumption configs are
# honored. We instantiate BedrockBatchesConfig (which extends
# BaseAWSLLM) lazily to avoid a circular import at module load.
from litellm.llms.bedrock.batches.transformation import BedrockBatchesConfig
creds = BedrockBatchesConfig().get_credentials(
aws_access_key_id=kwargs.get("aws_access_key_id"),
aws_secret_access_key=kwargs.get("aws_secret_access_key"),
aws_session_token=kwargs.get("aws_session_token"),
aws_region_name=region,
aws_session_name=kwargs.get("aws_session_name"),
aws_profile_name=kwargs.get("aws_profile_name"),
aws_role_name=kwargs.get("aws_role_name"),
aws_web_identity_token=kwargs.get("aws_web_identity_token"),
aws_sts_endpoint=kwargs.get("aws_sts_endpoint"),
aws_external_id=kwargs.get("aws_external_id"),
)
client = boto3.client(
"bedrock",
region_name=region,
aws_access_key_id=creds.access_key,
aws_secret_access_key=creds.secret_key,
aws_session_token=creds.token,
)
if logging_obj is not None:
# Use the bare job id in the logged URL so we don't double up the
# `model-invocation-job/` segment when `batch_id` is a full ARN.
# `GetModelInvocationJob` accepts either form, but only the bare id
# produces a sensible-looking URL in logs.
url_path_id = _extract_job_id_from_arn(batch_id) or batch_id
logging_obj.pre_call(
input=batch_id,
api_key="",
additional_args={
"complete_input_dict": {"jobIdentifier": batch_id},
"api_base": (
f"https://bedrock.{region}.amazonaws.com/"
f"model-invocation-job/{url_path_id}"
),
},
)
response = client.get_model_invocation_job(jobIdentifier=batch_id)
if logging_obj is not None:
logging_obj.post_call(
input=batch_id,
api_key="",
original_response=response,
additional_args={"complete_input_dict": {"jobIdentifier": batch_id}},
)
bedrock_status = str(response.get("status", ""))
openai_status = cast(
Any,
_BEDROCK_MIJ_STATUS_TO_OPENAI.get(bedrock_status, "in_progress"),
)
input_uri = (
response.get("inputDataConfig", {})
.get("s3InputDataConfig", {})
.get("s3Uri", "")
)
output_prefix = (
response.get("outputDataConfig", {})
.get("s3OutputDataConfig", {})
.get("s3Uri", "")
)
# Bedrock returns the output *prefix* the user supplied at job creation.
# Actual results land at <prefix>/<job-id>/<basename(input)>.out — we
# surface that single-file URI as `output_file_id` so the OpenAI-style
# download flow works without an extra S3 listing call. We deliberately
# do NOT fall back to the bare prefix when prediction fails: a prefix
# is not a downloadable object, so handing it back as `output_file_id`
# would reproduce the very NoSuchKey bug this handler exists to fix.
# The bare prefix is preserved in metadata for callers that want the
# `manifest.json.out` or want to do their own listing.
job_arn = response.get("jobArn", batch_id)
job_id = _extract_job_id_from_arn(job_arn)
output_file_uri = _predict_output_file_uri(output_prefix, input_uri, job_id)
completed_at = _to_epoch(response.get("endTime"))
# Note: metadata uses "" (not None) for unknown URIs to satisfy the
# OpenAI Batch metadata schema, which is `dict[str, str]`. The
# `output_file_id` field on the LiteLLMBatch itself does carry None
# correctly (see below), so callers should branch on that, not on
# `metadata["output_file_uri"]`.
openai_batch_metadata: OpenAIBatchMetadata = {
"model_arn": response.get("modelId", ""),
"job_arn": job_arn,
"job_name": response.get("jobName", ""),
"failure_message": response.get("message") or "",
"input_s3_uri": input_uri,
"output_s3_uri": output_prefix,
"output_file_uri": output_file_uri or "",
}
return LiteLLMBatch(
id=job_arn,
object="batch",
status=openai_status,
created_at=_to_epoch(response.get("submitTime")) or 0,
in_progress_at=_to_epoch(response.get("lastModifiedTime")),
completed_at=completed_at if openai_status == "completed" else None,
failed_at=completed_at if openai_status == "failed" else None,
cancelled_at=completed_at if openai_status == "cancelled" else None,
expired_at=completed_at if openai_status == "expired" else None,
request_counts=BatchRequestCounts(total=0, completed=0, failed=0),
metadata=openai_batch_metadata,
completion_window="24h",
endpoint="/v1/chat/completions",
input_file_id=input_uri,
output_file_id=output_file_uri if openai_status == "completed" else None,
)

Some files were not shown because too many files have changed in this diff Show more