scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-04 02:31:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 69

Merge pull request #27650 from BerriAI/litellm_internal_staging

Browse source 
merge main
This commit is contained in:
Sameer Kankute 2026-05-11 20:42:22 +05:30 • committed by GitHub
commit ea432f2042
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
597 changed files with 24241 additions and 6854 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

341
.circleci/config.yml
View file

@ -350,7 +350,15 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -v tests/local_testing -x --junitxml=test-results/junit.xml --durations=5 -k "langfuse"
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--junitxml=test-results/junit.xml \
--durations=5 \
-k \"langfuse\""
no_output_timeout: 15m
# Store test results
- store_test_results:
@ -395,7 +403,15 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -v tests/proxy_admin_ui_tests -x --junitxml=test-results/junit.xml --durations=5 -n 2
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/proxy_admin_ui_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 2"
no_output_timeout: 15m
# Store test results
@ -471,7 +487,15 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -v tests/router_unit_tests -x --junitxml=test-results/junit.xml --durations=5 -n 4
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/router_unit_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
no_output_timeout: 15m
# Store test results
- store_test_results:
@ -495,7 +519,15 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest tests/local_testing/ -v -k "assistants" -x --junitxml=test-results/junit.xml --durations=5
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--junitxml=test-results/junit.xml \
--durations=5 \
-k \"assistants\""
no_output_timeout: 15m
# Store test results
- store_test_results:
@ -528,14 +560,19 @@ jobs:
# Add --timeout to kill hanging tests after 120s (2 min)
# Add --durations=20 to show 20 slowest tests for debugging
# Subdirectories with dedicated jobs (maintain this list as new jobs are added)
IGNORE_DIRS=(
"tests/llm_translation/realtime"
)
IGNORE_ARGS=""
for dir in "${IGNORE_DIRS[@]}"; do
IGNORE_ARGS="$IGNORE_ARGS --ignore=$dir"
done
uv run --no-sync python -m pytest -v tests/llm_translation $IGNORE_ARGS --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread --retries 2 --retry-delay 5 --max-worker-restart=5
mkdir -p test-results
# Glob excludes the realtime/ subdirectory since it has its own job
TEST_FILES=$(circleci tests glob "tests/llm_translation/**/test_*.py" | grep -v "^tests/llm_translation/realtime/")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v \
--junitxml=test-results/junit.xml \
--durations=20 \
-n 4 \
--timeout=120 --timeout_method=thread \
--retries 2 --retry-delay 5 \
--max-worker-restart=5"
no_output_timeout: 15m
# Store test results
@ -560,7 +597,17 @@ jobs:
command: |
# Add --timeout to kill hanging tests after 120s (2 min)
# Add --durations=20 to show 20 slowest tests for debugging
uv run --no-sync python -m pytest -vv tests/llm_translation/realtime --cov=litellm --cov-report=xml -v --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/llm_translation/realtime/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=20 \
-n 4 \
--timeout=120 --timeout_method=thread"
no_output_timeout: 15m
- run:
name: Rename the coverage files
@ -593,7 +640,15 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -vv tests/agent_tests --ignore=tests/agent_tests/local_only_agent_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/agent_tests/**/test_*.py" | grep -v "^tests/agent_tests/local_only_agent_tests/")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
- run:
name: Rename the coverage files
@ -626,7 +681,18 @@ jobs:
- run:
name: Run tests
command: |
LITELLM_LOG=WARNING uv run --no-sync python -m pytest tests/guardrails_tests -vv --cov=litellm --cov-report=xml --junitxml=test-results/junit.xml --durations=5 -n 2 --timeout=120 --timeout_method=thread
mkdir -p test-results
export LITELLM_LOG=WARNING
TEST_FILES=$(circleci tests glob "tests/guardrails_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 2 \
--timeout=120 --timeout_method=thread"
no_output_timeout: 15m
- run:
name: Rename the coverage files
@ -660,7 +726,16 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -vv tests/unified_google_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 --retries 3 --retry-delay 5
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/unified_google_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
--retries 3 --retry-delay 5"
no_output_timeout: 15m
- run:
name: Rename the coverage files
@ -702,7 +777,15 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -v tests/llm_responses_api_testing -x --junitxml=test-results/junit.xml --durations=5 -n 8
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/llm_responses_api_testing/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 8"
no_output_timeout: 15m
# Store test results
@ -725,7 +808,16 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -vv tests/ocr_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/ocr_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--cov=litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
no_output_timeout: 15m
- run:
name: Rename the coverage files
@ -758,7 +850,16 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -vv tests/search_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/search_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--cov=litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
no_output_timeout: 15m
- run:
name: Rename the coverage files
@ -793,7 +894,15 @@ jobs:
name: Run enterprise tests
command: |
uv run --no-sync python -m prisma generate
uv run --no-sync python -m pytest -v tests/enterprise -x --junitxml=test-results/junit-enterprise.xml --durations=10 -n 4
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/enterprise/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--junitxml=test-results/junit-enterprise.xml \
--durations=10 \
-n 4"
no_output_timeout: 15m
# Store test results
- store_test_results:
@ -815,7 +924,16 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -vv tests/batches_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 -n 2
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/batches_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 2"
no_output_timeout: 15m
- run:
name: Rename the coverage files
@ -848,7 +966,16 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -vv tests/litellm_utils_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 -n 2
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/litellm_utils_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 2"
no_output_timeout: 15m
- run:
name: Rename the coverage files
@ -882,7 +1009,16 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -vv tests/pass_through_unit_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/pass_through_unit_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--cov=litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
no_output_timeout: 15m
- run:
name: Rename the coverage files
@ -916,7 +1052,15 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -v tests/image_gen_tests -n 4 -x --junitxml=test-results/junit.xml --durations=5
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/image_gen_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
no_output_timeout: 15m
# Store test results
- store_test_results:
@ -939,7 +1083,18 @@ jobs:
- run:
name: Run tests
command: |
LITELLM_LOG=WARNING uv run --no-sync python -m pytest tests/logging_callback_tests -vv --cov=litellm --cov-report=xml -n 4 --junitxml=test-results/junit.xml --durations=5 --timeout=120 --timeout_method=thread
mkdir -p test-results
export LITELLM_LOG=WARNING
TEST_FILES=$(circleci tests glob "tests/logging_callback_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=litellm --cov-report=xml \
-n 4 \
--junitxml=test-results/junit.xml \
--durations=5 \
--timeout=120 --timeout_method=thread"
no_output_timeout: 15m
- run:
name: Rename the coverage files
@ -972,7 +1127,15 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -vv tests/audio_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/audio_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
- run:
name: Rename the coverage files
@ -1012,14 +1175,19 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -vv \
mkdir -p test-results
TEST_FILES=$(printf "%s\n" \
tests/local_testing/test_dual_cache.py \
tests/local_testing/test_redis_batch_optimizations.py \
tests/local_testing/test_router_utils.py \
--cov=litellm --cov-report=xml \
-x -s -v --junitxml=test-results/junit.xml \
--durations=5 -n 2 \
--reruns 2 --reruns-delay 1
tests/local_testing/test_router_utils.py)
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 -n 2 \
--reruns 2 --reruns-delay 1"
no_output_timeout: 20m
- run:
name: Rename the coverage files
@ -1260,8 +1428,17 @@ jobs:
- run:
name: Run Basic Proxy Startup Tests (Health Readiness and Chat Completion)
command: |
uv run --no-sync python -m pytest -v tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--junitxml=test-results/junit-2.xml \
--durations=5"
no_output_timeout: 15m
- store_test_results:
path: test-results
build_and_test:
machine:
@ -1331,7 +1508,18 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -s -v tests/*.py -x --junitxml=test-results/junit.xml -n 4 --durations=5 --ignore=tests/otel_tests --ignore=tests/spend_tracking_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation --ignore=tests/llm_responses_api_testing --ignore=tests/mcp_tests --ignore=tests/guardrails_tests --ignore=tests/image_gen_tests --ignore=tests/pass_through_unit_tests
mkdir -p test-results
# Original used `tests/*.py` (top-level only); the `--ignore=...`
# flags were vestigial since shell globbing did not descend into
# subdirectories. Replicate by globbing only top-level test files.
TEST_FILES=$(circleci tests glob "tests/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-s -v -x \
--junitxml=test-results/junit.xml \
-n 4 \
--durations=5"
no_output_timeout: 15m
# Store test results
@ -1406,7 +1594,14 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -s -vv tests/openai_endpoints_tests --junitxml=test-results/junit.xml --durations=5
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/openai_endpoints_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-s -vv \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
# Store test results
@ -1475,7 +1670,14 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -v tests/otel_tests --junitxml=test-results/junit.xml --durations=5
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/otel_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
# Clean up first container
- run:
@ -1518,7 +1720,14 @@ jobs:
- run:
name: Run second round of tests
command: |
uv run --no-sync python -m pytest -v tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--junitxml=test-results/junit-2.xml \
--durations=5"
no_output_timeout: 15m
# Store test results
@ -1587,8 +1796,17 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -vv tests/spend_tracking_tests -x --junitxml=test-results/junit.xml --durations=5
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/spend_tracking_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
- store_test_results:
path: test-results
- run:
name: Stop and remove first container
when: always
@ -1676,7 +1894,14 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -vv tests/multi_instance_e2e_tests -x --junitxml=test-results/junit.xml --durations=5
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/multi_instance_e2e_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
# Clean up first container
# Store test results
@ -1732,7 +1957,14 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -vv tests/store_model_in_db_tests -x --junitxml=test-results/junit.xml --durations=5
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/store_model_in_db_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
- run:
name: Stop and remove containers
@ -1805,9 +2037,18 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -vv tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--junitxml=test-results/junit-2.xml \
--durations=5"
no_output_timeout: 15m
# Clean up first container
- store_test_results:
path: test-results
- run:
name: Stop and remove first container
command: |
@ -1940,7 +2181,14 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -v tests/pass_through_tests/ -x --junitxml=test-results/junit.xml --durations=5
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/pass_through_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
# Store test results
@ -1997,9 +2245,16 @@ jobs:
- run:
name: Run Claude Agent SDK E2E Tests
command: |
mkdir -p test-results
export LITELLM_PROXY_URL="http://localhost:4000"
export LITELLM_API_KEY="sk-1234"
uv run --no-sync python -m pytest -vv tests/proxy_e2e_anthropic_messages_tests/ -x -s --junitxml=test-results/junit.xml --durations=5
TEST_FILES=$(circleci tests glob "tests/proxy_e2e_anthropic_messages_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
# Store test results

BIN
.evidence/main_header_repro.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 60 KiB

1
.github/workflows/test-unit-proxy-db.yml vendored
View file

@ -141,6 +141,7 @@ jobs:
tests/proxy_unit_tests/test_server_root_path.py
tests/proxy_unit_tests/test_proxy_pass_user_config.py
tests/proxy_unit_tests/test_proxy_token_counter.py
tests/proxy_unit_tests/test_request_size_limit_middleware.py
workers: 4
dist: loadscope
timeout: 15

3
.gitignore vendored
View file

@ -100,4 +100,5 @@ STABILIZATION_TODO.md
**/playwright-report
**/*.storageState.json
**/coverage
test-config
test-config
.vscode

21
AGENTS.md
View file

@ -241,10 +241,27 @@ When opening issues or pull requests, follow these templates:
### Running the proxy server
Start the proxy with a config file:
Create a minimal config file and start the proxy:
```yaml
# config.yaml
model_list:
- model_name: fake-openai-endpoint
litellm_params:
model: openai/fake-model
api_key: fake-key
api_base: https://fake-api.example.com
general_settings:
master_key: sk-1234
litellm_settings:
drop_params: True
telemetry: False
```
```bash
uv run litellm --config dev_config.yaml --port 4000
uv run litellm --config config.yaml --port 4000
```
The proxy takes ~15-20 seconds to fully start (it runs Prisma migrations on boot). Wait for `/health` to return before sending requests. Without a PostgreSQL `DATABASE_URL`, the proxy connects to a default Neon dev database embedded in the `litellm-proxy-extras` package.

2
CLAUDE.md
View file

@ -146,7 +146,7 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components:
- **Bound large result sets.** Prisma materializes full results in memory. For results over ~10 MB, paginate with `take`/`skip` or `cursor`/`take`, always with an explicit `order`. Prefer cursor-based pagination (`skip` is O(n)). Don't paginate naturally small result sets.
- **Limit fetched columns on wide tables.** Use `select` to fetch only needed fields — returns a partial object, so downstream code must not access unselected fields.
- **Check index coverage.** For new or modified queries, check `schema.prisma` for a supporting index. Prefer extending an existing index (e.g. `@@index([a])` → `@@index([a, b])`) over adding a new one, unless it's a `@@unique`. Only add indexes for large/frequent queries.
- **Keep schema files in sync.** Apply schema changes to all `schema.prisma` copies (`schema.prisma`, `litellm/proxy/`, `litellm-proxy-extras/`, `litellm-js/spend-logs/` for SpendLogs) with a migration under `litellm-proxy-extras/litellm_proxy_extras/migrations/`.
- **Keep schema files in sync.** Apply schema changes to all `schema.prisma` copies (`schema.prisma`, `litellm/proxy/`, `litellm-proxy-extras/`) with a migration under `litellm-proxy-extras/litellm_proxy_extras/migrations/`.
### Setup Wizard (`litellm/setup_wizard.py`)
- The wizard is implemented as a single `SetupWizard` class with `@staticmethod` methods — keep it that way. No module-level functions except `run_setup_wizard()` (the public entrypoint) and pure helpers (color, ANSI).

20
Dockerfile
View file

@ -1,8 +1,8 @@
# Base image for building
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:3258be472764337fd13095bcbb3182da170243b5819fd67ad4c0754590588b31
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:3258be472764337fd13095bcbb3182da170243b5819fd67ad4c0754590588b31
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin
@ -68,8 +68,8 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
USER root
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 libsndfile supervisor && \
npm install -g npm@11.12.1 tar@7.5.11 glob@13.0.6 @isaacs/brace-expansion@5.0.1 brace-expansion@5.0.5 minimatch@10.2.4 diff@8.0.3 picomatch@4.0.4 && \
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 libsndfile && \
npm install -g npm@11.14.0 tar@7.5.11 glob@13.0.6 @isaacs/brace-expansion@5.0.1 brace-expansion@5.0.5 minimatch@10.2.4 diff@8.0.3 picomatch@4.0.4 && \
GLOBAL="$(npm root -g)" && \
for pkg in tar glob @isaacs/brace-expansion brace-expansion minimatch diff picomatch; do \
name="${pkg##*/}"; \
@ -85,17 +85,17 @@ ENV PATH="/app/.venv/bin:${PATH}"
COPY --from=builder /app /app
# Prisma binaries live in $HOME/.cache (default prisma-python location),
# which is /root/.cache here. Copy them from the builder so they survive
# deployments that volume-mount /app/.cache (e.g. readOnlyRootFilesystem
# + emptyDir) — otherwise the mount would shadow the baked-in query engine.
COPY --from=builder /root/.cache /root/.cache
# which is /root/.cache here. Copy only the Prisma subdirs — copying the
# whole /root/.cache drags in the uv build cache (~660 MB, includes a
# setuptools wheel that surfaces as a CVE finding even though it's not
# on the runtime sys.path).
COPY --from=builder /root/.cache/prisma /root/.cache/prisma
COPY --from=builder /root/.cache/prisma-python /root/.cache/prisma-python
RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
find /app/.venv -type d -path "*/tornado/test" -delete
EXPOSE 4000/tcp
COPY docker/supervisord.conf /etc/supervisord.conf
ENTRYPOINT ["docker/prod_entrypoint.sh"]
CMD ["--port", "4000"]

7
codecov.yaml
View file

@ -1,3 +1,8 @@
codecov:
require_ci_to_pass: false # post coverage status even if CI has unrelated failures
notify:
wait_for_ci: false # post as soon as expected uploads arrive, don't wait on CI
component_management:
individual_components:
- component_id: "Router"
@ -28,7 +33,7 @@ coverage:
project:
default:
target: auto
threshold: 1% # at maximum allow project coverage to drop by 1%
threshold: 0% # do not allow project coverage to drop
patch:
default:
target: auto

18
deploy/Dockerfile.ghcr_base
View file

@ -1,18 +0,0 @@
# Use the provided base image
FROM ghcr.io/berriai/litellm:main-latest@sha256:7c311546c25e7bb6e8cafede9fcd3d0d622ac636b5c9418befaa32e85dfb0186
# Set the working directory to /app
WORKDIR /app
# Copy the configuration file into the container at /app
COPY config.yaml .
# Make sure your docker/entrypoint.sh is executable
# Convert Windows line endings to Unix
RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
# Expose the necessary port
EXPOSE 4000/tcp
# Override the CMD instruction with your desired command and arguments
CMD ["--port", "4000", "--config", "config.yaml", "--detailed_debug", "--run_gunicorn"]

44
deploy/charts/litellm-helm/templates/deployment.yaml
View file

@ -100,6 +100,16 @@ spec:
- name: DATABASE_URL
value: {{ .Values.db.url | quote }}
{{- end }}
{{- if and .Values.db.useExisting .Values.db.secret.readReplicaUrlKey }}
- name: DATABASE_URL_READ_REPLICA
valueFrom:
secretKeyRef:
name: {{ .Values.db.secret.name }}
key: {{ .Values.db.secret.readReplicaUrlKey }}
{{- else if .Values.db.readReplicaUrl }}
- name: DATABASE_URL_READ_REPLICA
value: {{ .Values.db.readReplicaUrl | quote }}
{{- end }}
- name: PROXY_MASTER_KEY
valueFrom:
secretKeyRef:
@ -116,21 +126,32 @@ spec:
name: {{ include "redis.secretName" .Subcharts.redis }}
key: {{include "redis.secretPasswordKey" .Subcharts.redis }}
{{- end }}
{{- /*
Inject LITELLM_LOG only when envVars does not already define it.
*/}}
{{- if and .Values.logLevel (not (hasKey (default dict .Values.envVars) "LITELLM_LOG")) }}
- name: LITELLM_LOG
value: {{ .Values.logLevel | quote }}
{{- end }}
{{- if .Values.envVars }}
{{- range $key, $val := .Values.envVars }}
- name: {{ $key }}
value: {{ $val | quote }}
{{- end }}
{{- end }}
{{- if .Values.separateHealthApp }}
- name: SEPARATE_HEALTH_APP
value: "1"
- name: SEPARATE_HEALTH_PORT
value: {{ .Values.separateHealthPort | default "8081" | quote }}
{{- end }}
{{- with .Values.extraEnvVars }}
{{- toYaml . | nindent 12 }}
{{- end }}
{{- if .Values.migrationJob.enabled }}
# Schema updates are owned by the dedicated migrations Job; skip
# the proxy's startup `prisma db push` so N replicas don't race
# one DB on every rollout. Placed last (after envVars and
# extraEnvVars) so this override can't be silently shadowed by a
# user-supplied DISABLE_SCHEMA_UPDATE under last-wins duplicate-env
# semantics — same pattern the migrations Job uses.
- name: DISABLE_SCHEMA_UPDATE
value: "true"
{{- end }}
envFrom:
{{- range .Values.environmentSecrets }}
- secretRef:
@ -158,15 +179,10 @@ spec:
- name: http
containerPort: {{ .Values.service.port }}
protocol: TCP
{{- if .Values.separateHealthApp }}
- name: health
containerPort: {{ .Values.separateHealthPort | default 8081 }}
protocol: TCP
{{- end }}
livenessProbe:
httpGet:
path: {{ .Values.livenessProbe.path | quote }}
port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
port: "http"
initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.livenessProbe.periodSeconds }}
timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }}
@ -175,7 +191,7 @@ spec:
readinessProbe:
httpGet:
path: {{ .Values.readinessProbe.path | quote }}
port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
port: "http"
initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }}
@ -184,7 +200,7 @@ spec:
startupProbe:
httpGet:
path: {{ .Values.startupProbe.path | quote }}
port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
port: "http"
initialDelaySeconds: {{ .Values.startupProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.startupProbe.periodSeconds }}
timeoutSeconds: {{ .Values.startupProbe.timeoutSeconds }}

10
deploy/charts/litellm-helm/tests/deployment_tests.yaml
View file

@ -257,16 +257,16 @@ tests:
value: 0
- equal:
path: spec.template.spec.containers[0].livenessProbe.periodSeconds
value: 10
value: 15
- equal:
path: spec.template.spec.containers[0].livenessProbe.timeoutSeconds
value: 1
value: 5
- equal:
path: spec.template.spec.containers[0].livenessProbe.successThreshold
value: 1
- equal:
path: spec.template.spec.containers[0].livenessProbe.failureThreshold
value: 3
value: 5
- equal:
path: spec.template.spec.containers[0].readinessProbe.httpGet.path
value: /health/readiness
@ -278,7 +278,7 @@ tests:
value: 10
- equal:
path: spec.template.spec.containers[0].readinessProbe.timeoutSeconds
value: 1
value: 5
- equal:
path: spec.template.spec.containers[0].readinessProbe.successThreshold
value: 1
@ -296,7 +296,7 @@ tests:
value: 10
- equal:
path: spec.template.spec.containers[0].startupProbe.timeoutSeconds
value: 1
value: 5
- equal:
path: spec.template.spec.containers[0].startupProbe.successThreshold
value: 1

49
deploy/charts/litellm-helm/values.yaml
View file

@ -88,26 +88,20 @@ service:
# optionally specify loadBalancerClass
# loadBalancerClass: tailscale
# Separate health app configuration
# When enabled, health checks will use a separate port and the application
# will receive SEPARATE_HEALTH_APP=1 and SEPARATE_HEALTH_PORT from environment variables
separateHealthApp: false
separateHealthPort: 8081
# Probe tuning for proxy container
# Probes for LiteLLM gateway container
livenessProbe:
path: /health/liveliness
initialDelaySeconds: 0
periodSeconds: 10
timeoutSeconds: 1
periodSeconds: 15
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 3
failureThreshold: 5
readinessProbe:
path: /health/readiness
initialDelaySeconds: 0
periodSeconds: 10
timeoutSeconds: 1
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 3
@ -115,7 +109,7 @@ startupProbe:
path: /health/readiness
initialDelaySeconds: 0
periodSeconds: 10
timeoutSeconds: 1
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 30
@ -258,6 +252,26 @@ db:
passwordKey: password
# Optional: when set, DATABASE_HOST will be sourced from this secret key instead of db.endpoint
endpointKey: ""
# Optional: when set, DATABASE_URL_READ_REPLICA will be sourced from this
# secret key instead of db.readReplicaUrl. Prefer this over the plain
# value: read-replica URLs typically embed credentials, and a value
# written to db.readReplicaUrl ends up visible in the rendered pod spec
# and the Helm release secret.
readReplicaUrlKey: ""
# Optional read-replica routing. When set, the proxy sends read-only
# queries (find_*, count, group_by, query_raw/_first) to this URL while
# writes continue to go to db.url. Useful for Aurora-style clusters with
# separate reader/writer endpoints. Leave empty to keep single-DB behavior.
# When IAM_TOKEN_DB_AUTH is enabled, the reader URL is auto-refreshed
# alongside the writer (host/port/user/db are parsed from this URL once
# at startup; only the IAM token rotates).
#
# If the URL embeds credentials, prefer db.secret.readReplicaUrlKey over
# this field — the plain value is rendered into the pod spec and the
# Helm release secret. This field is intended for credential-less URLs
# only (e.g. when IAM_TOKEN_DB_AUTH supplies the token at runtime).
readReplicaUrl: ""
# Use the Stackgres Helm chart to deploy an instance of a Stackgres cluster.
# The Stackgres Operator must already be installed within the target
@ -329,6 +343,17 @@ migrationJob:
helm:
enabled: false
# Log level for the litellm proxy (sets LITELLM_LOG in the deployment env).
# Rendered as a direct `env:` entry, which in Kubernetes takes precedence over
# any `envFrom:` source. If you currently source LITELLM_LOG from an
# environmentSecret or environmentConfigMap, set `logLevel: ""` here to
# disable injection — otherwise this value silently overrides your secret /
# configmap entry.
#
# Setting LITELLM_LOG inside `envVars:` below also wins: the template skips
# this injection entirely when envVars already defines LITELLM_LOG.
logLevel: INFO
# Additional environment variables to be added to the deployment as a map of key-value pairs
envVars: {}

56
deploy/kubernetes/kub.yaml
View file

@ -1,56 +0,0 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: litellm-deployment
spec:
replicas: 3
selector:
matchLabels:
app: litellm
template:
metadata:
labels:
app: litellm
spec:
containers:
- name: litellm-container
image: ghcr.io/berriai/litellm:main-latest
imagePullPolicy: Always
env:
- name: AZURE_API_KEY
value: "d6f****"
- name: AZURE_API_BASE
value: "https://openai"
- name: LITELLM_MASTER_KEY
value: "sk-1234"
- name: DATABASE_URL
value: "postgresql://ishaan*********"
args:
- "--config"
- "/app/proxy_config.yaml" # Update the path to mount the config file
volumeMounts: # Define volume mount for proxy_config.yaml
- name: config-volume
mountPath: /app
readOnly: true
livenessProbe:
httpGet:
path: /health/liveliness
port: 4000
initialDelaySeconds: 120
periodSeconds: 15
successThreshold: 1
failureThreshold: 3
timeoutSeconds: 10
readinessProbe:
httpGet:
path: /health/readiness
port: 4000
initialDelaySeconds: 120
periodSeconds: 15
successThreshold: 1
failureThreshold: 3
timeoutSeconds: 10
volumes: # Define volume to mount proxy_config.yaml
- name: config-volume
configMap:
name: litellm-config

12
deploy/kubernetes/service.yaml
View file

@ -1,12 +0,0 @@
apiVersion: v1
kind: Service
metadata:
name: litellm-service
spec:
selector:
app: litellm
ports:
- protocol: TCP
port: 4000
targetPort: 4000
type: LoadBalancer

13
dev_config.yaml
View file

@ -1,13 +0,0 @@
model_list:
- model_name: fake-openai-endpoint
litellm_params:
model: openai/fake-model
api_key: fake-key
api_base: https://exampleopenaiendpoint-production.up.railway.app/
general_settings:
master_key: sk-1234
litellm_settings:
drop_params: True
telemetry: False

5
docker-compose.yml
View file

@ -16,6 +16,11 @@ services:
- "4000:4000" # Map the container port to the host, change the host port if necessary
environment:
DATABASE_URL: "postgresql://llmproxy:dbpassword9090@db:5432/litellm"
# Optional: route read-only queries (find_*, count, group_by, query_raw/_first)
# to a separate reader endpoint, e.g. an Aurora reader. Leave unset for
# single-DB deployments. With IAM_TOKEN_DB_AUTH enabled, the reader URL
# is auto-refreshed alongside the writer.
# DATABASE_URL_READ_REPLICA: "postgresql://llmproxy:dbpassword9090@db-reader:5432/litellm"
STORE_MODEL_IN_DB: "True" # allows adding models to proxy via UI
env_file:
- .env # Load local .env file

68
docker/Dockerfile.alpine
View file

@ -1,68 +0,0 @@
# Base image for building
ARG LITELLM_BUILD_IMAGE=python:3.11-alpine@sha256:f07e2ace46f560f09a6eeec7b4913b80ee99546e749ef82342a419a326620856
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=python:3.11-alpine@sha256:f07e2ace46f560f09a6eeec7b4913b80ee99546e749ef82342a419a326620856
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin
FROM $LITELLM_BUILD_IMAGE AS builder
WORKDIR /app
COPY --from=uvbin /uv /usr/local/bin/uv
COPY --from=uvbin /uvx /usr/local/bin/uvx
RUN apk add --no-cache gcc python3-dev musl-dev nodejs npm libsndfile
ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
UV_PROJECT_ENVIRONMENT=/app/.venv \
UV_LINK_MODE=copy \
XDG_CACHE_HOME=/app/.cache \
PATH="/app/.venv/bin:${PATH}"
# Copy dependency metadata first for layer caching
COPY pyproject.toml uv.lock ./
COPY enterprise/pyproject.toml enterprise/
COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/
# Install third-party dependencies (cached unless pyproject.toml/uv.lock change)
RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--extra extra_proxy \
--extra semantic-router \
--python python3
# Copy full source tree
COPY . .
# Install project and workspace packages (fast - deps already cached)
RUN uv sync --frozen --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--extra extra_proxy \
--extra semantic-router \
--python python3
RUN prisma generate --schema=./schema.prisma
RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
FROM $LITELLM_RUNTIME_IMAGE AS runtime
RUN apk upgrade --no-cache && apk add --no-cache libsndfile nodejs npm
WORKDIR /app
ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
XDG_CACHE_HOME=/app/.cache \
PATH="/app/.venv/bin:${PATH}"
COPY --from=builder /app /app
EXPOSE 4000/tcp
ENTRYPOINT ["docker/prod_entrypoint.sh"]
CMD ["--port", "4000"]

86
docker/Dockerfile.custom_ui
View file

@ -1,86 +0,0 @@
# Use the provided base image
# NOTE: This is a dev/branch-specific tag. Update digest when the base image is rebuilt.
FROM ghcr.io/berriai/litellm:litellm_fwd_server_root_path-dev
# Set the working directory to /app
WORKDIR /app
# Install Node.js and npm (adjust version as needed)
RUN apt-get update && apt-get upgrade -y \
libxml2 \
libexpat1 \
openssl \
libssl3 \
git \
libkrb5-3 \
libglib2.0-0 \
wget \
libaom3 \
libxslt1.1 \
libgnutls30 \
libc6 && \
apt-get install -y --no-install-recommends nodejs npm && \
npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
GLOBAL="$(npm root -g)" && \
find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
done && \
find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
done && \
find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
done && \
find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
done && \
find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
done && \
find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \
npm cache clean --force && \
apt-get purge -y npm
# Copy the UI source into the container
COPY ./ui/litellm-dashboard /app/ui/litellm-dashboard
# Set an environment variable for UI_BASE_PATH
# This can be overridden at build time
# set UI_BASE_PATH to "<your server root path>/ui"
ENV UI_BASE_PATH="/prod/ui"
# Build the UI with the specified UI_BASE_PATH
WORKDIR /app/ui/litellm-dashboard
RUN npm ci
RUN UI_BASE_PATH=$UI_BASE_PATH npm run build
# Create the destination directory
RUN mkdir -p /app/litellm/proxy/_experimental/out
# Move the built files to the appropriate location
# Assuming the build output is in ./out directory
RUN rm -rf /app/litellm/proxy/_experimental/out/* && \
mv ./out/* /app/litellm/proxy/_experimental/out/
# Switch back to the main app directory
WORKDIR /app
# Make sure your docker/entrypoint.sh is executable
# Convert Windows line endings to Unix for entrypoint scripts
RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
# Run as non-root user
RUN groupadd --gid 1000 appuser && useradd --uid 1000 --gid 1000 --no-create-home appuser \
&& chown -R appuser:appuser /app
USER appuser
# Expose the necessary port
EXPOSE 4000/tcp
HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:4000/health')"]
# Override the CMD instruction with your desired command and arguments
CMD ["--port", "4000", "--config", "config.yaml", "--detailed_debug"]

4
docker/Dockerfile.database
View file

@ -66,7 +66,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
USER root
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 libsndfile supervisor && \
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 libsndfile && \
npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
GLOBAL="$(npm root -g)" && \
find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
@ -102,7 +102,5 @@ RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
EXPOSE 4000/tcp
COPY docker/supervisord.conf /etc/supervisord.conf
ENTRYPOINT ["docker/prod_entrypoint.sh"]
CMD ["--port", "4000"]

121
docker/Dockerfile.dev
View file

@ -1,121 +0,0 @@
# Base image for building
ARG LITELLM_BUILD_IMAGE=python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin
FROM $LITELLM_BUILD_IMAGE AS builder
WORKDIR /app
USER root
COPY --from=uvbin /uv /usr/local/bin/uv
COPY --from=uvbin /uvx /usr/local/bin/uvx
RUN apt-get update && apt-get install -y --no-install-recommends \
gcc \
g++ \
python3-dev \
libssl-dev \
pkg-config \
nodejs \
npm \
&& rm -rf /var/lib/apt/lists/*
ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
UV_PROJECT_ENVIRONMENT=/app/.venv \
UV_LINK_MODE=copy \
XDG_CACHE_HOME=/app/.cache \
PATH="/app/.venv/bin:${PATH}"
# Copy dependency metadata first for layer caching
COPY pyproject.toml uv.lock ./
COPY enterprise/pyproject.toml enterprise/
COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/
# Install third-party dependencies (cached unless pyproject.toml/uv.lock change)
RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--extra extra_proxy \
--extra semantic-router \
--python python
# Copy full source tree
COPY . .
# Build Admin UI before final sync
RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
# Install project and workspace packages (fast - deps already cached)
RUN uv sync --frozen --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--extra extra_proxy \
--extra semantic-router \
--python python
RUN prisma generate --schema=./schema.prisma
RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
FROM $LITELLM_RUNTIME_IMAGE AS runtime
USER root
RUN apt-get update && apt-get upgrade -y \
libxml2 \
libexpat1 \
openssl \
libssl3 \
git \
libkrb5-3 \
libglib2.0-0 \
wget \
libaom3 \
libxslt1.1 \
libgnutls30 \
libc6 \
&& apt-get install -y --no-install-recommends \
libssl3 \
libatomic1 \
nodejs \
npm \
&& rm -rf /var/lib/apt/lists/* \
&& npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \
&& GLOBAL="$(npm root -g)" \
&& find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
done \
&& find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
done \
&& find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
done \
&& find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
done \
&& find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
done \
&& find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null \
&& npm cache clean --force \
&& apt-get purge -y npm
WORKDIR /app
ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
XDG_CACHE_HOME=/app/.cache \
PATH="/app/.venv/bin:${PATH}"
COPY --from=builder /app /app
EXPOSE 4000/tcp
ENTRYPOINT ["docker/prod_entrypoint.sh"]
CMD ["--port", "4000"]

30
docker/Dockerfile.health_check
View file

@ -1,30 +0,0 @@
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin
FROM python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d
WORKDIR /app
# Copy the uv binary and the health check script.
COPY --from=uvbin /uv /usr/local/bin/uv
COPY pyproject.toml uv.lock /app/
COPY scripts/health_check/health_check_client.py /app/health_check_client.py
# Resolve and install the health-check dependencies from the project lockfile
# so the runtime image stays self-contained and reproducible.
RUN uv export --frozen --no-default-groups --only-group healthcheck --no-emit-project --no-hashes --output-file /tmp/health-check-requirements.txt \
&& uv pip install --system -r /tmp/health-check-requirements.txt \
&& rm /tmp/health-check-requirements.txt \
&& rm /app/pyproject.toml /app/uv.lock \
&& chmod +x /app/health_check_client.py
# Run as non-root user
RUN groupadd --gid 1000 appuser && useradd --uid 1000 --gid 1000 --no-create-home appuser
USER appuser
# Health check
HEALTHCHECK --interval=30s --timeout=5s --start-period=5s --retries=3 \
CMD ["python", "/app/health_check_client.py", "--help"]
# Set entrypoint
ENTRYPOINT ["python", "/app/health_check_client.py"]

3
docker/Dockerfile.non_root
View file

@ -103,13 +103,12 @@ RUN for i in 1 2 3; do \
apk upgrade --no-cache && break || sleep 5; \
done && \
for i in 1 2 3; do \
apk add --no-cache python3 bash openssl tzdata supervisor libsndfile nodejs && break || sleep 5; \
apk add --no-cache python3 bash openssl tzdata libsndfile nodejs && break || sleep 5; \
done
COPY --from=builder /app /app
COPY --from=builder /var/lib/litellm/ui /var/lib/litellm/ui
COPY --from=builder /var/lib/litellm/assets /var/lib/litellm/assets
COPY --from=builder /app/docker/supervisord.conf /etc/supervisord.conf
ENV PATH="/app/.venv/bin:${PATH}" \
PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \

8
docker/prod_entrypoint.sh
View file

@ -1,14 +1,8 @@
#!/bin/sh
if [ "$SEPARATE_HEALTH_APP" = "1" ]; then
export LITELLM_ARGS="$@"
export SUPERVISORD_STOPWAITSECS="${SUPERVISORD_STOPWAITSECS:-3600}"
exec supervisord -c /etc/supervisord.conf
fi
if [ "$USE_DDTRACE" = "true" ]; then
export DD_TRACE_OPENAI_ENABLED="False"
exec ddtrace-run litellm "$@"
else
exec litellm "$@"
fi
fi

46
docker/supervisord.conf
View file

@ -1,46 +0,0 @@
[supervisord]
nodaemon=true
loglevel=info
logfile=/tmp/supervisord.log
pidfile=/tmp/supervisord.pid
[group:litellm]
programs=main,health
[program:main]
command=sh -c 'if [ "$USE_DDTRACE" = "true" ]; then export DD_TRACE_OPENAI_ENABLED="False"; exec ddtrace-run python -m litellm.proxy.proxy_cli --host 0.0.0.0 --port=4000 $LITELLM_ARGS; else exec python -m litellm.proxy.proxy_cli --host 0.0.0.0 --port=4000 $LITELLM_ARGS; fi'
autostart=true
autorestart=true
startretries=3
priority=1
exitcodes=0
stopasgroup=true
killasgroup=true
stopwaitsecs=%(ENV_SUPERVISORD_STOPWAITSECS)s
stdout_logfile=/dev/stdout
stderr_logfile=/dev/stderr
stdout_logfile_maxbytes = 0
stderr_logfile_maxbytes = 0
environment=PYTHONUNBUFFERED=true
[program:health]
command=sh -c '[ "$SEPARATE_HEALTH_APP" = "1" ] && exec uvicorn litellm.proxy.health_endpoints.health_app_factory:build_health_app --factory --host 0.0.0.0 --port=${SEPARATE_HEALTH_PORT:-4001} || exit 0'
autostart=true
autorestart=true
startretries=3
priority=2
exitcodes=0
stopasgroup=true
killasgroup=true
stopwaitsecs=%(ENV_SUPERVISORD_STOPWAITSECS)s
stdout_logfile=/dev/stdout
stderr_logfile=/dev/stderr
stdout_logfile_maxbytes = 0
stderr_logfile_maxbytes = 0
environment=PYTHONUNBUFFERED=true
[eventlistener:process_monitor]
command=python -c "from supervisor import childutils; import os, signal; [os.kill(os.getppid(), signal.SIGTERM) for h,p in iter(lambda: childutils.listener.wait(), None) if h['eventname'] in ['PROCESS_STATE_FATAL', 'PROCESS_STATE_EXITED'] and dict([x.split(':') for x in p.split(' ')])['processname'] in ['main', 'health'] or childutils.listener.ok()]"
events=PROCESS_STATE_EXITED,PROCESS_STATE_FATAL
autostart=true
autorestart=true

BIN
docs/images/local-testing/hosted-vllm-custom-tool-local-test.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 82 KiB

4
enterprise/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-enterprise"
version = "0.1.39"
version = "0.1.40"
description = "Package for LiteLLM Enterprise features"
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.1.39"
version = "0.1.40"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-enterprise==",

108
index.yaml
View file

@ -1,108 +0,0 @@
apiVersion: v1
entries:
litellm-helm:
- apiVersion: v2
appVersion: v1.43.18
created: "2024-08-19T23:58:25.331689+08:00"
dependencies:
- condition: db.deployStandalone
name: postgresql
repository: oci://registry-1.docker.io/bitnamicharts
version: '>=13.3.0'
- condition: redis.enabled
name: redis
repository: oci://registry-1.docker.io/bitnamicharts
version: '>=18.0.0'
description: Call all LLM APIs using the OpenAI format
digest: 0411df3dc42868be8af3ad3e00cb252790e6bd7ad15f5b77f1ca5214573a8531
name: litellm-helm
type: application
urls:
- https://berriai.github.io/litellm/litellm-helm-0.2.3.tgz
version: 0.2.3
postgresql:
- annotations:
category: Database
images: |
- name: os-shell
image: docker.io/bitnami/os-shell:12-debian-12-r16
- name: postgres-exporter
image: docker.io/bitnami/postgres-exporter:0.15.0-debian-12-r14
- name: postgresql
image: docker.io/bitnami/postgresql:16.2.0-debian-12-r6
licenses: Apache-2.0
apiVersion: v2
appVersion: 16.2.0
created: "2024-08-19T23:58:25.335716+08:00"
dependencies:
- name: common
repository: oci://registry-1.docker.io/bitnamicharts
tags:
- bitnami-common
version: 2.x.x
description: PostgreSQL (Postgres) is an open source object-relational database
known for reliability and data integrity. ACID-compliant, it supports foreign
keys, joins, views, triggers and stored procedures.
digest: 3c8125526b06833df32e2f626db34aeaedb29d38f03d15349db6604027d4a167
home: https://bitnami.com
icon: https://bitnami.com/assets/stacks/postgresql/img/postgresql-stack-220x234.png
keywords:
- postgresql
- postgres
- database
- sql
- replication
- cluster
maintainers:
- name: VMware, Inc.
url: https://github.com/bitnami/charts
name: postgresql
sources:
- https://github.com/bitnami/charts/tree/main/bitnami/postgresql
urls:
- https://berriai.github.io/litellm/charts/postgresql-14.3.1.tgz
version: 14.3.1
redis:
- annotations:
category: Database
images: |
- name: kubectl
image: docker.io/bitnami/kubectl:1.29.2-debian-12-r3
- name: os-shell
image: docker.io/bitnami/os-shell:12-debian-12-r16
- name: redis
image: docker.io/bitnami/redis:7.2.4-debian-12-r9
- name: redis-exporter
image: docker.io/bitnami/redis-exporter:1.58.0-debian-12-r4
- name: redis-sentinel
image: docker.io/bitnami/redis-sentinel:7.2.4-debian-12-r7
licenses: Apache-2.0
apiVersion: v2
appVersion: 7.2.4
created: "2024-08-19T23:58:25.339392+08:00"
dependencies:
- name: common
repository: oci://registry-1.docker.io/bitnamicharts
tags:
- bitnami-common
version: 2.x.x
description: Redis(R) is an open source, advanced key-value store. It is often
referred to as a data structure server since keys can contain strings, hashes,
lists, sets and sorted sets.
digest: b2fa1835f673a18002ca864c54fadac3c33789b26f6c5e58e2851b0b14a8f984
home: https://bitnami.com
icon: https://bitnami.com/assets/stacks/redis/img/redis-stack-220x234.png
keywords:
- redis
- keyvalue
- database
maintainers:
- name: VMware, Inc.
url: https://github.com/bitnami/charts
name: redis
sources:
- https://github.com/bitnami/charts/tree/main/bitnami/redis
urls:
- https://berriai.github.io/litellm/charts/redis-18.19.1.tgz
version: 18.19.1
generated: "2024-08-19T23:58:25.322532+08:00"

5
litellm-js/proxy/.npmrc
View file

@ -1,5 +0,0 @@
# Supply-chain hardening
# Packages needing lifecycle scripts: npm rebuild <pkg>
ignore-scripts=true
# Protects local npm install only — npm ci (used in CI) ignores this
min-release-age=3

8
litellm-js/proxy/README.md
View file

@ -1,8 +0,0 @@
```
npm install
npm run dev
```
```
npm run deploy
```

2054
litellm-js/proxy/package-lock.json generated
View file

File diff suppressed because it is too large Load diff

14
litellm-js/proxy/package.json
View file

@ -1,14 +0,0 @@
{
"scripts": {
"dev": "wrangler dev src/index.ts",
"deploy": "wrangler deploy --minify src/index.ts"
},
"dependencies": {
"hono": "4.12.16",
"openai": "4.29.2"
},
"devDependencies": {
"@cloudflare/workers-types": "4.20260501.1",
"wrangler": "4.87.0"
}
}

59
litellm-js/proxy/src/index.ts
View file

@ -1,59 +0,0 @@
import { Hono } from 'hono'
import { Context } from 'hono';
import { bearerAuth } from 'hono/bearer-auth'
import OpenAI from "openai";
const openai = new OpenAI({
apiKey: "sk-1234",
baseURL: "https://openai-endpoint.ishaanjaffer0324.workers.dev"
});
async function call_proxy() {
const completion = await openai.chat.completions.create({
messages: [{ role: "system", content: "You are a helpful assistant." }],
model: "gpt-3.5-turbo",
});
return completion
}
const app = new Hono()
// Middleware for API Key Authentication
const apiKeyAuth = async (c: Context, next: Function) => {
const apiKey = c.req.header('Authorization');
if (!apiKey || apiKey !== 'Bearer sk-1234') {
return c.text('Unauthorized', 401);
}
await next();
};
app.use('/*', apiKeyAuth)
app.get('/', (c) => {
return c.text('Hello Hono!')
})
// Handler for chat completions
const chatCompletionHandler = async (c: Context) => {
// Assuming your logic for handling chat completion goes here
// For demonstration, just returning a simple JSON response
const response = await call_proxy()
return c.json(response);
};
// Register the above handler for different POST routes with the apiKeyAuth middleware
app.post('/v1/chat/completions', chatCompletionHandler);
app.post('/chat/completions', chatCompletionHandler);
// Example showing how you might handle dynamic segments within the URL
// Here, using ':model*' to capture the rest of the path as a parameter 'model'
app.post('/openai/deployments/:model*/chat/completions', chatCompletionHandler);
export default app

17
litellm-js/proxy/tsconfig.json
View file

@ -1,17 +0,0 @@
{
"compilerOptions": {
"target": "ESNext",
"module": "ESNext",
"moduleResolution": "Bundler",
"strict": true,
"lib": [
"ESNext"
],
"types": [
"@cloudflare/workers-types"
],
"jsx": "react-jsx",
"jsxImportSource": "hono/jsx",
"skipLibCheck": true
},
}

18
litellm-js/proxy/wrangler.toml
View file

@ -1,18 +0,0 @@
name = "my-app"
compatibility_date = "2023-12-01"
# [vars]
# MY_VAR = "my-variable"
# [[kv_namespaces]]
# binding = "MY_KV_NAMESPACE"
# id = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
# [[r2_buckets]]
# binding = "MY_BUCKET"
# bucket_name = "my-bucket"
# [[d1_databases]]
# binding = "DB"
# database_name = "my-database"
# database_id = ""

5
litellm-js/spend-logs/.npmrc
View file

@ -1,5 +0,0 @@
# Supply-chain hardening
# Packages needing lifecycle scripts: npm rebuild <pkg>
ignore-scripts=true
# Protects local npm install only — npm ci (used in CI) ignores this
min-release-age=3

26
litellm-js/spend-logs/Dockerfile
View file

@ -1,26 +0,0 @@
# Use the specific Node.js v20.11.0 image
FROM node:20.18.1-alpine3.20
# Set the working directory inside the container
WORKDIR /app
# Copy package.json and package-lock.json to the working directory
COPY ./litellm-js/spend-logs/package*.json ./
# Install dependencies
RUN npm ci
# Install Prisma globally
RUN npm install -g prisma
# Copy the rest of the application code
COPY ./litellm-js/spend-logs .
# Generate Prisma client
RUN npx prisma generate
# Expose the port that the Node.js server will run on
EXPOSE 3000
# Command to run the Node.js app with npm run dev
CMD ["npm", "run", "dev"]

8
litellm-js/spend-logs/README.md
View file

@ -1,8 +0,0 @@
```
npm install
npm run dev
```
```
open http://localhost:3000
```

597
litellm-js/spend-logs/package-lock.json generated
View file

@ -1,597 +0,0 @@
{
"name": "spend-logs",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"dependencies": {
"@hono/node-server": "1.19.13",
"hono": "4.12.16"
},
"devDependencies": {
"@types/node": "20.19.25",
"tsx": "4.20.6"
}
},
"node_modules/@esbuild/aix-ppc64": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz",
"integrity": "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==",
"cpu": [
"ppc64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"aix"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/android-arm": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.12.tgz",
"integrity": "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==",
"cpu": [
"arm"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"android"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/android-arm64": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.12.tgz",
"integrity": "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==",
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"android"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/android-x64": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.12.tgz",
"integrity": "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==",
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"android"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/darwin-arm64": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.12.tgz",
"integrity": "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==",
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/darwin-x64": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.12.tgz",
"integrity": "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==",
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/freebsd-arm64": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.12.tgz",
"integrity": "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==",
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"freebsd"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/freebsd-x64": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.12.tgz",
"integrity": "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==",
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"freebsd"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/linux-arm": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.12.tgz",
"integrity": "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==",
"cpu": [
"arm"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/linux-arm64": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.12.tgz",
"integrity": "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==",
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/linux-ia32": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.12.tgz",
"integrity": "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==",
"cpu": [
"ia32"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/linux-loong64": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.12.tgz",
"integrity": "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==",
"cpu": [
"loong64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/linux-mips64el": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.12.tgz",
"integrity": "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==",
"cpu": [
"mips64el"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/linux-ppc64": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.12.tgz",
"integrity": "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==",
"cpu": [
"ppc64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/linux-riscv64": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.12.tgz",
"integrity": "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==",
"cpu": [
"riscv64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/linux-s390x": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.12.tgz",
"integrity": "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==",
"cpu": [
"s390x"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/linux-x64": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.12.tgz",
"integrity": "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==",
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/netbsd-arm64": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.12.tgz",
"integrity": "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==",
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"netbsd"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/netbsd-x64": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.12.tgz",
"integrity": "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==",
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"netbsd"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/openbsd-arm64": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.12.tgz",
"integrity": "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==",
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"openbsd"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/openbsd-x64": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.12.tgz",
"integrity": "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==",
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"openbsd"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/openharmony-arm64": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.12.tgz",
"integrity": "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==",
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"openharmony"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/sunos-x64": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.12.tgz",
"integrity": "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==",
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"sunos"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/win32-arm64": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.12.tgz",
"integrity": "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==",
"cpu": [
"arm64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"win32"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/win32-ia32": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.12.tgz",
"integrity": "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==",
"cpu": [
"ia32"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"win32"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@esbuild/win32-x64": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.12.tgz",
"integrity": "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==",
"cpu": [
"x64"
],
"dev": true,
"license": "MIT",
"optional": true,
"os": [
"win32"
],
"engines": {
"node": ">=18"
}
},
"node_modules/@hono/node-server": {
"version": "1.19.13",
"resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-1.19.13.tgz",
"integrity": "sha512-TsQLe4i2gvoTtrHje625ngThGBySOgSK3Xo2XRYOdqGN1teR8+I7vchQC46uLJi8OF62YTYA3AhSpumtkhsaKQ==",
"license": "MIT",
"engines": {
"node": ">=18.14.1"
},
"peerDependencies": {
"hono": "^4"
}
},
"node_modules/@types/node": {
"version": "20.19.25",
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.25.tgz",
"integrity": "sha512-ZsJzA5thDQMSQO788d7IocwwQbI8B5OPzmqNvpf3NY/+MHDAS759Wo0gd2WQeXYt5AAAQjzcrTVC6SKCuYgoCQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"undici-types": "~6.21.0"
}
},
"node_modules/esbuild": {
"version": "0.25.12",
"resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.12.tgz",
"integrity": "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==",
"dev": true,
"hasInstallScript": true,
"license": "MIT",
"bin": {
"esbuild": "bin/esbuild"
},
"engines": {
"node": ">=18"
},
"optionalDependencies": {
"@esbuild/aix-ppc64": "0.25.12",
"@esbuild/android-arm": "0.25.12",
"@esbuild/android-arm64": "0.25.12",
"@esbuild/android-x64": "0.25.12",
"@esbuild/darwin-arm64": "0.25.12",
"@esbuild/darwin-x64": "0.25.12",
"@esbuild/freebsd-arm64": "0.25.12",
"@esbuild/freebsd-x64": "0.25.12",
"@esbuild/linux-arm": "0.25.12",
"@esbuild/linux-arm64": "0.25.12",
"@esbuild/linux-ia32": "0.25.12",
"@esbuild/linux-loong64": "0.25.12",
"@esbuild/linux-mips64el": "0.25.12",
"@esbuild/linux-ppc64": "0.25.12",
"@esbuild/linux-riscv64": "0.25.12",
"@esbuild/linux-s390x": "0.25.12",
"@esbuild/linux-x64": "0.25.12",
"@esbuild/netbsd-arm64": "0.25.12",
"@esbuild/netbsd-x64": "0.25.12",
"@esbuild/openbsd-arm64": "0.25.12",
"@esbuild/openbsd-x64": "0.25.12",
"@esbuild/openharmony-arm64": "0.25.12",
"@esbuild/sunos-x64": "0.25.12",
"@esbuild/win32-arm64": "0.25.12",
"@esbuild/win32-ia32": "0.25.12",
"@esbuild/win32-x64": "0.25.12"
}
},
"node_modules/fsevents": {
"version": "2.3.3",
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz",
"integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==",
"dev": true,
"hasInstallScript": true,
"license": "MIT",
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
}
},
"node_modules/get-tsconfig": {
"version": "4.13.0",
"resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.13.0.tgz",
"integrity": "sha512-1VKTZJCwBrvbd+Wn3AOgQP/2Av+TfTCOlE4AcRJE72W1ksZXbAx8PPBR9RzgTeSPzlPMHrbANMH3LbltH73wxQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"resolve-pkg-maps": "^1.0.0"
},
"funding": {
"url": "https://github.com/privatenumber/get-tsconfig?sponsor=1"
}
},
"node_modules/hono": {
"version": "4.12.16",
"resolved": "https://registry.npmjs.org/hono/-/hono-4.12.16.tgz",
"integrity": "sha512-jN0ZewiNAWSe5khM3EyCmBb250+b40wWbwNILNfEvq84VREWwOIkuUsFONk/3i3nqkz7Oe1PcpM2mwQEK2L9Kg==",
"license": "MIT",
"engines": {
"node": ">=16.9.0"
}
},
"node_modules/resolve-pkg-maps": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz",
"integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==",
"dev": true,
"license": "MIT",
"funding": {
"url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1"
}
},
"node_modules/tsx": {
"version": "4.20.6",
"resolved": "https://registry.npmjs.org/tsx/-/tsx-4.20.6.tgz",
"integrity": "sha512-ytQKuwgmrrkDTFP4LjR0ToE2nqgy886GpvRSpU0JAnrdBYppuY5rLkRUYPU1yCryb24SsKBTL/hlDQAEFVwtZg==",
"dev": true,
"license": "MIT",
"dependencies": {
"esbuild": "~0.25.0",
"get-tsconfig": "^4.7.5"
},
"bin": {
"tsx": "dist/cli.mjs"
},
"engines": {
"node": ">=18.0.0"
},
"optionalDependencies": {
"fsevents": "~2.3.3"
}
},
"node_modules/undici-types": {
"version": "6.21.0",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz",
"integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==",
"dev": true,
"license": "MIT"
}
}
}

13
litellm-js/spend-logs/package.json
View file

@ -1,13 +0,0 @@
{
"scripts": {
"dev": "tsx watch src/index.ts"
},
"dependencies": {
"@hono/node-server": "1.19.13",
"hono": "4.12.16"
},
"devDependencies": {
"@types/node": "20.19.25",
"tsx": "4.20.6"
}
}

29
litellm-js/spend-logs/schema.prisma
View file

@ -1,29 +0,0 @@
generator client {
provider = "prisma-client-js"
}
datasource client {
provider = "postgresql"
url = env("DATABASE_URL")
}
model LiteLLM_SpendLogs {
request_id String @id
call_type String
api_key String @default("")
spend Float @default(0.0)
total_tokens Int @default(0)
prompt_tokens Int @default(0)
completion_tokens Int @default(0)
startTime DateTime
endTime DateTime
model String @default("")
api_base String @default("")
user String @default("")
metadata Json @default("{}")
cache_hit String @default("")
cache_key String @default("")
request_tags Json @default("[]")
team_id String?
end_user String?
}

32
litellm-js/spend-logs/src/_types.ts
View file

@ -1,32 +0,0 @@
export type LiteLLM_IncrementSpend = {
key_transactions: Array<LiteLLM_IncrementObject>, // [{"key": spend},..]
user_transactions: Array<LiteLLM_IncrementObject>,
team_transactions: Array<LiteLLM_IncrementObject>,
spend_logs_transactions: Array<LiteLLM_SpendLogs>
}
export type LiteLLM_IncrementObject = {
key: string,
spend: number
}
export type LiteLLM_SpendLogs = {
request_id: string; // @id means it's a unique identifier
call_type: string;
api_key: string; // @default("") means it defaults to an empty string if not provided
spend: number; // Float in Prisma corresponds to number in TypeScript
total_tokens: number; // Int in Prisma corresponds to number in TypeScript
prompt_tokens: number;
completion_tokens: number;
startTime: Date; // DateTime in Prisma corresponds to Date in TypeScript
endTime: Date;
model: string; // @default("") means it defaults to an empty string if not provided
api_base: string;
user: string;
metadata: any; // Json type in Prisma is represented by any in TypeScript; could also use a more specific type if the structure of JSON is known
cache_hit: string;
cache_key: string;
request_tags: any; // Similarly, this could be an array or a more specific type depending on the expected structure
team_id?: string | null; // ? indicates it's optional and can be undefined, but could also be null if not provided
end_user?: string | null;
};

84
litellm-js/spend-logs/src/index.ts
View file

@ -1,84 +0,0 @@
import { serve } from '@hono/node-server'
import { Hono } from 'hono'
import { PrismaClient } from '@prisma/client'
import {LiteLLM_SpendLogs, LiteLLM_IncrementSpend, LiteLLM_IncrementObject} from './_types'
const app = new Hono()
const prisma = new PrismaClient()
// In-memory storage for logs
let spend_logs: LiteLLM_SpendLogs[] = [];
const key_logs: LiteLLM_IncrementObject[] = [];
const user_logs: LiteLLM_IncrementObject[] = [];
const transaction_logs: LiteLLM_IncrementObject[] = [];
app.get('/', (c) => {
return c.text('Hello Hono!')
})
const MIN_LOGS = 1; // Minimum number of logs needed to initiate a flush
const FLUSH_INTERVAL = 5000; // Time in ms to wait before trying to flush again
const BATCH_SIZE = 100; // Preferred size of each batch to write to the database
const MAX_LOGS_PER_INTERVAL = 1000; // Maximum number of logs to flush in a single interval
const flushLogsToDb = async () => {
if (spend_logs.length >= MIN_LOGS) {
// Limit the logs to process in this interval to MAX_LOGS_PER_INTERVAL or less
const logsToProcess = spend_logs.slice(0, MAX_LOGS_PER_INTERVAL);
for (let i = 0; i < logsToProcess.length; i += BATCH_SIZE) {
// Create subarray for current batch, ensuring it doesn't exceed the BATCH_SIZE
const batch = logsToProcess.slice(i, i + BATCH_SIZE);
// Convert datetime strings to Date objects
const batchWithDates = batch.map(entry => ({
...entry,
startTime: new Date(entry.startTime),
endTime: new Date(entry.endTime),
// Repeat for any other DateTime fields you may have
}));
await prisma.liteLLM_SpendLogs.createMany({
data: batchWithDates,
});
console.log(`Flushed ${batch.length} logs to the DB.`);
}
// Remove the processed logs from spend_logs
spend_logs = spend_logs.slice(logsToProcess.length);
console.log(`${logsToProcess.length} logs processed. Remaining in queue: ${spend_logs.length}`);
} else {
// This will ensure it doesn't falsely claim "No logs to flush." when it's merely below the MIN_LOGS threshold.
if(spend_logs.length > 0) {
console.log(`Accumulating logs. Currently at ${spend_logs.length}, waiting for at least ${MIN_LOGS}.`);
} else {
console.log("No logs to flush.");
}
}
};
// Setup interval for attempting to flush the logs
setInterval(flushLogsToDb, FLUSH_INTERVAL);
// Route to receive log messages
app.post('/spend/update', async (c) => {
const incomingLogs = await c.req.json<LiteLLM_SpendLogs[]>();
spend_logs.push(...incomingLogs);
console.log(`Received and stored ${incomingLogs.length} logs. Total logs in memory: ${spend_logs.length}`);
return c.json({ message: `Successfully stored ${incomingLogs.length} logs` });
});
const port = 3000
console.log(`Server is running on port ${port}`)
serve({
fetch: app.fetch,
port
})

13
litellm-js/spend-logs/tsconfig.json
View file

@ -1,13 +0,0 @@
{
"compilerOptions": {
"target": "ESNext",
"module": "ESNext",
"moduleResolution": "Bundler",
"strict": true,
"types": [
"node"
],
"jsx": "react-jsx",
"jsxImportSource": "hono/jsx",
}
}

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
version = "0.4.70"
version = "0.4.71"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.4.70"
version = "0.4.71"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-proxy-extras==",

13
litellm/__init__.py
View file

@ -206,6 +206,7 @@ add_user_information_to_llm_headers: Optional[bool] = (
)
store_audit_logs = False # Enterprise feature, allow users to see audit logs
skip_system_message_in_guardrail: bool = False
skip_tool_message_in_guardrail: bool = False
### end of callbacks #############
email: Optional[str] = (
@ -388,6 +389,7 @@ anthropic_beta_headers_url: str = os.getenv(
suppress_debug_info = False
dynamodb_table_name: Optional[str] = None
s3_callback_params: Optional[Dict] = None
s3_audit_callback_params: Optional[Dict] = None
datadog_llm_observability_params: Optional[Union[DatadogLLMObsInitParams, Dict]] = None
datadog_params: Optional[Union[DatadogInitParams, Dict]] = None
aws_sqs_callback_params: Optional[Dict] = None
@ -414,6 +416,9 @@ custom_prometheus_metadata_labels: List[str] = []
custom_prometheus_tags: List[str] = []
prometheus_metrics_config: Optional[List] = None
prometheus_emit_stream_label: bool = False
prometheus_end_user_metrics_max_series_per_metric: Optional[int] = 10000
prometheus_end_user_metrics_ttl_seconds: Optional[float] = 3600.0
prometheus_end_user_metrics_cleanup_interval_seconds: Optional[float] = 60.0
disable_add_prefix_to_prompt: bool = (
False # used by anthropic, to disable adding prefix to prompt
)
@ -586,6 +591,7 @@ anyscale_models: Set = set()
cerebras_models: Set = set()
galadriel_models: Set = set()
nvidia_nim_models: Set = set()
nvidia_riva_models: Set = set()
sambanova_models: Set = set()
sambanova_embedding_models: Set = set()
novita_models: Set = set()
@ -812,6 +818,8 @@ def add_known_models(model_cost_map: Optional[Dict] = None):
galadriel_models.add(key)
elif value.get("litellm_provider") == "nvidia_nim":
nvidia_nim_models.add(key)
elif value.get("litellm_provider") == "nvidia_riva":
nvidia_riva_models.add(key)
elif value.get("litellm_provider") == "sambanova":
sambanova_models.add(key)
elif value.get("litellm_provider") == "sambanova-embedding-models":
@ -971,6 +979,7 @@ model_list = list(
| cerebras_models
| galadriel_models
| nvidia_nim_models
| nvidia_riva_models
| sambanova_models
| azure_text_models
| novita_models
@ -1067,6 +1076,7 @@ models_by_provider: dict = {
"cerebras": cerebras_models,
"galadriel": galadriel_models,
"nvidia_nim": nvidia_nim_models,
"nvidia_riva": nvidia_riva_models,
"sambanova": sambanova_models | sambanova_embedding_models,
"novita": novita_models,
"nebius": nebius_models | nebius_embedding_models,
@ -1618,6 +1628,9 @@ if TYPE_CHECKING:
from .llms.deepgram.audio_transcription.transformation import (
DeepgramAudioTranscriptionConfig as DeepgramAudioTranscriptionConfig,
)
from .llms.nvidia_riva.audio_transcription.transformation import (
NvidiaRivaAudioTranscriptionConfig as NvidiaRivaAudioTranscriptionConfig,
)
from .llms.topaz.image_variations.transformation import (
TopazImageVariationConfig as TopazImageVariationConfig,
)

219
litellm/_redis.py
View file

@ -19,6 +19,7 @@ import redis.asyncio as async_redis # type: ignore
from litellm import get_secret, get_secret_str
from litellm._redis_credential_provider import (
AzureADCredentialProvider,
GCPIAMCredentialProvider,
_generate_gcp_iam_access_token,
)
@ -27,6 +28,8 @@ from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker
from ._logging import verbose_logger
AZURE_REDIS_SCOPE = "https://redis.azure.com/.default"
def _get_redis_kwargs():
arg_spec = inspect.getfullargspec(redis.Redis)
@ -43,6 +46,10 @@ def _get_redis_kwargs():
"redis_connect_func",
"gcp_service_account",
"gcp_ssl_ca_certs",
"azure_redis_ad_token",
"azure_client_id",
"azure_tenant_id",
"azure_client_secret",
]
available_args = [x for x in arg_spec.args if x not in exclude_args] + include_args
@ -89,6 +96,10 @@ def _get_redis_cluster_kwargs(client=None):
) # Needed for sync clusters and IAM detection
available_args.append("gcp_service_account")
available_args.append("gcp_ssl_ca_certs")
available_args.append("azure_redis_ad_token")
available_args.append("azure_client_id")
available_args.append("azure_tenant_id")
available_args.append("azure_client_secret")
available_args.append("max_connections")
return available_args
@ -155,6 +166,125 @@ def create_gcp_iam_redis_connect_func(
return iam_connect
def _build_azure_credential(
azure_client_id: Optional[str] = None,
azure_tenant_id: Optional[str] = None,
azure_client_secret: Optional[str] = None,
):
"""
Build a long-lived Azure credential object.
Azure SDK credentials cache tokens internally and handle expiry/refresh
transparently, so this should be called once and the result reused.
"""
try:
from azure.identity import (
ClientSecretCredential,
DefaultAzureCredential,
ManagedIdentityCredential,
)
except ImportError:
raise ImportError(
"azure-identity is required for Azure AD Redis authentication. "
"Install it with: pip install azure-identity"
)
_client_id = azure_client_id or os.environ.get("AZURE_CLIENT_ID")
_tenant_id = azure_tenant_id or os.environ.get("AZURE_TENANT_ID")
_client_secret = azure_client_secret or os.environ.get("AZURE_CLIENT_SECRET")
if _client_id and _tenant_id and _client_secret:
return ClientSecretCredential(
client_id=_client_id,
tenant_id=_tenant_id,
client_secret=_client_secret,
)
elif _client_id:
return ManagedIdentityCredential(client_id=_client_id)
else:
return DefaultAzureCredential()
def _generate_azure_ad_redis_token(
azure_client_id: Optional[str] = None,
azure_tenant_id: Optional[str] = None,
azure_client_secret: Optional[str] = None,
) -> str:
"""
One-shot helper that builds a credential and fetches a single Azure AD
access token for Redis. Each call rebuilds the credential and performs a
network round-trip, so it should not be used in steady-state Redis flows
— the sync (``create_azure_ad_redis_connect_func``) and async paths
(``AzureADCredentialProvider``) keep the credential alive across
connections so the Azure SDK's internal cache + silent refresh apply.
"""
credential = _build_azure_credential(
azure_client_id=azure_client_id,
azure_tenant_id=azure_tenant_id,
azure_client_secret=azure_client_secret,
)
token = credential.get_token(AZURE_REDIS_SCOPE)
return token.token
def create_azure_ad_redis_connect_func(
azure_client_id: Optional[str] = None,
azure_tenant_id: Optional[str] = None,
azure_client_secret: Optional[str] = None,
) -> Callable:
"""
Creates a custom Redis connection function for Azure AD authentication.
Used for sync Redis clients. The credential is created once (captured by the
closure) and reused across connections — the Azure SDK handles token caching
and silent renewal internally. Only ``get_token`` is called per connection.
"""
credential = _build_azure_credential(
azure_client_id=azure_client_id,
azure_tenant_id=azure_tenant_id,
azure_client_secret=azure_client_secret,
)
def ad_connect(self):
"""Initialize the connection and authenticate using Azure AD"""
from redis.exceptions import (
AuthenticationError,
AuthenticationWrongNumberOfArgsError,
)
from redis.utils import str_if_bytes
self._parser.on_connect(self)
access_token = credential.get_token(AZURE_REDIS_SCOPE).token
# Only include username when explicitly set — sending AUTH "" <token>
# is invalid for most ACL-configured Azure Redis instances.
username = os.environ.get("REDIS_USERNAME", "")
if username:
auth_args = (username, access_token)
else:
auth_args = (access_token,)
self.send_command("AUTH", *auth_args, check_health=False)
try:
auth_response = self.read_response()
except AuthenticationWrongNumberOfArgsError:
# Fallback: try with just the token (Redis < 6 / no ACL)
self.send_command("AUTH", access_token, check_health=False)
auth_response = self.read_response()
if str_if_bytes(auth_response) != "OK":
raise AuthenticationError("Azure AD authentication failed for Redis")
# Attach the live credential object so async paths can wrap it in
# AzureADCredentialProvider for refresh-aware token retrieval. The raw
# client_id/tenant_id/secret are intentionally NOT exposed here — the
# credential closure already holds them.
ad_connect._azure_credential = credential # type: ignore[attr-defined]
return ad_connect
def get_redis_url_from_environment():
if "REDIS_URL" in os.environ:
return os.environ["REDIS_URL"]
@ -179,7 +309,7 @@ def get_redis_url_from_environment():
return f"{redis_protocol}://{auth_part}{os.environ['REDIS_HOST']}:{os.environ['REDIS_PORT']}"
def _get_redis_client_logic(**env_overrides):
def _get_redis_client_logic(**env_overrides): # noqa: PLR0915
"""
Common functionality across sync + async redis client implementations
"""
@ -253,6 +383,52 @@ def _get_redis_client_logic(**env_overrides):
if _gcp_ssl_ca_certs and redis_kwargs.get("ssl", False):
redis_kwargs["ssl_ca_certs"] = _gcp_ssl_ca_certs
# Handle Azure AD authentication (after GCP IAM block)
_azure_redis_ad_token = redis_kwargs.get("azure_redis_ad_token") or get_secret(
"REDIS_AZURE_AD_TOKEN"
)
_azure_ad_enabled = (
_azure_redis_ad_token is not None
and str(_azure_redis_ad_token).lower() == "true"
)
if _azure_ad_enabled and _gcp_service_account is not None:
verbose_logger.warning(
"Both GCP IAM (gcp_service_account) and Azure AD (azure_redis_ad_token) are configured for Redis. "
"Using GCP IAM. Remove one to avoid misconfiguration."
)
if _azure_ad_enabled and _gcp_service_account is None:
_azure_client_id = redis_kwargs.get("azure_client_id") or get_secret_str(
"AZURE_CLIENT_ID"
)
_azure_tenant_id = redis_kwargs.get("azure_tenant_id") or get_secret_str(
"AZURE_TENANT_ID"
)
_azure_client_secret = redis_kwargs.get(
"azure_client_secret"
) or get_secret_str("AZURE_CLIENT_SECRET")
verbose_logger.debug("Setting up Azure AD authentication for Redis.")
redis_kwargs["redis_connect_func"] = create_azure_ad_redis_connect_func(
azure_client_id=_azure_client_id,
azure_tenant_id=_azure_tenant_id,
azure_client_secret=_azure_client_secret,
)
# Marker for async paths to detect Azure AD auth. The live credential
# object is attached separately as `_azure_credential` by
# `create_azure_ad_redis_connect_func`; the raw client_id/tenant_id/secret
# are intentionally NOT exposed on the function to avoid leaking
# credentials via inspection or logging.
redis_kwargs["redis_connect_func"]._azure_redis_ad_token = True # type: ignore[attr-defined]
# Always remove Azure-specific kwargs that shouldn't be passed to Redis client
redis_kwargs.pop("azure_redis_ad_token", None)
redis_kwargs.pop("azure_client_id", None)
redis_kwargs.pop("azure_tenant_id", None)
redis_kwargs.pop("azure_client_secret", None)
if "url" in redis_kwargs and redis_kwargs["url"] is not None:
# Only strip host/port/db/password when not routing to a cluster.
# When startup_nodes is also present the cluster path takes priority and
@ -373,7 +549,7 @@ def get_redis_client(**env_overrides):
return redis.Redis(**redis_kwargs)
def get_redis_async_client(
def get_redis_async_client( # noqa: PLR0915
connection_pool: Optional[async_redis.BlockingConnectionPool] = None,
**env_overrides,
) -> Union[async_redis.Redis, async_redis.RedisCluster]:
@ -398,6 +574,14 @@ def get_redis_async_client(
cluster_kwargs["credential_provider"] = GCPIAMCredentialProvider(
redis_connect_func._gcp_service_account
)
# Handle Azure AD authentication for async clusters via CredentialProvider
# so the credential's internal cache + silent refresh runs per connection
# (mirrors GCP IAM above; avoids static-token-baked-in-pool expiry).
elif redis_connect_func and hasattr(redis_connect_func, "_azure_credential"):
cluster_kwargs["credential_provider"] = AzureADCredentialProvider(
redis_connect_func._azure_credential,
username=os.environ.get("REDIS_USERNAME") or None,
)
new_startup_nodes: List[ClusterNode] = []
@ -431,6 +615,22 @@ def get_redis_async_client(
# Check for Redis Sentinel
if "sentinel_nodes" in redis_kwargs and "service_name" in redis_kwargs:
return _init_async_redis_sentinel(redis_kwargs)
# Wrap GCP / Azure AD auth in a CredentialProvider for the standard async
# Redis client. The async client doesn't support redis_connect_func, but it
# does honour credential_provider — which is called per connection, so the
# underlying SDK can refresh tokens silently before they expire.
redis_connect_func = redis_kwargs.pop("redis_connect_func", None)
if redis_connect_func and hasattr(redis_connect_func, "_azure_credential"):
redis_kwargs["credential_provider"] = AzureADCredentialProvider(
redis_connect_func._azure_credential,
username=os.environ.get("REDIS_USERNAME") or None,
)
elif redis_connect_func and hasattr(redis_connect_func, "_gcp_service_account"):
redis_kwargs["credential_provider"] = GCPIAMCredentialProvider(
redis_connect_func._gcp_service_account
)
_pretty_print_redis_config(redis_kwargs=redis_kwargs)
if connection_pool is not None:
@ -464,6 +664,21 @@ def get_redis_connection_pool(
redis_kwargs["max_connections"],
)
return async_redis.BlockingConnectionPool.from_url(**pool_kwargs)
# Wrap GCP / Azure AD auth in a CredentialProvider so pool-managed
# connections re-fetch tokens via the SDK's internal cache + silent refresh
# rather than reusing a single token captured at pool creation.
redis_connect_func = redis_kwargs.pop("redis_connect_func", None)
if redis_connect_func and hasattr(redis_connect_func, "_azure_credential"):
redis_kwargs["credential_provider"] = AzureADCredentialProvider(
redis_connect_func._azure_credential,
username=os.environ.get("REDIS_USERNAME") or None,
)
elif redis_connect_func and hasattr(redis_connect_func, "_gcp_service_account"):
redis_kwargs["credential_provider"] = GCPIAMCredentialProvider(
redis_connect_func._gcp_service_account
)
connection_class = async_redis.Connection
if "ssl" in redis_kwargs:
connection_class = async_redis.SSLConnection

35
litellm/_redis_credential_provider.py
View file

@ -1,10 +1,13 @@
import asyncio
import threading
import time
from typing import Dict, Tuple
from typing import Any, Dict, Optional, Tuple, Union
from redis.credentials import CredentialProvider # type: ignore[attr-defined]
# Azure AD scope for Redis Cache for Azure.
AZURE_REDIS_SCOPE = "https://redis.azure.com/.default"
# GCP IAM tokens are valid for 1 hour. Cache for 55 minutes to refresh before expiry.
_GCP_IAM_TOKEN_TTL_SECONDS = 3300
@ -101,3 +104,33 @@ class GCPIAMCredentialProvider(CredentialProvider):
_get_cached_gcp_iam_token, self._gcp_service_account
)
return (token,)
class AzureADCredentialProvider(CredentialProvider):
"""
redis.credentials.CredentialProvider implementation that supplies Azure AD
tokens for Redis authentication.
Wraps an azure-identity credential object so the Azure SDK's internal token
cache and silent refresh are honoured on every Redis connection. This avoids
the static-token-baked-in-pool issue where pool-managed connections would
fail authentication after the initial token expired (~1 hour TTL).
"""
def __init__(self, credential: Any, username: Optional[str] = None) -> None:
self._credential = credential
self._username = username
def get_credentials(self) -> Union[Tuple[str], Tuple[str, str]]:
token = self._credential.get_token(AZURE_REDIS_SCOPE).token
if self._username:
return (self._username, token)
return (token,)
async def get_credentials_async(self) -> Union[Tuple[str], Tuple[str, str]]:
token_obj = await asyncio.to_thread(
self._credential.get_token, AZURE_REDIS_SCOPE
)
if self._username:
return (self._username, token_obj.token)
return (token_obj.token,)

11
litellm/constants.py
View file

@ -161,6 +161,11 @@ MCP_STDIO_ALLOWED_COMMANDS: frozenset = frozenset(
| (set(_MCP_STDIO_EXTRA_COMMANDS.split(",")) - {""})
)
# MCP OAuth2 Token Exchange (OBO) Defaults
MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE = int(
os.getenv("MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE", "500")
)
LITELLM_UI_ALLOW_HEADERS = [
"x-litellm-semantic-filter",
"x-litellm-semantic-filter-tools",
@ -1457,6 +1462,12 @@ KEY_ROTATION_JOB_NAME = "litellm_key_rotation_job"
EXPIRED_UI_SESSION_KEY_CLEANUP_JOB_NAME = "litellm_expired_ui_session_key_cleanup_job"
SPEND_LOG_RUN_LOOPS = int(os.getenv("SPEND_LOG_RUN_LOOPS", 500))
SPEND_LOG_CLEANUP_BATCH_SIZE = int(os.getenv("SPEND_LOG_CLEANUP_BATCH_SIZE", 1000))
SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES = int(
os.getenv("SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES", 3)
)
SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS = float(
os.getenv("SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS", 0.5)
)
SPEND_LOG_QUEUE_SIZE_THRESHOLD = int(os.getenv("SPEND_LOG_QUEUE_SIZE_THRESHOLD", 100))
SPEND_LOG_QUEUE_POLL_INTERVAL = float(os.getenv("SPEND_LOG_QUEUE_POLL_INTERVAL", 2.0))
SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE = int(

2
litellm/experimental_mcp_client/client.py
View file

@ -366,6 +366,8 @@ class MCPClient:
headers["Authorization"] = f"Bearer {self._mcp_auth_value}"
elif self.auth_type == MCPAuth.token:
headers["Authorization"] = f"token {self._mcp_auth_value}"
elif self.auth_type == MCPAuth.oauth2_token_exchange:
headers["Authorization"] = f"Bearer {self._mcp_auth_value}"
elif isinstance(self._mcp_auth_value, dict):
headers.update(self._mcp_auth_value)
# Note: aws_sigv4 auth is not handled here — SigV4 requires per-request

139
litellm/integrations/azure_sentinel/azure_sentinel.py
View file

@ -14,16 +14,18 @@ For batching specific details see CustomBatchLogger class
import asyncio
import os
import time
import traceback
from typing import List, Optional
from typing import List, Optional, Union
from litellm._logging import verbose_logger
from litellm.integrations.custom_batch_logger import CustomBatchLogger
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
)
from litellm.types.utils import StandardLoggingPayload
from litellm.types.utils import StandardAuditLogPayload, StandardLoggingPayload
class AzureSentinelLogger(CustomBatchLogger):
@ -39,6 +41,7 @@ class AzureSentinelLogger(CustomBatchLogger):
tenant_id: Optional[str] = None,
client_id: Optional[str] = None,
client_secret: Optional[str] = None,
audit_stream_name: Optional[str] = None,
**kwargs,
):
"""
@ -57,57 +60,77 @@ class AzureSentinelLogger(CustomBatchLogger):
If not provided, will use AZURE_SENTINEL_CLIENT_ID or AZURE_CLIENT_ID env var.
client_secret (str, optional): Azure Client Secret for OAuth2 authentication.
If not provided, will use AZURE_SENTINEL_CLIENT_SECRET or AZURE_CLIENT_SECRET env var.
audit_stream_name (str, optional): Stream name from DCR for audit logs.
If not provided, audit logs use the standard stream name.
"""
self.async_httpx_client = get_async_httpx_client(
llm_provider=httpxSpecialProvider.LoggingCallback
)
self.dcr_immutable_id = dcr_immutable_id or os.getenv(
resolved_dcr_immutable_id = dcr_immutable_id or os.getenv(
"AZURE_SENTINEL_DCR_IMMUTABLE_ID"
)
self.stream_name = stream_name or os.getenv(
"AZURE_SENTINEL_STREAM_NAME", "Custom-LiteLLM"
resolved_stream_name = (
stream_name or os.getenv("AZURE_SENTINEL_STREAM_NAME") or "Custom-LiteLLM"
)
self.endpoint = endpoint or os.getenv("AZURE_SENTINEL_ENDPOINT")
self.tenant_id = (
resolved_audit_stream_name = audit_stream_name or resolved_stream_name
resolved_endpoint = endpoint or os.getenv("AZURE_SENTINEL_ENDPOINT")
resolved_tenant_id = (
tenant_id
or os.getenv("AZURE_SENTINEL_TENANT_ID")
or os.getenv("AZURE_TENANT_ID")
)
self.client_id = (
resolved_client_id = (
client_id
or os.getenv("AZURE_SENTINEL_CLIENT_ID")
or os.getenv("AZURE_CLIENT_ID")
)
self.client_secret = (
resolved_client_secret = (
client_secret
or os.getenv("AZURE_SENTINEL_CLIENT_SECRET")
or os.getenv("AZURE_CLIENT_SECRET")
)
if not self.dcr_immutable_id:
if not resolved_dcr_immutable_id:
raise ValueError(
"AZURE_SENTINEL_DCR_IMMUTABLE_ID is required. Set it as an environment variable or pass dcr_immutable_id parameter."
)
if not self.endpoint:
if not resolved_endpoint:
raise ValueError(
"AZURE_SENTINEL_ENDPOINT is required. Set it as an environment variable or pass endpoint parameter."
)
if not self.tenant_id:
if not resolved_tenant_id:
raise ValueError(
"AZURE_SENTINEL_TENANT_ID or AZURE_TENANT_ID is required. Set it as an environment variable or pass tenant_id parameter."
)
if not self.client_id:
if not resolved_client_id:
raise ValueError(
"AZURE_SENTINEL_CLIENT_ID or AZURE_CLIENT_ID is required. Set it as an environment variable or pass client_id parameter."
)
if not self.client_secret:
if not resolved_client_secret:
raise ValueError(
"AZURE_SENTINEL_CLIENT_SECRET or AZURE_CLIENT_SECRET is required. Set it as an environment variable or pass client_secret parameter."
)
self.dcr_immutable_id = resolved_dcr_immutable_id
self.stream_name = resolved_stream_name
self.audit_stream_name = resolved_audit_stream_name
self.endpoint = resolved_endpoint
self.tenant_id = resolved_tenant_id
self.client_id = resolved_client_id
self.client_secret = resolved_client_secret
# Build API endpoint: {Endpoint}/dataCollectionRules/{DCR Immutable ID}/streams/{Stream Name}?api-version=2023-01-01
self.api_endpoint = f"{self.endpoint.rstrip('/')}/dataCollectionRules/{self.dcr_immutable_id}/streams/{self.stream_name}?api-version=2023-01-01"
self.api_endpoint = self._build_api_endpoint(
endpoint=resolved_endpoint,
dcr_immutable_id=resolved_dcr_immutable_id,
stream_name=resolved_stream_name,
)
self.audit_api_endpoint = self._build_api_endpoint(
endpoint=resolved_endpoint,
dcr_immutable_id=resolved_dcr_immutable_id,
stream_name=resolved_audit_stream_name,
)
# OAuth2 scope for Azure Monitor
self.oauth_scope = "https://monitor.azure.com/.default"
@ -118,6 +141,13 @@ class AzureSentinelLogger(CustomBatchLogger):
super().__init__(**kwargs, flush_lock=self.flush_lock)
asyncio.create_task(self.periodic_flush())
self.log_queue: List[StandardLoggingPayload] = []
self.audit_log_queue: List[StandardAuditLogPayload] = []
@staticmethod
def _build_api_endpoint(
endpoint: str, dcr_immutable_id: str, stream_name: str
) -> str:
return f"{endpoint.rstrip('/')}/dataCollectionRules/{dcr_immutable_id}/streams/{stream_name}?api-version=2023-01-01"
async def _get_oauth_token(self) -> str:
"""
@ -126,9 +156,6 @@ class AzureSentinelLogger(CustomBatchLogger):
Returns:
Bearer token string
"""
# Check if we have a valid cached token
import time
if (
self.oauth_token
and self.oauth_token_expires_at
@ -170,9 +197,6 @@ class AzureSentinelLogger(CustomBatchLogger):
if not self.oauth_token:
raise Exception("OAuth2 token response did not contain access_token")
# Cache token expiry time
import time
self.oauth_token_expires_at = time.time() + expires_in
return self.oauth_token
@ -246,6 +270,34 @@ class AzureSentinelLogger(CustomBatchLogger):
)
pass
async def async_log_audit_log_event(
self, audit_log: StandardAuditLogPayload
) -> None:
"""
Async log LiteLLM audit log events to Azure Sentinel.
Audit logs are queued separately from standard LLM logs so mixed callback
usage never sends schema-mismatched records in the same ingestion batch.
"""
try:
verbose_logger.debug(
"Azure Sentinel: Logging audit event id=%s action=%s table=%s",
audit_log.get("id"),
audit_log.get("action"),
audit_log.get("table_name"),
)
self.audit_log_queue.append(audit_log)
if len(self.audit_log_queue) >= self.batch_size:
await self.async_send_audit_batch()
except Exception as e:
verbose_logger.exception(
f"Azure Sentinel Audit Log Layer Error - {str(e)}\n{traceback.format_exc()}"
)
pass
async def async_send_batch(self):
"""
Sends the batch of logs to Azure Monitor Logs Ingestion API
@ -253,22 +305,42 @@ class AzureSentinelLogger(CustomBatchLogger):
Raises:
Raises a NON Blocking verbose_logger.exception if an error occurs
"""
await self._async_send_batch_to_api(
log_queue=self.log_queue,
api_endpoint=self.api_endpoint,
log_type="logs",
)
async def async_send_audit_batch(self):
"""
Sends the batch of audit logs to Azure Monitor Logs Ingestion API
"""
await self._async_send_batch_to_api(
log_queue=self.audit_log_queue,
api_endpoint=self.audit_api_endpoint,
log_type="audit logs",
)
async def _async_send_batch_to_api(
self,
log_queue: List[Union[StandardLoggingPayload, StandardAuditLogPayload]],
api_endpoint: str,
log_type: str,
) -> None:
try:
if not self.log_queue:
if not log_queue:
return
verbose_logger.debug(
"Azure Sentinel - about to flush %s events", len(self.log_queue)
"Azure Sentinel - about to flush %s %s", len(log_queue), log_type
)
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
# Get OAuth2 token
bearer_token = await self._get_oauth_token()
# Convert log queue to JSON array format expected by Logs Ingestion API
# Each log entry should be a JSON object in the array
body = safe_dumps(self.log_queue)
body = safe_dumps(log_queue)
# Set headers for Logs Ingestion API
headers = {
@ -278,7 +350,7 @@ class AzureSentinelLogger(CustomBatchLogger):
# Send the request
response = await self.async_httpx_client.post(
url=self.api_endpoint, data=body.encode("utf-8"), headers=headers
url=api_endpoint, data=body.encode("utf-8"), headers=headers
)
if response.status_code not in [200, 204]:
@ -301,4 +373,15 @@ class AzureSentinelLogger(CustomBatchLogger):
f"Azure Sentinel Error sending batch API - {str(e)}\n{traceback.format_exc()}"
)
finally:
self.log_queue.clear()
log_queue.clear()
async def flush_queue(self):
if self.flush_lock is None:
return
async with self.flush_lock:
if self.log_queue:
await self.async_send_batch()
if self.audit_log_queue:
await self.async_send_audit_batch()
self.last_flush_time = time.time()

261
litellm/integrations/opentelemetry.py
View file

@ -57,6 +57,17 @@ LITELLM_PROXY_REQUEST_SPAN_NAME = "Received Proxy Server Request"
RAW_REQUEST_SPAN_NAME = "raw_gen_ai_request"
LITELLM_REQUEST_SPAN_NAME = "litellm_request"
CAPTURE_MODE_NO_CONTENT = "NO_CONTENT"
CAPTURE_MODE_SPAN_ONLY = "SPAN_ONLY"
CAPTURE_MODE_EVENT_ONLY = "EVENT_ONLY"
CAPTURE_MODE_SPAN_AND_EVENT = "SPAN_AND_EVENT"
_VALID_CAPTURE_MODES = {
CAPTURE_MODE_NO_CONTENT,
CAPTURE_MODE_SPAN_ONLY,
CAPTURE_MODE_EVENT_ONLY,
CAPTURE_MODE_SPAN_AND_EVENT,
}
@dataclass
class OpenTelemetryConfig:
@ -71,6 +82,9 @@ class OpenTelemetryConfig:
ignore_context_propagation: Optional[bool] = None
# When True, create a private TracerProvider instead of reusing or setting the global one.
skip_set_global: bool = False
# Programmatic override for OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT.
# One of NO_CONTENT, SPAN_ONLY, EVENT_ONLY, SPAN_AND_EVENT (or "true" as legacy alias).
capture_message_content: Optional[str] = None
def __post_init__(self) -> None:
# If endpoint is specified but exporter is still the default "console",
@ -182,6 +196,9 @@ class OpenTelemetry(CustomLogger):
super().__init__(**kwargs)
self._init_metrics(meter_provider)
self._init_logs(logger_provider)
# Sample env-var / config / message_logging at init so subsequent
# _capture_in_span / _capture_in_event calls are deterministic.
self._capture_mode_cached = self._compute_capture_mode_from_init_state()
self._init_otel_logger_on_litellm_proxy()
@staticmethod
@ -306,6 +323,62 @@ class OpenTelemetry(CustomLogger):
hasattr(self, "callback_name") and self.callback_name == "langfuse_otel"
)
def _compute_capture_mode_from_init_state(self) -> Optional[str]:
"""Sample explicit settings at init. Returns the resolved mode or
None if nothing explicit is set (in which case the legacy
``self.message_logging`` flag is consulted dynamically per request).
``"true"``/``"1"`` map to ``EVENT_ONLY`` per the contrib convention.
``"false"``/``"0"`` map to ``NO_CONTENT``.
Unknown values are ignored.
"""
explicit = self.config.capture_message_content or os.getenv(
"OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT"
)
if not explicit:
return None
normalized = explicit.upper()
if normalized in ("TRUE", "1"):
return CAPTURE_MODE_EVENT_ONLY
if normalized in ("FALSE", "0"):
return CAPTURE_MODE_NO_CONTENT
if normalized in _VALID_CAPTURE_MODES:
return normalized
return None
def _resolve_capture_mode(self) -> str:
"""Return the active capture mode for this request.
Precedence:
1. ``litellm.turn_off_message_logging=True`` forces ``NO_CONTENT``
(kill-switch checked dynamically).
2. Explicit setting sampled at init from
``OpenTelemetryConfig.capture_message_content`` or
``OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT``.
3. Legacy ``self.message_logging`` (checked dynamically).
"""
if litellm.turn_off_message_logging:
return CAPTURE_MODE_NO_CONTENT
if self._capture_mode_cached is not None:
return self._capture_mode_cached
return (
CAPTURE_MODE_SPAN_AND_EVENT
if self.message_logging
else CAPTURE_MODE_NO_CONTENT
)
def _capture_in_span(self) -> bool:
return self._resolve_capture_mode() in (
CAPTURE_MODE_SPAN_ONLY,
CAPTURE_MODE_SPAN_AND_EVENT,
)
def _capture_in_event(self) -> bool:
return self._resolve_capture_mode() in (
CAPTURE_MODE_EVENT_ONLY,
CAPTURE_MODE_SPAN_AND_EVENT,
)
def _init_tracing(self, tracer_provider):
from opentelemetry import trace
from opentelemetry.sdk.trace import TracerProvider
@ -825,8 +898,7 @@ class OpenTelemetry(CustomLogger):
from opentelemetry import trace
from opentelemetry.trace import Status, StatusCode
# only log raw LLM request/response if message_logging is on and not globally turned off
if litellm.turn_off_message_logging or not self.message_logging:
if not self._capture_in_span():
return
litellm_params = kwargs.get("litellm_params", {})
@ -1117,9 +1189,14 @@ class OpenTelemetry(CustomLogger):
}
if role == "tool" and msg.get("id"):
attrs["id"] = msg["id"]
if self.message_logging and msg.get("content"):
capture_event_content = self._capture_in_event()
if capture_event_content and msg.get("content"):
attrs["gen_ai.prompt"] = msg["content"]
body = msg.copy()
if not capture_event_content:
body.pop("content", None)
log_record = SdkLogRecord(
timestamp=self._to_ns(datetime.now()),
trace_id=parent_ctx.trace_id,
@ -1127,7 +1204,7 @@ class OpenTelemetry(CustomLogger):
trace_flags=parent_ctx.trace_flags,
severity_number=SeverityNumber.INFO,
severity_text="INFO",
body=msg.copy(),
body=body,
attributes=attrs,
)
otel_logger.emit(log_record)
@ -1141,14 +1218,15 @@ class OpenTelemetry(CustomLogger):
"finish_reason": choice.get("finish_reason"),
}
body_msg = choice.get("message", {})
if self.message_logging and body_msg.get("content"):
capture_event_content = self._capture_in_event()
if capture_event_content and body_msg.get("content"):
attrs["message.content"] = body_msg["content"]
body = {
"index": idx,
"finish_reason": choice.get("finish_reason"),
"message": {"role": body_msg.get("role", "assistant")},
}
if self.message_logging and body_msg.get("content"):
if capture_event_content and body_msg.get("content"):
body["message"]["content"] = body_msg["content"]
log_record = SdkLogRecord(
@ -1674,9 +1752,7 @@ class OpenTelemetry(CustomLogger):
########## LLM Request Medssages / tools / content Attributes ###########
#########################################################################
if litellm.turn_off_message_logging is True:
return
if self.message_logging is not True:
if not self._capture_in_span():
return
if optional_params.get("tools"):
@ -1695,17 +1771,41 @@ class OpenTelemetry(CustomLogger):
value=safe_dumps(transformed_messages),
)
if kwargs.get("system_instructions"):
transformed_system_instructions = (
self._transform_messages_to_otel_semantic_conventions(
kwargs.get("system_instructions")
# Coalesce the different kwarg names that carry the system
# prompt depending on the call path:
# - "system_instructions" — Vertex AI Gemini chat-completion
# - "instructions" — OpenAI Responses API
# - "system" — Anthropic Messages API
# Use `is not None` rather than truthiness to avoid falsy
# values (e.g. []) falling through to the wrong kwarg.
system_instructions = (
kwargs.get("system_instructions")
if kwargs.get("system_instructions") is not None
else (
kwargs.get("instructions")
if kwargs.get("instructions") is not None
else kwargs.get("system")
)
)
if system_instructions:
if isinstance(system_instructions, str):
# Plain text system prompt — no transformation needed
self.safe_set_attribute(
span=span,
key=SpanAttributes.GEN_AI_SYSTEM_INSTRUCTIONS.value,
value=system_instructions,
)
else:
transformed_system_instructions = (
self._transform_messages_to_otel_semantic_conventions(
system_instructions
)
)
self.safe_set_attribute(
span=span,
key=SpanAttributes.GEN_AI_SYSTEM_INSTRUCTIONS.value,
value=safe_dumps(transformed_system_instructions),
)
)
self.safe_set_attribute(
span=span,
key=SpanAttributes.GEN_AI_SYSTEM_INSTRUCTIONS.value,
value=safe_dumps(transformed_system_instructions),
)
self.safe_set_attribute(
span=span,
@ -1764,6 +1864,57 @@ class OpenTelemetry(CustomLogger):
value=value,
)
elif response_obj.get("output"):
# Responses API: ResponsesAPIResponse has an "output"
# list instead of "choices". Each item with
# type="message" contains a "content" list of
# OutputText objects (type="output_text").
output_items = response_obj.get("output")
output_messages = self._transform_responses_api_output_to_otel(
output_items
)
if output_messages:
self.safe_set_attribute(
span=span,
key=SpanAttributes.GEN_AI_OUTPUT_MESSAGES.value,
value=safe_dumps(output_messages),
)
# Emit per-tool-call span attributes (parity with
# the choices branch that calls _tool_calls_kv_pair).
# Convert Responses API function_call items to the
# ChatCompletionMessageToolCall format expected by
# _tool_calls_kv_pair.
tool_calls = []
for out_item in output_items:
item_d = self._to_dict(out_item)
if item_d and item_d.get("type") == "function_call":
tool_calls.append(
{
"function": {
"name": item_d.get("name", ""),
"arguments": item_d.get("arguments", ""),
}
}
)
if tool_calls:
kv_pairs = OpenTelemetry._tool_calls_kv_pair(tool_calls) # type: ignore
for key, value in kv_pairs.items():
self.safe_set_attribute(
span=span,
key=key,
value=value,
)
# Extract finish reason from ResponsesAPIResponse.status
status = response_obj.get("status")
if status:
self.safe_set_attribute(
span=span,
key=SpanAttributes.GEN_AI_RESPONSE_FINISH_REASONS.value,
value=safe_dumps([status]),
)
except Exception as e:
self.handle_callback_failure(
callback_name=self.callback_name or "opentelemetry"
@ -1859,6 +2010,78 @@ class OpenTelemetry(CustomLogger):
transformed.append(transformed_msg)
return transformed
@staticmethod
def _to_dict(obj) -> Optional[dict]:
"""Normalize an object to a plain dict.
Handles three forms that appear in practice:
1. Plain ``dict`` — returned as-is.
2. LiteLLM's ``BaseLiteLLMOpenAIResponseObject`` — exposes a
``.get()`` method that delegates to ``__dict__``.
3. Raw Pydantic v2 models from the ``openai`` SDK (e.g.
``ResponseOutputMessage``, ``ResponseOutputText``) — these do
**not** have ``.get()`` but do have ``.model_dump()``.
Returns ``None`` for anything else so callers can skip it.
"""
if isinstance(obj, dict):
return obj
if hasattr(obj, "get"):
# BaseLiteLLMOpenAIResponseObject duck-type
return obj # type: ignore[return-value]
if hasattr(obj, "model_dump"):
# Raw Pydantic v2 model (e.g. openai SDK types)
return obj.model_dump() # type: ignore[union-attr]
return None
def _transform_responses_api_output_to_otel(self, output: List) -> List[dict]:
"""
Transform Responses API output items into OTEL GenAI 1.38 format.
The Responses API returns output as a list of items, each with a
``type`` field. Message items (``type="message"``) contain a
``content`` list of ``OutputText`` objects with ``type="output_text"``
and ``text`` fields.
Items may be plain dicts, LiteLLM wrapper objects (with ``.get()``),
or raw Pydantic v2 models from the ``openai`` SDK (with
``.model_dump()``). We normalize each item to a dict via
``_to_dict`` before processing.
This method converts them to the same ``{"role": ..., "parts": [...]}``
format used by ``_transform_choices_to_otel_semantic_conventions``.
"""
transformed = []
for raw_item in output:
item = self._to_dict(raw_item)
if item is None:
continue
if item.get("type") == "message":
role = item.get("role", "assistant")
parts = []
for raw_content in item.get("content", []):
content = self._to_dict(raw_content)
if content is None:
continue
if content.get("type") == "output_text":
text = content.get("text", "")
if text:
parts.append({"type": "text", "content": text})
if parts:
transformed.append({"role": role, "parts": parts})
elif item.get("type") == "function_call":
# Surface tool calls from Responses API output
part: dict = {
"type": "tool_call",
"name": item.get("name", ""),
"arguments": item.get("arguments", ""),
}
if item.get("call_id"):
part["id"] = item["call_id"]
transformed.append({"role": "assistant", "parts": [part]})
return transformed
def set_raw_request_attributes(self, span: Span, kwargs, response_obj):
try:
# Only set provider-specific raw payload attributes on this span.

190
litellm/integrations/prometheus.py
View file

@ -25,6 +25,9 @@ from typing import (
import litellm
from litellm._logging import print_verbose, verbose_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.prometheus_helpers.bounded_prometheus_series_tracker import (
BoundedPrometheusSeriesTracker,
)
from litellm.integrations.prometheus_helpers import (
PrometheusLabelFactoryContext,
_get_cached_end_user_id_for_cost_tracking,
@ -81,6 +84,7 @@ class PrometheusLogger(CustomLogger):
if _custom_buckets is not None
else LATENCY_BUCKETS
)
self._bounded_prometheus_series_tracker = BoundedPrometheusSeriesTracker()
# Create metric factory functions
self._counter_factory = self._create_metric_factory(Counter)
@ -984,6 +988,40 @@ class PrometheusLogger(CustomLogger):
return filtered_labels
def _track_end_user_metric_series(
self,
metric: Any,
metric_name: DEFINED_PROMETHEUS_METRICS,
labels: Dict[str, Optional[str]],
) -> None:
"""
Cap the cardinality of metrics that include the ``end_user`` label.
Called *after* ``metric.labels(...).inc()/observe()`` so the emission is
recorded in prometheus-client's child map before any eviction runs.
Series that get evicted before the next scrape lose updates accrued
since the last scrape — this is inherent to any cardinality cap.
"""
labelnames = self.get_labels_for_metric(metric_name)
if UserAPIKeyLabelNames.END_USER.value not in labelnames:
return
if labels.get(UserAPIKeyLabelNames.END_USER.value) is None:
return
max_series = litellm.prometheus_end_user_metrics_max_series_per_metric
ttl_seconds = litellm.prometheus_end_user_metrics_ttl_seconds
if max_series is None and ttl_seconds is None:
return
self._bounded_prometheus_series_tracker.track_series(
metric=metric,
metric_name=metric_name,
label_values=tuple(labels.get(label) for label in labelnames),
max_series=max_series,
ttl_seconds=ttl_seconds,
cleanup_interval_seconds=litellm.prometheus_end_user_metrics_cleanup_interval_seconds,
)
def _inc_labeled_counter(
self,
counter: Any,
@ -998,6 +1036,7 @@ class PrometheusLogger(CustomLogger):
label_context=label_context,
)
counter.labels(**_labels).inc(amount)
self._track_end_user_metric_series(counter, metric_name, _labels)
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
# Define prometheus client
@ -1047,21 +1086,9 @@ class PrometheusLogger(CustomLogger):
output_tokens = standard_logging_payload["completion_tokens"]
tokens_used = standard_logging_payload["total_tokens"]
response_cost = standard_logging_payload["response_cost"]
_requester_metadata: Optional[dict] = standard_logging_payload["metadata"].get(
"requester_metadata"
combined_metadata = _get_combined_custom_metadata_from_standard_logging_payload(
standard_logging_payload=standard_logging_payload
)
user_api_key_auth_metadata: Optional[dict] = standard_logging_payload[
"metadata"
].get("user_api_key_auth_metadata")
spend_logs_metadata: Optional[dict] = standard_logging_payload["metadata"].get(
"spend_logs_metadata"
)
combined_metadata: Dict[str, Any] = {
**(_requester_metadata if _requester_metadata else {}),
**(user_api_key_auth_metadata if user_api_key_auth_metadata else {}),
**(spend_logs_metadata if spend_logs_metadata else {}),
}
if standard_logging_payload is not None and isinstance(
standard_logging_payload, dict
):
@ -1416,26 +1443,46 @@ class PrometheusLogger(CustomLogger):
)
remaining_tokens_variable_name = f"litellm-key-remaining-tokens-{model_group}"
remaining_requests = (
metadata.get(remaining_requests_variable_name, sys.maxsize) or sys.maxsize
remaining_requests = metadata.get(remaining_requests_variable_name)
if remaining_requests is None:
remaining_requests = sys.maxsize
remaining_tokens = metadata.get(remaining_tokens_variable_name)
if remaining_tokens is None:
remaining_tokens = sys.maxsize
enum_values = UserAPIKeyLabelValues(
hashed_api_key=user_api_key,
api_key_alias=user_api_key_alias,
model=model_group,
model_id=model_id,
custom_metadata_labels=get_custom_labels_from_metadata(
metadata=_get_combined_custom_metadata_from_standard_logging_payload(
standard_logging_payload=kwargs.get("standard_logging_object")
)
),
)
remaining_tokens = (
metadata.get(remaining_tokens_variable_name, sys.maxsize) or sys.maxsize
label_context = PrometheusLabelFactoryContext(enum_values)
requests_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
"litellm_remaining_api_key_requests_for_model"
),
enum_values=enum_values,
label_context=label_context,
)
self.litellm_remaining_api_key_requests_for_model.labels(**requests_labels).set(
remaining_requests
)
self.litellm_remaining_api_key_requests_for_model.labels(
_sanitize_prometheus_label_value(user_api_key),
_sanitize_prometheus_label_value(user_api_key_alias),
_sanitize_prometheus_label_value(model_group),
_sanitize_prometheus_label_value(model_id),
).set(remaining_requests)
self.litellm_remaining_api_key_tokens_for_model.labels(
_sanitize_prometheus_label_value(user_api_key),
_sanitize_prometheus_label_value(user_api_key_alias),
_sanitize_prometheus_label_value(model_group),
_sanitize_prometheus_label_value(model_id),
).set(remaining_tokens)
tokens_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
"litellm_remaining_api_key_tokens_for_model"
),
enum_values=enum_values,
label_context=label_context,
)
self.litellm_remaining_api_key_tokens_for_model.labels(**tokens_labels).set(
remaining_tokens
)
def _set_latency_metrics(
self,
@ -1471,6 +1518,11 @@ class PrometheusLogger(CustomLogger):
self.litellm_llm_api_time_to_first_token_metric.labels(
**_ttft_labels
).observe(time_to_first_token_seconds)
self._track_end_user_metric_series(
self.litellm_llm_api_time_to_first_token_metric,
"litellm_llm_api_time_to_first_token_metric",
_ttft_labels,
)
else:
verbose_logger.debug(
"Time to first token metric not emitted, stream option in model_parameters is not True"
@ -1491,6 +1543,11 @@ class PrometheusLogger(CustomLogger):
self.litellm_llm_api_latency_metric.labels(**_labels).observe(
api_call_total_time_seconds
)
self._track_end_user_metric_series(
self.litellm_llm_api_latency_metric,
"litellm_llm_api_latency_metric",
_labels,
)
# total request latency
total_time_seconds = self._safe_duration_seconds(
@ -1508,6 +1565,11 @@ class PrometheusLogger(CustomLogger):
self.litellm_request_total_latency_metric.labels(**_labels).observe(
total_time_seconds
)
self._track_end_user_metric_series(
self.litellm_request_total_latency_metric,
"litellm_request_total_latency_metric",
_labels,
)
# request queue time (time from arrival to processing start)
_litellm_params = kwargs.get("litellm_params", {}) or {}
@ -1525,6 +1587,11 @@ class PrometheusLogger(CustomLogger):
self.litellm_request_queue_time_metric.labels(**_labels).observe(
queue_time_seconds
)
self._track_end_user_metric_series(
self.litellm_request_queue_time_metric,
"litellm_request_queue_time_seconds",
_labels,
)
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
verbose_logger.debug(
@ -1561,18 +1628,27 @@ class PrometheusLogger(CustomLogger):
)
try:
self.litellm_llm_api_failed_requests_metric.labels(
_sanitize_prometheus_label_value(end_user_id),
_sanitize_prometheus_label_value(user_api_key),
_sanitize_prometheus_label_value(user_api_key_alias),
_sanitize_prometheus_label_value(model),
_sanitize_prometheus_label_value(user_api_team),
_sanitize_prometheus_label_value(user_api_team_alias),
_sanitize_prometheus_label_value(user_id),
_sanitize_prometheus_label_value(
standard_logging_payload.get("model_id", "")
enum_values = UserAPIKeyLabelValues(
end_user=end_user_id,
hashed_api_key=user_api_key,
api_key_alias=user_api_key_alias,
model=model,
team=user_api_team,
team_alias=user_api_team_alias,
user=user_id,
model_id=standard_logging_payload.get("model_id", ""),
custom_metadata_labels=get_custom_labels_from_metadata(
metadata=_get_combined_custom_metadata_from_standard_logging_payload(
standard_logging_payload=standard_logging_payload
)
),
).inc()
)
PrometheusLogger._inc_labeled_counter(
self,
self.litellm_llm_api_failed_requests_metric,
"litellm_llm_api_failed_requests_metric",
enum_values,
)
self.set_llm_deployment_failure_metrics(kwargs)
await self._set_org_budget_metrics_after_api_request(
org_id=user_api_key_org_id,
@ -3622,6 +3698,36 @@ def get_custom_labels_from_metadata(metadata: dict) -> Dict[str, str]:
return result
def _get_combined_custom_metadata_from_standard_logging_payload(
standard_logging_payload: Optional[dict],
) -> Dict[str, Any]:
"""
Combine the metadata sources that can supply custom Prometheus labels.
"""
if not isinstance(standard_logging_payload, dict):
return {}
standard_logging_metadata = standard_logging_payload.get("metadata") or {}
if not isinstance(standard_logging_metadata, dict):
return {}
requester_metadata = standard_logging_metadata.get("requester_metadata")
user_api_key_auth_metadata = standard_logging_metadata.get(
"user_api_key_auth_metadata"
)
spend_logs_metadata = standard_logging_metadata.get("spend_logs_metadata")
return {
**(requester_metadata if isinstance(requester_metadata, dict) else {}),
**(
user_api_key_auth_metadata
if isinstance(user_api_key_auth_metadata, dict)
else {}
),
**(spend_logs_metadata if isinstance(spend_logs_metadata, dict) else {}),
}
def _tag_matches_wildcard_configured_pattern(
tags: Sequence[str], configured_tag: str
) -> bool:

107
litellm/integrations/prometheus_helpers/bounded_prometheus_series_tracker.py Normal file
View file

@ -0,0 +1,107 @@
from __future__ import annotations
import time
from collections import OrderedDict
from threading import RLock
from typing import Any, Dict, Optional
class BoundedPrometheusSeriesTracker:
"""
Tracks Prometheus child series and removes stale/excess labelsets.
The tracker is label-agnostic: callers decide which series should be tracked
and pass the full label tuple used by the Prometheus metric.
"""
def __init__(self) -> None:
self._series: Dict[str, OrderedDict[tuple[Optional[str], ...], float]] = {}
self._last_ttl_cleanup: Dict[str, float] = {}
self.lock = RLock()
def track_series(
self,
metric: Any,
metric_name: str,
label_values: tuple[Optional[str], ...],
max_series: Optional[int],
ttl_seconds: Optional[float],
cleanup_interval_seconds: Optional[float],
) -> None:
if max_series is None and ttl_seconds is None:
return
now = time.monotonic()
with self.lock:
series = self._series.setdefault(metric_name, OrderedDict())
series[label_values] = now
series.move_to_end(label_values)
if ttl_seconds is not None and self._should_run_ttl_cleanup(
metric_name=metric_name,
now=now,
cleanup_interval_seconds=cleanup_interval_seconds,
):
expired_label_values = [
tracked_label_values
for tracked_label_values, last_seen in series.items()
if now - last_seen > ttl_seconds
]
for tracked_label_values in expired_label_values:
self._remove_metric_series(metric, series, tracked_label_values)
# max_series <= 0 is treated as "unlimited" so a misconfigured zero
# value cannot silently drop every emission for this metric.
if max_series is not None and max_series > 0:
while len(series) > max_series:
tracked_label_values = next(iter(series))
if not self._remove_metric_child(metric, tracked_label_values):
break
del series[tracked_label_values]
def _should_run_ttl_cleanup(
self,
metric_name: str,
now: float,
cleanup_interval_seconds: Optional[float],
) -> bool:
if cleanup_interval_seconds is None or cleanup_interval_seconds <= 0:
self._last_ttl_cleanup[metric_name] = now
return True
last_cleanup = self._last_ttl_cleanup.get(metric_name)
if last_cleanup is None or now - last_cleanup >= cleanup_interval_seconds:
self._last_ttl_cleanup[metric_name] = now
return True
return False
def _remove_metric_series(
self,
metric: Any,
series: OrderedDict[tuple[Optional[str], ...], float],
label_values: tuple[Optional[str], ...],
) -> None:
if self._remove_metric_child(metric, label_values):
series.pop(label_values, None)
@staticmethod
def _remove_metric_child(
metric: Any, label_values: tuple[Optional[str], ...]
) -> bool:
"""
Remove the Prometheus child for ``label_values`` and report whether the
tracker should commit the matching state change.
Returns ``True`` when the child is no longer present in Prometheus
(either it was just removed or it was already gone), and ``False`` when
``metric.remove()`` raised an unexpected error and the child likely
still exists.
"""
try:
metric.remove(*label_values)
return True
except KeyError:
return True
except (AttributeError, ValueError):
return False

100
litellm/integrations/s3_v2.py
View file

@ -16,6 +16,7 @@ from litellm._logging import print_verbose, verbose_logger
from litellm.constants import DEFAULT_S3_BATCH_SIZE, DEFAULT_S3_FLUSH_INTERVAL_SECONDS
from litellm.integrations.s3 import get_s3_object_key
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.custom_httpx.http_handler import (
_get_httpx_client,
@ -53,15 +54,25 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
s3_strip_base64_files: bool = False,
s3_use_key_prefix: bool = False,
s3_use_virtual_hosted_style: bool = False,
s3_callback_params_override: Optional[dict] = None,
**kwargs,
):
try:
verbose_logger.debug(
f"in init s3 logger - s3_callback_params {litellm.s3_callback_params}"
)
_masker = SensitiveDataMasker()
if s3_callback_params_override is not None:
verbose_logger.debug(
f"in init s3 logger (audit override) - "
f"{_masker.mask_dict(dict(s3_callback_params_override))}"
)
else:
verbose_logger.debug(
f"in init s3 logger - s3_callback_params "
f"{_masker.mask_dict(dict(litellm.s3_callback_params or {}))}"
)
# Initialize S3 params first to get the correct s3_verify value
self._init_s3_params(
params_source=s3_callback_params_override,
s3_bucket_name=s3_bucket_name,
s3_region_name=s3_region_name,
s3_api_version=s3_api_version,
@ -139,94 +150,85 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
s3_strip_base64_files: bool = False,
s3_use_key_prefix: bool = False,
s3_use_virtual_hosted_style: bool = False,
params_source: Optional[dict] = None,
):
"""
Initialize the s3 params for this logging callback
Initialize the s3 params for this logging callback. Reads from
`params_source` if given (e.g. `s3_audit_callback_params` for the
audit-log instance), otherwise falls back to `litellm.s3_callback_params`.
Resolves `os.environ/X` markers into a local dict; never mutates the source.
"""
litellm.s3_callback_params = litellm.s3_callback_params or {}
# read in .env variables - example os.environ/AWS_BUCKET_NAME
for key, value in litellm.s3_callback_params.items():
if isinstance(value, str) and value.startswith("os.environ/"):
litellm.s3_callback_params[key] = litellm.get_secret(value)
if params_source is None:
params_source = litellm.s3_callback_params or {}
params: dict = {
key: (
litellm.get_secret(value)
if isinstance(value, str) and value.startswith("os.environ/")
else value
)
for key, value in params_source.items()
}
self.s3_bucket_name = (
litellm.s3_callback_params.get("s3_bucket_name") or s3_bucket_name
)
self.s3_region_name = (
litellm.s3_callback_params.get("s3_region_name") or s3_region_name
)
self.s3_api_version = (
litellm.s3_callback_params.get("s3_api_version") or s3_api_version
)
self.s3_bucket_name = params.get("s3_bucket_name") or s3_bucket_name
self.s3_region_name = params.get("s3_region_name") or s3_region_name
self.s3_api_version = params.get("s3_api_version") or s3_api_version
self.s3_use_ssl = (
litellm.s3_callback_params.get("s3_use_ssl", True)
if litellm.s3_callback_params.get("s3_use_ssl") is not None
params.get("s3_use_ssl", True)
if params.get("s3_use_ssl") is not None
else s3_use_ssl
)
self.s3_verify = (
litellm.s3_callback_params.get("s3_verify")
if litellm.s3_callback_params.get("s3_verify") is not None
params.get("s3_verify")
if params.get("s3_verify") is not None
else s3_verify
)
self.s3_endpoint_url = (
litellm.s3_callback_params.get("s3_endpoint_url") or s3_endpoint_url
)
self.s3_endpoint_url = params.get("s3_endpoint_url") or s3_endpoint_url
self.s3_aws_access_key_id = (
litellm.s3_callback_params.get("s3_aws_access_key_id")
or s3_aws_access_key_id
params.get("s3_aws_access_key_id") or s3_aws_access_key_id
)
self.s3_aws_secret_access_key = (
litellm.s3_callback_params.get("s3_aws_secret_access_key")
or s3_aws_secret_access_key
params.get("s3_aws_secret_access_key") or s3_aws_secret_access_key
)
self.s3_aws_session_token = (
litellm.s3_callback_params.get("s3_aws_session_token")
or s3_aws_session_token
params.get("s3_aws_session_token") or s3_aws_session_token
)
self.s3_aws_session_name = (
litellm.s3_callback_params.get("s3_aws_session_name") or s3_aws_session_name
params.get("s3_aws_session_name") or s3_aws_session_name
)
self.s3_aws_profile_name = (
litellm.s3_callback_params.get("s3_aws_profile_name") or s3_aws_profile_name
params.get("s3_aws_profile_name") or s3_aws_profile_name
)
self.s3_aws_role_name = (
litellm.s3_callback_params.get("s3_aws_role_name") or s3_aws_role_name
)
self.s3_aws_role_name = params.get("s3_aws_role_name") or s3_aws_role_name
self.s3_aws_web_identity_token = (
litellm.s3_callback_params.get("s3_aws_web_identity_token")
or s3_aws_web_identity_token
params.get("s3_aws_web_identity_token") or s3_aws_web_identity_token
)
self.s3_aws_sts_endpoint = (
litellm.s3_callback_params.get("s3_aws_sts_endpoint") or s3_aws_sts_endpoint
params.get("s3_aws_sts_endpoint") or s3_aws_sts_endpoint
)
self.s3_config = litellm.s3_callback_params.get("s3_config") or s3_config
self.s3_path = litellm.s3_callback_params.get("s3_path") or s3_path
# done reading litellm.s3_callback_params
self.s3_config = params.get("s3_config") or s3_config
self.s3_path = params.get("s3_path") or s3_path
self.s3_use_team_prefix = (
bool(litellm.s3_callback_params.get("s3_use_team_prefix", False))
or s3_use_team_prefix
bool(params.get("s3_use_team_prefix", False)) or s3_use_team_prefix
)
self.s3_use_key_prefix = (
bool(litellm.s3_callback_params.get("s3_use_key_prefix", False))
or s3_use_key_prefix
bool(params.get("s3_use_key_prefix", False)) or s3_use_key_prefix
)
self.s3_strip_base64_files = (
bool(litellm.s3_callback_params.get("s3_strip_base64_files", False))
or s3_strip_base64_files
bool(params.get("s3_strip_base64_files", False)) or s3_strip_base64_files
)
self.s3_use_virtual_hosted_style = (
bool(litellm.s3_callback_params.get("s3_use_virtual_hosted_style", False))
bool(params.get("s3_use_virtual_hosted_style", False))
or s3_use_virtual_hosted_style
)

2
litellm/litellm_core_utils/custom_logger_registry.py
View file

@ -14,6 +14,7 @@ from litellm import _custom_logger_compatible_callbacks_literal
from litellm.integrations.agentops import AgentOps
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
from litellm.integrations.argilla import ArgillaLogger
from litellm.integrations.azure_sentinel.azure_sentinel import AzureSentinelLogger
from litellm.integrations.azure_storage.azure_storage import AzureBlobStorageLogger
from litellm.integrations.bitbucket import BitBucketPromptManager
from litellm.integrations.braintrust_logging import BraintrustLogger
@ -73,6 +74,7 @@ class CustomLoggerRegistry:
"opik": OpikLogger,
"argilla": ArgillaLogger,
"opentelemetry": OpenTelemetry,
"azure_sentinel": AzureSentinelLogger,
"azure_storage": AzureBlobStorageLogger,
"humanloop": HumanloopLogger,
# OTEL compatible loggers

12
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -621,6 +621,18 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915
or "https://integrate.api.nvidia.com/v1"
) # type: ignore
dynamic_api_key = api_key or get_secret_str("NVIDIA_NIM_API_KEY")
elif custom_llm_provider == "nvidia_riva":
# NVIDIA Riva is gRPC-based; api_base must be a host:port like
# `grpc.nvcf.nvidia.com:443` or `localhost:50051`. There is no
# public-default endpoint, so we do not fill one in here.
api_base = api_base or get_secret_str("NVIDIA_RIVA_API_BASE") # type: ignore
# Fall back to NVIDIA_NIM_API_KEY because users running both NVCF
# services typically reuse the same nvapi-* key.
dynamic_api_key = (
api_key
or get_secret_str("NVIDIA_RIVA_API_KEY")
or get_secret_str("NVIDIA_NIM_API_KEY")
)
elif custom_llm_provider == "cerebras":
api_base = (
api_base or get_secret("CEREBRAS_API_BASE") or "https://api.cerebras.ai/v1"

35
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -436,12 +436,21 @@ def update_messages_with_model_file_ids(
"""
Updates messages with model file ids.
For managed files (unified file IDs), uses model_file_id_mapping if it
resolves the id, otherwise decodes the base64-encoded unified file ID
and extracts the llm_output_file_id directly. Mirrors the Responses-API
sibling `update_responses_input_with_model_file_ids`.
model_file_id_mapping: Dict[str, Dict[str, str]] = {
"litellm_proxy/file_id": {
"model_id": "provider_file_id"
}
}
"""
from litellm.proxy.openai_files_endpoints.common_utils import (
_is_base64_encoded_unified_file_id,
convert_b64_uid_to_unified_uid,
)
for message in messages:
if message.get("role") == "user":
@ -450,7 +459,13 @@ def update_messages_with_model_file_ids(
if isinstance(content, str):
continue
for c in content:
if c["type"] == "file":
if not isinstance(c, dict):
# Content list items aren't always dicts. e.g.
# text_completion forwards a token-ids list/list-of-
# lists through this path. Skip non-dict items
# instead of indexing into them.
continue
if c.get("type") == "file":
file_object = cast(ChatCompletionFileObject, c)
file_object_file_field = file_object.get("file")
if not isinstance(file_object_file_field, dict):
@ -468,9 +483,23 @@ def update_messages_with_model_file_ids(
if file_id:
provider_file_id = (
model_file_id_mapping.get(file_id, {}).get(model_id)
or file_id
if model_file_id_mapping
else None
)
if (
not provider_file_id
and _is_base64_encoded_unified_file_id(file_id)
):
unified_file_id = convert_b64_uid_to_unified_uid(
file_id
)
if "llm_output_file_id," in unified_file_id:
provider_file_id = unified_file_id.split(
"llm_output_file_id,"
)[1].split(";")[0]
file_object_file_field["file_id"] = (
provider_file_id or file_id
)
file_object_file_field["file_id"] = provider_file_id
if format:
file_object_file_field["format"] = format
return messages

69
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -2124,27 +2124,62 @@ def anthropic_process_openai_file_message(
)
_EMPTY_TEXT_PLACEHOLDER = (
"[System: Empty message content sanitised to satisfy protocol]"
)
def _sanitize_empty_text_content(
message: AllMessageValues,
) -> AllMessageValues:
"""
Case C: Sanitize empty text content
- Replace empty or whitespace-only text content with a placeholder message.
- Handles both string content and list-of-blocks content (rewriting only
the empty text blocks in place; non-text blocks like images are left
untouched).
Returns:
The message with sanitized content if needed, otherwise the original message
"""
if message.get("role") in ["user", "assistant"]:
content = message.get("content")
if isinstance(content, str):
if not content or not content.strip():
message = cast(AllMessageValues, dict(message)) # Make a copy
message["content"] = (
"[System: Empty message content sanitised to satisfy protocol]"
)
verbose_logger.debug(
f"_sanitize_empty_text_content: Replaced empty text content in {message.get('role')} message"
)
if message.get("role") not in ["user", "assistant"]:
return message
content = message.get("content")
if isinstance(content, str):
if not content or not content.strip():
message = cast(AllMessageValues, dict(message)) # Make a copy
message["content"] = _EMPTY_TEXT_PLACEHOLDER
verbose_logger.debug(
f"_sanitize_empty_text_content: Replaced empty text content in {message.get('role')} message"
)
return message
if isinstance(content, list):
# Walk the blocks and rewrite any empty text blocks. We rewrite (rather
# than drop) so callers don't end up with an entirely empty content
# list, which Anthropic also rejects.
new_blocks: List[Any] = []
rewrote_any = False
for block in content:
if isinstance(block, dict) and block.get("type") == "text":
text = block.get("text")
if not isinstance(text, str) or not text or not text.strip():
new_block = dict(block)
new_block["text"] = _EMPTY_TEXT_PLACEHOLDER
new_blocks.append(new_block)
rewrote_any = True
continue
new_blocks.append(block)
if rewrote_any:
message = cast(AllMessageValues, dict(message)) # Make a copy
message["content"] = new_blocks # type: ignore
verbose_logger.debug(
f"_sanitize_empty_text_content: Replaced empty text block(s) in {message.get('role')} message"
)
return message
@ -2427,6 +2462,18 @@ def anthropic_messages_pt( # noqa: PLR0915
# Sanitize messages for tool calling issues when modify_params=True
messages = sanitize_messages_for_tool_calling(messages)
# Anthropic rejects empty text content blocks with:
# "messages: text content blocks must be non-empty"
# OpenAI/other providers silently tolerate `{"role": "user", "content": ""}`,
# so callers (and upstream agent frameworks like pydantic-ai) routinely
# send empty user/assistant turns. We always rewrite these to a placeholder
# for Anthropic-shaped requests, independent of `litellm.modify_params`,
# because there is no way to "pass through" an empty text block — the
# request will always 400 otherwise. The richer tool-call sanitization
# (Cases A/B/D in `sanitize_messages_for_tool_calling`) remains gated on
# `modify_params` because it actually mutates conversation structure.
messages = [_sanitize_empty_text_content(m) for m in messages]
# add role=tool support to allow function call result/error submission
user_message_types = {"user", "tool", "function"}
# reformat messages to ensure user/assistant are alternating, if there's either 2 consecutive 'user' messages or 2 consecutive 'assistant' message, merge them.

308
litellm/litellm_core_utils/realtime_streaming.py
View file

@ -31,6 +31,8 @@ DefaultLoggedRealTimeEventTypes = [
"session.created",
"response.create",
"response.done",
"conversation.item.added", # GA
"conversation.item.done", # GA
]
@ -44,6 +46,7 @@ class RealTimeStreaming:
model: str = "",
user_api_key_dict: Optional[Any] = None,
request_data: Optional[Dict] = None,
backend_uses_beta_protocol: Optional[bool] = None,
):
self.websocket = websocket
self.backend_ws = backend_ws
@ -54,6 +57,14 @@ class RealTimeStreaming:
self.session_tools: List[Dict] = []
self.tool_calls: List[Dict] = []
# Detect whether the client is explicitly opting into the beta protocol.
self._client_wants_beta = self._detect_beta_header(websocket)
self._backend_uses_beta_protocol = (
self._client_wants_beta
if backend_uses_beta_protocol is None
else backend_uses_beta_protocol
)
_logged_real_time_event_types = litellm.logged_real_time_event_types
if _logged_real_time_event_types is None:
@ -76,6 +87,27 @@ class RealTimeStreaming:
# response.create can be rewritten to include the failure context.
self._pending_guardrail_message: Optional[str] = None
_SESSION_EVENT_TYPES = frozenset(["session.created", "session.updated"])
_AUDIO_FORMAT_MAP: Dict[str, Dict[str, Any]] = {
"pcm16": {"type": "audio/pcm", "rate": 24000},
"g711_ulaw": {"type": "audio/G711-ulaw", "rate": 8000},
"g711_alaw": {"type": "audio/G711-alaw", "rate": 8000},
}
# GA name → beta name (when client WebSocket includes OpenAI-Beta: realtime=v1)
_GA_TO_BETA_EVENT_TYPES: Dict[str, str] = {
"conversation.item.added": "conversation.item.created",
"response.output_text.delta": "response.text.delta",
"response.output_audio.delta": "response.audio.delta",
"response.output_audio_transcript.delta": "response.audio_transcript.delta",
"response.output_text.done": "response.text.done",
"response.output_audio.done": "response.audio.done",
"response.output_audio_transcript.done": "response.audio_transcript.done",
}
_GA_TO_BETA_CONTENT_TYPES: Dict[str, str] = {
"output_text": "text",
"output_audio": "audio",
}
def _should_store_message(
self,
message_obj: Union[dict, OpenAIRealtimeEvents],
@ -92,24 +124,27 @@ class RealTimeStreaming:
if isinstance(message, bytes):
message = message.decode("utf-8")
if isinstance(message, dict):
message_obj = message
# TypedDict union members do not narrow to plain dict for mypy.
message_obj: Dict[str, Any] = cast(Dict[str, Any], message)
else:
message_obj = json.loads(message)
message_obj = cast(Dict[str, Any], json.loads(cast(str, message)))
self._collect_tool_calls_from_response_done(cast(dict, message_obj))
try:
if (
not isinstance(message, dict)
or message_obj.get("type") == "session.created"
or message_obj.get("type") == "session.updated"
):
message_obj = OpenAIRealtimeStreamSessionEvents(**message_obj) # type: ignore
elif not isinstance(message, dict):
message_obj = OpenAIRealtimeStreamResponseBaseObject(**message_obj) # type: ignore
event_type = message_obj.get("type", "")
if event_type in self._SESSION_EVENT_TYPES:
typed_obj = OpenAIRealtimeStreamSessionEvents(**message_obj) # type: ignore
else:
# Use the base object as a safe catch-all for all other event types
# (both beta and GA), so unknown/new event names never raise here.
typed_obj = OpenAIRealtimeStreamResponseBaseObject(**message_obj) # type: ignore
except Exception as e:
verbose_logger.debug(f"Error parsing message for logging: {e}")
raise e
if self._should_store_message(message_obj):
self.messages.append(message_obj)
# Don't re-raise — a parse failure must not drop or delay the message
if self._should_store_message(message_obj):
self.messages.append(message_obj) # type: ignore[arg-type]
return
if self._should_store_message(typed_obj):
self.messages.append(typed_obj)
def _collect_user_input_from_client_event(self, message: Union[str, dict]) -> None:
"""Extract user text content from client WebSocket events for spend logging."""
@ -147,6 +182,8 @@ class RealTimeStreaming:
tools = session.get("tools")
if tools and isinstance(tools, list):
self.session_tools = tools
# GA: session.type is required; log it for traceability but no action needed
verbose_logger.debug(f"Realtime session.type: {session.get('type')}")
except (json.JSONDecodeError, AttributeError, TypeError):
pass
@ -228,6 +265,23 @@ class RealTimeStreaming:
else:
await self.backend_ws.send(message) # type: ignore[union-attr, attr-defined]
def _make_disable_auto_response_message(self) -> str:
"""Return a session.update that disables VAD auto-response."""
if self._backend_uses_beta_protocol:
session: Dict[str, Any] = {
"turn_detection": {"create_response": False},
}
else:
session = {
"type": "realtime",
"audio": {
"input": {
"turn_detection": {"create_response": False},
}
},
}
return json.dumps({"type": "session.update", "session": session})
def _has_realtime_guardrails(self) -> bool:
"""Return True if any callback is registered for realtime guardrail event types."""
from litellm.integrations.custom_guardrail import CustomGuardrail
@ -435,14 +489,7 @@ class RealTimeStreaming:
):
self.store_message(event_str)
await self.websocket.send_text(event_str)
await self._send_to_backend(
json.dumps(
{
"type": "session.update",
"session": {"turn_detection": {"create_response": False}},
}
)
)
await self._send_to_backend(self._make_disable_auto_response_message())
continue
## GUARDRAIL: run on transcription events in provider_config path too
if (
@ -484,14 +531,7 @@ class RealTimeStreaming:
):
self.store_message(raw_response)
await self.websocket.send_text(raw_response)
await self._send_to_backend(
json.dumps(
{
"type": "session.update",
"session": {"turn_detection": {"create_response": False}},
}
)
)
await self._send_to_backend(self._make_disable_auto_response_message())
return True
if (
@ -542,7 +582,20 @@ class RealTimeStreaming:
continue
## LOGGING
self.store_message(raw_response)
await self.websocket.send_text(raw_response)
# If the client opted into beta protocol, translate GA event
# names/shapes back to the beta equivalents before forwarding.
if self._client_wants_beta:
try:
event_dict = json.loads(raw_response)
translated = self._translate_event_to_beta(event_dict)
if translated is None:
continue # drop GA-only events (e.g. conversation.item.done)
await self.websocket.send_text(json.dumps(translated))
except Exception:
await self.websocket.send_text(raw_response)
else:
await self.websocket.send_text(raw_response)
except websockets.exceptions.ConnectionClosed as e: # type: ignore
verbose_logger.exception(
@ -553,6 +606,183 @@ class RealTimeStreaming:
finally:
await self.log_messages()
@staticmethod
def _detect_beta_header(websocket: Any) -> bool:
"""Return True if the client sent 'OpenAI-Beta: realtime=v1'.
Checks the raw ASGI scope headers so it works for both FastAPI WebSocket
objects and any test doubles that expose a .scope dict.
"""
try:
headers = websocket.scope.get("headers", [])
for name, value in headers:
if isinstance(name, bytes):
name = name.decode("latin-1")
if isinstance(value, bytes):
value = value.decode("latin-1")
if name.lower() == "openai-beta" and "realtime=v1" in value.lower():
return True
except Exception:
pass
return False
@staticmethod
def _remap_beta_session_to_ga(session: dict) -> dict:
"""
Convert a beta-style session.update payload to the GA nested schema.
Beta → GA field mappings
─────────────────────────────────────────────────────────────────────
session.type (inject "realtime" if absent)
session.modalities → session.output_modalities
session.voice → session.audio.output.voice
session.input_audio_format → session.audio.input.format (with type/rate)
session.output_audio_format → session.audio.output.format (with type/rate)
session.turn_detection → session.audio.input.turn_detection
session.input_audio_transcription → session.audio.input.transcription
─────────────────────────────────────────────────────────────────────
Fields not in the mapping (instructions, tools, etc.) are passed through.
GA clients that already use the nested shape are unaffected.
"""
# Work on a shallow copy so we don't mutate the caller's dict
session = dict(session)
# 1. Ensure session.type is present
if "type" not in session:
session["type"] = "realtime"
# 2. Rename modalities → output_modalities and normalise combinations.
# Beta allowed ["audio", "text"] together; GA only supports ["audio"] or
# ["text"] as single-element lists. When both are present we prefer
# ["audio"] because audio mode already delivers transcripts via events.
if "modalities" in session:
mods = session.pop("modalities")
if "output_modalities" not in session:
mods_set = {m.lower() for m in (mods or [])}
if "audio" in mods_set:
session["output_modalities"] = ["audio"]
elif "text" in mods_set:
session["output_modalities"] = ["text"]
# 3-7. Lift flat audio fields into the nested audio object
audio: Dict[str, Any] = {}
inp: Dict[str, Any] = {}
out: Dict[str, Any] = {}
# voice → audio.output.voice
if "voice" in session:
out["voice"] = session.pop("voice")
# input_audio_format → audio.input.format
if "input_audio_format" in session:
raw = session.pop("input_audio_format")
inp["format"] = (
RealTimeStreaming._AUDIO_FORMAT_MAP.get(raw, raw)
if isinstance(raw, str)
else raw
)
# output_audio_format → audio.output.format
if "output_audio_format" in session:
raw = session.pop("output_audio_format")
out["format"] = (
RealTimeStreaming._AUDIO_FORMAT_MAP.get(raw, raw)
if isinstance(raw, str)
else raw
)
# turn_detection → audio.input.turn_detection
if "turn_detection" in session:
inp["turn_detection"] = session.pop("turn_detection")
# input_audio_transcription → audio.input.transcription
if "input_audio_transcription" in session:
inp["transcription"] = session.pop("input_audio_transcription")
if inp:
audio["input"] = inp
if out:
audio["output"] = out
if audio:
# Merge with any existing GA-style `audio` block the client already set,
# letting the remapped values take precedence within each sub-key.
existing = session.get("audio") or {}
for sub_key, sub_val in audio.items():
if (
sub_key in existing
and isinstance(existing[sub_key], dict)
and isinstance(sub_val, dict)
):
existing[sub_key] = {**existing[sub_key], **sub_val}
else:
existing[sub_key] = sub_val
session["audio"] = existing
return session
@staticmethod
def _translate_event_to_beta(event: dict) -> Optional[dict]:
"""Translate a single GA event dict to its beta equivalent.
Returns None if the event should be dropped entirely (e.g. the GA-only
conversation.item.done has no beta counterpart).
Returns the (possibly mutated copy of the) event otherwise.
"""
event_type = event.get("type", "")
# conversation.item.done has no beta equivalent — the client already
# received conversation.item.created (translated from .added).
if event_type == "conversation.item.done":
return None
# Shallow-copy so we don't mutate the stored message
translated = dict(event)
# Rename the type field
if event_type in RealTimeStreaming._GA_TO_BETA_EVENT_TYPES:
translated["type"] = RealTimeStreaming._GA_TO_BETA_EVENT_TYPES[event_type]
# Fix content block types inside items (response.done output list,
# conversation.item.created item content, etc.)
if "item" in translated and isinstance(translated["item"], dict):
translated["item"] = RealTimeStreaming._translate_item_content_types(
dict(translated["item"])
)
if "response" in translated and isinstance(translated["response"], dict):
resp = dict(translated["response"])
if "output" in resp and isinstance(resp["output"], list):
resp["output"] = [
(
RealTimeStreaming._translate_item_content_types(dict(o))
if isinstance(o, dict)
else o
)
for o in resp["output"]
]
translated["response"] = resp
return translated
@staticmethod
def _translate_item_content_types(item: dict) -> dict:
"""Replace GA content type names with beta names inside a single item."""
if "content" not in item or not isinstance(item["content"], list):
return item
new_content = []
for block in item["content"]:
if (
isinstance(block, dict)
and block.get("type") in RealTimeStreaming._GA_TO_BETA_CONTENT_TYPES
):
block = dict(block)
block["type"] = RealTimeStreaming._GA_TO_BETA_CONTENT_TYPES[
block["type"]
]
new_content.append(block)
item["content"] = new_content
return item
async def client_ack_messages(self):
try:
while True:
@ -594,6 +824,19 @@ class RealTimeStreaming:
self._pending_guardrail_message = None
continue
# GA compatibility: remap beta-style session fields only when
# the upstream is in GA mode. Beta upstreams expect the flat
# session shape unchanged.
if (
msg_type == "session.update"
and not self._backend_uses_beta_protocol
):
session = msg_obj.get("session", {})
if isinstance(session, dict):
session = self._remap_beta_session_to_ga(session)
msg_obj["session"] = session
message = json.dumps(msg_obj)
except (json.JSONDecodeError, AttributeError):
pass
@ -627,3 +870,8 @@ class RealTimeStreaming:
await forward_task
except asyncio.CancelledError:
pass
def client_sent_openai_beta_realtime_header(websocket: Any) -> bool:
"""True when the client WebSocket includes ``OpenAI-Beta: realtime=v1``."""
return RealTimeStreaming._detect_beta_header(websocket)

12
litellm/llms/anthropic/chat/guardrail_translation/handler.py
View file

@ -23,7 +23,9 @@ from litellm.llms.anthropic.experimental_pass_through.adapters.transformation im
from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
from litellm.llms.base_llm.guardrail_translation.utils import (
effective_skip_system_message_for_guardrail,
effective_skip_tool_message_for_guardrail,
openai_messages_without_system,
openai_messages_without_tool,
)
from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler import (
AnthropicPassthroughLoggingHandler,
@ -108,6 +110,7 @@ class AnthropicMessagesHandler(BaseTranslation):
return data
skip_system = effective_skip_system_message_for_guardrail(guardrail_to_apply)
skip_tool = effective_skip_tool_message_for_guardrail(guardrail_to_apply)
chat_completion_compatible_request = self._translate_to_openai(data)
@ -117,6 +120,8 @@ class AnthropicMessagesHandler(BaseTranslation):
)
if skip_system:
structured_messages = openai_messages_without_system(structured_messages)
if skip_tool:
structured_messages = openai_messages_without_tool(structured_messages)
texts_to_check: List[str] = []
images_to_check: List[str] = []
@ -134,6 +139,7 @@ class AnthropicMessagesHandler(BaseTranslation):
images_to_check=images_to_check,
task_mappings=task_mappings,
skip_system_message=skip_system,
skip_tool_message=skip_tool,
)
# Step 2: Apply guardrail to all texts in batch
@ -198,13 +204,17 @@ class AnthropicMessagesHandler(BaseTranslation):
images_to_check: List[str],
task_mappings: List[Tuple[int, Optional[int]]],
skip_system_message: bool = False,
skip_tool_message: bool = False,
) -> None:
"""
Extract text content and images from a message.
Override this method to customize text/image extraction logic.
"""
if skip_system_message and str(message.get("role") or "").lower() == "system":
role = str(message.get("role") or "").lower()
if skip_system_message and role == "system":
return
if skip_tool_message and role == "tool":
return
content = message.get("content", None)

52
litellm/llms/anthropic/chat/handler.py
View file

@ -65,7 +65,7 @@ from litellm.types.utils import (
from ...base import BaseLLM
from ..common_utils import AnthropicError, process_anthropic_headers
from .transformation import AnthropicConfig
from .transformation import ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY, AnthropicConfig
if TYPE_CHECKING:
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
@ -83,6 +83,7 @@ async def make_call(
timeout: Optional[Union[float, httpx.Timeout]],
json_mode: bool,
speed: Optional[str] = None,
tool_name_reverse_map: Optional[Dict[str, str]] = None,
) -> Tuple[Any, httpx.Headers]:
if client is None:
client = litellm.module_level_aclient
@ -117,6 +118,7 @@ async def make_call(
sync_stream=False,
json_mode=json_mode,
speed=speed,
tool_name_reverse_map=tool_name_reverse_map,
)
# LOGGING
@ -141,6 +143,7 @@ def make_sync_call(
timeout: Optional[Union[float, httpx.Timeout]],
json_mode: bool,
speed: Optional[str] = None,
tool_name_reverse_map: Optional[Dict[str, str]] = None,
) -> Tuple[Any, httpx.Headers]:
if client is None:
client = litellm.module_level_client # re-use a module level client
@ -183,6 +186,7 @@ def make_sync_call(
sync_stream=True,
json_mode=json_mode,
speed=speed,
tool_name_reverse_map=tool_name_reverse_map,
)
# LOGGING
@ -237,6 +241,11 @@ class AnthropicChatCompletion(BaseLLM):
timeout=timeout,
json_mode=json_mode,
speed=optional_params.get("speed") if optional_params else None,
tool_name_reverse_map=(
litellm_params.get(ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY)
if isinstance(litellm_params, dict)
else None
),
)
streamwrapper = CustomStreamWrapper(
completion_stream=completion_stream,
@ -462,6 +471,11 @@ class AnthropicChatCompletion(BaseLLM):
timeout=timeout,
json_mode=json_mode,
speed=optional_params.get("speed") if optional_params else None,
tool_name_reverse_map=(
litellm_params.get(ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY)
if isinstance(litellm_params, dict)
else None
),
)
return CustomStreamWrapper(
completion_stream=completion_stream,
@ -526,6 +540,7 @@ class ModelResponseIterator:
sync_stream: bool,
json_mode: Optional[bool] = False,
speed: Optional[str] = None,
tool_name_reverse_map: Optional[Dict[str, str]] = None,
):
self.streaming_response = streaming_response
self.response_iterator = self.streaming_response
@ -533,6 +548,13 @@ class ModelResponseIterator:
self.tool_index = -1
self.json_mode = json_mode
self.speed = speed
# rewritten-name -> caller's original. Built per-request from the
# forward map in AnthropicConfig._build_request_tool_name_maps; only
# contains entries we actually rewrote, so a tool legitimately named
# `foo_bar` is *not* reverse-mapped just because some other tool was
# rewritten to `foo_bar` in a different request. Empty/None is the
# common case (no '/' or other invalid chars in any tool name).
self.tool_name_reverse_map: Dict[str, str] = tool_name_reverse_map or {}
# Generate response ID once per stream to match OpenAI-compatible behavior
self.response_id = _generate_id()
@ -557,6 +579,10 @@ class ModelResponseIterator:
# Accumulate compaction blocks for multi-turn reconstruction
self.compaction_blocks: List[Dict[str, Any]] = []
# Accumulate streamed thinking text so final usage can split reasoning
# tokens from regular output tokens.
self.reasoning_content_chunks: List[str] = []
# Track server tool use inputs and results for code_interpreter_results
self._server_tool_inputs: Dict[str, Any] = {}
self.tool_results: List[Dict[str, Any]] = []
@ -587,9 +613,14 @@ class ModelResponseIterator:
return False
def _handle_usage(self, anthropic_usage_chunk: Union[dict, UsageDelta]) -> Usage:
reasoning_content = (
"".join(self.reasoning_content_chunks)
if self.reasoning_content_chunks
else None
)
return AnthropicConfig().calculate_usage(
usage_object=cast(dict, anthropic_usage_chunk),
reasoning_content=None,
reasoning_content=reasoning_content,
speed=self.speed,
)
@ -636,10 +667,13 @@ class ModelResponseIterator:
"thinking" in content_block["delta"]
or "signature" in content_block["delta"]
):
thinking_content = content_block["delta"].get("thinking")
if isinstance(thinking_content, str) and thinking_content:
self.reasoning_content_chunks.append(thinking_content)
thinking_blocks = [
ChatCompletionThinkingBlock(
type="thinking",
thinking=content_block["delta"].get("thinking") or "",
thinking=thinking_content or "",
signature=str(content_block["delta"].get("signature") or ""),
)
]
@ -792,6 +826,16 @@ class ModelResponseIterator:
or content_block_start["content_block"]["type"] == "server_tool_use"
):
self.tool_index += 1
# Reverse-map the (sanitized) tool name back to the
# caller's original. No-op when the map is empty.
_stream_tool_name = content_block_start["content_block"]["name"]
if (
self.tool_name_reverse_map
and _stream_tool_name in self.tool_name_reverse_map
):
_stream_tool_name = self.tool_name_reverse_map[
_stream_tool_name
]
# Use empty string for arguments in content_block_start - actual arguments
# come in subsequent content_block_delta chunks and get accumulated.
# Using str(input) here would prepend '{}' causing invalid JSON accumulation.
@ -799,7 +843,7 @@ class ModelResponseIterator:
id=content_block_start["content_block"]["id"],
type="function",
function=ChatCompletionToolCallFunctionChunk(
name=content_block_start["content_block"]["name"],
name=_stream_tool_name,
arguments="",
),
index=self.tool_index,

364
litellm/llms/anthropic/chat/transformation.py
View file

@ -105,6 +105,115 @@ else:
LoggingClass = Any
# Anthropic requires tool names to match ^[a-zA-Z0-9_-]{1,128}$. Any other
# character (commonly '/' or '.' from OpenAPI-derived MCP tools, e.g.
# "actions/download-job-logs-for-workflow-run") must be replaced before
# the request is sent.
#
# A naive "replace [^a-zA-Z0-9_-] with _" is unsafe because it's lossy:
# `foo/bar` and `foo_bar` both collapse to `foo_bar`. Two tools with the
# same sanitized name would either 400 at Anthropic (duplicate) or, worse,
# cause the response side to mis-translate `foo_bar` (a name the caller
# really did register) back to `foo/bar`.
#
# Instead we build a *per-request* forward map (original -> sanitized)
# whose codomain is unique within the request: when two originals collapse
# to the same candidate, or when a sanitized name collides with an already-
# valid name elsewhere in the request, we append numeric suffixes
# (`_2`, `_3`, ...) until the result is free.
#
# The reverse map (sanitized -> original) only contains entries where the
# original was actually rewritten. So a tool whose name is already valid
# round-trips identically and is *never* mistakenly re-mapped on the
# response side.
_ANTHROPIC_TOOL_NAME_INVALID_CHARS = re.compile(r"[^a-zA-Z0-9_-]")
_ANTHROPIC_TOOL_NAME_MAX_LEN = 128
# Single, internal-only key on ``litellm_params`` used to thread the per-
# request reverse map (sanitized -> original) from request build to response
# parsing. ``litellm_params`` is never serialized to a provider; ``optional_
# params`` IS (it becomes the JSON body via ``data = {**optional_params}``).
# Keep these two channels strictly separate -- never stash internal
# coordination state in ``optional_params``.
ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY = "_anthropic_tool_name_map"
def _basic_sanitize_anthropic_tool_name(name: str) -> str:
"""Lossy: replace [^a-zA-Z0-9_-] with '_' and truncate to 128.
Used as a candidate generator for the per-request forward map.
Callers should NOT use this directly for translation -- always go
through the forward map so collisions are resolved.
"""
if not isinstance(name, str) or not name:
return name
return _ANTHROPIC_TOOL_NAME_INVALID_CHARS.sub("_", name)[
:_ANTHROPIC_TOOL_NAME_MAX_LEN
]
def _build_anthropic_tool_name_maps(
original_names: List[str],
) -> Tuple[Dict[str, str], Dict[str, str]]:
"""Build (forward, reverse) tool-name maps for a single request.
forward[original] = sanitized -- only present when name was rewritten
reverse[sanitized] = original -- inverse of `forward`
Properties:
- All sanitized names satisfy ^[a-zA-Z0-9_-]{1,128}$.
- Sanitized names are unique within the request (no two originals
collide on the wire).
- A name that's already valid AND doesn't collide with another tool's
sanitized form passes through untouched and is absent from the maps.
That's the key correctness property: response-side translation only
runs on entries we actually rewrote, so a tool legitimately named
`foo_bar` is never incorrectly retyped to `foo/bar` just because
some *other* request had that pair.
- Order-dependent: when two originals would clash, the *second* one
seen gets the disambiguating suffix. Callers should preserve the
caller's tool order (we do).
"""
forward: Dict[str, str] = {}
used: set = set()
# First pass: reserve slots for names that are already valid so they
# always have priority regardless of input order.
for original in original_names:
if not isinstance(original, str) or not original:
continue
candidate = _basic_sanitize_anthropic_tool_name(original)
if candidate == original:
used.add(candidate)
# Second pass: sanitize/disambiguate names that need rewriting.
for original in original_names:
if not isinstance(original, str) or not original:
continue
candidate = _basic_sanitize_anthropic_tool_name(original)
if candidate == original:
continue
# Skip duplicates of the same original name. Without this guard the
# second pass would assign a fresh suffix and overwrite the forward
# map entry, causing every reference to map to the suffixed name and
# leaving the original sanitized slot orphaned in `used` with no
# reverse mapping.
if original in forward:
continue
# Disambiguate against names already chosen this request.
unique = candidate
n = 1
while unique in used:
n += 1
suffix = f"_{n}"
# Keep within the 128-char cap.
head = candidate[: _ANTHROPIC_TOOL_NAME_MAX_LEN - len(suffix)]
unique = f"{head}{suffix}"
forward[original] = unique
used.add(unique)
reverse = {v: k for k, v in forward.items()}
return forward, reverse
REASONING_EFFORT_TO_OUTPUT_CONFIG_EFFORT: Dict[str, str] = {
"low": "low",
"minimal": "low",
@ -486,7 +595,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
}
def _map_tool_choice(
self, tool_choice: Optional[str], parallel_tool_use: Optional[bool]
self,
tool_choice: Optional[str],
parallel_tool_use: Optional[bool],
) -> Optional[AnthropicMessagesToolChoice]:
_tool_choice: Optional[AnthropicMessagesToolChoice] = None
if tool_choice == "auto":
@ -527,7 +638,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
return _tool_choice
def _map_tool_helper( # noqa: PLR0915
self, tool: ChatCompletionToolParam
self,
tool: ChatCompletionToolParam,
) -> Tuple[Optional[AllAnthropicToolsValues], Optional[AnthropicMcpServerTool]]:
returned_tool: Optional[AllAnthropicToolsValues] = None
mcp_server: Optional[AnthropicMcpServerTool] = None
@ -783,7 +895,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
return initial_tool
def _map_tools(
self, tools: List
self,
tools: List,
) -> Tuple[List[AllAnthropicToolsValues], List[AnthropicMcpServerTool]]:
anthropic_tools = []
mcp_servers = []
@ -799,6 +912,174 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
mcp_servers.append(mcp_server_tool)
return anthropic_tools, mcp_servers
@staticmethod
def _rewrite_tool_names_in_messages(
messages: List[AllMessageValues],
name_forward_map: Dict[str, str],
) -> List[AllMessageValues]:
"""Return a copy of `messages` with tool_call/function_call names
rewritten using the per-request forward map.
Only mutates messages whose tool_call/function_call name is *in* the
forward map. Names absent from the map (already valid, no collision)
round-trip untouched. We only deep-copy the entries we actually
change to keep this O(turns-with-rewritten-tools), not O(history).
"""
if not name_forward_map:
return messages
new_messages: List[AllMessageValues] = []
for msg in messages:
if not isinstance(msg, dict):
new_messages.append(msg)
continue
tool_calls = msg.get("tool_calls")
function_call = msg.get("function_call")
if not tool_calls and not function_call:
new_messages.append(msg)
continue
new_msg = dict(msg)
if isinstance(tool_calls, list):
new_calls = []
for tc in tool_calls:
if not isinstance(tc, dict):
new_calls.append(tc)
continue
fn = tc.get("function")
fn_name = fn.get("name") if isinstance(fn, dict) else None
if (
isinstance(fn, dict)
and isinstance(fn_name, str)
and fn_name in name_forward_map
):
new_fn = dict(fn)
new_fn["name"] = name_forward_map[fn_name]
new_tc = dict(tc)
new_tc["function"] = new_fn
new_calls.append(new_tc)
else:
new_calls.append(tc)
new_msg["tool_calls"] = new_calls
fc_name = (
function_call.get("name") if isinstance(function_call, dict) else None
)
if (
isinstance(function_call, dict)
and isinstance(fc_name, str)
and fc_name in name_forward_map
):
new_fc = dict(function_call)
new_fc["name"] = name_forward_map[fc_name]
new_msg["function_call"] = new_fc
new_messages.append(cast(AllMessageValues, new_msg))
return new_messages
@staticmethod
def _build_request_tool_name_maps(
tools: List,
) -> Tuple[Dict[str, str], Dict[str, str]]:
"""Build the (forward, reverse) tool-name maps for an OpenAI tools list.
Operates on **OpenAI-format** tool dicts (pre-``_map_tools``). The
production sanitization path uses ``_sanitize_tool_names_in_request``
instead, which operates on **Anthropic-format** tools (post-
``_map_tools``, where ``type == "custom"``). This helper exists for
callers that need to compute the maps from the raw OpenAI shape --
e.g. test setup or future pre-mapping consumers.
See _build_anthropic_tool_name_maps for the collision rules. Pulls
the original name out of either ``{"function": {"name": ...}}``
(legacy OpenAI shape) or ``{"name": ...}`` (rare top-level shape).
"""
original_names: List[str] = []
for tool in tools or []:
if not isinstance(tool, dict):
continue
original = (
tool.get("function", {}).get("name")
if isinstance(tool.get("function"), dict)
else None
)
if original is None:
original = tool.get("name")
if isinstance(original, str) and original:
original_names.append(original)
return _build_anthropic_tool_name_maps(original_names)
@staticmethod
def _sanitize_tool_names_in_request(
optional_params: Dict[str, Any],
) -> Tuple[Dict[str, str], Dict[str, str]]:
"""Sanitize ``optional_params['tools']`` and ``optional_params['tool_choice']``
in place so every name matches Anthropic's ``^[a-zA-Z0-9_-]{1,128}$``.
Returns ``(forward, reverse)`` for use by message-history rewriting
and response translation. ``forward[original] = sanitized`` is only
populated for names that were actually rewritten -- i.e. either
contained an invalid character or collided with another tool's
sanitized form. Names already valid AND unique pass through and are
absent from both maps.
Only ``type == "custom"`` tools (the OpenAI function-tool shape) are
considered. Hosted tools (``web_search``, ``bash``, ``code_execution``,
``computer_*``, ``mcp``, ...) own reserved names defined by Anthropic
and must not be touched.
"""
tools = optional_params.get("tools")
if not isinstance(tools, list) or not tools:
return {}, {}
# 1. Collect originals from the Anthropic-shaped custom-tool entries.
# Order matters: the first occurrence wins the canonical slot;
# later collisions get numeric suffixes (see
# ``_build_anthropic_tool_name_maps``).
original_names: List[str] = []
for t in tools:
if not isinstance(t, dict):
continue
if t.get("type") != "custom":
continue
name = t.get("name")
if isinstance(name, str) and name:
original_names.append(name)
if not original_names:
return {}, {}
forward, reverse = _build_anthropic_tool_name_maps(original_names)
if not forward:
# Every name was already valid -- nothing to do.
return forward, reverse
# 2. Apply forward map. Build a new list with copy-on-change entries
# so a caller reusing the same tool list/dicts across requests
# doesn't see its inputs permanently rewritten (which would also
# drop the original key from `forward` on the next request).
new_tools: List[Any] = []
for t in tools:
if (
isinstance(t, dict)
and t.get("type") == "custom"
and isinstance(t.get("name"), str)
and t["name"] in forward
):
new_tools.append({**t, "name": forward[t["name"]]})
else:
new_tools.append(t)
optional_params["tools"] = new_tools
# 3. Same for ``tool_choice`` when it targets a named tool. Copy
# rather than mutate for the same reason as above.
tool_choice = optional_params.get("tool_choice")
if isinstance(tool_choice, dict) and tool_choice.get("type") == "tool":
tc_name = tool_choice.get("name")
if isinstance(tc_name, str) and tc_name in forward:
optional_params["tool_choice"] = {
**tool_choice,
"name": forward[tc_name],
}
return forward, reverse
def _detect_tool_search_tools(self, tools: Optional[List]) -> bool:
"""Check if tool search tools are present in the tools list."""
if not tools:
@ -1125,6 +1406,17 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
non_default_params=non_default_params
)
# NB: ``map_openai_params`` deliberately does NOT sanitize tool names
# here. Names are the *original* OpenAI names at this stage, and must
# remain so until ``transform_request`` -- which is the single
# chokepoint where Anthropic, Bedrock-Anthropic, and Vertex-Anthropic
# all pass through. Doing it there guarantees:
# 1. one source of truth for the per-request forward/reverse maps,
# 2. the maps land on ``litellm_params`` (internal), never on
# ``optional_params`` (which is serialized into the request body
# via ``data = {**optional_params}`` and would 400 with
# ``Extra inputs are not permitted``).
for param, value in non_default_params.items():
if param == "max_tokens":
optional_params["max_tokens"] = (
@ -1135,7 +1427,6 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
value if isinstance(value, int) else max(1, int(round(value)))
)
elif param == "tools":
# check if optional params already has tools
anthropic_tools, mcp_servers = self._map_tools(value)
optional_params = self._add_tools_to_optional_params(
optional_params=optional_params, tools=anthropic_tools
@ -1565,6 +1856,34 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
headers=headers, optional_params=optional_params
)
# === Tool-name sanitization (single chokepoint) ===
# Anthropic enforces ^[a-zA-Z0-9_-]{1,128}$ on every tool name. We
# sanitize *here* -- not in map_openai_params -- because:
#
# - This function is the single boundary shared by AnthropicConfig,
# AmazonAnthropicConfig (Bedrock invoke), VertexAIAnthropicConfig,
# and AzureAnthropicConfig (all call ``super().transform_request``
# or ``AnthropicConfig.transform_request(self, ...)``). Sanitizing
# once here covers every Anthropic-shaped request.
# - The forward/reverse maps are coordination state; they belong on
# ``litellm_params`` (internal-only), never on ``optional_params``
# (which becomes the JSON body via ``{**optional_params}``).
# - It keeps ``map_openai_params`` a pure param translator with no
# side-channel state.
#
# The reverse map only contains entries for names that were actually
# rewritten -- so a tool legitimately named ``foo_bar`` is never
# incorrectly retyped to ``foo/bar`` on the response side.
# See _build_anthropic_tool_name_maps for the collision-handling
# rules and rationale.
_name_forward_map, _name_reverse_map = self._sanitize_tool_names_in_request(
optional_params=optional_params,
)
if _name_forward_map:
messages = self._rewrite_tool_names_in_messages(messages, _name_forward_map)
if _name_reverse_map and isinstance(litellm_params, dict):
litellm_params[ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY] = _name_reverse_map
# Separate system prompt from rest of message
anthropic_system_message_list = self.translate_system_message(messages=messages)
# Handling anthropic API Prompt Caching
@ -1837,8 +2156,16 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
speed: Optional[str] = None,
) -> Usage:
# NOTE: Sometimes the usage object has None set explicitly for token counts, meaning .get() & key access returns None, and we need to account for this
prompt_tokens = usage_object.get("input_tokens", 0) or 0
completion_tokens = usage_object.get("output_tokens", 0) or 0
raw_prompt_tokens = usage_object.get("input_tokens", 0) or 0
prompt_tokens: int = (
int(raw_prompt_tokens) if isinstance(raw_prompt_tokens, (int, float)) else 0
)
raw_completion_tokens = usage_object.get("output_tokens", 0) or 0
completion_tokens: int = (
int(raw_completion_tokens)
if isinstance(raw_completion_tokens, (int, float))
else 0
)
_usage = usage_object
cache_creation_input_tokens: int = 0
cache_read_input_tokens: int = 0
@ -1907,11 +2234,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
text_tokens=raw_input_tokens,
)
# Always populate completion_token_details, not just when there's reasoning_content
reasoning_tokens = (
estimated_reasoning_tokens = (
token_counter(text=reasoning_content, count_response_tokens=True)
if reasoning_content
else 0
)
reasoning_tokens = min(estimated_reasoning_tokens, completion_tokens)
completion_token_details = CompletionTokensDetailsWrapper(
reasoning_tokens=reasoning_tokens if reasoning_tokens > 0 else 0,
text_tokens=(
@ -2041,6 +2369,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
json_mode: Optional[bool] = None,
prefix_prompt: Optional[str] = None,
speed: Optional[str] = None,
tool_name_reverse_map: Optional[Dict[str, str]] = None,
):
_hidden_params: Dict = {}
_hidden_params["additional_headers"] = process_anthropic_headers(
@ -2065,6 +2394,21 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
compaction_blocks,
) = self.extract_response_content(completion_response=completion_response)
# Reverse-map rewritten tool names back to caller's originals so a
# downstream OpenAI-style dispatcher can match on the registered name.
# See _build_anthropic_tool_name_maps for why this is keyed on the
# per-request reverse map (so a tool legitimately named `foo_bar` is
# never incorrectly retyped to `foo/bar`). No-op when the map is
# empty (the common case).
if tool_name_reverse_map and tool_calls:
for tc in tool_calls:
fn = tc.get("function") if isinstance(tc, dict) else None
if fn is None:
continue
_name = fn.get("name")
if isinstance(_name, str) and _name in tool_name_reverse_map:
fn["name"] = tool_name_reverse_map[_name]
if (
prefix_prompt is not None
and not text_content.startswith(prefix_prompt)
@ -2191,6 +2535,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
prefix_prompt = self.get_prefix_prompt(messages=messages)
speed = optional_params.get("speed")
tool_name_reverse_map: Optional[Dict[str, str]] = None
if isinstance(litellm_params, dict):
_candidate = litellm_params.get(ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY)
if isinstance(_candidate, dict):
tool_name_reverse_map = _candidate
model_response = self.transform_parsed_response(
completion_response=completion_response,
@ -2199,6 +2548,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
json_mode=json_mode,
prefix_prompt=prefix_prompt,
speed=speed,
tool_name_reverse_map=tool_name_reverse_map,
)
return model_response

6
litellm/llms/azure/realtime/handler.py
View file

@ -89,9 +89,10 @@ class AzureOpenAIRealtime(AzureChatCompletion):
if api_base is None:
raise ValueError("api_base is required for Azure OpenAI calls")
if api_version is None and (
backend_uses_beta_protocol = (
realtime_protocol is None or realtime_protocol.upper() not in ("GA", "V1")
):
)
if api_version is None and backend_uses_beta_protocol:
raise ValueError("api_version is required for Azure OpenAI calls")
url = self._construct_url(
@ -114,6 +115,7 @@ class AzureOpenAIRealtime(AzureChatCompletion):
logging_obj,
user_api_key_dict=user_api_key_dict,
request_data={"litellm_metadata": litellm_metadata or {}},
backend_uses_beta_protocol=backend_uses_beta_protocol,
)
await realtime_streaming.bidirectional_forward()

15
litellm/llms/base_llm/guardrail_translation/utils.py
View file

@ -14,7 +14,22 @@ def effective_skip_system_message_for_guardrail(guardrail_to_apply: Any) -> bool
return bool(getattr(litellm, "skip_system_message_in_guardrail", False))
def effective_skip_tool_message_for_guardrail(guardrail_to_apply: Any) -> bool:
per = getattr(guardrail_to_apply, "skip_tool_message_in_guardrail", None)
if per is not None:
return bool(per)
import litellm
return bool(getattr(litellm, "skip_tool_message_in_guardrail", False))
def openai_messages_without_system(
messages: List[AllMessageValues],
) -> List[AllMessageValues]:
return [m for m in messages if str((m or {}).get("role") or "").lower() != "system"]
def openai_messages_without_tool(
messages: List[AllMessageValues],
) -> List[AllMessageValues]:
return [m for m in messages if str((m or {}).get("role") or "").lower() != "tool"]

205
litellm/llms/bedrock/base_aws_llm.py
View file

@ -7,6 +7,8 @@ from datetime import datetime
from typing import (
TYPE_CHECKING,
Any,
Callable,
ClassVar,
Dict,
List,
Literal,
@ -63,8 +65,16 @@ class AwsAuthError(Exception):
class BaseAWSLLM:
# Process-wide IAM credential cache (shared across instances — Bedrock passthrough is per-request).
# Storage is in-process memory only: default ``DualCache()`` has no Redis backend unless attached
# elsewhere. Entry TTL: static access-key + secret + region use ``_get_default_ttl_for_boto3_credentials``
# (~59 minutes); ambient env (``_auth_with_env_vars`` returns ``ttl=None``) uses ``InMemoryCache``'s
# ``default_ttl`` (600 seconds / 10 minutes). AssumeRole, web identity, profiles, and explicit
# session-token tuples are not cached — see ``get_credentials`` and ``_get_or_set_cached_credentials``.
_shared_iam_cache: ClassVar[DualCache] = DualCache()
def __init__(self) -> None:
self.iam_cache = DualCache()
self.iam_cache = BaseAWSLLM._shared_iam_cache
super().__init__()
self.aws_authentication_params = [
"aws_access_key_id",
@ -103,6 +113,79 @@ class BaseAWSLLM:
credential_str = json.dumps(credential_args, sort_keys=True)
return hashlib.sha256(credential_str.encode()).hexdigest()
def _get_or_set_cached_credentials(
self,
credential_args: Dict[str, Optional[str]],
credential_fetcher: Callable[[], Tuple[Any, Optional[int]]],
) -> Any:
"""
Read-through IAM cache on the process-wide ``DualCache``.
Only the in-memory layer is used by default (no Redis on ``_shared_iam_cache`` unless
configured globally). TTL on write: static access-key fetches pass
``_get_default_ttl_for_boto3_credentials()`` (~59 minutes); ambient env passes ``ttl=None``,
which ``InMemoryCache.set_cache`` resolves to ``default_ttl`` (600 seconds / 10 minutes by
default).
Used only for static access-key credentials and ambient credentials from
``_auth_with_env_vars`` (including when skipping AssumeRole because the runtime identity
already matches ``aws_role_name``).
AssumeRole, web identity exchange, profiles, and explicit session-token tuples are not
cached here — shared ``Credentials`` / refresh state must not span logical sessions.
"""
cache_key = self.get_cache_key(credential_args)
_cached = self.iam_cache.get_cache(cache_key)
if _cached:
return _cached
credentials, ttl = credential_fetcher()
self.iam_cache.set_cache(cache_key, credentials, ttl=ttl)
return credentials
@staticmethod
def _is_auth_with_web_identity_token(
aws_web_identity_token: Optional[str],
aws_role_name: Optional[str],
aws_session_name: Optional[str],
) -> bool:
return (
aws_web_identity_token is not None
and aws_role_name is not None
and aws_session_name is not None
)
@staticmethod
def _is_auth_with_aws_role(aws_role_name: Optional[str]) -> bool:
return aws_role_name is not None
@staticmethod
def _is_auth_with_aws_profile(aws_profile_name: Optional[str]) -> bool:
return aws_profile_name is not None
@staticmethod
def _is_auth_with_aws_session_token_tuple(
aws_access_key_id: Optional[str],
aws_secret_access_key: Optional[str],
aws_session_token: Optional[str],
) -> bool:
return (
aws_access_key_id is not None
and aws_secret_access_key is not None
and aws_session_token is not None
)
@staticmethod
def _is_auth_with_access_key_and_secret_key(
aws_access_key_id: Optional[str],
aws_secret_access_key: Optional[str],
aws_region_name: Optional[str],
) -> bool:
return (
aws_access_key_id is not None
and aws_secret_access_key is not None
and aws_region_name is not None
)
@tracer.wrap()
def get_credentials(
self,
@ -184,95 +267,97 @@ class BaseAWSLLM:
aws_external_id,
)
# create cache key for non-expiring auth flows
args = {
k: v
for k, v in locals().items()
if k.startswith("aws_") or k == "ssl_verify"
}
cache_key = self.get_cache_key(args)
_cached_credentials = self.iam_cache.get_cache(cache_key)
if _cached_credentials:
return _cached_credentials
#########################################################
# Handle diff boto3 auth flows
# for each helper
# Return:
# Credentials - boto3.Credentials
# cache ttl - Optional[int]. If None, the credentials are not cached. Some auth flows have no expiry time.
#
# iam_cache: static keys and ambient env only (including skip-AssumeRole path).
# Do not cache AssumeRole / web identity / profile / explicit session-token paths here.
#########################################################
if (
aws_web_identity_token is not None
and aws_role_name is not None
and aws_session_name is not None
if self._is_auth_with_web_identity_token(
aws_web_identity_token,
aws_role_name,
aws_session_name,
):
credentials, _cache_ttl = self._auth_with_web_identity_token(
aws_web_identity_token=aws_web_identity_token,
aws_role_name=aws_role_name,
aws_session_name=aws_session_name,
aws_web_identity_token=cast(str, aws_web_identity_token),
aws_role_name=cast(str, aws_role_name),
aws_session_name=cast(str, aws_session_name),
aws_region_name=aws_region_name,
aws_sts_endpoint=aws_sts_endpoint,
aws_external_id=aws_external_id,
)
elif aws_role_name is not None:
# Check if we're already running as the target role and can skip assumption
# This handles IRSA (EKS), ECS task roles, and EC2 instance profiles
if self._is_already_running_as_role(aws_role_name, ssl_verify=ssl_verify):
return credentials
elif self._is_auth_with_aws_role(aws_role_name):
# Same role (IRSA/ECS/EC2): ambient creds via _get_or_set_cached_credentials like the
# default env branch; never pre-read cache (must run _is_already_running_as_role first).
if self._is_already_running_as_role(
cast(str, aws_role_name), ssl_verify=ssl_verify
):
verbose_logger.debug(
"Already running as target role %s, using ambient credentials",
aws_role_name,
)
credentials, _cache_ttl = self._auth_with_env_vars()
else:
verbose_logger.debug(
"Using role assumption: calling _auth_with_aws_role"
return self._get_or_set_cached_credentials(
args, self._auth_with_env_vars
)
# If aws_session_name is not provided, generate a default one
if aws_session_name is None:
aws_session_name = (
f"litellm-session-{int(datetime.now().timestamp())}"
)
credentials, _cache_ttl = self._auth_with_aws_role(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_session_token=aws_session_token,
aws_role_name=aws_role_name,
aws_session_name=aws_session_name,
aws_region_name=aws_region_name,
aws_sts_endpoint=aws_sts_endpoint,
aws_external_id=aws_external_id,
ssl_verify=ssl_verify,
)
elif aws_profile_name is not None: ### CHECK SESSION ###
credentials, _cache_ttl = self._auth_with_aws_profile(aws_profile_name)
elif (
aws_access_key_id is not None
and aws_secret_access_key is not None
and aws_session_token is not None
):
credentials, _cache_ttl = self._auth_with_aws_session_token(
verbose_logger.debug("Using role assumption: calling _auth_with_aws_role")
# If aws_session_name is not provided, generate a default one
if aws_session_name is None:
aws_session_name = f"litellm-session-{int(datetime.now().timestamp())}"
credentials, _assume_ttl = self._auth_with_aws_role(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_session_token=aws_session_token,
)
elif (
aws_access_key_id is not None
and aws_secret_access_key is not None
and aws_region_name is not None
):
credentials, _cache_ttl = self._auth_with_access_key_and_secret_key(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_role_name=cast(str, aws_role_name),
aws_session_name=aws_session_name,
aws_region_name=aws_region_name,
aws_sts_endpoint=aws_sts_endpoint,
aws_external_id=aws_external_id,
ssl_verify=ssl_verify,
)
return credentials
elif self._is_auth_with_aws_profile(aws_profile_name):
credentials, _cache_ttl = self._auth_with_aws_profile(
cast(str, aws_profile_name)
)
return credentials
elif self._is_auth_with_aws_session_token_tuple(
aws_access_key_id,
aws_secret_access_key,
aws_session_token,
):
credentials, _cache_ttl = self._auth_with_aws_session_token(
aws_access_key_id=cast(str, aws_access_key_id),
aws_secret_access_key=cast(str, aws_secret_access_key),
aws_session_token=cast(str, aws_session_token),
)
return credentials
elif self._is_auth_with_access_key_and_secret_key(
aws_access_key_id,
aws_secret_access_key,
aws_region_name,
):
return self._get_or_set_cached_credentials(
args,
lambda: self._auth_with_access_key_and_secret_key(
aws_access_key_id=cast(str, aws_access_key_id),
aws_secret_access_key=cast(str, aws_secret_access_key),
aws_region_name=cast(str, aws_region_name),
),
)
else:
credentials, _cache_ttl = self._auth_with_env_vars()
self.iam_cache.set_cache(cache_key, credentials, ttl=_cache_ttl)
return credentials
return self._get_or_set_cached_credentials(args, self._auth_with_env_vars)
def _get_aws_region_from_model_arn(self, model: Optional[str]) -> Optional[str]:
try:

24
litellm/llms/bedrock/chat/invoke_agent/transformation.py
View file

@ -299,29 +299,9 @@ class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM):
)
def _get_response_stream_shape(self):
"""Get the response stream shape for parsing, reusing existing logic."""
try:
# Try to reuse the cached shape from the existing decoder
from litellm.llms.bedrock.chat.invoke_handler import (
get_response_stream_shape,
)
from litellm.llms.bedrock.common_utils import BEDROCK_RESPONSE_STREAM_SHAPE
return get_response_stream_shape()
except ImportError:
# Fallback: create our own shape
try:
from botocore.loaders import Loader
from botocore.model import ServiceModel
loader = Loader()
bedrock_service_dict = loader.load_service_model(
"bedrock-runtime", "service-2"
)
bedrock_service_model = ServiceModel(bedrock_service_dict)
return bedrock_service_model.shape_for("ResponseStream")
except Exception as e:
verbose_logger.warning(f"Could not load response stream shape: {e}")
return None
return BEDROCK_RESPONSE_STREAM_SHAPE
def _extract_response_content(self, events: InvokeAgentEventList) -> str:
"""Extract the final response content from parsed events."""

34
litellm/llms/bedrock/chat/invoke_handler.py
View file

@ -67,9 +67,13 @@ from litellm.types.utils import (
from litellm.utils import CustomStreamWrapper, get_secret
from ..base_aws_llm import BaseAWSLLM
from ..common_utils import BedrockError, ModelResponseIterator, get_bedrock_tool_name
from ..common_utils import (
BEDROCK_RESPONSE_STREAM_SHAPE,
BedrockError,
ModelResponseIterator,
get_bedrock_tool_name,
)
_response_stream_shape_cache = None
bedrock_tool_name_mappings: InMemoryCache = InMemoryCache(
max_size_in_memory=50, default_ttl=600
)
@ -1391,20 +1395,6 @@ class BedrockLLM(BaseAWSLLM):
return None
def get_response_stream_shape():
global _response_stream_shape_cache
if _response_stream_shape_cache is None:
from botocore.loaders import Loader
from botocore.model import ServiceModel
loader = Loader()
bedrock_service_dict = loader.load_service_model("bedrock-runtime", "service-2")
bedrock_service_model = ServiceModel(bedrock_service_dict)
_response_stream_shape_cache = bedrock_service_model.shape_for("ResponseStream")
return _response_stream_shape_cache
class AWSEventStreamDecoder:
def __init__(self, model: str, json_mode: Optional[bool] = False) -> None:
from botocore.parsers import EventStreamJSONParser
@ -1838,8 +1828,18 @@ class AWSEventStreamDecoder:
yield self._chunk_parser(chunk_data=_data)
def _parse_message_from_event(self, event) -> Optional[str]:
if BEDROCK_RESPONSE_STREAM_SHAPE is None:
raise BedrockError(
status_code=500,
message=(
"Bedrock event-stream shape could not be loaded from botocore. "
"Ensure botocore is correctly installed."
),
)
response_dict = event.to_response_dict()
parsed_response = self.parser.parse(response_dict, get_response_stream_shape())
parsed_response = self.parser.parse(
response_dict, BEDROCK_RESPONSE_STREAM_SHAPE
)
if response_dict["status_code"] != 200:
decoded_body = response_dict["body"].decode()

58
litellm/llms/bedrock/common_utils.py
View file

@ -14,6 +14,7 @@ if TYPE_CHECKING:
import httpx
import litellm
from litellm import verbose_logger
from litellm.llms.base_llm.anthropic_messages.transformation import (
BaseAnthropicMessagesConfig,
)
@ -917,38 +918,57 @@ def get_bedrock_chat_config(model: str):
return litellm.AmazonInvokeConfig()
def _load_bedrock_response_stream_shape():
"""
Load the ResponseStream shape from botocore's bundled bedrock-runtime schema.
Called once at module import time; the result is stored in
``BEDROCK_RESPONSE_STREAM_SHAPE`` and reused for the process lifetime.
Returns ``None`` if botocore is unavailable or the service model cannot be
loaded, so the module still imports cleanly.
"""
try:
from botocore.loaders import Loader
from botocore.model import ServiceModel
loader = Loader()
service_dict = loader.load_service_model("bedrock-runtime", "service-2")
return ServiceModel(service_dict).shape_for("ResponseStream")
except Exception as e:
verbose_logger.warning(
"litellm: could not pre-load bedrock-runtime response stream shape "
"— Bedrock event-stream decoding will be unavailable. Error: %s",
e,
)
return None
# Eagerly resolved once per process — avoids per-instance or per-request disk I/O.
BEDROCK_RESPONSE_STREAM_SHAPE = _load_bedrock_response_stream_shape()
class BedrockEventStreamDecoderBase:
"""
Base class for event stream decoding for Bedrock
"""
_response_stream_shape_cache = None
def __init__(self):
from botocore.parsers import EventStreamJSONParser
self.parser = EventStreamJSONParser()
def get_response_stream_shape(self):
if self._response_stream_shape_cache is None:
from botocore.loaders import Loader
from botocore.model import ServiceModel
loader = Loader()
bedrock_service_dict = loader.load_service_model(
"bedrock-runtime", "service-2"
)
bedrock_service_model = ServiceModel(bedrock_service_dict)
self._response_stream_shape_cache = bedrock_service_model.shape_for(
"ResponseStream"
)
return self._response_stream_shape_cache
def _parse_message_from_event(self, event) -> Optional[str]:
if BEDROCK_RESPONSE_STREAM_SHAPE is None:
raise BedrockError(
status_code=500,
message=(
"Bedrock event-stream shape could not be loaded from botocore. "
"Ensure botocore is correctly installed."
),
)
response_dict = event.to_response_dict()
parsed_response = self.parser.parse(
response_dict, self.get_response_stream_shape()
response_dict, BEDROCK_RESPONSE_STREAM_SHAPE
)
if response_dict["status_code"] != 200:

51
litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py
View file

@ -408,6 +408,47 @@ class AmazonAnthropicClaudeMessagesConfig(
if self._supports_tool_search_on_bedrock(model):
beta_set.add("tool-search-tool-2025-10-19")
@staticmethod
def _filter_context_management_for_bedrock_invoke(
anthropic_messages_request: Dict,
beta_set: set,
) -> None:
"""
Bedrock InvokeModel accepts ``context_management`` only when it carries
``compact_20260112`` edits paired with the ``compact-2026-01-12``
anthropic-beta header. Other edit types (notably ``clear_thinking_20251015``,
which Claude Code sends on every request) are LiteLLM-internal and would
cause Bedrock to 400 with ``"context_management: Extra inputs are not
permitted"``.
Filter the edits list to the supported subset, add the beta header when
compact edits remain, and drop ``context_management`` entirely when no
supported edits are left so the safety-net allowlist can pass it through.
Ref: https://github.com/BerriAI/litellm/issues/27532
"""
cm = anthropic_messages_request.get("context_management")
if not isinstance(cm, dict):
return
edits = cm.get("edits")
if not isinstance(edits, list):
anthropic_messages_request.pop("context_management", None)
return
compact_edits = [
e
for e in edits
if isinstance(e, dict) and e.get("type") == "compact_20260112"
]
if compact_edits:
beta_set.add("compact-2026-01-12")
anthropic_messages_request["context_management"] = {
**cm,
"edits": compact_edits,
}
else:
anthropic_messages_request.pop("context_management", None)
def _convert_output_format_to_inline_schema(
self,
output_format: Dict,
@ -551,6 +592,11 @@ class AmazonAnthropicClaudeMessagesConfig(
if injected_thinking_for_clear_thinking:
beta_set.add("interleaved-thinking-2025-05-14")
self._filter_context_management_for_bedrock_invoke(
anthropic_messages_request=anthropic_messages_request,
beta_set=beta_set,
)
self._get_tool_search_beta_header_for_bedrock(
model=model,
tool_search_used=tool_search_used,
@ -597,8 +643,9 @@ class AmazonAnthropicClaudeMessagesConfig(
anthropic_messages_request.pop("output_config", None)
# 7. Final safety net: filter top-level fields to the Bedrock Invoke allowlist.
# Catches Anthropic-only extensions (context_management, output_config, speed,
# mcp_servers, ...) and any future additions Claude Code may start sending.
# Catches Anthropic-only extensions (output_config, speed, mcp_servers, ...)
# and any future additions Claude Code may start sending. ``context_management``
# has already been pre-filtered to its Bedrock-supported subset above.
allowed = self.BEDROCK_INVOKE_ALLOWED_TOP_LEVEL_FIELDS
stripped = sorted(k for k in anthropic_messages_request if k not in allowed)
if stripped:

6
litellm/llms/bedrock/passthrough/transformation.py
View file

@ -10,14 +10,12 @@ from ..base_aws_llm import BaseAWSLLM
from ..common_utils import BedrockEventStreamDecoderBase, BedrockModelInfo
if TYPE_CHECKING:
from httpx import URL
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.types.utils import CostResponseTypes
if TYPE_CHECKING:
from httpx import URL
class BedrockPassthroughConfig(
BaseAWSLLM, BedrockModelInfo, BedrockEventStreamDecoderBase, BasePassthroughConfig
):

53
litellm/llms/custom_httpx/http_handler.py
View file

@ -1,4 +1,5 @@
import asyncio
import concurrent.futures
import inspect
import os
import socket
@ -133,6 +134,11 @@ _DEFAULT_TIMEOUT = httpx.Timeout(
timeout=COMPLETION_HTTP_FALLBACK_SECONDS,
connect=HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS,
)
_STREAMING_ERROR_BODY_READ_TIMEOUT_SECONDS = 5.0
_STREAMING_ERROR_BODY_READ_EXECUTOR = concurrent.futures.ThreadPoolExecutor(
max_workers=50,
thread_name_prefix="litellm-streaming-error-body-read",
)
def _prepare_request_data_and_content(
@ -386,17 +392,30 @@ def _safe_get_response_text(response: httpx.Response) -> str:
return ""
async def _safe_aread_response(response: httpx.Response) -> bytes:
async def _safe_aread_response(
response: httpx.Response, timeout: Optional[float] = None
) -> bytes:
"""Safely read async response body, falling back to empty bytes on errors."""
try:
if timeout is not None:
return await asyncio.wait_for(response.aread(), timeout=timeout)
return await response.aread()
except Exception:
return b""
def _safe_read_response(response: httpx.Response) -> bytes:
def _safe_read_response(
response: httpx.Response, timeout: Optional[float] = None
) -> bytes:
"""Safely read sync response body, falling back to empty bytes on errors."""
try:
if timeout is not None:
future = _STREAMING_ERROR_BODY_READ_EXECUTOR.submit(response.read)
try:
return future.result(timeout=timeout)
except Exception:
response.close()
return b""
return response.read()
except Exception:
return b""
@ -405,8 +424,19 @@ def _safe_read_response(response: httpx.Response) -> bytes:
def _raise_masked_sync_error(e: httpx.HTTPStatusError, stream: bool) -> None:
"""Raise a MaskedHTTPStatusError for sync HTTP handlers."""
if stream:
_body = mask_sensitive_info(_safe_read_response(e.response))
raise MaskedHTTPStatusError(e, message=_body, text=_body) from None
try:
_body = mask_sensitive_info(
_safe_read_response(
e.response,
timeout=_STREAMING_ERROR_BODY_READ_TIMEOUT_SECONDS,
)
)
raise MaskedHTTPStatusError(e, message=_body, text=_body) from None
finally:
try:
e.response.close()
except Exception:
pass
_text = mask_sensitive_info(_safe_get_response_text(e.response))
raise MaskedHTTPStatusError(e, message=_text, text=_text) from None
@ -414,8 +444,19 @@ def _raise_masked_sync_error(e: httpx.HTTPStatusError, stream: bool) -> None:
async def _raise_masked_async_error(e: httpx.HTTPStatusError, stream: bool) -> None:
"""Raise a MaskedHTTPStatusError for async HTTP handlers."""
if stream:
_body = mask_sensitive_info(await _safe_aread_response(e.response))
raise MaskedHTTPStatusError(e, message=_body, text=_body) from None
try:
_body = mask_sensitive_info(
await _safe_aread_response(
e.response,
timeout=_STREAMING_ERROR_BODY_READ_TIMEOUT_SECONDS,
)
)
raise MaskedHTTPStatusError(e, message=_body, text=_body) from None
finally:
try:
await e.response.aclose()
except Exception:
pass
_text = mask_sensitive_info(_safe_get_response_text(e.response))
raise MaskedHTTPStatusError(e, message=_text, text=_text) from None

1
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -5255,7 +5255,6 @@ class BaseLLMHTTPHandler:
headers = {
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json",
"OpenAI-Beta": "realtime=v1",
}
if extra_headers:

70
litellm/llms/hosted_vllm/chat/transformation.py
View file

@ -2,7 +2,18 @@
Translate from OpenAI's `/v1/chat/completions` to VLLM's `/v1/chat/completions`
"""
from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, cast, overload
from typing import (
Any,
Coroutine,
Dict,
List,
Literal,
Optional,
Tuple,
Union,
cast,
overload,
)
from litellm.litellm_core_utils.prompt_templates.common_utils import (
_get_image_mime_type_from_url,
@ -21,6 +32,61 @@ from ...openai.chat.gpt_transformation import OpenAIGPTConfig
class HostedVLLMChatConfig(OpenAIGPTConfig):
def _convert_custom_tools_to_function_tools(
self, tools: List[Dict[str, Any]]
) -> List[Dict[str, Any]]:
"""
vLLM chat completions currently accepts only OpenAI function tools.
Convert custom tools into function tools so request validation does not fail.
"""
converted_tools: List[Dict[str, Any]] = []
for idx, tool in enumerate(tools):
if not isinstance(tool, dict):
converted_tools.append(tool)
continue
if tool.get("type") != "custom":
converted_tools.append(tool)
continue
custom_tool = tool.get("custom", {})
if not isinstance(custom_tool, dict):
custom_tool = {}
tool_name = (
custom_tool.get("name") or tool.get("name") or f"custom_tool_{idx}"
)
tool_description = custom_tool.get("description") or tool.get("description")
tool_parameters = custom_tool.get("input_schema") or tool.get(
"input_schema"
)
if not isinstance(tool_parameters, dict):
tool_parameters = {
"type": "object",
"properties": {
"input": {
"type": "string",
"description": "Raw tool input payload.",
}
},
"required": ["input"],
}
function_tool: Dict[str, Any] = {
"type": "function",
"function": {
"name": str(tool_name),
"parameters": tool_parameters,
},
}
if isinstance(tool_description, str):
function_tool["function"]["description"] = tool_description
converted_tools.append(function_tool)
return converted_tools
def get_supported_openai_params(self, model: str) -> List[str]:
params = super().get_supported_openai_params(model)
params.extend(["reasoning_effort", "thinking"])
@ -39,6 +105,8 @@ class HostedVLLMChatConfig(OpenAIGPTConfig):
_tools = _remove_additional_properties(_tools)
# remove 'strict' from tools
_tools = _remove_strict_from_schema(_tools)
if isinstance(_tools, list):
_tools = self._convert_custom_tools_to_function_tools(_tools)
if _tools is not None:
non_default_params["tools"] = _tools

0
litellm/llms/nvidia_riva/__init__.py Normal file
View file

0
litellm/llms/nvidia_riva/audio_transcription/__init__.py Normal file
View file

232
litellm/llms/nvidia_riva/audio_transcription/audio_utils.py Normal file
View file

@ -0,0 +1,232 @@
"""
Audio resampling utilities for the NVIDIA Riva STT provider.
We intentionally avoid a hard dependency on ``ffmpeg`` so this works in
slim Python environments. Format coverage:
- ``soundfile`` handles wav / flac / ogg out of the box (libsndfile).
- ``audioread`` is tried for everything ``soundfile`` cannot decode (mp3,
m4a, mp4, webm, ...). This is a soft optional dependency.
If neither library can decode the input we raise a clear error instructing
the caller to convert the audio upstream.
"""
import io
import os
import tempfile
from dataclasses import dataclass
from typing import Any, Tuple, cast
from litellm.llms.nvidia_riva.audio_transcription.transformation import (
RIVA_TARGET_NUM_CHANNELS,
RIVA_TARGET_SAMPLE_RATE_HZ,
)
from litellm.llms.nvidia_riva.common_utils import NvidiaRivaException
# Keep this as Any: the module intentionally avoids importing numpy at module
# import time (optional dependency), and project-wide mypy config evaluates this
# file in contexts where conditional type aliases can degrade to "FloatArray?".
FloatArray = Any
_INSTALL_HINT = (
"Install Riva STT extras to enable automatic audio resampling: "
"`pip install 'litellm[stt-nvidia-riva]'`"
)
@dataclass
class ResampledAudio:
pcm_bytes: bytes
duration_seconds: float
sample_rate_hz: int
num_channels: int
def resample_to_riva_pcm(file_bytes: bytes) -> ResampledAudio:
"""
Decode ``file_bytes`` and produce 16 kHz mono LINEAR_PCM (int16 little
endian) suitable for streaming to Riva, plus the audio duration in
seconds (used for cost calculation when Riva does not return usage).
"""
try:
import numpy as np # type: ignore
except ImportError as e:
raise NvidiaRivaException(
status_code=500,
message=f"numpy is required for Riva audio resampling. {_INSTALL_HINT}",
) from e
samples_float, source_rate = _decode_to_float32(file_bytes)
# Downmix to mono by averaging channels.
if samples_float.ndim == 2 and samples_float.shape[1] > 1:
samples_float = samples_float.mean(axis=1)
elif samples_float.ndim == 2:
samples_float = samples_float[:, 0]
samples_float = np.asarray(samples_float, dtype=np.float32).ravel()
if source_rate != RIVA_TARGET_SAMPLE_RATE_HZ:
samples_float = _resample(
samples_float, source_rate, RIVA_TARGET_SAMPLE_RATE_HZ
)
# Clip + convert float [-1, 1] to int16 little-endian PCM.
np.clip(samples_float, -1.0, 1.0, out=samples_float)
pcm_int16 = (samples_float * 32767.0).astype("<i2")
pcm_bytes = pcm_int16.tobytes()
duration_seconds = float(pcm_int16.size) / float(RIVA_TARGET_SAMPLE_RATE_HZ)
return ResampledAudio(
pcm_bytes=pcm_bytes,
duration_seconds=duration_seconds,
sample_rate_hz=RIVA_TARGET_SAMPLE_RATE_HZ,
num_channels=RIVA_TARGET_NUM_CHANNELS,
)
def _decode_to_float32(file_bytes: bytes) -> Tuple["FloatArray", int]:
"""
Decode arbitrary audio bytes into a float32 array shaped either
``(n_samples,)`` (mono) or ``(n_samples, n_channels)`` plus the source
sample rate.
Tries ``soundfile`` first (wav/flac/ogg), then falls back to
``audioread`` for compressed formats. Raises a clear error if neither
works.
"""
import numpy as np # type: ignore
sf_error: Exception | None = None
try:
import soundfile as sf # type: ignore
with io.BytesIO(file_bytes) as buf:
data, source_rate = sf.read(buf, dtype="float32", always_2d=False)
return cast("FloatArray", data), int(source_rate)
except ImportError as e:
sf_error = e
except Exception as e:
# soundfile raises RuntimeError / LibsndfileError for formats it
# cannot decode (mp3 on older libsndfile, m4a, webm, ...).
sf_error = e
try:
import audioread # type: ignore
except ImportError as e:
raise NvidiaRivaException(
status_code=400,
message=(
"Could not decode audio for Riva STT. Install audio extras "
f"(`pip install 'litellm[stt-nvidia-riva]'`) or convert your "
f"audio to wav/flac/ogg before calling the API. "
f"Underlying error: {sf_error}"
),
) from e
# audioread backends (FFmpeg subprocess, GStreamer, Core Audio) require a
# filesystem path, so spill the bytes to a temp file. mkstemp is portable
# to Windows where re-opening a NamedTemporaryFile is not allowed.
fd, tmp_path = tempfile.mkstemp(suffix=".audio")
try:
with os.fdopen(fd, "wb") as tmp_file:
tmp_file.write(file_bytes)
try:
with audioread.audio_open(tmp_path) as src:
source_rate = int(src.samplerate)
channels = int(src.channels)
chunks = []
for buf in src:
chunks.append(np.frombuffer(buf, dtype=np.int16))
if not chunks:
raise NvidiaRivaException(
status_code=400,
message="Audio decode produced no samples.",
)
interleaved = np.concatenate(chunks).astype(np.float32) / 32768.0
if channels > 1:
interleaved = interleaved.reshape(-1, channels)
return cast("FloatArray", interleaved), source_rate
except NvidiaRivaException:
raise
except Exception as e:
raise NvidiaRivaException(
status_code=400,
message=(
"Could not decode audio for Riva STT. Convert your audio to "
f"wav/flac/ogg before calling the API. Underlying error: {e}"
),
) from e
finally:
try:
os.unlink(tmp_path)
except OSError:
pass
def _resample(
samples: "FloatArray", source_rate: int, target_rate: int
) -> "FloatArray":
"""
Resample mono float32 ``samples`` from ``source_rate`` to ``target_rate``.
Prefers high-quality polyphase resampling when ``soxr`` or ``scipy`` is
available (anti-aliased, important for downsampling 44.1/48 kHz -> 16 kHz
where naive interpolation folds high frequencies back into the speech
band). Falls back to linear interpolation if neither is installed —
acceptable for speech-only mono input but lossy for wideband content.
"""
import numpy as np # type: ignore
if source_rate == target_rate or samples.size == 0:
return samples
try:
import soxr # type: ignore
return cast(
"FloatArray",
np.asarray(
soxr.resample(samples, source_rate, target_rate), dtype=np.float32
),
)
except ImportError:
pass
try:
from math import gcd
from scipy.signal import resample_poly # type: ignore
g = gcd(int(source_rate), int(target_rate))
up = int(target_rate) // g
down = int(source_rate) // g
return cast(
"FloatArray", np.asarray(resample_poly(samples, up, down), dtype=np.float32)
)
except ImportError:
pass
return _linear_resample(samples, source_rate, target_rate)
def _linear_resample(
samples: "FloatArray", source_rate: int, target_rate: int
) -> "FloatArray":
"""Linear-interpolation fallback. See :func:`_resample` for caveats."""
import numpy as np # type: ignore
duration = samples.size / float(source_rate)
target_length = int(round(duration * target_rate))
if target_length <= 1:
return samples.astype(np.float32)
src_indices = np.linspace(0, samples.size - 1, num=target_length, dtype=np.float64)
left = np.floor(src_indices).astype(np.int64)
right = np.minimum(left + 1, samples.size - 1)
frac = (src_indices - left).astype(np.float32)
return ((1.0 - frac) * samples[left] + frac * samples[right]).astype(np.float32)

444
litellm/llms/nvidia_riva/audio_transcription/handler.py Normal file
View file

@ -0,0 +1,444 @@
"""
NVIDIA Riva STT handler.
This module bridges litellm's transcription dispatch to NVIDIA Riva's gRPC
streaming ASR API. We do *not* go through ``base_llm_http_handler`` because
Riva is gRPC-only: HTTP-shaped abstractions (``httpx.Response``,
``api_base/v1/...`` URLs, multipart bodies) do not apply.
The handler is intentionally a thin orchestration layer:
1. Resample the inbound audio to 16 kHz mono LINEAR_PCM (Riva's required
wire format).
2. Build ``RecognitionConfig`` / ``StreamingRecognitionConfig`` protobufs
from the structured dict produced by
:class:`NvidiaRivaAudioTranscriptionConfig`.
3. Construct ``riva.client.Auth`` honoring NVCF (function-id metadata + TLS)
vs self-hosted (any host:port, optional TLS) modes.
4. Stream the audio through Riva's ``streaming_response_generator`` and
aggregate ``is_final`` results into a single transcript.
5. Return a normalized ``TranscriptionResponse`` with ``duration`` exposed
on ``_hidden_params`` so cost calculation works.
``riva-client`` is imported lazily so ``litellm`` core remains usable
without the optional STT extras installed.
"""
import asyncio
import inspect
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
from litellm.litellm_core_utils.audio_utils.utils import (
get_audio_file_name,
process_audio_file,
)
from litellm.llms.nvidia_riva.audio_transcription.audio_utils import (
resample_to_riva_pcm,
)
from litellm.llms.nvidia_riva.audio_transcription.transformation import (
NvidiaRivaAudioTranscriptionConfig,
RIVA_TARGET_NUM_CHANNELS,
RIVA_TARGET_SAMPLE_RATE_HZ,
)
from litellm.llms.nvidia_riva.common_utils import (
NvidiaRivaException,
grpc_error_to_litellm_exception,
)
from litellm.types.utils import FileTypes, TranscriptionResponse
from litellm.utils import convert_to_model_response_object
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import (
Logging as LiteLLMLoggingObj,
)
# Stream audio to Riva in ~50 ms slices (1600 samples at 16 kHz). Matches
# NVIDIA's recommended chunk size for streaming ASR — small enough for
# responsive endpointing, large enough to keep per-RPC overhead low.
_DEFAULT_CHUNK_SAMPLES = 1600
_DEFAULT_CHUNK_BYTES = _DEFAULT_CHUNK_SAMPLES * 2 # int16 = 2 bytes/sample
_RIVA_INSTALL_HINT = (
"NVIDIA Riva client is not installed. "
"Install with `pip install 'litellm[stt-nvidia-riva]'`."
)
class NvidiaRivaAudioTranscription:
"""Sync + async entry point for Riva ASR."""
def audio_transcriptions(
self,
model: str,
audio_file: FileTypes,
optional_params: dict,
litellm_params: dict,
model_response: TranscriptionResponse,
timeout: float,
logging_obj: "LiteLLMLoggingObj",
api_key: Optional[str],
api_base: Optional[str],
atranscription: bool = False,
provider_config: Optional[NvidiaRivaAudioTranscriptionConfig] = None,
):
if provider_config is None:
provider_config = NvidiaRivaAudioTranscriptionConfig()
if atranscription:
return self.async_audio_transcriptions(
model=model,
audio_file=audio_file,
optional_params=optional_params,
litellm_params=litellm_params,
model_response=model_response,
timeout=timeout,
logging_obj=logging_obj,
api_key=api_key,
api_base=api_base,
provider_config=provider_config,
)
return self._run_sync(
model=model,
audio_file=audio_file,
optional_params=optional_params,
litellm_params=litellm_params,
model_response=model_response,
timeout=timeout,
logging_obj=logging_obj,
api_key=api_key,
api_base=api_base,
provider_config=provider_config,
atranscription=atranscription,
)
async def async_audio_transcriptions(
self,
model: str,
audio_file: FileTypes,
optional_params: dict,
litellm_params: dict,
model_response: TranscriptionResponse,
timeout: float,
logging_obj: "LiteLLMLoggingObj",
api_key: Optional[str],
api_base: Optional[str],
provider_config: Optional[NvidiaRivaAudioTranscriptionConfig] = None,
) -> TranscriptionResponse:
# ``riva-client`` exposes a sync streaming generator, so we offload
# the blocking call to a worker thread to keep the event loop free.
return await asyncio.to_thread(
self._run_sync,
model=model,
audio_file=audio_file,
optional_params=optional_params,
litellm_params=litellm_params,
model_response=model_response,
timeout=timeout,
logging_obj=logging_obj,
api_key=api_key,
api_base=api_base,
provider_config=provider_config or NvidiaRivaAudioTranscriptionConfig(),
atranscription=True,
)
def _run_sync(
self,
model: str,
audio_file: FileTypes,
optional_params: dict,
litellm_params: dict,
model_response: TranscriptionResponse,
timeout: float,
logging_obj: "LiteLLMLoggingObj",
api_key: Optional[str],
api_base: Optional[str],
provider_config: NvidiaRivaAudioTranscriptionConfig,
atranscription: bool = False,
) -> TranscriptionResponse:
if not api_base:
raise NvidiaRivaException(
status_code=400,
message=(
"NVIDIA Riva requires `api_base` (host:port for the gRPC "
"endpoint, e.g. `grpc.nvcf.nvidia.com:443` or "
"`localhost:50051`). Set it in litellm_params or via "
"NVIDIA_RIVA_API_BASE."
),
)
processed = process_audio_file(audio_file)
resampled = resample_to_riva_pcm(processed.file_content)
request_payload = provider_config.transform_audio_transcription_request(
model=model,
audio_file=audio_file,
optional_params=optional_params,
litellm_params={
**litellm_params,
"api_base": api_base,
"api_key": api_key,
},
).data
if not isinstance(request_payload, dict):
raise NvidiaRivaException(
status_code=500,
message="NvidiaRivaAudioTranscriptionConfig produced an unexpected request payload type.",
)
recognition_config_dict: Dict[str, Any] = request_payload["recognition_config"]
# The wire format is fixed by our resampler; override anything stale
# the caller passed in so the gRPC config matches the bytes we send.
recognition_config_dict["sample_rate_hertz"] = RIVA_TARGET_SAMPLE_RATE_HZ
recognition_config_dict["audio_channel_count"] = RIVA_TARGET_NUM_CHANNELS
recognition_config_dict["encoding"] = "LINEAR_PCM"
response_format = request_payload.get("response_format") or "json"
timestamp_granularities = request_payload.get("timestamp_granularities")
riva_module, riva_asr_module = _import_riva()
auth_obj = self._construct_auth(
riva_module=riva_module,
api_base=api_base,
api_key=api_key,
optional_params=optional_params,
)
recognition_config = self._build_recognition_config_proto(
riva_asr_module=riva_asr_module,
recognition_config_dict=recognition_config_dict,
)
streaming_config = riva_asr_module.StreamingRecognitionConfig(
config=recognition_config, interim_results=False
)
logging_obj.pre_call(
input=None,
api_key=api_key,
additional_args={
"api_base": api_base,
"atranscription": atranscription,
"complete_input_dict": {
"recognition_config": recognition_config_dict,
"nvcf_function_id_set": bool(
optional_params.get("nvcf_function_id")
),
"use_ssl": optional_params.get("use_ssl"),
},
},
)
try:
asr_service = riva_module.ASRService(auth_obj)
audio_chunks = self._iter_audio_chunks(resampled.pcm_bytes)
stream_kwargs: Dict[str, Any] = {
"audio_chunks": audio_chunks,
"streaming_config": streaming_config,
}
# Forward the deadline so the stream cannot block forever if the
# server stalls. Older riva-client versions do not accept a
# ``timeout`` kwarg, so pass it only when supported.
if timeout is not None and self._supports_timeout_kwarg(
asr_service.streaming_response_generator
):
stream_kwargs["timeout"] = float(timeout)
stream = asr_service.streaming_response_generator(**stream_kwargs)
final_results = self._collect_final_results(stream)
except NvidiaRivaException:
raise
except Exception as e:
raise grpc_error_to_litellm_exception(e) from e
transcription = NvidiaRivaAudioTranscriptionConfig.build_transcription_response(
final_results=final_results,
response_format=response_format,
duration_seconds=resampled.duration_seconds,
timestamp_granularities=timestamp_granularities,
)
stringified_response = dict(transcription)
logging_obj.post_call(
input=get_audio_file_name(audio_file),
api_key=api_key,
additional_args={"complete_input_dict": recognition_config_dict},
original_response=stringified_response,
)
hidden_params = {
"model": model,
"custom_llm_provider": "nvidia_riva",
"audio_transcription_duration": resampled.duration_seconds,
}
final_response: TranscriptionResponse = convert_to_model_response_object( # type: ignore
response_object=stringified_response,
model_response_object=model_response,
hidden_params=hidden_params,
response_type="audio_transcription",
)
return final_response
def _construct_auth(
self,
riva_module: Any,
api_base: str,
api_key: Optional[str],
optional_params: dict,
) -> Any:
"""
Build a ``riva.client.Auth`` object.
- When ``nvcf_function_id`` is provided we attach the NVCF
``function-id`` and bearer ``authorization`` metadata, and default
``use_ssl`` to True (NVCF endpoints are TLS-only).
- Otherwise (self-hosted) we default ``use_ssl`` to False but still
honor an explicit override — self-hosted Riva behind an ingress
with TLS termination is a real deployment topology.
"""
nvcf_function_id = optional_params.get("nvcf_function_id")
use_ssl_override = optional_params.get("use_ssl")
use_ssl = (
bool(use_ssl_override)
if use_ssl_override is not None
else bool(nvcf_function_id)
)
metadata: List[Tuple[str, str]] = []
if nvcf_function_id:
metadata.append(("function-id", str(nvcf_function_id)))
if api_key:
metadata.append(("authorization", f"Bearer {api_key}"))
try:
return riva_module.Auth(
uri=api_base, use_ssl=use_ssl, metadata_args=metadata
)
except TypeError:
# Older riva-client signatures used positional-only args.
return riva_module.Auth(None, use_ssl, api_base, metadata)
def _build_recognition_config_proto(
self, riva_asr_module: Any, recognition_config_dict: Dict[str, Any]
):
encoding_name = (
recognition_config_dict.get("encoding") or "LINEAR_PCM"
).upper()
encoding_enum = getattr(
riva_asr_module.AudioEncoding,
encoding_name,
riva_asr_module.AudioEncoding.LINEAR_PCM,
)
config = riva_asr_module.RecognitionConfig(
encoding=encoding_enum,
sample_rate_hertz=int(recognition_config_dict["sample_rate_hertz"]),
language_code=recognition_config_dict["language_code"],
audio_channel_count=int(recognition_config_dict["audio_channel_count"]),
enable_automatic_punctuation=bool(
recognition_config_dict.get("enable_automatic_punctuation", True)
),
enable_word_time_offsets=bool(
recognition_config_dict.get("enable_word_time_offsets", False)
),
max_alternatives=int(recognition_config_dict.get("max_alternatives", 1)),
model=recognition_config_dict.get("model", "") or "",
verbatim_transcripts=bool(
recognition_config_dict.get("verbatim_transcripts", False)
),
profanity_filter=bool(
recognition_config_dict.get("profanity_filter", False)
),
)
endpointing = recognition_config_dict.get("endpointing_config")
if isinstance(endpointing, dict) and endpointing:
try:
ep = riva_asr_module.EndpointingConfig(**endpointing)
config.endpointing_config.CopyFrom(ep)
except Exception:
# If the user supplied an unknown EndpointingConfig field
# (older Riva server), fall back to Riva's defaults rather
# than failing the whole request.
pass
return config
@staticmethod
def _supports_timeout_kwarg(callable_obj: Any) -> bool:
try:
sig = inspect.signature(callable_obj)
except (TypeError, ValueError):
return False
params = sig.parameters
if "timeout" in params:
return True
return any(p.kind == inspect.Parameter.VAR_KEYWORD for p in params.values())
@staticmethod
def _iter_audio_chunks(pcm_bytes: bytes):
for offset in range(0, len(pcm_bytes), _DEFAULT_CHUNK_BYTES):
chunk = pcm_bytes[offset : offset + _DEFAULT_CHUNK_BYTES]
if not chunk:
continue
yield chunk
@staticmethod
def _collect_final_results(stream) -> List[Dict[str, Any]]:
"""
Walk the gRPC stream, ignore empty / non-final chunks, and return a
list of normalized final-result dicts. Matching the user's note: the
``id`` blocks with no ``results`` are streaming heartbeats and must
be skipped.
"""
final_results: List[Dict[str, Any]] = []
for response in stream:
results = getattr(response, "results", None) or []
for result in results:
if not getattr(result, "is_final", False):
continue
alternatives = getattr(result, "alternatives", None) or []
if not alternatives:
continue
top = alternatives[0]
transcript = getattr(top, "transcript", "") or ""
words_proto = getattr(top, "words", None) or []
words = []
for word in words_proto:
words.append(
{
"word": getattr(word, "word", ""),
"start_time_ms": int(getattr(word, "start_time", 0) or 0),
"end_time_ms": int(getattr(word, "end_time", 0) or 0),
}
)
final_results.append({"transcript": transcript, "words": words})
return final_results
def _import_riva():
"""
Lazy import of ``riva.client`` and ``riva.client.proto.riva_asr_pb2``.
We try the SDK first (preferred) and fall back to importing the proto
module separately when the SDK packaging changes between versions.
"""
try:
import riva.client as riva_client # type: ignore
except ImportError as e:
raise NvidiaRivaException(status_code=500, message=_RIVA_INSTALL_HINT) from e
riva_asr_module = riva_client
if not hasattr(riva_asr_module, "RecognitionConfig"):
try:
import riva.client.proto.riva_asr_pb2 as riva_asr_pb2 # type: ignore
riva_asr_module = riva_asr_pb2
except ImportError as e:
raise NvidiaRivaException(
status_code=500, message=_RIVA_INSTALL_HINT
) from e
return riva_client, riva_asr_module

284
litellm/llms/nvidia_riva/audio_transcription/transformation.py Normal file
View file

@ -0,0 +1,284 @@
"""
Translates from OpenAI's `/v1/audio/transcriptions` to NVIDIA Riva's gRPC
streaming recognize API.
Riva is gRPC-only, so unlike most providers in this directory the request
"transformation" produced here is a structured dict consumed directly by the
gRPC handler (rather than HTTP form-data). The handler builds Riva
``RecognitionConfig`` / ``StreamingRecognitionConfig`` protobufs from this
dict at call time.
Reference: https://docs.nvidia.com/deeplearning/riva/user-guide/docs/asr/asr-overview.html
"""
from typing import Any, Dict, List, Optional, Union
from httpx import Headers, Response
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.llms.openai import (
AllMessageValues,
OpenAIAudioTranscriptionOptionalParams,
)
from litellm.types.utils import FileTypes, TranscriptionResponse
from ...base_llm.audio_transcription.transformation import (
AudioTranscriptionRequestData,
BaseAudioTranscriptionConfig,
)
from ..common_utils import NvidiaRivaException
# Riva expects a fixed wire format for the audio chunks we stream in.
RIVA_TARGET_SAMPLE_RATE_HZ = 16000
RIVA_TARGET_NUM_CHANNELS = 1
RIVA_TARGET_ENCODING = "LINEAR_PCM"
class NvidiaRivaAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
"""
Config for NVIDIA Riva ASR (gRPC).
Supports both NVCF-hosted (``api_base=grpc.nvcf.nvidia.com:443`` +
``nvcf_function_id``) and self-hosted deployments (any ``host:port``,
optional TLS via ``use_ssl``).
"""
def get_supported_openai_params(
self, model: str
) -> List[OpenAIAudioTranscriptionOptionalParams]:
# Riva natively understands language + word timestamps.
# `response_format` is honored at response-shaping time in the handler.
return ["language", "response_format", "timestamp_granularities"]
def map_openai_params(
self,
non_default_params: dict,
optional_params: dict,
model: str,
drop_params: bool,
) -> dict:
for key, value in non_default_params.items():
if value is None:
continue
if key == "language":
optional_params["language_code"] = self._normalize_language_code(value)
elif key == "timestamp_granularities":
# OpenAI accepts ["word"], ["segment"], or both. Riva only
# natively exposes word timing, so we toggle it on whenever
# "word" is requested. Segment timing is reconstructed in the
# response transformer.
if isinstance(value, list) and "word" in value:
optional_params["enable_word_time_offsets"] = True
optional_params["timestamp_granularities"] = value
elif key == "response_format":
# Stored verbatim; consumed by transform_audio_transcription_response.
optional_params["response_format"] = value
else:
optional_params[key] = value
return optional_params
def get_error_class(
self, error_message: str, status_code: int, headers: Union[dict, Headers]
) -> BaseLLMException:
return NvidiaRivaException(
message=error_message, status_code=status_code, headers=headers
)
def transform_audio_transcription_request(
self,
model: str,
audio_file: FileTypes,
optional_params: dict,
litellm_params: dict,
) -> AudioTranscriptionRequestData:
"""
Build a structured dict that the gRPC handler consumes. We do *not*
construct protobufs here, so this module remains importable without
``nvidia-riva-client`` being installed (matching how other providers
defer SDK imports to handler-call time).
"""
recognition_config = self._build_recognition_config_dict(
model=model,
optional_params=optional_params,
)
endpointing_config = self._build_endpointing_config_dict(optional_params)
if endpointing_config is not None:
recognition_config["endpointing_config"] = endpointing_config
request_payload: Dict[str, Any] = {
"recognition_config": recognition_config,
"response_format": optional_params.get("response_format") or "json",
"timestamp_granularities": optional_params.get("timestamp_granularities"),
}
return AudioTranscriptionRequestData(data=request_payload, files=None)
def transform_audio_transcription_response(
self,
raw_response: Response,
) -> TranscriptionResponse:
# Not used: Riva responses come from a gRPC stream, not an httpx
# response. The handler calls _build_transcription_response directly.
raise NotImplementedError(
"NvidiaRivaAudioTranscriptionConfig.transform_audio_transcription_response "
"is not used. The handler builds the TranscriptionResponse directly "
"from Riva's gRPC streaming results."
)
def validate_environment(
self,
headers: dict,
model: str,
messages: List[AllMessageValues],
optional_params: dict,
litellm_params: dict,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
) -> dict:
# gRPC auth is constructed in the handler, not via HTTP headers.
return headers
def _build_recognition_config_dict(
self, model: str, optional_params: dict
) -> Dict[str, Any]:
"""
Build the Riva ``RecognitionConfig`` shape as a plain dict.
``model`` is intentionally left empty when the user has not supplied
``riva_model_name``. Riva auto-selects the right deployment from
``language_code`` + ``sample_rate_hertz``. NVIDIA's internal
deployment names (e.g. ``parakeet-1.1b-en-US-asr-streaming-...``)
change across NIM versions, regions, and self-hosted builds, so
hardcoding any name here would break unpredictably.
"""
return {
"language_code": optional_params.get("language_code", "en-US"),
"sample_rate_hertz": optional_params.get(
"sample_rate_hertz", RIVA_TARGET_SAMPLE_RATE_HZ
),
"encoding": optional_params.get("encoding", RIVA_TARGET_ENCODING),
"audio_channel_count": optional_params.get(
"audio_channel_count", RIVA_TARGET_NUM_CHANNELS
),
"enable_automatic_punctuation": optional_params.get(
"enable_automatic_punctuation", True
),
"enable_word_time_offsets": bool(
optional_params.get("enable_word_time_offsets", False)
),
"max_alternatives": optional_params.get("max_alternatives", 1),
"model": optional_params.get("riva_model_name", ""),
"verbatim_transcripts": optional_params.get("verbatim_transcripts", False),
"profanity_filter": optional_params.get("profanity_filter", False),
}
def _build_endpointing_config_dict(
self, optional_params: dict
) -> Optional[Dict[str, Any]]:
"""
Translate an OpenAI-style ``chunking_strategy`` into Riva's
``EndpointingConfig`` shape, or pass through an explicit
``endpointing_config`` dict.
Returns ``None`` when neither is provided so Riva uses its built-in
VAD defaults.
"""
explicit = optional_params.get("endpointing_config")
if isinstance(explicit, dict):
return dict(explicit)
chunking = optional_params.get("chunking_strategy")
if chunking in (None, "auto"):
return None
if isinstance(chunking, dict) and chunking.get("type") == "server_vad":
config: Dict[str, Any] = {}
if "threshold" in chunking:
threshold = float(chunking["threshold"])
config["start_threshold"] = threshold
config["stop_threshold"] = threshold
if "silence_duration_ms" in chunking:
config["stop_history"] = int(chunking["silence_duration_ms"])
if "prefix_padding_ms" in chunking:
config["stop_history_eou"] = int(chunking["prefix_padding_ms"])
return config or None
return None
@staticmethod
def _normalize_language_code(language: str) -> str:
"""
OpenAI accepts bare ISO-639 codes like ``en``; Riva requires BCP-47
like ``en-US``. Normalize the most common bare codes; pass through
anything that already looks like BCP-47.
"""
if not isinstance(language, str) or not language:
return "en-US"
if "-" in language:
return language
bare_to_bcp47 = {
"en": "en-US",
"es": "es-ES",
"de": "de-DE",
"fr": "fr-FR",
"it": "it-IT",
"pt": "pt-BR",
"ja": "ja-JP",
"ko": "ko-KR",
"zh": "zh-CN",
"ru": "ru-RU",
"hi": "hi-IN",
"ar": "ar-SA",
}
return bare_to_bcp47.get(language.lower(), language)
@staticmethod
def build_transcription_response(
final_results: List[Dict[str, Any]],
response_format: str,
duration_seconds: Optional[float],
timestamp_granularities: Optional[List[str]],
) -> TranscriptionResponse:
"""
Aggregate a list of normalized "final result" dicts into a
``TranscriptionResponse`` shaped for the requested ``response_format``.
Each entry in ``final_results`` is expected to look like::
{
"transcript": str,
"words": [{"word": str, "start_time_ms": int, "end_time_ms": int}, ...],
}
which the handler produces by walking the gRPC stream and keeping
only ``result.is_final`` entries (empty/non-final chunks are
ignored).
"""
full_transcript = "".join(
(item.get("transcript") or "") for item in final_results
).strip()
response = TranscriptionResponse(text=full_transcript)
response["task"] = "transcribe"
if response_format == "verbose_json":
words: List[Dict[str, Any]] = []
if timestamp_granularities and "word" in timestamp_granularities:
for item in final_results:
for word in item.get("words", []) or []:
words.append(
{
"word": word.get("word", ""),
"start": (float(word.get("start_time_ms", 0)) / 1000.0),
"end": float(word.get("end_time_ms", 0)) / 1000.0,
}
)
if words:
response["words"] = words
if duration_seconds is not None:
response["duration"] = duration_seconds
return response

92
litellm/llms/nvidia_riva/common_utils.py Normal file
View file

@ -0,0 +1,92 @@
"""
Common utilities and exceptions for the NVIDIA Riva STT provider
"""
from typing import Any, Optional
from litellm.llms.base_llm.chat.transformation import BaseLLMException
class NvidiaRivaException(BaseLLMException):
"""
Exception raised for NVIDIA Riva (gRPC) errors.
``status_code`` is an HTTP-equivalent code derived from the underlying
gRPC ``StatusCode`` (when available) so that litellm's existing error
classifiers (RateLimitError, AuthenticationError, etc.) keep working.
"""
pass
# Mapping from grpc.StatusCode.name -> equivalent HTTP status code.
# Kept as a plain dict (rather than importing grpc enums) so this module is
# importable without grpc installed.
_GRPC_STATUS_CODE_TO_HTTP: dict = {
"OK": 200,
"CANCELLED": 499,
"UNKNOWN": 500,
"INVALID_ARGUMENT": 400,
"DEADLINE_EXCEEDED": 504,
"NOT_FOUND": 404,
"ALREADY_EXISTS": 409,
"PERMISSION_DENIED": 403,
"RESOURCE_EXHAUSTED": 429,
"FAILED_PRECONDITION": 400,
"ABORTED": 409,
"OUT_OF_RANGE": 400,
"UNIMPLEMENTED": 501,
"INTERNAL": 500,
"UNAVAILABLE": 503,
"DATA_LOSS": 500,
"UNAUTHENTICATED": 401,
}
def _extract_grpc_status_name(error: Any) -> Optional[str]:
"""
Best-effort extraction of a gRPC StatusCode name from an arbitrary error.
Works for ``grpc.RpcError`` instances (which expose ``.code()``) as well
as plain exceptions whose string representation contains a status name.
"""
code_fn = getattr(error, "code", None)
if callable(code_fn):
try:
code = code_fn()
except Exception:
code = None
name = getattr(code, "name", None)
if isinstance(name, str):
return name
return None
def _extract_grpc_details(error: Any) -> Optional[str]:
"""Best-effort extraction of a human-readable detail string from a gRPC error."""
details_fn = getattr(error, "details", None)
if callable(details_fn):
try:
details = details_fn()
except Exception:
details = None
if isinstance(details, str) and details:
return details
return None
def grpc_error_to_litellm_exception(error: Exception) -> NvidiaRivaException:
"""
Convert a gRPC error (or any exception raised from the Riva client) into
a ``NvidiaRivaException`` with an appropriate HTTP-equivalent status code.
"""
status_name = _extract_grpc_status_name(error)
http_status = _GRPC_STATUS_CODE_TO_HTTP.get(status_name or "", 500)
detail = _extract_grpc_details(error) or str(error)
message = (
f"NVIDIA Riva gRPC error ({status_name}): {detail}"
if status_name
else f"NVIDIA Riva error: {detail}"
)
return NvidiaRivaException(status_code=http_status, message=message)

24
litellm/llms/openai/chat/guardrail_translation/handler.py
View file

@ -21,7 +21,9 @@ from litellm._logging import verbose_proxy_logger
from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
from litellm.llms.base_llm.guardrail_translation.utils import (
effective_skip_system_message_for_guardrail,
effective_skip_tool_message_for_guardrail,
openai_messages_without_system,
openai_messages_without_tool,
)
from litellm.main import stream_chunk_builder
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam
@ -73,6 +75,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
return data
skip_system = effective_skip_system_message_for_guardrail(guardrail_to_apply)
skip_tool = effective_skip_tool_message_for_guardrail(guardrail_to_apply)
texts_to_check: List[str] = []
images_to_check: List[str] = []
@ -91,6 +94,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
text_task_mappings=text_task_mappings,
tool_call_task_mappings=tool_call_task_mappings,
skip_system_message=skip_system,
skip_tool_message=skip_tool,
)
# Step 2: Apply guardrail to all texts and tool calls in batch
@ -102,11 +106,15 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
inputs["tool_calls"] = tool_calls_to_check # type: ignore
structured_messages = self.get_structured_messages(data)
if structured_messages:
inputs["structured_messages"] = (
openai_messages_without_system(structured_messages)
if skip_system
else structured_messages
)
if skip_system:
structured_messages = openai_messages_without_system(
structured_messages
)
if skip_tool:
structured_messages = openai_messages_without_tool(
structured_messages
)
inputs["structured_messages"] = structured_messages
# Pass tools (function definitions) to the guardrail
tools = data.get("tools")
if tools:
@ -176,13 +184,17 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
text_task_mappings: List[Tuple[int, Optional[int]]],
tool_call_task_mappings: List[Tuple[int, int]],
skip_system_message: bool = False,
skip_tool_message: bool = False,
) -> None:
"""
Extract text content, images, and tool calls from a message.
Override this method to customize text/image/tool call extraction logic.
"""
if skip_system_message and str(message.get("role") or "").lower() == "system":
role = str(message.get("role") or "").lower()
if skip_system_message and role == "system":
return
if skip_tool_message and role == "tool":
return
content = message.get("content", None)

46
litellm/llms/openai/realtime/handler.py
View file

@ -6,12 +6,15 @@ This requires websockets, and is currently only supported on LiteLLM Proxy.
from typing import Any, Optional, cast
from litellm._logging import _redact_string
from litellm._logging import _redact_string, verbose_logger
from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
from litellm.types.realtime import RealtimeQueryParams
from ....litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from ....litellm_core_utils.realtime_streaming import RealTimeStreaming
from ....litellm_core_utils.realtime_streaming import (
RealTimeStreaming,
client_sent_openai_beta_realtime_header,
)
from ....llms.custom_httpx.http_handler import get_shared_realtime_ssl_context
from ..openai import OpenAIChatCompletion
@ -33,21 +36,24 @@ class OpenAIRealtime(OpenAIChatCompletion):
"""
return "https://api.openai.com/"
def _get_additional_headers(self, api_key: str) -> dict:
def _get_additional_headers(
self,
api_key: str,
*,
openai_beta_realtime: bool = False,
) -> dict:
"""
Get additional headers beyond Authorization.
Override this in subclasses to customize headers (e.g., remove OpenAI-Beta).
Headers for the upstream OpenAI Realtime WebSocket.
Args:
api_key: API key for authentication
Returns:
Dictionary of additional headers
When the client sent ``OpenAI-Beta: realtime=v1`` on the proxy WebSocket,
``openai_beta_realtime`` is True and the same header is forwarded upstream
so the legacy beta API is used. GA clients omit that header on the client
connection and must send GA-shaped ``session.update`` payloads.
"""
return {
"Authorization": f"Bearer {api_key}",
"OpenAI-Beta": "realtime=v1",
}
headers: dict = {"Authorization": f"Bearer {api_key}"}
if openai_beta_realtime:
headers["OpenAI-Beta"] = "realtime=v1"
return headers
def _get_ssl_config(self, url: str) -> Any:
"""
@ -120,8 +126,16 @@ class OpenAIRealtime(OpenAIChatCompletion):
# Get provider-specific SSL configuration
ssl_config = self._get_ssl_config(url)
# Get provider-specific headers
headers = self._get_additional_headers(api_key)
openai_beta_realtime = client_sent_openai_beta_realtime_header(websocket)
if not openai_beta_realtime:
verbose_logger.debug(
"OpenAI Realtime: connecting with GA protocol (no OpenAI-Beta header). "
"If your client expects beta event names, add 'OpenAI-Beta: realtime=v1' "
"to the WebSocket headers sent to the LiteLLM proxy."
)
headers = self._get_additional_headers(
api_key, openai_beta_realtime=openai_beta_realtime
)
# Log a masked request preview consistent with other endpoints.
logging_obj.pre_call(

12
litellm/llms/ovhcloud/audio_transcription/transformation.py
View file

@ -156,5 +156,17 @@ class OVHCloudAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
text = response_json.get("text") or response_json.get("transcript") or ""
response = TranscriptionResponse(text=text)
# OVHCloud field migration (deadline: 2026-05-11):
# `duration` is replaced by `seconds` in STT responses.
# Prefer `seconds`, fall back to `duration`, normalize to `duration`
# so downstream consumers see a consistent key.
duration = (
response_json["seconds"]
if "seconds" in response_json and response_json["seconds"] is not None
else response_json.get("duration")
)
if duration is not None:
response_json["duration"] = duration
response._hidden_params = response_json
return response

15
litellm/llms/ovhcloud/chat/transformation.py
View file

@ -13,6 +13,7 @@ from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
from litellm.llms.ovhcloud.utils import OVHCloudException
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.llms.openai import AllMessageValues
@ -98,10 +99,16 @@ class OVHCloudChatCompletionStreamingHandler(BaseModelResponseIterator):
new_choices = []
for choice in chunk["choices"]:
if "delta" in choice and "reasoning" in choice["delta"]:
choice["delta"]["reasoning_content"] = choice["delta"].get(
"reasoning"
)
if "delta" in choice:
delta = choice["delta"]
# OVHCloud field migration (deadline: 2026-05-11):
# `reasoning_content` is replaced by `reasoning`.
# Normalise to `reasoning_content` so downstream consumers
# see a consistent key during the transition window.
reasoning_new = delta.get("reasoning")
reasoning_legacy = delta.get("reasoning_content")
if reasoning_new is not None and reasoning_legacy is None:
delta["reasoning_content"] = reasoning_new
new_choices.append(choice)
return ModelResponseStream(

51
litellm/llms/sagemaker/common_utils.py
View file

@ -9,7 +9,27 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.utils import GenericStreamingChunk as GChunk
from litellm.types.utils import StreamingChatCompletionChunk
_response_stream_shape_cache = None
def _load_sagemaker_response_stream_shape():
try:
from botocore.loaders import Loader
from botocore.model import ServiceModel
loader = Loader()
service_dict = loader.load_service_model("sagemaker-runtime", "service-2")
return ServiceModel(service_dict).shape_for(
"InvokeEndpointWithResponseStreamOutput"
)
except Exception as e:
verbose_logger.warning(
"litellm: could not pre-load sagemaker-runtime response stream shape "
"— SageMaker event-stream decoding will be unavailable. Error: %s",
e,
)
return None
SAGEMAKER_RESPONSE_STREAM_SHAPE = _load_sagemaker_response_stream_shape()
class SagemakerError(BaseLLMException):
@ -187,8 +207,18 @@ class AWSEventStreamDecoder:
verbose_logger.error(f"Final error parsing accumulated JSON: {e}")
def _parse_message_from_event(self, event) -> Optional[str]:
if SAGEMAKER_RESPONSE_STREAM_SHAPE is None:
raise SagemakerError(
status_code=500,
message=(
"SageMaker event-stream shape could not be loaded from botocore. "
"Ensure botocore is correctly installed."
),
)
response_dict = event.to_response_dict()
parsed_response = self.parser.parse(response_dict, get_response_stream_shape())
parsed_response = self.parser.parse(
response_dict, SAGEMAKER_RESPONSE_STREAM_SHAPE
)
if response_dict["status_code"] != 200:
raise ValueError(f"Bad response code, expected 200: {response_dict}")
@ -204,20 +234,3 @@ class AWSEventStreamDecoder:
return None
return chunk.decode() # type: ignore[no-any-return]
def get_response_stream_shape():
global _response_stream_shape_cache
if _response_stream_shape_cache is None:
from botocore.loaders import Loader
from botocore.model import ServiceModel
loader = Loader()
sagemaker_service_dict = loader.load_service_model(
"sagemaker-runtime", "service-2"
)
sagemaker_service_model = ServiceModel(sagemaker_service_dict)
_response_stream_shape_cache = sagemaker_service_model.shape_for(
"InvokeEndpointWithResponseStreamOutput"
)
return _response_stream_shape_cache

7
litellm/llms/xai/realtime/handler.py
View file

@ -28,7 +28,12 @@ class XAIRealtime(OpenAIRealtime):
"""xAI uses a different API base URL."""
return XAI_API_BASE
def _get_additional_headers(self, api_key: str) -> dict:
def _get_additional_headers(
self,
api_key: str,
*,
openai_beta_realtime: bool = False,
) -> dict:
"""
xAI does NOT require the OpenAI-Beta header.
Only send Authorization header.

45
litellm/main.py
View file

@ -1,3 +1,5 @@
# LiteLLM main module: public completion, embedding, streaming, and moderation entrypoints.
#
# +-----------------------------------------------+
# | |
# | Give Feedback / Get Help |
@ -211,6 +213,12 @@ from .llms.oobabooga.chat import oobabooga
from .llms.openai.completion.handler import OpenAITextCompletion
from .llms.openai.image_variations.handler import OpenAIImageVariationsHandler
from .llms.openai.openai import OpenAIChatCompletion
from .llms.nvidia_riva.audio_transcription.handler import (
NvidiaRivaAudioTranscription,
)
from .llms.nvidia_riva.audio_transcription.transformation import (
NvidiaRivaAudioTranscriptionConfig,
)
from .llms.openai.transcriptions.handler import OpenAIAudioTranscription
from .llms.openai_like.chat.handler import OpenAILikeChatHandler
from .llms.openai_like.embedding.handler import OpenAILikeEmbeddingHandler
@ -266,6 +274,7 @@ from .types.utils import (
openai_chat_completions = OpenAIChatCompletion()
openai_text_completions = OpenAITextCompletion()
openai_audio_transcriptions = OpenAIAudioTranscription()
nvidia_riva_audio_transcriptions = NvidiaRivaAudioTranscription()
openai_image_variations = OpenAIImageVariationsHandler()
groq_chat_completions = GroqChatCompletion()
sap_gen_ai_hub_chat_completions = GenAIHubOrchestration()
@ -1452,14 +1461,14 @@ def completion( # type: ignore # noqa: PLR0915
if eos_token:
custom_prompt_dict[model]["eos_token"] = eos_token
if kwargs.get("model_file_id_mapping"):
messages = update_messages_with_model_file_ids(
messages=messages,
model_id=kwargs.get("model_info", {}).get("id", None),
model_file_id_mapping=cast(
Dict[str, Dict[str, str]], kwargs.get("model_file_id_mapping")
),
)
messages = update_messages_with_model_file_ids(
messages=messages,
model_id=kwargs.get("model_info", {}).get("id", None),
model_file_id_mapping=cast(
Dict[str, Dict[str, str]],
kwargs.get("model_file_id_mapping") or {},
),
)
provider_config: Optional[BaseConfig] = None
if custom_llm_provider is not None and custom_llm_provider in [
@ -6605,6 +6614,26 @@ def transcription(
litellm_params=litellm_params_dict,
shared_session=shared_session,
)
elif custom_llm_provider == "nvidia_riva":
# NVIDIA Riva is gRPC-based, not HTTP. It has its own dedicated handler
# rather than going through base_llm_http_handler.
response = nvidia_riva_audio_transcriptions.audio_transcriptions(
model=model,
audio_file=file,
optional_params=optional_params,
litellm_params=litellm_params_dict,
model_response=model_response,
atranscription=atranscription,
timeout=timeout,
logging_obj=litellm_logging_obj,
api_base=api_base,
api_key=api_key,
provider_config=(
provider_config
if isinstance(provider_config, NvidiaRivaAudioTranscriptionConfig)
else None
),
)
elif provider_config is not None:
response = base_llm_http_handler.audio_transcriptions(
model=model,

69
litellm/model_prices_and_context_window_backup.json
View file

@ -27187,6 +27187,20 @@
"supports_reasoning": true,
"supports_tool_choice": true
},
"openrouter/qwen/qwen3.6-plus": {
"input_cost_per_token": 3.25e-07,
"litellm_provider": "openrouter",
"max_input_tokens": 1000000,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_token": 1.95e-06,
"source": "https://openrouter.ai/qwen/qwen3.6-plus",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
},
"openrouter/qwen/qwen3.5-35b-a3b": {
"input_cost_per_token": 2.5e-07,
"litellm_provider": "openrouter",
@ -28874,6 +28888,19 @@
"mode": "chat",
"output_cost_per_token": 0.0
},
"sambanova/MiniMax-M2.7": {
"input_cost_per_token": 3e-07,
"litellm_provider": "sambanova",
"max_input_tokens": 204800,
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"source": "https://cloud.sambanova.ai/plans/pricing",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true
},
"sambanova/DeepSeek-R1": {
"input_cost_per_token": 5e-06,
"litellm_provider": "sambanova",
@ -34927,6 +34954,48 @@
"supports_vision": true,
"supports_web_search": true
},
"xai/grok-4.3": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"input_cost_per_token": 1.25e-06,
"input_cost_per_token_above_200k_tokens": 2.5e-06,
"litellm_provider": "xai",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 2.5e-06,
"output_cost_per_token_above_200k_tokens": 5e-06,
"source": "https://docs.x.ai/docs/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true
},
"xai/grok-4.3-latest": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"input_cost_per_token": 1.25e-06,
"input_cost_per_token_above_200k_tokens": 2.5e-06,
"litellm_provider": "xai",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 2.5e-06,
"output_cost_per_token_above_200k_tokens": 5e-06,
"source": "https://docs.x.ai/docs/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true
},
"xai/grok-beta": {
"input_cost_per_token": 5e-06,
"litellm_provider": "xai",

196
litellm/proxy/_experimental/mcp_server/auth/token_exchange.py Normal file
View file

@ -0,0 +1,196 @@
"""
OAuth 2.0 Token Exchange (RFC 8693) handler for MCP servers.
Exchanges a user's incoming JWT (subject_token) for a scoped access token
at an IDP's token exchange endpoint. The exchanged token is then used to
authenticate requests to the upstream MCP server.
See: https://datatracker.ietf.org/doc/html/rfc8693
"""
import asyncio
import hashlib
import weakref
from typing import TYPE_CHECKING, Dict, Tuple
import httpx
from litellm._logging import verbose_logger
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.constants import (
MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL,
MCP_OAUTH2_TOKEN_CACHE_MIN_TTL,
MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS,
MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE,
)
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.types.llms.custom_http import httpxSpecialProvider
if TYPE_CHECKING:
from litellm.types.mcp_server.mcp_server_manager import MCPServer
# RFC 8693 grant type constant
TOKEN_EXCHANGE_GRANT_TYPE = "urn:ietf:params:oauth:grant-type:token-exchange"
DEFAULT_SUBJECT_TOKEN_TYPE = "urn:ietf:params:oauth:token-type:access_token"
class TokenExchangeHandler:
"""Handles OAuth 2.0 Token Exchange (RFC 8693) for MCP servers.
Caches exchanged tokens keyed by ``hash(subject_token + server_id)`` so
repeated calls with the same user token skip the IDP round-trip.
"""
def __init__(self) -> None:
self._cache = InMemoryCache(
max_size_in_memory=MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE,
default_ttl=MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL,
)
# WeakValueDictionary so locks are GC'd once no coroutine holds a reference,
# preventing unbounded growth with many rotating user tokens.
self._locks: weakref.WeakValueDictionary[str, asyncio.Lock] = (
weakref.WeakValueDictionary()
)
def _get_lock(self, cache_key: str) -> asyncio.Lock:
lock = self._locks.get(cache_key)
if lock is None:
lock = asyncio.Lock()
self._locks[cache_key] = lock
return lock
@staticmethod
def _cache_key(subject_token: str, server_id: str) -> str:
raw = f"{subject_token}:{server_id}"
return hashlib.sha256(raw.encode()).hexdigest()
async def exchange_token(
self,
subject_token: str,
server: "MCPServer",
) -> str:
"""Exchange *subject_token* for a scoped access token.
Returns the exchanged ``access_token`` string (suitable for a
``Bearer`` header).
Raises ``ValueError`` on configuration or IDP errors.
"""
cache_key = self._cache_key(subject_token, server.server_id)
# Fast path
cached = self._cache.get_cache(cache_key)
if cached is not None:
return cached
# Slow path — one exchange at a time per (user, server) pair
async with self._get_lock(cache_key):
cached = self._cache.get_cache(cache_key)
if cached is not None:
return cached
token, ttl = await self._do_exchange(subject_token, server)
self._cache.set_cache(cache_key, token, ttl=ttl)
return token
async def _do_exchange(
self,
subject_token: str,
server: "MCPServer",
) -> Tuple[str, int]:
"""POST to the token exchange endpoint with RFC 8693 parameters.
Returns ``(access_token, ttl_seconds)``.
"""
endpoint = server.token_exchange_endpoint or server.token_url
if not endpoint:
raise ValueError(
f"MCP server '{server.server_id}' has auth_type=oauth2_token_exchange "
f"but no token_exchange_endpoint or token_url configured"
)
if not server.client_id or not server.client_secret:
raise ValueError(
f"MCP server '{server.server_id}' has auth_type=oauth2_token_exchange "
f"but missing client_id or client_secret"
)
data: Dict[str, str] = {
"grant_type": TOKEN_EXCHANGE_GRANT_TYPE,
"subject_token": subject_token,
"subject_token_type": server.subject_token_type
or DEFAULT_SUBJECT_TOKEN_TYPE,
"client_id": server.client_id,
"client_secret": server.client_secret,
}
if server.audience:
data["audience"] = server.audience
if server.scopes:
data["scope"] = " ".join(server.scopes)
verbose_logger.debug(
"Exchanging token for MCP server %s at %s (audience=%s)",
server.server_id,
endpoint,
server.audience,
)
client = get_async_httpx_client(llm_provider=httpxSpecialProvider.MCP)
try:
response = await client.post(endpoint, data=data)
response.raise_for_status()
except httpx.HTTPStatusError as exc:
verbose_logger.debug(
"Token exchange IDP error for MCP server %s (status %d)",
server.server_id,
exc.response.status_code,
)
raise ValueError(
f"Token exchange for MCP server '{server.server_id}' "
f"failed with status {exc.response.status_code}"
) from exc
body = response.json()
if not isinstance(body, dict):
raise ValueError(
f"Token exchange response for MCP server '{server.server_id}' "
f"returned non-object JSON (got {type(body).__name__})"
)
access_token = body.get("access_token")
if not access_token:
raise ValueError(
f"Token exchange response for MCP server '{server.server_id}' "
f"missing 'access_token'"
)
raw_expires_in = body.get("expires_in")
try:
expires_in = (
int(raw_expires_in)
if raw_expires_in is not None
else MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL
)
except (TypeError, ValueError):
expires_in = MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL
ttl = max(
expires_in - MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS,
MCP_OAUTH2_TOKEN_CACHE_MIN_TTL,
)
verbose_logger.info(
"Token exchange succeeded for MCP server %s (expires in %ds)",
server.server_id,
expires_in,
)
return access_token, ttl
def invalidate(self, subject_token: str, server_id: str) -> None:
"""Remove a cached exchanged token (e.g. after a 401)."""
cache_key = self._cache_key(subject_token, server_id)
self._cache.delete_cache(cache_key)
# Module-level singleton
mcp_token_exchange_handler = TokenExchangeHandler()

79
litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py
View file

@ -12,7 +12,8 @@ from litellm.llms.custom_httpx.http_handler import (
)
from litellm.proxy._experimental.mcp_server.oauth_utils import (
TOKEN_NO_CACHE_HEADERS,
validate_loopback_redirect_uri,
get_request_base_url,
validate_trusted_redirect_uri,
)
from litellm.proxy.auth.ip_address_utils import IPAddressUtils
from litellm.proxy.common_utils.encrypt_decrypt_utils import (
@ -29,51 +30,6 @@ router = APIRouter(
)
def get_request_base_url(request: Request) -> str:
"""
Get the base URL for the request, considering X-Forwarded-* headers.
X-Forwarded-Proto / X-Forwarded-Host / X-Forwarded-Port are only honoured
when the request comes from a configured trusted proxy
(``use_x_forwarded_for`` enabled AND caller in ``mcp_trusted_proxy_ranges``).
Otherwise the request's literal ``base_url`` is returned, so an
untrusted caller cannot poison OAuth-discovery / redirect_uri values
by injecting headers.
Args:
request: FastAPI Request object
Returns:
The reconstructed base URL (e.g., "https://proxy.example.com")
"""
base_url = str(request.base_url).rstrip("/")
parsed = urlparse(base_url)
if not IPAddressUtils.is_request_from_trusted_proxy(request):
return base_url
x_forwarded_proto = request.headers.get("X-Forwarded-Proto")
x_forwarded_host = request.headers.get("X-Forwarded-Host")
x_forwarded_port = request.headers.get("X-Forwarded-Port")
scheme = x_forwarded_proto if x_forwarded_proto else parsed.scheme
if x_forwarded_host:
# X-Forwarded-Host may already include port (e.g., "example.com:8080")
if ":" in x_forwarded_host and not x_forwarded_host.startswith("["):
netloc = x_forwarded_host
elif x_forwarded_port:
netloc = f"{x_forwarded_host}:{x_forwarded_port}"
else:
netloc = x_forwarded_host
else:
netloc = parsed.netloc
if x_forwarded_port and ":" not in netloc:
netloc = f"{netloc}:{x_forwarded_port}"
return urlunparse((scheme, netloc, parsed.path, "", "", ""))
def encode_state_with_base_url(
base_url: str,
original_state: str,
@ -127,12 +83,14 @@ def decode_state_hash(encrypted_state: str) -> dict:
return state_data
def _get_validated_client_redirect_uri(state_data: Dict[str, Any]) -> str:
"""Return a loopback client redirect URI from OAuth state."""
def _get_validated_client_redirect_uri(
request: Request, state_data: Dict[str, Any]
) -> str:
"""Return a trusted (same-origin or loopback) client redirect URI from OAuth state."""
redirect_uri = state_data.get("client_redirect_uri") or state_data.get("base_url")
if not redirect_uri or not isinstance(redirect_uri, str):
raise HTTPException(status_code=400, detail="Invalid redirect URI")
validate_loopback_redirect_uri(redirect_uri)
validate_trusted_redirect_uri(request, redirect_uri)
return redirect_uri
@ -338,12 +296,12 @@ async def authorize_with_server(
status_code=400, detail="MCP server authorization url is not set"
)
# Loopback-only redirect_uri. The URI is encrypted into the OAuth
# state and decoded on /callback to redirect the user back; a non-
# loopback URI would be an open-redirect + code-theft primitive
# (VERIA-57 root cause B). MCP clients are native apps — loopback is
# the spec-compliant callback pattern.
validate_loopback_redirect_uri(redirect_uri)
# Loopback OR same-origin redirect_uri. The URI is encrypted into the
# OAuth state and decoded on /callback to redirect the user back;
# restricting to trusted origins blocks the open-redirect +
# code-theft primitive (VERIA-57 root cause B). Loopback supports
# native MCP clients; same-origin supports the proxy's own UI callback.
validate_trusted_redirect_uri(request, redirect_uri)
parsed = urlparse(redirect_uri)
base_url = urlunparse(parsed._replace(query=""))
request_base_url = get_request_base_url(request)
@ -660,17 +618,18 @@ async def token_endpoint(
@router.get("/callback")
async def callback(code: str, state: str):
async def callback(request: Request, code: str, state: str):
try:
state_data = decode_state_hash(state)
original_state = state_data["original_state"]
# Re-validate loopback at the sink. /authorize rejects non-loopback
# Re-validate at the sink. /authorize rejects untrusted
# redirect_uri before encoding into state, but encrypted states
# minted before that check was added have no expiry and remain
# valid indefinitely. Validating here blocks the open-redirect +
# code-theft primitive even for pre-fix states.
redirect_uri = _get_validated_client_redirect_uri(state_data)
# valid indefinitely. Validating here (same-origin OR loopback)
# blocks the open-redirect + code-theft primitive even for pre-fix
# states while allowing the UI's same-origin callback to work.
redirect_uri = _get_validated_client_redirect_uri(request, state_data)
params = {"code": code, "state": original_state}
complete_returned_url = _append_query_params(redirect_uri, params)

177
litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
View file

@ -411,6 +411,15 @@ class MCPServerManager:
aws_role_name=server_config.get("aws_role_name", None),
aws_session_name=server_config.get("aws_session_name", None),
instructions=server_config.get("instructions", None),
# Token Exchange (OBO) fields
token_exchange_endpoint=server_config.get(
"token_exchange_endpoint", None
),
audience=server_config.get("audience", None),
subject_token_type=server_config.get(
"subject_token_type",
"urn:ietf:params:oauth:token-type:access_token",
),
)
self._assign_unique_short_prefix(new_server)
self.config_mcp_servers[server_id] = new_server
@ -497,7 +506,8 @@ class MCPServerManager:
# Add any static headers from server config.
#
# Note: `extra_headers` on MCPServer is a List[str] of header names to forward
# from the client request (not available in this OpenAPI tool generation step).
# from each client MCP request; values are applied at call time via
# `_request_extra_headers` in server.py (not baked in here).
# `static_headers` is a dict of concrete headers to always send.
headers = (
merge_mcp_headers(
@ -765,10 +775,23 @@ class MCPServerManager:
aws_role_name=aws_creds.get("aws_role_name"),
aws_session_name=aws_creds.get("aws_session_name"),
instructions=mcp_server.instructions,
# Token Exchange (OBO) fields — read from credentials JSON blob
token_exchange_endpoint=(
credentials_dict.get("token_exchange_endpoint")
if credentials_dict
else None
),
audience=(credentials_dict.get("audience") if credentials_dict else None),
subject_token_type=(
credentials_dict.get("subject_token_type") if credentials_dict else None
)
or "urn:ietf:params:oauth:token-type:access_token",
)
return new_server
async def _maybe_register_openapi_tools(self, server: MCPServer):
async def _maybe_register_openapi_tools(
self, server: MCPServer, *, initialize_mapping: bool = True
):
"""Register OpenAPI tools if the server has a spec_path configured."""
if server.spec_path:
verbose_logger.info(
@ -779,7 +802,8 @@ class MCPServerManager:
server=server,
base_url=server.url or "",
)
self.initialize_tool_name_to_mcp_server_name_mapping()
if initialize_mapping:
self.initialize_tool_name_to_mcp_server_name_mapping()
async def add_server(self, mcp_server: LiteLLM_MCPServerTable):
try:
@ -1136,6 +1160,29 @@ class MCPServerManager:
#########################################################
# Methods that call the upstream MCP servers
#########################################################
@staticmethod
def _extract_bearer_token(
oauth2_headers: Optional[Dict[str, str]],
raw_headers: Optional[Dict[str, str]],
) -> Optional[str]:
"""Extract the bare Bearer token from oauth2_headers or raw_headers.
Returns the token string without the ``Bearer `` prefix, or ``None``
if no Authorization header is found.
"""
auth_value: Optional[str] = None
if oauth2_headers and "Authorization" in oauth2_headers:
auth_value = oauth2_headers["Authorization"]
elif raw_headers:
# raw_headers may have lowercase keys depending on the ASGI server
normalized = {k.lower(): v for k, v in raw_headers.items()}
auth_value = normalized.get("authorization")
if auth_value:
if auth_value.startswith("Bearer "):
return auth_value[len("Bearer ") :]
return auth_value
return None
def _build_stdio_env(
self,
server: MCPServer,
@ -1169,25 +1216,30 @@ class MCPServerManager:
mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None,
extra_headers: Optional[Dict[str, str]] = None,
stdio_env: Optional[Dict[str, str]] = None,
subject_token: Optional[str] = None,
) -> MCPClient:
"""
Create an MCPClient instance for the given server.
Auth resolution (single place for all auth logic):
1. ``mcp_auth_header`` — per-request/per-user override
2. OAuth2 client_credentials token — auto-fetched and cached
3. ``server.authentication_token`` — static token from config/DB
2. OAuth2 Token Exchange (OBO) — exchange user token for scoped token
3. OAuth2 client_credentials token — auto-fetched and cached
4. ``server.authentication_token`` — static token from config/DB
Args:
server: The server configuration.
mcp_auth_header: Optional per-request auth override.
extra_headers: Additional headers to forward.
stdio_env: Environment variables for stdio transport.
subject_token: Optional user JWT for token exchange (OBO) flow.
Returns:
Configured MCP client instance.
"""
auth_value = await resolve_mcp_auth(server, mcp_auth_header)
auth_value = await resolve_mcp_auth(
server, mcp_auth_header, subject_token=subject_token
)
transport = server.transport or MCPTransport.sse
@ -1978,7 +2030,11 @@ class MCPServerManager:
_SHORT_PREFIX_MAX_REHASH_ATTEMPTS = 1024
def _assign_unique_short_prefix(self, server: MCPServer) -> None:
def _assign_unique_short_prefix(
self,
server: MCPServer,
registry: Optional[Dict[str, MCPServer]] = None,
) -> None:
"""Resolve and cache a collision-free short tool prefix on ``server``.
Called at registration time for every MCP server entering the
@ -2002,7 +2058,8 @@ class MCPServerManager:
return
used: Dict[str, str] = {}
for other in self.get_registry().values():
registry_for_collision_check = registry or self.get_registry()
for other in registry_for_collision_check.values():
if other.server_id == server.server_id:
continue
if other.short_prefix:
@ -2534,9 +2591,12 @@ class MCPServerManager:
if server_auth_header is None:
server_auth_header = mcp_auth_header
# oauth2 headers
# Extract subject token for OAuth2 Token Exchange (OBO) flow
subject_token: Optional[str] = None
extra_headers: Optional[Dict[str, str]] = None
if mcp_server.auth_type == MCPAuth.oauth2:
if mcp_server.auth_type == MCPAuth.oauth2_token_exchange:
subject_token = self._extract_bearer_token(oauth2_headers, raw_headers)
elif mcp_server.auth_type == MCPAuth.oauth2:
if mcp_server.has_client_credentials:
# For M2M OAuth servers, Authorization must come from token fetch.
extra_headers = None
@ -2604,6 +2664,7 @@ class MCPServerManager:
mcp_auth_header=server_auth_header,
extra_headers=extra_headers,
stdio_env=stdio_env,
subject_token=subject_token,
)
call_tool_params = MCPCallToolRequestParams(
@ -2916,46 +2977,72 @@ class MCPServerManager:
# against the *full* set so dedup is deterministic regardless of
# iteration order.
for server in db_mcp_servers:
existing_server = previous_registry.get(server.server_id)
try:
existing_server = previous_registry.get(server.server_id)
if (
existing_server is not None
and existing_server.updated_at is not None
and server.updated_at is not None
and existing_server.updated_at == server.updated_at
):
# Re-use existing server instance to avoid re-running build_mcp_server_from_table()
# which can perform network discovery for OAuth2 servers.
new_registry[server.server_id] = existing_server
continue
if (
existing_server is not None
and existing_server.updated_at is not None
and server.updated_at is not None
and existing_server.updated_at == server.updated_at
):
# Re-use existing server instance to avoid re-running build_mcp_server_from_table()
# which can perform network discovery for OAuth2 servers.
new_registry[server.server_id] = existing_server
continue
_warn_on_server_name_fields(
server_id=server.server_id,
alias=getattr(server, "alias", None),
server_name=getattr(server, "server_name", None),
)
verbose_logger.debug(
f"Building server from DB: {server.server_id} ({server.server_name})"
)
new_server = await self.build_mcp_server_from_table(server)
# Carry the cached short_prefix from the previous registry entry
# (if any) so the prefix is stable across reloads.
if existing_server is not None and existing_server.short_prefix:
new_server.short_prefix = existing_server.short_prefix
new_registry[server.server_id] = new_server
_warn_on_server_name_fields(
server_id=server.server_id,
alias=getattr(server, "alias", None),
server_name=getattr(server, "server_name", None),
)
verbose_logger.debug(
f"Building server from DB: {server.server_id} ({server.server_name})"
)
new_server = await self.build_mcp_server_from_table(server)
# Carry the cached short_prefix from the previous registry entry
# (if any) so the prefix is stable across reloads.
if existing_server is not None and existing_server.short_prefix:
new_server.short_prefix = existing_server.short_prefix
new_registry[server.server_id] = new_server
except Exception as e:
verbose_logger.exception(
"Skipping MCP server %s (%s) during DB reload: %s",
server.server_id,
getattr(server, "alias", None),
e,
)
# Swap in the new registry first so _assign_unique_short_prefix
# sees the complete set when checking for collisions.
self.registry = new_registry
for new_server in new_registry.values():
self._assign_unique_short_prefix(new_server)
# Register OpenAPI tools *after* the final short prefix is assigned
# so the tools are stored in the global registry under the same
# prefix that lookups will use.
await self._maybe_register_openapi_tools(new_server)
# Assign short prefixes against the full candidate set without
# publishing the staged registry to concurrent callers.
registered_registry: Dict[str, MCPServer] = {}
registered_openapi_tools = False
for server_id, new_server in new_registry.items():
try:
self._assign_unique_short_prefix(new_server, registry=new_registry)
# Register OpenAPI tools *after* the final short prefix is assigned
# so the tools are stored in the global registry under the same
# prefix that lookups will use.
await self._maybe_register_openapi_tools(
new_server, initialize_mapping=False
)
registered_registry[server_id] = new_server
if new_server.spec_path:
registered_openapi_tools = True
except Exception as e:
verbose_logger.exception(
"Skipping MCP server %s (%s) during DB reload: %s",
new_server.server_id,
getattr(new_server, "alias", None),
e,
)
self.registry = registered_registry
if registered_openapi_tools:
self.initialize_tool_name_to_mcp_server_name_mapping()
verbose_logger.debug(
"MCP registry refreshed (%s servers in registry)", len(new_registry)
"MCP registry refreshed (%s servers in registry)", len(registered_registry)
)
def get_mcp_servers_from_ids(self, server_ids: List[str]) -> List[MCPServer]:

42
litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py
View file

@ -26,6 +26,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import (
decrypt_value_helper,
encrypt_value_helper,
)
from litellm.proxy._experimental.mcp_server.auth import token_exchange
from litellm.types.llms.custom_http import httpxSpecialProvider
if TYPE_CHECKING:
@ -50,12 +51,23 @@ class MCPOAuth2TokenCache(InMemoryCache):
def _get_lock(self, server_id: str) -> asyncio.Lock:
return self._locks.setdefault(server_id, asyncio.Lock())
async def async_get_token(self, server: "MCPServer") -> Optional[str]:
@staticmethod
def _has_client_credentials_config(server: "MCPServer") -> bool:
return bool(server.client_id and server.client_secret and server.token_url)
async def async_get_token(
self,
server: "MCPServer",
*,
require_client_credentials_flow: bool = True,
) -> Optional[str]:
"""Return a valid access token, fetching or refreshing as needed.
Returns ``None`` when the server lacks client credentials config.
"""
if not server.has_client_credentials:
if require_client_credentials_flow and not server.has_client_credentials:
return None
if not self._has_client_credentials_config(server):
return None
server_id = server.server_id
@ -263,16 +275,38 @@ mcp_per_user_token_cache = MCPPerUserTokenCache()
async def resolve_mcp_auth(
server: "MCPServer",
mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None,
subject_token: Optional[str] = None,
) -> Optional[Union[str, Dict[str, str]]]:
"""Resolve the auth value for an MCP server.
Priority:
1. ``mcp_auth_header`` — per-request/per-user override
2. OAuth2 client_credentials token — auto-fetched and cached
3. ``server.authentication_token`` — static token from config/DB
2. OAuth2 Token Exchange (OBO / RFC 8693) — exchange user token for scoped token
3. OAuth2 client_credentials token — auto-fetched and cached
4. ``server.authentication_token`` — static token from config/DB
"""
if mcp_auth_header:
return mcp_auth_header
if server.has_token_exchange_config:
if subject_token:
return await token_exchange.mcp_token_exchange_handler.exchange_token(
subject_token, server
)
# No subject_token — fall back to client_credentials using the same client
# credentials and token_url so M2M scenarios still work.
if server.client_id and server.client_secret and server.token_url:
return await mcp_oauth2_token_cache.async_get_token(
server,
require_client_credentials_flow=False,
)
# OBO configured but no subject_token and missing client credentials — warn
# rather than silently proceeding unauthenticated.
verbose_logger.warning(
"MCP server '%s' is configured for token exchange (OBO) but no subject_token "
"was provided and client credentials (client_id/client_secret/token_url) are "
"incomplete. The request will proceed without authentication.",
server.server_id,
)
if server.has_client_credentials:
return await mcp_oauth2_token_cache.async_get_token(server)
return server.authentication_token

109
litellm/proxy/_experimental/mcp_server/oauth_utils.py
View file

@ -2,15 +2,63 @@
(BYOK + discoverable / pass-through OAuth proxy)."""
from ipaddress import ip_address
from urllib.parse import urlparse
from urllib.parse import urlparse, urlunparse
from fastapi import HTTPException
from fastapi import HTTPException, Request
from litellm._logging import verbose_logger
from litellm.proxy.auth.ip_address_utils import IPAddressUtils
# RFC 6749 §5.1 / OAuth 2.1 draft-15 §4.1.3: token-endpoint responses
# must not be cached — both success and error bodies may reveal secrets.
TOKEN_NO_CACHE_HEADERS = {"Cache-Control": "no-store", "Pragma": "no-cache"}
def get_request_base_url(request: Request) -> str:
"""
Get the base URL for the request, considering X-Forwarded-* headers.
X-Forwarded-Proto / X-Forwarded-Host / X-Forwarded-Port are only honoured
when the request comes from a configured trusted proxy
(``use_x_forwarded_for`` enabled AND caller in ``mcp_trusted_proxy_ranges``).
Otherwise the request's literal ``base_url`` is returned, so an
untrusted caller cannot poison OAuth-discovery / redirect_uri values
by injecting headers.
Args:
request: FastAPI Request object
Returns:
The reconstructed base URL (e.g., "https://proxy.example.com")
"""
base_url = str(request.base_url).rstrip("/")
parsed = urlparse(base_url)
if not IPAddressUtils.is_request_from_trusted_proxy(request):
return base_url
x_forwarded_proto = request.headers.get("X-Forwarded-Proto")
x_forwarded_host = request.headers.get("X-Forwarded-Host")
x_forwarded_port = request.headers.get("X-Forwarded-Port")
scheme = x_forwarded_proto if x_forwarded_proto else parsed.scheme
if x_forwarded_host:
# X-Forwarded-Host may already include port (e.g., "example.com:8080")
if ":" in x_forwarded_host and not x_forwarded_host.startswith("["):
netloc = x_forwarded_host
elif x_forwarded_port:
netloc = f"{x_forwarded_host}:{x_forwarded_port}"
else:
netloc = x_forwarded_host
else:
netloc = parsed.netloc
if x_forwarded_port and ":" not in netloc:
netloc = f"{netloc}:{x_forwarded_port}"
return urlunparse((scheme, netloc, parsed.path, "", "", ""))
def validate_loopback_redirect_uri(redirect_uri: str) -> None:
"""Require a loopback ``redirect_uri`` (OAuth 2.1 §4.1.2.1 + RFC 8252
§7.3 native-app pattern). MCP clients are native apps that listen on
@ -46,3 +94,60 @@ def validate_loopback_redirect_uri(redirect_uri: str) -> None:
# don't let it bubble up as a 500.
pass
raise HTTPException(status_code=400, detail="invalid_request")
def validate_trusted_redirect_uri(request: Request, redirect_uri: str) -> None:
"""Accept same-origin (proxy's own origin) OR loopback ``redirect_uri``.
Same-origin is required for the LiteLLM UI's OAuth flow: the UI
redirects to ``<proxy>/ui/mcp/oauth/callback`` which is not loopback
but is on the proxy's own trusted HTTPS origin. An attacker cannot
host content on the proxy's own origin without already owning the
proxy, so the open-redirect / code-theft primitive that motivated
:func:`validate_loopback_redirect_uri` does not apply here.
Loopback continues to be accepted for native MCP clients (per
OAuth 2.1 §4.1.2.1 + RFC 8252 §7.3).
Use this in the discoverable OAuth proxy endpoints that serve both
native clients and the proxy's own UI. BYOK endpoints that only
support native clients should keep
:func:`validate_loopback_redirect_uri`.
"""
try:
parsed = urlparse(redirect_uri)
except ValueError:
raise HTTPException(status_code=400, detail="invalid_request")
if parsed.scheme not in ("http", "https"):
raise HTTPException(status_code=400, detail="invalid_request")
if parsed.fragment:
raise HTTPException(status_code=400, detail="invalid_request")
# Same-origin: scheme + netloc (host[:port]) must match the proxy's
# own base URL at this request (honouring trusted X-Forwarded-*).
try:
proxy_base = urlparse(get_request_base_url(request))
if (
parsed.netloc
and parsed.scheme == proxy_base.scheme
and parsed.netloc.lower() == proxy_base.netloc.lower()
):
return
except Exception as exc:
# If we can't determine the proxy's origin, fall through to
# loopback. Log so the failure is diagnosable in production.
verbose_logger.warning(
"validate_trusted_redirect_uri: could not determine proxy origin, "
"falling back to loopback-only check. error=%s",
exc,
)
host = (parsed.hostname or "").lower()
if host == "localhost":
return
try:
if ip_address(host).is_loopback:
return
except ValueError:
pass
raise HTTPException(status_code=400, detail="invalid_request")

109
litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py
View file

@ -6,10 +6,34 @@ import asyncio
import contextvars
import json
import os
import re
from pathlib import PurePosixPath
from typing import Any, Dict, List, Optional
from urllib.parse import quote
# Tool names emitted from OpenAPI specs must work across all major LLM providers.
# OpenAI/Anthropic/Bedrock all enforce a character class roughly equivalent to
# ^[a-zA-Z0-9_-]+$ on tool names. Many specs (notably GitHub's REST API) use
# tag-namespaced operationIds like "actions/download-job-logs-for-workflow-run"
# which include '/'. Sanitize here so the same regex passes everywhere downstream.
_OPENAPI_TOOL_NAME_INVALID_CHARS = re.compile(r"[^a-zA-Z0-9_-]")
_OPENAPI_TOOL_NAME_MAX_LEN = 128
def sanitize_openapi_tool_name(raw_name: str) -> str:
"""Map an OpenAPI operationId / fallback to a provider-safe tool name.
Replaces any character outside ``[a-zA-Z0-9_-]`` with ``_`` and caps the
result at 128 chars (the most restrictive of the major providers).
Lowercased to match the existing convention in
``register_tools_from_openapi``.
"""
if not raw_name:
return raw_name
sanitized = _OPENAPI_TOOL_NAME_INVALID_CHARS.sub("_", raw_name).lower()
return sanitized[:_OPENAPI_TOOL_NAME_MAX_LEN]
from litellm._logging import verbose_logger
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
@ -31,6 +55,13 @@ _request_auth_header: contextvars.ContextVar[Optional[str]] = contextvars.Contex
"_request_auth_header", default=None
)
# Per-request extra headers forwarded from the client request.
# Populated from MCPServer.extra_headers names matched against raw request
# headers in server.py before dispatching to a local/OpenAPI tool handler.
_request_extra_headers: contextvars.ContextVar[Optional[Dict[str, str]]] = (
contextvars.ContextVar("_request_extra_headers", default=None)
)
def _sanitize_path_parameter_value(param_value: Any, param_name: str) -> str:
"""Ensure path params cannot introduce directory traversal."""
@ -273,6 +304,46 @@ def build_input_schema(operation: Dict[str, Any]) -> Dict[str, Any]:
}
def _merge_openapi_tool_request_headers(
static_headers: Dict[str, str]
) -> Dict[str, str]:
"""Merge static closure headers with per-request ContextVar overrides.
Precedence (highest to lowest):
1. ``_request_auth_header`` — BYOK override of ``Authorization``
2. ``static_headers`` — operator-configured headers baked into the
tool closure at registration time
3. ``_request_extra_headers`` — per-request headers forwarded from
the MCP caller (allowlisted by ``MCPServer.extra_headers``)
This matches the existing MCP invariant in
:func:`litellm.proxy._experimental.mcp_server.utils.merge_mcp_headers`
and the managed MCP path, where ``static_headers`` always wins over
caller-forwarded headers. Keeping the same precedence here prevents an
authenticated caller from overriding an operator-configured value
(e.g. a tenant id or upstream API key) by sending the same header name.
Header names are compared case-insensitively so different casing cannot
bypass the precedence rules.
"""
request_extra = _request_extra_headers.get() or {}
static = static_headers or {}
static_lower_names = {k.lower() for k in static}
effective_headers: Dict[str, str] = {
k: v for k, v in request_extra.items() if k.lower() not in static_lower_names
}
effective_headers.update(static)
override_auth = _request_auth_header.get()
if override_auth:
for existing in [k for k in effective_headers if k.lower() == "authorization"]:
del effective_headers[existing]
effective_headers["Authorization"] = override_auth
return effective_headers
def create_tool_function(
path: str,
method: str,
@ -310,14 +381,7 @@ def create_tool_function(
The function safely handles parameter names that aren't valid Python identifiers
by using **kwargs instead of named parameters.
"""
# Allow per-request auth override (e.g. BYOK credential set via ContextVar).
# The ContextVar holds the full Authorization header value, including the
# correct prefix (Bearer / ApiKey / Basic) formatted by the caller in
# server.py based on the server's configured auth_type.
effective_headers = dict(headers)
override_auth = _request_auth_header.get()
if override_auth:
effective_headers["Authorization"] = override_auth
effective_headers = _merge_openapi_tool_request_headers(headers)
# Build URL from base_url and path
url = base_url + path
@ -399,17 +463,36 @@ def create_tool_function(
def register_tools_from_openapi(spec: Dict[str, Any], base_url: str):
"""Register MCP tools from OpenAPI specification."""
paths = spec.get("paths", {})
used_names: set = set()
for path, path_item in paths.items():
for method in ["get", "post", "put", "delete", "patch"]:
if method in path_item:
operation = path_item[method]
# Generate tool name
operation_id = operation.get(
"operationId", f"{method}_{path.replace('/', '_')}"
)
tool_name = operation_id.replace(" ", "_").lower()
# Generate tool name. Sanitize to ^[a-zA-Z0-9_-]+$ (lowercase)
# so the resulting name is valid across OpenAI/Anthropic/Bedrock.
# Many specs (e.g. GitHub REST) use tag-namespaced operationIds
# like "actions/download-job-logs-for-workflow-run" which
# contain '/' and would 400 at the LLM provider boundary.
operation_id = operation.get("operationId", f"{method}_{path}")
tool_name = sanitize_openapi_tool_name(operation_id)
# Disambiguate collisions: two operationIds that differ only
# by sanitized characters (e.g. "foo/list" and "foo.list")
# would both become "foo_list". Append _2, _3, … to keep
# every tool reachable, mirroring the Anthropic-side logic
# in _build_anthropic_tool_name_maps.
unique = tool_name
n = 1
while unique in used_names:
n += 1
suffix = f"_{n}"
unique = (
tool_name[: _OPENAPI_TOOL_NAME_MAX_LEN - len(suffix)] + suffix
)
tool_name = unique
used_names.add(tool_name)
# Get description
description = operation.get(

28
litellm/proxy/_experimental/mcp_server/rest_endpoints.py
View file

@ -857,6 +857,7 @@ if MCP_AVAILABLE:
########################################################
from litellm.proxy.management_endpoints.mcp_management_endpoints import (
NewMCPServerRequest,
_inherit_credentials_from_existing_server,
)
def _extract_credentials(
@ -975,9 +976,11 @@ if MCP_AVAILABLE:
async def _preview_openapi_tools(spec_path: str) -> dict:
"""Generate tool previews from an OpenAPI spec without creating a server."""
from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import (
_OPENAPI_TOOL_NAME_MAX_LEN,
build_input_schema,
load_openapi_spec_async,
resolve_operation_params,
sanitize_openapi_tool_name,
)
try:
@ -985,8 +988,9 @@ if MCP_AVAILABLE:
paths = spec.get("paths", {})
components = spec.get("components", {})
tools: List[dict] = []
used_names: set = set()
for path, path_item in paths.items():
for method in ("get", "post", "put", "patch", "delete"):
for method in ("get", "post", "put", "delete", "patch"):
operation = path_item.get(method)
if operation is None:
continue
@ -995,7 +999,23 @@ if MCP_AVAILABLE:
operation, path_item, components
)
op_id = operation.get("operationId", f"{method}_{path}")
raw_op_id = operation.get("operationId", f"{method}_{path}")
# Match what register_tools_from_openapi does so the preview
# the user sees in the dashboard equals the names that get
# registered (and shipped to LLM providers, which enforce
# ^[a-zA-Z0-9_-]+$). See sanitize_openapi_tool_name docstring.
op_id = sanitize_openapi_tool_name(raw_op_id)
unique = op_id
n = 1
while unique in used_names:
n += 1
suffix = f"_{n}"
unique = (
op_id[: _OPENAPI_TOOL_NAME_MAX_LEN - len(suffix)] + suffix
)
op_id = unique
used_names.add(op_id)
summary = operation.get("summary", "")
description = operation.get("description", summary)
input_schema = build_input_schema(resolved_op)
@ -1068,6 +1088,10 @@ if MCP_AVAILABLE:
},
)
new_mcp_server_request = _inherit_credentials_from_existing_server(
new_mcp_server_request
)
# For OpenAPI spec servers, generate tools from the spec directly
if new_mcp_server_request.spec_path:
return await _preview_openapi_tools(new_mcp_server_request.spec_path)

30
litellm/proxy/_experimental/mcp_server/server.py
View file

@ -161,6 +161,7 @@ if MCP_AVAILABLE:
)
from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import (
_request_auth_header,
_request_extra_headers,
)
from litellm.proxy._experimental.mcp_server.sse_transport import SseServerTransport
from litellm.proxy._experimental.mcp_server.tool_registry import (
@ -2276,11 +2277,40 @@ if MCP_AVAILABLE:
auth_header_value = f"Basic {mcp_auth_header}"
else:
auth_header_value = f"Bearer {mcp_auth_header}"
# Forward named client headers to OpenAPI tool upstream requests.
# MCPServer.extra_headers lists header names to copy from raw_headers.
# OAuth2 M2M: never take Authorization from the caller (matches
# _prepare_mcp_server_headers for managed MCP).
forwarded_headers: Optional[Dict[str, str]] = None
if mcp_server and mcp_server.extra_headers and raw_headers:
normalized_raw = {
str(k).lower(): v
for k, v in raw_headers.items()
if isinstance(k, str)
}
skip_caller_authorization = bool(mcp_server.has_client_credentials)
for header_name in mcp_server.extra_headers:
if not isinstance(header_name, str):
continue
if (
skip_caller_authorization
and header_name.lower() == "authorization"
):
continue
value = normalized_raw.get(header_name.lower())
if value is not None:
if forwarded_headers is None:
forwarded_headers = {}
forwarded_headers[header_name] = value
_auth_token = _request_auth_header.set(auth_header_value)
_extra_token = _request_extra_headers.set(forwarded_headers)
try:
local_content = await _handle_local_mcp_tool(name, arguments)
finally:
_request_auth_header.reset(_auth_token)
_request_extra_headers.reset(_extra_token)
response = CallToolResult(content=cast(Any, local_content), isError=False)
# Try managed MCP server tool (pass the full prefixed name)

2
litellm/proxy/_experimental/out/404.html
View file

File diff suppressed because one or more lines are too long

6
litellm/proxy/_experimental/out/__next.__PAGE__.txt
View file

@ -1,10 +1,10 @@
1:"$Sreact.fragment"
2:I[347257,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"ClientPageRoot"]
3:I[952683,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","/litellm-asset-prefix/_next/static/chunks/0493aafc4891dd29.js","/litellm-asset-prefix/_next/static/chunks/e9906ef85805c46e.js","/litellm-asset-prefix/_next/static/chunks/b3d198d6c56a21b8.js","/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","/litellm-asset-prefix/_next/static/chunks/37e77c06e99eb8ff.js","/litellm-asset-prefix/_next/static/chunks/adb8beb738574863.js","/litellm-asset-prefix/_next/static/chunks/0549bc9afa7d4888.js","/litellm-asset-prefix/_next/static/chunks/0b470ffc60999bf4.js","/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","/litellm-asset-prefix/_next/static/chunks/baadbd26839e7b66.js","/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","/litellm-asset-prefix/_next/static/chunks/6eee262391715440.js","/litellm-asset-prefix/_next/static/chunks/4e17b625d75327a7.js","/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","/litellm-asset-prefix/_next/static/chunks/a06cc76a774dd182.js","/litellm-asset-prefix/_next/static/chunks/264fd32eefec52b6.js","/litellm-asset-prefix/_next/static/chunks/86828bdbafb8b581.js","/litellm-asset-prefix/_next/static/chunks/10dc4591ef08a91f.js","/litellm-asset-prefix/_next/static/chunks/e099566e8bd4ee4e.js","/litellm-asset-prefix/_next/static/chunks/fbe12a36d22e9554.js","/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","/litellm-asset-prefix/_next/static/chunks/679dbd657c8b5aef.js","/litellm-asset-prefix/_next/static/chunks/94f7208f5087e27c.js","/litellm-asset-prefix/_next/static/chunks/43f6fc3c2ab9cf23.js","/litellm-asset-prefix/_next/static/chunks/4e06277331e725da.js","/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","/litellm-asset-prefix/_next/static/chunks/ac3cf77acb5bf234.js","/litellm-asset-prefix/_next/static/chunks/fb125648f2dae104.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/a09028cd611c08ef.js","/litellm-asset-prefix/_next/static/chunks/6967a3b4ecbd3785.js","/litellm-asset-prefix/_next/static/chunks/3e917c79aadd945b.js","/litellm-asset-prefix/_next/static/chunks/9bbebdeb3f1cb03f.js","/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","/litellm-asset-prefix/_next/static/chunks/5f2d62a75803a3f7.js","/litellm-asset-prefix/_next/static/chunks/ca5fbafaf3826374.js","/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","/litellm-asset-prefix/_next/static/chunks/9b0ee76cbdef1a2a.js","/litellm-asset-prefix/_next/static/chunks/1bc2898be56acd1b.js","/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","/litellm-asset-prefix/_next/static/chunks/878832edb30e99a4.js","/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","/litellm-asset-prefix/_next/static/chunks/e641179604eae8c8.js","/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","/litellm-asset-prefix/_next/static/chunks/0cdfadbcf4b8c9e4.js","/litellm-asset-prefix/_next/static/chunks/8f3bf592254c6c3b.js","/litellm-asset-prefix/_next/static/chunks/8c17e934bd227606.js","/litellm-asset-prefix/_next/static/chunks/b98447395b5d37ef.js"],"default"]
3:I[952683,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","/litellm-asset-prefix/_next/static/chunks/0493aafc4891dd29.js","/litellm-asset-prefix/_next/static/chunks/f7e1d08418645368.js","/litellm-asset-prefix/_next/static/chunks/b3d198d6c56a21b8.js","/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","/litellm-asset-prefix/_next/static/chunks/37e77c06e99eb8ff.js","/litellm-asset-prefix/_next/static/chunks/adb8beb738574863.js","/litellm-asset-prefix/_next/static/chunks/0549bc9afa7d4888.js","/litellm-asset-prefix/_next/static/chunks/0b470ffc60999bf4.js","/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","/litellm-asset-prefix/_next/static/chunks/baadbd26839e7b66.js","/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","/litellm-asset-prefix/_next/static/chunks/6eee262391715440.js","/litellm-asset-prefix/_next/static/chunks/4e17b625d75327a7.js","/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","/litellm-asset-prefix/_next/static/chunks/a06cc76a774dd182.js","/litellm-asset-prefix/_next/static/chunks/264fd32eefec52b6.js","/litellm-asset-prefix/_next/static/chunks/86828bdbafb8b581.js","/litellm-asset-prefix/_next/static/chunks/10dc4591ef08a91f.js","/litellm-asset-prefix/_next/static/chunks/e099566e8bd4ee4e.js","/litellm-asset-prefix/_next/static/chunks/fbe12a36d22e9554.js","/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","/litellm-asset-prefix/_next/static/chunks/679dbd657c8b5aef.js","/litellm-asset-prefix/_next/static/chunks/94f7208f5087e27c.js","/litellm-asset-prefix/_next/static/chunks/43f6fc3c2ab9cf23.js","/litellm-asset-prefix/_next/static/chunks/4e06277331e725da.js","/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","/litellm-asset-prefix/_next/static/chunks/ac3cf77acb5bf234.js","/litellm-asset-prefix/_next/static/chunks/fb125648f2dae104.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/a09028cd611c08ef.js","/litellm-asset-prefix/_next/static/chunks/6967a3b4ecbd3785.js","/litellm-asset-prefix/_next/static/chunks/3e917c79aadd945b.js","/litellm-asset-prefix/_next/static/chunks/9bbebdeb3f1cb03f.js","/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","/litellm-asset-prefix/_next/static/chunks/5f2d62a75803a3f7.js","/litellm-asset-prefix/_next/static/chunks/ca5fbafaf3826374.js","/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","/litellm-asset-prefix/_next/static/chunks/9b0ee76cbdef1a2a.js","/litellm-asset-prefix/_next/static/chunks/1bc2898be56acd1b.js","/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","/litellm-asset-prefix/_next/static/chunks/878832edb30e99a4.js","/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","/litellm-asset-prefix/_next/static/chunks/8e3d0ce9505a304f.js","/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","/litellm-asset-prefix/_next/static/chunks/0cdfadbcf4b8c9e4.js","/litellm-asset-prefix/_next/static/chunks/8f3bf592254c6c3b.js","/litellm-asset-prefix/_next/static/chunks/8c17e934bd227606.js","/litellm-asset-prefix/_next/static/chunks/b98447395b5d37ef.js"],"default"]
1a:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"OutletBoundary"]
1b:"$Sreact.suspense"
:HL["/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","style"]
0:{"buildId":"lVPGJ4SMG3ZUqEHCANFji","rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0493aafc4891dd29.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/e9906ef85805c46e.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/b3d198d6c56a21b8.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/37e77c06e99eb8ff.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/adb8beb738574863.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/0549bc9afa7d4888.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0b470ffc60999bf4.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/baadbd26839e7b66.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/6eee262391715440.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/4e17b625d75327a7.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/a06cc76a774dd182.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/264fd32eefec52b6.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/86828bdbafb8b581.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/10dc4591ef08a91f.js","async":true}],["$","script","script-18",{"src":"/litellm-asset-prefix/_next/static/chunks/e099566e8bd4ee4e.js","async":true}],["$","script","script-19",{"src":"/litellm-asset-prefix/_next/static/chunks/fbe12a36d22e9554.js","async":true}],["$","script","script-20",{"src":"/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","async":true}],["$","script","script-21",{"src":"/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","async":true}],["$","script","script-22",{"src":"/litellm-asset-prefix/_next/static/chunks/679dbd657c8b5aef.js","async":true}],["$","script","script-23",{"src":"/litellm-asset-prefix/_next/static/chunks/94f7208f5087e27c.js","async":true}],["$","script","script-24",{"src":"/litellm-asset-prefix/_next/static/chunks/43f6fc3c2ab9cf23.js","async":true}],["$","script","script-25",{"src":"/litellm-asset-prefix/_next/static/chunks/4e06277331e725da.js","async":true}],["$","script","script-26",{"src":"/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","async":true}],["$","script","script-27",{"src":"/litellm-asset-prefix/_next/static/chunks/ac3cf77acb5bf234.js","async":true}],["$","script","script-28",{"src":"/litellm-asset-prefix/_next/static/chunks/fb125648f2dae104.js","async":true}],["$","script","script-29",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true}],["$","script","script-30",{"src":"/litellm-asset-prefix/_next/static/chunks/a09028cd611c08ef.js","async":true}],["$","script","script-31",{"src":"/litellm-asset-prefix/_next/static/chunks/6967a3b4ecbd3785.js","async":true}],["$","script","script-32",{"src":"/litellm-asset-prefix/_next/static/chunks/3e917c79aadd945b.js","async":true}],["$","script","script-33",{"src":"/litellm-asset-prefix/_next/static/chunks/9bbebdeb3f1cb03f.js","async":true}],"$L6","$L7","$L8","$L9","$La","$Lb","$Lc","$Ld","$Le","$Lf","$L10","$L11","$L12","$L13","$L14","$L15","$L16","$L17","$L18"],"$L19"]}],"loading":null,"isPartial":false}
0:{"buildId":"8TZ2JbOi7SZ6BCj9ScTHW","rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0493aafc4891dd29.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/f7e1d08418645368.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/b3d198d6c56a21b8.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/37e77c06e99eb8ff.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/adb8beb738574863.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/0549bc9afa7d4888.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0b470ffc60999bf4.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/baadbd26839e7b66.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/6eee262391715440.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/4e17b625d75327a7.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/a06cc76a774dd182.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/264fd32eefec52b6.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/86828bdbafb8b581.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/10dc4591ef08a91f.js","async":true}],["$","script","script-18",{"src":"/litellm-asset-prefix/_next/static/chunks/e099566e8bd4ee4e.js","async":true}],["$","script","script-19",{"src":"/litellm-asset-prefix/_next/static/chunks/fbe12a36d22e9554.js","async":true}],["$","script","script-20",{"src":"/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","async":true}],["$","script","script-21",{"src":"/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","async":true}],["$","script","script-22",{"src":"/litellm-asset-prefix/_next/static/chunks/679dbd657c8b5aef.js","async":true}],["$","script","script-23",{"src":"/litellm-asset-prefix/_next/static/chunks/94f7208f5087e27c.js","async":true}],["$","script","script-24",{"src":"/litellm-asset-prefix/_next/static/chunks/43f6fc3c2ab9cf23.js","async":true}],["$","script","script-25",{"src":"/litellm-asset-prefix/_next/static/chunks/4e06277331e725da.js","async":true}],["$","script","script-26",{"src":"/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","async":true}],["$","script","script-27",{"src":"/litellm-asset-prefix/_next/static/chunks/ac3cf77acb5bf234.js","async":true}],["$","script","script-28",{"src":"/litellm-asset-prefix/_next/static/chunks/fb125648f2dae104.js","async":true}],["$","script","script-29",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true}],["$","script","script-30",{"src":"/litellm-asset-prefix/_next/static/chunks/a09028cd611c08ef.js","async":true}],["$","script","script-31",{"src":"/litellm-asset-prefix/_next/static/chunks/6967a3b4ecbd3785.js","async":true}],["$","script","script-32",{"src":"/litellm-asset-prefix/_next/static/chunks/3e917c79aadd945b.js","async":true}],["$","script","script-33",{"src":"/litellm-asset-prefix/_next/static/chunks/9bbebdeb3f1cb03f.js","async":true}],"$L6","$L7","$L8","$L9","$La","$Lb","$Lc","$Ld","$Le","$Lf","$L10","$L11","$L12","$L13","$L14","$L15","$L16","$L17","$L18"],"$L19"]}],"loading":null,"isPartial":false}
4:{}
5:"$0:rsc:props:children:0:props:serverProvidedParams:params"
6:["$","script","script-34",{"src":"/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","async":true}]
@ -17,7 +17,7 @@ c:["$","script","script-40",{"src":"/litellm-asset-prefix/_next/static/chunks/1b
d:["$","script","script-41",{"src":"/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","async":true}]
e:["$","script","script-42",{"src":"/litellm-asset-prefix/_next/static/chunks/878832edb30e99a4.js","async":true}]
f:["$","script","script-43",{"src":"/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","async":true}]
10:["$","script","script-44",{"src":"/litellm-asset-prefix/_next/static/chunks/e641179604eae8c8.js","async":true}]
10:["$","script","script-44",{"src":"/litellm-asset-prefix/_next/static/chunks/8e3d0ce9505a304f.js","async":true}]
11:["$","script","script-45",{"src":"/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","async":true}]
12:["$","script","script-46",{"src":"/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","async":true}]
13:["$","script","script-47",{"src":"/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","async":true}]

6
litellm/proxy/_experimental/out/__next._full.txt
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/__next._head.txt
View file

@ -3,4 +3,4 @@
3:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"MetadataBoundary"]
4:"$Sreact.suspense"
5:I[27201,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"IconMark"]
0:{"buildId":"lVPGJ4SMG3ZUqEHCANFji","rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"./favicon.ico"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"loading":null,"isPartial":false}
0:{"buildId":"8TZ2JbOi7SZ6BCj9ScTHW","rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"./favicon.ico"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"loading":null,"isPartial":false}

2
litellm/proxy/_experimental/out/__next._index.txt
View file

@ -5,4 +5,4 @@
5:I[837457,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
:HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"]
:HL["/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","style"]
0:{"buildId":"lVPGJ4SMG3ZUqEHCANFji","rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]]}],"loading":null,"isPartial":false}
0:{"buildId":"8TZ2JbOi7SZ6BCj9ScTHW","rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]]}],"loading":null,"isPartial":false}

Some files were not shown because too many files have changed in this diff Show more